From 3f3e3f4f0513a48c01a4e187463120c948f4ada2 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 10:45:24 +0800 Subject: [PATCH 01/51] perf(get): avoid memory body stream wrapper (#6163) Use MemoryTrackedBytesStream directly as an s3s ByteStream so in-memory GET bodies avoid the generic StreamingBlob::wrap adapter while preserving exact remaining length, request lifecycle tracking, and length-mismatch failure semantics. Co-authored-by: heihutu --- rustfs/src/app/object_usecase.rs | 39 ++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 913d6eb52..643b4fb6e 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -1074,7 +1074,7 @@ where } impl futures::Stream for MemoryTrackedBytesStream { - type Item = std::io::Result; + type Item = Result; fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); @@ -1105,7 +1105,8 @@ impl futures::Stream for MemoryTrackedBytesStream { return Poll::Ready(Some(Err(std::io::Error::new( std::io::ErrorKind::InvalidData, format!("materialized GET body length mismatch: expected {}, got {}", this.expected, actual), - )))); + ) + .into()))); } let Some(bytes) = this.bytes.take() else { @@ -1132,6 +1133,16 @@ impl futures::Stream for MemoryTrackedBytesStream { } } +impl ByteStream for MemoryTrackedBytesStream { + fn remaining_length(&self) -> RemainingLength { + if self.emitted || self.bytes.is_none() { + RemainingLength::new_exact(0) + } else { + RemainingLength::new_exact(self.expected) + } + } +} + impl Drop for MemoryTrackedBytesStream { fn drop(&mut self) { if self.lifecycle.is_finished() { @@ -4149,7 +4160,7 @@ impl DefaultObjectUsecase { let bytes_len = bytes.len(); let guard = rustfs_io_metrics::track_get_object_buffered_bytes(bytes_len); let remaining = usize::try_from(response_content_length.max(0)).unwrap_or(usize::MAX); - let blob = StreamingBlob::wrap(MemoryTrackedBytesStream::new(bytes, remaining, source, guard, lifecycle)); + let blob = StreamingBlob::new(MemoryTrackedBytesStream::new(bytes, remaining, source, guard, lifecycle)); if let Some(handoff_start) = handoff_start { rustfs_io_metrics::record_get_object_response_handoff( "single_chunk", @@ -12882,7 +12893,10 @@ mod tests { .await .expect("mismatched memory body must yield an item") .expect_err("a short memory body must fail the stream instead of serving a truncated body"); - assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert_eq!( + err.downcast_ref::().map(std::io::Error::kind), + Some(std::io::ErrorKind::InvalidData) + ); assert!(stream.next().await.is_none(), "stream must terminate after the error"); } @@ -12901,7 +12915,22 @@ mod tests { .await .expect("mismatched memory body must yield an item") .expect_err("an over-long memory body must fail the stream instead of serving mismatched bytes"); - assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert_eq!( + err.downcast_ref::().map(std::io::Error::kind), + Some(std::io::ErrorKind::InvalidData) + ); + } + + #[test] + fn memory_blob_preserves_exact_remaining_length() { + let blob = DefaultObjectUsecase::build_memory_bytes_blob( + Bytes::from_static(b"hello"), + 5, + GET_MEMORY_BODY_SOURCE_BUFFERED_BODY, + GetObjectBodyLifecycle::disabled(), + ); + + assert_eq!(blob.remaining_length().exact(), Some(5)); } #[tokio::test] From 7db38827776f2efb4a879a192242b35aa8a30067 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Mon, 17 Aug 2026 11:32:23 +0800 Subject: [PATCH 02/51] chore(obs): adjudicate 19 bare dead_code allows (#6162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backlog#1823 step 10, batch 2. Eighteen of the nineteen suppress nothing and are deleted; one was real and keeps an allow that now says why. Rotation::Never is constructed only by the rolling-appender tests at rolling.rs:456, 477 and 498, so the lib target reports it as never constructed. Its allow is restored with that reason. Finding it corrected the method used for batch 1. Removing all nineteen and running cargo check -p rustfs-obs --tests reported zero warnings even after touching every source file, while clippy --lib --tests -D warnings caught Rotation::Never. cargo's warning output is not a reliable completeness check — it does not re-emit for cached compilations, and touching the sources did not cover the lib target here. Later batches should treat clippy -D warnings as the gate; batch 1's six crates were re-checked under clippy and are clean. Taken with #6086, which cleared this crate's 44 module-level blankets and left six real items, obs has now had 63 dead-code suppressions examined, of which seven were suppressing anything at all. The rest sat on items that are publicly reachable, where dead_code never applied — the same shape as the swift module and kms's dek.rs. Verification: clippy --lib --tests -D warnings clean in the default, gpu and pyroscope lanes; cargo nextest run -p rustfs-obs 324 passed; make pre-commit exit 0. Ref rustfs/backlog#1823 (step 10). --- crates/obs/src/metrics/config.rs | 1 - crates/obs/src/metrics/report.rs | 3 --- crates/obs/src/metrics/schema/entry/descriptor.rs | 3 --- crates/obs/src/metrics/schema/entry/metric_name.rs | 2 -- crates/obs/src/metrics/schema/entry/metric_type.rs | 3 --- crates/obs/src/metrics/schema/entry/mod.rs | 1 - crates/obs/src/metrics/schema/entry/namespace.rs | 1 - crates/obs/src/metrics/schema/entry/path_utils.rs | 1 - crates/obs/src/metrics/schema/entry/subsystem.rs | 3 --- crates/obs/src/telemetry/rolling.rs | 5 ++++- 10 files changed, 4 insertions(+), 19 deletions(-) diff --git a/crates/obs/src/metrics/config.rs b/crates/obs/src/metrics/config.rs index 26b73d6b1..c43e0c1e1 100644 --- a/crates/obs/src/metrics/config.rs +++ b/crates/obs/src/metrics/config.rs @@ -17,7 +17,6 @@ use std::time::Duration; /// Environment variable key for the global default metrics interval (seconds). pub const ENV_DEFAULT_METRICS_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL_SEC"; /// Default interval for metrics collection if not specified otherwise. -#[allow(dead_code)] pub const DEFAULT_METRICS_INTERVAL: Duration = Duration::from_secs(60); /// Environment variable key for cluster metrics interval (seconds). diff --git a/crates/obs/src/metrics/report.rs b/crates/obs/src/metrics/report.rs index b03630272..b3a421f55 100644 --- a/crates/obs/src/metrics/report.rs +++ b/crates/obs/src/metrics/report.rs @@ -145,21 +145,18 @@ impl PrometheusMetric { } #[inline] - #[allow(dead_code)] pub fn with_label(mut self, key: &'static str, value: impl Into>) -> Self { self.labels.push((key, value.into())); self } #[inline] - #[allow(dead_code)] pub fn with_label_owned(mut self, key: &'static str, value: String) -> Self { self.labels.push((key, Cow::Owned(value))); self } #[inline] - #[allow(dead_code)] pub fn with_labels(mut self, labels: Vec<(&'static str, Cow<'static, str>)>) -> Self { self.labels = labels; self diff --git a/crates/obs/src/metrics/schema/entry/descriptor.rs b/crates/obs/src/metrics/schema/entry/descriptor.rs index c4612e1f8..fce80d998 100644 --- a/crates/obs/src/metrics/schema/entry/descriptor.rs +++ b/crates/obs/src/metrics/schema/entry/descriptor.rs @@ -16,7 +16,6 @@ use crate::{MetricName, MetricNamespace, MetricSubsystem, MetricType}; use std::collections::HashSet; /// MetricDescriptor - Metric descriptors -#[allow(dead_code)] #[derive(Debug, Clone)] pub struct MetricDescriptor { pub name: MetricName, @@ -52,7 +51,6 @@ impl MetricDescriptor { } /// Get the full metric name in Prometheus style: __ - #[allow(dead_code)] pub fn get_full_metric_name(&self) -> String { let namespace = self.namespace.as_str(); let formatted_subsystem = self.subsystem.as_str(); @@ -61,7 +59,6 @@ impl MetricDescriptor { } /// check whether the label is in the label set - #[allow(dead_code)] pub fn has_label(&mut self, label: &str) -> bool { self.get_label_set().contains(label) } diff --git a/crates/obs/src/metrics/schema/entry/metric_name.rs b/crates/obs/src/metrics/schema/entry/metric_name.rs index 7d22eaa62..d9ec407b3 100644 --- a/crates/obs/src/metrics/schema/entry/metric_name.rs +++ b/crates/obs/src/metrics/schema/entry/metric_name.rs @@ -13,7 +13,6 @@ // limitations under the License. /// The metric name is the individual name of the metric -#[allow(dead_code)] #[derive(Debug, Clone, PartialEq, Eq)] pub enum MetricName { // The generic metric name @@ -443,7 +442,6 @@ pub enum MetricName { } impl MetricName { - #[allow(dead_code)] pub fn as_str(&self) -> String { match self { Self::AuthTotal => "auth_total".to_string(), diff --git a/crates/obs/src/metrics/schema/entry/metric_type.rs b/crates/obs/src/metrics/schema/entry/metric_type.rs index 33d8a78dd..b16f4cd12 100644 --- a/crates/obs/src/metrics/schema/entry/metric_type.rs +++ b/crates/obs/src/metrics/schema/entry/metric_type.rs @@ -13,7 +13,6 @@ // limitations under the License. /// MetricType - Indicates the type of indicator -#[allow(dead_code)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MetricType { Counter, @@ -23,7 +22,6 @@ pub enum MetricType { impl MetricType { /// convert the metric type to a string representation - #[allow(dead_code)] pub fn as_str(&self) -> &'static str { match self { Self::Counter => "counter", @@ -34,7 +32,6 @@ impl MetricType { /// Convert the metric type to the Prometheus value type /// In a Rust implementation, this might return the corresponding Prometheus Rust client type - #[allow(dead_code)] pub fn as_prom(&self) -> &'static str { match self { Self::Counter => "counter.", diff --git a/crates/obs/src/metrics/schema/entry/mod.rs b/crates/obs/src/metrics/schema/entry/mod.rs index 87215d15c..c10b48b45 100644 --- a/crates/obs/src/metrics/schema/entry/mod.rs +++ b/crates/obs/src/metrics/schema/entry/mod.rs @@ -56,7 +56,6 @@ pub fn new_gauge_md( } /// create a new histogram indicator descriptor -#[allow(dead_code)] pub fn new_histogram_md( name: impl Into, help: impl Into, diff --git a/crates/obs/src/metrics/schema/entry/namespace.rs b/crates/obs/src/metrics/schema/entry/namespace.rs index 31c3ce590..fe9a5df1f 100644 --- a/crates/obs/src/metrics/schema/entry/namespace.rs +++ b/crates/obs/src/metrics/schema/entry/namespace.rs @@ -19,7 +19,6 @@ pub enum MetricNamespace { } impl MetricNamespace { - #[allow(dead_code)] pub fn as_str(&self) -> &'static str { match self { Self::RustFS => "rustfs", diff --git a/crates/obs/src/metrics/schema/entry/path_utils.rs b/crates/obs/src/metrics/schema/entry/path_utils.rs index f8b63da28..d2abc7bfd 100644 --- a/crates/obs/src/metrics/schema/entry/path_utils.rs +++ b/crates/obs/src/metrics/schema/entry/path_utils.rs @@ -14,7 +14,6 @@ /// Format the path to the metric name format /// Replace '/' and '-' with '_' -#[allow(dead_code)] pub fn format_path_to_metric_name(path: &str) -> String { path.trim_start_matches('/').replace(['/', '-'], "_") } diff --git a/crates/obs/src/metrics/schema/entry/subsystem.rs b/crates/obs/src/metrics/schema/entry/subsystem.rs index 1c313f6e8..7b0606890 100644 --- a/crates/obs/src/metrics/schema/entry/subsystem.rs +++ b/crates/obs/src/metrics/schema/entry/subsystem.rs @@ -102,7 +102,6 @@ impl MetricSubsystem { } /// Get the formatted metric name format string - #[allow(dead_code)] pub fn as_str(&self) -> String { format_path_to_metric_name(self.path()) } @@ -151,7 +150,6 @@ impl MetricSubsystem { } /// A convenient way to create custom subsystems directly - #[allow(dead_code)] pub fn new(path: impl Into) -> Self { Self::Custom(path.into()) } @@ -176,7 +174,6 @@ impl std::fmt::Display for MetricSubsystem { } } -#[allow(dead_code)] pub mod subsystems { use super::MetricSubsystem; diff --git a/crates/obs/src/telemetry/rolling.rs b/crates/obs/src/telemetry/rolling.rs index 3ff661bfd..c5680ac61 100644 --- a/crates/obs/src/telemetry/rolling.rs +++ b/crates/obs/src/telemetry/rolling.rs @@ -38,7 +38,10 @@ pub enum Rotation { Minutely, Hourly, Daily, - #[allow(dead_code)] + #[allow( + dead_code, + reason = "constructed only by this file's rolling-appender tests; the lib target cannot see them (backlog#1823)" + )] Never, } From a9691b67978c56cb38081f453fc4959e568275a6 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Mon, 17 Aug 2026 11:34:47 +0800 Subject: [PATCH 03/51] chore: adjudicate 19 bare dead_code allows across six leaf crates (#6161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backlog#1823 step 10, batch 1 of the repo-wide item-allow sweep. 227 bare #[allow(dead_code)] remain across 83 files; this takes the 19 in utils, notify, checksums, policy, keystone and trusted-proxies, which are small enough to verify end to end. Removing all 19 first, before writing any reason, matters: 8 of them suppress nothing. Every allow in utils, one in policy and three in notify sit on items that are publicly reachable, so dead_code never applied to them — the same shape as the swift module and kms's dek.rs. Writing a reason onto a no-op allow would dress noise up as considered judgement, so those are simply deleted. Three items are genuinely dead and go with their allows: notify's new_target_id_set, the AWS metadata fetcher's get_metadata_token, and policy's empty `pub struct Value;`, none of which is referenced anywhere in the tree. The remaining eight keep an allow, now saying why the item survives rather than who calls it. Two are exercised only by their own crate's tests (checksums' MD5_HEADER_NAME, policy's is_match_as_pattern_prefix). Four are fields written but never read back: keystone's verify_ssl, parsed from config after the reqwest client is already built; keystone's client handle, which keeps the Keystone client alive for the mapper's lifetime; the AWS IMDS endpoint, kept beside the client while requests build their own URLs; and notify's rules_map, whose own comment retains it for snapshot-time judgements no code performs. checksums' Md5 needed the most care. Crc32, Sha256 and seven others each have an arm in ChecksumAlgorithm::into_impl, and Md5 has none, which reads like a missing algorithm. It is not: ChecksumAlgorithm has no Md5 variant at all. S3 carries Content-MD5 as its own header, separate from the x-amz-checksum-* family, and this impl exists so both paths share the Checksum trait. The reason records that, so the next reader does not re-derive it. One measurement note for anyone continuing this sweep: cargo does not re-emit warnings for cached compilations, so a per-crate loop of `cargo check -p ` under-reports. checksums showed zero that way while actually carrying three. Touch the sources and check the crates in one invocation, then attribute by path. Verification: the six crates are warning-free under cargo check --tests; clippy --lib --tests -D warnings clean; cargo nextest run 1096 passed; make pre-commit exit 0. Ref rustfs/backlog#1823 (step 10). --- crates/checksums/src/http.rs | 5 +- crates/checksums/src/lib.rs | 10 +++- crates/keystone/src/client.rs | 5 +- crates/keystone/src/identity.rs | 5 +- crates/notify/src/rules/config.rs | 5 +- crates/notify/src/rules/rules_map.rs | 3 -- crates/notify/src/rules/target_id_set.rs | 6 --- crates/policy/src/policy/function.rs | 4 -- crates/policy/src/policy/utils/wildcard.rs | 6 ++- .../trusted-proxies/src/cloud/metadata/aws.rs | 53 ++----------------- crates/trusted-proxies/src/config/env.rs | 1 - crates/utils/src/io.rs | 1 - crates/utils/src/net.rs | 1 - crates/utils/src/os/fs_type.rs | 1 - crates/utils/src/path.rs | 1 - 15 files changed, 32 insertions(+), 75 deletions(-) diff --git a/crates/checksums/src/http.rs b/crates/checksums/src/http.rs index 1a369a42d..ef21bc3ba 100644 --- a/crates/checksums/src/http.rs +++ b/crates/checksums/src/http.rs @@ -38,7 +38,10 @@ pub const XXHASH_3_HEADER_NAME: &str = "x-amz-checksum-xxhash3"; pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64"; pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128"; -#[allow(dead_code)] +#[allow( + dead_code, + reason = "Content-MD5 wire name, resolved by header_name() below and asserted by this crate's tests (backlog#1823)" +)] pub(crate) static MD5_HEADER_NAME: &str = "content-md5"; pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] = diff --git a/crates/checksums/src/lib.rs b/crates/checksums/src/lib.rs index a8da44545..5b566fe83 100644 --- a/crates/checksums/src/lib.rs +++ b/crates/checksums/src/lib.rs @@ -476,13 +476,19 @@ impl Checksum for Xxhash64 { } } -#[allow(dead_code)] #[derive(Debug, Default)] +#[allow( + dead_code, + reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)" +)] struct Md5 { hasher: md5::Md5, } -#[allow(dead_code)] +#[allow( + dead_code, + reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)" +)] impl Md5 { fn update(&mut self, bytes: &[u8]) { use md5::Digest; diff --git a/crates/keystone/src/client.rs b/crates/keystone/src/client.rs index d5df169e8..bd1d36d30 100644 --- a/crates/keystone/src/client.rs +++ b/crates/keystone/src/client.rs @@ -31,7 +31,10 @@ pub struct KeystoneClient { admin_password: Option, admin_project: Option, admin_domain: String, - #[allow(dead_code)] + #[allow( + dead_code, + reason = "TLS verification flag parsed from config; the reqwest client is built before it is consulted, so nothing reads it back (backlog#1823)" + )] verify_ssl: bool, /// Request timeout applied to the underlying HTTP client. timeout: std::time::Duration, diff --git a/crates/keystone/src/identity.rs b/crates/keystone/src/identity.rs index 96e45de17..f61ca5c2b 100644 --- a/crates/keystone/src/identity.rs +++ b/crates/keystone/src/identity.rs @@ -20,7 +20,10 @@ use tracing::{debug, info}; /// Maps Keystone identities to RustFS concepts pub struct KeystoneIdentityMapper { - #[allow(dead_code)] + #[allow( + dead_code, + reason = "keeps the Keystone client alive for the mapper's lifetime; the mapping paths do not call through it yet (backlog#1823)" + )] client: Arc, role_policy_map: HashMap, enable_tenant_prefix: bool, diff --git a/crates/notify/src/rules/config.rs b/crates/notify/src/rules/config.rs index 42ca136fa..624f99787 100644 --- a/crates/notify/src/rules/config.rs +++ b/crates/notify/src/rules/config.rs @@ -40,7 +40,10 @@ impl RuleEvents for RuleView { #[derive(Debug)] struct CompiledRules { // Keep RulesMap (can be used later if you want to make more complex judgments during the snapshot reading phase) - #[allow(dead_code)] + #[allow( + dead_code, + reason = "speculative retention: the comment above keeps it for richer snapshot-time judgements that no code performs yet (backlog#1823)" + )] rules_map: RulesMap, // for RulesContainer::iter_rules rule_views: Vec, diff --git a/crates/notify/src/rules/rules_map.rs b/crates/notify/src/rules/rules_map.rs index 9ae40ad85..7d1314893 100644 --- a/crates/notify/src/rules/rules_map.rs +++ b/crates/notify/src/rules/rules_map.rs @@ -187,7 +187,6 @@ impl RulesMap { /// # Parameters /// * `event_name` - The EventName from which to remove the rule. /// * `pattern` - The pattern of the rule to be removed. - #[allow(dead_code)] pub fn remove_rule(&mut self, event_name: &EventName, pattern: &str) { let mut remove_event = false; @@ -209,7 +208,6 @@ impl RulesMap { /// /// # Parameters /// * `event_names` - A slice of EventNames to be removed. - #[allow(dead_code)] pub fn remove_rules(&mut self, event_names: &[EventName]) { for event_name in event_names { self.map.remove(event_name); @@ -223,7 +221,6 @@ impl RulesMap { /// * `event_name` - The EventName to update. /// * `pattern` - The pattern of the rule to be updated. /// * `target_id` - The TargetID to be added. - #[allow(dead_code)] pub fn update_rule(&mut self, event_name: EventName, pattern: String, target_id: TargetID) { self.map.entry(event_name).or_default().add(pattern, target_id); self.total_events_mask |= event_name.mask(); // Update only the relevant bitmask diff --git a/crates/notify/src/rules/target_id_set.rs b/crates/notify/src/rules/target_id_set.rs index d5036975c..06c2c2f87 100644 --- a/crates/notify/src/rules/target_id_set.rs +++ b/crates/notify/src/rules/target_id_set.rs @@ -18,12 +18,6 @@ use rustfs_targets::arn::TargetID; /// TargetIDSet - A collection representation of TargetID. pub type TargetIdSet = HashSet; -/// Provides a Go-like method for TargetIdSet (can be implemented as trait if needed) -#[allow(dead_code)] -pub(crate) fn new_target_id_set(target_ids: Vec) -> TargetIdSet { - target_ids.into_iter().collect() -} - // HashSet has built-in clone, union, difference and other operations. // But the Go version of the method returns a new Set, and the HashSet method is usually iterator or modify itself. // If you need to exactly match Go's API style, you can add wrapper functions. diff --git a/crates/policy/src/policy/function.rs b/crates/policy/src/policy/function.rs index b1fa5d1ee..55bf7e13a 100644 --- a/crates/policy/src/policy/function.rs +++ b/crates/policy/src/policy/function.rs @@ -219,10 +219,6 @@ impl PartialEq for Functions { } } -#[derive(Clone, Serialize, Deserialize)] -#[allow(dead_code)] -pub struct Value; - #[cfg(test)] mod tests { use crate::policy::Functions; diff --git a/crates/policy/src/policy/utils/wildcard.rs b/crates/policy/src/policy/utils/wildcard.rs index 915cb153e..0b8fa3bb2 100644 --- a/crates/policy/src/policy/utils/wildcard.rs +++ b/crates/policy/src/policy/utils/wildcard.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[allow(dead_code)] pub fn is_simple_match(pattern: P, name: N) -> bool where P: AsRef, @@ -29,7 +28,10 @@ where inner_match(pattern, name, false) } -#[allow(dead_code)] +#[allow( + dead_code, + reason = "prefix-matcher asserted by this file's tests; no production caller yet (backlog#1823)" +)] pub fn is_match_as_pattern_prefix(pattern: P, text: N) -> bool where P: AsRef, diff --git a/crates/trusted-proxies/src/cloud/metadata/aws.rs b/crates/trusted-proxies/src/cloud/metadata/aws.rs index 506eca925..c971cfa4d 100644 --- a/crates/trusted-proxies/src/cloud/metadata/aws.rs +++ b/crates/trusted-proxies/src/cloud/metadata/aws.rs @@ -27,6 +27,10 @@ use crate::CloudMetadataFetcher; #[derive(Debug, Clone)] pub struct AwsMetadataFetcher { client: Client, + #[allow( + dead_code, + reason = "IMDS endpoint retained beside the client it configures; requests build their own URLs (backlog#1823)" + )] metadata_endpoint: String, } @@ -46,55 +50,6 @@ impl AwsMetadataFetcher { metadata_endpoint: "http://169.254.169.254".to_string(), } } - - /// Retrieves an IMDSv2 token for secure metadata access. - #[allow(dead_code)] - async fn get_metadata_token(&self) -> Result { - let url = format!("{}/latest/api/token", self.metadata_endpoint); - - match self - .client - .put(&url) - .header("X-aws-ec2-metadata-token-ttl-seconds", "21600") - .send() - .await - { - Ok(response) => { - if response.status().is_success() { - let token = response - .text() - .await - .map_err(|e| AppError::cloud(format!("Failed to read IMDSv2 token: {}", e)))?; - Ok(token) - } else { - debug!( - event = "trusted_proxies.cloud_metadata", - component = "trusted_proxies", - subsystem = "aws_metadata", - provider = "aws", - operation = "imdsv2_token", - result = "http_error", - status = %response.status(), - "trusted proxy cloud metadata request failed" - ); - Err(AppError::cloud("Failed to obtain IMDSv2 token")) - } - } - Err(e) => { - debug!( - event = "trusted_proxies.cloud_metadata", - component = "trusted_proxies", - subsystem = "aws_metadata", - provider = "aws", - operation = "imdsv2_token", - result = "request_failed", - error = %e, - "trusted proxy cloud metadata request failed" - ); - Err(AppError::cloud(format!("IMDSv2 request failed: {}", e))) - } - } - } } #[async_trait] diff --git a/crates/trusted-proxies/src/config/env.rs b/crates/trusted-proxies/src/config/env.rs index a982ae0d8..2a8a3ff6a 100644 --- a/crates/trusted-proxies/src/config/env.rs +++ b/crates/trusted-proxies/src/config/env.rs @@ -68,7 +68,6 @@ pub fn is_env_set(key: &str) -> bool { } /// Returns a list of all proxy-related environment variables and their current values. -#[allow(dead_code)] pub fn get_all_proxy_env_vars() -> Vec<(String, String)> { let vars = [ ENV_TRUSTED_PROXY_ENABLED, diff --git a/crates/utils/src/io.rs b/crates/utils/src/io.rs index 92e69e5db..44388b0f5 100644 --- a/crates/utils/src/io.rs +++ b/crates/utils/src/io.rs @@ -68,7 +68,6 @@ pub async fn read_full_or_eof( /// Read exactly buf.len() bytes into buf, or return an error if EOF is reached before any bytes are read. /// Like Go's io.ReadFull. -#[allow(dead_code)] pub async fn read_full(reader: R, buf: &mut [u8]) -> std::io::Result { match read_full_or_eof(reader, buf).await? { Some(n) => Ok(n), diff --git a/crates/utils/src/net.rs b/crates/utils/src/net.rs index 873a6d940..3489ba41c 100644 --- a/crates/utils/src/net.rs +++ b/crates/utils/src/net.rs @@ -431,7 +431,6 @@ pub fn parse_and_resolve_address(addr_str: &str) -> std::io::Result Ok(resolved_addr) } -#[allow(dead_code)] pub fn bytes_stream(stream: S, content_length: usize) -> impl Stream> + Send + 'static where S: Stream> + Send + 'static, diff --git a/crates/utils/src/os/fs_type.rs b/crates/utils/src/os/fs_type.rs index a3ae1045c..3650cb793 100644 --- a/crates/utils/src/os/fs_type.rs +++ b/crates/utils/src/os/fs_type.rs @@ -16,7 +16,6 @@ /// /// The table follows Linux `include/uapi/linux/magic.h`; filesystem magic /// values without a stable Linux uapi source stay `UNKNOWN`. -#[allow(dead_code)] pub(crate) fn get_fs_type(fs_type: u64) -> &'static str { // Magic numbers for various filesystems. match fs_type { diff --git a/crates/utils/src/path.rs b/crates/utils/src/path.rs index c383dc822..8b53e764b 100644 --- a/crates/utils/src/path.rs +++ b/crates/utils/src/path.rs @@ -70,7 +70,6 @@ pub fn is_dir_object(object: &str) -> bool { /// /// If the object name ends with `GLOBAL_DIR_SUFFIX`, it is replaced with a slash. /// Otherwise, the name is returned as is. -#[allow(dead_code)] pub fn decode_dir_object(object: &str) -> String { if has_suffix(object, GLOBAL_DIR_SUFFIX) { format!("{}{}", object.trim_end_matches(GLOBAL_DIR_SUFFIX), SLASH_SEPARATOR) From c04ee41cf088b86150b6bb75f13fed8913fcf0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Mon, 17 Aug 2026 14:12:04 +0800 Subject: [PATCH 04/51] feat(site-replication): drain the retry queue from the reconcile tick (#6131) --- crates/madmin/src/site_replication.rs | 6 +- rustfs/src/admin/handlers/site_replication.rs | 1057 ++++++++++++++++- 2 files changed, 1051 insertions(+), 12 deletions(-) diff --git a/crates/madmin/src/site_replication.rs b/crates/madmin/src/site_replication.rs index a8813f99c..740924405 100644 --- a/crates/madmin/src/site_replication.rs +++ b/crates/madmin/src/site_replication.rs @@ -258,7 +258,7 @@ pub struct SRLDAPUser { pub api_version: Option, } -#[derive(Debug, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct SRIAMUser { #[serde(rename = "accessKey", default)] pub access_key: String, @@ -270,7 +270,7 @@ pub struct SRIAMUser { pub api_version: Option, } -#[derive(Debug, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct SRGroupInfo { #[serde(rename = "updateReq", default)] pub update_req: GroupAddRemove, @@ -346,7 +346,7 @@ pub struct SRCredInfo { pub api_version: Option, } -#[derive(Debug, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct SRIAMItem { #[serde(default)] pub r#type: String, diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index d9604fe6f..fba311ba9 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -2986,6 +2986,18 @@ fn reconcile_site_replication_wiring() -> std::pin::Pin { if state.pending_endpoint_refresh.is_some() || state.pending_remove.is_some() || state.pending_rotation.is_some() @@ -3017,6 +3029,9 @@ fn reconcile_site_replication_wiring() -> std::pin::Pin upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None), None => { - dequeue_site_replication_retry_events(&mut state.retry_queue, &peer, &path); + dequeue_site_replication_retry_events_including_escalated(&mut state.retry_queue, &peer, &path); } } Ok(()) @@ -6071,10 +6086,96 @@ fn retry_event_matches(event: &SiteReplicationRetryEvent, peer: &PeerInfo, path: (event.peer_deployment_id == peer.deployment_id || event.peer_endpoint == peer.endpoint) && event.path == path } +const SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH: &str = "internal:retry-snapshot:iam"; +const SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH: &str = "internal:retry-snapshot:bucket-metadata"; + +fn collapsed_retry_queue_path(path: &str) -> Option<&'static str> { + let base_path = path.split_once('?').map(|(base, _)| base).unwrap_or(path); + match base_path { + "/rustfs/admin/v3/site-replication/peer/iam-item" | SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH => { + Some(SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH) + } + "/rustfs/admin/v3/site-replication/peer/bucket-meta" | SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH => { + Some(SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH) + } + _ => None, + } +} + +fn normalize_collapsed_retry_queue_paths(queue: &mut Vec) -> bool { + let mut changed = false; + let mut normalized: Vec = Vec::with_capacity(queue.len()); + for mut event in queue.drain(..) { + if let Some(path) = collapsed_retry_queue_path(&event.path) + && event.path != path + { + event.path = path.to_string(); + changed = true; + } + + let duplicate = normalized.iter().position(|existing| { + existing.path == event.path + && (existing.peer_deployment_id == event.peer_deployment_id || existing.peer_endpoint == event.peer_endpoint) + }); + let Some(index) = duplicate else { + normalized.push(event); + continue; + }; + + changed = true; + let existing = &mut normalized[index]; + let event_is_newer = match (event.updated_at, existing.updated_at) { + (Some(event), Some(existing)) => event >= existing, + (Some(_), None) => true, + _ => false, + }; + if event_is_newer { + let retry_count = existing.retry_count.max(event.retry_count); + *existing = event; + existing.retry_count = retry_count; + } else { + existing.retry_count = existing.retry_count.max(event.retry_count); + } + existing.failed = existing.retry_count >= SITE_REPLICATION_RETRY_FAILED_AFTER; + } + *queue = normalized; + changed +} + +async fn migrate_collapsed_retry_queue_paths() -> S3Result<()> { + update_site_replication_state_when_changed(|state| { + Ok(if normalize_collapsed_retry_queue_paths(&mut state.retry_queue) { + StateCommit::Changed(()) + } else { + StateCommit::Unchanged(()) + }) + }) + .await +} + +#[cfg(test)] fn dequeue_site_replication_retry_events(queue: &mut Vec, peer: &PeerInfo, path: &str) -> usize { settle_site_replication_retry_events(queue, peer, path, None) } +/// Repair-path settlement: also clears snapshot-escalated entries. Running a +/// repair is the operator's explicit accountability transfer for the +/// possibly-unreplayed deletion the marker records; ordinary delivery +/// successes must not clear it (see [`settle_site_replication_retry_events`]). +fn dequeue_site_replication_retry_events_including_escalated( + queue: &mut Vec, + peer: &PeerInfo, + path: &str, +) -> usize { + let before = queue.len(); + let collapsed_path = collapsed_retry_queue_path(path); + queue.retain(|event| { + !retry_event_matches(event, peer, path) + && !collapsed_path.is_some_and(|collapsed_path| retry_event_matches(event, peer, collapsed_path)) + }); + before.saturating_sub(queue.len()) +} + /// Remove the retry events for (peer, path) that `generation` is entitled to /// settle. A successful delivery only proves the peer reached the state the /// delivery carried: while it was in flight another edit can commit, fail its @@ -6090,10 +6191,24 @@ fn settle_site_replication_retry_events( generation: Option, ) -> usize { let before = queue.len(); + let collapsed_path = collapsed_retry_queue_path(path); queue.retain(|event| { if !retry_event_matches(event, peer, path) { return true; } + // A wire-path success identifies no IAM or bucket-metadata entity. + // This also protects legacy rows until the startup migration moves + // them under their internal snapshot path. + if collapsed_path.is_some() { + return true; + } + // A snapshot-escalated entry records a possibly-unreplayed deletion. + // Collapsed paths are shared by every entity, so a later successful + // delivery of a DIFFERENT item proves nothing about the deleted one — + // only a repair settles it (dequeue_..._including_escalated). + if event.last_error == SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { + return true; + } match (generation, event.edit_generation) { (Some(settled), Some(failed)) => failed > settled, _ => false, @@ -6109,6 +6224,7 @@ fn upsert_site_replication_retry_event( error: &str, generation: Option, ) { + let path = collapsed_retry_queue_path(path).unwrap_or(path); let now = OffsetDateTime::now_utc(); let detail = summarize_peer_error_detail(error); if let Some(event) = queue.iter_mut().find(|event| retry_event_matches(event, peer, path)) { @@ -6171,7 +6287,12 @@ async fn enqueue_site_replication_retry_event_for_generation( let path_owned = path.to_string(); let error_text = error.to_string(); let result = update_site_replication_state(move |state| { - upsert_site_replication_retry_event(&mut state.retry_queue, &peer_owned, &path_owned, &error_text, generation); + // A peer that left the state can never drain its entries again + // (remove_sites already pruned them); recording a late failure for it + // would only pollute retry_stats until the queue cap evicts it. + if state.peers.contains_key(&peer_owned.deployment_id) { + upsert_site_replication_retry_event(&mut state.retry_queue, &peer_owned, &path_owned, &error_text, generation); + } Ok(()) }) .await; @@ -6205,6 +6326,595 @@ fn retry_event_replayed_by_bootstrap(event: &SiteReplicationRetryEvent) -> bool ) } +/// Exponential backoff base for the background retry drain, aligned with the +/// reconcile cadence (`site_replication_reconcile::RECONCILE_INTERVAL`). +const SITE_REPLICATION_RETRY_DRAIN_BASE_BACKOFF_SECS: i64 = 600; +/// Backoff ceiling: a permanently failed peer is still probed daily. +const SITE_REPLICATION_RETRY_DRAIN_MAX_BACKOFF_SECS: i64 = 86_400; + +/// What the background drain may do for one retry event. Everything not +/// representable here is operator territory (manual repair). +#[derive(Debug, Clone, PartialEq, Eq)] +enum RetryDrainAction { + /// Constant-path IAM item deliveries collapse into one queue entry per + /// peer and their bodies are not persisted; the only faithful replay is + /// the current IAM snapshot from the bootstrap plan. + IamSnapshot, + /// Same collapse for bucket-meta deliveries: replay the bucket metadata + /// snapshot from the bootstrap plan. + BucketMetadataSnapshot, + /// A self-contained bucket op the bootstrap plan can re-derive for its + /// bucket (`make-with-versioning` / `configure-replication`). + BucketOpReplay { operation: String, bucket: String }, + /// Re-send the current peer records under a fresh edit generation. + PeerEdit, +} + +#[derive(Clone)] +enum RetrySnapshot { + Iam(Vec), + BucketMetadata(Vec), +} + +impl RetrySnapshot { + fn from_plan(action: &RetryDrainAction, plan: &SiteReplicationBootstrapPlan) -> Option { + match action { + RetryDrainAction::IamSnapshot => Some(Self::Iam(plan.iam_items.clone())), + RetryDrainAction::BucketMetadataSnapshot => Some(Self::BucketMetadata(plan.bucket_items.clone())), + _ => None, + } + } + + fn fingerprint(&self) -> S3Result>> { + let mut payloads = match self { + Self::Iam(items) => items.iter().map(serde_json::to_vec).collect::, _>>(), + Self::BucketMetadata(items) => items.iter().map(serde_json::to_vec).collect::, _>>(), + } + .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize retry snapshot failed: {err}")))?; + payloads.sort_unstable(); + Ok(payloads) + } + + fn replay_after_change(previous: &Self, fresh: &Self, observed_at: OffsetDateTime) -> Self { + match (previous, fresh) { + (Self::Iam(previous), Self::Iam(fresh)) => { + let fresh_keys: HashSet = fresh.iter().filter_map(iam_snapshot_key).collect(); + let mut replay = fresh.clone(); + for item in previous { + if iam_snapshot_key(item).is_some_and(|key| !fresh_keys.contains(&key)) { + replay.extend(iam_snapshot_tombstones(item, observed_at)); + } + } + Self::Iam(replay) + } + (Self::BucketMetadata(previous), Self::BucketMetadata(fresh)) => { + let fresh_keys: HashSet<(&str, &str)> = fresh + .iter() + .map(|item| (item.bucket.as_str(), item.r#type.as_str())) + .collect(); + let mut replay = fresh.clone(); + for item in previous { + if !fresh_keys.contains(&(item.bucket.as_str(), item.r#type.as_str())) { + replay.push(bucket_metadata_snapshot_tombstone(item, observed_at)); + } + } + Self::BucketMetadata(replay) + } + _ => fresh.clone(), + } + } + + async fn send(&self, transport: &PeerTransport, access_key: &str, secret_key: &str) -> S3Result<()> { + match self { + Self::Iam(items) => { + for item in items { + SiteReplicationRepairTask::Iam(item) + .send(transport, access_key, secret_key) + .await?; + } + } + Self::BucketMetadata(items) => { + for item in items { + SiteReplicationRepairTask::BucketMetadata(item) + .send(transport, access_key, secret_key) + .await?; + } + } + } + Ok(()) + } +} + +#[derive(Hash, PartialEq, Eq)] +enum IamSnapshotKey { + Policy(String), + User(String), + Group(String), + PolicyMapping { target: String, user_type: i64, is_group: bool }, +} + +fn iam_snapshot_key(item: &SRIAMItem) -> Option { + match item.r#type.as_str() { + "policy" => Some(IamSnapshotKey::Policy(item.name.clone())), + "iam-user" => item + .iam_user + .as_ref() + .map(|user| IamSnapshotKey::User(user.access_key.clone())), + "group-info" => item + .group_info + .as_ref() + .map(|group| IamSnapshotKey::Group(group.update_req.group.clone())), + "policy-mapping" => item.policy_mapping.as_ref().map(|mapping| IamSnapshotKey::PolicyMapping { + target: mapping.user_or_group.clone(), + user_type: mapping.user_type, + is_group: mapping.is_group, + }), + _ => None, + } +} + +fn iam_snapshot_tombstones(item: &SRIAMItem, observed_at: OffsetDateTime) -> Vec { + let mut tombstone = item.clone(); + tombstone.updated_at = Some(observed_at); + match item.r#type.as_str() { + "policy" => tombstone.policy = None, + "iam-user" => { + if let Some(user) = tombstone.iam_user.as_mut() { + user.is_delete_req = true; + user.user_req = None; + } + } + "group-info" => { + let Some(group) = tombstone.group_info.as_mut() else { + return Vec::new(); + }; + group.update_req.is_remove = true; + if group.update_req.members.is_empty() { + return vec![tombstone]; + } + let mut delete = tombstone.clone(); + if let Some(group) = delete.group_info.as_mut() { + group.update_req.members.clear(); + } + return vec![tombstone, delete]; + } + "policy-mapping" => { + if let Some(mapping) = tombstone.policy_mapping.as_mut() { + mapping.policy.clear(); + } + } + _ => return Vec::new(), + } + vec![tombstone] +} + +fn bucket_metadata_snapshot_tombstone(item: &SRBucketMeta, observed_at: OffsetDateTime) -> SRBucketMeta { + SRBucketMeta { + r#type: item.r#type.clone(), + bucket: item.bucket.clone(), + updated_at: Some(observed_at), + expiry_updated_at: Some(observed_at), + api_version: item.api_version.clone(), + ..Default::default() + } +} + +const SITE_REPLICATION_RETRY_SNAPSHOT_STABILITY_ATTEMPTS: usize = 3; + +fn classify_site_replication_retry_event(event: &SiteReplicationRetryEvent) -> Option { + let snapshot_action = match event.path.as_str() { + SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH => Some(RetryDrainAction::IamSnapshot), + SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH => Some(RetryDrainAction::BucketMetadataSnapshot), + _ => None, + }; + if snapshot_action.is_some() && event.last_error != SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { + return snapshot_action; + } + if event.path.starts_with("internal:") { + // Marker records store payloads in `last_error` (legacy + // pending-endpoint-refresh backup and snapshot liabilities); they are + // not drainable delivery failures. + return None; + } + if event.last_error == SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { + // Already snapshot-replayed once for this failure episode; a possible + // deletion cannot be replayed from a snapshot, so re-sending daily + // proves nothing. A new hook failure overwrites the marker. + return None; + } + let base_path = event.path.split_once('?').map(|(base, _)| base).unwrap_or(&event.path); + match base_path { + "/rustfs/admin/v3/site-replication/peer/iam-item" => Some(RetryDrainAction::IamSnapshot), + "/rustfs/admin/v3/site-replication/peer/bucket-meta" => Some(RetryDrainAction::BucketMetadataSnapshot), + SITE_REPLICATION_PEER_EDIT_PATH => Some(RetryDrainAction::PeerEdit), + SITE_REPLICATION_PEER_BUCKET_OPS_PATH => { + let operation = retry_bucket_operation(&event.path)?; + if !matches!( + operation.as_str(), + SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING | SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION + ) { + // Destructive ops (delete-bucket / force-delete-bucket) are + // operator territory: replaying them against a peer whose + // bucket was since recreated is irreversible. + return None; + } + let bucket = retry_bucket_name(&event.path)?; + Some(RetryDrainAction::BucketOpReplay { operation, bucket }) + } + _ => None, + } +} + +fn retry_bucket_name(path: &str) -> Option { + let (_, query) = path.split_once('?')?; + form_urlencoded::parse(query.as_bytes()) + .find_map(|(key, value)| (key == "bucket" && !value.is_empty()).then(|| value.into_owned())) +} + +/// A collapsed retry event after a stable snapshot resend is escalated with +/// this marker instead of being cleared: the snapshot contains no task for a +/// failed deletion, so remote absence remains operator-visible. Collapsed +/// failures use an internal queue path so ordinary successes and older nodes +/// cannot settle an unrelated entity's liability. +const SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER: &str = "snapshot replayed; a failed deletion cannot be replayed from a snapshot — run site replication repair or re-deliver to settle"; + +/// Escalate a collapsed retry event after its snapshot resend succeeded, +/// unless a newer failure was recorded after `snapshot_updated_at` (that +/// failure belongs to a newer local commit the snapshot did not contain and +/// must keep the entry drain-eligible). +fn escalate_site_replication_retry_events_up_to( + queue: &mut Vec, + peer: &PeerInfo, + path: &str, + snapshot_updated_at: Option, +) -> usize { + let Some(marker_path) = collapsed_retry_queue_path(path) else { + return 0; + }; + + if path != marker_path { + queue.retain(|event| { + if !retry_event_matches(event, peer, path) { + return true; + } + matches!((event.updated_at, snapshot_updated_at), (Some(current), Some(seen)) if current > seen) + || matches!((event.updated_at, snapshot_updated_at), (Some(_), None)) + }); + } + + let marker_index = queue.iter().position(|event| retry_event_matches(event, peer, marker_path)); + let marker_index = marker_index.unwrap_or_else(|| { + queue.push(SiteReplicationRetryEvent { + id: Uuid::new_v4().to_string(), + peer_deployment_id: peer.deployment_id.clone(), + peer_endpoint: peer.endpoint.clone(), + path: marker_path.to_string(), + updated_at: snapshot_updated_at, + ..Default::default() + }); + queue.len() - 1 + }); + let event = &mut queue[marker_index]; + let newer_failure_recorded = match (event.updated_at, snapshot_updated_at) { + (Some(current), Some(seen)) => current > seen, + (Some(_), None) => true, + (None, _) => false, + }; + if newer_failure_recorded && event.last_error != SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER { + return 0; + } + event.failed = true; + event.retry_count = event.retry_count.max(SITE_REPLICATION_RETRY_FAILED_AFTER); + event.last_error = SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER.to_string(); + event.updated_at = Some(OffsetDateTime::now_utc()); + 1 +} + +async fn escalate_site_replication_retry_event_up_to(peer: &PeerInfo, path: &str, snapshot_updated_at: Option) { + let peer_owned = peer.clone(); + let path_owned = path.to_string(); + let result = update_site_replication_state(move |state| { + escalate_site_replication_retry_events_up_to(&mut state.retry_queue, &peer_owned, &path_owned, snapshot_updated_at); + Ok(()) + }) + .await; + + if let Err(err) = result { + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + peer = %peer.endpoint, + deployment_id = %peer.deployment_id, + path, + error = ?err, + "failed to escalate site replication retry event" + ); + } +} + +/// Whether the drain may attempt this event now. +fn site_replication_retry_backoff_elapsed(event: &SiteReplicationRetryEvent, now: OffsetDateTime) -> bool { + let Some(updated_at) = event.updated_at else { + return true; + }; + // 600 * 2^8 already exceeds the daily ceiling; capping the shift keeps + // the arithmetic overflow-free for any persisted retry_count. + let exponent = event.retry_count.saturating_sub(1).min(8); + let delay = (SITE_REPLICATION_RETRY_DRAIN_BASE_BACKOFF_SECS << exponent).min(SITE_REPLICATION_RETRY_DRAIN_MAX_BACKOFF_SECS); + now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) >= delay +} + +/// The subset of the retry queue the background drain is allowed to touch. +fn actionable_site_replication_retry_events(state: &SiteReplicationState, now: OffsetDateTime) -> Vec { + state + .retry_queue + .iter() + .filter(|event| classify_site_replication_retry_event(event).is_some()) + .filter(|event| state.peers.contains_key(&event.peer_deployment_id)) + .filter(|event| site_replication_retry_backoff_elapsed(event, now)) + .cloned() + .collect() +} + +/// Background consumer for the retry queue, run from the reconcile tick. +/// +/// Scope: this settles "delivered once and failed" entries whose replay is +/// faithful (bucket ops, peer edits). Collapsed iam-item / bucket-meta +/// entries are snapshot-resent and then *escalated*, not cleared — a failed +/// deletion leaves no task in the snapshot, so remote absence stays unproven +/// until a later delivery or a manual repair. A hook that never fired (crash +/// between the local commit and the send) leaves no entry at all, so the +/// drain is not a full cross-site diff-heal; manual repair remains the +/// authoritative catch-all. +async fn drain_site_replication_retry_queue() { + if let Err(err) = drain_site_replication_retry_queue_inner().await { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "retry_drain_failed", + error = ?err, + "admin site replication state" + ); + } +} + +async fn drain_site_replication_retry_queue_inner() -> S3Result<()> { + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + let actionable = actionable_site_replication_retry_events(&runtime.state, OffsetDateTime::now_utc()); + if actionable.is_empty() { + return Ok(()); + } + let Some(store) = current_object_store_handle() else { + return Ok(()); + }; + if runtime.state.pending_endpoint_refresh.is_some() + || runtime.state.pending_remove.is_some() + || runtime.state.pending_rotation.is_some() + { + // The tick-level gate ran before the reconcilers; a multi-step flow + // (endpoint refresh commits its pending marker without the lifecycle + // guard) may have started since. Re-check on the fresh state. + return Ok(()); + } + // Serialize against operator repair execution. This does NOT close the + // dry-run -> execute window (dry-run takes no lock): a drain settling a + // replayable bucket-op entry in that window changes the preflight token + // and execute fails safe with "preflight is stale" — the operator + // re-runs the dry-run. Lock order matches repair: lifecycle guard (held + // by the reconcile tick) -> repair execution lock -> state object lock + // inside the send bookkeeping. An operator repair holding the lock makes + // this tick skip after the lock-acquire timeout. + with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { + drain_site_replication_retry_queue_locked(runtime, actionable).await + }) + .await + .map_err(ApiError::from)? +} + +async fn drain_site_replication_retry_queue_locked( + runtime: SiteReplicationRuntime, + events: Vec, +) -> S3Result<()> { + let needs_plan = events + .iter() + .any(|event| !matches!(classify_site_replication_retry_event(event), Some(RetryDrainAction::PeerEdit))); + // The plan is a full local snapshot (buckets + IAM); build it once per + // tick and only when a snapshot resend is actually due. + let plan = if needs_plan { + let info = build_sr_info(&runtime.state, &runtime.local_peer).await?; + Some(site_replication_bootstrap_plan(&info)?) + } else { + None + }; + + let mut events_by_peer: BTreeMap> = BTreeMap::new(); + for event in events { + events_by_peer + .entry(event.peer_deployment_id.clone()) + .or_default() + .push(event); + } + + let mut settled = 0usize; + let mut failures = 0usize; + for (deployment_id, peer_events) in events_by_peer { + let Some(peer) = runtime.state.peers.get(&deployment_id) else { + continue; + }; + if deployment_id == runtime.local_peer.deployment_id + || same_identity_endpoint(&peer.endpoint, &runtime.local_peer.endpoint) + { + continue; + } + let transport = match PeerTransport::for_runtime_peer(peer).await { + Ok(transport) => transport, + Err(err) => { + // Record the attempt so backoff advances for an unreachable + // peer instead of re-dialing it every tick. + for event in &peer_events { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + } + failures += peer_events.len(); + continue; + } + }; + for event in peer_events { + let Some(action) = classify_site_replication_retry_event(&event) else { + continue; + }; + match drain_one_site_replication_retry_event(&runtime, peer, &transport, &event, action, plan.as_ref()).await { + Ok(true) => settled += 1, + Ok(false) => {} + Err(_) => failures += 1, + } + } + } + + if settled > 0 || failures > 0 { + info!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "retry_drain_settled", + settled, + failures, + "admin site replication state" + ); + } + Ok(()) +} + +/// Replay one retry event against its peer. Returns `Ok(true)` when the +/// event was settled (delivered, or provably stale), `Ok(false)` when it was +/// skipped, and `Err` after a failed delivery (already re-queued with an +/// incremented retry count). +async fn drain_one_site_replication_retry_event( + runtime: &SiteReplicationRuntime, + peer: &PeerInfo, + transport: &PeerTransport, + event: &SiteReplicationRetryEvent, + action: RetryDrainAction, + plan: Option<&SiteReplicationBootstrapPlan>, +) -> S3Result { + let access_key = &runtime.state.service_account_access_key; + let secret_key = &runtime.service_account_secret_key; + match action.clone() { + RetryDrainAction::IamSnapshot | RetryDrainAction::BucketMetadataSnapshot => { + let Some(plan) = plan else { + return Ok(false); + }; + let mut current_snapshot = RetrySnapshot::from_plan(&action, plan).expect("snapshot action has a snapshot"); + let mut replay = current_snapshot.clone(); + for _ in 0..SITE_REPLICATION_RETRY_SNAPSHOT_STABILITY_ATTEMPTS { + let current_fingerprint = current_snapshot.fingerprint()?; + if let Err(err) = replay.send(transport, access_key, secret_key).await { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + return Err(err); + } + let fresh_info = build_sr_info(&runtime.state, &runtime.local_peer).await?; + let fresh_plan = site_replication_bootstrap_plan(&fresh_info)?; + let fresh_snapshot = RetrySnapshot::from_plan(&action, &fresh_plan).expect("snapshot action has a snapshot"); + if fresh_snapshot.fingerprint()? == current_fingerprint { + escalate_site_replication_retry_event_up_to(peer, &event.path, event.updated_at).await; + return Ok(true); + } + replay = RetrySnapshot::replay_after_change(¤t_snapshot, &fresh_snapshot, OffsetDateTime::now_utc()); + current_snapshot = fresh_snapshot; + } + Ok(false) + } + RetryDrainAction::BucketOpReplay { operation, bucket } => { + let Some(plan) = plan else { + return Ok(false); + }; + // Replay from the CURRENT plan, never the recorded path: the + // recorded query can carry an expired one-shot bootstrap token or + // a stale createdAt. + let make_op = operation == SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING; + let paths = if make_op { + &plan.bucket_make_ops + } else { + &plan.bucket_configure_ops + }; + let tasks: Vec> = paths + .iter() + .filter(|path| retry_bucket_name(path).as_deref() == Some(bucket.as_str())) + .map(|path| { + if make_op { + SiteReplicationRepairTask::BucketMake(path) + } else { + SiteReplicationRepairTask::Replication(path) + } + }) + .collect(); + if tasks.is_empty() { + // The bucket left the plan (deleted, or replication no longer + // configured): the recorded intent is stale, settle it. + dequeue_site_replication_retry_event(peer, &event.path).await; + return Ok(true); + } + for task in &tasks { + if let Err(err) = task.send(transport, access_key, secret_key).await { + enqueue_site_replication_retry_event(peer, &event.path, &err).await; + return Err(err); + } + } + dequeue_site_replication_retry_event(peer, &event.path).await; + Ok(true) + } + RetryDrainAction::PeerEdit => { + // The recorded generation is stale by definition — the receiver + // fences it. Allocate a fresh generation and re-send the current + // peer records (a superset of the failed body; the receiver + // upserts), all inside one state transaction so the fence and the + // bodies agree. + let target_id = peer.deployment_id.clone(); + let (generation, bodies) = update_site_replication_state(move |state| { + if !state.peers.contains_key(&target_id) { + return Ok((None, Vec::new())); + } + Ok((Some(next_peer_edit_generation(state)), state.peers.values().cloned().collect::>())) + }) + .await?; + let Some(generation) = generation else { + // Peer left between the snapshot and now; the queue entry was + // already pruned by remove_sites. + return Ok(false); + }; + let local_deployment_id = Some(runtime.local_peer.deployment_id.as_str()).filter(|id| !id.is_empty()); + let edit_path = peer_edit_path_with_fence(local_deployment_id, generation); + let delivery_fence = local_deployment_id.is_some().then_some(generation); + for body in &bodies { + if let Err(err) = send_peer_admin_request_with_client( + &transport.client, + &transport.connection, + &edit_path, + access_key, + secret_key, + body, + ) + .await + { + enqueue_site_replication_retry_event_for_generation( + peer, + SITE_REPLICATION_PEER_EDIT_PATH, + &err, + delivery_fence, + ) + .await; + return Err(err); + } + } + dequeue_site_replication_retry_event_for_generation(peer, SITE_REPLICATION_PEER_EDIT_PATH, delivery_fence).await; + Ok(true) + } + } +} + /// Remove a retry event for (peer, path) from the queue on successful delivery. /// This is a no-op (load + no-op persist skipped) when no matching entry exists, /// avoiding unnecessary I/O on the common path. @@ -11839,6 +12549,320 @@ mod tests { assert!(target_state.peers["remote"].skip_tls_verify); } + fn drain_event(peer: &str, path: &str, retry_count: u32, updated_at: Option) -> SiteReplicationRetryEvent { + SiteReplicationRetryEvent { + id: format!("evt-{peer}"), + peer_deployment_id: peer.to_string(), + peer_endpoint: format!("https://{peer}.example.com"), + path: path.to_string(), + retry_count, + failed: retry_count >= SITE_REPLICATION_RETRY_FAILED_AFTER, + last_error: "remote-operation-failed".to_string(), + updated_at, + edit_generation: None, + } + } + + /// P1-3 red-light: the drain must only ever act on deliveries it can + /// replay faithfully. IAM / bucket-meta entries collapse per (peer, path) + /// with no body persisted — only a snapshot resend is truthful; bucket + /// makes/replication configs are re-derivable; destructive bucket ops and + /// unrelated `internal:` marker records are never background-replayed. + #[test] + fn test_classify_site_replication_retry_event_actions() { + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let classify = |path: &str| classify_site_replication_retry_event(&drain_event("remote", path, 1, Some(now))); + + assert_eq!( + classify("/rustfs/admin/v3/site-replication/peer/iam-item"), + Some(RetryDrainAction::IamSnapshot) + ); + assert_eq!( + classify("/rustfs/admin/v3/site-replication/peer/bucket-meta"), + Some(RetryDrainAction::BucketMetadataSnapshot) + ); + assert_eq!(classify(SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH), Some(RetryDrainAction::IamSnapshot)); + assert_eq!( + classify(SITE_REPLICATION_RETRY_BUCKET_METADATA_SNAPSHOT_PATH), + Some(RetryDrainAction::BucketMetadataSnapshot) + ); + assert_eq!(classify(SITE_REPLICATION_PEER_EDIT_PATH), Some(RetryDrainAction::PeerEdit)); + assert_eq!( + classify( + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning&createdAt=1" + ), + Some(RetryDrainAction::BucketOpReplay { + operation: SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING.to_string(), + bucket: "photos".to_string(), + }) + ); + assert_eq!( + classify("/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=configure-replication"), + Some(RetryDrainAction::BucketOpReplay { + operation: SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION.to_string(), + bucket: "photos".to_string(), + }) + ); + // Destructive ops are operator territory: replaying a bucket delete + // against a peer whose bucket was since recreated is irreversible. + assert_eq!( + classify("/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=delete-bucket"), + None + ); + assert_eq!( + classify("/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=force-delete-bucket"), + None + ); + // `internal:` records store payloads in `last_error`, not failures. + assert_eq!(classify(SITE_REPLICATION_ENDPOINT_REFRESH_RETRY_PATH), None); + assert_eq!(classify("internal:some-future-marker"), None); + assert_eq!(classify("/rustfs/admin/v3/site-replication/peer/unknown"), None); + } + + #[test] + fn test_retry_snapshot_fingerprint_detects_concurrent_iam_change() { + let old = SRIAMItem { + r#type: "policy".to_string(), + name: "readwrite".to_string(), + updated_at: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")), + ..Default::default() + }; + let mut new = old.clone(); + new.updated_at = Some(OffsetDateTime::from_unix_timestamp(1_700_000_001).expect("timestamp")); + + let sent = RetrySnapshot::Iam(vec![old]); + let changed = RetrySnapshot::Iam(vec![new]); + assert_ne!(sent.fingerprint().unwrap(), changed.fingerprint().unwrap()); + } + + #[test] + fn test_retry_snapshot_replays_a_concurrent_deletion_as_a_tombstone() { + let observed_at = OffsetDateTime::from_unix_timestamp(1_700_000_010).expect("timestamp"); + let policy = SRIAMItem { + r#type: "policy".to_string(), + name: "readwrite".to_string(), + policy: Some(serde_json::json!({"Version": "2012-10-17"})), + ..Default::default() + }; + let replay = + RetrySnapshot::replay_after_change(&RetrySnapshot::Iam(vec![policy]), &RetrySnapshot::Iam(Vec::new()), observed_at); + let RetrySnapshot::Iam(items) = replay else { + panic!("IAM snapshot expected"); + }; + assert_eq!(items.len(), 1); + assert_eq!(items[0].name, "readwrite"); + assert!(items[0].policy.is_none()); + assert_eq!(items[0].updated_at, Some(observed_at)); + + let bucket = SRBucketMeta { + r#type: "tags".to_string(), + bucket: "photos".to_string(), + tags: Some("encoded-tags".to_string()), + ..Default::default() + }; + let replay = RetrySnapshot::replay_after_change( + &RetrySnapshot::BucketMetadata(vec![bucket]), + &RetrySnapshot::BucketMetadata(Vec::new()), + observed_at, + ); + let RetrySnapshot::BucketMetadata(items) = replay else { + panic!("bucket metadata snapshot expected"); + }; + assert_eq!(items.len(), 1); + assert_eq!(items[0].bucket, "photos"); + assert_eq!(items[0].r#type, "tags"); + assert!(items[0].tags.is_none()); + assert_eq!(items[0].updated_at, Some(observed_at)); + } + + /// Exponential backoff gates every attempt: without it a dead peer's + /// entries hit `failed` (retry_count >= 3) within 30 minutes of reconcile + /// ticks and the retry stats lose their signal. + #[test] + fn test_site_replication_retry_backoff_schedule() { + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let at = |secs_ago: i64| Some(now - time::Duration::seconds(secs_ago)); + let elapsed = |retry_count: u32, secs_ago: i64| { + site_replication_retry_backoff_elapsed(&drain_event("remote", "/p", retry_count, at(secs_ago)), now) + }; + + // No record of when it failed: attempt now. + assert!(site_replication_retry_backoff_elapsed(&drain_event("remote", "/p", 1, None), now)); + // First failure: one reconcile interval. + assert!(!elapsed(1, 599)); + assert!(elapsed(1, 601)); + // Third failure: 600 * 2^2 = 2400s. + assert!(!elapsed(3, 1200)); + assert!(elapsed(3, 2401)); + // Ceiling: a long-dead peer is still probed daily, never less often. + assert!(!elapsed(30, 86_000)); + assert!(elapsed(30, 86_401)); + } + + /// The actionable subset respects classification, peer membership and + /// backoff; everything else stays untouched in the queue. + #[test] + fn test_actionable_site_replication_retry_events_filters() { + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let old = Some(now - time::Duration::seconds(700)); + let mut state = SiteReplicationState::default(); + state + .peers + .insert("remote".to_string(), peer("remote", "https://remote.example.com")); + + state.retry_queue = vec![ + // Eligible: known peer, replayable, past backoff. + drain_event("remote", SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, 1, old), + // Not yet due. + drain_event("remote", "/rustfs/admin/v3/site-replication/peer/bucket-meta", 2, Some(now)), + // Unknown peer (removed since the failure was recorded). + drain_event("gone", "/rustfs/admin/v3/site-replication/peer/iam-item", 1, old), + // Marker record, not a delivery failure. + drain_event("remote", SITE_REPLICATION_ENDPOINT_REFRESH_RETRY_PATH, 0, old), + // Destructive op: operator-only. + drain_event( + "remote", + "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=delete-bucket", + 1, + old, + ), + ]; + + let actionable = actionable_site_replication_retry_events(&state, now); + assert_eq!(actionable.len(), 1, "only the due, replayable, known-peer event is actionable"); + assert_eq!(actionable[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); + } + + /// The drain settles a peer-edit success under a freshly allocated + /// generation; legacy queue entries carry `edit_generation: None` and + /// must be cleared by that generation-scoped settlement (`(Some, None)` + /// falls through to removal), or the drain would spin on them forever. + #[test] + fn test_settle_clears_legacy_none_generation_event_for_generation_scoped_success() { + let target = peer("remote", "https://remote.example.com"); + let mut queue = vec![drain_event("remote", SITE_REPLICATION_PEER_EDIT_PATH, 1, None)]; + assert!(queue[0].edit_generation.is_none()); + + let settled = settle_site_replication_retry_events(&mut queue, &target, SITE_REPLICATION_PEER_EDIT_PATH, Some(42)); + + assert_eq!(settled, 1, "a legacy None-generation event must settle under a newer generation"); + assert!(queue.is_empty()); + } + + /// A successful snapshot resend cannot prove a failed *deletion* was + /// replayed, so the collapsed entry is escalated (operator-visible, + /// drain-idle) instead of cleared — unless a newer failure was stamped + /// during the delivery window, which keeps the entry drain-eligible. + #[test] + fn test_escalate_up_to_marks_snapshot_replayed_and_keeps_newer_failures() { + let target = peer("remote", "https://remote.example.com"); + let path = "/rustfs/admin/v3/site-replication/peer/iam-item"; + let snapshot_at = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + + // Failure re-stamped after the snapshot: untouched, still eligible. + let mut queue = vec![drain_event( + "remote", + SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, + 2, + Some(snapshot_at + time::Duration::seconds(5)), + )]; + assert_eq!( + escalate_site_replication_retry_events_up_to( + &mut queue, + &target, + SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, + Some(snapshot_at), + ), + 0 + ); + assert!(!queue[0].failed); + assert!( + classify_site_replication_retry_event(&queue[0]).is_some(), + "a newer failure must stay drain-eligible" + ); + + // Unchanged since the snapshot: escalated, kept, drain-idle. + let mut queue = vec![drain_event( + "remote", + SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, + 2, + Some(snapshot_at), + )]; + assert_eq!( + escalate_site_replication_retry_events_up_to(&mut queue, &target, path, Some(snapshot_at)), + 1 + ); + assert_eq!(queue.len(), 1, "the entry must survive until remote absence is proven"); + assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); + assert!(queue[0].failed); + assert_eq!(queue[0].last_error, SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER); + assert!( + classify_site_replication_retry_event(&queue[0]).is_none(), + "a snapshot-replayed entry must not be re-sent daily" + ); + // Ordinary success dequeues must not clear the marker: collapsed + // paths are shared by every entity, so a successful Bob update + // proves nothing about a failed Alice deletion (second review + // round). + assert_eq!(dequeue_site_replication_retry_events(&mut queue, &target, path), 0); + assert_eq!(queue.len(), 1, "an escalated entry must survive an ordinary delivery success"); + // Only a repair — the operator's accountability transfer — settles it. + assert_eq!(dequeue_site_replication_retry_events_including_escalated(&mut queue, &target, path), 1); + assert!(queue.is_empty()); + + // A failed Alice deletion is stored under the internal path, so a + // successful Bob update on the shared wire path cannot erase it even + // before the drain runs. + let mut queue = Vec::new(); + upsert_site_replication_retry_event(&mut queue, &target, path, "alice delete failed", None); + assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); + assert_eq!(dequeue_site_replication_retry_events(&mut queue, &target, path), 0); + assert_eq!(queue.len(), 1); + assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); + + // A later hook failure overwrites the marker and re-arms the drain. + let mut queue = vec![drain_event("remote", path, 2, Some(snapshot_at))]; + escalate_site_replication_retry_events_up_to(&mut queue, &target, path, Some(snapshot_at)); + upsert_site_replication_retry_event(&mut queue, &target, path, "peer offline", None); + assert!(classify_site_replication_retry_event(&queue[0]).is_some()); + + // Legacy entry without a timestamp: escalated. + let mut queue = vec![drain_event("remote", path, 2, None)]; + assert_eq!( + escalate_site_replication_retry_events_up_to(&mut queue, &target, path, Some(snapshot_at)), + 1 + ); + + // A cloned event can disappear during replay; escalation recreates + // the internal liability while leaving another peer's row untouched. + let mut queue = vec![drain_event("other", path, 2, Some(snapshot_at))]; + assert_eq!( + escalate_site_replication_retry_events_up_to(&mut queue, &target, path, Some(snapshot_at)), + 1 + ); + assert!(!queue[0].failed); + assert_eq!(queue.len(), 2); + assert_eq!(queue[1].peer_deployment_id, target.deployment_id); + assert_eq!(queue[1].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); + } + + #[test] + fn test_collapsed_retry_queue_migration_preserves_legacy_liability() { + let peer = PeerInfo { + deployment_id: "remote-dep".to_string(), + ..peer("remote", "https://remote.example.com") + }; + let wire_path = "/rustfs/admin/v3/site-replication/peer/iam-item"; + let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp"); + let mut queue = vec![drain_event("remote-dep", wire_path, 2, Some(now))]; + + assert_eq!(dequeue_site_replication_retry_events(&mut queue, &peer, wire_path), 0); + assert!(normalize_collapsed_retry_queue_paths(&mut queue)); + assert_eq!(queue.len(), 1); + assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); + assert!(!normalize_collapsed_retry_queue_paths(&mut queue)); + } + #[test] fn test_pending_endpoint_refresh_retry_summary_redacts_pem() { let pem = "-----BEGIN CERTIFICATE-----\nsecret-marker\n-----END CERTIFICATE-----"; @@ -13545,6 +14569,7 @@ mod tests { upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "third", None); assert_eq!(queue.len(), 1); + assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); assert_eq!(queue[0].retry_count, SITE_REPLICATION_RETRY_FAILED_AFTER); assert!(queue[0].failed); assert_eq!(queue[0].last_error, "third"); @@ -13586,12 +14611,12 @@ mod tests { ); assert!(queue.is_empty()); - // Broadcast paths carry no generation and keep settling unconditionally - // — their retry events live under their own path and never collide - // with a peer-edit delivery. + // Collapsed broadcast failures live under an internal snapshot path; + // an unrelated success on their shared wire path cannot settle them. let iam_path = "/rustfs/admin/v3/site-replication/peer/iam-item"; upsert_site_replication_retry_event(&mut queue, &peer, iam_path, "peer offline", None); - assert_eq!(dequeue_site_replication_retry_events(&mut queue, &peer, iam_path), 1); + assert_eq!(dequeue_site_replication_retry_events(&mut queue, &peer, iam_path), 0); + assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); } /// P1-15 review follow-up: the receiving side of the ordering fence. Two @@ -13910,7 +14935,7 @@ mod tests { deployment_id: "current-dep".to_string(), ..peer("remote", "https://remote.example.com") }; - let path = "/rustfs/admin/v3/site-replication/peer/iam-item"; + let path = SITE_REPLICATION_PEER_EDIT_PATH; let mut queue = vec![ SiteReplicationRetryEvent { id: "same-endpoint".to_string(), @@ -17167,17 +18192,31 @@ mod tests { async fn test_retry_event_persist_must_not_wipe_concurrent_locked_rmw() { publish_ready_iam_context().await; + const ROUNDS: usize = 8; let seed = SiteReplicationState { pending_rotation: Some(PendingRotation { id: "rot-1".to_string(), access_key: "svc-account".to_string(), ..Default::default() }), + // Retry events are only recorded for current peers; seed them so + // the concurrency assertion below exercises the persist path. + peers: (0..ROUNDS) + .map(|round| { + let deployment_id = format!("peer-{round}-deployment"); + ( + deployment_id.clone(), + PeerInfo { + endpoint: format!("https://peer-{round}.example:9000"), + deployment_id, + ..Default::default() + }, + ) + }) + .collect(), ..Default::default() }; save_site_replication_state(&seed).await.expect("seed state"); - - const ROUNDS: usize = 8; for round in 0..ROUNDS { let peer = PeerInfo { endpoint: format!("https://peer-{round}.example:9000"), From d091554ffe73d7d82b29f4c32433f101cd7130fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Mon, 17 Aug 2026 14:12:39 +0800 Subject: [PATCH 05/51] fix(kms): resolve Vault auth from the environment at startup and add Kubernetes auth (#6095) --- crates/kms/src/api_types.rs | 58 +++ crates/kms/src/backends/vault.rs | 1 + crates/kms/src/backends/vault_credentials.rs | 276 ++++++++++++++- crates/kms/src/backends/vault_transit.rs | 1 + crates/kms/src/backup/vault_restore.rs | 4 + crates/kms/src/config.rs | 349 ++++++++++++++++--- docs/operations/kms-backend-security.md | 2 +- docs/operations/vault-kms-authentication.md | 54 ++- rustfs/src/admin/handlers/kms_backup.rs | 6 +- rustfs/src/init.rs | 215 ++++++++++-- 10 files changed, 862 insertions(+), 104 deletions(-) diff --git a/crates/kms/src/api_types.rs b/crates/kms/src/api_types.rs index b53aa4f34..6faf6cacc 100644 --- a/crates/kms/src/api_types.rs +++ b/crates/kms/src/api_types.rs @@ -293,6 +293,15 @@ enum StrictVaultAuthMethod { #[serde(default)] refresh_safety_window_secs: Option, }, + Kubernetes { + role: String, + #[serde(default)] + mount: Option, + #[serde(default)] + jwt_path: Option, + #[serde(default)] + refresh_safety_window_secs: Option, + }, TokenFile { path: std::path::PathBuf, #[serde(default)] @@ -319,6 +328,17 @@ impl From for VaultAuthMethod { mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()), refresh_safety_window_secs, }, + StrictVaultAuthMethod::Kubernetes { + role, + mount, + jwt_path, + refresh_safety_window_secs, + } => Self::Kubernetes { + role, + mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()), + jwt_path: jwt_path.unwrap_or_else(|| std::path::PathBuf::from(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH)), + refresh_safety_window_secs, + }, StrictVaultAuthMethod::TokenFile { path, poll_interval_secs, @@ -499,6 +519,7 @@ impl From<&KmsConfig> for KmsConfigSummary { auth_method_type: match &vault_config.auth_method { VaultAuthMethod::Token { .. } => "token".to_string(), VaultAuthMethod::AppRole { .. } => "approle".to_string(), + VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(), VaultAuthMethod::TokenFile { .. } => "token_file".to_string(), }, has_stored_credentials: true, @@ -513,6 +534,7 @@ impl From<&KmsConfig> for KmsConfigSummary { auth_method_type: match &vault_config.auth_method { VaultAuthMethod::Token { .. } => "token".to_string(), VaultAuthMethod::AppRole { .. } => "approle".to_string(), + VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(), VaultAuthMethod::TokenFile { .. } => "token_file".to_string(), }, has_stored_credentials: true, @@ -901,6 +923,42 @@ mod tests { assert!(request.to_kms_config().validate().is_ok()); } + /// The admin API reaches Kubernetes auth with the role alone; the mount and + /// the projected token path fall back to the cluster defaults, so a Tenant + /// manifest carries no credential and no cluster-specific paths. + #[test] + fn test_deserialize_vault_configure_request_accepts_kubernetes_auth() { + let raw = serde_json::json!({ + "backend_type": "vault-transit", + "address": "https://vault.example.com:8200", + "mount_path": "rustfs", + "auth_method": { "Kubernetes": { "role": "rustfs" } } + }); + + let request: ConfigureKmsRequest = serde_json::from_value(raw).expect("kubernetes auth should deserialize"); + let config = request.to_kms_config(); + config.validate().expect("kubernetes auth must validate"); + + let vault = config.vault_transit_config().expect("vault transit backend config"); + let VaultAuthMethod::Kubernetes { + role, mount, jwt_path, .. + } = &vault.auth_method + else { + panic!("expected Kubernetes auth, got {:?}", vault.auth_method); + }; + assert_eq!(role, "rustfs"); + assert_eq!(mount, crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT); + assert_eq!(jwt_path, std::path::Path::new(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH)); + + let unknown_field = serde_json::json!({ + "backend_type": "vault-transit", + "address": "https://vault.example.com:8200", + "auth_method": { "Kubernetes": { "role": "rustfs", "service_account": "rustfs" } } + }); + serde_json::from_value::(unknown_field) + .expect_err("an unknown auth field must be rejected rather than silently dropped"); + } + #[test] fn test_deserialize_aws_configure_request_accepts_type_aliases() { for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] { diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index 84ba8e6fa..56d2a856e 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -550,6 +550,7 @@ impl VaultKmsClient { address: config.address.clone(), namespace: config.namespace.clone(), attempt_timeout: kms_config.effective_timeout(), + skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify), }; let source = token_source_for(&config.auth_method, &settings)?; let policy = VaultCredentialPolicy::from_kms_config( diff --git a/crates/kms/src/backends/vault_credentials.rs b/crates/kms/src/backends/vault_credentials.rs index b8766442b..d5d8a564f 100644 --- a/crates/kms/src/backends/vault_credentials.rs +++ b/crates/kms/src/backends/vault_credentials.rs @@ -326,6 +326,97 @@ impl fmt::Debug for AppRoleLogin { } } +/// Token source for [`VaultAuthMethod::Kubernetes`]: exchanges the pod's +/// projected ServiceAccount token for a lease-bound Vault token. +/// +/// The JWT is re-read on every login because the kubelet rotates a projected +/// token well inside the pod's lifetime; caching it would strand the source on +/// an expired assertion once the current Vault token can no longer be renewed. +/// +/// Unlike [`TokenFileSource`], the file mode is not checked: the kubelet owns +/// the projected token and mounts it world-readable by default, so rejecting +/// group/other bits would refuse every standard pod rather than catch a +/// deployment error. +pub(crate) struct KubernetesLogin { + /// Unauthenticated client used only for the login exchange. + login_client: VaultClient, + mount: String, + role: String, + jwt_path: PathBuf, +} + +impl KubernetesLogin { + pub(crate) fn new(settings: &VaultConnectionSettings, mount: String, role: String, jwt_path: PathBuf) -> Result { + Ok(Self { + login_client: settings.build_login_client()?, + mount, + role, + jwt_path, + }) + } + + /// Read the ServiceAccount token for one login attempt. + /// + /// Mirrors [`AppRoleLogin::resolve_secret_id`]: a read failure is fatal for + /// the attempt but the refresh loop keeps retrying, so a token the kubelet + /// has not projected yet heals the source without a restart. + async fn resolve_jwt(&self) -> AttemptResult { + let mut raw = tokio::fs::read_to_string(&self.jwt_path) + .await + .map_err(|error| AttemptError { + class: ErrorClass::Fatal, + error: KmsError::configuration_error(format!( + "Failed to read Kubernetes ServiceAccount token {}: {error}", + self.jwt_path.display() + )), + })?; + let trimmed = raw.trim(); + if trimmed.is_empty() { + raw.zeroize(); + return Err(AttemptError { + class: ErrorClass::Fatal, + error: KmsError::configuration_error(format!( + "Kubernetes ServiceAccount token {} is empty", + self.jwt_path.display() + )), + }); + } + let jwt = SecretString::new(trimmed.to_string()); + raw.zeroize(); + Ok(jwt) + } +} + +#[async_trait] +impl TokenSource for KubernetesLogin { + async fn acquire(&self) -> AttemptResult { + let jwt = self.resolve_jwt().await?; + let auth = vaultrs::auth::kubernetes::login(&self.login_client, &self.mount, &self.role, jwt.expose()) + .await + .map_err(|error| attempt_error("Kubernetes login", error))?; + Ok(TokenLease::from_auth(auth)) + } + + async fn renew(&self, client: &VaultClient) -> AttemptResult { + let auth = vaultrs::token::renew_self(client, None) + .await + .map_err(|error| attempt_error("token renewal", error))?; + Ok(TokenLease::from_auth(auth)) + } +} + +impl fmt::Debug for KubernetesLogin { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // The login client embeds Vault client settings and must stay out of + // Debug output; the role name is not a secret, and the JWT is never held. + f.debug_struct("KubernetesLogin") + .field("mount", &self.mount) + .field("role", &self.role) + .field("jwt_path", &self.jwt_path) + .finish_non_exhaustive() + } +} + /// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed /// token file (for example a Vault Agent auto-auth sink). /// @@ -464,6 +555,9 @@ pub(crate) fn token_source_for( secret_id.clone(), secret_id_file.clone(), )?)), + VaultAuthMethod::Kubernetes { + role, mount, jwt_path, .. + } => Ok(Box::new(KubernetesLogin::new(settings, mount.clone(), role.clone(), jwt_path.clone())?)), VaultAuthMethod::TokenFile { path, poll_interval_secs, @@ -486,6 +580,9 @@ pub(crate) struct VaultConnectionSettings { pub(crate) namespace: Option, /// Per-attempt HTTP timeout applied to the underlying reqwest client. pub(crate) attempt_timeout: Duration, + /// Whether to accept an unverified Vault server certificate. Gated on + /// `allow_insecure_dev_defaults` by `KmsConfig::validate`. + pub(crate) skip_tls_verify: bool, } impl VaultConnectionSettings { @@ -499,6 +596,11 @@ impl VaultConnectionSettings { // operation-level retry policy. settings_builder.timeout(Some(self.attempt_timeout)); settings_builder.token(token); + // Always set explicitly: left unset, vaultrs derives this from its own + // VAULT_SKIP_VERIFY variable, so a stray value in the environment would + // disable certificate verification behind the KMS configuration and its + // insecure-defaults gate. + settings_builder.verify(!self.skip_tls_verify); if let Some(namespace) = &self.namespace { settings_builder.namespace(Some(namespace.clone())); @@ -551,6 +653,10 @@ impl VaultCredentialPolicy { refresh_safety_window_secs: Some(secs), .. } + | VaultAuthMethod::Kubernetes { + refresh_safety_window_secs: Some(secs), + .. + } | VaultAuthMethod::TokenFile { refresh_safety_window_secs: Some(secs), .. @@ -584,15 +690,25 @@ pub(crate) struct VaultClientHandle { impl VaultClientHandle { /// Absolute expiry of this generation's token. + /// + /// `lease.ttl` is built from the `lease_duration` the Vault server sent, so + /// a value too large to add to `issued_at` would panic on the bare `+`. A + /// TTL that cannot be represented is indistinguishable from no expiry, so it + /// collapses to `None` — the same answer already given for the zero-lease + /// tokens Vault issues, which keeps the token in use and still fully + /// validated by Vault on every call. fn expires_at(&self) -> Option { - self.lease.map(|lease| self.issued_at + lease.ttl) + self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl)) } /// When the renewal task should refresh this generation: half the TTL, /// leaving the second half as budget for retries before the fail-closed /// window is reached. + /// + /// Unrepresentable TTLs collapse to `None` as in [`Self::expires_at`], + /// leaving a token that never expires with nothing to renew. fn renew_at(&self) -> Option { - self.lease.map(|lease| self.issued_at + lease.ttl / 2) + self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl / 2)) } } @@ -662,7 +778,7 @@ impl VaultCredentialProvider { let handle = self.current.load_full(); if let Some(expires_at) = handle.expires_at() { let now = Instant::now(); - if now + self.policy.safety_window >= expires_at { + if self.inside_safety_window(now, expires_at) { return Err(KmsError::credentials_unavailable(format!( "Vault token (generation {}) is within {:?} of expiry and has not been refreshed; refusing to use it", handle.generation, self.policy.safety_window @@ -672,6 +788,18 @@ impl VaultCredentialProvider { Ok(handle) } + /// Whether the token expiring at `expires_at` is close enough to refuse. + /// + /// `safety_window` reaches here from persisted configuration, so it is not + /// guaranteed to have passed this version's validation: a window too large + /// to add to the current instant would panic on the bare `+`. Such a window + /// means every token is always inside it, so saturating to "refuse" is both + /// the fail-closed answer and the one the arithmetic was reaching for. + fn inside_safety_window(&self, now: Instant, expires_at: Instant) -> bool { + now.checked_add(self.policy.safety_window) + .is_none_or(|deadline| deadline >= expires_at) + } + /// Publish the credential gauges for the generation currently installed. /// /// The fail-closed gauge re-evaluates the very gate @@ -683,7 +811,7 @@ impl VaultCredentialProvider { let fail_closed = match handle.expires_at() { Some(expires_at) => { metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64()); - now + self.policy.safety_window >= expires_at + self.inside_safety_window(now, expires_at) } // A generation without an expiry has no remaining TTL to report // and can never lapse, so it can never fail closed either. @@ -860,7 +988,7 @@ impl Drop for CredentialTaskHandle { #[cfg(test)] mod tests { use super::*; - use crate::config::REDACTED_SECRET; + use crate::config::{DEFAULT_VAULT_KUBERNETES_MOUNT, REDACTED_SECRET}; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; const TEST_TOKEN: &str = "vault-token-debug-leak-canary"; @@ -871,6 +999,7 @@ mod tests { address: "http://127.0.0.1:8200".to_string(), namespace: Some("team-namespace".to_string()), attempt_timeout: Duration::from_secs(30), + skip_tls_verify: false, } } @@ -1057,6 +1186,143 @@ mod tests { assert!(format!("{source:?}").contains("AppRoleLogin")); } + #[tokio::test] + async fn test_kubernetes_auth_method_maps_to_login_source() { + let settings = test_settings(); + let source = token_source_for(&VaultAuthMethod::kubernetes("rustfs".to_string()), &settings) + .expect("kubernetes auth must map to a login source"); + + assert!(format!("{source:?}").contains("KubernetesLogin")); + } + + /// `refresh_safety_window_secs` is operator-supplied and reaches the request + /// path from persisted configuration, so the fail-closed comparison must + /// survive a window too large to add to the current instant. Before the + /// checked arithmetic this panicked with "overflow when adding duration to + /// instant" on the first request after a lease-bearing login. + #[tokio::test] + async fn test_current_refuses_rather_than_panics_on_an_unrepresentable_safety_window() { + let (provider, _state) = scripted_provider( + Duration::from_secs(60), + true, + test_policy(Duration::from_secs(u64::MAX), Duration::from_secs(5)), + ) + .await; + + let error = provider + .current() + .expect_err("a window wider than any lease must refuse the token"); + assert!( + matches!(error, KmsError::CredentialsUnavailable { .. }), + "expected CredentialsUnavailable, got {error:?}" + ); + } + + /// `lease_duration` is a bare u64 straight off the Vault response and forms + /// the other side of the same comparison, so an absurd one must not panic + /// either. It is indistinguishable from a non-expiring token, which is how + /// the zero-lease case already behaves. + #[tokio::test] + async fn test_an_unrepresentable_lease_is_treated_as_non_expiring() { + let (provider, _state) = scripted_provider( + Duration::from_secs(u64::MAX), + true, + test_policy(Duration::from_secs(30), Duration::from_secs(5)), + ) + .await; + + provider + .current() + .expect("a token whose expiry cannot be represented must stay usable"); + } + + /// The configured flag has to reach the HTTP client, not just the config + /// struct: every generation (authenticated and login) builds its own client, + /// and a Vault with a self-signed certificate fails the handshake unless + /// each one carries the setting. + #[test] + fn test_skip_tls_verify_reaches_every_vault_client_generation() { + for skip_tls_verify in [false, true] { + let settings = VaultConnectionSettings { + address: "https://vault.example.com:8200".to_string(), + namespace: None, + attempt_timeout: Duration::from_secs(30), + skip_tls_verify, + }; + + let authenticated = settings.build_client(TEST_TOKEN).expect("authenticated client must build"); + assert_eq!(authenticated.settings.verify, !skip_tls_verify); + + let login = settings.build_login_client().expect("login client must build"); + assert_eq!(login.settings.verify, !skip_tls_verify); + } + } + + /// vaultrs derives `verify` from its own VAULT_SKIP_VERIFY variable when the + /// builder leaves it unset, which would disable certificate verification + /// without passing the KMS insecure-defaults gate. + #[test] + fn test_vaultrs_skip_verify_env_cannot_override_the_configured_setting() { + temp_env::with_var("VAULT_SKIP_VERIFY", Some("true"), || { + let client = test_settings().build_client(TEST_TOKEN).expect("client must build"); + assert!( + client.settings.verify, + "a stray VAULT_SKIP_VERIFY must not disable verification behind the KMS configuration" + ); + }); + } + + /// The projected token is read fresh per login attempt and trimmed, so a + /// kubelet rotation is picked up without a restart and a trailing newline + /// does not corrupt the assertion sent to Vault. + #[tokio::test] + async fn test_kubernetes_login_rereads_and_trims_the_service_account_token() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("token"); + tokio::fs::write(&path, " first-jwt\n").await.expect("write token"); + + let login = KubernetesLogin::new( + &test_settings(), + DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(), + "rustfs".to_string(), + path.clone(), + ) + .expect("login source must build"); + + assert_eq!(login.resolve_jwt().await.expect("first read").expose(), "first-jwt"); + + tokio::fs::write(&path, "rotated-jwt").await.expect("rotate token"); + assert_eq!( + login.resolve_jwt().await.expect("second read").expose(), + "rotated-jwt", + "a rotated projected token must be picked up without a restart" + ); + } + + /// The ServiceAccount token is re-read per attempt, so an unreadable or + /// empty one fails that attempt without reaching Vault; the refresh loop + /// keeps retrying, which is what lets a late projection heal the source. + #[tokio::test] + async fn test_kubernetes_login_rejects_an_unusable_service_account_token() { + let dir = tempfile::tempdir().expect("temp dir"); + let missing = dir.path().join("absent-token"); + let empty = dir.path().join("empty-token"); + tokio::fs::write(&empty, " \n").await.expect("write empty token"); + + for (path, expected) in [(missing, "Failed to read"), (empty, "is empty")] { + let login = + KubernetesLogin::new(&test_settings(), DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(), "rustfs".to_string(), path) + .expect("login source must build"); + + let error = login + .acquire() + .await + .expect_err("an unusable ServiceAccount token must fail the attempt"); + assert!(matches!(error.class, ErrorClass::Fatal)); + assert!(error.error.to_string().contains(expected), "got {}", error.error); + } + } + #[tokio::test(start_paused = true)] async fn test_renewal_task_renews_at_half_ttl() { let (provider, state) = scripted_provider( diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 095fcf455..602c53154 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -415,6 +415,7 @@ impl VaultTransitKmsClient { address: config.address.clone(), namespace: config.namespace.clone(), attempt_timeout: kms_config.effective_timeout(), + skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify), }; let source = token_source_for(&config.auth_method, &settings)?; let policy = VaultCredentialPolicy::from_kms_config( diff --git a/crates/kms/src/backup/vault_restore.rs b/crates/kms/src/backup/vault_restore.rs index b7d4884cc..b033c4d5f 100644 --- a/crates/kms/src/backup/vault_restore.rs +++ b/crates/kms/src/backup/vault_restore.rs @@ -450,6 +450,10 @@ impl VaultRestoreClient { address: target.address.clone(), namespace: target.namespace.clone(), attempt_timeout: kms_config.effective_timeout(), + // A restore target carries no TLS settings, so certificates are + // always verified: recovery is the last path that should accept an + // unauthenticated Vault. + skip_tls_verify: false, }; let source = token_source_for(&target.auth_method, &settings)?; let policy = VaultCredentialPolicy::from_kms_config( diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index c33d1ae0a..bd6c4a0e1 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -25,6 +25,10 @@ use url::Url; pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS"; pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION"; +pub const ENV_KMS_VAULT_ADDRESS: &str = "RUSTFS_KMS_VAULT_ADDRESS"; +pub const ENV_KMS_VAULT_TOKEN: &str = "RUSTFS_KMS_VAULT_TOKEN"; +pub const ENV_KMS_VAULT_NAMESPACE: &str = "RUSTFS_KMS_VAULT_NAMESPACE"; +pub const ENV_KMS_VAULT_MOUNT_PATH: &str = "RUSTFS_KMS_VAULT_MOUNT_PATH"; pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY"; pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT"; pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX"; @@ -35,6 +39,9 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE"; pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT"; pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE"; +pub const ENV_KMS_VAULT_KUBERNETES_ROLE: &str = "RUSTFS_KMS_VAULT_KUBERNETES_ROLE"; +pub const ENV_KMS_VAULT_KUBERNETES_MOUNT: &str = "RUSTFS_KMS_VAULT_KUBERNETES_MOUNT"; +pub const ENV_KMS_VAULT_KUBERNETES_JWT_PATH: &str = "RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH"; pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION"; pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL"; /// Age in whole seconds beyond which a key is reported as due for rotation; @@ -45,6 +52,9 @@ pub const ENV_KMS_ROTATION_MAX_WRAPS: &str = "RUSTFS_KMS_ROTATION_MAX_WRAPS"; pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret"; pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata"; pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle"; +pub const DEFAULT_VAULT_KUBERNETES_MOUNT: &str = "kubernetes"; +/// Where the kubelet projects a pod's ServiceAccount token by default. +pub const DEFAULT_VAULT_KUBERNETES_JWT_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token"; /// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior. /// @@ -84,6 +94,14 @@ fn default_vault_approle_mount() -> String { DEFAULT_VAULT_APPROLE_MOUNT.to_string() } +fn default_vault_kubernetes_mount() -> String { + DEFAULT_VAULT_KUBERNETES_MOUNT.to_string() +} + +fn default_vault_kubernetes_jwt_path() -> PathBuf { + PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH) +} + pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[ RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"), RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"), @@ -490,6 +508,23 @@ pub enum VaultAuthMethod { #[serde(default)] refresh_safety_window_secs: Option, }, + /// Kubernetes authentication: the pod's ServiceAccount token is exchanged + /// for a lease-bound Vault token that is renewed in the background. + Kubernetes { + /// Vault role bound to this ServiceAccount. + role: String, + /// Kubernetes auth engine mount path. + #[serde(default = "default_vault_kubernetes_mount")] + mount: String, + /// Projected ServiceAccount token to present. Re-read on every login so + /// a token the kubelet rotates is picked up without a restart. + #[serde(default = "default_vault_kubernetes_jwt_path")] + jwt_path: PathBuf, + /// Fail-closed margin in seconds, as on `AppRole`. Defaults to the + /// per-attempt timeout. + #[serde(default)] + refresh_safety_window_secs: Option, + }, /// Agent-managed token file (for example a Vault Agent auto-auth sink): /// the token is read from `path` and re-read periodically so a token /// rotated by the agent is picked up without a restart. @@ -520,6 +555,16 @@ impl VaultAuthMethod { } } + /// Kubernetes authentication with the default mount and projected token path. + pub fn kubernetes(role: String) -> Self { + Self::Kubernetes { + role, + mount: default_vault_kubernetes_mount(), + jwt_path: default_vault_kubernetes_jwt_path(), + refresh_safety_window_secs: None, + } + } + /// Agent-managed token file with the default poll interval. pub fn token_file(path: PathBuf) -> Self { Self::TokenFile { @@ -548,6 +593,20 @@ impl fmt::Debug for VaultAuthMethod { .field("mount", mount) .field("refresh_safety_window_secs", refresh_safety_window_secs) .finish(), + // No redaction: the role and mount name a Vault binding, and the + // ServiceAccount token itself is never held on this type. + Self::Kubernetes { + role, + mount, + jwt_path, + refresh_safety_window_secs, + } => f + .debug_struct("Kubernetes") + .field("role", role) + .field("mount", mount) + .field("jwt_path", jwt_path) + .field("refresh_safety_window_secs", refresh_safety_window_secs) + .finish(), Self::TokenFile { path, poll_interval_secs, @@ -1028,50 +1087,12 @@ impl KmsConfig { }); } KmsBackend::VaultKv2 => { - let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200"); - let auth_method = vault_auth_method_from_env()?; - let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false); - - let mount_path = match get_env_opt_str("RUSTFS_KMS_VAULT_MOUNT_PATH") { - Some(path) => { - tracing::warn!( - "RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused" - ); - path - } - None => default_vault_kv2_mount_path(), - }; - - config.backend_config = BackendConfig::VaultKv2(Box::new(VaultConfig { - address, - auth_method, - namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"), - mount_path, - kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"), - key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"), - tls: vault_tls_config(skip_tls_verify), - })); + config.backend_config = + BackendConfig::VaultKv2(Box::new(vault_kv2_config_from_env(VaultCliOverrides::default())?)); } KmsBackend::VaultTransit => { - let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200"); - let auth_method = vault_auth_method_from_env()?; - let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false); - - config.backend_config = BackendConfig::VaultTransit(Box::new(VaultTransitConfig { - address, - auth_method, - namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"), - mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"), - metadata_kv_mount: get_env_str( - ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT, - DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT, - ), - metadata_key_prefix: get_env_str( - ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX, - DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX, - ), - tls: vault_tls_config(skip_tls_verify), - })); + config.backend_config = + BackendConfig::VaultTransit(Box::new(vault_transit_config_from_env(VaultCliOverrides::default())?)); } KmsBackend::Static => { // Read from file first, then fall back to direct env var @@ -1202,6 +1223,78 @@ fn is_under_temp_dir(path: &Path) -> bool { path.starts_with(std::env::temp_dir()) } +/// Command-line values that take precedence over the matching environment +/// variables when assembling a Vault backend configuration. +/// +/// Every field has a `RUSTFS_KMS_VAULT_*` equivalent that the CLI layer already +/// reads, so these are only set when the operator passed an explicit flag. +/// +/// Deliberately not `Debug`: `token` holds the raw Vault token, and the +/// redacting `Debug` impls elsewhere in this module exist because a derived one +/// would print it. Denying the derive makes a future `{overrides:?}` a compile +/// error instead of a leak. +#[derive(Default, Clone, Copy)] +pub struct VaultCliOverrides<'a> { + pub address: Option<&'a str>, + pub token: Option<&'a str>, + pub mount_path: Option<&'a str>, +} + +/// Assemble the Vault KV2 backend configuration from the environment. +/// +/// Shared by [`KmsConfig::from_env`] and the server's command-line startup path +/// so both resolve the same auth method, namespace, TLS and mount settings. +pub fn vault_kv2_config_from_env(overrides: VaultCliOverrides<'_>) -> Result { + let mount_path = match overrides + .mount_path + .map(str::to_string) + .or_else(|| get_env_opt_str(ENV_KMS_VAULT_MOUNT_PATH)) + { + Some(path) => { + tracing::warn!( + "RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused" + ); + path + } + None => default_vault_kv2_mount_path(), + }; + + Ok(VaultConfig { + address: vault_address_from_env(overrides.address), + auth_method: vault_auth_method_from_env(overrides.token)?, + namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE), + mount_path, + kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"), + key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"), + tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)), + }) +} + +/// Assemble the Vault Transit backend configuration from the environment. +/// +/// Companion to [`vault_kv2_config_from_env`]; see there for why both entry +/// points share it. +pub fn vault_transit_config_from_env(overrides: VaultCliOverrides<'_>) -> Result { + Ok(VaultTransitConfig { + address: vault_address_from_env(overrides.address), + auth_method: vault_auth_method_from_env(overrides.token)?, + namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE), + mount_path: overrides + .mount_path + .map(str::to_string) + .unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_MOUNT_PATH, "transit")), + metadata_kv_mount: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT), + metadata_key_prefix: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX), + tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)), + }) +} + +fn vault_address_from_env(override_value: Option<&str>) -> String { + override_value + .map(str::to_string) + .unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_ADDRESS, "http://localhost:8200")) +} + /// Resolve the Vault auth method from environment variables. /// /// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication; @@ -1209,27 +1302,59 @@ fn is_under_temp_dir(path: &Path) -> bool { /// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE` /// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the /// file taking precedence. Without a role id the legacy token flow applies. -fn vault_auth_method_from_env() -> Result { +/// +/// `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` selects Kubernetes authentication, which +/// presents the pod's projected ServiceAccount token. +/// +/// `token_override` carries a token supplied on the command line; it stands in +/// for `RUSTFS_KMS_VAULT_TOKEN` everywhere below, including the conflict checks, +/// so a flag and the variable it mirrors select the same method. +fn vault_auth_method_from_env(token_override: Option<&str>) -> Result { + let token = token_override + .map(str::to_string) + .or_else(|| get_env_opt_str(ENV_KMS_VAULT_TOKEN)); + let role_id = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID); + let kubernetes_role = get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_ROLE); + if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) { // A token file names one authoritative credential source; combining it // with another one would leave the effective identity ambiguous, so // that is a configuration error rather than a precedence rule. - if get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID).is_some() { - return Err(KmsError::configuration_error(format!( - "{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method" - ))); - } - if get_env_opt_str("RUSTFS_KMS_VAULT_TOKEN").is_some() { - return Err(KmsError::configuration_error(format!( - "{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with RUSTFS_KMS_VAULT_TOKEN; configure exactly one Vault auth method" - ))); + for (name, configured) in [ + (ENV_KMS_VAULT_APPROLE_ROLE_ID, role_id.is_some()), + (ENV_KMS_VAULT_KUBERNETES_ROLE, kubernetes_role.is_some()), + (ENV_KMS_VAULT_TOKEN, token.is_some()), + ] { + if configured { + return Err(KmsError::configuration_error(format!( + "{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {name}; configure exactly one Vault auth method" + ))); + } } return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file))); } - let Some(role_id) = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID) else { + if let Some(role) = kubernetes_role { + // Unlike a leftover static token, a second login method is never a + // stale remnant: both were configured deliberately and neither can be + // ranked over the other. + if role_id.is_some() { + return Err(KmsError::configuration_error(format!( + "{ENV_KMS_VAULT_KUBERNETES_ROLE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method" + ))); + } + return Ok(VaultAuthMethod::Kubernetes { + role, + mount: get_env_str(ENV_KMS_VAULT_KUBERNETES_MOUNT, DEFAULT_VAULT_KUBERNETES_MOUNT), + jwt_path: get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_JWT_PATH) + .map_or_else(default_vault_kubernetes_jwt_path, PathBuf::from), + refresh_safety_window_secs: None, + }); + } + + let Some(role_id) = role_id else { return Ok(VaultAuthMethod::Token { - token: get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"), + token: token.unwrap_or_else(|| "dev-token".to_string()), }); }; @@ -1273,6 +1398,22 @@ fn validate_vault_auth_method(backend_name: &str, auth_method: &VaultAuthMethod) } Ok(()) } + VaultAuthMethod::Kubernetes { + role, mount, jwt_path, .. + } => { + if role.is_empty() { + return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes role cannot be empty"))); + } + if mount.is_empty() { + return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes mount cannot be empty"))); + } + if jwt_path.as_os_str().is_empty() { + return Err(KmsError::configuration_error(format!( + "{backend_name} Kubernetes ServiceAccount token path cannot be empty" + ))); + } + Ok(()) + } VaultAuthMethod::TokenFile { path, poll_interval_secs, @@ -1976,6 +2117,106 @@ mod tests { .expect("well-formed token file auth must validate"); } + /// A Kubernetes role alone configures the method: the credential is the + /// pod's projected ServiceAccount token, so nothing secret is in the + /// environment and the mount and token path fall back to the cluster + /// defaults. + #[test] + fn test_from_env_selects_kubernetes() { + with_vars( + vec![ + ("RUSTFS_KMS_BACKEND", Some("vault-transit")), + (ENV_KMS_VAULT_ADDRESS, Some("https://vault.example.com")), + (ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")), + (ENV_KMS_VAULT_KUBERNETES_MOUNT, None), + (ENV_KMS_VAULT_KUBERNETES_JWT_PATH, None), + (ENV_KMS_VAULT_TOKEN, None), + (ENV_KMS_VAULT_TOKEN_FILE, None), + (ENV_KMS_VAULT_APPROLE_ROLE_ID, None), + ], + || { + let config = KmsConfig::from_env().expect("kms config should load from env"); + let vault = config.vault_transit_config().expect("vault transit backend config"); + let VaultAuthMethod::Kubernetes { + role, + mount, + jwt_path, + refresh_safety_window_secs, + } = &vault.auth_method + else { + panic!( + "a kubernetes role in the environment must select Kubernetes auth, got {:?}", + vault.auth_method + ); + }; + assert_eq!(role, "rustfs"); + assert_eq!(mount, DEFAULT_VAULT_KUBERNETES_MOUNT); + assert_eq!(jwt_path, Path::new(DEFAULT_VAULT_KUBERNETES_JWT_PATH)); + assert_eq!(refresh_safety_window_secs, &None); + }, + ); + } + + #[test] + fn test_from_env_kubernetes_is_mutually_exclusive_with_other_auth() { + with_vars( + vec![ + ("RUSTFS_KMS_BACKEND", Some("vault-transit")), + (ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")), + (ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")), + (ENV_KMS_VAULT_TOKEN, None), + (ENV_KMS_VAULT_TOKEN_FILE, None), + ], + || { + let error = KmsConfig::from_env().expect_err("kubernetes combined with approle must be rejected"); + assert!(error.to_string().contains(ENV_KMS_VAULT_KUBERNETES_ROLE)); + assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_ROLE_ID)); + }, + ); + } + + #[test] + fn test_validate_rejects_bad_kubernetes_settings() { + let vault_config = |auth_method: VaultAuthMethod| KmsConfig { + backend: KmsBackend::VaultTransit, + backend_config: BackendConfig::VaultTransit(Box::new(VaultTransitConfig { + address: "https://vault.example.com:8200".to_string(), + auth_method, + ..Default::default() + })), + ..Default::default() + }; + + let error = vault_config(VaultAuthMethod::kubernetes(String::new())) + .validate() + .expect_err("an empty kubernetes role must be rejected"); + assert!(error.to_string().contains("role"), "got {error}"); + + let error = vault_config(VaultAuthMethod::Kubernetes { + role: "rustfs".to_string(), + mount: String::new(), + jwt_path: PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH), + refresh_safety_window_secs: None, + }) + .validate() + .expect_err("an empty kubernetes mount must be rejected"); + assert!(error.to_string().contains("mount"), "got {error}"); + + let error = vault_config(VaultAuthMethod::Kubernetes { + role: "rustfs".to_string(), + mount: DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(), + jwt_path: PathBuf::new(), + refresh_safety_window_secs: None, + }) + .validate() + .expect_err("an empty ServiceAccount token path must be rejected"); + assert!(error.to_string().contains("token path"), "got {error}"); + + vault_config(VaultAuthMethod::kubernetes("rustfs".to_string())) + .validate() + .expect("well-formed kubernetes auth must validate"); + } + /// Every KV2 read, write and listing is routed through `kv_mount`, so an /// empty one names a path no Vault engine answers. The Transit backend /// already rejects its own empty mounts; this closes the same gap on the diff --git a/docs/operations/kms-backend-security.md b/docs/operations/kms-backend-security.md index 4fe4aad07..3f1dfdc7e 100644 --- a/docs/operations/kms-backend-security.md +++ b/docs/operations/kms-backend-security.md @@ -2,7 +2,7 @@ RustFS ships several KMS backends. They differ not only in deployment effort but in **where master key material lives and who can read it**. Pick a backend based on the confidentiality boundary you need, not on the name alone. -For how the Vault backends authenticate (static token, AppRole, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md). For which RustFS identities may manage or use a given key, see [Per-key KMS authorization](kms-per-key-authorization.md). If you are migrating from MinIO, read [Migrating from MinIO: encrypted objects do not carry over](#migrating-from-minio-encrypted-objects-do-not-carry-over) first. +For how the Vault backends authenticate (static token, AppRole, Kubernetes, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md). For which RustFS identities may manage or use a given key, see [Per-key KMS authorization](kms-per-key-authorization.md). If you are migrating from MinIO, read [Migrating from MinIO: encrypted objects do not carry over](#migrating-from-minio-encrypted-objects-do-not-carry-over) first. ## Backend comparison diff --git a/docs/operations/vault-kms-authentication.md b/docs/operations/vault-kms-authentication.md index 6ee0a0c2c..adabd2918 100644 --- a/docs/operations/vault-kms-authentication.md +++ b/docs/operations/vault-kms-authentication.md @@ -8,9 +8,12 @@ This runbook covers how the RustFS Vault KMS backends (KV2 and Transit) authenti | --- | --- | --- | --- | --- | | Static token | `Token` | Whatever the operator provisioned; RustFS never renews it | None | Development; short-lived experiments | | AppRole | `AppRole` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production without a Vault Agent sidecar | +| Kubernetes | `Kubernetes` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production on Kubernetes, with no credential to distribute | | Agent token file | `TokenFile` | Owned by Vault Agent; RustFS only re-reads the sink file | File re-read once per poll interval | Production with a Vault Agent (or equivalent) managing auth | -Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` or an explicit `RUSTFS_KMS_VAULT_TOKEN` is rejected at startup with a configuration error, because the effective identity would be ambiguous. +Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with any other method, or `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID`, is rejected at startup with a configuration error, because the effective identity would be ambiguous. A leftover `RUSTFS_KMS_VAULT_TOKEN` alongside a configured login method is tolerated and ignored, so a stale variable cannot silently downgrade the identity. + +All of these are read the same way whether the service is started with `RUSTFS_KMS_ENABLE=true` or configured later through `POST /rustfs/admin/v3/kms/configure`. The default `dev-token` fallback for `RUSTFS_KMS_VAULT_TOKEN` is rejected outside explicit development mode (`RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS=true`), as are plain-HTTP Vault addresses and disabled TLS verification. @@ -56,7 +59,44 @@ Deliver the SecretID out of band — a secrets-manager-mounted file, an init-con The secret_id file is re-read on every login attempt, so rotating the SecretID is a two-step operation with no restart: generate a new SecretID (`vault write -f auth/approle/role/rustfs-kms/secret-id`), atomically replace the file, then revoke the old SecretID accessor. The already-issued token keeps renewing; the new SecretID is only needed at the next full re-login. -An empty or missing secret_id file fails the login attempt immediately (no Vault round trip) and is retried on the normal refresh cadence, so repairing the file heals the backend without a restart. +An empty or missing secret_id file fails the login attempt immediately (no Vault round trip). At startup the error is fatal — provider construction fails and the process exits — so a file missing at boot is recovered by restarting the process, not by an in-process retry. Once RustFS is running, the same failure is retried on the normal refresh cadence, so repairing the file mid-run heals the backend without a restart. + +## Kubernetes authentication + +On Kubernetes this is the method to prefer: the pod's own ServiceAccount is the identity, so there is no credential to distribute, rotate, or leak into a Secret. + +### Vault-side setup + +```shell +vault auth enable kubernetes + +vault write auth/kubernetes/config \ + kubernetes_host="https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT" + +vault write auth/kubernetes/role/rustfs \ + bound_service_account_names=rustfs \ + bound_service_account_namespaces=rustfs \ + token_policies=rustfs-kms \ + token_ttl=1h +``` + +As with AppRole, keep `token_ttl` comfortably above the RustFS per-attempt timeout (default 30s). + +### RustFS configuration + +```shell +RUSTFS_KMS_BACKEND=vault-transit # or "vault" for the KV2 backend +RUSTFS_KMS_VAULT_ADDRESS=https://vault.vault.svc.cluster.local:8200 +RUSTFS_KMS_VAULT_KUBERNETES_ROLE=rustfs +# Optional, defaults to "kubernetes": +# RUSTFS_KMS_VAULT_KUBERNETES_MOUNT=kubernetes +# Optional, defaults to the kubelet's projected token path: +# RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH=/var/run/secrets/kubernetes.io/serviceaccount/token +``` + +RustFS logs in at startup and renews the token at half its TTL, falling back to a fresh login exactly as AppRole does. The ServiceAccount token is re-read from disk on every login rather than cached, so a projected token the kubelet rotates is picked up without a restart. + +A missing or empty token file fails the login attempt immediately (no Vault round trip). At startup the error is fatal — provider construction fails and the process exits — so a token projected late during a slow pod start is recovered by the pod restart loop, not by an in-process retry. Once RustFS is running, a token file that goes missing or turns empty is retried on the normal refresh cadence and heals the backend on its own. ## Vault Agent token file @@ -101,13 +141,13 @@ If the agent stops refreshing the file that is fine — RustFS re-reads the same ## Fail-closed window -For lease-bound credentials (AppRole tokens, token files), `current()` refuses to hand out a token that is within the safety window of its expiry and has not been refreshed. Requests then fail with `KMS credentials unavailable: ...` instead of being sent with a token that could lapse mid-flight and fail unpredictably on the Vault side. +For lease-bound credentials (AppRole and Kubernetes tokens, token files), `current()` refuses to hand out a token that is within the safety window of its expiry and has not been refreshed. Requests then fail with `KMS credentials unavailable: ...` instead of being sent with a token that could lapse mid-flight and fail unpredictably on the Vault side. - Default window: one per-attempt timeout (`RUSTFS_KMS_TIMEOUT_SECS`, default 30s) — a request issued now can legitimately stay in flight that long, so the token must outlive it. -- Override: `refresh_safety_window_secs` on the `AppRole` or `TokenFile` auth configuration. +- Override: `refresh_safety_window_secs` on the `AppRole`, `Kubernetes` or `TokenFile` auth configuration. - Static tokens never trip the window: they carry no lease and are assumed valid until Vault says otherwise. -The window is a symptom threshold, not the fault itself: by the time it trips, refresh has been failing for roughly half the token TTL (AppRole) or two poll intervals (token file). +The window is a symptom threshold, not the fault itself: by the time it trips, refresh has been failing for roughly half the token TTL (AppRole, Kubernetes) or two poll intervals (token file). ### Troubleshooting @@ -117,6 +157,8 @@ The window is a symptom threshold, not the fault itself: by the time it trips, r | Renewal succeeded but re-login later fails | `Vault token renewal failed; falling back to a fresh login` followed by login errors | SecretID expired/revoked or AppRole role changed; rotate the secret_id file | | Token file mode error at startup or during polls | `has insecure permissions` in the error | Fix the sink `mode` (0600) and the file owner; the next poll heals the provider | | Token file missing/empty errors | `Failed to read Vault token file` / `token file ... is empty` | Vault Agent down or sink misconfigured; restart the agent, the next poll heals the provider | -| Startup fails immediately with a configuration error naming two env vars | — | Two auth methods configured at once; keep exactly one of token, AppRole, token file | +| Kubernetes login fails with a permission error | `Vault Kubernetes login failed` | The pod's ServiceAccount is not in the role's `bound_service_account_names`/`_namespaces`, or `auth/kubernetes/config` names the wrong API server | +| Kubernetes ServiceAccount token errors | `Failed to read Kubernetes ServiceAccount token` / `ServiceAccount token ... is empty` | The token is not projected into the pod (check `automountServiceAccountToken` and the volume mount); the next refresh cycle heals the provider | +| Startup fails immediately with a configuration error naming two env vars | — | Two auth methods configured at once; keep exactly one of token, AppRole, Kubernetes, token file | When diagnosing, confirm three clocks/lifetimes in order: the Vault token TTL (`vault token lookup` with the token's accessor), the RustFS refresh cadence (half TTL or the poll interval), and the fail-closed window. The renewal task logs every failed cycle, so a silent gap in warnings combined with `CredentialsUnavailable` errors points at the process clock or a paused runtime rather than Vault. diff --git a/rustfs/src/admin/handlers/kms_backup.rs b/rustfs/src/admin/handlers/kms_backup.rs index 02384f487..02682e7da 100644 --- a/rustfs/src/admin/handlers/kms_backup.rs +++ b/rustfs/src/admin/handlers/kms_backup.rs @@ -286,6 +286,7 @@ fn auth_method_kind(auth: &VaultAuthMethod) -> String { match auth { VaultAuthMethod::Token { .. } => "token", VaultAuthMethod::AppRole { .. } => "approle", + VaultAuthMethod::Kubernetes { .. } => "kubernetes", VaultAuthMethod::TokenFile { .. } => "token-file", } .to_string() @@ -484,7 +485,10 @@ fn business_trust_root_secrets(config: &KmsConfig) -> Vec> { secrets.push(Zeroizing::new(role_id.clone())); secrets.push(Zeroizing::new(secret_id.clone())); } - VaultAuthMethod::TokenFile { .. } => {} + // Kubernetes and TokenFile hold no inline plaintext credential: the + // ServiceAccount token and the agent-managed token live in files, and + // the role names a Vault binding rather than half a credential pair. + VaultAuthMethod::Kubernetes { .. } | VaultAuthMethod::TokenFile { .. } => {} }; match &config.backend_config { diff --git a/rustfs/src/init.rs b/rustfs/src/init.rs index 67c889c1a..34a1d6ca0 100644 --- a/rustfs/src/init.rs +++ b/rustfs/src/init.rs @@ -304,30 +304,37 @@ fn build_local_kms_config(cfg: &config::Config) -> std::io::Result( + cfg: &'a config::Config, + backend_name: &str, +) -> std::io::Result> { + let address = cfg + .kms_vault_address + .as_deref() + .ok_or_else(|| Error::other(format!("Vault address is required for {backend_name} backend")))?; + + Ok(rustfs_kms::config::VaultCliOverrides { + address: Some(address), + token: cfg.kms_vault_token.as_deref(), + mount_path: cfg.kms_vault_mount_path.as_deref(), + }) +} + /// Build KMS configuration for Vault backend fn build_vault_kms_config(cfg: &config::Config) -> std::io::Result { - let vault_address = cfg - .kms_vault_address - .as_ref() - .ok_or_else(|| Error::other("Vault address is required for vault backend"))?; - let vault_token = cfg - .kms_vault_token - .as_ref() - .ok_or_else(|| Error::other("Vault token is required for vault backend"))?; + let backend_config = rustfs_kms::config::vault_kv2_config_from_env(vault_cli_overrides(cfg, "vault")?) + .map_err(|e| Error::other(format!("Vault KMS configuration failed: {e}")))?; let kms_config = rustfs_kms::config::KmsConfig { backend: rustfs_kms::config::KmsBackend::VaultKv2, - backend_config: rustfs_kms::config::BackendConfig::VaultKv2(Box::new(rustfs_kms::config::VaultConfig { - address: vault_address.clone(), - auth_method: rustfs_kms::config::VaultAuthMethod::Token { - token: vault_token.clone(), - }, - namespace: None, - mount_path: cfg.kms_vault_mount_path.clone().unwrap_or_else(|| "transit".to_string()), - kv_mount: "secret".to_string(), - key_path_prefix: "rustfs/kms/keys".to_string(), - tls: None, - })), + backend_config: rustfs_kms::config::BackendConfig::VaultKv2(Box::new(backend_config)), allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults, allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(), default_key_id: cfg.kms_default_key_id.clone(), @@ -344,26 +351,12 @@ fn build_vault_kms_config(cfg: &config::Config) -> std::io::Result std::io::Result { - let vault_address = cfg - .kms_vault_address - .as_ref() - .ok_or_else(|| Error::other("Vault address is required for vault-transit backend"))?; - let vault_token = cfg - .kms_vault_token - .as_ref() - .ok_or_else(|| Error::other("Vault token is required for vault-transit backend"))?; + let backend_config = rustfs_kms::config::vault_transit_config_from_env(vault_cli_overrides(cfg, "vault-transit")?) + .map_err(|e| Error::other(format!("Vault Transit KMS configuration failed: {e}")))?; let kms_config = rustfs_kms::config::KmsConfig { backend: rustfs_kms::config::KmsBackend::VaultTransit, - backend_config: rustfs_kms::config::BackendConfig::VaultTransit(Box::new(rustfs_kms::config::VaultTransitConfig { - address: vault_address.clone(), - auth_method: rustfs_kms::config::VaultAuthMethod::Token { - token: vault_token.clone(), - }, - namespace: None, - mount_path: cfg.kms_vault_mount_path.clone().unwrap_or_else(|| "transit".to_string()), - ..rustfs_kms::config::VaultTransitConfig::default() - })), + backend_config: rustfs_kms::config::BackendConfig::VaultTransit(Box::new(backend_config)), allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults, allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(), default_key_id: cfg.kms_default_key_id.clone(), @@ -1405,7 +1398,10 @@ pub async fn init_sftp_system() -> Result, Box crate::config::Config { + let mut config = crate::config::Config::new("127.0.0.1:9000", vec!["/tmp/rustfs-vault-kms".to_string()]); + config.kms_enable = true; + config.kms_backend = backend.to_string(); + config.kms_vault_address = Some("https://vault.example.com:8200".to_string()); + config + } + + /// The Vault auth method and the settings the CLI has no flag for come from + /// the environment, so startup and `KmsConfig::from_env` cannot disagree. + /// Regression: startup used to hardcode token auth and require a token, + /// which made every non-token method unreachable through `RUSTFS_KMS_ENABLE`. + #[test] + fn build_vault_transit_kms_config_resolves_auth_and_mounts_from_env() { + let config = temp_env::with_vars( + [ + ("RUSTFS_KMS_VAULT_TOKEN", None), + ("RUSTFS_KMS_VAULT_TOKEN_FILE", None), + ("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None), + ("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", Some("env-role-id")), + ("RUSTFS_KMS_VAULT_APPROLE_SECRET_ID", Some("env-secret-id")), + ("RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE", None), + ("RUSTFS_KMS_VAULT_NAMESPACE", Some("team-a")), + ("RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT", Some("rustfs-kv")), + ], + || { + build_vault_transit_kms_config(&vault_kms_test_config("vault-transit")) + .expect("vault transit KMS configuration should build") + }, + ); + + let vault = config.vault_transit_config().expect("vault transit backend config"); + let rustfs_kms::config::VaultAuthMethod::AppRole { role_id, secret_id, .. } = &vault.auth_method else { + panic!("approle in the environment must select AppRole auth, got {:?}", vault.auth_method); + }; + assert_eq!(role_id, "env-role-id"); + assert_eq!(secret_id, "env-secret-id"); + assert_eq!(vault.namespace.as_deref(), Some("team-a")); + assert_eq!(vault.metadata_kv_mount, "rustfs-kv"); + } + + /// Kubernetes auth needs no credential in the environment at all: the role + /// selects it and the pod's projected ServiceAccount token supplies the rest. + #[test] + fn build_vault_transit_kms_config_selects_kubernetes_auth() { + let config = temp_env::with_vars( + [ + ("RUSTFS_KMS_VAULT_TOKEN", None), + ("RUSTFS_KMS_VAULT_TOKEN_FILE", None), + ("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None), + ("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", Some("rustfs")), + ("RUSTFS_KMS_VAULT_KUBERNETES_MOUNT", None), + ("RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH", None), + ], + || { + build_vault_transit_kms_config(&vault_kms_test_config("vault-transit")) + .expect("vault transit KMS configuration should build") + }, + ); + + let vault = config.vault_transit_config().expect("vault transit backend config"); + let rustfs_kms::config::VaultAuthMethod::Kubernetes { + role, mount, jwt_path, .. + } = &vault.auth_method + else { + panic!( + "a kubernetes role in the environment must select Kubernetes auth, got {:?}", + vault.auth_method + ); + }; + assert_eq!(role, "rustfs"); + assert_eq!(mount, rustfs_kms::config::DEFAULT_VAULT_KUBERNETES_MOUNT); + assert_eq!(jwt_path, std::path::Path::new(rustfs_kms::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH)); + } + + /// Two credential sources leave the effective identity ambiguous, so + /// startup refuses rather than picking one. + #[test] + fn build_vault_kms_config_refuses_two_auth_methods() { + temp_env::with_vars( + [ + ("RUSTFS_KMS_VAULT_TOKEN", None), + ("RUSTFS_KMS_VAULT_TOKEN_FILE", Some("/run/vault-agent/token")), + ("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None), + ("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", Some("rustfs")), + ], + || { + let error = build_vault_kms_config(&vault_kms_test_config("vault")) + .expect_err("two Vault auth methods must not start the server"); + assert!(error.to_string().contains("exactly one"), "unexpected error: {error}"); + }, + ); + } + + /// The KV2 backend has its own builder, so the key-location settings have + /// to be proven separately from the Transit one: pointing at the wrong KV + /// mount or prefix makes existing keys look absent. + #[test] + fn build_vault_kms_config_resolves_kv_mount_and_prefix_from_env() { + let config = temp_env::with_vars( + [ + ("RUSTFS_KMS_VAULT_TOKEN", Some("a-real-token")), + ("RUSTFS_KMS_VAULT_TOKEN_FILE", None), + ("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None), + ("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None), + ("RUSTFS_KMS_VAULT_KV_MOUNT", Some("rustfs-kv")), + ("RUSTFS_KMS_VAULT_KEY_PREFIX", Some("tenant/keys")), + ], + || build_vault_kms_config(&vault_kms_test_config("vault")).expect("vault KV2 KMS configuration should build"), + ); + + let vault = config.vault_config().expect("vault kv2 backend config"); + assert_eq!(vault.kv_mount, "rustfs-kv"); + assert_eq!(vault.key_path_prefix, "tenant/keys"); + } + + /// Skipping TLS verification was silently dropped on this path before, so + /// an operator who asked for it still got a verified connection. Now that it + /// is honoured it must fail closed without the development opt-in, rather + /// than quietly downgrading the Vault connection. + #[test] + fn build_vault_transit_kms_config_refuses_skip_tls_verify_without_opt_in() { + let vars = [ + ("RUSTFS_KMS_VAULT_TOKEN", Some("a-real-token")), + ("RUSTFS_KMS_VAULT_TOKEN_FILE", None), + ("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None), + ("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None), + ("RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY", Some("true")), + ]; + + temp_env::with_vars(vars, || { + let error = build_vault_transit_kms_config(&vault_kms_test_config("vault-transit")) + .expect_err("skipping TLS verification must not start the server"); + assert!(error.to_string().contains("TLS"), "unexpected error: {error}"); + }); + + temp_env::with_vars(vars, || { + let mut cfg = vault_kms_test_config("vault-transit"); + cfg.kms_allow_insecure_dev_defaults = true; + let config = build_vault_transit_kms_config(&cfg).expect("the development opt-in should accept skip-verify"); + let vault = config.vault_transit_config().expect("vault transit backend config"); + assert!(vault.tls.as_ref().is_some_and(|tls| tls.skip_verify)); + }); + } + fn aws_kms_test_config() -> crate::config::Config { let mut config = crate::config::Config::new("127.0.0.1:9000", vec!["/tmp/rustfs-aws-kms".to_string()]); config.kms_enable = true; From 89e25132055efa3f316b237b404966cc52364fdb Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 15:04:23 +0800 Subject: [PATCH 06/51] feat(ecstore): pin bitrot algorithms with a startup self-test (HS-11) (#6165) feat(ecstore): pin bitrot algorithms with a startup self-test A drifted HighwayHash implementation fails silently: every shard reads back corrupt, heal rewrites healthy data, and cross-platform clusters disagree about which copy is good. Mirror MinIO's bitrotSelfTest by verifying, once at process start: - known-answer digests for HighwayHash256S / HighwayHash256SLegacy over a deterministic 4096-byte xorshift64* payload, plus the externally verifiable FIPS SHA-256 "abc" vector guarding the HashAlgorithm plumbing itself; - an end-to-end roundtrip per streaming variant (encode -> size formula -> bitrot_verify -> BitrotReader read-back), over full blocks and a partial tail; - tamper detection: one flipped byte in the final data block and one in the leading hash must both be rejected as a hash mismatch, not by an incidental read error. The check costs microseconds and runs inline in init_background_service_runtime before any shard can be written or verified. Outcome surfaces as one structured bitrot_selftest log event, the rustfs_bitrot_selftest_status gauge (1=passed / 0=failed / 2=skipped), a bitrotSelftest field on the admin server-info response, and RUSTFS_BITROT_SELFTEST_STRICT=on turns a failure into a startup error (MinIO Fatal parity; the default only degrades the status so a bad build cannot brick an existing fleet on upgrade). Closes rustfs/backlog#1873 (HS-11). Co-authored-by: heihutu --- crates/ecstore/src/api/mod.rs | 4 +- crates/ecstore/src/erasure/coding/bitrot.rs | 303 +++++++++++++++++++- rustfs/src/admin/handlers/system.rs | 29 ++ rustfs/src/bitrot_selftest.rs | 181 ++++++++++++ rustfs/src/lib.rs | 1 + rustfs/src/module_switches.rs | 14 + rustfs/src/startup_background.rs | 11 +- rustfs/src/storage/storage_api.rs | 4 + rustfs/src/storage_api.rs | 4 +- 9 files changed, 535 insertions(+), 16 deletions(-) create mode 100644 rustfs/src/bitrot_selftest.rs diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index f18da095a..5526b7d00 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -373,8 +373,8 @@ pub mod error { pub mod erasure { pub use crate::erasure::coding::{ - BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, ReedSolomonEncoder, - calc_shard_size, calc_shard_size_legacy, + BitrotReader, BitrotSelfTestError, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, + ReedSolomonEncoder, bitrot_self_test, calc_shard_size, calc_shard_size_legacy, }; } diff --git a/crates/ecstore/src/erasure/coding/bitrot.rs b/crates/ecstore/src/erasure/coding/bitrot.rs index 7b3166fbc..698b1b02c 100644 --- a/crates/ecstore/src/erasure/coding/bitrot.rs +++ b/crates/ecstore/src/erasure/coding/bitrot.rs @@ -820,10 +820,263 @@ impl BitrotWriterWrapper { } } +// --- startup bitrot self-test (rustfs/backlog#1873, MinIO bitrotSelfTest parity) --- +// +// A broken hash implementation (bad SIMD feature combination, platform drift, a +// key-handling regression) fails silently: every shard reads back "corrupt", +// heal rewrites data that was fine, and cross-platform clusters disagree about +// which copy is healthy. The self-test below pins the algorithms the moment a +// process starts, so a drifted build announces itself instead of quietly +// rewriting objects. See docs/rustfs-heal-scanner-vs-minio-comprehensive- +// analysis-2026-08-16.md §6 HS-11. + +/// Length of the deterministic self-test payload. +pub const BITROT_SELF_TEST_PAYLOAD_LEN: usize = 4096; + +/// Known-answer digest of [`bitrot_self_test_payload`] under `HighwayHash256S` +/// (the production default). Pinned so any platform or build where the +/// implementation drifts fails startup instead of mis-hashing shards. +const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S: [u8; 32] = [ + 0xb9, 0x32, 0xa2, 0xaa, 0x4a, 0xb7, 0x33, 0x6a, 0xa3, 0xca, 0x7e, 0x61, 0x9d, 0x86, 0x52, 0x14, 0x6e, 0x7f, 0xd8, 0x9e, 0xea, + 0x08, 0xd9, 0x8c, 0x33, 0x85, 0x87, 0x19, 0x30, 0xd6, 0xed, 0x06, +]; + +/// Known-answer digest of the same payload under `HighwayHash256SLegacy`. +const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY: [u8; 32] = [ + 0x98, 0x24, 0x71, 0x4f, 0x16, 0xbb, 0x48, 0x39, 0xed, 0x68, 0xfa, 0x63, 0x5e, 0xd9, 0x07, 0x61, 0xdf, 0x0a, 0xff, 0xcf, 0x7d, + 0x8c, 0xa8, 0xc7, 0xc0, 0xb6, 0x6f, 0x05, 0xdb, 0xda, 0x5a, 0x22, +]; + +/// FIPS 180-2 test vector: SHA-256 of the ASCII string "abc". Unlike the +/// Highway digests above this one is externally verifiable, so it guards the +/// whole `HashAlgorithm` plumbing even for readers who distrust pinned +/// self-computed constants. +const BITROT_SELF_TEST_KAT_SHA256_ABC: [u8; 32] = [ + 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, + 0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad, +]; + +/// Deterministic self-test payload: xorshift64* from a fixed seed, so every +/// platform and every run hashes the same 4096 bytes. +fn bitrot_self_test_payload() -> [u8; BITROT_SELF_TEST_PAYLOAD_LEN] { + let mut state = 0x9E37_79B9_7F4A_7C15u64; + let mut payload = [0u8; BITROT_SELF_TEST_PAYLOAD_LEN]; + for byte in payload.iter_mut() { + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + *byte = state.wrapping_mul(0x2545_F491_4F6C_DD1D) as u8; + } + payload +} + +/// Why a bitrot self-test failed. +#[derive(Debug)] +pub enum BitrotSelfTestError { + /// A known-answer digest mismatched the pinned constant. + KnownAnswerMismatch { + algorithm: &'static str, + got: String, + want: String, + }, + /// A freshly encoded shard failed `bitrot_verify`. + RoundtripVerify { algorithm: &'static str, detail: String }, + /// A verified roundtrip read back different bytes than were written. + RoundtripReadback { algorithm: &'static str }, + /// A deliberately tampered shard was not rejected by `bitrot_verify`. + TamperNotRejected { + algorithm: &'static str, + tampered: &'static str, + }, +} + +impl std::fmt::Display for BitrotSelfTestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::KnownAnswerMismatch { algorithm, got, want } => { + write!(f, "known-answer mismatch for {algorithm}: got {got}, want {want}") + } + Self::RoundtripVerify { algorithm, detail } => write!(f, "{algorithm} roundtrip shard failed verification: {detail}"), + Self::RoundtripReadback { algorithm } => write!(f, "{algorithm} roundtrip read back different bytes"), + Self::TamperNotRejected { algorithm, tampered } => { + write!(f, "{algorithm} tampered shard ({tampered}) was not rejected") + } + } + } +} + +impl std::error::Error for BitrotSelfTestError {} + +fn self_test_hex(bytes: &[u8]) -> String { + rustfs_utils::hex(bytes) +} + +// (kept as a named one-liner so every KAT failure site reads the same; the +// underlying formatter is the shared `rustfs_utils::hex`) + +/// Compare a digest against its pinned constant. Split out so a test can drive +/// it with a wrong constant and prove the mismatch path fires. +fn bitrot_kat_check( + algorithm: &'static str, + algo: &HashAlgorithm, + payload: &[u8], + expected: &[u8; 32], +) -> Result<(), BitrotSelfTestError> { + let digest = algo.hash_encode(payload); + let digest = digest.as_ref(); + if digest.len() != expected.len() || digest != expected.as_slice() { + return Err(BitrotSelfTestError::KnownAnswerMismatch { + algorithm, + got: self_test_hex(digest), + want: self_test_hex(expected), + }); + } + Ok(()) +} + +/// Encode `payload` with `shard_size` blocks, verify it end to end, and read +/// every block back through `BitrotReader` comparing bytes. +async fn bitrot_roundtrip_check( + algorithm: &'static str, + algo: HashAlgorithm, + payload: &[u8], + shard_size: usize, +) -> Result<(), BitrotSelfTestError> { + let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::::new()), shard_size, algo.clone()); + for chunk in payload.chunks(shard_size) { + writer + .write(chunk) + .await + .map_err(|err| BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: format!("encode failed: {err}"), + })?; + } + let encoded = writer.into_inner().into_inner(); + + let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone()); + if encoded.len() != on_disk { + return Err(BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: format!("encoded {} bytes, size formula says {on_disk}", encoded.len()), + }); + } + bitrot_verify(std::io::Cursor::new(encoded.clone()), on_disk, payload.len(), algo.clone(), shard_size) + .await + .map_err(|err| BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: err.to_string(), + })?; + + let mut reader = BitrotReader::new(std::io::Cursor::new(encoded), shard_size, algo, false); + let mut offset = 0usize; + while offset < payload.len() { + let want = shard_size.min(payload.len() - offset); + let mut buf = vec![0u8; want]; + let read = reader + .read(&mut buf) + .await + .map_err(|err| BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: format!("read back failed at offset {offset}: {err}"), + })?; + if read != want || buf[..read] != payload[offset..offset + read] { + return Err(BitrotSelfTestError::RoundtripReadback { algorithm }); + } + offset += read; + } + Ok(()) +} + +/// Flip one byte and require `bitrot_verify` to reject the result. +async fn bitrot_tamper_check( + algorithm: &'static str, + algo: HashAlgorithm, + payload: &[u8], + shard_size: usize, + tampered: &'static str, + flip_at: usize, +) -> Result<(), BitrotSelfTestError> { + let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::::new()), shard_size, algo.clone()); + for chunk in payload.chunks(shard_size) { + writer.write(chunk).await.expect("self-test encode should not fail"); + } + let mut corrupt = writer.into_inner().into_inner(); + let flip_index = flip_at % corrupt.len(); + corrupt[flip_index] ^= 0x80; + + let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone()); + match bitrot_verify(std::io::Cursor::new(corrupt), on_disk, payload.len(), algo, shard_size).await { + // The flipped byte must be rejected as a hash mismatch specifically, not + // by any incidental read error: an in-memory cursor cannot fail reads, + // so accepting any other failure here would mask a verify path that + // errors out before it ever compares hashes. + Err(err) if err.to_string().contains("hash mismatch") => Ok(()), + Ok(()) => Err(BitrotSelfTestError::TamperNotRejected { algorithm, tampered }), + Err(err) => Err(BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: format!("tampered shard rejected with an unexpected error: {err}"), + }), + } +} + +/// Verify every bitrot algorithm this crate can write or verify in production: +/// both streaming Highway variants roundtrip end to end (encode → size formula +/// → `bitrot_verify` → read back) and reject a flipped byte in both the data +/// and the leading hash, while all three hashed algorithms reproduce their +/// pinned known-answer digests. +/// +/// Runs in well under a millisecond on 4 KiB of data; callers may run it inline +/// at startup. Pure CPU, no allocation beyond a few KiB of scratch. +pub async fn bitrot_self_test() -> Result<(), BitrotSelfTestError> { + let payload = bitrot_self_test_payload(); + + // Externally verifiable vector first: it guards the HashAlgorithm plumbing + // itself, before any self-pinned constants are consulted. + let abc = HashAlgorithm::SHA256.hash_encode(b"abc"); + if abc.as_ref() != BITROT_SELF_TEST_KAT_SHA256_ABC.as_slice() { + return Err(BitrotSelfTestError::KnownAnswerMismatch { + algorithm: "SHA256", + got: self_test_hex(abc.as_ref()), + want: self_test_hex(&BITROT_SELF_TEST_KAT_SHA256_ABC), + }); + } + + bitrot_kat_check( + "HighwayHash256S", + &HashAlgorithm::HighwayHash256S, + &payload, + &BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S, + )?; + bitrot_kat_check( + "HighwayHash256SLegacy", + &HashAlgorithm::HighwayHash256SLegacy, + &payload, + &BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY, + )?; + + for (algorithm, algo) in [ + ("HighwayHash256S", HashAlgorithm::HighwayHash256S), + ("HighwayHash256SLegacy", HashAlgorithm::HighwayHash256SLegacy), + ] { + // Full blocks plus a partial tail, exactly like a real part stripe. + let tail_len = 2 * 1024 + 333; + bitrot_roundtrip_check(algorithm, algo.clone(), &payload, 1024).await?; + bitrot_roundtrip_check(algorithm, algo.clone(), &payload[..tail_len], 1024).await?; + // One flipped byte in the final data block, one in the first leading + // hash: both must fail verification. + bitrot_tamper_check(algorithm, algo.clone(), &payload, 1024, "final data byte", payload.len() - 1).await?; + bitrot_tamper_check(algorithm, algo, &payload, 1024, "leading hash byte", 0).await?; + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::{ - BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_shard_file_size, bitrot_verify, write_all_vectored, + BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_kat_check, bitrot_self_test, + bitrot_self_test_payload, bitrot_shard_file_size, bitrot_verify, write_all_vectored, }; use super::{MAX_RETAINED_CHUNKS_PER_BLOCK, ShardChunkRead, ShardSource}; use bytes::Bytes; @@ -1090,6 +1343,32 @@ mod tests { } } + #[test] + fn bitrot_self_test_payload_is_deterministic() { + // Two independent builds of the payload must agree byte for byte, or + // the pinned known-answer digests below would be meaningless. + assert_eq!(bitrot_self_test_payload(), bitrot_self_test_payload()); + } + + #[test] + fn bitrot_self_test_rejects_a_wrong_known_answer_digest() { + let payload = bitrot_self_test_payload(); + let wrong = [0u8; 32]; + let err = bitrot_kat_check("HighwayHash256S", &HashAlgorithm::HighwayHash256S, &payload, &wrong) + .expect_err("a zeroed digest must never match"); + match err { + super::BitrotSelfTestError::KnownAnswerMismatch { algorithm, .. } => assert_eq!(algorithm, "HighwayHash256S"), + other => panic!("expected KnownAnswerMismatch, got {other:?}"), + } + } + + #[tokio::test] + async fn bitrot_self_test_passes() { + bitrot_self_test() + .await + .expect("the pinned digests and roundtrip checks must all pass on this platform"); + } + #[tokio::test] async fn vectored_test_writers_cover_fallback_flush_and_shutdown_paths() { let mut counting = VectoredCountingWriter::default(); @@ -1189,7 +1468,7 @@ mod tests { let last = corrupt.len() - 1; corrupt[last] ^= 0x80; let err = bitrot_verify( - Cursor::new(corrupt), + std::io::Cursor::new(corrupt), super::bitrot_shard_file_size(data.len(), shard_size, algo.clone()), data.len(), algo, @@ -1282,7 +1561,7 @@ mod tests { #[tokio::test] async fn bitrot_reader_rejects_output_buffers_larger_than_shard_size() { - let mut reader = BitrotReader::new(Cursor::new(Vec::::new()), 4, HashAlgorithm::None, false); + let mut reader = BitrotReader::new(std::io::Cursor::new(Vec::::new()), 4, HashAlgorithm::None, false); let mut out = [0u8; 5]; let err = reader .read(&mut out) @@ -1407,7 +1686,7 @@ mod tests { (HashAlgorithm::HighwayHash256, true), ] { let label = format!("{algo:?}"); - let writer = Cursor::new(Vec::::new()); + let writer = std::io::Cursor::new(Vec::::new()); let mut w = BitrotWriter::new(writer, shard_size, algo.clone()); w.write(&[7u8; 16]).await.unwrap(); let written = w.into_inner().into_inner(); @@ -1492,7 +1771,7 @@ mod tests { } async fn encode_one_block(payload: &[u8], shard_size: usize, algo: HashAlgorithm) -> Vec { - let mut w = BitrotWriter::new(Cursor::new(Vec::::new()), shard_size, algo); + let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::::new()), shard_size, algo); w.write(payload).await.unwrap(); w.into_inner().into_inner() } @@ -1600,7 +1879,7 @@ mod tests { for algo in [HashAlgorithm::HighwayHash256S, HashAlgorithm::HighwayHash256SLegacy] { for &size in &[1usize, 16, 17, 32, 40, 48] { let payload: Vec = (0..size).map(|i| i as u8).collect(); - let mut w = BitrotWriter::new(Cursor::new(Vec::::new()), shard_size, algo.clone()); + let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::::new()), shard_size, algo.clone()); for chunk in payload.chunks(shard_size) { w.write(chunk).await.unwrap(); } @@ -1674,14 +1953,14 @@ mod tests { w.write(&data).await.expect("write shard"); let mut via_read = vec![0u8; SHARD]; - let n1 = BitrotReader::new(Cursor::new(encoded.clone()), SHARD, algo.clone(), false) + let n1 = BitrotReader::new(std::io::Cursor::new(encoded.clone()), SHARD, algo.clone(), false) .read(&mut via_read) .await .expect("read"); // A buffer with only capacity — no initialized bytes at all. let mut via_append: Vec = Vec::with_capacity(SHARD); - let n2 = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false) + let n2 = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false) .read_appending(&mut via_append, SHARD) .await .expect("read_appending"); @@ -1706,7 +1985,7 @@ mod tests { encoded.truncate(encoded.len() - 1); let mut out: Vec = Vec::with_capacity(SHARD); - let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false) + let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false) .read_appending(&mut out, SHARD) .await .expect_err("a truncated shard must not succeed"); @@ -1732,7 +2011,7 @@ mod tests { encoded[last] ^= 0xff; let mut out: Vec = Vec::with_capacity(SHARD); - let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo, false) + let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false) .read_appending(&mut out, SHARD) .await .expect_err("a corrupt shard must not verify"); @@ -1844,7 +2123,7 @@ mod tests { "Cursor must be able to hand out a block, otherwise the fast path is dead code" ); assert_eq!(mem.position(), 8, "taking a block must advance like a read of the same length"); - let mut streamed = Cursor::new(encoded.clone()); + let mut streamed = std::io::Cursor::new(encoded.clone()); assert!( ShardSource::try_take_block(&mut streamed, 8).is_none(), "a non-Bytes source must stay on the streaming path" @@ -1872,7 +2151,7 @@ mod tests { ); let mut via_stream: Vec = Vec::with_capacity(SHARD); - BitrotReader::new(Cursor::new(encoded), SHARD, algo, false) + BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false) .read_appending(&mut via_stream, SHARD) .await .expect("streaming read"); diff --git a/rustfs/src/admin/handlers/system.rs b/rustfs/src/admin/handlers/system.rs index 96054ada5..c7ad93b54 100644 --- a/rustfs/src/admin/handlers/system.rs +++ b/rustfs/src/admin/handlers/system.rs @@ -417,6 +417,13 @@ struct SystemAdminDiscovery { struct ServerInfoResponse { info: InfoMessage, admin_discovery: SystemAdminDiscovery, + /// Startup bitrot algorithm self-test outcome (rustfs/backlog#1873): + /// `passed` (algorithms verified at boot), `failed` (a drifted hash + /// implementation — the process is serving with degraded integrity + /// checking unless `RUSTFS_BITROT_SELFTEST_STRICT` aborted it), or + /// `unknown` (not yet run or disabled). + #[serde(rename = "bitrotSelftest")] + bitrot_selftest: &'static str, } #[derive(Serialize)] @@ -433,6 +440,14 @@ fn system_admin_discovery(usecase: &DefaultAdminUsecase) -> SystemAdminDiscovery } } +fn bitrot_selftest_status_str() -> &'static str { + match crate::bitrot_selftest::bitrot_selftest_passed() { + Some(true) => "passed", + Some(false) => "failed", + None => "unknown", + } +} + #[async_trait::async_trait] impl Operation for ServerInfoHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { @@ -464,6 +479,7 @@ impl Operation for ServerInfoHandler { let response = ServerInfoResponse { info, admin_discovery: system_admin_discovery(&usecase), + bitrot_selftest: bitrot_selftest_status_str(), }; let data = serde_json::to_vec(&response).map_err(|e| { @@ -1535,6 +1551,18 @@ mod tests { ); } + /// The startup bitrot self-test outcome must surface in server info as one + /// of three closed-set strings, never an internal enum or a null + /// (rustfs/backlog#1873). This test pins the string mapping; whether the + /// process-global cell holds Some(true)/Some(false)/None is owned by + /// `crate::bitrot_selftest`'s own tests. + #[test] + fn bitrot_selftest_status_str_is_a_closed_set_of_operators_strings() { + let rendered = super::bitrot_selftest_status_str(); + assert!(matches!(rendered, "passed" | "failed" | "unknown")); + assert_eq!(super::bitrot_selftest_status_str(), rendered); + } + #[test] fn server_info_response_exposes_admin_discovery_paths() { let usecase = DefaultAdminUsecase::without_context(); @@ -1556,6 +1584,7 @@ mod tests { pools: None, }, admin_discovery: system_admin_discovery(&usecase), + bitrot_selftest: super::bitrot_selftest_status_str(), }; let value = serde_json::to_value(response).expect("server info response should serialize"); diff --git a/rustfs/src/bitrot_selftest.rs b/rustfs/src/bitrot_selftest.rs new file mode 100644 index 000000000..4b8c1d1c4 --- /dev/null +++ b/rustfs/src/bitrot_selftest.rs @@ -0,0 +1,181 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Startup bitrot algorithm self-test (rustfs/backlog#1873). +//! +//! A drifted hash implementation fails silently in production: every shard +//! reads back "corrupt", heal rewrites healthy data, and cross-platform +//! clusters disagree about which copy is good. [`run_startup_bitrot_self_test`] +//! pins the algorithms once at process start — the check itself runs in well +//! under a millisecond on 4 KiB, so it executes inline before background +//! services come up and the result is published before the server accepts +//! traffic. +//! +//! Outcome surface: +//! - one structured `bitrot_selftest` log event (`passed`/`failed`/`skipped`), +//! - the `rustfs_bitrot_selftest_status` gauge (1=passed, 0=failed, 2=skipped), +//! - [`bitrot_selftest_passed`] for admin/health surfaces, +//! - `RUSTFS_BITROT_SELFTEST_STRICT=on` turns a failure into a startup error +//! (MinIO `bitrotSelfTest` Fatal parity); the default only degrades the +//! status so a bad build cannot brick an existing fleet on upgrade. + +use crate::storage_api::startup::background::{BitrotSelfTestError, bitrot_self_test}; +use metrics::gauge; +use std::future::Future; +use std::io; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::time::Instant; +use tracing::{debug, error, info}; + +const LOG_COMPONENT_MAIN: &str = "main"; +const LOG_SUBSYSTEM_STARTUP: &str = "startup"; +const EVENT_BITROT_SELFTEST: &str = "bitrot_selftest"; +const METRIC_BITROT_SELFTEST_STATUS: &str = "rustfs_bitrot_selftest_status"; + +/// Gauge values for [`METRIC_BITROT_SELFTEST_STATUS`]. +const STATUS_PASSED: f64 = 1.0; +const STATUS_FAILED: f64 = 0.0; +const STATUS_SKIPPED: f64 = 2.0; + +/// Internal cell values for [`BITROT_SELF_TEST_STATUS`]. +const STATUS_CELL_UNSET: u8 = 0; +const STATUS_CELL_PASSED: u8 = 1; +const STATUS_CELL_FAILED: u8 = 2; + +static BITROT_SELF_TEST_STATUS: AtomicU8 = AtomicU8::new(STATUS_CELL_UNSET); + +/// Last recorded self-test outcome: `None` before the first run, then +/// `Some(true)` on a passing check and `Some(false)` on a failed one (a +/// skipped check never publishes, so it cannot read as a pass). The cell is +/// last-writer-wins rather than set-once: production runs the self-test once, +/// and last-writer-wins keeps tests that exercise both outcomes +/// order-independent. +pub fn bitrot_selftest_passed() -> Option { + match BITROT_SELF_TEST_STATUS.load(Ordering::Acquire) { + STATUS_CELL_UNSET => None, + STATUS_CELL_PASSED => Some(true), + STATUS_CELL_FAILED => Some(false), + _ => None, + } +} + +/// Run the bitrot self-test and publish the outcome. In strict mode a failure +/// is returned as an error so the caller aborts startup. +pub(crate) async fn run_startup_bitrot_self_test(enabled: bool, strict: bool) -> io::Result<()> { + run_startup_bitrot_self_test_with(enabled, strict, bitrot_self_test).await +} + +async fn run_startup_bitrot_self_test_with(enabled: bool, strict: bool, run_check: F) -> io::Result<()> +where + F: FnOnce() -> Fut, + Fut: Future>, +{ + if !enabled { + gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_SKIPPED); + debug!( + target: "rustfs::main::run", + event = EVENT_BITROT_SELFTEST, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "skipped", + reason = "disabled", + "Bitrot self-test skipped" + ); + return Ok(()); + } + + let started = Instant::now(); + match run_check().await { + Ok(()) => { + BITROT_SELF_TEST_STATUS.store(STATUS_CELL_PASSED, Ordering::Release); + gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_PASSED); + info!( + target: "rustfs::main::run", + event = EVENT_BITROT_SELFTEST, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "passed", + duration_us = started.elapsed().as_micros() as u64, + "Bitrot self-test passed" + ); + } + Err(err) => { + BITROT_SELF_TEST_STATUS.store(STATUS_CELL_FAILED, Ordering::Release); + gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_FAILED); + error!( + target: "rustfs::main::run", + event = EVENT_BITROT_SELFTEST, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "failed", + duration_us = started.elapsed().as_micros() as u64, + error = %err, + "Bitrot self-test failed" + ); + if strict { + return Err(io::Error::other(format!("bitrot self-test failed: {err}"))); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{BITROT_SELF_TEST_STATUS, STATUS_CELL_UNSET, bitrot_selftest_passed, run_startup_bitrot_self_test_with}; + use crate::storage_api::startup::background::BitrotSelfTestError; + use std::future::ready; + use std::sync::atomic::Ordering; + + fn failing_check() -> impl Future> { + ready(Err(BitrotSelfTestError::RoundtripReadback { + algorithm: "HighwayHash256S", + })) + } + + /// All scenarios run sequentially inside one test: the status cell is + /// process-global, so parallel per-scenario tests would race the reset and + /// read each other's outcomes (the exact order-dependent flake class this + /// module exists to avoid). + #[tokio::test] + async fn startup_self_test_publishes_outcome_and_strict_gates_abort() { + BITROT_SELF_TEST_STATUS.store(STATUS_CELL_UNSET, Ordering::Release); + + // Skipped: publishes nothing, never fails, never aborts. + run_startup_bitrot_self_test_with(false, true, || async { Ok(()) }) + .await + .expect("a disabled self-test must not fail even in strict mode"); + assert_eq!(bitrot_selftest_passed(), None, "a skipped run must leave the status unset"); + + // Passing: publishes Some(true), never fails. + run_startup_bitrot_self_test_with(true, false, || async { Ok(()) }) + .await + .expect("a passing check must never fail startup"); + assert_eq!(bitrot_selftest_passed(), Some(true), "a passing run must publish Some(true)"); + + // Failing, non-strict: publishes Some(false) but startup continues. + run_startup_bitrot_self_test_with(true, false, failing_check) + .await + .expect("a failed check must not abort startup in non-strict mode"); + assert_eq!(bitrot_selftest_passed(), Some(false), "a failing run must publish Some(false)"); + + // Failing, strict: startup error carries the failure and the published + // outcome stays a failure. + let err = run_startup_bitrot_self_test_with(true, true, failing_check) + .await + .expect_err("strict mode must turn a failed check into a startup error"); + assert!(err.to_string().contains("bitrot self-test failed")); + assert_eq!(bitrot_selftest_passed(), Some(false)); + } +} diff --git a/rustfs/src/lib.rs b/rustfs/src/lib.rs index f8e9d893d..b60a2bd20 100644 --- a/rustfs/src/lib.rs +++ b/rustfs/src/lib.rs @@ -76,6 +76,7 @@ pub mod allocator_reclaim; pub mod app; pub mod auth; pub mod auth_keystone; +pub(crate) mod bitrot_selftest; pub mod capacity; pub mod cluster_snapshot; pub mod config; diff --git a/rustfs/src/module_switches.rs b/rustfs/src/module_switches.rs index fcb0ffcef..ced58b0ec 100644 --- a/rustfs/src/module_switches.rs +++ b/rustfs/src/module_switches.rs @@ -33,6 +33,8 @@ pub(crate) const ENV_SCANNER_ENABLED: &str = "RUSTFS_SCANNER_ENABLED"; pub(crate) const ENV_SCANNER_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_SCANNER"; pub(crate) const ENV_HEAL_ENABLED: &str = "RUSTFS_HEAL_ENABLED"; pub(crate) const ENV_HEAL_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_HEAL"; +pub(crate) const ENV_BITROT_SELFTEST_ENABLE: &str = "RUSTFS_BITROT_SELFTEST_ENABLE"; +pub(crate) const ENV_BITROT_SELFTEST_STRICT: &str = "RUSTFS_BITROT_SELFTEST_STRICT"; static AUDIT_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_AUDIT_ENABLE); static NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE); @@ -47,6 +49,18 @@ pub(crate) fn heal_enabled_from_env() -> bool { get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true) } +/// Whether the startup bitrot algorithm self-test runs, defaulting to on +/// (rustfs/backlog#1873). +pub(crate) fn bitrot_selftest_enabled_from_env() -> bool { + rustfs_utils::get_env_bool(ENV_BITROT_SELFTEST_ENABLE, true) +} + +/// Whether a failed bitrot self-test aborts startup instead of only logging +/// and exposing a failed status, defaulting to off. +pub(crate) fn bitrot_selftest_strict_from_env() -> bool { + rustfs_utils::get_env_bool(ENV_BITROT_SELFTEST_STRICT, false) +} + /// Last published audit-module state. pub fn is_audit_module_enabled() -> bool { AUDIT_MODULE_ENABLED.load(Ordering::Relaxed) diff --git a/rustfs/src/startup_background.rs b/rustfs/src/startup_background.rs index 8cee2c502..fd5366740 100644 --- a/rustfs/src/startup_background.rs +++ b/rustfs/src/startup_background.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::module_switches::{heal_enabled_from_env, scanner_enabled_from_env}; +use crate::bitrot_selftest::run_startup_bitrot_self_test; +use crate::module_switches::{ + bitrot_selftest_enabled_from_env, bitrot_selftest_strict_from_env, heal_enabled_from_env, scanner_enabled_from_env, +}; use crate::storage_api::startup::background::{ECStore, set_workload_admission_snapshot_provider}; use crate::workload_admission::RustFsWorkloadAdmissionSnapshotProvider; use rustfs_concurrency::WorkloadAdmissionSnapshotProvider; @@ -27,6 +30,12 @@ const LOG_SUBSYSTEM_STARTUP: &str = "startup"; const EVENT_BACKGROUND_SERVICES_CONFIGURED: &str = "background_services_configured"; pub(crate) async fn init_background_service_runtime(store: Arc) -> Result { + // Pin the bitrot algorithms before anything can write or verify a shard: + // the check costs well under a millisecond, and in strict mode a drifted + // build must abort here rather than after it has touched data + // (rustfs/backlog#1873). + run_startup_bitrot_self_test(bitrot_selftest_enabled_from_env(), bitrot_selftest_strict_from_env()).await?; + let _ = create_ahm_services_cancel_token(); let enable_scanner = scanner_enabled_from_env(); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 5d217f18e..d1e7c0f3a 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -569,6 +569,10 @@ pub(crate) mod ecstore_erasure { pub(crate) use rustfs_ecstore::api::erasure::{BitrotReader, Erasure}; } +/// Startup bitrot algorithm self-test (rustfs/backlog#1873), re-exported for +/// the root facade's background-startup section. +pub(crate) use rustfs_ecstore::api::erasure::{BitrotSelfTestError, bitrot_self_test}; + pub(crate) mod ecstore_storage { #[cfg(test)] pub(crate) use rustfs_ecstore::api::storage::init_local_disks; diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index b9b8d3c64..e90c1f528 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -214,7 +214,9 @@ pub(crate) mod startup { } pub(crate) mod background { - pub(crate) use crate::storage::storage_api::{ECStore, set_workload_admission_snapshot_provider}; + pub(crate) use crate::storage::storage_api::{ + BitrotSelfTestError, ECStore, bitrot_self_test, set_workload_admission_snapshot_provider, + }; } pub(crate) mod bucket_metadata { From 23b17c2d5ac9d6e4e7ebdb44ee4efb0bad102f5c Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 15:04:49 +0800 Subject: [PATCH 07/51] feat(madmin): add a SigV4-signed admin client for heal and scanner APIs (HS-05) (#6166) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feat(madmin): add a SigV4-signed admin client for heal and scanner APIs The madmin crate held only wire types; automation and mc-style tooling had no way to drive the heal/scanner admin surface without hand-rolled HTTP. Add `AdminClient`, which signs with the same rustfs-signer path the server authenticates (UNSIGNED-PAYLOAD marker, matching RustFS peer admin calls) and wraps: - heal_start / heal_status / heal_stop over POST /rustfs/admin/v3/heal/ (bucket/prefix path params percent-encoded per segment; stop models the server's two cancel branches: token-scoped task status vs path-scoped start-success receipt); - background_heal_status, scanner_status (freshness typed), plus ilm_expiry_status / replacement_recovery_status passthroughs; - a public get_json escape hatch for endpoints not wrapped yet. Wire types follow the madmin-go model (SDK-owned mirrors pinned by round-trip tests): HealOpts with serde defaults so partial settings objects decode, HealScanMode accepting both the numeric and name encodings, and status structs that type the fields operators branch on while flattening unknown nested payloads verbatim so server additions cannot break the client. Errors map to a closed AdminClientError enum (InvalidEndpoint / Transport / HttpStatus with body / Decode). Tests cover wire round-trips, path building, both stop branches, error mapping, and — via a dependency-free raw-TCP test server — that signed requests carry a SigV4 Authorization header, the right method/path/ query, and the expected JSON body. Closes rustfs/backlog#1869 (first increment; single-sourcing the wire structs server-side and an embedded-server e2e roundtrip are noted as follow-ups there). Co-authored-by: heihutu --- Cargo.lock | 5 + crates/madmin/Cargo.toml | 5 + crates/madmin/src/client.rs | 851 ++++++++++++++++++++++++++++++++++++ crates/madmin/src/lib.rs | 2 + 4 files changed, 863 insertions(+) create mode 100644 crates/madmin/src/client.rs diff --git a/Cargo.lock b/Cargo.lock index 38637d173..b521a731d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9825,14 +9825,19 @@ name = "rustfs-madmin" version = "1.0.0-rc.2" dependencies = [ "hotpath", + "http 1.5.0", "humantime", "hyper", "jiff", + "reqwest", "rmp-serde", + "rustfs-signer", + "s3s", "serde", "serde_json", "sysinfo", "time", + "tokio", ] [[package]] diff --git a/crates/madmin/Cargo.toml b/crates/madmin/Cargo.toml index 6a1fe0ff5..dfd346334 100644 --- a/crates/madmin/Cargo.toml +++ b/crates/madmin/Cargo.toml @@ -37,7 +37,11 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] [dependencies] hotpath.workspace = true humantime.workspace = true +http.workspace = true hyper = { workspace = true, features = ["http2", "http1", "server"] } +reqwest = { workspace = true, features = ["json"] } +rustfs-signer.workspace = true +s3s.workspace = true jiff = { workspace = true, features = ["serde"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value"] } @@ -49,3 +53,4 @@ doctest = false [dev-dependencies] rmp-serde.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net"] } diff --git a/crates/madmin/src/client.rs b/crates/madmin/src/client.rs new file mode 100644 index 000000000..273f922a3 --- /dev/null +++ b/crates/madmin/src/client.rs @@ -0,0 +1,851 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Admin API HTTP client for heal and scanner management (rustfs/backlog#1869). +//! +//! [`AdminClient`] speaks the `/rustfs/admin/v3` surface with S3 SigV4 +//! request signing (the same scheme the server's admin router authenticates), +//! so `mc`-style tooling and automation can drive heal start/query/cancel and +//! read background-heal / scanner status without hand-rolling HTTP. +//! +//! Wire structs in this module mirror the server-side shapes +//! (`rustfs/src/admin/handlers/heal.rs`, `handlers/scanner.rs`, +//! `rustfs-common/src/heal_channel.rs`), following the madmin-go model where +//! the SDK owns its own copies and round-trip tests pin the encoding. Deeply +//! nested status payloads that the server composes from runtime types are +//! carried through as `serde_json::Value` and flattened maps rather than +//! duplicated field-for-field, so the client cannot silently drift on fields +//! it never interprets. + +use crate::heal_commands::HealResultItem; +use http::Method; +use serde::{Deserialize, Serialize, de}; +use std::time::Duration; + +/// Default admin API path prefix on a RustFS endpoint. +pub const DEFAULT_ADMIN_API_PREFIX: &str = "/rustfs/admin"; +/// Default SigV4 region when the server has no explicit region configured. +pub const DEFAULT_REGION: &str = "us-east-1"; + +/// Scan mode for a heal request, mirroring the server's numeric-or-name wire +/// encoding (`0` unknown/default, `1` normal, `2` deep). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum HealScanMode { + /// Server default; behaves as [`HealScanMode::Normal`]. + #[default] + Unknown, + /// Metadata-level checks only. + Normal, + /// Full bitrot verification while healing. + Deep, +} + +impl HealScanMode { + fn wire_number(self) -> u8 { + match self { + Self::Unknown => 0, + Self::Normal => 1, + Self::Deep => 2, + } + } + + fn from_wire_number(value: u8) -> Option { + match value { + 0 => Some(Self::Unknown), + 1 => Some(Self::Normal), + 2 => Some(Self::Deep), + _ => None, + } + } + + fn from_wire_name(value: &str) -> Option { + match value { + "unknown" => Some(Self::Unknown), + "normal" => Some(Self::Normal), + "deep" => Some(Self::Deep), + _ => None, + } + } +} + +impl Serialize for HealScanMode { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_u8(self.wire_number()) + } +} + +impl<'de> Deserialize<'de> for HealScanMode { + fn deserialize>(deserializer: D) -> Result { + struct HealScanModeVisitor; + + impl de::Visitor<'_> for HealScanModeVisitor { + type Value = HealScanMode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a heal scan mode number or name") + } + + fn visit_u64(self, value: u64) -> Result { + u8::try_from(value) + .ok() + .and_then(HealScanMode::from_wire_number) + .ok_or_else(|| E::custom(format!("unknown heal scan mode number: {value}"))) + } + + fn visit_str(self, value: &str) -> Result { + HealScanMode::from_wire_name(value).ok_or_else(|| E::custom(format!("unknown heal scan mode name: {value}"))) + } + } + + deserializer.deserialize_any(HealScanModeVisitor) + } +} + +/// Heal options for an admin heal request (mirror of the server body type). +/// Fields default on decode: a client should tolerate a server response whose +/// settings object omits fields it never set. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct HealOpts { + #[serde(default)] + pub recursive: bool, + #[serde(rename = "dryRun", default)] + pub dry_run: bool, + #[serde(default)] + pub remove: bool, + #[serde(default)] + pub recreate: bool, + #[serde(rename = "scanMode", default)] + pub scan_mode: HealScanMode, + #[serde(rename = "updateParity", default)] + pub update_parity: bool, + #[serde(rename = "nolock", default)] + pub no_lock: bool, + #[serde(rename = "pool", default)] + pub pool: Option, + #[serde(rename = "set", default)] + pub set: Option, +} + +/// Successful heal start / path-scoped cancel response. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HealStartSuccess { + pub client_token: String, + pub client_address: String, + #[serde(default)] + pub start_time: String, +} + +/// Heal task status response (query, cancel-with-token, start-then-poll). +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HealTaskStatus { + /// `running` | `finished` | `stopped` | `notFound`. + pub summary: String, + /// Failure detail for stopped tasks; empty otherwise. + #[serde(rename = "detail", default)] + pub failure_detail: String, + #[serde(default)] + pub start_time: String, + #[serde(default)] + pub settings: HealOpts, + #[serde(default)] + pub items: Vec, + #[serde(default)] + pub truncated: bool, + /// Live progress snapshot; the exact shape is owned by the heal runtime. + #[serde(default)] + pub progress: Option, +} + +/// `POST /v3/background-heal/status` response. Known top-level fields are +/// typed; the flattened heal info and operations matrix pass through verbatim. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BackgroundHealStatus { + /// `disabled` | `uninitialized` | `idle` | `active` | `degraded`. + pub state: String, + #[serde(default)] + pub heal_queue_length: u64, + #[serde(default)] + pub heal_active_tasks: u64, + #[serde(default)] + pub cluster_status_complete: bool, + #[serde(default)] + pub progress: Option, + /// Remaining wire fields (flattened `BackgroundHealInfo` plus the + /// priority-by-source operations matrix), carried verbatim. + #[serde(flatten)] + pub extra: serde_json::Map, +} + +/// `GET /v3/scanner/status` response, typed at the fields operators branch +/// on; everything else passes through verbatim. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScannerStatus { + pub enabled: bool, + /// `fresh` | `stale` | `unknown`; absent when the scanner never completed + /// a cycle. + #[serde(default)] + pub freshness: Option, + #[serde(flatten)] + pub extra: serde_json::Map, +} + +/// Freshness block of the scanner status response. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScannerFreshness { + /// `fresh` | `stale` | `unknown`. + pub state: String, +} + +impl ScannerStatus { + /// Convenience accessor for the freshness state string. + pub fn freshness(&self) -> &str { + self.freshness + .as_ref() + .map(|freshness| freshness.state.as_str()) + .unwrap_or("unknown") + } +} + +/// Everything that can go wrong in an admin client call. +#[derive(Debug)] +pub enum AdminClientError { + /// The endpoint URL could not be parsed. + InvalidEndpoint(String), + /// Request build/send failed (DNS, connect, timeout, body read). + Transport(reqwest::Error), + /// The server answered a non-2xx status. + HttpStatus { status: u16, body: String }, + /// The response body did not decode into the expected shape. + Decode { message: String }, +} + +impl std::fmt::Display for AdminClientError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidEndpoint(message) => write!(f, "invalid admin endpoint: {message}"), + Self::Transport(err) => write!(f, "admin request transport failure: {err}"), + Self::HttpStatus { status, body } => write!(f, "admin request failed with HTTP {status}: {body}"), + Self::Decode { message } => write!(f, "admin response decode failure: {message}"), + } + } +} + +impl std::error::Error for AdminClientError {} + +impl From for AdminClientError { + fn from(err: reqwest::Error) -> Self { + Self::Transport(err) + } +} + +/// A signed client for a RustFS admin API. +#[derive(Debug, Clone)] +pub struct AdminClient { + endpoint: reqwest::Url, + access_key: String, + secret_key: String, + session_token: String, + region: String, + api_prefix: String, + http: reqwest::Client, +} + +impl AdminClient { + /// Build a client for `endpoint` (e.g. `http://127.0.0.1:9000`) using root + /// or admin credentials. Requests are SigV4-signed with the same scheme + /// the server's admin router authenticates. + pub fn new(endpoint: &str, access_key: &str, secret_key: &str) -> Result { + let url = reqwest::Url::parse(endpoint).map_err(|err| AdminClientError::InvalidEndpoint(err.to_string()))?; + if url.host_str().is_none() { + return Err(AdminClientError::InvalidEndpoint("endpoint has no host".to_string())); + } + let http = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(30)) + .build() + .map_err(AdminClientError::Transport)?; + Ok(Self { + endpoint: url, + access_key: access_key.to_string(), + secret_key: secret_key.to_string(), + session_token: String::new(), + region: DEFAULT_REGION.to_string(), + api_prefix: DEFAULT_ADMIN_API_PREFIX.to_string(), + http, + }) + } + + /// Attach an STS session token (signed as `x-amz-security-token`). + pub fn with_session_token(mut self, session_token: impl Into) -> Self { + self.session_token = session_token.into(); + self + } + + /// Override the SigV4 region (defaults to `us-east-1`, matching a + /// region-less RustFS deployment). + pub fn with_region(mut self, region: impl Into) -> Self { + self.region = region.into(); + self + } + + /// Override the admin API path prefix (defaults to `/rustfs/admin`). + pub fn with_api_prefix(mut self, prefix: impl Into) -> Self { + self.api_prefix = prefix.into(); + self + } + + /// Start a heal. `bucket` empty and `prefix` empty heals the whole + /// deployment (requires `recursive` or a `pool`/`set` pair in `opts`, + /// enforced server-side); a bucket alone heals the bucket (the server + /// forces `recursive` for bucket heals). + pub async fn heal_start( + &self, + bucket: Option<&str>, + prefix: Option<&str>, + opts: &HealOpts, + force_start: bool, + ) -> Result { + let body = serde_json::to_vec(opts).map_err(|err| AdminClientError::Decode { + message: err.to_string(), + })?; + let mut query = Vec::new(); + if force_start { + query.push(("forceStart", "true".to_string())); + } + self.post_json(&heal_path(bucket, prefix), &query, body).await + } + + /// Query the status of the heal identified by `client_token` (the token + /// returned by [`Self::heal_start`]) at the path it was started on. + pub async fn heal_status( + &self, + bucket: Option<&str>, + prefix: Option<&str>, + client_token: &str, + ) -> Result { + self.post_json(&heal_path(bucket, prefix), &[("clientToken", client_token.to_string())], Vec::new()) + .await + } + + /// Stop a heal: with a `client_token` only that task is cancelled and its + /// final status returned; without one, every heal task at the path is + /// cancelled (the server answers with a start-success-shaped receipt). + pub async fn heal_stop( + &self, + bucket: Option<&str>, + prefix: Option<&str>, + client_token: Option<&str>, + ) -> Result { + let mut query = vec![("forceStop", "true".to_string())]; + if let Some(token) = client_token { + query.push(("clientToken", token.to_string())); + } + match client_token { + Some(_) => { + let status: HealTaskStatus = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?; + Ok(HealStopOutcome::Stopped(status)) + } + None => { + let success: HealStartSuccess = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?; + Ok(HealStopOutcome::PathStopped(success)) + } + } + } + + /// Cluster-aggregated background heal status. + pub async fn background_heal_status(&self) -> Result { + self.get_json("/v3/background-heal/status").await + } + + /// Data scanner status (enabled state, freshness, runtime config). + pub async fn scanner_status(&self) -> Result { + self.get_json("/v3/scanner/status").await + } + + /// ILM expiry worker status. The payload is owned by the expiry + /// subsystem and still evolving; returned verbatim. + pub async fn ilm_expiry_status(&self) -> Result { + self.get_json("/v3/ilm/expiry/status").await + } + + /// Durable replacement-recovery status (admin v4). The payload is owned + /// by the heal runtime; returned verbatim. + pub async fn replacement_recovery_status(&self) -> Result { + self.get_json("/v4/heal/replacement-recovery").await + } + + /// Signed GET returning a decoded JSON body; escape hatch for endpoints + /// this client does not wrap yet. + pub async fn get_json Deserialize<'de>>(&self, path: &str) -> Result { + let url = self.url_for(path, &[])?; + let request = self.sign_and_build(Method::GET, url, Vec::new(), None).await?; + self.execute(request).await + } + + /// Signed POST returning a decoded JSON body. + async fn post_json Deserialize<'de>>( + &self, + path: &str, + query: &[(&str, String)], + body: Vec, + ) -> Result { + let content_type = if body.is_empty() { None } else { Some("application/json") }; + let url = self.url_for(path, query)?; + let request = self.sign_and_build(Method::POST, url, body, content_type).await?; + self.execute(request).await + } + + fn url_for(&self, path: &str, query: &[(&str, String)]) -> Result { + let mut url = self + .endpoint + .join(&format!("{}{}", self.api_prefix.trim_end_matches('/'), path)) + .map_err(|err| AdminClientError::InvalidEndpoint(err.to_string()))?; + if !query.is_empty() { + let mut pairs = url.query_pairs_mut(); + for (key, value) in query { + pairs.append_pair(key, value); + } + } + Ok(url) + } + + /// Build a SigV4-signed request via the same signer the server trusts, + /// then hand the signed headers to the HTTP client. The signature covers + /// method, path, query, and an unsigned-payload marker — the same shape + /// RustFS itself sends for peer admin calls. + async fn sign_and_build( + &self, + method: Method, + url: reqwest::Url, + body: Vec, + content_type: Option<&str>, + ) -> Result { + let authority = match (url.host_str(), url.port_or_known_default()) { + (Some(host), Some(port)) => format!("{host}:{port}"), + _ => return Err(AdminClientError::InvalidEndpoint("endpoint has no authority".to_string())), + }; + let mut builder = http::Request::builder() + .method(method.clone()) + .uri(url.as_str()) + .header(http::header::HOST, &authority) + .header("x-amz-content-sha256", rustfs_signer::constants::UNSIGNED_PAYLOAD); + if let Some(content_type) = content_type { + builder = builder.header(http::header::CONTENT_TYPE, content_type); + } + let unsigned = builder + .body(s3s::Body::empty()) + .map_err(|err| AdminClientError::InvalidEndpoint(format!("build request failed: {err}")))?; + let signed = rustfs_signer::sign_v4( + unsigned, + body.len() as i64, + &self.access_key, + &self.secret_key, + &self.session_token, + &self.region, + ); + + let mut request = self + .http + .request(method, url) + .body(body) + .build() + .map_err(AdminClientError::Transport)?; + let headers = request.headers_mut(); + for (name, value) in signed.headers().iter() { + // HOST is owned by the HTTP client; the signed value above was + // built from the same URL authority, so they always agree. + if name == http::header::HOST { + continue; + } + headers.insert(name, value.clone()); + } + Ok(request) + } + + async fn execute Deserialize<'de>>(&self, request: reqwest::Request) -> Result { + let response = self.http.execute(request).await?; + let status = response.status(); + let bytes = response.bytes().await?; + if !status.is_success() { + return Err(AdminClientError::HttpStatus { + status: status.as_u16(), + body: String::from_utf8_lossy(&bytes).into_owned(), + }); + } + serde_json::from_slice(&bytes).map_err(|err| AdminClientError::Decode { + message: err.to_string(), + }) + } +} + +/// Response of [`AdminClient::heal_stop`]: cancelling a single tokened task +/// answers with that task's status, cancelling a whole path answers with a +/// start-success-shaped receipt. +#[derive(Debug, Clone)] +pub enum HealStopOutcome { + Stopped(HealTaskStatus), + PathStopped(HealStartSuccess), +} + +fn heal_path(bucket: Option<&str>, prefix: Option<&str>) -> String { + match (bucket, prefix) { + (Some(bucket), Some(prefix)) if !bucket.is_empty() && !prefix.is_empty() => { + format!("/v3/heal/{}/{}", percent_encode_path_segment(bucket), percent_encode_path_segment(prefix)) + } + (Some(bucket), Some(_)) | (Some(bucket), None) if !bucket.is_empty() => { + format!("/v3/heal/{}", percent_encode_path_segment(bucket)) + } + _ => "/v3/heal/".to_string(), + } +} + +/// Encode a single path segment (slashes are content, not separators, inside +/// bucket/prefix path params). +fn percent_encode_path_segment(segment: &str) -> String { + let mut out = String::with_capacity(segment.len()); + for byte in segment.bytes() { + match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => out.push(byte as char), + _ => out.push_str(&format!("%{byte:02X}")), + } + } + out +} + +#[cfg(test)] +mod tests { + use super::{ + AdminClient, AdminClientError, BackgroundHealStatus, HealOpts, HealScanMode, HealStartSuccess, HealTaskStatus, + ScannerStatus, heal_path, percent_encode_path_segment, + }; + use serde_json::json; + use std::sync::{Arc, Mutex}; + + #[test] + fn heal_paths_cover_root_bucket_and_prefix() { + assert_eq!(heal_path(None, None), "/v3/heal/"); + assert_eq!(heal_path(Some(""), Some("")), "/v3/heal/"); + assert_eq!(heal_path(Some("bucket"), None), "/v3/heal/bucket"); + assert_eq!(heal_path(Some("bucket"), Some("pre/fix")), "/v3/heal/bucket/pre%2Ffix"); + } + + #[test] + fn path_segments_percent_encode_reserved_characters() { + assert_eq!(percent_encode_path_segment("a b"), "a%20b"); + assert_eq!(percent_encode_path_segment("a/b"), "a%2Fb"); + assert_eq!(percent_encode_path_segment("ü"), "%C3%BC"); + } + + #[test] + fn heal_opts_round_trip_through_the_server_wire_shape() { + let opts = HealOpts { + recursive: true, + dry_run: false, + remove: true, + recreate: false, + scan_mode: HealScanMode::Deep, + update_parity: true, + no_lock: false, + pool: Some(1), + set: Some(2), + }; + let wire = serde_json::to_value(&opts).unwrap(); + assert_eq!(wire["scanMode"], json!(2), "the server body decodes scanMode as a number"); + let back: HealOpts = serde_json::from_value(wire).unwrap(); + assert_eq!(back.scan_mode, HealScanMode::Deep); + assert_eq!(back.pool, Some(1)); + } + + #[test] + fn heal_scan_mode_accepts_both_wire_encodings() { + assert_eq!(serde_json::from_value::(json!(1)).unwrap(), HealScanMode::Normal); + assert_eq!(serde_json::from_value::(json!("deep")).unwrap(), HealScanMode::Deep); + assert!(serde_json::from_value::(json!(9)).is_err()); + assert!(serde_json::from_value::(json!("sideways")).is_err()); + } + + #[test] + fn heal_task_status_decodes_the_server_response_shape() { + let raw = json!({ + "summary": "finished", + "detail": "", + "startTime": "2026-08-17T00:00:00Z", + "settings": {"recursive": false, "scanMode": 1}, + "items": [{ + "resultId": 1, "type": "object", "bucket": "b", "object": "o", "versionId": "", "detail": "", + "parityBlocks": 2, "dataBlocks": 2, "diskCount": 4, "setCount": 1, + "before": {"drives": []}, "after": {"drives": []}, "objectSize": 128 + }], + "truncated": false + }); + let status: HealTaskStatus = serde_json::from_value(raw).unwrap(); + assert_eq!(status.summary, "finished"); + assert_eq!(status.items.len(), 1); + assert_eq!(status.settings.scan_mode, HealScanMode::Normal); + assert!(status.progress.is_none()); + } + + #[test] + fn background_heal_status_types_known_fields_and_passes_the_rest_through() { + let raw = json!({ + "state": "active", + "bitrotStartTime": "t", + "healQueueLength": 3, + "healActiveTasks": 1, + "healOperations": {"queueLength": 3}, + "clusterStatusComplete": true + }); + let status: BackgroundHealStatus = serde_json::from_value(raw).unwrap(); + assert_eq!(status.state, "active"); + assert_eq!(status.heal_queue_length, 3); + assert!(status.cluster_status_complete); + assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through"); + } + + #[test] + fn scanner_status_defaults_freshness_to_unknown() { + let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}}); + let status: ScannerStatus = serde_json::from_value(raw).unwrap(); + assert_eq!(status.freshness(), "stale"); + let bare: ScannerStatus = serde_json::from_value(json!({"enabled": false})).unwrap(); + assert_eq!(bare.freshness(), "unknown"); + } + + #[test] + fn invalid_endpoint_is_rejected_without_io() { + let err = AdminClient::new("not a url", "ak", "sk").unwrap_err(); + assert!(matches!(err, AdminClientError::InvalidEndpoint(_))); + } + + #[tokio::test] + async fn signed_requests_carry_sigv4_authorization_and_correct_target() { + let server = TestServer::spawn(r#"{"clientToken":"token-1","clientAddress":"127.0.0.1:9","startTime":"t"}"#, 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "minioadmin", "minioadmin") + .expect("client builds against the test server"); + + let start: HealStartSuccess = client + .heal_start( + Some("bucket"), + None, + &HealOpts { + recursive: true, + ..Default::default() + }, + false, + ) + .await + .expect("signed heal start decodes"); + + assert_eq!(start.client_token, "token-1"); + let request = server.recorded(); + assert_eq!(request.method, "POST"); + assert_eq!(request.path, "/rustfs/admin/v3/heal/bucket"); + assert!(!request.query.contains("forceStart"), "absent flags must not be sent"); + let auth = request.header("authorization").expect("request must be signed"); + assert!(auth.starts_with("AWS4-HMAC-SHA256"), "SigV4 scheme, got: {auth}"); + assert!(auth.contains("Credential=minioadmin/"), "credentials must be in the Authorization header"); + assert_eq!( + request.header("x-amz-content-sha256").as_deref(), + Some("UNSIGNED-PAYLOAD"), + "the client signs the same payload marker RustFS peer calls use" + ); + assert_eq!(request.header("content-type").as_deref(), Some("application/json")); + assert!(request.body.contains("\"recursive\":true")); + } + + #[tokio::test] + async fn query_sends_client_token_on_the_same_path() { + let body = r#"{"summary":"running","detail":"","settings":{"recursive":false},"items":[],"truncated":false}"#; + let server = TestServer::spawn(body, 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + + let status = client + .heal_status(Some("bucket"), None, "token-1") + .await + .expect("status decodes"); + assert_eq!(status.summary, "running"); + let request = server.recorded(); + assert_eq!(request.path, "/rustfs/admin/v3/heal/bucket"); + assert!(request.query.contains("clientToken=token-1")); + assert!(!request.query.contains("forceStop")); + } + + #[tokio::test] + async fn stop_without_token_takes_the_path_cancel_branch() { + let server = TestServer::spawn(r#"{"clientToken":"path","clientAddress":"c","startTime":"t"}"#, 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + + let outcome = client.heal_stop(Some("bucket"), None, None).await.expect("path stop decodes"); + assert!(matches!(outcome, super::HealStopOutcome::PathStopped(_))); + let request = server.recorded(); + assert!(request.query.contains("forceStop=true")); + assert!(!request.query.contains("clientToken")); + } + + #[tokio::test] + async fn http_error_status_maps_to_a_typed_error_with_body() { + let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + let err = client.scanner_status().await.unwrap_err(); + match err { + AdminClientError::HttpStatus { status, body } => { + assert_eq!(status, 403); + assert!(body.contains("AccessDenied")); + } + other => panic!("expected HttpStatus, got {other:?}"), + } + } + + #[tokio::test] + async fn malformed_success_body_maps_to_a_decode_error() { + let server = TestServer::spawn("not json", 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + assert!(matches!(client.scanner_status().await.unwrap_err(), AdminClientError::Decode { .. })); + } + + /// One recorded request, parsed off the wire with the minimum needed for + /// assertions: method, path, query, headers, body. + #[derive(Debug, Clone)] + struct RecordedRequest { + method: String, + path: String, + query: String, + headers: Vec<(String, String)>, + body: String, + } + + impl RecordedRequest { + fn header(&self, name: &str) -> Option { + self.headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.clone()) + } + } + + /// Minimal HTTP/1.1 server: one canned response per connection, every + /// request recorded behind an `Arc`. Deliberately dependency-free — + /// the assertions only need the raw request bytes. + struct TestServer { + addr: std::net::SocketAddr, + requests: Arc>>, + } + + impl TestServer { + async fn spawn(response_body: &'static str, status: u16) -> Self { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral port"); + let addr = listener.local_addr().expect("local addr"); + let requests: Arc>> = Arc::new(Mutex::new(Vec::new())); + + let recorded = requests.clone(); + tokio::spawn(async move { + let reason = if status == 200 { "OK" } else { "Forbidden" }; + let response = format!( + "HTTP/1.1 {status} {reason}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{response_body}", + response_body.len() + ); + // Each request is a fresh connection (connection: close); a + // bounded loop serves every call a test makes while letting + // the task exit instead of lingering for the whole process. + for _ in 0..16 { + let Ok((mut stream, _)) = listener.accept().await else { + break; + }; + let mut buffer = Vec::with_capacity(2048); + let mut chunk = [0u8; 2048]; + // Read headers plus content-length body, or stop on close. + loop { + if let Some(end) = find_header_end(&buffer) { + let content_length = extract_content_length(&buffer[..end]); + if buffer.len() >= end + content_length { + break; + } + } + let n = match stream.read(&mut chunk).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + buffer.extend_from_slice(&chunk[..n]); + if buffer.len() > 64 * 1024 { + break; + } + } + if let Some(request) = parse_request(&buffer) { + recorded.lock().expect("recorded lock").push(request); + } + let _ = stream.write_all(response.as_bytes()).await; + let _ = stream.shutdown().await; + } + }); + + Self { addr, requests } + } + + fn recorded(&self) -> RecordedRequest { + self.requests + .lock() + .expect("recorded lock") + .last() + .cloned() + .expect("the client call must have produced one recorded request") + } + } + + fn find_header_end(buffer: &[u8]) -> Option { + buffer.windows(4).position(|window| window == b"\r\n\r\n").map(|pos| pos + 4) + } + + fn extract_content_length(headers: &[u8]) -> usize { + let text = String::from_utf8_lossy(headers).to_ascii_lowercase(); + text.lines() + .find_map(|line| line.strip_prefix("content-length:")) + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(0) + } + + fn parse_request(raw: &[u8]) -> Option { + let end = find_header_end(raw)?; + let head = String::from_utf8_lossy(&raw[..end]); + let body = String::from_utf8_lossy(&raw[end..]).into_owned(); + let mut lines = head.lines(); + let request_line = lines.next()?; + let mut parts = request_line.split_whitespace(); + let method = parts.next()?.to_string(); + let target = parts.next()?.to_string(); + let (path, query) = match target.split_once('?') { + Some((path, query)) => (path.to_string(), query.to_string()), + None => (target, String::new()), + }; + let headers = lines + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.trim().to_string(), value.trim().to_string())) + .collect(); + Some(RecordedRequest { + method, + path, + query, + headers, + body, + }) + } +} diff --git a/crates/madmin/src/lib.rs b/crates/madmin/src/lib.rs index 154663b11..a9d4bd8b9 100644 --- a/crates/madmin/src/lib.rs +++ b/crates/madmin/src/lib.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod client; pub mod group; pub mod heal_commands; pub mod health; @@ -25,6 +26,7 @@ pub mod trace; pub mod user; pub mod utils; +pub use client::*; pub use group::*; pub use info_commands::*; pub use policy::*; From 984c7057132936a4772d29c1e209903cca3db85a Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 16:24:54 +0800 Subject: [PATCH 08/51] docs(ecstore): fix bitrot comment typo (#6168) Co-authored-by: heihutu --- _typos.toml | 1 + crates/ecstore/src/erasure/coding/bitrot.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/_typos.toml b/_typos.toml index 12d50d60e..7ff843137 100644 --- a/_typos.toml +++ b/_typos.toml @@ -40,6 +40,7 @@ mak = "mak" gae = "gae" GAE = "GAE" thr = "thr" +mis = "mis" # s3-tests original test names (cannot be changed) nonexisted = "nonexisted" consts = "consts" diff --git a/crates/ecstore/src/erasure/coding/bitrot.rs b/crates/ecstore/src/erasure/coding/bitrot.rs index 698b1b02c..947bbb68e 100644 --- a/crates/ecstore/src/erasure/coding/bitrot.rs +++ b/crates/ecstore/src/erasure/coding/bitrot.rs @@ -835,7 +835,7 @@ pub const BITROT_SELF_TEST_PAYLOAD_LEN: usize = 4096; /// Known-answer digest of [`bitrot_self_test_payload`] under `HighwayHash256S` /// (the production default). Pinned so any platform or build where the -/// implementation drifts fails startup instead of mis-hashing shards. +/// implementation drifts fails startup instead of miss-hashing shards. const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S: [u8; 32] = [ 0xb9, 0x32, 0xa2, 0xaa, 0x4a, 0xb7, 0x33, 0x6a, 0xa3, 0xca, 0x7e, 0x61, 0x9d, 0x86, 0x52, 0x14, 0x6e, 0x7f, 0xd8, 0x9e, 0xea, 0x08, 0xd9, 0x8c, 0x33, 0x85, 0x87, 0x19, 0x30, 0xd6, 0xed, 0x06, From e0b87b0e7e48d5db66358dde292a80498df6e6ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Mon, 17 Aug 2026 17:47:36 +0800 Subject: [PATCH 09/51] fix(site-replication): admit only verifiable peer-edit fences (#6123) --- rustfs/src/admin/handlers/site_replication.rs | 219 +++++++++++++++++- 1 file changed, 214 insertions(+), 5 deletions(-) diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index fba311ba9..18eef001b 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -6023,7 +6023,10 @@ fn edit_generation_wall_clock() -> u64 { /// node's clock behind the clock that fed the previous lifetime) mints /// below the stale mark and the origin stays fenced — but only until real /// time passes the previous lifetime's last allocation, because every later -/// allocation takes the wall-clock floor again. Bounded by the skew, +/// allocation takes the wall-clock floor again (and never longer than +/// [`PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS`]: a regression past the window +/// leaves the mark implausibly distant and the origin runs unfenced +/// immediately). Bounded by the skew, /// self-healing, and no rollback window beyond the plain counter's: a /// delivery applies only at or above the receiver's mark, so the one /// cross-lifetime interleaving that can apply stale content — a @@ -6063,6 +6066,52 @@ fn peer_edit_fence(queries: &HashMap) -> Option<(String, u64)> { Some((origin.clone(), generation)) } +/// How far below the recorded high-water mark a delivery may sit and still +/// be fenced as stale. The distance a GENUINE superseded delivery can trail +/// its origin's mark is small: retransmissions re-run the sender flow and +/// mint a fresh generation (the retry queue keys on the bare path and never +/// replays a fenced URL), so only an in-flight straggler of the losing +/// fan-out race trails the mark, by delivery latency — minutes at the +/// outside. A mark further above than this window cannot be explained by +/// any genuine race, only by a forged fence (the shared service account +/// lets any peer stamp any origin) or by a persisted clock excursion the +/// origin has since left behind — and fencing on it would silently drop the +/// origin's real edits, so the stale check ignores it instead. +const PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS: u64 = 24 * 60 * 60 * 1_000_000_000; + +/// Whether an incoming fence may be honoured, as far as this site can vouch +/// for it. The sender's identity is unverifiable (shared service account), +/// so the check runs over what the receiving state knows: the claimed origin +/// must be a site this state currently replicates with — the same membership +/// rule the load-time mark pruning applies, so every mark recorded behind +/// this check is one a reload would keep — and not this site itself, which +/// never delivers edits to itself. The caller IGNORES an inadmissible fence +/// rather than failing the request: the delivery applies exactly as an +/// unstamped (pre-fence) delivery would, no high-water mark is read or +/// written, and the worst a forged fence achieves is forfeiting an ordering +/// guarantee its sender was never owed. The generation itself is NOT +/// bounded here: a genuine origin whose hybrid clock persisted a wall-clock +/// excursion allocates arbitrarily far in the future, and refusing to +/// record its marks would strip the ordering fence from exactly the +/// deliveries that still race — the staleness window on the read side is +/// what defuses forged marks instead. +fn peer_edit_fence_is_admissible(state: &SiteReplicationState, local_deployment_id: &str, fence: &(String, u64)) -> bool { + let (origin, generation) = fence; + if origin != local_deployment_id && state.peers.contains_key(origin) { + return true; + } + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "fence_origin_not_a_remote_peer", + origin = %origin, + generation = *generation, + "ignoring inadmissible peer-edit fence" + ); + false +} + /// True when a strictly newer edit from the same origin site already landed /// here. No lock on the sending side can order deliveries issued by two /// nodes of that site, so ordering is decided here, on the generation the @@ -6070,11 +6119,42 @@ fn peer_edit_fence(queries: &HashMap) -> Option<(String, u64)> { /// stale: one edit legitimately fans out several deliveries under a single /// generation (the ILM-expiry edit sends every peer's record), and a replay of /// an applied delivery re-applies the same edit idempotently. +/// +/// A mark more than [`PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS`] above the +/// delivery is implausible and does NOT fence: the shared service account +/// means any peer can stamp any origin, so a forged `u64::MAX`-scale mark +/// would otherwise silently swallow the origin's genuine edits for good. +/// Bounding the fence by distance instead of by an absolute ceiling keeps +/// ordering intact wherever the origin's clock actually operates — two +/// racing deliveries trail each other by seconds whether the hybrid clock +/// tracks wall time or persists a long-gone excursion far ahead of it — +/// while a mark no genuine race can explain merely downgrades the origin to +/// unfenced (pre-fence) delivery instead of dropping its edits. (One genuine +/// shape does land out here: a plain-counter straggler arriving after its +/// origin's first hybrid-clock edit. It gets the same downgrade — applied +/// unfenced — once, at upgrade time; fencing it instead would silence the +/// mirror case, a hybrid-clock origin downgraded back to the plain counter.) fn peer_edit_delivery_is_stale(state: &SiteReplicationState, origin: &str, generation: u64) -> bool { - state - .applied_edit_generations - .get(origin) - .is_some_and(|applied| *applied > generation) + let Some(applied) = state.applied_edit_generations.get(origin) else { + return false; + }; + if *applied <= generation { + return false; + } + if *applied - generation > PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "fence_mark_beyond_staleness_window", + origin, + generation, + applied_mark = *applied, + "ignoring implausibly distant peer-edit high-water mark" + ); + return false; + } + true } fn record_applied_peer_edit_generation(state: &mut SiteReplicationState, origin: &str, generation: u64) { @@ -10698,6 +10778,11 @@ impl Operation for SRPeerEditHandler { let outcome = update_site_replication_state_when_changed(move |state| { let mut incoming = incoming; let local_peer = local_peer_at_endpoint(commit_endpoint, state); + // The fence is self-reported — the shared service account means + // the sender cannot be identified — so it is honoured only after + // the admissibility check, against the same state it will gate. + let commit_fence = + commit_fence.filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence)); // Ordering fence: the sending site allocates the generation under // its state-object lock, so a delivery that lost the race carries // a generation this site has already passed. Applying it would @@ -13393,6 +13478,15 @@ mod tests { handler_block.contains("record_applied_peer_edit_generation(state, origin, *generation);"), "SRPeerEditHandler must record the applied generation so later stale deliveries are recognised" ); + // Fence hardening: origin and generation are self-reported by a + // caller the shared service account cannot identify, so the handler + // must pass the fence through the admissibility check — against the + // same state the fence gates, i.e. inside the transaction — before + // reading or raising any high-water mark. + assert!( + handler_block.contains(".filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence))"), + "SRPeerEditHandler must admit a fence only through peer_edit_fence_is_admissible inside the state transaction" + ); // P1-15 PR2: both halves of the fence and the edit they fence share // ONE transaction. Checking the fence against a state read outside the // lock would let the check pass on one snapshot and the write land on @@ -14769,6 +14863,121 @@ mod tests { assert!(peer_edit_delivery_is_stale(&state, origin, generation - 1)); } + /// A fence is self-reported: every site authenticates peer traffic with + /// the same site-replicator credential, so a compromised peer can stamp + /// ANY origin with ANY generation. An origin the receiver does not + /// replicate with — or the receiver itself — is ignored and plants no + /// mark; a mark a compromised peer plants for a CURRENT origin cannot + /// silence that origin, because the staleness window refuses to fence on + /// a mark implausibly far above the genuine deliveries. + #[test] + fn forged_peer_edit_fences_cannot_poison_the_high_water_marks() { + let mut state = SiteReplicationState { + peers: BTreeMap::from([ + ( + "site-local".to_string(), + PeerInfo { + deployment_id: "site-local".to_string(), + ..peer("local", "https://local.example:9000") + }, + ), + ( + "site-victim".to_string(), + PeerInfo { + deployment_id: "site-victim".to_string(), + ..peer("victim", "https://victim.example:9000") + }, + ), + ]), + ..Default::default() + }; + // An origin outside the current membership is refused outright... + let unknown = ("site-unknown".to_string(), 4u64); + assert!(!peer_edit_fence_is_admissible(&state, "site-local", &unknown)); + + // No site delivers edits to itself: a fence claiming the receiver as + // its origin is forged by construction, current peer or not. + let own = ("site-local".to_string(), 4u64); + assert!(!peer_edit_fence_is_admissible(&state, "site-local", &own)); + + // A current remote peer's fence is admitted and works end to end. + let genuine = ("site-victim".to_string(), 1u64); + assert!(peer_edit_fence_is_admissible(&state, "site-local", &genuine)); + assert!(!peer_edit_delivery_is_stale(&state, &genuine.0, genuine.1)); + record_applied_peer_edit_generation(&mut state, &genuine.0, genuine.1); + assert_eq!(state.applied_edit_generations.get("site-victim"), Some(&1)); + + // A forged u64::MAX-scale mark CAN be recorded — the shared service + // account means the receiver cannot tell the stamp was forged — but + // it is inert: the victim's genuine hybrid-clock deliveries sit far + // more than the staleness window below it, so they keep applying + // instead of being silently acked-and-dropped. + record_applied_peer_edit_generation(&mut state, "site-victim", u64::MAX); + assert!(!peer_edit_delivery_is_stale(&state, "site-victim", edit_generation_wall_clock())); + } + + /// The staleness window bounds the fence by DISTANCE from the mark, not + /// by an absolute clock ceiling, so ordering must hold wherever the + /// origin's hybrid clock actually operates. The regression that matters: + /// a temporary wall-clock excursion far in the future is persisted by + /// `next_peer_edit_generation` (`max(now, prev + 1)` never comes back + /// down), and two later edits g+1 then g can arrive in reverse order — + /// g must still be fenced, even though both generations dwarf the + /// receiver's clock. Conversely a mark further above a delivery than any + /// genuine race can explain must not fence it. + #[test] + fn peer_edit_fence_orders_a_persisted_future_clock_and_defuses_distant_marks() { + let mut state = SiteReplicationState { + peers: BTreeMap::from([( + "site-origin".to_string(), + PeerInfo { + deployment_id: "site-origin".to_string(), + ..peer("origin", "https://origin.example:9000") + }, + )]), + ..Default::default() + }; + + // The origin's clock once jumped ten years ahead; the hybrid clock + // keeps allocating from there long after the clock was corrected. + let excursion = edit_generation_wall_clock() + 10 * 365 * 24 * 60 * 60 * 1_000_000_000; + let fence = ("site-origin".to_string(), excursion + 1); + assert!(peer_edit_fence_is_admissible(&state, "site-local", &fence)); + record_applied_peer_edit_generation(&mut state, &fence.0, fence.1); + + // The reverse delivery of the race: g arrives after g+1 landed. + // Without the fence it would commit last and roll g+1 back. + assert!(peer_edit_delivery_is_stale(&state, "site-origin", excursion)); + // Equal generation (same edit's fan-out or a replay) still applies, + // as does the next edit. + assert!(!peer_edit_delivery_is_stale(&state, "site-origin", excursion + 1)); + assert!(!peer_edit_delivery_is_stale(&state, "site-origin", excursion + 2)); + + // The window's exact boundary: a delivery trailing the mark by the + // full window is still fenced; one nanosecond further is not — that + // distance is no longer explicable by a genuine race, only by a + // forged mark or an excursion the origin has left behind. + let mark = fence.1; + // A straggler trailing by a concrete hour must still be fenced — + // pins the window's real magnitude, not just its symbolic boundary. + assert!(peer_edit_delivery_is_stale(&state, "site-origin", mark - 60 * 60 * 1_000_000_000)); + assert!(peer_edit_delivery_is_stale( + &state, + "site-origin", + mark - PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS + )); + assert!(!peer_edit_delivery_is_stale( + &state, + "site-origin", + mark - PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS - 1 + )); + + // A pre-hybrid plain-counter origin trails such a mark by eons: it + // is not fenced (the rc.2-era downgrade case), it just runs + // unfenced until its counter regime catches up. + assert!(!peer_edit_delivery_is_stale(&state, "site-origin", 3)); + } + /// P1-15 review follow-up: a site that leaves the mesh drops below two /// peers, which clears its state object and restarts its generation /// counter at zero. A mark left over from its previous membership would From 59b7d13095e780fccbff829a19770a713f593884 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 19:40:56 +0800 Subject: [PATCH 10/51] feat(scanner): expose prefix-level bucket usage via admin API (HS-08) (#6171) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feat(scanner): expose prefix-level bucket usage via admin API The scanner's per-bucket, per-set usage caches already hold a path-keyed prefix tree, but dui() flattened it only to bucket names — consoles and operators had no way to ask "what does this prefix hold" without an S3 listing sweep (rustfs/backlog#1872, MinIO loadPrefixUsageFromBackend parity). Add: - data-usage: prefix_usage_in_cache — a shared aggregation over the entry map (arbitrary prefix, full counters, one-level sub-prefix breakdown with names recovered from the literal-path cache keys), hardened like the scanner's checked flatten: cycles, dangling child links, over-deep trees, and overflowing counters yield None rather than unbounded recursion or wrapped totals. - ecstore: ECStore::all_set_disks — iterate every erasure set so a query can read each set's own cache copy; the hash-routed store path would always land on one set. - scanner: bucket_prefix_usage — per-set loads (5s budget each, a slow set degrades to not-reporting instead of stalling the caller), merged across sets with partial/compacted/truncated flags, served from a bounded 30s cache (128 entries, hard-capped) that bucket writes invalidate through the dirty-usage hook. - admin: GET /rustfs/admin/v3/usage/{bucket}?prefix=&max-entries= behind the same any-of gate as datausageinfo (DataUsageInfoAdminAction OR ListBucketAction), rejecting unknown query parameters and clamping max-entries to 1..=10000. Route registered in the policy table (deferred MultipleActions, matching datausageinfo) and the route matrix test. Closes rustfs/backlog#1872. Co-authored-by: heihutu --- crates/data-usage/src/data_usage.rs | 286 ++++++++++++++++ crates/ecstore/src/store/mod.rs | 10 + crates/scanner/src/data_usage_define.rs | 10 +- crates/scanner/src/lib.rs | 2 + crates/scanner/src/prefix_usage.rs | 349 ++++++++++++++++++++ crates/scanner/src/scanner_io.rs | 4 + rustfs/src/admin/handlers/mod.rs | 1 + rustfs/src/admin/handlers/system.rs | 8 +- rustfs/src/admin/handlers/usage_prefix.rs | 142 ++++++++ rustfs/src/admin/mod.rs | 4 +- rustfs/src/admin/route_policy.rs | 5 + rustfs/src/admin/route_registration_test.rs | 1 + 12 files changed, 816 insertions(+), 6 deletions(-) create mode 100644 crates/scanner/src/prefix_usage.rs create mode 100644 rustfs/src/admin/handlers/usage_prefix.rs diff --git a/crates/data-usage/src/data_usage.rs b/crates/data-usage/src/data_usage.rs index ebbd03261..b08281d0f 100644 --- a/crates/data-usage/src/data_usage.rs +++ b/crates/data-usage/src/data_usage.rs @@ -870,6 +870,157 @@ pub struct DataUsageCacheInfo { pub snapshot_complete: bool, } +/// Prefix-level usage over a raw entry map — the shared core behind +/// [`DataUsageCache::prefix_usage`], usable by any cache-shaped reader (the +/// scanner's writer-side cache has the same map type). +/// +/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix +/// names come straight off the child keys — no reverse mapping exists or is +/// needed. A compacted prefix carries its aggregate but no children, which +/// the `compacted` flag reports so callers can say why the breakdown is +/// empty. `truncated` is set when the breakdown exceeded `max_entries` and +/// was cut (largest first). +pub fn prefix_usage_in_cache( + cache: &HashMap, + bucket: &str, + prefix: &str, + max_entries: usize, +) -> Option { + let prefix = prefix.trim_matches('/'); + let root = if prefix.is_empty() { + bucket.to_string() + } else { + format!("{bucket}/{prefix}") + }; + let entry = cache.get(&hash_path(&root).key())?.clone(); + + let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?); + + let child_prefix = format!("{root}/"); + let mut sub_prefixes: Vec = entry + .children + .iter() + .filter_map(|child_key| { + let child = cache.get(child_key)?; + let child_flat = flatten_entry(cache, child, 1)?; + // Child keys are literal `bucket/pre/name` paths; a trailing + // slash marks a directory object and is display-only here. + let name = child_key + .strip_prefix(child_prefix.as_str()) + .unwrap_or(child_key.as_str()) + .trim_end_matches('/') + .to_string(); + Some(PrefixUsageEntry { + prefix: name, + usage: PrefixUsageSummary::from_entry(&child_flat), + }) + }) + .collect(); + sub_prefixes.sort_by(|left, right| { + right + .usage + .size + .cmp(&left.usage.size) + .then_with(|| left.prefix.cmp(&right.prefix)) + }); + let truncated = sub_prefixes.len() > max_entries; + sub_prefixes.truncate(max_entries); + + Some(PrefixUsageQuery { + usage, + compacted: entry.compacted, + truncated, + sub_prefixes, + }) +} + +/// Maximum subtree depth [`flatten_entry`] will walk before declaring the +/// cache corrupt — the same bound the scanner's checked flatten uses. +const PREFIX_USAGE_MAX_DEPTH: usize = 1024; + +/// Flatten one entry's subtree into an aggregate: the free-function twin of +/// [`DataUsageCache::flatten`], carrying the scanner checked-flatten +/// hardening so a corrupt cache (cycles, over-deep trees, overflowing +/// counters) yields `None` instead of unbounded recursion or wrapped totals. +fn flatten_entry(cache: &HashMap, root: &DataUsageEntry, depth: usize) -> Option { + if depth > PREFIX_USAGE_MAX_DEPTH { + return None; + } + let mut flattened = DataUsageEntry::default(); + if !flattened.checked_merge(root) { + return None; + } + flattened.compacted = root.compacted; + // The root itself is not pre-seeded: it is merged above, and a corrupt + // child edge pointing back at the root's own key is still terminated by + // the visited set on first encounter. + let mut visited: HashSet<&str> = HashSet::new(); + let mut pending: Vec<(&String, usize)> = root.children.iter().map(|child| (child, depth + 1)).collect(); + while let Some((key, child_depth)) = pending.pop() { + if child_depth > PREFIX_USAGE_MAX_DEPTH || !visited.insert(key.as_str()) { + return None; + } + let entry = cache.get(key)?; + if !flattened.checked_merge(entry) { + return None; + } + pending.extend(entry.children.iter().map(|child| (child, child_depth + 1))); + } + flattened.children.clear(); + Some(flattened) +} + +/// Flattened counters of one prefix subtree, as returned by +/// [`DataUsageCache::prefix_usage`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PrefixUsageSummary { + pub size: u64, + pub objects: u64, + pub versions: u64, + pub delete_markers: u64, +} + +impl PrefixUsageSummary { + fn from_entry(entry: &DataUsageEntry) -> Self { + Self { + size: entry.size as u64, + objects: entry.objects as u64, + versions: entry.versions as u64, + delete_markers: entry.delete_markers as u64, + } + } + + /// Add another set's counters into this one (entries are partitioned by + /// set, so per-set results sum). + pub fn merge(&mut self, other: &Self) { + self.size = self.size.saturating_add(other.size); + self.objects = self.objects.saturating_add(other.objects); + self.versions = self.versions.saturating_add(other.versions); + self.delete_markers = self.delete_markers.saturating_add(other.delete_markers); + } +} + +/// One first-level sub-prefix row of a [`PrefixUsageQuery`]. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] +pub struct PrefixUsageEntry { + pub prefix: String, + pub usage: PrefixUsageSummary, +} + +/// Result of [`DataUsageCache::prefix_usage`]. +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PrefixUsageQuery { + pub usage: PrefixUsageSummary, + /// The prefix entry was compacted by the scanner: its aggregate is valid + /// but no sub-prefix breakdown exists on disk. + pub compacted: bool, + /// The breakdown had more entries than `max_entries`; the largest remain. + pub truncated: bool, + pub sub_prefixes: Vec, +} + /// Read-only projection of a scanner-written `.usage-cache.bin` file. /// /// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`) @@ -997,6 +1148,21 @@ impl DataUsageCache { } } + /// Prefix-level usage for one bucket subtree, plus the one-level + /// breakdown below it (rustfs/backlog#1872, MinIO + /// `loadPrefixUsageFromBackend` parity and beyond: arbitrary prefixes and + /// full counters instead of first-level sizes only). + /// + /// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix + /// names come straight off the child keys — no reverse mapping exists or + /// is needed. A compacted prefix carries its aggregate but no children, + /// which the `compacted` flag reports so callers can say why the + /// breakdown is empty. `truncated` is set when the breakdown exceeded + /// `max_entries` and was cut (largest first). + pub fn prefix_usage(&self, bucket: &str, prefix: &str, max_entries: usize) -> Option { + prefix_usage_in_cache(&self.cache, bucket, prefix, max_entries) + } + pub fn force_compact(&mut self, limit: usize) { if self.cache.len() < limit { return; @@ -1898,6 +2064,126 @@ mod tests { ); } + /// Build a cache shaped like `bucket/{a,b/{c,d}},bucket/loose` with + /// distinct counters so aggregation is observable. + fn prefix_usage_fixture_cache() -> DataUsageCache { + let mut cache = DataUsageCache::default(); + let mut insert = |path: &str, parent: &str, size: usize, objects: usize, versions: usize, delete_markers: usize| { + cache.replace( + path, + parent, + DataUsageEntry { + size, + objects, + versions, + delete_markers, + ..Default::default() + }, + ); + }; + insert("bucket", "", 0, 0, 0, 0); + insert("bucket/a", "bucket", 100, 1, 1, 0); + insert("bucket/b", "bucket", 0, 0, 0, 0); + insert("bucket/b/c", "bucket/b", 200, 2, 2, 1); + insert("bucket/b/d", "bucket/b", 40, 1, 3, 0); + insert("bucket/loose", "bucket", 10, 1, 1, 1); + cache + } + + #[test] + fn prefix_usage_aggregates_bucket_root_and_one_level_below() { + let cache = prefix_usage_fixture_cache(); + + let root = cache + .prefix_usage("bucket", "", 100) + .expect("root query must find the bucket entry"); + assert_eq!(root.usage.size, 350, "root aggregate flattens the whole subtree"); + assert_eq!(root.usage.objects, 5); + assert_eq!(root.usage.versions, 7); + assert_eq!(root.usage.delete_markers, 2); + assert!(!root.compacted); + assert!(!root.truncated); + // Breakdown is one level: b (240) before a (100) before loose (10), + // each flattened to its own subtree total. + let names: Vec<(&str, u64)> = root + .sub_prefixes + .iter() + .map(|entry| (entry.prefix.as_str(), entry.usage.size)) + .collect(); + assert_eq!(names, vec![("b", 240), ("a", 100), ("loose", 10)]); + } + + #[test] + fn prefix_usage_drills_into_arbitrary_prefixes() { + let cache = prefix_usage_fixture_cache(); + + let b = cache.prefix_usage("bucket", "b", 100).expect("nested prefix must resolve"); + assert_eq!(b.usage.size, 240); + assert_eq!(b.usage.versions, 5); + let names: Vec<&str> = b.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect(); + assert_eq!(names, vec!["c", "d"]); + + // Prefix slashes are normalized away. + let slashed = cache.prefix_usage("bucket", "/b/", 100).expect("slash-insensitive lookup"); + assert_eq!(slashed.usage.size, 240); + + assert!(cache.prefix_usage("bucket", "absent", 100).is_none(), "unknown prefix must be a miss"); + assert!(cache.prefix_usage("other", "", 100).is_none(), "unknown bucket must be a miss"); + } + + #[test] + fn prefix_usage_reports_and_respects_truncation() { + let cache = prefix_usage_fixture_cache(); + let capped = cache.prefix_usage("bucket", "", 2).expect("root query"); + assert!(capped.truncated, "three children capped to two must flag truncation"); + let names: Vec<&str> = capped.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect(); + assert_eq!(names, vec!["b", "a"], "largest prefixes survive the cut"); + } + + #[test] + fn prefix_usage_marks_compacted_entries() { + let mut cache = DataUsageCache::default(); + cache.replace( + "bucket", + "", + DataUsageEntry { + size: 999, + objects: 9, + compacted: true, + ..Default::default() + }, + ); + + let compacted = cache.prefix_usage("bucket", "", 100).expect("compacted root resolves"); + assert!(compacted.compacted, "compaction must be visible to callers"); + assert_eq!(compacted.usage.size, 999); + assert!(compacted.sub_prefixes.is_empty(), "a compacted entry carries no children"); + } + + #[test] + fn prefix_usage_rejects_cyclic_and_dangling_caches() { + // A self-referencing child (corrupt cache) must yield a miss for the + // whole query, not unbounded recursion. + let mut cache = prefix_usage_fixture_cache(); + if let Some(entry) = cache.cache.get_mut("bucket/b") { + entry.children.insert("bucket/b".to_string()); + } + assert!(cache.prefix_usage("bucket", "b", 100).is_none(), "a cyclic subtree must be rejected"); + // The unaffected sibling still answers. + assert!(cache.prefix_usage("bucket", "a", 100).is_some()); + + // A child key with no entry (dangling link) is rejected rather than + // silently dropped: half a tree would under-report usage. + let mut dangling = prefix_usage_fixture_cache(); + if let Some(entry) = dangling.cache.get_mut("bucket/b") { + entry.children.insert("bucket/b/ghost".to_string()); + } + assert!( + dangling.prefix_usage("bucket", "b", 100).is_none(), + "a dangling child link must be rejected" + ); + } + #[test] fn hash_path_uses_portable_slash_semantics() { for (input, expected) in [ diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index f51fa6df5..8d02ef441 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -216,6 +216,16 @@ impl std::fmt::Debug for ECStore { /// These delegate to the process-global statics. No local state — the globals /// remain the single source of truth until the migration is complete. impl ECStore { + /// Every erasure set across all pools, pool-major order. + /// + /// Read-only queries that must consult each set's own copy of a + /// per-bucket object (e.g. the scanner's `.usage-cache.bin`) iterate + /// this instead of the hash-routed store path, which would always land + /// on one set (rustfs/backlog#1872). + pub fn all_set_disks(&self) -> Vec> { + self.pools.iter().flat_map(|pool| pool.disk_set.iter().cloned()).collect() + } + /// Get server configuration (delegates to global) pub fn get_server_config(&self) -> Option { runtime_sources::server_config() diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 67a199661..8ac714314 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -28,7 +28,8 @@ use rustfs_common::heal_channel::HealScanMode; use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS; pub use rustfs_data_usage::{ AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME, - DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, TierStats, hash_path, + DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, PrefixUsageEntry, + PrefixUsageQuery, PrefixUsageSummary, TierStats, hash_path, prefix_usage_in_cache, }; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use tokio::time::{Duration, Instant, sleep, timeout}; @@ -430,6 +431,13 @@ pub(crate) enum DataUsageCachePrepareOutcome { } impl DataUsageCache { + /// Prefix-level usage query over this (writer-side) cache; see + /// [`prefix_usage_in_cache`] for the semantics + /// (rustfs/backlog#1872). + pub fn prefix_usage(&self, bucket: &str, prefix: &str, max_entries: usize) -> Option { + prefix_usage_in_cache(&self.cache, bucket, prefix, max_entries) + } + pub(crate) fn prepare_for_scan( &mut self, name: &str, diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index 4a5cc7543..36cc6b817 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -53,6 +53,7 @@ use tokio_util::sync::CancellationToken; pub mod data_usage_define; pub mod error; +pub mod prefix_usage; mod remote_scanner; pub mod runtime_config; pub mod scanner; @@ -64,6 +65,7 @@ pub(crate) mod storage_api; pub use data_usage_define::*; pub use error::ScannerError; +pub use prefix_usage::{BucketPrefixUsageResponse, bucket_prefix_usage, invalidate_prefix_usage_cache}; pub use remote_scanner::{ NS_SCANNER_MAX_REQUEST_BODY_SIZE, RemoteScannerAdmission, RemoteScannerRequest, admit_remote_scanner_request, claim_remote_scanner_request, decode_remote_scanner_request, preflight_remote_scanner_request, diff --git a/crates/scanner/src/prefix_usage.rs b/crates/scanner/src/prefix_usage.rs new file mode 100644 index 000000000..9f94957de --- /dev/null +++ b/crates/scanner/src/prefix_usage.rs @@ -0,0 +1,349 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Prefix-level bucket usage for admin/console consumers (rustfs/backlog#1872, +//! MinIO `loadPrefixUsageFromBackend` parity). +//! +//! The per-bucket, per-set `.usage-cache.bin` objects already hold a +//! path-keyed prefix tree; this module reads every set's copy through that +//! set's own object layer (the hash-routed store path would always land on +//! one set), aggregates the overlapping trees, and serves the result from a +//! bounded 30-second cache. Bucket writes poke the cache through the +//! dirty-usage hook so a fresh scan is visible immediately. + +use crate::data_usage_define::{DATA_USAGE_CACHE_NAME, DataUsageCache}; +use crate::error::ScannerError; +use crate::storage_api::owner::{ + EcstoreSetDisks, EcstoreStore, ecstore_is_reserved_or_invalid_bucket, ecstore_resolve_object_store_handle, +}; +use futures::future::join_all; +use rustfs_data_usage::{PrefixUsageEntry, PrefixUsageSummary}; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime}; +use tracing::{debug, warn}; + +const LOG_COMPONENT_SCANNER: &str = "scanner"; +const LOG_SUBSYSTEM_PREFIX_USAGE: &str = "prefix_usage"; +const EVENT_PREFIX_USAGE_CACHE_STATE: &str = "prefix_usage_cache_state"; + +/// How long a computed breakdown stays fresh. MinIO uses the same 30s for +/// its prefix-usage cache; bucket writes additionally invalidate on the spot. +const CACHE_TTL: Duration = Duration::from_secs(30); +/// Hard entry cap for the result cache; exceeded, expired entries go first +/// and the map clears rather than growing past the bound. +const CACHE_MAX_ENTRIES: usize = 128; +/// Per-set cache read budget. The underlying loader retries for up to a +/// minute per attempt on backend errors — far too long for an admin GET, so +/// a slow set degrades to "not reporting" instead of stalling the caller. +const PER_SET_LOAD_TIMEOUT: Duration = Duration::from_secs(5); + +/// Aggregated prefix-usage answer across every erasure set. +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BucketPrefixUsageResponse { + pub bucket: String, + pub prefix: String, + pub usage: PrefixUsageSummary, + /// Every reporting set's prefix entry was compacted: the aggregate is + /// valid, the sub-prefix breakdown is empty on disk. + pub compacted: bool, + /// The sub-prefix breakdown is incomplete: at least one reporting set + /// had the prefix compacted (or absent while others found it), so its + /// objects cannot be attributed to a sub-prefix. + pub sub_prefixes_partial: bool, + /// The breakdown exceeded the caller's entry limit; largest remain. + pub truncated: bool, + pub sub_prefixes: Vec, + /// Sets whose cache held this bucket and prefix. + pub sets_reporting: usize, + pub sets_total: usize, + /// Newest `last_update` across reporting sets, unix seconds. + pub last_update_unix_secs: Option, +} + +#[derive(Clone)] +struct CachedResponse { + computed_at: std::time::Instant, + response: Arc, +} + +/// Cache key: (lowercased bucket, normalized prefix, max entries). +type PrefixUsageCacheKey = (String, String, usize); +type PrefixUsageCacheMap = Option>; + +static PREFIX_USAGE_CACHE: Mutex = Mutex::new(None); + +/// Drop cached results for `bucket` (empty string clears everything). Wired +/// into the dirty-usage recording path so a write makes the next prefix +/// query recompute instead of serving up to `CACHE_TTL` seconds of stale +/// numbers. +pub fn invalidate_prefix_usage_cache(bucket: &str) { + let mut guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner()); + let Some(map) = guard.as_mut() else { + return; + }; + if bucket.is_empty() { + map.clear(); + return; + } + map.retain(|(cached_bucket, ..), _| !cached_bucket.eq_ignore_ascii_case(bucket)); +} + +/// Query prefix usage for `bucket` (arbitrary `prefix`, empty = whole +/// bucket), merging every erasure set's own cache copy. `max_entries` bounds +/// the sub-prefix rows (largest first). +pub async fn bucket_prefix_usage( + bucket: &str, + prefix: &str, + max_entries: usize, +) -> Result { + if ecstore_is_reserved_or_invalid_bucket(bucket, true) { + return Err(ScannerError::Other(format!("invalid bucket name: {bucket}"))); + } + let normalized_prefix = prefix.trim_matches('/').to_string(); + let cache_key = (bucket.to_ascii_lowercase(), normalized_prefix.clone(), max_entries); + if let Some(response) = lookup_cached(&cache_key) { + return Ok((*response).clone()); + } + + let store = ecstore_resolve_object_store_handle() + .ok_or_else(|| ScannerError::Other("object store is not initialized".to_string()))?; + let response = Arc::new(compute_prefix_usage(store, bucket, &normalized_prefix, max_entries).await); + store_cached(cache_key, response.clone()); + Ok((*response).clone()) +} + +async fn compute_prefix_usage( + store: Arc, + bucket: &str, + prefix: &str, + max_entries: usize, +) -> BucketPrefixUsageResponse { + let sets: Vec> = store.all_set_disks(); + let sets_total = sets.len(); + let cache_name = format!("{bucket}/{DATA_USAGE_CACHE_NAME}"); + + let per_set = join_all(sets.into_iter().map(|set| { + let cache_name = cache_name.clone(); + async move { + let mut cache = DataUsageCache::default(); + // A set that has never scanned this bucket (or cannot be read + // within the budget) reports nothing — the remaining sets still + // produce a usable, flagged answer. + let loaded = match tokio::time::timeout(PER_SET_LOAD_TIMEOUT, cache.load(set, &cache_name)).await { + Ok(Ok(())) => cache, + Ok(Err(err)) => { + debug!( + target: "rustfs::scanner::prefix_usage", + event = EVENT_PREFIX_USAGE_CACHE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_PREFIX_USAGE, + bucket = %bucket, + state = "set_load_failed", + error = %err, + "Prefix usage set cache load failed" + ); + return None; + } + Err(_) => { + warn!( + target: "rustfs::scanner::prefix_usage", + event = EVENT_PREFIX_USAGE_CACHE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_PREFIX_USAGE, + bucket = %bucket, + state = "set_load_timeout", + "Prefix usage set cache load timed out" + ); + return None; + } + }; + if loaded.info.name != bucket { + // Empty or stale-scoped cache: this set has no data for the bucket. + return None; + } + let last_update = loaded.info.last_update; + let query = loaded.prefix_usage(bucket, prefix, max_entries); + Some((query, last_update)) + } + })) + .await; + + let mut usage = PrefixUsageSummary::default(); + let mut sub_prefix_map: HashMap = HashMap::new(); + let mut sets_reporting = 0usize; + let mut reporting_but_absent = 0usize; + let mut any_compacted = false; + let mut all_compacted = true; + let mut truncated = false; + let mut last_update: Option = None; + + for (query, set_last_update) in per_set.into_iter().flatten() { + // last_update counts every set that has scanned the bucket, even + // when the prefix itself is absent on that set. + if let Some(set_last_update) = set_last_update + && last_update.map(|current| set_last_update > current).unwrap_or(true) + { + last_update = Some(set_last_update); + } + let Some(query) = query else { + // The set knows the bucket but not this prefix: legitimate when + // the prefix's objects all hash to other sets, but it means the + // breakdown below cannot attribute that set's (zero) objects. + reporting_but_absent += 1; + continue; + }; + sets_reporting += 1; + usage.merge(&query.usage); + if query.compacted { + any_compacted = true; + } else { + all_compacted = false; + } + truncated |= query.truncated; + for entry in query.sub_prefixes { + sub_prefix_map.entry(entry.prefix).or_default().merge(&entry.usage); + } + } + + let mut sub_prefixes: Vec = sub_prefix_map + .into_iter() + .map(|(prefix, usage)| PrefixUsageEntry { prefix, usage }) + .collect(); + sub_prefixes.sort_by(|left, right| { + right + .usage + .size + .cmp(&left.usage.size) + .then_with(|| left.prefix.cmp(&right.prefix)) + }); + // Merged rows can exceed max_entries only when per-set truncation + // already flagged; enforce the caller bound on the merged view too. + if sub_prefixes.len() > max_entries { + truncated = true; + sub_prefixes.truncate(max_entries); + } + + let found = sets_reporting > 0; + BucketPrefixUsageResponse { + bucket: bucket.to_string(), + prefix: prefix.to_string(), + usage, + compacted: found && all_compacted, + sub_prefixes_partial: any_compacted || reporting_but_absent > 0, + truncated, + sub_prefixes, + sets_reporting, + sets_total, + last_update_unix_secs: last_update + .and_then(|time| time.duration_since(SystemTime::UNIX_EPOCH).ok()) + .map(|dur| dur.as_secs()), + } +} + +fn lookup_cached(key: &(String, String, usize)) -> Option> { + let mut guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner()); + let map = guard.as_mut()?; + let cached = map.get(key)?; + if cached.computed_at.elapsed() > CACHE_TTL { + map.remove(key); + return None; + } + Some(cached.response.clone()) +} + +fn store_cached(key: (String, String, usize), response: Arc) { + let mut guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner()); + let map = guard.get_or_insert_with(HashMap::new); + // Bound the cache: drop expired entries first, and if the cap is still + // exceeded clear wholesale — the next queries recompute in milliseconds. + if map.len() >= CACHE_MAX_ENTRIES { + map.retain(|_, cached| cached.computed_at.elapsed() <= CACHE_TTL); + if map.len() >= CACHE_MAX_ENTRIES { + map.clear(); + } + } + map.insert( + key, + CachedResponse { + computed_at: std::time::Instant::now(), + response, + }, + ); +} + +#[cfg(test)] +mod tests { + use super::{CACHE_MAX_ENTRIES, PREFIX_USAGE_CACHE, invalidate_prefix_usage_cache, store_cached}; + use rustfs_data_usage::PrefixUsageSummary; + + fn response(bucket: &str) -> super::BucketPrefixUsageResponse { + super::BucketPrefixUsageResponse { + bucket: bucket.to_string(), + prefix: String::new(), + usage: PrefixUsageSummary::default(), + compacted: false, + sub_prefixes_partial: false, + truncated: false, + sub_prefixes: Vec::new(), + sets_reporting: 1, + sets_total: 1, + last_update_unix_secs: None, + } + } + + fn seed(bucket: &str, prefix: &str) { + store_cached( + (bucket.to_ascii_lowercase(), prefix.to_string(), 10), + std::sync::Arc::new(response(bucket)), + ); + } + + fn contains(bucket: &str, prefix: &str) -> bool { + PREFIX_USAGE_CACHE + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .as_ref() + .is_some_and(|map| map.contains_key(&(bucket.to_ascii_lowercase(), prefix.to_string(), 10))) + } + + /// All cache tests run inside one test to keep the process-global map + /// free of cross-test ordering (the flake class this module avoids). + #[test] + fn invalidation_scopes_to_bucket_and_cache_stays_bounded() { + invalidate_prefix_usage_cache(""); + seed("alpha", "x"); + seed("beta", "y"); + + // Case-insensitive bucket scoping. + invalidate_prefix_usage_cache("ALPHA"); + assert!(!contains("alpha", "x")); + assert!(contains("beta", "y")); + + // Wholesale clear. + invalidate_prefix_usage_cache(""); + assert!(!contains("beta", "y")); + + // Hard cap: overflow clears rather than grows. + for index in 0..=(CACHE_MAX_ENTRIES / 2) { + let bucket = format!("cap-bucket-{index}"); + seed(&bucket, "a"); + seed(&bucket, "b"); + } + let guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner()); + let map = guard.as_ref().expect("seeded"); + assert!(map.len() <= CACHE_MAX_ENTRIES, "cache must stay bounded, got {}", map.len()); + } +} diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 15ee9cca0..f722ff186 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -231,6 +231,10 @@ pub fn record_dirty_usage_bucket(bucket: &str) { dirty_buckets.len() }; global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets)); + // A write invalidates this bucket's prefix-usage answers on the spot so + // admin/console consumers never ride the full TTL after a change + // (rustfs/backlog#1872). + crate::prefix_usage::invalidate_prefix_usage_cache(bucket); DIRTY_USAGE_BUCKET_NOTIFY.notify_one(); } diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index 8c837eb05..f0a32f402 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -64,6 +64,7 @@ mod target_descriptor; pub mod tier; pub mod tls_debug; pub mod trace; +pub mod usage_prefix; pub mod user; pub mod user_iam; pub mod user_lifecycle; diff --git a/rustfs/src/admin/handlers/system.rs b/rustfs/src/admin/handlers/system.rs index c7ad93b54..0909568e5 100644 --- a/rustfs/src/admin/handlers/system.rs +++ b/rustfs/src/admin/handlers/system.rs @@ -1158,10 +1158,10 @@ impl Operation for RuntimeCapabilitiesHandler { } } -/// Authorization gate for GET datausageinfo: any-of the dedicated admin action -/// OR the bucket listing action. Pinned by a unit test so the gate cannot -/// silently narrow or widen (rustfs/backlog#1306). -fn data_usage_info_gate_actions() -> Vec { +/// Authorization gate for GET datausageinfo (and prefix usage): any-of the +/// dedicated admin action OR the bucket listing action. Pinned by a unit test +/// so the gate cannot silently narrow or widen (rustfs/backlog#1306). +pub(crate) fn data_usage_info_gate_actions() -> Vec { vec![ Action::AdminAction(AdminAction::DataUsageInfoAdminAction), Action::S3Action(S3Action::ListBucketAction), diff --git a/rustfs/src/admin/handlers/usage_prefix.rs b/rustfs/src/admin/handlers/usage_prefix.rs new file mode 100644 index 000000000..10cce50d5 --- /dev/null +++ b/rustfs/src/admin/handlers/usage_prefix.rs @@ -0,0 +1,142 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Prefix-level bucket usage admin handler (rustfs/backlog#1872). +//! +//! `GET /rustfs/admin/v3/usage/{bucket}?prefix=&max-entries=` answers +//! "what does this bucket / this prefix hold" from the scanner's per-set +//! usage caches, with a one-level sub-prefix breakdown — the data console +//! buckets view MinIO serves from `loadPrefixUsageFromBackend`. + +use crate::admin::auth::validate_admin_request; +use crate::admin::handlers::system::data_usage_info_gate_actions; +use crate::admin::router::{AdminOperation, Operation, S3Router}; +use crate::auth::{check_key_valid, get_session_token}; +use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use http::{HeaderMap, HeaderValue, StatusCode}; +use hyper::Method; +use matchit::Params; +use s3s::header::CONTENT_TYPE; +use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; + +const JSON_CONTENT_TYPE: &str = "application/json"; +const DEFAULT_MAX_ENTRIES: usize = 1000; +const MAX_ENTRIES_LIMIT: usize = 10_000; + +pub struct BucketPrefixUsageHandler {} + +pub fn register_usage_prefix_route(r: &mut S3Router) -> std::io::Result<()> { + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/usage/{bucket}").as_str(), + AdminOperation(&BucketPrefixUsageHandler {}), + )?; + Ok(()) +} + +/// Parse `prefix` and `max-entries` from the query string. Unknown keys are +/// rejected so a typo'd parameter cannot silently change the answer's shape. +fn parse_usage_prefix_query(query: Option<&str>) -> S3Result<(String, usize)> { + let mut prefix: Option = None; + let mut max_entries: Option = None; + for (key, value) in url::form_urlencoded::parse(query.unwrap_or_default().as_bytes()) { + match key.as_ref() { + "prefix" => prefix = Some(value.into_owned()), + "max-entries" => { + max_entries = Some( + value + .parse::() + .map_err(|_| s3_error!(InvalidArgument, "max-entries must be a positive integer"))?, + ); + } + other => return Err(s3_error!(InvalidArgument, "unknown query parameter: {other}")), + } + } + let max_entries = max_entries.unwrap_or(DEFAULT_MAX_ENTRIES).clamp(1, MAX_ENTRIES_LIMIT); + Ok((prefix.unwrap_or_default(), max_entries)) +} + +#[async_trait::async_trait] +impl Operation for BucketPrefixUsageHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let remote_addr = req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)); + validate_admin_request(&req.headers, &cred, owner, false, data_usage_info_gate_actions(), remote_addr).await?; + + let bucket = params.get("bucket").unwrap_or_default().to_string(); + if bucket.is_empty() { + return Err(s3_error!(InvalidRequest, "bucket path parameter is required")); + } + let (prefix, max_entries) = parse_usage_prefix_query(req.uri.query())?; + + // Authorization is bucket-scoped by the same any-of gate as the + // datausageinfo route; the bucket name itself is validated by the + // scanner layer, which rejects reserved/invalid names. + let response = rustfs_scanner::bucket_prefix_usage(&bucket, &prefix, max_entries) + .await + .map_err(|err| s3_error!(InvalidArgument, "{}", err))?; + + let data = serde_json::to_vec(&response) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "parse prefix usage failed"))?; + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, HeaderValue::from_static(JSON_CONTENT_TYPE)); + + Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header)) + } +} + +#[cfg(test)] +mod tests { + use super::{DEFAULT_MAX_ENTRIES, MAX_ENTRIES_LIMIT, parse_usage_prefix_query}; + use s3s::S3Error; + + fn query(raw: &str) -> Result<(String, usize), S3Error> { + parse_usage_prefix_query(Some(raw)) + } + + #[test] + fn defaults_apply_when_no_query_is_given() { + assert_eq!(parse_usage_prefix_query(None).unwrap(), (String::new(), DEFAULT_MAX_ENTRIES)); + assert_eq!(query("").unwrap(), (String::new(), DEFAULT_MAX_ENTRIES)); + } + + #[test] + fn prefix_round_trips_url_encoded_characters() { + let (prefix, _) = query("prefix=pre%2Ffix%20name").unwrap(); + assert_eq!(prefix, "pre/fix name"); + } + + #[test] + fn max_entries_parses_and_clamps_to_documented_bounds() { + assert_eq!(query("max-entries=5").unwrap().1, 5); + assert_eq!(query("max-entries=0").unwrap().1, 1, "zero must clamp up, not mean unlimited"); + assert_eq!(query("max-entries=99999999").unwrap().1, MAX_ENTRIES_LIMIT); + assert!(query("max-entries=-3").is_err()); + assert!(query("max-entries=abc").is_err()); + } + + #[test] + fn unknown_parameters_are_rejected_not_ignored() { + assert!( + query("prefixes=x").is_err(), + "a typo'd parameter must fail the request, not widen the query" + ); + } +} diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index d61abf932..fe07d4056 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -40,7 +40,8 @@ use handlers::{ audit, batch_job, bucket_meta, cluster_snapshot, config_admin, diagnostics, durability as durability_handler, extensions, heal, health, idp_compat, ilm_transition, inspect_archive, kms, module_switch, object_data_cache, object_zip_download, oidc, plugins_catalog, plugins_instances, pools, profile_admin, quota as quota_handler, rebalance, - replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, user, + replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, usage_prefix, + user, }; use router::{AdminOperation, S3Router}; use s3s::route::S3Route; @@ -80,6 +81,7 @@ fn register_admin_routes(r: &mut S3Router) -> std::io::Result<() bucket_meta::register_bucket_meta_route(r)?; config_admin::register_config_route(r)?; scanner::register_scanner_route(r)?; + usage_prefix::register_usage_prefix_route(r)?; ilm_transition::register_ilm_transition_route(r)?; object_data_cache::register_object_data_cache_route(r)?; audit::register_audit_target_route(r)?; diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 474d3ad0f..2c1ad5e1b 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -1558,6 +1558,11 @@ pub const DEFERRED_ADMIN_ROUTE_POLICIES: &[DeferredAdminRoutePolicy] = &[ "/rustfs/admin/v3/datausageinfo", DeferredRoutePolicyReason::MultipleActions, ), + deferred( + HttpMethod::Get, + "/rustfs/admin/v3/usage/{bucket}", + DeferredRoutePolicyReason::MultipleActions, + ), deferred( HttpMethod::Post, "/rustfs/admin/v3/object-zip-downloads", diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index f829b8c94..e81f13d17 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -172,6 +172,7 @@ fn expected_admin_route_matrix() -> Vec { admin_route(Method::POST, "/v4/inspect/archive"), admin_route(Method::GET, "/v3/storageinfo"), admin_route(Method::GET, "/v3/datausageinfo"), + admin_route_sample(Method::GET, "/v3/usage/{bucket}", "/v3/usage/test-bucket"), admin_route(Method::GET, "/v3/metrics"), admin_route(Method::GET, "/v3/object-data-cache/stats"), admin_route(Method::POST, "/v3/object-data-cache/flush"), From beb6e1383e9982dac6a1b06f8e7a2af735336094 Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Tue, 18 Aug 2026 01:21:15 +0800 Subject: [PATCH 11/51] feat(helm): add TLSRoute passthrough support for gateway api (#6169) Add an optional TLS passthrough listener to the Gateway API support. When gatewayApi.listeners.tls.enabled is true, the Gateway gets a TLS listener with tls.mode: Passthrough and a TLSRoute is rendered to the RustFS service so TLS terminates at the backend (end-to-end encryption). Refs rustfs/rustfs#3862. --- helm/README.md | 6 +++++ helm/rustfs/templates/gateway-api/gateway.yml | 10 ++++++++ .../rustfs/templates/gateway-api/tlsroute.yml | 25 +++++++++++++++++++ helm/rustfs/values.yaml | 6 +++++ 4 files changed, 47 insertions(+) create mode 100644 helm/rustfs/templates/gateway-api/tlsroute.yml diff --git a/helm/README.md b/helm/README.md index 8bdd8b531..79b7c6fd6 100644 --- a/helm/README.md +++ b/helm/README.md @@ -273,6 +273,10 @@ uer. `ClusterIssuer` or `Issuer`. | | gatewayApi.listeners.http.port| int | `8000` | Gateway API http listener port. | | gatewayApi.listeners.https.name | string | `websecure` | Gateway API https listener name. | | gatewayApi.listeners.https.port| int | `8443` | Gateway API https listener port. | +| gatewayApi.listeners.tls.enabled | bool | `false` | Enable a TLS passthrough listener and generate a TLSRoute. | +| gatewayApi.listeners.tls.name | string | `tls` | Gateway API TLS passthrough listener name. | +| gatewayApi.listeners.tls.port | int | `443` | Gateway API TLS passthrough listener port. | +| gatewayApi.listeners.tls.backendPort | int | `null` | Backend service port that terminates TLS; defaults to the console port. | | gatewayApi.hostname | string | Hostname to access RustFS via gateway api. | | gatewayApi.secretName | string | Secret tls to via RustFS using HTTPS. | | gatewayApi.existingGateway.name | string | `""` | The existing gateway name, instead of creating a new one. | @@ -447,6 +451,8 @@ rustfs-route ["example.rustfs.com"] 172m Then, via RustFS instance via `https://example.rustfs.com` or `http://example.rustfs.com`. +For end-to-end encryption, set `gatewayApi.listeners.tls.enabled` to `true`. The chart then adds a `TLS` listener with `tls.mode: Passthrough` to the `Gateway` and generates a `TLSRoute` that forwards the encrypted stream to the RustFS service, where TLS is terminated on the backend side. Note that backend TLS termination must be configured on RustFS itself (for example `RUSTFS_TLS_PATH` pointing to server certificates), and the installed Gateway API CRDs must include `TLSRoute`. + # Uninstall Uninstalling the rustfs installation with command, diff --git a/helm/rustfs/templates/gateway-api/gateway.yml b/helm/rustfs/templates/gateway-api/gateway.yml index 207637b3e..8de988beb 100644 --- a/helm/rustfs/templates/gateway-api/gateway.yml +++ b/helm/rustfs/templates/gateway-api/gateway.yml @@ -26,5 +26,15 @@ spec: - name: {{ include "rustfs.fullname" $ }}-tls kind: Secret {{- end }} + {{- if .tls.enabled }} + - name: {{ .tls.name }} + port: {{ .tls.port }} + protocol: TLS + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: Same + {{- end }} {{- end }} {{- end }} diff --git a/helm/rustfs/templates/gateway-api/tlsroute.yml b/helm/rustfs/templates/gateway-api/tlsroute.yml new file mode 100644 index 000000000..e35f86c06 --- /dev/null +++ b/helm/rustfs/templates/gateway-api/tlsroute.yml @@ -0,0 +1,25 @@ +{{- if and .Values.gatewayApi.enabled .Values.gatewayApi.listeners.tls.enabled }} +apiVersion: gateway.networking.k8s.io/v1 +kind: TLSRoute +metadata: + name: {{ include "rustfs.fullname" . }}-tlsroute + namespace: {{ .Release.Namespace }} +spec: + parentRefs: + {{- if .Values.gatewayApi.existingGateway.name }} + - name: {{ .Values.gatewayApi.existingGateway.name }} + {{- if .Values.gatewayApi.existingGateway.namespace }} + namespace: {{ .Values.gatewayApi.existingGateway.namespace }} + {{- end }} + sectionName: {{ .Values.gatewayApi.listeners.tls.name }} + {{- else }} + - name: {{ include "rustfs.fullname" $ }}-gateway + sectionName: {{ .Values.gatewayApi.listeners.tls.name }} + {{- end }} + hostnames: + - {{ .Values.gatewayApi.hostname }} + rules: + - backendRefs: + - name: {{ include "rustfs.fullname" . }}-svc + port: {{ .Values.gatewayApi.listeners.tls.backendPort | default .Values.service.console.port }} +{{- end }} diff --git a/helm/rustfs/values.yaml b/helm/rustfs/values.yaml index 19826df96..d57317243 100644 --- a/helm/rustfs/values.yaml +++ b/helm/rustfs/values.yaml @@ -369,6 +369,12 @@ gatewayApi: https: name: websecure port: 8443 + tls: # Optional TLS passthrough listener; renders a TLSRoute so TLS terminates at the RustFS backend. + enabled: false + name: tls + port: 443 + # Service port that terminates TLS on the backend; defaults to the console port. + backendPort: null hostname: example.rustfs.com httpToHttpsRedirect: true existingGateway: From 7cb91a019084be368bbcaa750306628c1a127dce Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 07:51:56 +0800 Subject: [PATCH 12/51] chore(ecstore): adjudicate 32 bare dead_code allows (#6173) Replace every bare `#[allow(dead_code)]` in ecstore with either a deletion or a per-item allow carrying a `reason`. Blanket allows at module, struct, and impl level silence the lint for future members too, so each is narrowed to the members that are actually dead. Delete the dead cluster in `config/heal.rs` (`Config`, its three methods, `RUSTFS_BITROT_CYCLE_IN_MONTHS`, `parse_bitrot_config`) rather than annotate it: it has no callers and is unreachable outside the crate, and `parse_bitrot_config` would panic on its disabled path via `Duration::from_secs_f64(-1.0)`. `DEFAULT_KVS` stays, since the config registry uses it. Correct two `reason` strings on `Checksum::new` and `PutObjReader::md5_current_hex_string`, which are methods but carried a field-only rationale. Refs backlog#1823 Co-authored-by: houseme --- .../ecstore/src/bucket/bucket_target_sys.rs | 4 +- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 9 ++- .../bucket/lifecycle/tier_last_day_stats.rs | 5 +- .../src/bucket/lifecycle/tier_sweeper.rs | 10 +++- crates/ecstore/src/bucket/quota/mod.rs | 2 - crates/ecstore/src/client/api_get_object.rs | 4 +- crates/ecstore/src/client/api_get_options.rs | 1 - crates/ecstore/src/client/api_list.rs | 1 - crates/ecstore/src/client/api_put_object.rs | 4 +- crates/ecstore/src/client/api_remove.rs | 5 +- crates/ecstore/src/client/checksum.rs | 6 +- crates/ecstore/src/client/object_api_utils.rs | 3 +- crates/ecstore/src/config/audit.rs | 3 - crates/ecstore/src/config/heal.rs | 59 ------------------- crates/ecstore/src/config/mod.rs | 1 - crates/ecstore/src/core/pools.rs | 12 +++- crates/ecstore/src/disk/local.rs | 2 +- .../ecstore/src/services/rebalance/types.rs | 1 - .../ecstore/src/services/tier/tier_config.rs | 17 +++++- crates/ecstore/src/services/tier/tier_gen.rs | 1 - 20 files changed, 59 insertions(+), 91 deletions(-) diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index 0a9b0f41c..00cf124f1 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -1549,8 +1549,8 @@ impl Default for PutObjectOptions { } } -#[allow(dead_code)] impl PutObjectOptions { + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn set_match_etag(&mut self, etag: &str) { if etag == "*" { self.custom_header @@ -1561,6 +1561,7 @@ impl PutObjectOptions { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn set_match_etag_except(&mut self, etag: &str) { if etag == "*" { self.custom_header @@ -1696,6 +1697,7 @@ impl PutObjectOptions { header } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn validate(&self, _c: Arc) -> Result<(), std::io::Error> { //if self.checksum.is_set() { /*if !self.trailing_header_support { diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 0366551e0..ec5aaf74e 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -456,16 +456,23 @@ impl<'a> LifecycleExpiryTrace<'a> { } } -#[allow(dead_code)] impl ExpiryStats { pub fn missed_tasks(&self) -> i64 { self.missed_expiry_tasks.load(Ordering::SeqCst) } + #[allow( + dead_code, + reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)" + )] fn missed_free_vers_tasks(&self) -> i64 { self.missed_freevers_tasks.load(Ordering::SeqCst) } + #[allow( + dead_code, + reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)" + )] fn missed_tier_journal_tasks(&self) -> i64 { self.missed_tier_journal_tasks.load(Ordering::SeqCst) } diff --git a/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs b/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs index b32fb32f5..2d8d6c2ea 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs @@ -80,7 +80,10 @@ impl LastDayTierStats { } } - #[allow(dead_code)] + #[allow( + dead_code, + reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)" + )] fn merge(&self, m: LastDayTierStats) -> LastDayTierStats { let mut cl = self.clone(); let mut cm = m; diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs index 2ce68fd03..0bde1a23c 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -177,9 +177,10 @@ fn should_record_remote_delete_failure(err: &std::io::Error) -> bool { } #[derive(Default)] -#[allow(dead_code)] struct ObjSweeper { + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] object: String, + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] bucket: String, version_id: Option, versioned: bool, @@ -191,9 +192,9 @@ struct ObjSweeper { remote_object: String, } -#[allow(dead_code)] impl ObjSweeper { #[allow(clippy::new_ret_no_self)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub async fn new(bucket: &str, object: &str) -> Result { Ok(Self { object: object.into(), @@ -202,17 +203,20 @@ impl ObjSweeper { }) } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn with_version(&mut self, vid: Option) -> &Self { self.version_id = vid.clone(); self } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self { self.versioned = versioned; self.suspended = suspended; self } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn get_opts(&self) -> lifecycle::ObjectOpts { let mut opts = ObjectOpts { version_id: self.version_id.clone(), @@ -226,6 +230,7 @@ impl ObjSweeper { opts } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn set_transition_state(&mut self, info: TransitionedObject) { self.transition_tier = info.tier; self.transition_status = info.status; @@ -266,6 +271,7 @@ impl ObjSweeper { None } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub async fn sweep(&self, api: Arc) { let Some(je) = self.should_remove_remote_object() else { return; diff --git a/crates/ecstore/src/bucket/quota/mod.rs b/crates/ecstore/src/bucket/quota/mod.rs index 0ce5a3a52..3fed86728 100644 --- a/crates/ecstore/src/bucket/quota/mod.rs +++ b/crates/ecstore/src/bucket/quota/mod.rs @@ -312,9 +312,7 @@ mod tests { } #[derive(Deserialize)] struct LegacyBucketQuota { - #[allow(dead_code)] quota: Option, - #[allow(dead_code)] quota_type: LegacyQuotaType, } let legacy = serde_json::from_slice::(&json) diff --git a/crates/ecstore/src/client/api_get_object.rs b/crates/ecstore/src/client/api_get_object.rs index a9c69aad0..1eb6122ca 100644 --- a/crates/ecstore/src/client/api_get_object.rs +++ b/crates/ecstore/src/client/api_get_object.rs @@ -95,7 +95,6 @@ impl TransitionClient { } #[derive(Default)] -#[allow(dead_code)] pub struct GetRequest { pub buffer: Vec, pub offset: i64, @@ -107,11 +106,12 @@ pub struct GetRequest { pub setting_object_info: bool, } -#[allow(dead_code)] pub struct GetResponse { pub size: i64, //pub error: error, + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] pub did_read: bool, + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] pub object_info: ObjectInfo, } diff --git a/crates/ecstore/src/client/api_get_options.rs b/crates/ecstore/src/client/api_get_options.rs index 622c5a4c2..503b44f3a 100644 --- a/crates/ecstore/src/client/api_get_options.rs +++ b/crates/ecstore/src/client/api_get_options.rs @@ -27,7 +27,6 @@ use tracing::warn; use crate::client::api_error_response::err_invalid_argument; #[derive(Default)] -#[allow(dead_code)] pub struct AdvancedGetOptions { pub replication_delete_marker: bool, pub is_replication_ready_for_delete_marker: bool, diff --git a/crates/ecstore/src/client/api_list.rs b/crates/ecstore/src/client/api_list.rs index 6bd8591c3..a7f894a00 100644 --- a/crates/ecstore/src/client/api_list.rs +++ b/crates/ecstore/src/client/api_list.rs @@ -360,7 +360,6 @@ impl TransitionClient { } #[derive(Default)] -#[allow(dead_code)] pub struct ListObjectsOptions { reverse_versions: bool, with_versions: bool, diff --git a/crates/ecstore/src/client/api_put_object.rs b/crates/ecstore/src/client/api_put_object.rs index 5fbc3fd2c..bf9efe4a2 100644 --- a/crates/ecstore/src/client/api_put_object.rs +++ b/crates/ecstore/src/client/api_put_object.rs @@ -137,8 +137,8 @@ impl Default for PutObjectOptions { } } -#[allow(dead_code)] impl PutObjectOptions { + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn set_match_etag(&mut self, etag: &str) { if etag == "*" { self.custom_header.insert("If-Match", HeaderValue::from_static("*")); @@ -149,6 +149,7 @@ impl PutObjectOptions { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn set_match_etag_except(&mut self, etag: &str) { if etag == "*" { self.custom_header.insert("If-None-Match", HeaderValue::from_static("*")); @@ -259,6 +260,7 @@ impl PutObjectOptions { header } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> { //if self.checksum.is_set() { /*if !self.trailing_header_support { diff --git a/crates/ecstore/src/client/api_remove.rs b/crates/ecstore/src/client/api_remove.rs index 25c71d80a..573a2eed1 100644 --- a/crates/ecstore/src/client/api_remove.rs +++ b/crates/ecstore/src/client/api_remove.rs @@ -55,7 +55,6 @@ pub struct RemoveBucketOptions { const DELETE_RESPONSE_PREVIEW_LEN: usize = 1024; #[derive(Debug)] -#[allow(dead_code)] pub struct AdvancedRemoveOptions { pub replication_delete_marker: bool, pub replication_status: ReplicationStatus, @@ -465,10 +464,10 @@ impl TransitionClient { } #[derive(Debug, Default)] -#[allow(dead_code)] pub struct RemoveObjectError { + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] object_name: String, - #[allow(dead_code)] + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] version_id: String, err: Option, } diff --git a/crates/ecstore/src/client/checksum.rs b/crates/ecstore/src/client/checksum.rs index c71394210..7bb96a6a3 100644 --- a/crates/ecstore/src/client/checksum.rs +++ b/crates/ecstore/src/client/checksum.rs @@ -372,8 +372,8 @@ pub struct Checksum { computed: bool, } -#[allow(dead_code)] impl Checksum { + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn new(t: ChecksumMode, b: &[u8]) -> Checksum { if t.is_set() && b.len() == t.raw_byte_len() { return Checksum { @@ -385,7 +385,7 @@ impl Checksum { Checksum::default() } - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn new_checksum_string(t: ChecksumMode, s: &str) -> Result { let b = match base64_decode(s.as_bytes()) { Ok(b) => b, @@ -412,7 +412,7 @@ impl Checksum { base64_encode(&self.r) } - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn raw(&self) -> Option> { if !self.is_set() { return None; diff --git a/crates/ecstore/src/client/object_api_utils.rs b/crates/ecstore/src/client/object_api_utils.rs index 484233fad..b4bd3e0b5 100644 --- a/crates/ecstore/src/client/object_api_utils.rs +++ b/crates/ecstore/src/client/object_api_utils.rs @@ -37,16 +37,17 @@ pub struct PutObjReader { //pub sealMD5Fn: SealMD5CurrFn, } -#[allow(dead_code)] impl PutObjReader { pub fn new(reader: HashReader) -> Self { Self { reader } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn md5_current_hex_string(&self) -> String { self.reader.checksum().map(|v| v.encoded).unwrap_or_default() } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> { self.reader = enc_reader; diff --git a/crates/ecstore/src/config/audit.rs b/crates/ecstore/src/config/audit.rs index b7ed83323..63c63b700 100644 --- a/crates/ecstore/src/config/audit.rs +++ b/crates/ecstore/src/config/audit.rs @@ -39,7 +39,6 @@ use rustfs_config::{ }; use std::sync::LazyLock; -#[allow(dead_code)] #[allow(clippy::declare_interior_mutable_const)] /// Default KVS for audit webhook settings. pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock = LazyLock::new(|| { @@ -117,7 +116,6 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock = LazyLock::new(|| { ]) }); -#[allow(dead_code)] #[allow(clippy::declare_interior_mutable_const)] /// Default KVS for audit MQTT settings. pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock = LazyLock::new(|| { @@ -375,7 +373,6 @@ pub static DEFAULT_AUDIT_NATS_KVS: LazyLock = LazyLock::new(|| { ]) }); -#[allow(dead_code)] pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock = LazyLock::new(|| { KVS(vec![ KV { diff --git a/crates/ecstore/src/config/heal.rs b/crates/ecstore/src/config/heal.rs index 4505d3ce0..8ef41efa1 100644 --- a/crates/ecstore/src/config/heal.rs +++ b/crates/ecstore/src/config/heal.rs @@ -12,12 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::error::{Error, Result}; use rustfs_config::server_config::{KV, KVS}; use rustfs_config::{DEFAULT_HEAL_BITROT_CYCLE_SECS, HEAL_BITROT_CYCLE}; -use rustfs_utils::string::parse_bool; use std::sync::LazyLock; -use std::time::Duration; pub static DEFAULT_KVS: LazyLock = LazyLock::new(|| { KVS(vec![KV { @@ -26,59 +23,3 @@ pub static DEFAULT_KVS: LazyLock = LazyLock::new(|| { hidden_if_empty: false, }]) }); - -#[derive(Debug, Default)] -pub struct Config { - pub bitrot: String, - pub sleep: Duration, - pub io_count: usize, - pub drive_workers: usize, - pub cache: Duration, -} - -impl Config { - pub fn bitrot_scan_cycle(&self) -> Duration { - self.cache - } - - pub fn get_workers(&self) -> usize { - self.drive_workers - } - - pub fn update(&mut self, nopts: &Config) { - self.bitrot = nopts.bitrot.clone(); - self.io_count = nopts.io_count; - self.sleep = nopts.sleep; - self.drive_workers = nopts.drive_workers; - } -} - -const RUSTFS_BITROT_CYCLE_IN_MONTHS: u64 = 1; - -fn parse_bitrot_config(s: &str) -> Result { - match parse_bool(s) { - Ok(enabled) => { - if enabled { - Ok(Duration::from_secs_f64(0.0)) - } else { - Ok(Duration::from_secs_f64(-1.0)) - } - } - Err(_) => { - if !s.ends_with("m") { - return Err(Error::other("unknown format")); - } - - match s.trim_end_matches('m').parse::() { - Ok(months) => { - if months < RUSTFS_BITROT_CYCLE_IN_MONTHS { - return Err(Error::other(format!("minimum bitrot cycle is {RUSTFS_BITROT_CYCLE_IN_MONTHS} month(s)"))); - } - - Ok(Duration::from_secs(months * 30 * 24 * 60)) - } - Err(err) => Err(Error::other(err)), - } - } - } -} diff --git a/crates/ecstore/src/config/mod.rs b/crates/ecstore/src/config/mod.rs index 5e6e3a3e3..ff97d03e0 100644 --- a/crates/ecstore/src/config/mod.rs +++ b/crates/ecstore/src/config/mod.rs @@ -16,7 +16,6 @@ mod audit; pub mod com; -#[allow(dead_code)] pub mod heal; mod notify; mod oidc; diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index f35017760..ec9da3708 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -1996,11 +1996,11 @@ impl PoolMeta { Ok(false) } - #[allow(dead_code)] pub fn validate(&self, pools: Vec>) -> Result { struct PoolInfo { position: usize, completed: bool, + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] decom_started: bool, } @@ -4958,13 +4958,19 @@ fn is_disk_online_state(state: &str) -> bool { } #[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")] -#[allow(dead_code)] +#[allow( + dead_code, + reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)" +)] fn fallback_total_capacity(disks: &[rustfs_madmin::Disk]) -> usize { fallback_total_capacity_dedup(disks) } #[deprecated(since = "0.1.0", note = "Use fallback_free_capacity_dedup instead")] -#[allow(dead_code)] +#[allow( + dead_code, + reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)" +)] fn fallback_free_capacity(disks: &[rustfs_madmin::Disk]) -> usize { fallback_free_capacity_dedup(disks) } diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 2cab189aa..1932744e8 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -6562,7 +6562,7 @@ impl LocalDisk { Ok(f) } - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn get_metrics(&self) -> DiskMetrics { DiskMetrics::default() } diff --git a/crates/ecstore/src/services/rebalance/types.rs b/crates/ecstore/src/services/rebalance/types.rs index b43e075b8..5f79275dc 100644 --- a/crates/ecstore/src/services/rebalance/types.rs +++ b/crates/ecstore/src/services/rebalance/types.rs @@ -132,7 +132,6 @@ impl RebalanceStopPropagationRecord { } } -#[allow(dead_code)] #[derive(Debug, Clone, Default)] pub struct DiskStat { pub total_space: u64, diff --git a/crates/ecstore/src/services/tier/tier_config.rs b/crates/ecstore/src/services/tier/tier_config.rs index a5be98866..149f96ef6 100644 --- a/crates/ecstore/src/services/tier/tier_config.rs +++ b/crates/ecstore/src/services/tier/tier_config.rs @@ -16,8 +16,16 @@ use serde::{Deserialize, Serialize}; use std::{fmt::Display, io}; use tracing::info; +#[allow( + dead_code, + reason = "tier config wire version stamped by the parity constructors below (backlog#1823)" +)] const C_TIER_CONFIG_VER: &str = "v1"; +#[allow( + dead_code, + reason = "tier-name validation message reached only from the parity constructors below (backlog#1823)" +)] const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty"; const WASABI_US_EAST_ENDPOINT: &str = "https://s3.wasabisys.com"; const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[ @@ -264,7 +272,6 @@ impl Clone for TierConfig { } } -#[allow(dead_code)] impl TierConfig { pub(crate) fn clone_with_credentials(&self) -> Self { Self { @@ -284,6 +291,7 @@ impl TierConfig { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn endpoint(&self) -> String { match self.tier_type { TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(), @@ -303,6 +311,7 @@ impl TierConfig { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn bucket(&self) -> String { match self.tier_type { TierType::S3 => self.s3.as_ref().map(|s| s.bucket.clone()).unwrap_or_default(), @@ -322,6 +331,7 @@ impl TierConfig { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn prefix(&self) -> String { match self.tier_type { TierType::S3 => self.s3.as_ref().map(|s| s.prefix.clone()).unwrap_or_default(), @@ -341,6 +351,7 @@ impl TierConfig { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn region(&self) -> String { match self.tier_type { TierType::S3 => self.s3.as_ref().map(|s| s.region.clone()).unwrap_or_default(), @@ -457,7 +468,7 @@ impl TierWasabi { } impl TierS3 { - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn create( name: &str, access_key: &str, @@ -528,7 +539,7 @@ pub struct TierMinIO { } impl TierMinIO { - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn create( name: &str, endpoint: &str, diff --git a/crates/ecstore/src/services/tier/tier_gen.rs b/crates/ecstore/src/services/tier/tier_gen.rs index 400466b47..63a692b85 100644 --- a/crates/ecstore/src/services/tier/tier_gen.rs +++ b/crates/ecstore/src/services/tier/tier_gen.rs @@ -14,7 +14,6 @@ use crate::services::tier::tier::TierConfigMgr; -#[allow(dead_code)] impl TierConfigMgr { pub fn msg_size(&self) -> usize { 100 From 360bceafced5bd63a39f013f7fc30aef162c50f2 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 08:29:29 +0800 Subject: [PATCH 13/51] feat(heal): add progress and trace observability (#6179) * feat(heal): track erasure set progress baseline Record erasure-set heal byte progress from per-object results and seed progress totals from complete usage-cache snapshots when available. Keep usage-cache failures observational so heal execution continues without a baseline. Co-Authored-By: heihutu * feat(heal): skip filtered erasure set versions Skip erasure-set versions written after the durable heal start time, and queue lifecycle-expired versions for expiry before skipping them. Track new-version and ILM-expired skips separately so progress can explain completed baseline work without treating these skips as retry-blocking failures. Co-Authored-By: heihutu * feat(heal): wire abandoned data-dir cleanup check Connect check_abandoned_parts through ECStore, pool, and set layers so heal can invoke the existing orphan data-dir reclaim path instead of returning NotImplemented. Add dry-run support to the reclaim scan and cover dry-run plus scoped set behavior with regression tests. Co-Authored-By: heihutu * feat(obs): add heal scanner trace bus Introduce an in-process broadcast trace bus with typed heal and scanner events, lazy event construction, and bounded lagged-subscriber behavior. Cover zero-subscriber publishing, subscription delivery, drop accounting, and lagged receivers with focused common-crate tests. Co-Authored-By: heihutu * feat(obs): stream heal trace events from admin API Wire the admin trace endpoint to the common trace bus for heal/scanner events, including kind, regex, and threshold filtering. Co-Authored-By: heihutu * feat(obs): emit heal trace events Publish heal task lifecycle and abandoned-parts cleanup events through the common trace bus so the admin trace stream has live heal diagnostics. Co-Authored-By: heihutu * feat(obs): emit scanner trace events Publish scanner folder, lifecycle action, and heal-candidate events through the common trace bus for live admin scanner diagnostics. Co-Authored-By: heihutu * fix(heal): route data usage loader through storage api Keep ECStore data-usage facade access behind the heal storage_api boundary so architecture migration guards can validate the heal progress path. Co-Authored-By: heihutu * perf(heal): avoid lifecycle snapshots on ordinary heal pages Only request lifecycle object snapshots when the heal pass has lifecycle expiry context. This keeps ordinary listing and disk-walk pages from cloning FileInfo/ObjectInfo payloads while preserving the skip path that queues expired versions. Co-Authored-By: heihutu * test(heal): update bug-fix mocks for lifecycle snapshots Carry the lifecycle snapshot opt-in argument through the remaining heal bug-fix test mocks so all-targets clippy covers the updated storage trait. Co-Authored-By: heihutu * test(rustfs): sync heal storage mock signature Update the rustfs storage RPC test mock for the lifecycle snapshot opt-in argument and cover it with rustfs all-targets clippy. Co-Authored-By: heihutu * test(e2e): allocate smoke ports across nextest processes Serialize E2E port selection with a small /tmp allocator so nextest workers do not reuse the same just-released ephemeral port before RustFS binds it. Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- Cargo.lock | 1 + crates/common/Cargo.toml | 1 + crates/common/src/lib.rs | 1 + crates/common/src/trace_bus.rs | 333 ++++++++++++++ crates/e2e_test/src/common.rs | 83 +++- crates/ecstore/src/api/mod.rs | 1 + crates/ecstore/src/bucket/lifecycle/mod.rs | 2 +- crates/ecstore/src/core/pools.rs | 91 ++++ crates/ecstore/src/core/sets.rs | 29 +- .../src/set_disk/core/io_primitives.rs | 22 + crates/ecstore/src/set_disk/mod.rs | 109 ++++- crates/ecstore/src/set_disk/ops/heal.rs | 61 ++- crates/ecstore/src/set_disk/ops/heal_walk.rs | 50 ++- crates/ecstore/src/store/heal.rs | 42 +- crates/ecstore/src/store/heal_walk.rs | 3 +- crates/heal/src/heal/channel.rs | 1 + crates/heal/src/heal/erasure_healer.rs | 344 ++++++++++++-- crates/heal/src/heal/manager.rs | 30 ++ crates/heal/src/heal/progress.rs | 166 ++++++- crates/heal/src/heal/storage.rs | 176 +++++++- crates/heal/src/heal/storage_api.rs | 13 +- crates/heal/src/heal/task.rs | 238 +++++++++- crates/heal/src/lib.rs | 1 + .../heal_b5_versioned_regression_test.rs | 2 +- .../tests/heal_b920_subquorum_union_test.rs | 6 +- crates/heal/tests/heal_bug_fixes_test.rs | 2 + crates/madmin/src/service_commands.rs | 10 +- crates/scanner/src/scanner_folder.rs | 254 +++++++++++ rustfs/src/admin/handlers/profile_admin.rs | 422 ++++++++++++++++-- rustfs/src/storage/rpc/node_service.rs | 1 + 30 files changed, 2383 insertions(+), 112 deletions(-) create mode 100644 crates/common/src/trace_bus.rs diff --git a/Cargo.lock b/Cargo.lock index b521a731d..85c91d523 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9280,6 +9280,7 @@ dependencies = [ "s3s", "serde", "serde_json", + "smallvec", "tokio", "tonic", "tracing", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 5c9094d61..ddc02cf2e 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -42,6 +42,7 @@ chrono = { workspace = true, features = ["serde"] } jiff = { workspace = true, features = ["serde"] } metrics = { workspace = true } serde = { workspace = true, features = ["derive"] } +smallvec = { workspace = true } rmp-serde = { workspace = true } s3s = { workspace = true, features = ["minio"] } tracing = { workspace = true } diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 09240e25b..1ae200f24 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -19,6 +19,7 @@ pub mod last_minute; pub mod metrics; mod readiness; pub mod table_catalog; +pub mod trace_bus; pub use globals::*; pub use readiness::{GlobalReadiness, SystemStage}; diff --git a/crates/common/src/trace_bus.rs b/crates/common/src/trace_bus.rs new file mode 100644 index 000000000..5e6a90d60 --- /dev/null +++ b/crates/common/src/trace_bus.rs @@ -0,0 +1,333 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use smallvec::SmallVec; +use std::{ + sync::{ + Arc, OnceLock, + atomic::{AtomicUsize, Ordering}, + }, + time::{Duration, SystemTime}, +}; +use tokio::sync::broadcast; + +const DEFAULT_TRACE_BUS_CAPACITY: usize = 1024; +const TRACE_ATTR_INLINE_CAPACITY: usize = 8; + +static GLOBAL_TRACE_BUS: OnceLock = OnceLock::new(); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TraceKind { + Heal, + Scanner, +} + +impl TraceKind { + pub const fn as_str(self) -> &'static str { + match self { + Self::Heal => "heal", + Self::Scanner => "scanner", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TraceFunc { + HealTask, + HealBucket, + HealObject, + HealCheckAbandonedParts, + HealErasureSetPage, + ScannerFolder, + ScannerIlmAction, + ScannerHealCandidate, + Dropped, +} + +impl TraceFunc { + pub const fn as_str(self) -> &'static str { + match self { + Self::HealTask => "heal.Task", + Self::HealBucket => "heal.Bucket", + Self::HealObject => "heal.Object", + Self::HealCheckAbandonedParts => "heal.CheckAbandonedParts", + Self::HealErasureSetPage => "heal.ErasureSetPage", + Self::ScannerFolder => "scanner.Folder", + Self::ScannerIlmAction => "scanner.IlmAction", + Self::ScannerHealCandidate => "scanner.HealCandidate", + Self::Dropped => "trace.Dropped", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TraceVal { + Bool(bool), + U64(u64), + I64(i64), + Str(Arc), +} + +impl From for TraceVal { + fn from(value: bool) -> Self { + Self::Bool(value) + } +} + +impl From for TraceVal { + fn from(value: u64) -> Self { + Self::U64(value) + } +} + +impl From for TraceVal { + fn from(value: i64) -> Self { + Self::I64(value) + } +} + +impl From<&str> for TraceVal { + fn from(value: &str) -> Self { + Self::Str(Arc::from(value)) + } +} + +impl From for TraceVal { + fn from(value: String) -> Self { + Self::Str(Arc::from(value)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TraceAttr { + pub key: &'static str, + pub value: TraceVal, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TraceEvent { + pub kind: TraceKind, + pub func: TraceFunc, + pub time: SystemTime, + pub bucket: Option>, + pub object: Option>, + pub duration: Duration, + pub bytes: u64, + pub attrs: SmallVec<[TraceAttr; TRACE_ATTR_INLINE_CAPACITY]>, +} + +impl TraceEvent { + pub fn new(kind: TraceKind, func: TraceFunc) -> Self { + Self { + kind, + func, + time: SystemTime::now(), + bucket: None, + object: None, + duration: Duration::ZERO, + bytes: 0, + attrs: SmallVec::new(), + } + } + + pub fn with_bucket(mut self, bucket: impl Into>) -> Self { + self.bucket = Some(bucket.into()); + self + } + + pub fn with_object(mut self, object: impl Into>) -> Self { + self.object = Some(object.into()); + self + } + + pub fn with_duration(mut self, duration: Duration) -> Self { + self.duration = duration; + self + } + + pub fn with_bytes(mut self, bytes: u64) -> Self { + self.bytes = bytes; + self + } + + pub fn with_attr(mut self, key: &'static str, value: impl Into) -> Self { + self.attrs.push(TraceAttr { + key, + value: value.into(), + }); + self + } +} + +#[derive(Debug)] +pub struct TraceBus { + sender: broadcast::Sender>, + subscriber_count: Arc, +} + +impl TraceBus { + pub fn new(capacity: usize) -> Self { + let capacity = capacity.max(1); + let (sender, _receiver) = broadcast::channel(capacity); + Self { + sender, + subscriber_count: Arc::new(AtomicUsize::new(0)), + } + } + + pub fn subscriber_count(&self) -> usize { + self.subscriber_count.load(Ordering::Acquire) + } + + pub fn subscribe(&self) -> TraceSubscription { + let receiver = self.sender.subscribe(); + self.subscriber_count.fetch_add(1, Ordering::AcqRel); + TraceSubscription { + receiver, + subscriber_count: Arc::clone(&self.subscriber_count), + } + } + + pub fn emit(&self, build: impl FnOnce() -> TraceEvent) -> bool { + if self.subscriber_count() == 0 { + return false; + } + + self.sender.send(Arc::new(build())).is_ok() + } +} + +impl Default for TraceBus { + fn default() -> Self { + Self::new(DEFAULT_TRACE_BUS_CAPACITY) + } +} + +#[derive(Debug)] +pub struct TraceSubscription { + receiver: broadcast::Receiver>, + subscriber_count: Arc, +} + +impl TraceSubscription { + pub async fn recv(&mut self) -> Result, broadcast::error::RecvError> { + self.receiver.recv().await + } + + pub fn try_recv(&mut self) -> Result, broadcast::error::TryRecvError> { + self.receiver.try_recv() + } +} + +impl Drop for TraceSubscription { + fn drop(&mut self) { + self.subscriber_count.fetch_sub(1, Ordering::AcqRel); + } +} + +pub fn global_trace_bus() -> &'static TraceBus { + GLOBAL_TRACE_BUS.get_or_init(TraceBus::default) +} + +pub fn subscribe_trace_events() -> TraceSubscription { + global_trace_bus().subscribe() +} + +pub fn trace_emit(build: impl FnOnce() -> TraceEvent) -> bool { + global_trace_bus().emit(build) +} + +pub fn trace_subscriber_count() -> usize { + global_trace_bus().subscriber_count() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicUsize; + + #[test] + fn trace_emit_skips_builder_without_subscribers() { + let bus = TraceBus::new(4); + let built = AtomicUsize::new(0); + + let sent = bus.emit(|| { + built.fetch_add(1, Ordering::Relaxed); + TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask) + }); + + assert!(!sent); + assert_eq!(built.load(Ordering::Relaxed), 0); + } + + #[tokio::test] + async fn trace_subscriber_receives_event() { + let bus = TraceBus::new(4); + let mut subscription = bus.subscribe(); + + assert!(bus.emit(|| { + TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject) + .with_bucket("bucket") + .with_object("object") + .with_duration(Duration::from_millis(7)) + .with_bytes(11) + .with_attr("dry", true) + })); + + let event = subscription + .recv() + .await + .expect("subscriber should receive emitted trace event"); + + assert_eq!(event.kind, TraceKind::Heal); + assert_eq!(event.func, TraceFunc::HealObject); + assert_eq!(event.bucket.as_deref(), Some("bucket")); + assert_eq!(event.object.as_deref(), Some("object")); + assert_eq!(event.duration, Duration::from_millis(7)); + assert_eq!(event.bytes, 11); + assert_eq!( + event.attrs.as_slice(), + &[TraceAttr { + key: "dry", + value: TraceVal::Bool(true) + }] + ); + } + + #[test] + fn trace_subscription_drop_decrements_count() { + let bus = TraceBus::new(4); + let subscription = bus.subscribe(); + + assert_eq!(bus.subscriber_count(), 1); + drop(subscription); + assert_eq!(bus.subscriber_count(), 0); + } + + #[tokio::test] + async fn lagged_subscriber_drops_events_without_blocking_publishers() { + let bus = TraceBus::new(2); + let mut subscription = bus.subscribe(); + + for index in 0_u64..4 { + assert!(bus.emit(|| { TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder).with_attr("index", index) })); + } + + let err = subscription + .recv() + .await + .expect_err("receiver should observe lag instead of blocking publishers"); + assert!(matches!(err, broadcast::error::RecvError::Lagged(_))); + } +} diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index f1fcaa20a..460c764ed 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -32,6 +32,7 @@ use rustfs_signer::sign_v4; use s3s::Body; use std::ffi::OsStr; use std::fs as stdfs; +use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; use std::sync::Once; @@ -51,6 +52,11 @@ pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] = &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")]; pub const TEST_BUCKET: &str = "e2e-test-bucket"; const RUSTFS_FULL_FEATURE: &str = "full"; +const TEST_PORT_MIN: u16 = 20_000; +const TEST_PORT_RANGE: u16 = 40_000; +const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port"; +const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock"; +const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30); fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option { let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy(); @@ -67,6 +73,64 @@ fn configured_capture_log_path(temp_dir: &str) -> Option { capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned()) } +struct PortAllocatorGuard; + +impl PortAllocatorGuard { + async fn acquire() -> Result> { + loop { + match stdfs::create_dir(TEST_PORT_LOCK_DIR) { + Ok(()) => return Ok(Self), + Err(err) if err.kind() == ErrorKind::AlreadyExists => { + remove_stale_port_allocator_lock(); + sleep(Duration::from_millis(10)).await; + } + Err(err) => return Err(err.into()), + } + } + } +} + +impl Drop for PortAllocatorGuard { + fn drop(&mut self) { + let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR); + } +} + +fn advance_test_port(port: u16) -> u16 { + let offset = (port - TEST_PORT_MIN + 1) % TEST_PORT_RANGE; + TEST_PORT_MIN + offset +} + +fn seeded_test_port() -> u16 { + let offset = (Uuid::new_v4().as_u128() % u128::from(TEST_PORT_RANGE)) as u16; + TEST_PORT_MIN + offset +} + +fn read_next_test_port() -> u16 { + stdfs::read_to_string(TEST_PORT_COUNTER_PATH) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .filter(|port| (TEST_PORT_MIN..TEST_PORT_MIN + TEST_PORT_RANGE).contains(port)) + .unwrap_or_else(seeded_test_port) +} + +fn remove_stale_port_allocator_lock() { + let Ok(metadata) = stdfs::metadata(TEST_PORT_LOCK_DIR) else { + return; + }; + let Ok(modified) = metadata.modified() else { + return; + }; + if modified.elapsed().is_ok_and(|elapsed| elapsed > TEST_PORT_LOCK_STALE_AFTER) { + let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR); + } +} + +fn write_next_test_port(port: u16) -> Result<(), Box> { + stdfs::write(TEST_PORT_COUNTER_PATH, port.to_string())?; + Ok(()) +} + pub(crate) fn capture_command_logs( command: &mut Command, log_path: Option<&str>, @@ -508,10 +572,21 @@ impl RustFSTestEnvironment { /// Find an available port for the test pub async fn find_available_port() -> Result> { use std::net::TcpListener; - let listener = TcpListener::bind("127.0.0.1:0")?; - let port = listener.local_addr()?.port(); - drop(listener); - Ok(port) + let _guard = PortAllocatorGuard::acquire().await?; + let mut next_port = read_next_test_port(); + + for _ in 0..TEST_PORT_RANGE { + let port = next_port; + next_port = advance_test_port(next_port); + write_next_test_port(next_port)?; + + if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) { + drop(listener); + return Ok(port); + } + } + + Err("no available E2E test port found".into()) } /// Kill any existing RustFS processes diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 5526b7d00..4b8bc3249 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -483,6 +483,7 @@ pub mod store_list { } pub mod storage { + pub use crate::core::pools::HealLifecycleExpiryContext; pub use crate::store::HealWalkVersion; pub use crate::store::{ ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks, diff --git a/crates/ecstore/src/bucket/lifecycle/mod.rs b/crates/ecstore/src/bucket/lifecycle/mod.rs index 823263072..6d8e64f1b 100644 --- a/crates/ecstore/src/bucket/lifecycle/mod.rs +++ b/crates/ecstore/src/bucket/lifecycle/mod.rs @@ -19,7 +19,7 @@ pub mod core; pub mod evaluator; pub mod manual_transition_job; mod metadata_boundary; -pub(crate) use metadata_boundary::get_expiry_configs; +pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs}; mod object_lock_boundary; pub use self::core as lifecycle; mod replication_sink; diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index ec9da3708..02fe70f29 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -16,6 +16,7 @@ use crate::bucket::replication::replication_state_from_filemeta; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::bucket::{ lifecycle::{ + LifecycleExpiryConfigs, bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::{ LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle, @@ -2335,6 +2336,10 @@ fn lifecycle_action_removes_data_movement_version(action: IlmAction) -> bool { ) } +fn lifecycle_action_skips_heal_version(action: IlmAction) -> bool { + action.delete() +} + fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result { if !apply_actions || applied { return Ok(true); @@ -2385,7 +2390,80 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement( } } +pub struct HealLifecycleExpiryContext { + configs: LifecycleExpiryConfigs, +} + impl ECStore { + pub async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result> { + if bucket == RUSTFS_META_BUCKET { + return Ok(None); + } + + let configs = get_expiry_configs(self, bucket).await?; + if configs.lifecycle.is_none() { + return Ok(None); + } + + Ok(Some(HealLifecycleExpiryContext { configs })) + } + + pub async fn enqueue_heal_lifecycle_expiry( + self: &Arc, + context: &HealLifecycleExpiryContext, + bucket: &str, + object: &str, + version_id: Option<&str>, + object_info: Option<&crate::object_api::ObjectInfo>, + ) -> Result { + let Some(lifecycle_config) = context.configs.lifecycle.as_ref() else { + return Ok(false); + }; + + let object_info = if let Some(object_info) = object_info { + if object_info.bucket != bucket || object_info.name != object { + return Ok(false); + } + let snapshot_version_id = object_info + .version_id + .filter(|version_id| !version_id.is_nil()) + .map(|version_id| version_id.to_string()); + if snapshot_version_id.as_deref() != version_id { + return Ok(false); + } + object_info.clone() + } else { + match self + .get_object_info( + bucket, + object, + &ObjectOptions { + version_id: version_id.map(str::to_string), + versioned: version_id.is_some(), + expected_bucket_incarnation_id: Some(context.configs.bucket_incarnation_id), + ..Default::default() + }, + ) + .await + { + Ok(object_info) => object_info, + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => return Ok(false), + Err(err) => return Err(err), + } + }; + + let event = eval_action_from_lifecycle(lifecycle_config, context.configs.object_lock.as_deref(), &object_info).await; + if !lifecycle_action_skips_heal_version(event.action) { + return Ok(false); + } + + if lifecycle_delete_all_versions_blocked_by_replication(self.clone(), bucket, &object_info.name, event.action).await? { + return Ok(false); + } + + Ok(apply_expiry_rule_in(self.clone(), &event, &LcEventSrc::Scanner, &object_info).await) + } + async fn save_current_pool_meta(&self) -> Result<()> { let _save_guard = self.pool_meta_save_gate.lock().await; let snapshot = { @@ -4287,6 +4365,19 @@ mod tests { )); } + #[test] + fn lifecycle_action_skips_heal_version_for_every_delete_action() { + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteVersionAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredVersionAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAllVersionsAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DelMarkerDeleteAllVersionsAction)); + assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionAction)); + assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionVersionAction)); + assert!(!lifecycle_action_skips_heal_version(IlmAction::NoneAction)); + } + #[test] fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() { let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false) diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index acb8b53b9..1d1bcedeb 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -1140,11 +1140,11 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets { Err(Error::DiskNotFound) } - #[tracing::instrument(skip(self))] - async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> { - // Multipart orphan reconciliation is intentionally retained above the pool/set layers - // until there is a concrete caller and a stable lower-level contract to implement. - Err(StorageError::NotImplemented) + #[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))] + async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> { + self.get_disks_for_heal_object(object, opts)? + .check_abandoned_parts(bucket, object, opts) + .await } } @@ -1996,7 +1996,7 @@ mod tests { } #[tokio::test] - async fn sets_check_abandoned_parts_returns_typed_not_implemented_error() { + async fn sets_check_abandoned_parts_rejects_invalid_set_scope() { let format = FormatV3::new(1, 1); let sets = Sets { id: format.id, @@ -2021,10 +2021,21 @@ mod tests { }; let err = sets - .check_abandoned_parts("bucket", "object", &HealOpts::default()) + .check_abandoned_parts( + "bucket", + "object", + &HealOpts { + set: Some(1), + ..Default::default() + }, + ) .await - .expect_err("abandoned-parts ownership should stay above the pool/set storage layers"); - assert!(matches!(err, StorageError::NotImplemented)); + .expect_err("out-of-range abandoned-parts set scope must fail closed"); + assert!( + matches!(err, StorageError::InvalidArgument(_, ref field, ref reason) + if field == "set" && reason.contains("invalid heal set index 1")), + "unexpected invalid set error: {err:?}" + ); } // Builds a single-set `Sets` over `SET_DRIVE_COUNT` local temp-dir disks, diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 41bd6dac7..7af25a64c 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -4860,6 +4860,14 @@ impl SetDisks { /// is best-effort maintenance: individual delete failures are logged and /// skipped rather than propagated. pub(crate) async fn reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result { + self.reclaim_orphan_data_dirs_inner(bucket, object, false).await + } + + pub(crate) async fn dry_run_reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result { + self.reclaim_orphan_data_dirs_inner(bucket, object, true).await + } + + async fn reclaim_orphan_data_dirs_inner(&self, bucket: &str, object: &str, dry_run: bool) -> disk::error::Result { let disks = self.get_disks_internal().await; // Phase 1 (read-only): build the referenced-data-dir union and record the @@ -4967,6 +4975,20 @@ impl SetDisks { continue; } let stray = format!("{object}/{dir}"); + if dry_run { + removed += 1; + debug!( + target: "rustfs_ecstore::set_disk", + event = "heal_abandoned_parts", + component = "ecstore", + subsystem = "heal", + state = "dry_run_matched", + result = "matched", + bucket, object, data_dir = %dir, + "Heal abandoned parts dry-run matched orphaned data directory" + ); + continue; + } match disk .delete( bucket, diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index a04c5db82..5a56ea36e 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -6998,6 +6998,100 @@ mod tests { assert!(object_dir.join(STORAGE_FORMAT_FILE).exists(), "metadata must be preserved"); } + async fn recv_abandoned_parts_trace( + trace: &mut rustfs_common::trace_bus::TraceSubscription, + bucket: &str, + object: &str, + state: &str, + ) -> rustfs_common::trace_bus::TraceEvent { + for _ in 0..32 { + let event = tokio::time::timeout(std::time::Duration::from_secs(1), trace.recv()) + .await + .expect("abandoned-parts trace event should arrive") + .expect("trace bus should stay open"); + if event.kind == rustfs_common::trace_bus::TraceKind::Heal + && event.func == rustfs_common::trace_bus::TraceFunc::HealCheckAbandonedParts + && event.bucket.as_deref() == Some(bucket) + && event.object.as_deref() == Some(object) + && trace_attr_string(&event, "state").as_deref() == Some(state) + { + return (*event).clone(); + } + } + + panic!("expected abandoned-parts trace state {state} for {bucket}/{object}"); + } + + fn trace_attr_string(event: &rustfs_common::trace_bus::TraceEvent, key: &str) -> Option { + event.attrs.iter().find_map(|attr| { + if attr.key != key { + return None; + } + Some(match &attr.value { + rustfs_common::trace_bus::TraceVal::Bool(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::U64(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::I64(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::Str(value) => value.to_string(), + }) + }) + } + + #[tokio::test] + async fn check_abandoned_parts_dry_run_counts_without_deleting() { + let mut trace = rustfs_common::trace_bus::subscribe_trace_events(); + let (dir, disk) = make_single_local_disk().await; + let live = Uuid::new_v4(); + let orphan = Uuid::new_v4(); + + let object_dir = dir.path().join("bucket").join("obj"); + write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await; + fs::create_dir_all(object_dir.join(live.to_string())) + .await + .expect("live data dir should be created"); + fs::create_dir_all(object_dir.join(orphan.to_string())) + .await + .expect("orphan data dir should be created"); + + let set = make_set_disks_with(vec![Some(disk)]).await; + set.check_abandoned_parts( + "bucket", + "obj", + &HealOpts { + dry_run: true, + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("dry-run abandoned-parts check should succeed"); + let dry_run_trace = recv_abandoned_parts_trace(&mut trace, "bucket", "obj", "dry_run_matched").await; + assert_eq!(trace_attr_string(&dry_run_trace, "dry_run").as_deref(), Some("true")); + assert_eq!(trace_attr_string(&dry_run_trace, "data_dirs").as_deref(), Some("1")); + + assert!(object_dir.join(live.to_string()).exists(), "referenced data dir must be preserved"); + assert!(object_dir.join(orphan.to_string()).exists(), "dry-run must not remove orphaned data dir"); + + set.check_abandoned_parts( + "bucket", + "obj", + &HealOpts { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("abandoned-parts check should reclaim stale data dir"); + let reclaim_trace = recv_abandoned_parts_trace(&mut trace, "bucket", "obj", "reclaimed").await; + assert_eq!(trace_attr_string(&reclaim_trace, "dry_run").as_deref(), Some("false")); + assert_eq!(trace_attr_string(&reclaim_trace, "data_dirs").as_deref(), Some("1")); + + assert!( + object_dir.join(live.to_string()).exists(), + "referenced data dir must remain after reclaim" + ); + assert!(!object_dir.join(orphan.to_string()).exists(), "orphaned data dir must be removed"); + } + #[tokio::test] async fn reclaim_orphan_data_dirs_recovers_deferred_cleanup_after_restart() { let (dir, disk) = make_single_local_disk().await; @@ -12233,11 +12327,18 @@ mod tests { .expect_err("unsupported copy_object_part should return a typed error"); assert!(matches!(copy_part_err, StorageError::NotImplemented)); - let abandoned_err = set_disks - .check_abandoned_parts("bucket", "object", &HealOpts::default()) + set_disks + .check_abandoned_parts( + "bucket", + "object", + &HealOpts { + dry_run: true, + no_lock: true, + ..Default::default() + }, + ) .await - .expect_err("abandoned-parts check should stay in the upper reconciliation layer"); - assert!(matches!(abandoned_err, StorageError::NotImplemented)); + .expect("abandoned-parts check should be callable on empty disk sets"); } #[tokio::test] diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 274b33f43..de688fbdb 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -16,6 +16,7 @@ use super::super::*; use crate::disk::disk_store::DiskStoreRenameDataExt; use crate::io_support::bitrot::object_mmap_read_enabled; use crate::storage_api_contracts::namespace::NamespaceLocking as _; +use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit}; use tracing::trace; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; @@ -2057,11 +2058,61 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks { Err(Error::DiskNotFound) } - #[tracing::instrument(skip(self))] - async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> { - // Multipart orphan reconciliation is intentionally retained above the set layer - // until there is a concrete caller and a stable lower-level contract to implement. - Err(StorageError::NotImplemented) + #[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))] + async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> { + let started_at = std::time::Instant::now(); + let _write_lock_guard = if !opts.no_lock { + let ns_lock = self.new_ns_lock(bucket, object).await?; + Some( + ns_lock + .get_write_lock(get_lock_acquire_timeout()) + .await + .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?, + ) + } else { + None + }; + + let removed = if opts.dry_run { + self.dry_run_reclaim_orphan_data_dirs(bucket, object).await? + } else { + self.reclaim_orphan_data_dirs(bucket, object).await? + }; + let state = if opts.dry_run && removed > 0 { + "dry_run_matched" + } else if removed > 0 { + "reclaimed" + } else { + "checked" + }; + let data_dirs = u64::try_from(removed).unwrap_or(u64::MAX); + + trace_emit(|| { + TraceEvent::new(TraceKind::Heal, TraceFunc::HealCheckAbandonedParts) + .with_bucket(bucket) + .with_object(object) + .with_duration(started_at.elapsed()) + .with_attr("state", state) + .with_attr("dry_run", opts.dry_run) + .with_attr("data_dirs", data_dirs) + }); + + if removed > 0 { + trace!( + event = "heal_abandoned_parts", + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_HEAL, + state = if opts.dry_run { "dry_run_matched" } else { "reclaimed" }, + result = "ok", + bucket, + object, + dry_run = opts.dry_run, + data_dirs = removed, + "Heal abandoned parts checked object data directories" + ); + } + + Ok(()) } } diff --git a/crates/ecstore/src/set_disk/ops/heal_walk.rs b/crates/ecstore/src/set_disk/ops/heal_walk.rs index a39a9abe7..ea1ea66cf 100644 --- a/crates/ecstore/src/set_disk/ops/heal_walk.rs +++ b/crates/ecstore/src/set_disk/ops/heal_walk.rs @@ -23,6 +23,7 @@ //! per-version `SetDisks::heal_object`. use super::super::*; +use crate::object_api::ObjectInfo; use std::collections::HashSet; use std::sync::Mutex; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -39,12 +40,16 @@ const BACKGROUND_WALKDIR_STALL_TIMEOUT: Duration = Duration::from_secs(60); /// it must not gate healing logic — the delete-marker vs data path is chosen /// inside `ops/heal.rs` from the resolved latest metadata. `version_id` is /// normalized (nil/absent UUID => `None`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct HealWalkVersion { /// object key pub name: String, /// normalized version id (`None` when the version is nil/absent) pub version_id: Option, + /// version modification time as Unix nanoseconds + pub mod_time_unix_nanos: Option, + /// object snapshot for lifecycle evaluation + pub lifecycle_object_info: Option, /// whether this version is a delete marker (observability only) pub is_delete_marker: bool, } @@ -63,6 +68,7 @@ struct HealWalkCollector { bucket: String, batch_objects: usize, version_budget: usize, + include_lifecycle_object_info: bool, objects: Mutex>, decode_error: Mutex>, version_total: AtomicUsize, @@ -116,10 +122,25 @@ impl HealWalkCollector { let mut versions = Vec::with_capacity(fiv.versions.len() + fiv.free_versions.len()); for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) { + let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil()); + let lifecycle_object_info = if self.include_lifecycle_object_info { + let mut lifecycle_fi = fi.clone(); + lifecycle_fi.version_id = version_uuid; + Some(ObjectInfo::from_file_info( + &lifecycle_fi, + &self.bucket, + &entry.name, + version_uuid.is_some(), + )) + } else { + None + }; versions.push(HealWalkVersion { name: entry.name.clone(), // Normalize: nil/absent version id => None. - version_id: fi.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()), + version_id: version_uuid.map(|u| u.to_string()), + mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()), + lifecycle_object_info, is_delete_marker: fi.deleted, }); } @@ -173,11 +194,26 @@ impl HealWalkCollector { } }; for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) { - let vid = fi.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()); + let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil()); + let vid = version_uuid.map(|u| u.to_string()); if seen.insert(vid.clone()) { + let lifecycle_object_info = if self.include_lifecycle_object_info { + let mut lifecycle_fi = fi.clone(); + lifecycle_fi.version_id = version_uuid; + Some(ObjectInfo::from_file_info( + &lifecycle_fi, + &self.bucket, + &entry.name, + version_uuid.is_some(), + )) + } else { + None + }; versions.push(HealWalkVersion { name: entry.name.clone(), version_id: vid, + mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()), + lifecycle_object_info, is_delete_marker: fi.deleted, }); } @@ -255,6 +291,7 @@ impl SetDisks { forward_to: Option<&str>, batch_objects: usize, version_budget: usize, + include_lifecycle_object_info: bool, ) -> disk::error::Result<(Vec, Option, bool)> { assert!(batch_objects >= 2, "heal_walk_versions_page requires batch_objects >= 2"); @@ -264,6 +301,7 @@ impl SetDisks { bucket: bucket.to_string(), batch_objects, version_budget: version_budget.max(1), + include_lifecycle_object_info, objects: Mutex::new(Vec::new()), decode_error: Mutex::new(None), version_total: AtomicUsize::new(0), @@ -347,6 +385,7 @@ mod tests { bucket: "bucket".to_string(), batch_objects: 2, version_budget: 2, + include_lifecycle_object_info: false, objects: Mutex::new(Vec::new()), decode_error: Mutex::new(None), version_total: AtomicUsize::new(0), @@ -388,6 +427,8 @@ mod tests { HealWalkVersion { name: name.to_string(), version_id: Some(id.to_string()), + mod_time_unix_nanos: None, + lifecycle_object_info: None, is_delete_marker: dm, } } @@ -491,6 +532,7 @@ mod tests { bucket: "bucket".to_string(), batch_objects: 1000, version_budget: 10_000, + include_lifecycle_object_info: false, objects: Mutex::new(Vec::new()), version_total: AtomicUsize::new(0), decode_error: Mutex::new(None), @@ -567,7 +609,7 @@ mod tests { .expect("corrupt test metadata should be written"); let error = set_disks - .heal_walk_versions_page(bucket, "", None, 2, 2) + .heal_walk_versions_page(bucket, "", None, 2, 2, false) .await .expect_err("semantic metadata corruption must fail the heal disk walk"); diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index 3efa4ed5f..d10abe740 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -18,6 +18,7 @@ use tracing::trace; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_HEAL: &str = "heal"; +const EVENT_HEAL_ABANDONED_PARTS: &str = "heal_abandoned_parts"; const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed"; const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started"; @@ -256,13 +257,40 @@ impl ECStore { #[instrument(skip(self))] pub(super) async fn handle_check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> { - let _ = (bucket, object, opts); - // Stale multipart reconciliation is already owned by the lifecycle-driven - // background cleanup path in `bucket_lifecycle_ops.rs`. There is currently - // no stable object-heal contract that should fan this request out through - // pool/set storage layers, so keep the placeholder explicit at the ECStore - // boundary instead of dispatching into lower layers. - Err(StorageError::NotImplemented) + let object = encode_dir_object(object); + let pools = self.get_pools_for_heal_object(opts)?; + + let mut futures = Vec::with_capacity(pools.len()); + for pool in pools.iter() { + futures.push(pool.check_abandoned_parts(bucket, &object, opts)); + } + + let mut first_error = None; + for result in join_all(futures).await { + if let Err(err) = result + && first_error.is_none() + { + first_error = Some(err); + } + } + + if let Some(err) = first_error { + return Err(err); + } + + trace!( + event = EVENT_HEAL_ABANDONED_PARTS, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_HEAL, + state = "completed", + result = "ok", + bucket, + object, + dry_run = opts.dry_run, + "Heal abandoned parts completed" + ); + + Ok(()) } } diff --git a/crates/ecstore/src/store/heal_walk.rs b/crates/ecstore/src/store/heal_walk.rs index 39d50007d..191e87e54 100644 --- a/crates/ecstore/src/store/heal_walk.rs +++ b/crates/ecstore/src/store/heal_walk.rs @@ -34,6 +34,7 @@ impl ECStore { forward_to: Option<&str>, batch_objects: usize, version_budget: usize, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { if pool_idx >= self.pools.len() || set_idx >= self.pools[pool_idx].disk_set.len() { return Err(Error::other(format!( @@ -43,7 +44,7 @@ impl ECStore { } self.pools[pool_idx].disk_set[set_idx] - .heal_walk_versions_page(bucket, prefix, forward_to, batch_objects, version_budget) + .heal_walk_versions_page(bucket, prefix, forward_to, batch_objects, version_budget, include_lifecycle_object_info) .await .map_err(Error::from) } diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index d2be2545d..23cf2f168 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -767,6 +767,7 @@ mod tests { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> crate::Result<(Vec, Option, bool)> { Ok((vec![], None, false)) } diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index 8f1810bb6..04075837c 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -23,13 +23,14 @@ use crate::heal::{ }; use crate::{Error, Result}; use futures::{StreamExt, stream::FuturesUnordered}; -use metrics::gauge; +use metrics::{counter, gauge}; use rustfs_common::heal_channel::{HealOpts, HealRequestSource, HealScanMode}; use rustfs_madmin::heal_commands::HealResultItem; use std::sync::{ Arc, atomic::{AtomicUsize, Ordering}, }; +use std::time::{Duration, UNIX_EPOCH}; use tokio::sync::{RwLock, Semaphore}; use tracing::{debug, error, warn}; @@ -47,6 +48,21 @@ enum HealObjectOutcome { Failed, } +fn result_object_size_u64(result: &HealResultItem) -> u64 { + u64::try_from(result.object_size).unwrap_or(u64::MAX) +} + +const NEW_VERSION_SKIP_GRACE_SECS: u64 = 60; +const NANOS_PER_SECOND: i128 = 1_000_000_000; + +fn should_skip_new_version(mod_time_unix_nanos: Option, started_at_secs: u64) -> bool { + let Some(mod_time_unix_nanos) = mod_time_unix_nanos else { + return false; + }; + let cutoff_secs = started_at_secs.saturating_add(NEW_VERSION_SKIP_GRACE_SECS); + mod_time_unix_nanos > i128::from(cutoff_secs).saturating_mul(NANOS_PER_SECOND) +} + struct PageConcurrencyGuard { in_flight: Arc, set_label: String, @@ -492,6 +508,7 @@ impl ErasureSetHealer { &mut skipped_objects, resume_manager, checkpoint_manager, + state.start_time, ) .await; @@ -658,6 +675,7 @@ impl ErasureSetHealer { skipped_objects: &mut u64, resume_manager: &ResumeManager, checkpoint_manager: &CheckpointManager, + started_at_secs: u64, ) -> Result<()> { debug!( target: "rustfs::heal::erasure_healer", @@ -710,6 +728,7 @@ impl ErasureSetHealer { // The end-of-pass summary reports the full failed/skipped counts. let mut transient_skip_samples_logged = 0_u64; let mut failure_samples_logged = 0_u64; + let mut bytes_processed = self.progress.read().await.bytes_processed; // backlog#920: select the per-erasure-set DISK-WALK union enumerator when // the scan is Deep OR the request came from AutoHeal — these are the paths @@ -718,17 +737,25 @@ impl ErasureSetHealer { // which stays the default. let use_disk_walk = matches!(self.heal_opts.scan_mode, HealScanMode::Deep) || matches!(self.source, HealRequestSource::AutoHeal); + let lifecycle_expiry_context = self.storage.load_heal_lifecycle_expiry_context(bucket).await?; + let include_lifecycle_object_info = lifecycle_expiry_context.is_some(); loop { self.verify_replacement_identity_fence("page scan").await?; // Get one page of object versions let (objects, next_token, is_truncated) = if use_disk_walk { self.storage - .list_versions_for_heal_page_disk_walk(set_disk_id, bucket, "", continuation_token.as_deref()) + .list_versions_for_heal_page_disk_walk( + set_disk_id, + bucket, + "", + continuation_token.as_deref(), + include_lifecycle_object_info, + ) .await? } else { self.storage - .list_objects_for_heal_page(bucket, "", continuation_token.as_deref()) + .list_objects_for_heal_page(bucket, "", continuation_token.as_deref(), include_lifecycle_object_info) .await? }; let page_is_empty = objects.is_empty(); @@ -736,6 +763,7 @@ impl ErasureSetHealer { let page_resume_index = *current_object_index; let semaphore = Arc::new(Semaphore::new(page_concurrency_limit)); let mut page_tasks = FuturesUnordered::new(); + let mut completed_in_page = 0usize; // Capture the last version identity of this page for the anti-loop guard. let page_last = objects.last().map(|item| (item.name.clone(), item.version_id.clone())); @@ -751,6 +779,75 @@ impl ErasureSetHealer { continue; } + if should_skip_new_version(item.mod_time_unix_nanos, started_at_secs) { + checkpoint_manager.add_processed_object(key).await?; + *processed_objects = processed_objects.saturating_add(1); + completed_in_page = completed_in_page.saturating_add(1); + counter!("rustfs_heal_skipped_new_versions_total").increment(1); + { + let mut progress = self.progress.write().await; + progress.record_skipped_new_version(); + progress.set_current_object(Some(format!("skipped_new: {bucket}/{}", item.name))); + progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed); + } + debug!( + target: "rustfs::heal::erasure_healer", + event = EVENT_HEAL_ERASURE_OBJECT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_ERASURE_HEALER, + set_disk_id, + bucket, + object = %item.name, + version_id = ?item.version_id, + state = "skipped_new_version", + "Erasure set object version skipped because it was written after heal started" + ); + if completed_in_page.is_multiple_of(100) { + checkpoint_manager.update_position(bucket_index, page_resume_index).await?; + } + continue; + } + + if let Some(context) = lifecycle_expiry_context.as_ref() + && self + .storage + .enqueue_heal_lifecycle_expiry( + context, + bucket, + &item.name, + item.version_id.as_deref(), + item.lifecycle_object_info.as_ref(), + ) + .await? + { + checkpoint_manager.add_processed_object(key).await?; + *processed_objects = processed_objects.saturating_add(1); + completed_in_page = completed_in_page.saturating_add(1); + counter!("rustfs_heal_skipped_ilm_expired_total").increment(1); + { + let mut progress = self.progress.write().await; + progress.record_skipped_ilm_expired(); + progress.set_current_object(Some(format!("skipped_ilm: {bucket}/{}", item.name))); + progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed); + } + debug!( + target: "rustfs::heal::erasure_healer", + event = EVENT_HEAL_ERASURE_OBJECT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_ERASURE_HEALER, + set_disk_id, + bucket, + object = %item.name, + version_id = ?item.version_id, + state = "skipped_ilm_expired", + "Erasure set object version skipped because lifecycle expiry was queued" + ); + if completed_in_page.is_multiple_of(100) { + checkpoint_manager.update_position(bucket_index, page_resume_index).await?; + } + continue; + } + resume_manager .set_current_item(Some(bucket.to_string()), Some(item.name.clone())) .await?; @@ -777,7 +874,7 @@ impl ErasureSetHealer { let _permit = match permit { Ok(permit) => permit, - Err(err) => return (dedup_key, object_name, version_id, Err(err)), + Err(err) => return (dedup_key, object_name, version_id, (0, Err(err))), }; let _in_flight_guard = PageConcurrencyGuard::new(in_flight, set_label); @@ -788,7 +885,7 @@ impl ErasureSetHealer { // recorded as skipped-ok rather than failed. The delete-marker // vs data path is chosen internally in ops/heal.rs. let result = if cancel_token.is_cancelled() { - Err(Error::TaskCancelled) + (0, Err(Error::TaskCancelled)) } else { match storage .heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts) @@ -797,8 +894,9 @@ impl ErasureSetHealer { Ok((result, None)) if target_outcomes_complete(&result, &target_endpoints) => { + let object_size = result_object_size_u64(&result); if !replacement_commit_evidence_required { - Ok(true) + (object_size, Ok(true)) } else { match storage .replacement_targets_have_version( @@ -810,27 +908,42 @@ impl ErasureSetHealer { ) .await { - Ok(true) => Ok(true), - Ok(false) => Err(Error::transient_skip(format!( + Ok(true) => (object_size, Ok(true)), + Ok(false) => (object_size, Err(Error::transient_skip(format!( "Skipped heal for {bucket_name}/{object_name} because replacement target readback did not confirm the committed version" - ))), - Err(err) => Err(Error::transient_skip(format!( + )))), + Err(err) => (object_size, Err(Error::transient_skip(format!( "Skipped heal for {bucket_name}/{object_name} because replacement target readback failed: {err}" - ))), + )))), } } - } - Ok((_result, None)) if !target_endpoints.is_empty() => Err(Error::transient_skip(format!( - "Skipped heal for {bucket_name}/{object_name} because a replacement target was not committed" - ))), - Ok((_result, None)) => Ok(true), - Ok((_, Some(err))) if is_missing_object_dir_heal_result(&object_name, &err) => Ok(false), - Ok((_, Some(err))) | Err(err) => match Self::classify_heal_object_error(&err) { - HealObjectOutcome::Absent => Ok(false), - HealObjectOutcome::Transient => Err(Error::transient_skip(format!( - "Skipped heal for {bucket_name}/{object_name} due to transient error: {err}" + }, + Ok((result, None)) if !target_endpoints.is_empty() => ( + result_object_size_u64(&result), + Err(Error::transient_skip(format!( + "Skipped heal for {bucket_name}/{object_name} because a replacement target was not committed" ))), - HealObjectOutcome::Failed => Err(err), + ), + Ok((result, None)) => (result_object_size_u64(&result), Ok(true)), + Ok((result, Some(err))) if is_missing_object_dir_heal_result(&object_name, &err) => { + (result_object_size_u64(&result), Ok(false)) + } + Ok((result, Some(err))) => { + let object_size = result_object_size_u64(&result); + match Self::classify_heal_object_error(&err) { + HealObjectOutcome::Absent => (object_size, Ok(false)), + HealObjectOutcome::Transient => (object_size, Err(Error::transient_skip(format!( + "Skipped heal for {bucket_name}/{object_name} due to transient error: {err}" + )))), + HealObjectOutcome::Failed => (object_size, Err(err)), + } + } + Err(err) => match Self::classify_heal_object_error(&err) { + HealObjectOutcome::Absent => (0, Ok(false)), + HealObjectOutcome::Transient => (0, Err(Error::transient_skip(format!( + "Skipped heal for {bucket_name}/{object_name} due to transient error: {err}" + )))), + HealObjectOutcome::Failed => (0, Err(err)), }, } }; @@ -839,11 +952,12 @@ impl ErasureSetHealer { }); } - let mut completed_in_page = 0usize; while let Some((key, object, version_id, result)) = page_tasks.next().await { + let (object_size, result) = result; match result { Ok(true) => { *successful_objects += 1; + bytes_processed = bytes_processed.saturating_add(object_size); checkpoint_manager.add_processed_object(key).await?; debug!( target: "rustfs::heal::erasure_healer", @@ -861,6 +975,7 @@ impl ErasureSetHealer { Ok(false) => { checkpoint_manager.add_processed_object(key).await?; *successful_objects += 1; + bytes_processed = bytes_processed.saturating_add(object_size); debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_OBJECT_STATE, @@ -877,6 +992,7 @@ impl ErasureSetHealer { Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err), Err(Error::TransientSkip { message }) => { *skipped_objects += 1; + bytes_processed = bytes_processed.saturating_add(object_size); checkpoint_manager.add_skipped_object(key).await?; demote_to_debug_when!(!take_failure_log_sample(&mut transient_skip_samples_logged), warn, target: "rustfs::heal::erasure_healer", { event = EVENT_HEAL_ERASURE_OBJECT_STATE, @@ -893,6 +1009,7 @@ impl ErasureSetHealer { } Err(err) => { *failed_objects += 1; + bytes_processed = bytes_processed.saturating_add(object_size); checkpoint_manager.add_failed_object(key).await?; demote_to_debug_when!(!take_failure_log_sample(&mut failure_samples_logged), warn, target: "rustfs::heal::erasure_healer", { event = EVENT_HEAL_ERASURE_OBJECT_STATE, @@ -911,6 +1028,11 @@ impl ErasureSetHealer { *processed_objects += 1; completed_in_page += 1; + { + let mut progress = self.progress.write().await; + progress.set_current_object(Some(format!("{bucket}/{object}"))); + progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed); + } if completed_in_page.is_multiple_of(100) { checkpoint_manager.update_position(bucket_index, page_resume_index).await?; @@ -964,7 +1086,9 @@ impl ErasureSetHealer { progress.objects_scanned = state.total_objects; progress.objects_healed = state.successful_objects; progress.objects_failed = state.failed_objects; - progress.bytes_processed = 0; // set to 0 for now, can be extended later + progress.bytes_processed = 0; // Resume state tracks object counts, not byte counters. + progress.start_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.start_time)); + progress.last_update_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.last_update)); progress.set_current_object(state.current_object.clone()); } } @@ -1135,13 +1259,15 @@ mod resume_loop_tests { //! that emits programmable multi-version pages. These exercise the real loop //! logic (cursor seeding, per-version dedup, anti-loop guard, absence //! handling) — not merely a mock's own output. - use super::{ErasureSetHealer, target_outcomes_complete}; + use super::{ + ErasureSetHealer, NANOS_PER_SECOND, NEW_VERSION_SKIP_GRACE_SECS, should_skip_new_version, target_outcomes_complete, + }; use crate::heal::progress::HealProgress; use crate::heal::resume::{ CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils, compose_key, }; - use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo, HealStorageAPI}; + use crate::heal::storage::{DiskStatus, HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI}; use crate::heal::storage_api::status::BucketInfo; use crate::heal::{ BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, RUSTFS_META_BUCKET, new_disk, @@ -1149,7 +1275,7 @@ mod resume_loop_tests { use crate::{Error, Result}; use rustfs_common::heal_channel::{HealOpts, HealRequestSource}; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos}; - use std::collections::{HashMap, VecDeque}; + use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use tempfile::TempDir; @@ -1160,10 +1286,37 @@ mod resume_loop_tests { HealListItem { name: name.to_string(), version_id: version.map(str::to_string), + mod_time_unix_nanos: None, + lifecycle_object_info: None, is_delete_marker: delete_marker, } } + fn item_with_mod_time(name: &str, version: Option<&str>, mod_time_secs: u64) -> HealListItem { + HealListItem { + name: name.to_string(), + version_id: version.map(str::to_string), + mod_time_unix_nanos: Some(i128::from(mod_time_secs).saturating_mul(NANOS_PER_SECOND)), + lifecycle_object_info: None, + is_delete_marker: false, + } + } + + #[test] + fn new_version_filter_respects_grace_boundary() { + let started_at = 1_700_000_000; + + assert!(!should_skip_new_version(None, started_at)); + assert!(!should_skip_new_version( + Some(i128::from(started_at + NEW_VERSION_SKIP_GRACE_SECS).saturating_mul(NANOS_PER_SECOND)), + started_at, + )); + assert!(should_skip_new_version( + Some(i128::from(started_at + NEW_VERSION_SKIP_GRACE_SECS + 1).saturating_mul(NANOS_PER_SECOND)), + started_at, + )); + } + #[test] fn target_outcomes_require_each_requested_endpoint_once_and_ok() { let result = HealResultItem { @@ -1246,8 +1399,10 @@ mod resume_loop_tests { /// Target-specific physical readback evidence per `compose_key`; the /// fake models a healthy backend unless a test explicitly revokes it. replacement_commit_evidence: Mutex>, + lifecycle_expired: Mutex>, /// every heal_object call recorded as (name, version_id) heal_calls: Mutex)>>, + list_include_lifecycle_object_info: Mutex>, replacement_target_identity_sequences: Mutex>>, fail_listing: AtomicBool, } @@ -1274,9 +1429,15 @@ mod resume_loop_tests { .unwrap() .insert(compose_key(name, version), ReplacementCommitEvidence::Error(message.to_string())); } + fn set_lifecycle_expired(&self, name: &str, version: Option<&str>) { + self.lifecycle_expired.lock().unwrap().insert(compose_key(name, version)); + } fn calls(&self) -> Vec<(String, Option)> { self.heal_calls.lock().unwrap().clone() } + fn list_include_lifecycle_object_info_calls(&self) -> Vec { + self.list_include_lifecycle_object_info.lock().unwrap().clone() + } fn fail_listing(&self) { self.fail_listing.store(true, Ordering::SeqCst); } @@ -1330,6 +1491,23 @@ mod resume_loop_tests { async fn get_object_checksum(&self, _b: &str, _o: &str) -> Result> { Ok(None) } + async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result> { + Ok((!self.lifecycle_expired.lock().unwrap().is_empty()).then(HealLifecycleExpiryContext::test)) + } + async fn enqueue_heal_lifecycle_expiry( + &self, + _context: &HealLifecycleExpiryContext, + _bucket: &str, + object: &str, + version_id: Option<&str>, + _object_info: Option<&HealObjectInfo>, + ) -> Result { + Ok(self + .lifecycle_expired + .lock() + .unwrap() + .contains(&compose_key(object, version_id))) + } async fn heal_object( &self, _bucket: &str, @@ -1386,7 +1564,12 @@ mod resume_loop_tests { _bucket: &str, _prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { + self.list_include_lifecycle_object_info + .lock() + .unwrap() + .push(include_lifecycle_object_info); if self.fail_listing.load(Ordering::SeqCst) { return Err(Error::other("injected listing failure")); } @@ -1476,6 +1659,7 @@ mod resume_loop_tests { /// Drive one bucket heal pass; returns (processed, successful, failed, skipped, result). async fn run(env: &Env) -> (u64, u64, u64, u64, Result<()>) { + let state = env.resume.get_state().await; let mut current_object_index = 0usize; let mut processed = 0u64; let mut successful = 0u64; @@ -1494,6 +1678,7 @@ mod resume_loop_tests { &mut skipped, &env.resume, &env.checkpoint, + state.start_time, ) .await; (processed, successful, failed, skipped, result) @@ -1559,6 +1744,7 @@ mod resume_loop_tests { let mut successful = 0; let mut failed = 0; let mut skipped = 0; + let started_at = env.resume.get_state().await.start_time; let error = healer .heal_bucket_with_resume( @@ -1572,6 +1758,7 @@ mod resume_loop_tests { &mut skipped, &env.resume, &env.checkpoint, + started_at, ) .await .expect_err("a remounted target must not begin a new page scan"); @@ -1641,6 +1828,109 @@ mod resume_loop_tests { assert_eq!(skipped, 0); } + #[tokio::test] + async fn erasure_set_progress_accumulates_healed_object_bytes() { + let env = make_env().await; + env.storage.set_page( + None, + Page { + items: vec![item("first", Some("v1"), false), item("second", Some("v2"), false)], + next: None, + truncated: false, + }, + ); + env.storage.set_result( + "first", + Some("v1"), + HealResultItem { + object_size: 1024, + ..Default::default() + }, + ); + env.storage.set_result( + "second", + Some("v2"), + HealResultItem { + object_size: 2048, + ..Default::default() + }, + ); + + let (processed, successful, failed, skipped, result) = run(&env).await; + + result.expect("page heal should succeed"); + assert_eq!(processed, 2); + assert_eq!(successful, 2); + assert_eq!(failed, 0); + assert_eq!(skipped, 0); + let progress = env.healer.progress.read().await; + assert_eq!(progress.objects_scanned, 2); + assert_eq!(progress.objects_healed, 2); + assert_eq!(progress.objects_failed, 0); + assert_eq!(progress.bytes_processed, 3072); + assert!(matches!(progress.current_object.as_deref(), Some("b/first" | "b/second"))); + } + + #[tokio::test] + async fn erasure_set_skips_versions_written_after_heal_started() { + let env = make_env().await; + let started_at = env.resume.get_state().await.start_time; + env.storage.set_page( + None, + Page { + items: vec![ + item_with_mod_time("old", Some("v1"), started_at + NEW_VERSION_SKIP_GRACE_SECS), + item_with_mod_time("new", Some("v2"), started_at + NEW_VERSION_SKIP_GRACE_SECS + 1), + ], + next: None, + truncated: false, + }, + ); + + let (processed, successful, failed, skipped, result) = run(&env).await; + + result.expect("page heal should succeed"); + assert_eq!(processed, 2); + assert_eq!(successful, 1); + assert_eq!(failed, 0); + assert_eq!(skipped, 0); + assert_eq!(env.storage.calls(), vec![("old".to_string(), Some("v1".to_string()))]); + let progress = env.healer.progress.read().await; + assert_eq!(progress.skipped_new_versions, 1); + assert_eq!(progress.objects_scanned, 2); + assert_eq!(progress.objects_healed, 1); + assert_eq!(progress.objects_failed, 0); + } + + #[tokio::test] + async fn erasure_set_skips_versions_queued_for_lifecycle_expiry() { + let env = make_env().await; + env.storage.set_page( + None, + Page { + items: vec![item("expired", Some("v1"), false), item("kept", Some("v2"), false)], + next: None, + truncated: false, + }, + ); + env.storage.set_lifecycle_expired("expired", Some("v1")); + + let (processed, successful, failed, skipped, result) = run(&env).await; + + result.expect("page heal should succeed"); + assert_eq!(processed, 2); + assert_eq!(successful, 1); + assert_eq!(failed, 0); + assert_eq!(skipped, 0); + assert_eq!(env.storage.calls(), vec![("kept".to_string(), Some("v2".to_string()))]); + assert_eq!(env.storage.list_include_lifecycle_object_info_calls(), vec![true]); + let progress = env.healer.progress.read().await; + assert_eq!(progress.skipped_ilm_expired, 1); + assert_eq!(progress.objects_scanned, 2); + assert_eq!(progress.objects_healed, 1); + assert_eq!(progress.objects_failed, 0); + } + #[tokio::test] async fn bucket_listing_failure_does_not_mark_set_completed() { let env = make_env().await; diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 79ba6b169..66b8f637f 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -2385,8 +2385,27 @@ impl HealManager { snapshot.objects_scanned = snapshot.objects_scanned.saturating_add(progress.objects_scanned); snapshot.objects_healed = snapshot.objects_healed.saturating_add(progress.objects_healed); snapshot.objects_failed = snapshot.objects_failed.saturating_add(progress.objects_failed); + snapshot.skipped_new_versions = snapshot.skipped_new_versions.saturating_add(progress.skipped_new_versions); + snapshot.skipped_ilm_expired = snapshot.skipped_ilm_expired.saturating_add(progress.skipped_ilm_expired); + snapshot.objects_total_count = snapshot.objects_total_count.saturating_add(progress.objects_total_count); + snapshot.objects_total_size = snapshot.objects_total_size.saturating_add(progress.objects_total_size); snapshot.bytes_processed = snapshot.bytes_processed.saturating_add(progress.bytes_processed); + snapshot.start_time = match (snapshot.start_time, progress.start_time) { + (Some(current), Some(next)) => Some(current.min(next)), + (None, next) => next, + (current, None) => current, + }; + snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) { + (Some(current), Some(next)) => Some(current.max(next)), + (None, next) => next, + (current, None) => current, + }; + if progress.current_object.is_some() { + snapshot.current_object = progress.current_object; + } } + snapshot.refresh_progress_percentage(); + snapshot.refresh_estimated_completion_time(); Some(snapshot) } @@ -3208,6 +3227,7 @@ impl HealManager { } else { completed_task.get_status().await }; + let completed_progress = completed_task.get_progress().await; let completed_status_entry = CompletedHealStatus { heal_type: completed_task.heal_type.clone(), status: completed_status.clone(), @@ -3223,6 +3243,7 @@ impl HealManager { match completed_status { HealTaskStatus::Completed => { stats.update_task_completion(true); + stats.add_healed_objects(completed_progress.objects_healed, completed_progress.bytes_processed); } HealTaskStatus::Retrying { .. } => {} _ => { @@ -3749,6 +3770,7 @@ mod tests { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { Ok((Vec::new(), None, false)) } @@ -5396,6 +5418,8 @@ mod tests { )); { let mut progress = first.progress.write().await; + progress.start_time = Some(SystemTime::now() - Duration::from_secs(20)); + progress.set_total_baseline(12, 8192); progress.update_progress(7, 3, 1, 4096); } @@ -5405,6 +5429,8 @@ mod tests { )); { let mut progress = second.progress.write().await; + progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); + progress.set_total_baseline(8, 4096); progress.update_progress(11, 5, 2, 2048); } @@ -5419,7 +5445,11 @@ mod tests { assert_eq!(progress.objects_scanned, 18); assert_eq!(progress.objects_healed, 8); assert_eq!(progress.objects_failed, 3); + assert_eq!(progress.objects_total_count, 20); + assert_eq!(progress.objects_total_size, 12288); assert_eq!(progress.bytes_processed, 6144); + assert!((progress.progress_percentage - 50.0).abs() < 0.001); + assert!(progress.estimated_completion_time.is_some()); } #[tokio::test] diff --git a/crates/heal/src/heal/progress.rs b/crates/heal/src/heal/progress.rs index cb602b5de..981aa01fe 100644 --- a/crates/heal/src/heal/progress.rs +++ b/crates/heal/src/heal/progress.rs @@ -13,7 +13,7 @@ // limitations under the License. use serde::{Deserialize, Serialize}; -use std::time::SystemTime; +use std::time::{Duration, SystemTime}; #[derive(Debug, Default, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -24,6 +24,14 @@ pub struct HealProgress { pub objects_healed: u64, /// Objects failed pub objects_failed: u64, + /// Versions skipped because they were written after this heal started + pub skipped_new_versions: u64, + /// Versions skipped because lifecycle already selected them for expiry + pub skipped_ilm_expired: u64, + /// Baseline object count from the latest complete usage snapshot + pub objects_total_count: u64, + /// Baseline object bytes from the latest complete usage snapshot + pub objects_total_size: u64, /// Bytes processed pub bytes_processed: u64, /// Current object @@ -54,10 +62,56 @@ impl HealProgress { self.bytes_processed = bytes; self.last_update_time = Some(SystemTime::now()); - // calculate progress percentage - let total = scanned + healed + failed; + self.refresh_progress_percentage(); + self.refresh_estimated_completion_time(); + } + + pub fn set_total_baseline(&mut self, objects_total_count: u64, objects_total_size: u64) { + self.objects_total_count = objects_total_count; + self.objects_total_size = objects_total_size; + self.last_update_time = Some(SystemTime::now()); + self.refresh_progress_percentage(); + self.refresh_estimated_completion_time(); + } + + pub fn record_skipped_new_version(&mut self) { + self.skipped_new_versions = self.skipped_new_versions.saturating_add(1); + self.last_update_time = Some(SystemTime::now()); + self.refresh_progress_percentage(); + self.refresh_estimated_completion_time(); + } + + pub fn record_skipped_ilm_expired(&mut self) { + self.skipped_ilm_expired = self.skipped_ilm_expired.saturating_add(1); + self.last_update_time = Some(SystemTime::now()); + self.refresh_progress_percentage(); + self.refresh_estimated_completion_time(); + } + + fn completed_for_baseline(&self) -> u64 { + self.objects_healed + .saturating_add(self.objects_failed) + .saturating_add(self.skipped_new_versions) + .saturating_add(self.skipped_ilm_expired) + } + + pub(crate) fn refresh_progress_percentage(&mut self) { + if self.objects_total_size > 0 { + self.progress_percentage = ((self.bytes_processed as f64 / self.objects_total_size as f64) * 100.0).min(100.0); + return; + } + if self.objects_total_count > 0 { + let completed = self.completed_for_baseline(); + self.progress_percentage = ((completed as f64 / self.objects_total_count as f64) * 100.0).min(100.0); + return; + } + + let total = self + .objects_scanned + .saturating_add(self.objects_healed) + .saturating_add(self.objects_failed); if total > 0 { - self.progress_percentage = (healed as f64 / total as f64) * 100.0; + self.progress_percentage = (self.objects_healed as f64 / total as f64) * 100.0; } } @@ -66,9 +120,36 @@ impl HealProgress { self.last_update_time = Some(SystemTime::now()); } + pub fn refresh_estimated_completion_time(&mut self) { + let Some(start_time) = self.start_time else { + self.estimated_completion_time = None; + return; + }; + if self.is_completed() || !(0.0..100.0).contains(&self.progress_percentage) || self.bytes_processed == 0 { + self.estimated_completion_time = None; + return; + } + + let elapsed = match SystemTime::now().duration_since(start_time) { + Ok(elapsed) if !elapsed.is_zero() => elapsed, + _ => { + self.estimated_completion_time = None; + return; + } + }; + let estimated_total_secs = elapsed.as_secs_f64() * 100.0 / self.progress_percentage; + self.estimated_completion_time = start_time.checked_add(Duration::from_secs_f64(estimated_total_secs)); + } + pub fn is_completed(&self) -> bool { - self.progress_percentage >= 100.0 - || self.objects_scanned > 0 && self.objects_healed + self.objects_failed >= self.objects_scanned + if self.progress_percentage >= 100.0 { + return true; + } + if self.objects_total_count > 0 || self.objects_total_size > 0 { + return false; + } + + self.objects_scanned > 0 && self.objects_healed.saturating_add(self.objects_failed) >= self.objects_scanned } pub fn get_success_rate(&self) -> f64 { @@ -158,6 +239,10 @@ mod tests { assert_eq!(progress.objects_scanned, 0); assert_eq!(progress.objects_healed, 0); assert_eq!(progress.objects_failed, 0); + assert_eq!(progress.skipped_new_versions, 0); + assert_eq!(progress.skipped_ilm_expired, 0); + assert_eq!(progress.objects_total_count, 0); + assert_eq!(progress.objects_total_size, 0); assert_eq!(progress.bytes_processed, 0); assert_eq!(progress.progress_percentage, 0.0); assert!(progress.start_time.is_some()); @@ -181,6 +266,73 @@ mod tests { assert!(progress.last_update_time.is_some()); } + #[test] + fn test_heal_progress_estimates_completion_time_from_progress() { + let mut progress = HealProgress::new(); + progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); + + progress.update_progress(100, 25, 0, 4096); + + let eta = progress + .estimated_completion_time + .expect("partial byte progress should estimate completion"); + assert!(eta > SystemTime::now()); + } + + #[test] + fn test_heal_progress_uses_byte_baseline_for_percentage() { + let mut progress = HealProgress::new(); + progress.set_total_baseline(10, 8192); + + progress.update_progress(100, 25, 0, 4096); + + assert!((progress.progress_percentage - 50.0).abs() < 0.001); + } + + #[test] + fn test_heal_progress_uses_object_baseline_when_bytes_unknown() { + let mut progress = HealProgress::new(); + progress.set_total_baseline(10, 0); + + progress.update_progress(100, 3, 2, 0); + + assert!((progress.progress_percentage - 50.0).abs() < 0.001); + } + + #[test] + fn test_heal_progress_counts_skipped_versions_for_object_baseline() { + let mut progress = HealProgress::new(); + progress.set_total_baseline(10, 0); + + progress.update_progress(100, 3, 2, 0); + progress.record_skipped_new_version(); + + assert_eq!(progress.skipped_new_versions, 1); + assert!((progress.progress_percentage - 60.0).abs() < 0.001); + } + + #[test] + fn test_heal_progress_does_not_estimate_completion_without_bytes() { + let mut progress = HealProgress::new(); + progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); + + progress.update_progress(100, 25, 0, 0); + + assert!(progress.estimated_completion_time.is_none()); + } + + #[test] + fn test_heal_progress_with_baseline_is_not_completed_by_processed_count() { + let mut progress = HealProgress::new(); + progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); + progress.set_total_baseline(10, 8192); + + progress.update_progress(1, 1, 0, 1024); + + assert!(!progress.is_completed()); + assert!(progress.estimated_completion_time.is_some()); + } + #[test] fn test_heal_progress_update_progress_zero_total() { let mut progress = HealProgress::new(); @@ -251,6 +403,8 @@ mod tests { assert_eq!(json["objectsScanned"], 10); assert_eq!(json["objectsHealed"], 8); assert_eq!(json["objectsFailed"], 2); + assert_eq!(json["skippedNewVersions"], 0); + assert_eq!(json["skippedIlmExpired"], 0); assert_eq!(json["bytesProcessed"], 1024); assert_eq!(json["currentObject"], "test-bucket/test-object"); assert!(json["progressPercentage"].is_number()); diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index f5101ac1c..fbe51f050 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -22,6 +22,7 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; use tracing::{debug, error, warn}; +use super::storage_api::owner::{EcstoreHealLifecycleExpiryContext, ecstore_load_admin_data_usage_from_backend_cached}; use super::storage_api::storage::{ BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _, ObjectOperations as _, StorageAdminApi, @@ -29,6 +30,37 @@ use super::storage_api::storage::{ use super::{DiskStore, ECStore, Endpoint, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity}; pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader}; +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct HealBucketUsageBaseline { + pub objects_count: u64, + pub bytes: u64, +} + +pub struct HealLifecycleExpiryContext { + inner: HealLifecycleExpiryContextInner, +} + +enum HealLifecycleExpiryContextInner { + Ecstore(EcstoreHealLifecycleExpiryContext), + #[allow(dead_code)] + Test, +} + +impl HealLifecycleExpiryContext { + fn ecstore(inner: EcstoreHealLifecycleExpiryContext) -> Self { + Self { + inner: HealLifecycleExpiryContextInner::Ecstore(inner), + } + } + + #[cfg(test)] + pub(crate) fn test() -> Self { + Self { + inner: HealLifecycleExpiryContextInner::Test, + } + } +} + const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_STORAGE: &str = "storage"; const EVENT_HEAL_STORAGE_OBJECT_IO: &str = "heal_storage_object_io"; @@ -272,6 +304,10 @@ pub struct HealListItem { pub name: String, /// normalized version id (`None` when the version is nil/absent) pub version_id: Option, + /// version modification time as Unix nanoseconds + pub mod_time_unix_nanos: Option, + /// object snapshot for lifecycle evaluation + pub lifecycle_object_info: Option, /// whether this version is a delete marker (observability only) pub is_delete_marker: bool, } @@ -329,6 +365,28 @@ pub trait HealStorageAPI: Send + Sync { /// Get bucket info async fn get_bucket_info(&self, bucket: &str) -> Result>; + /// Aggregate usage-cache baselines for the requested buckets. + async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result> { + Ok(None) + } + + /// Load per-bucket lifecycle expiry context for heal skips. + async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result> { + Ok(None) + } + + /// Queue lifecycle expiry for a version that heal can skip. + async fn enqueue_heal_lifecycle_expiry( + &self, + _context: &HealLifecycleExpiryContext, + _bucket: &str, + _object: &str, + _version_id: Option<&str>, + _object_info: Option<&HealObjectInfo>, + ) -> Result { + Ok(false) + } + /// Fix bucket metadata async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()>; @@ -409,6 +467,7 @@ pub trait HealStorageAPI: Send + Sync { bucket: &str, prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)>; /// List versions for healing via a per-erasure-set DISK-WALK union enumerator @@ -427,8 +486,10 @@ pub trait HealStorageAPI: Send + Sync { bucket: &str, prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { - self.list_objects_for_heal_page(bucket, prefix, continuation_token).await + self.list_objects_for_heal_page(bucket, prefix, continuation_token, include_lifecycle_object_info) + .await } /// Get disk for resume functionality. @@ -1021,6 +1082,85 @@ impl HealStorageAPI for ECStoreHealStorage { } } + async fn erasure_set_usage_baseline(&self, buckets: &[String]) -> Result> { + if buckets.is_empty() { + return Ok(None); + } + + let info = match ecstore_load_admin_data_usage_from_backend_cached(self.ecstore.clone()).await { + Ok(info) if info.is_complete_bucket_usage_snapshot() => info, + Ok(_) | Err(_) => return Ok(None), + }; + + let mut baseline = HealBucketUsageBaseline::default(); + for bucket in buckets { + if let Some(usage) = info.buckets_usage.get(bucket) { + baseline.objects_count = baseline.objects_count.saturating_add(usage.objects_count); + baseline.bytes = baseline.bytes.saturating_add(usage.size); + } + } + + Ok(Some(baseline)) + } + + async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result> { + match self.ecstore.load_heal_lifecycle_expiry_context(bucket).await { + Ok(Some(context)) => Ok(Some(HealLifecycleExpiryContext::ecstore(context))), + Ok(None) => Ok(None), + Err(err) => { + debug!( + target: "rustfs::heal::storage", + event = EVENT_HEAL_STORAGE_ADMIN_OP, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_STORAGE, + operation = "load_heal_lifecycle_expiry_context", + bucket, + result = "failed", + error = %err, + "Heal storage lifecycle expiry context load failed" + ); + Ok(None) + } + } + } + + async fn enqueue_heal_lifecycle_expiry( + &self, + context: &HealLifecycleExpiryContext, + bucket: &str, + object: &str, + version_id: Option<&str>, + object_info: Option<&HealObjectInfo>, + ) -> Result { + let context = match &context.inner { + HealLifecycleExpiryContextInner::Ecstore(context) => context, + HealLifecycleExpiryContextInner::Test => return Ok(false), + }; + match self + .ecstore + .enqueue_heal_lifecycle_expiry(context, bucket, object, version_id, object_info) + .await + { + Ok(queued) => Ok(queued), + Err(err) => { + debug!( + target: "rustfs::heal::storage", + event = EVENT_HEAL_STORAGE_ADMIN_OP, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_STORAGE, + operation = "enqueue_heal_lifecycle_expiry", + bucket, + object, + version_id = ?version_id, + result = "failed", + error = %err, + "Heal storage lifecycle expiry check failed" + ); + Ok(false) + } + } + } + async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()> { debug!( target: "rustfs::heal::storage", @@ -1436,7 +1576,7 @@ impl HealStorageAPI for ECStoreHealStorage { loop { let (page_objects, next_token, is_truncated) = self - .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref()) + .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false) .await?; all_objects.extend(page_objects); @@ -1471,6 +1611,7 @@ impl HealStorageAPI for ECStoreHealStorage { bucket: &str, prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { debug!( target: "rustfs::heal::storage", @@ -1522,10 +1663,19 @@ impl HealStorageAPI for ECStoreHealStorage { let page_objects: Vec = list_info .objects .into_iter() - .map(|obj| HealListItem { - name: obj.name, - version_id: obj.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()), - is_delete_marker: obj.delete_marker, + .map(|mut obj| { + obj.version_id = obj.version_id.filter(|u| !u.is_nil()); + let version_id = obj.version_id.map(|u| u.to_string()); + let mod_time_unix_nanos = obj.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()); + let is_delete_marker = obj.delete_marker; + let lifecycle_object_info = include_lifecycle_object_info.then(|| obj.clone()); + HealListItem { + name: obj.name, + version_id, + mod_time_unix_nanos, + lifecycle_object_info, + is_delete_marker, + } }) .collect(); let page_count = page_objects.len(); @@ -1562,6 +1712,7 @@ impl HealStorageAPI for ECStoreHealStorage { bucket: &str, prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { // Per-page bounds for the disk-walk union enumerator. Objects are atomic // (never split across pages), so version_budget only bounds how many @@ -1590,7 +1741,16 @@ impl HealStorageAPI for ECStoreHealStorage { let (versions, next_forward, is_truncated) = self .ecstore - .heal_walk_versions_page(pool_idx, set_idx, bucket, prefix, forward_to.as_deref(), BATCH_OBJECTS, VERSION_BUDGET) + .heal_walk_versions_page( + pool_idx, + set_idx, + bucket, + prefix, + forward_to.as_deref(), + BATCH_OBJECTS, + VERSION_BUDGET, + include_lifecycle_object_info, + ) .await .map_err(|e| { error!( @@ -1614,6 +1774,8 @@ impl HealStorageAPI for ECStoreHealStorage { .map(|v| HealListItem { name: v.name, version_id: v.version_id, + mod_time_unix_nanos: v.mod_time_unix_nanos, + lifecycle_object_info: v.lifecycle_object_info, is_delete_marker: v.is_delete_marker, }) .collect(); diff --git a/crates/heal/src/heal/storage_api.rs b/crates/heal/src/heal/storage_api.rs index 417998e39..ff722b418 100644 --- a/crates/heal/src/heal/storage_api.rs +++ b/crates/heal/src/heal/storage_api.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub(crate) use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME; +pub(crate) use rustfs_ecstore::api::data_usage::{ + DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME, + load_admin_data_usage_from_backend_cached as ecstore_load_admin_data_usage_from_backend_cached, +}; pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint; pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult}; pub(crate) use rustfs_ecstore::api::disk::{ @@ -25,7 +28,9 @@ pub(crate) use rustfs_ecstore::api::disk::{ pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk}; pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError}; pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read; -pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore; +pub(crate) use rustfs_ecstore::api::storage::{ + ECStore as EcstoreStore, HealLifecycleExpiryContext as EcstoreHealLifecycleExpiryContext, +}; use rustfs_storage_api as storage_contracts; pub(crate) mod owner { @@ -34,8 +39,8 @@ pub(crate) mod owner { pub(crate) use super::{ ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, - EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, - ecstore_local_disk_map_read, + EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreHealLifecycleExpiryContext, + EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read, }; #[cfg(test)] diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index 62123c418..f6472eb65 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -19,11 +19,12 @@ use crate::heal::{ resume::{ CheckpointManager, ReplacementPhase, ReplacementTargetIdentity, ResumeManager, replacement_target_identities_match, }, - storage::{HealStorageAPI, next_heal_listing_token}, + storage::{HealBucketUsageBaseline, HealStorageAPI, next_heal_listing_token}, }; use crate::{Error, Result}; use metrics::{counter, histogram}; use rustfs_common::heal_channel::{HealOpts, HealRequestSource, HealScanMode}; +use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit}; use rustfs_madmin::heal_commands::HealResultItem; use rustfs_utils::path::SLASH_SEPARATOR; use serde::{Deserialize, Serialize}; @@ -178,6 +179,17 @@ pub enum HealPriority { Urgent = 3, } +impl HealPriority { + fn as_str(self) -> &'static str { + match self { + Self::Low => "low", + Self::Normal => "normal", + Self::High => "high", + Self::Urgent => "urgent", + } + } +} + /// Heal options #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HealOptions { @@ -498,6 +510,61 @@ impl HealTask { } } + fn emit_trace_task_state(&self, state: &'static str, duration: Duration, error: Option<&Error>) { + trace_emit(|| { + let mut event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask) + .with_duration(duration) + .with_attr("task_id", self.id.as_str()) + .with_attr("heal_type", self.heal_type.log_kind()) + .with_attr("state", state) + .with_attr("source", self.source.as_str()) + .with_attr("priority", self.priority.as_str()) + .with_attr("retry_attempts", u64::from(self.retry_attempts)) + .with_attr("dry_run", self.options.dry_run); + + event = match &self.heal_type { + HealType::Cluster => event, + HealType::Object { + bucket, + object, + version_id, + } => { + let event = event.with_bucket(bucket.as_str()).with_object(object.as_str()); + match version_id { + Some(version_id) => event.with_attr("version_id", version_id.as_str()), + None => event, + } + } + HealType::Bucket { bucket } => event.with_bucket(bucket.as_str()), + HealType::Prefix { bucket, prefix } => event.with_bucket(bucket.as_str()).with_object(prefix.as_str()), + HealType::ErasureSet { buckets, set_disk_id } => { + let bucket_count = u64::try_from(buckets.len()).unwrap_or(u64::MAX); + event + .with_attr("set_disk_id", set_disk_id.as_str()) + .with_attr("bucket_count", bucket_count) + } + HealType::Metadata { bucket, object } => event.with_bucket(bucket.as_str()).with_object(object.as_str()), + HealType::ECDecode { + bucket, + object, + version_id, + } => { + let event = event.with_bucket(bucket.as_str()).with_object(object.as_str()); + match version_id { + Some(version_id) => event.with_attr("version_id", version_id.as_str()), + None => event, + } + } + HealType::MRF { meta_path } => event.with_object(meta_path.as_str()), + }; + + match error { + Some(error) => event.with_attr("error", error.to_string()), + None => event, + } + }); + } + async fn remaining_timeout(&self) -> Result> { if let Some(total) = self.options.timeout { let start_instant = { *self.task_start_instant.read().await }; @@ -717,6 +784,7 @@ impl HealTask { queue_delay = ?queue_delay, "Heal task started" }); + self.emit_trace_task_state("started", Duration::ZERO, None); let result = match &self.heal_type { HealType::Cluster => self.heal_cluster().await, @@ -805,6 +873,14 @@ impl HealTask { } } + let terminal_state = match &result { + Ok(_) => "completed", + Err(Error::TaskCancelled) => "cancelled", + Err(Error::TaskTimeout) => "timed_out", + Err(_) => "failed", + }; + self.emit_trace_task_state(terminal_state, start_instant.elapsed(), result.as_ref().err()); + result } @@ -1535,7 +1611,7 @@ impl HealTask { let (objects, next_token, is_truncated) = self .await_with_control( self.storage - .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref()), + .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false), ) .await?; @@ -1697,6 +1773,23 @@ impl HealTask { Ok(()) } + async fn apply_erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<()> { + let baseline = match self + .await_with_control(self.storage.erasure_set_usage_baseline(buckets)) + .await + { + Ok(Some(baseline)) => baseline, + Ok(None) => return Ok(()), + Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err), + Err(_) => return Ok(()), + }; + + let HealBucketUsageBaseline { objects_count, bytes } = baseline; + let mut progress = self.progress.write().await; + progress.set_total_baseline(objects_count, bytes); + Ok(()) + } + async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> { debug!( target: "rustfs::heal::task", @@ -2298,6 +2391,8 @@ impl HealTask { None }; + self.apply_erasure_set_usage_baseline(&buckets).await?; + let healing_marker = format!("{set_disk_id}:{}", self.id); if let Some((disk, resume_manager, _)) = replacement_resume.as_ref() { let state = resume_manager.get_state().await; @@ -2602,7 +2697,8 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(4, 4, 0, 0); + let bytes_processed = progress.bytes_processed; + progress.update_progress(4, 4, 0, bytes_processed); } match result { @@ -2658,6 +2754,7 @@ mod tests { use super::super::{DiskOption, DiskStore, Endpoint, HealDiskExt as _, new_disk}; use super::*; use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo}; + use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events}; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos}; use std::collections::{HashMap, VecDeque}; use std::sync::Mutex; @@ -3203,6 +3300,8 @@ mod tests { block_heal_object: Mutex, resume_disk: Mutex>, replacement_resume_disk: Mutex>, + usage_baseline: Mutex>, + usage_baseline_error: Mutex, } #[test] @@ -3265,11 +3364,69 @@ mod tests { assert_eq!(samples_logged, MAX_BUCKET_FAILURE_LOG_SAMPLES); } + #[tokio::test] + async fn execute_emits_heal_trace_task_state() { + let mut trace = subscribe_trace_events(); + let storage = Arc::new(MockStorage::default()); + let task = HealTask::from_request( + HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())), + storage, + ); + + task.execute().await.expect("mock object heal should complete"); + + let started = recv_trace_task_state(&mut trace, &task.id, "started").await; + assert_eq!(started.kind, TraceKind::Heal); + assert_eq!(started.func, TraceFunc::HealTask); + assert_eq!(started.bucket.as_deref(), Some("bucket-a")); + assert_eq!(started.object.as_deref(), Some("object-a")); + assert_eq!(trace_attr_string(&started, "heal_type").as_deref(), Some("object")); + assert_eq!(trace_attr_string(&started, "source").as_deref(), Some("internal")); + assert_eq!(trace_attr_string(&started, "version_id").as_deref(), Some("version-a")); + + let completed = recv_trace_task_state(&mut trace, &task.id, "completed").await; + assert_eq!(completed.kind, TraceKind::Heal); + assert_eq!(completed.func, TraceFunc::HealTask); + assert_eq!(trace_attr_string(&completed, "state").as_deref(), Some("completed")); + } + + async fn recv_trace_task_state(trace: &mut TraceSubscription, task_id: &str, state: &str) -> TraceEvent { + for _ in 0..32 { + let event = tokio::time::timeout(Duration::from_secs(1), trace.recv()) + .await + .expect("trace event should arrive") + .expect("trace bus should stay open"); + if trace_attr_string(&event, "task_id").as_deref() == Some(task_id) + && trace_attr_string(&event, "state").as_deref() == Some(state) + { + return (*event).clone(); + } + } + + panic!("expected trace state {state} for task {task_id}"); + } + + fn trace_attr_string(event: &TraceEvent, key: &str) -> Option { + event.attrs.iter().find_map(|attr| { + if attr.key != key { + return None; + } + Some(match &attr.value { + TraceVal::Bool(value) => value.to_string(), + TraceVal::U64(value) => value.to_string(), + TraceVal::I64(value) => value.to_string(), + TraceVal::Str(value) => value.to_string(), + }) + }) + } + /// Build a latest, non-delete-marker heal list item with no version id. fn heal_item(name: &str) -> HealListItem { HealListItem { name: name.to_string(), version_id: None, + mod_time_unix_nanos: None, + lifecycle_object_info: None, is_delete_marker: false, } } @@ -3357,6 +3514,13 @@ mod tests { })) } + async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result> { + if *self.usage_baseline_error.lock().unwrap() { + return Err(Error::Other("usage baseline unavailable".to_string())); + } + Ok(*self.usage_baseline.lock().unwrap()) + } + async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<()> { Ok(()) } @@ -3540,6 +3704,7 @@ mod tests { bucket: &str, prefix: &str, continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { self.listed_prefixes.lock().unwrap().push(prefix.to_string()); if *self.truncate_without_token.lock().unwrap() { @@ -4654,6 +4819,73 @@ mod tests { assert!(storage.object_heal_opts.lock().unwrap().is_empty()); } + #[tokio::test] + async fn erasure_set_heal_applies_usage_baseline_to_progress() { + let temp = TempDir::new().expect("temporary directory should be created"); + let disk = make_resume_disk(&temp).await; + let storage = Arc::new(MockStorage { + resume_disk: Mutex::new(Some(disk)), + usage_baseline: Mutex::new(Some(HealBucketUsageBaseline { + objects_count: 10, + bytes: 8, + })), + ..Default::default() + }); + let request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-a".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ); + let task = HealTask::from_request(request, storage); + + task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string()) + .await + .expect("erasure set heal should complete"); + + let progress = task.get_progress().await; + assert_eq!(progress.objects_total_count, 10); + assert_eq!(progress.objects_total_size, 8); + assert_eq!(progress.bytes_processed, 2); + assert!((progress.progress_percentage - 25.0).abs() < 0.001); + } + + #[tokio::test] + async fn erasure_set_heal_ignores_usage_baseline_errors() { + let temp = TempDir::new().expect("temporary directory should be created"); + let disk = make_resume_disk(&temp).await; + let storage = Arc::new(MockStorage { + resume_disk: Mutex::new(Some(disk)), + usage_baseline_error: Mutex::new(true), + ..Default::default() + }); + let request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-a".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ); + let task = HealTask::from_request(request, storage); + + task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string()) + .await + .expect("usage baseline failures should not fail erasure set heal"); + + let progress = task.get_progress().await; + assert_eq!(progress.objects_total_count, 0); + assert_eq!(progress.objects_total_size, 0); + } + #[tokio::test] async fn resumable_erasure_set_execution_is_cancelled_while_object_heal_is_pending() { let temp = TempDir::new().expect("temporary directory should be created"); diff --git a/crates/heal/src/lib.rs b/crates/heal/src/lib.rs index 7c156304d..3dd29b064 100644 --- a/crates/heal/src/lib.rs +++ b/crates/heal/src/lib.rs @@ -445,6 +445,7 @@ mod tests { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool), Error> { Ok((Vec::new(), None, false)) } diff --git a/crates/heal/tests/heal_b5_versioned_regression_test.rs b/crates/heal/tests/heal_b5_versioned_regression_test.rs index 61a542955..95e0f0a90 100644 --- a/crates/heal/tests/heal_b5_versioned_regression_test.rs +++ b/crates/heal/tests/heal_b5_versioned_regression_test.rs @@ -176,7 +176,7 @@ async fn enumerate_all_versions(heal_storage: &Arc, bucket: let mut token: Option = None; loop { let (page, next, truncated) = heal_storage - .list_objects_for_heal_page(bucket, "", token.as_deref()) + .list_objects_for_heal_page(bucket, "", token.as_deref(), false) .await .expect("list_objects_for_heal_page failed"); items.extend(page); diff --git a/crates/heal/tests/heal_b920_subquorum_union_test.rs b/crates/heal/tests/heal_b920_subquorum_union_test.rs index 6d188b3f3..8d4ac65f2 100644 --- a/crates/heal/tests/heal_b920_subquorum_union_test.rs +++ b/crates/heal/tests/heal_b920_subquorum_union_test.rs @@ -166,7 +166,7 @@ async fn enumerate_b5(heal_storage: &Arc, bucket: &str) -> V let mut token: Option = None; loop { let (page, next, truncated) = heal_storage - .list_objects_for_heal_page(bucket, "", token.as_deref()) + .list_objects_for_heal_page(bucket, "", token.as_deref(), false) .await .expect("b5 list page failed"); items.extend(page); @@ -187,7 +187,7 @@ async fn enumerate_disk_walk(heal_storage: &Arc, bucket: &st let mut token: Option = None; loop { let (page, next, truncated) = heal_storage - .list_versions_for_heal_page_disk_walk(SET_DISK_ID, bucket, "", token.as_deref()) + .list_versions_for_heal_page_disk_walk(SET_DISK_ID, bucket, "", token.as_deref(), false) .await .expect("disk-walk list page failed"); items.extend(page); @@ -418,7 +418,7 @@ mod serial_tests { let mut pages = 0usize; loop { let (versions, next_forward, truncated) = ecstore - .heal_walk_versions_page(0, 0, bucket, "", forward.as_deref(), 2, 100_000) + .heal_walk_versions_page(0, 0, bucket, "", forward.as_deref(), 2, 100_000, false) .await .expect("heal_walk_versions_page failed"); pages += 1; diff --git a/crates/heal/tests/heal_bug_fixes_test.rs b/crates/heal/tests/heal_bug_fixes_test.rs index 7b8737f19..ba59142fd 100644 --- a/crates/heal/tests/heal_bug_fixes_test.rs +++ b/crates/heal/tests/heal_bug_fixes_test.rs @@ -242,6 +242,7 @@ fn test_heal_task_status_atomic_update() { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> rustfs_heal::Result<(Vec, Option, bool)> { Ok((vec![], None, false)) } @@ -385,6 +386,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> rustfs_heal::Result<(Vec, Option, bool)> { Ok((Vec::new(), None, false)) } diff --git a/crates/madmin/src/service_commands.rs b/crates/madmin/src/service_commands.rs index ddda978be..1b5d0b2a9 100644 --- a/crates/madmin/src/service_commands.rs +++ b/crates/madmin/src/service_commands.rs @@ -43,7 +43,7 @@ pub struct ServiceTraceOpts { #[allow(dead_code)] impl ServiceTraceOpts { - fn trace_types(&self) -> TraceType { + pub fn trace_types(&self) -> TraceType { let mut tt = TraceType::default(); tt.set_if(self.s3, &TraceType::S3); tt.set_if(self.internal, &TraceType::INTERNAL); @@ -72,6 +72,14 @@ impl ServiceTraceOpts { tt } + pub fn only_errors(&self) -> bool { + self.only_errors + } + + pub fn threshold(&self) -> Duration { + self.threshold + } + pub fn parse_params(&mut self, uri: &Uri) -> Result<(), String> { let query_pairs: HashMap<_, _> = uri .query() diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 25f1e758d..820cbb12f 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -41,6 +41,7 @@ use rustfs_common::metrics::{ CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource, UpdateCurrentPathFn, current_path_updater, global_metrics, }; +use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit, trace_subscriber_count}; use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration}; @@ -430,6 +431,113 @@ fn non_negative_i64_to_u64(value: i64) -> u64 { value.max(0) as u64 } +fn trace_start_instant() -> Option { + (trace_subscriber_count() > 0).then(Instant::now) +} + +fn emit_scanner_folder_trace(root: &str, folder: &str, objects: u64, started_at: Option, state: &'static str) { + let Some(started_at) = started_at else { + return; + }; + + trace_emit(|| { + let (bucket, prefix) = path2_bucket_object_with_base_path(root, folder); + TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder) + .with_bucket(bucket) + .with_object(prefix) + .with_duration(started_at.elapsed()) + .with_attr("state", state) + .with_attr("objects", objects) + }); +} + +fn emit_scanner_ilm_action_trace( + bucket: &str, + object: &str, + action: IlmAction, + count: u64, + queued: bool, + started_at: Option, +) { + let Some(started_at) = started_at else { + return; + }; + + let state = if queued { "queued" } else { "not_queued" }; + trace_emit(|| { + TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerIlmAction) + .with_bucket(bucket) + .with_object(object) + .with_duration(started_at.elapsed()) + .with_attr("state", state) + .with_attr("action", action.as_str()) + .with_attr("count", count) + .with_attr("queued", queued) + }); +} + +struct ScannerHealCandidateTraceContext { + bucket: String, + object: Option, + version_id: Option, + scan_mode: Option, + started_at: Instant, +} + +fn scanner_heal_candidate_trace_context(request: &HealChannelRequest) -> Option { + let started_at = trace_start_instant()?; + Some(ScannerHealCandidateTraceContext { + bucket: request.bucket.clone(), + object: request.object_prefix.clone(), + version_id: request.object_version_id.clone(), + scan_mode: request.scan_mode, + started_at, + }) +} + +struct ScannerHealCandidateTrace<'a> { + candidate_type: &'static str, + bucket: &'a str, + object: Option<&'a str>, + version_id: Option<&'a str>, + priority: HealChannelPriority, + scan_mode: Option, + result: Result, + started_at: Instant, +} + +fn emit_scanner_heal_candidate_trace(trace: ScannerHealCandidateTrace<'_>) { + trace_emit(|| { + let (state, admission, error) = match trace.result { + Ok(result) if result.is_admitted() => ("admitted", describe_heal_admission(result), None), + Ok(result) => ("not_admitted", describe_heal_admission(result), None), + Err(error) => ("submit_failed", "channel_error".to_string(), Some(error)), + }; + let mut event = TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerHealCandidate) + .with_bucket(trace.bucket) + .with_duration(trace.started_at.elapsed()) + .with_attr("state", state) + .with_attr("candidate_type", trace.candidate_type) + .with_attr("priority", heal_priority_label(trace.priority)) + .with_attr("admission", admission); + + if let Some(object) = trace.object { + event = event.with_object(object); + } + if let Some(version_id) = trace.version_id { + event = event.with_attr("version_id", version_id); + } + if let Some(scan_mode) = trace.scan_mode { + event = event.with_attr("scan_mode", scan_mode.as_str()); + } + if let Some(error) = error { + event = event.with_attr("error", error); + } + + event + }); +} + fn apply_scanner_size_summary(into: &mut DataUsageEntry, summary: &SizeSummary) { into.size = into.size.saturating_add(summary.total_size); into.versions = into.versions.saturating_add(summary.versions); @@ -677,9 +785,22 @@ async fn send_scanner_heal_request( request: HealChannelRequest, ) -> Result { let priority = request.priority; + let trace_context = scanner_heal_candidate_trace_context(&request); match send_heal_request_with_admission(request).await { Ok(result) => { record_heal_candidate_admission(candidate_type, priority, result); + if let Some(trace_context) = trace_context.as_ref() { + emit_scanner_heal_candidate_trace(ScannerHealCandidateTrace { + candidate_type, + bucket: &trace_context.bucket, + object: trace_context.object.as_deref(), + version_id: trace_context.version_id.as_deref(), + priority, + scan_mode: trace_context.scan_mode, + result: Ok(result), + started_at: trace_context.started_at, + }); + } Ok(result) } Err(err) => { @@ -690,6 +811,18 @@ async fn send_scanner_heal_request( "result" => "channel_error".to_string() ) .increment(1); + if let Some(trace_context) = trace_context.as_ref() { + emit_scanner_heal_candidate_trace(ScannerHealCandidateTrace { + candidate_type, + bucket: &trace_context.bucket, + object: trace_context.object.as_deref(), + version_id: trace_context.version_id.as_deref(), + priority, + scan_mode: trace_context.scan_mode, + result: Err(err.as_str()), + started_at: trace_context.started_at, + }); + } Err(ScannerError::Other(err)) } } @@ -905,7 +1038,9 @@ impl ScannerItem { "Scanner lifecycle action dispatched" ); let done_ilm = Metrics::time_ilm(event.action); + let trace_started_at = trace_start_instant(); let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await; + emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at); if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) { done_ilm(1)(); remaining_versions = 0; @@ -957,7 +1092,9 @@ impl ScannerItem { "Scanner lifecycle action dispatched" ); let done_ilm = Metrics::time_ilm(event.action); + let trace_started_at = trace_start_instant(); let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await; + emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at); if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) { done_ilm(1)(); if !versioning_config.prefix_enabled(&self.object_path()) && event.action == IlmAction::DeleteAction { @@ -995,7 +1132,9 @@ impl ScannerItem { "Scanner lifecycle action dispatched" ); let done_ilm = Metrics::time_ilm(event.action); + let trace_started_at = trace_start_instant(); let queued = apply_transition_rule(event, &LcEventSrc::Scanner, oi).await; + emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at); if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) { done_ilm(1)(); } @@ -1019,7 +1158,21 @@ impl ScannerItem { let action = event.action; let count = u64::try_from(to_delete_objs.len()).unwrap_or(u64::MAX); let done_ilm = Metrics::time_ilm(action); + let trace_started_at = trace_start_instant(); let queued = enqueue_runtime_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner).await; + if let Some(trace_started_at) = trace_started_at { + let state = if queued { "queued" } else { "not_queued" }; + trace_emit(|| { + TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerIlmAction) + .with_bucket(self.bucket.as_str()) + .with_object(self.object_path()) + .with_duration(trace_started_at.elapsed()) + .with_attr("state", state) + .with_attr("action", action.as_str()) + .with_attr("count", count) + .with_attr("queued", queued) + }); + } if record_scanner_ilm_action_if_queued(global_metrics(), action, count, queued) { done_ilm(count)(); remaining_versions = remaining_versions.saturating_sub(noncurrent_accounting.len()); @@ -1830,6 +1983,7 @@ impl FolderScanner { into: &mut DataUsageEntry, ) -> Result<(), ScannerError> { let done_folder = Metrics::time(Metric::ScanFolder); + let trace_started_at = trace_start_instant(); if ctx.is_cancelled() { return Err(ScannerError::Other("Operation cancelled".to_string())); @@ -2895,6 +3049,8 @@ impl FolderScanner { } done_folder(); + let scanned_objects = u64::try_from(into.objects).unwrap_or(u64::MAX); + emit_scanner_folder_trace(&self.root, &folder.name, scanned_objects, trace_started_at, "completed"); Ok(()) } @@ -4400,6 +4556,104 @@ mod tests { ); } + #[tokio::test] + async fn scanner_trace_helpers_emit_expected_events() { + let mut trace = rustfs_common::trace_bus::subscribe_trace_events(); + + emit_scanner_folder_trace( + "/tmp/rustfs-scanner-trace", + "/tmp/rustfs-scanner-trace/bucket-a/folder-a", + 7, + Some(Instant::now()), + "completed", + ); + let folder = recv_scanner_trace_event( + &mut trace, + TraceFunc::ScannerFolder, + Some("bucket-a"), + Some("folder-a"), + Some("completed"), + ) + .await; + assert_eq!(trace_attr_string(&folder, "objects").as_deref(), Some("7")); + + emit_scanner_ilm_action_trace("bucket-a", "object-a", IlmAction::DeleteAction, 2, true, Some(Instant::now())); + let ilm = recv_scanner_trace_event( + &mut trace, + TraceFunc::ScannerIlmAction, + Some("bucket-a"), + Some("object-a"), + Some("queued"), + ) + .await; + assert_eq!(trace_attr_string(&ilm, "action").as_deref(), Some("delete")); + assert_eq!(trace_attr_string(&ilm, "count").as_deref(), Some("2")); + assert_eq!(trace_attr_string(&ilm, "queued").as_deref(), Some("true")); + + emit_scanner_heal_candidate_trace(ScannerHealCandidateTrace { + candidate_type: "object", + bucket: "bucket-a", + object: Some("object-a"), + version_id: Some("version-a"), + priority: HealChannelPriority::High, + scan_mode: Some(HealScanMode::Deep), + result: Ok(HealAdmissionResult::Merged), + started_at: Instant::now(), + }); + let heal_candidate = recv_scanner_trace_event( + &mut trace, + TraceFunc::ScannerHealCandidate, + Some("bucket-a"), + Some("object-a"), + Some("admitted"), + ) + .await; + assert_eq!(trace_attr_string(&heal_candidate, "candidate_type").as_deref(), Some("object")); + assert_eq!(trace_attr_string(&heal_candidate, "priority").as_deref(), Some("high")); + assert_eq!(trace_attr_string(&heal_candidate, "scan_mode").as_deref(), Some("deep")); + assert_eq!(trace_attr_string(&heal_candidate, "version_id").as_deref(), Some("version-a")); + assert_eq!(trace_attr_string(&heal_candidate, "admission").as_deref(), Some("merged")); + } + + async fn recv_scanner_trace_event( + trace: &mut rustfs_common::trace_bus::TraceSubscription, + func: TraceFunc, + bucket: Option<&str>, + object: Option<&str>, + state: Option<&str>, + ) -> TraceEvent { + for _ in 0..32 { + let event = tokio::time::timeout(Duration::from_secs(1), trace.recv()) + .await + .expect("scanner trace event should arrive") + .expect("trace bus should stay open"); + if event.kind == TraceKind::Scanner + && event.func == func + && event.bucket.as_deref() == bucket + && event.object.as_deref() == object + && state.is_none_or(|state| trace_attr_string(&event, "state").as_deref() == Some(state)) + { + return (*event).clone(); + } + } + + panic!("expected scanner trace event {func:?} for bucket {bucket:?} object {object:?}"); + } + + fn trace_attr_string(event: &TraceEvent, key: &str) -> Option { + event.attrs.iter().find_map(|attr| { + if attr.key != key { + return None; + } + Some(match &attr.value { + rustfs_common::trace_bus::TraceVal::Bool(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::U64(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::I64(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::Str(value) => value.to_string(), + }) + }) + } + #[test] fn test_build_high_priority_heal_admission_error_contains_context() { let err = build_high_priority_heal_admission_error( diff --git a/rustfs/src/admin/handlers/profile_admin.rs b/rustfs/src/admin/handlers/profile_admin.rs index d4a85360a..d14af9f3b 100644 --- a/rustfs/src/admin/handlers/profile_admin.rs +++ b/rustfs/src/admin/handlers/profile_admin.rs @@ -23,18 +23,24 @@ use futures::{Stream, StreamExt}; use http::{HeaderMap, HeaderValue}; use hyper::{Method, StatusCode}; use matchit::Params; +use regex::Regex; +use rustfs_common::trace_bus::{TraceEvent, TraceKind, TraceVal, subscribe_trace_events}; use rustfs_madmin::service_commands::ServiceTraceOpts; +use rustfs_madmin::trace::TraceType; use rustfs_policy::policy::action::{Action, AdminAction}; use s3s::header::CONTENT_TYPE; use s3s::stream::{ByteStream, DynByteStream}; use s3s::{Body, S3Request, S3Response, S3Result, StdError, s3_error}; use serde::Serialize; +use std::collections::HashMap; use std::pin::Pin; use std::task::{Context, Poll}; -use std::time::Duration; +use std::time::{Duration, SystemTime}; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tracing::error; +use url::form_urlencoded; #[derive(Serialize)] struct ProfileStatus { @@ -206,16 +212,164 @@ impl Stream for TraceStream { impl ByteStream for TraceStream {} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct TraceKindFilter { + heal: bool, + scanner: bool, +} + +impl TraceKindFilter { + const ALL_SUPPORTED: Self = Self { + heal: true, + scanner: true, + }; + + fn from_request(uri: &hyper::Uri, trace_types: TraceType) -> S3Result { + let mut has_kind = false; + let mut filter = Self { + heal: false, + scanner: false, + }; + + for (key, value) in trace_query_pairs(uri) { + if key != "kind" { + continue; + } + has_kind = true; + for item in value.split(',') { + match item.trim().to_ascii_lowercase().as_str() { + "heal" | "healing" => filter.heal = true, + "scanner" => filter.scanner = true, + "all" => return Ok(Self::ALL_SUPPORTED), + _ => return Err(s3_error!(InvalidRequest, "invalid trace kind")), + } + } + } + + if has_kind { + return Ok(filter); + } + + if trace_types.mask() == 0 || trace_query_flag(uri, "all") { + return Ok(Self::ALL_SUPPORTED); + } + + Ok(Self { + heal: trace_types.overlaps(&TraceType::HEALING), + scanner: trace_types.overlaps(&TraceType::SCANNER), + }) + } + + const fn matches(self, kind: TraceKind) -> bool { + match kind { + TraceKind::Heal => self.heal, + TraceKind::Scanner => self.scanner, + } + } +} + +#[derive(Debug)] +struct TraceStreamFilter { + kinds: TraceKindFilter, + regex: Option, + threshold: Duration, +} + +impl TraceStreamFilter { + fn from_request(uri: &hyper::Uri, opts: &ServiceTraceOpts) -> S3Result { + if opts.only_errors() { + return Err(s3_error!( + InvalidRequest, + "trace error-only filter is not supported for heal/scanner trace" + )); + } + + Ok(Self { + kinds: TraceKindFilter::from_request(uri, opts.trace_types())?, + regex: trace_regex_filter(uri)?, + threshold: opts.threshold(), + }) + } + + fn matches_kind(&self, kind: TraceKind) -> bool { + self.kinds.matches(kind) + } + + fn matches_record(&self, record: &TraceWireRecord) -> bool { + record.duration >= self.threshold && self.regex.as_ref().is_none_or(|regex| record.matches_regex(regex)) + } +} + +#[derive(Serialize)] +struct TraceWireRecord { + #[serde(rename = "type")] + trace_type: u64, + #[serde(rename = "nodename")] + node_name: String, + #[serde(rename = "funcname")] + func_name: String, + #[serde(rename = "time")] + time: String, + #[serde(rename = "path")] + path: String, + #[serde(rename = "dur")] + duration: Duration, + #[serde(rename = "bytes", skip_serializing_if = "Option::is_none")] + bytes: Option, + #[serde(rename = "msg", skip_serializing_if = "Option::is_none")] + message: Option, + #[serde(rename = "custom", skip_serializing_if = "Option::is_none")] + custom: Option>, +} + +impl TraceWireRecord { + fn from_event(node_name: &str, event: &TraceEvent) -> Self { + Self { + trace_type: trace_type_mask(event.kind), + node_name: node_name.to_owned(), + func_name: event.func.as_str().to_owned(), + time: trace_time_string(event.time), + path: trace_path(event), + duration: event.duration, + bytes: trace_bytes(event.bytes), + message: None, + custom: trace_custom_attrs(event), + } + } + + fn dropped(node_name: &str, dropped: u64) -> Self { + let mut custom = HashMap::new(); + custom.insert("dropped_events".to_string(), dropped.to_string()); + + Self { + trace_type: 0, + node_name: node_name.to_owned(), + func_name: "trace.Dropped".to_string(), + time: trace_time_string(SystemTime::now()), + path: String::new(), + duration: Duration::ZERO, + bytes: None, + message: Some("trace subscriber lagged".to_string()), + custom: Some(custom), + } + } + + fn matches_regex(&self, regex: &Regex) -> bool { + regex.is_match(&self.func_name) + || regex.is_match(&self.path) + || self.message.as_ref().is_some_and(|message| regex.is_match(message)) + || self + .custom + .as_ref() + .is_some_and(|custom| custom.iter().any(|(key, value)| regex.is_match(key) || regex.is_match(value))) + } +} + /// `GET /v3/trace` — stream real-time server trace events. /// -/// RustFS emits diagnostics through the `tracing` pipeline but does not expose -/// an in-process subscriber that can fan trace events out to an admin client -/// (there is no request-trace broadcast channel). Rather than return an opaque -/// `501` — which would make `mc admin trace` fail to connect — this honors the -/// streaming NDJSON contract: it validates the requested trace filters, opens -/// the stream, emits a single capability record explaining that live tracing is -/// not wired, then holds the connection open with keep-alives. No fabricated -/// trace records are ever sent. +/// RustFS currently publishes heal and scanner diagnostics through the common +/// trace bus. The admin endpoint exposes those events as MinIO-shaped NDJSON +/// records while keeping unsupported trace classes filtered out. pub struct TraceHandler {} #[async_trait::async_trait] @@ -228,24 +382,13 @@ impl Operation for TraceHandler { let mut opts = ServiceTraceOpts::default(); opts.parse_params(&req.uri) .map_err(|_| s3_error!(InvalidRequest, "invalid trace parameters"))?; + let filter = TraceStreamFilter::from_request(&req.uri, &opts)?; let node_name = sysinfo::System::host_name().unwrap_or_else(|| "rustfs".to_string()); - let (tx, rx) = mpsc::channel::>(8); + let mut subscription = subscribe_trace_events(); + let (tx, rx) = mpsc::channel::>(64); spawn_traced(async move { - let notice = serde_json::json!({ - "nodename": node_name, - "funcname": "admin.Trace", - "msg": "RustFS does not expose an in-process trace-event subscriber; live tracing is not yet available", - "err": "trace_streaming_unsupported", - }); - if let Ok(mut encoded) = serde_json::to_vec(¬ice) { - encoded.push(b'\n'); - if tx.send(Ok(Bytes::from(encoded))).await.is_err() { - return; - } - } - let mut ticker = tokio::time::interval(Duration::from_secs(15)); ticker.tick().await; loop { @@ -256,6 +399,26 @@ impl Operation for TraceHandler { break; } } + received = subscription.recv() => { + match received { + Ok(event) => { + if !filter.matches_kind(event.kind) { + continue; + } + let record = TraceWireRecord::from_event(&node_name, &event); + if filter.matches_record(&record) && send_trace_record(&tx, &record).await.is_err() { + break; + } + } + Err(tokio::sync::broadcast::error::RecvError::Lagged(dropped)) => { + let record = TraceWireRecord::dropped(&node_name, dropped); + if send_trace_record(&tx, &record).await.is_err() { + break; + } + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + } + } } } }); @@ -269,17 +432,115 @@ impl Operation for TraceHandler { } } +async fn send_trace_record(tx: &mpsc::Sender>, record: &TraceWireRecord) -> Result<(), ()> { + let Some(encoded) = encode_ndjson(record) else { + return Ok(()); + }; + tx.send(Ok(encoded)).await.map_err(|_| ()) +} + +fn encode_ndjson(value: &impl Serialize) -> Option { + let mut encoded = serde_json::to_vec(value).ok()?; + encoded.push(b'\n'); + Some(Bytes::from(encoded)) +} + +fn trace_query_pairs(uri: &hyper::Uri) -> impl Iterator + '_ { + uri.query() + .into_iter() + .flat_map(|query| form_urlencoded::parse(query.as_bytes())) + .map(|(key, value)| (key.into_owned(), value.into_owned())) +} + +fn trace_query_flag(uri: &hyper::Uri, flag: &str) -> bool { + trace_query_pairs(uri).any(|(key, value)| key == flag && value == "true") +} + +fn trace_regex_filter(uri: &hyper::Uri) -> S3Result> { + trace_query_pairs(uri) + .find_map(|(key, value)| { + if key == "filter" && !value.is_empty() { + Some(value) + } else { + None + } + }) + .map(|pattern| Regex::new(&pattern).map_err(|_| s3_error!(InvalidRequest, "invalid trace filter"))) + .transpose() +} + +fn trace_type_mask(kind: TraceKind) -> u64 { + match kind { + TraceKind::Heal => TraceType::HEALING.mask(), + TraceKind::Scanner => TraceType::SCANNER.mask(), + } +} + +fn trace_time_string(time: SystemTime) -> String { + match OffsetDateTime::from(time).format(&Rfc3339) { + Ok(value) => value, + Err(_) => "1970-01-01T00:00:00Z".to_string(), + } +} + +fn trace_path(event: &TraceEvent) -> String { + match (event.bucket.as_deref(), event.object.as_deref()) { + (Some(bucket), Some(object)) if !object.is_empty() => format!("{bucket}/{object}"), + (Some(bucket), _) => bucket.to_owned(), + (None, Some(object)) => object.to_owned(), + (None, None) => String::new(), + } +} + +fn trace_bytes(bytes: u64) -> Option { + if bytes == 0 { + return None; + } + + match i64::try_from(bytes) { + Ok(value) => Some(value), + Err(_) => Some(i64::MAX), + } +} + +fn trace_custom_attrs(event: &TraceEvent) -> Option> { + if event.attrs.is_empty() { + return None; + } + + Some( + event + .attrs + .iter() + .map(|attr| (attr.key.to_string(), trace_value_string(&attr.value))) + .collect(), + ) +} + +fn trace_value_string(value: &TraceVal) -> String { + match value { + TraceVal::Bool(value) => value.to_string(), + TraceVal::U64(value) => value.to_string(), + TraceVal::I64(value) => value.to_string(), + TraceVal::Str(value) => value.to_string(), + } +} + #[cfg(test)] mod tests { use super::{ ProfileControlHandler, ProfileHandler, ProfileStatusHandler, ProfilingDownloadHandler, ProfilingStartHandler, - TraceHandler, + TraceHandler, TraceKindFilter, TraceStreamFilter, TraceWireRecord, }; use crate::admin::router::Operation; use http::{Extensions, HeaderMap, Uri}; use hyper::Method; use matchit::Params; - use s3s::{Body, S3ErrorCode, S3Request}; + use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind}; + use rustfs_madmin::service_commands::ServiceTraceOpts; + use rustfs_madmin::trace::TraceType; + use s3s::{Body, S3ErrorCode, S3Request, S3Result}; + use std::time::{Duration, UNIX_EPOCH}; fn build_profile_request(uri: &'static str) -> S3Request { S3Request { @@ -295,6 +556,13 @@ mod tests { } } + fn build_trace_stream_filter(uri: &'static str) -> S3Result { + let uri = Uri::from_static(uri); + let mut opts = ServiceTraceOpts::default(); + opts.parse_params(&uri).expect("test trace params should parse"); + TraceStreamFilter::from_request(&uri, &opts) + } + #[tokio::test] async fn profile_handler_rejects_missing_credentials() { let result = ProfileHandler {} @@ -358,4 +626,108 @@ mod tests { .expect_err("trace must reject anonymous requests"); assert_eq!(err.code(), &S3ErrorCode::AccessDenied); } + + #[test] + fn trace_kind_filter_supports_kind_query() { + let uri = Uri::from_static("/rustfs/admin/v3/trace?kind=heal"); + let filter = TraceKindFilter::from_request(&uri, TraceType::default()).expect("kind filter should parse"); + + assert!(filter.matches(TraceKind::Heal)); + assert!(!filter.matches(TraceKind::Scanner)); + } + + #[test] + fn trace_kind_filter_defaults_to_supported_events_without_type_flags() { + let uri = Uri::from_static("/rustfs/admin/v3/trace"); + let filter = TraceKindFilter::from_request(&uri, TraceType::default()).expect("empty filter should parse"); + + assert!(filter.matches(TraceKind::Heal)); + assert!(filter.matches(TraceKind::Scanner)); + } + + #[test] + fn trace_kind_filter_rejects_unknown_kind() { + let uri = Uri::from_static("/rustfs/admin/v3/trace?kind=s3"); + let err = TraceKindFilter::from_request(&uri, TraceType::default()).expect_err("unknown kind should fail"); + + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[test] + fn trace_stream_filter_matches_regex_against_path_and_attrs() { + let filter = build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&filter=data/.%2Bxl.meta") + .expect("regex filter should parse"); + let event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject) + .with_bucket("data") + .with_object("dir/xl.meta") + .with_attr("dry_run", true); + let record = TraceWireRecord::from_event("node-a", &event); + + assert!(filter.matches_kind(event.kind)); + assert!(filter.matches_record(&record)); + } + + #[test] + fn trace_stream_filter_rejects_invalid_regex() { + let err = build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&filter=[").expect_err("invalid regex should fail"); + + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[test] + fn trace_stream_filter_applies_threshold() { + let filter = + build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&threshold=10ms").expect("threshold should parse"); + let short = TraceWireRecord::from_event( + "node-a", + &TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject).with_duration(Duration::from_millis(9)), + ); + let long = TraceWireRecord::from_event( + "node-a", + &TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject).with_duration(Duration::from_millis(10)), + ); + + assert!(!filter.matches_record(&short)); + assert!(filter.matches_record(&long)); + } + + #[test] + fn trace_stream_filter_rejects_error_only_filter() { + let err = build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&err=true").expect_err("err filter should fail"); + + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[test] + fn trace_wire_record_contains_madmin_trace_fields() { + let event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject) + .with_bucket("bucket") + .with_object("object") + .with_duration(Duration::from_millis(3)) + .with_bytes(17) + .with_attr("dry", true); + let mut record = TraceWireRecord::from_event("node-a", &event); + record.time = "1970-01-01T00:00:00Z".to_string(); + + let value = serde_json::to_value(&record).expect("trace record should serialize"); + + assert_eq!(value["type"], TraceType::HEALING.mask()); + assert_eq!(value["nodename"], "node-a"); + assert_eq!(value["funcname"], "heal.Object"); + assert_eq!(value["time"], "1970-01-01T00:00:00Z"); + assert_eq!(value["path"], "bucket/object"); + assert_eq!(value["bytes"], 17); + assert_eq!(value["custom"]["dry"], "true"); + } + + #[test] + fn trace_wire_record_formats_epoch_time() { + let event = TraceEvent { + time: UNIX_EPOCH, + ..TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder) + }; + let record = TraceWireRecord::from_event("node-a", &event); + + assert_eq!(record.time, "1970-01-01T00:00:00Z"); + } } diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index d06e0892f..a39ca6c3a 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -2420,6 +2420,7 @@ mod tests { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> rustfs_heal::Result<(Vec, Option, bool)> { Ok((Vec::new(), None, false)) } From 35a30cd6144f5f37f3b03e2964b73655ed408274 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 08:46:32 +0800 Subject: [PATCH 14/51] feat(scanner): emit excess alerts as S3 notification events (HS-04) (#6176) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(scanner): emit excess alerts as S3 notification events The excess-versions / excess-version-size / excess-folders alerts were metrics-and-logs only; consoles and external auditors had no way to hear them (rustfs/backlog#1868, HS-04). MinIO emits s3:ObjectManyVersions / s3:ObjectLargeVersions / s3:PrefixManyFolders for the same conditions — RustFS carries those as EventName::Scanner* with s3:Scanner:* wire names that already existed unpublished. The three alert sites now also dispatch through the standard event pipeline (send_event via the storage_api owner facade), carrying the actual values and thresholds in req_params and UserAgent "Scanner". Without a cooldown a single over-threshold object would re-emit on every ~60s scan cycle, so emissions are edge-held per (kind, bucket, object) for 24h (RUSTFS_SCANNER_ALERT_COOLDOWN_SECS, 0 = every cycle), backed by a process-global map with a 4096-key hard cap that clears rather than grows. Metrics and structured logs stay level-triggered every cycle; only the notification events are held back. A restart resets the cooldown deliberately: one re-emission per still-hot key buys back visibility after the restarts that accompany incident response. Tests pin the edge-hold semantics (first fires, immediate re-check held, independent keys, cooldown expiry re-fires, zero cooldown always emits, hard bound) in one sequential test for the process-global map, and pin the emitted wire names against EventName's canonical string forms so a subscribed bucket notification can never silently stop matching. docs/operations/scanner-excess-alerts.md documents the three events, the metric-vs-event cadence difference, and the HS-15 threshold deltas (alert_excess_folders 65538 vs MinIO 50000 is deliberate: Proxmox Backup Server chunk layout compatibility). Closes rustfs/backlog#1868. Co-Authored-By: heihutu * docs(operations): split scanner excess alerts into English and Chinese pages The page shipped Chinese-only; keep it as scanner-excess-alerts_zh.md and add a faithful English translation at the original path, cross-linked at the top of both. Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- Cargo.lock | 1 + crates/ecstore/src/api/mod.rs | 2 +- crates/scanner/Cargo.toml | 3 + crates/scanner/src/scanner_folder.rs | 229 +++++++++++++++++++- crates/scanner/src/storage_api.rs | 7 +- docs/operations/scanner-excess-alerts.md | 37 ++++ docs/operations/scanner-excess-alerts_zh.md | 37 ++++ 7 files changed, 308 insertions(+), 8 deletions(-) create mode 100644 docs/operations/scanner-excess-alerts.md create mode 100644 docs/operations/scanner-excess-alerts_zh.md diff --git a/Cargo.lock b/Cargo.lock index 85c91d523..cda581ffb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10252,6 +10252,7 @@ dependencies = [ "rustfs-ecstore", "rustfs-filemeta", "rustfs-lock", + "rustfs-s3-types", "rustfs-storage-api", "rustfs-utils", "s3s", diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 4b8bc3249..3a031d60a 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -380,7 +380,7 @@ pub mod erasure { pub mod event { pub use crate::event::name::EventName; - pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook}; + pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook, send_event}; } pub mod global { diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index 16ff34a8e..0e2d039c6 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -108,6 +108,9 @@ temp-env = { workspace = true } tempfile = { workspace = true } uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] } tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] } +# Test-only: pins the emitted scanner alert wire names against the canonical +# EventName string forms subscribers configure (rustfs/backlog#1868). +rustfs-s3-types.workspace = true # Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via # the ecstore `api::tier::test_util` facade module (rustfs/backlog#1148 ilm-6). rustfs-ecstore = { workspace = true, features = ["test-util"] } diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 820cbb12f..15a01507c 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fs::FileType; use std::io::ErrorKind; -use std::sync::{Arc, Once}; +use std::sync::{Arc, Mutex, Once}; use std::time::{Duration, Instant, SystemTime}; use crate::ReplTargetSizeSummary; @@ -32,6 +32,7 @@ use crate::scanner_io::{ SCANNER_SKIP_FILE_ERROR, ScannerIODisk as _, is_scanner_metadata_corrupt_error, is_scanner_metadata_transient_error, }; use crate::sleeper::DynamicSleeper; +use crate::storage_api::owner::{EcstoreEventArgs, ecstore_send_event}; use metrics::{counter, describe_counter}; use rustfs_common::heal_channel::{ HEAL_DELETE_DANGLING, HealAdmissionDropReason, HealAdmissionResult, HealChannelPriority, HealChannelRequest, @@ -98,6 +99,101 @@ const METRIC_SCANNER_EXCESS_FOLDERS_TOTAL: &str = "rustfs_scanner_excess_folders const METRIC_SCANNER_PENDING_HEAL_PRUNE_TOTAL: &str = "rustfs_scanner_pending_heal_prune_total"; const METRIC_SCANNER_PENDING_HEAL_MALFORMED_TOTAL: &str = "rustfs_scanner_pending_heal_malformed_total"; const MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET: usize = 128; + +// --- scanner excess alerts as S3 notification events (rustfs/backlog#1868) -- +// +// The excess-versions / excess-version-size / excess-folders alerts were +// metrics-and-logs only; subscribers (consoles, external auditors) had no way +// to hear them. MinIO emits s3:ObjectManyVersions / s3:ObjectLargeVersions / +// s3:PrefixManyFolders for the same conditions — RustFS carries those as +// EventName::Scanner* with the wire names below. Without a cooldown a single +// over-threshold object would re-emit on every scan cycle (~a minute), so +// emissions are edge-held per (kind, bucket, object) for 24h. + +/// `s3:Scanner:ManyVersions` (MinIO `s3:ObjectManyVersions`). +pub const EVENT_SCANNER_MANY_VERSIONS: &str = "s3:Scanner:ManyVersions"; +/// `s3:Scanner:LargeVersions` (MinIO `s3:ObjectLargeVersions`). +pub const EVENT_SCANNER_LARGE_VERSIONS: &str = "s3:Scanner:LargeVersions"; +/// `s3:Scanner:BigPrefix` (MinIO `s3:PrefixManyFolders`). +pub const EVENT_SCANNER_BIG_PREFIX: &str = "s3:Scanner:BigPrefix"; +const ENV_SCANNER_ALERT_COOLDOWN_SECS: &str = "RUSTFS_SCANNER_ALERT_COOLDOWN_SECS"; +const DEFAULT_SCANNER_ALERT_COOLDOWN_SECS: u64 = 86_400; +/// Hard cap on distinct cooldown keys; a pathological number of over-threshold +/// objects clears the map wholesale instead of growing without bound (the +/// worst case is one re-emission per still-hot key per scan cycle). +const MAX_SCANNER_ALERT_COOLDOWN_KEYS: usize = 4096; + +/// Distinct alert kinds sharing one cooldown map. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum ScannerAlertKind { + ManyVersions, + LargeVersions, + BigPrefix, +} + +type ScannerAlertCooldownKey = (ScannerAlertKind, String, String); +type ScannerAlertCooldownMap = HashMap; + +static SCANNER_ALERT_EMISSION_COOLDOWN: Mutex> = Mutex::new(None); + +fn scanner_alert_cooldown() -> Duration { + let raw = std::env::var(ENV_SCANNER_ALERT_COOLDOWN_SECS) + .ok() + .and_then(|v| v.parse::().ok()); + Duration::from_secs(raw.unwrap_or(DEFAULT_SCANNER_ALERT_COOLDOWN_SECS)) +} + +/// Edge-held emission gate: returns `true` (and records the cooldown) only +/// when this (kind, bucket, object) last fired longer than the cooldown ago — +/// or never. Metrics and logs stay level-triggered every cycle; only the +/// notification events are held back. +fn scanner_alert_emission_allows(kind: ScannerAlertKind, bucket: &str, object: &str, cooldown: Duration) -> bool { + let key = (kind, bucket.to_string(), object.to_string()); + let mut guard = SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + let guard = guard.get_or_insert_with(ScannerAlertCooldownMap::new); + let now = Instant::now(); + // Expired entries leave first; the cap is still exceeded only when live + // keys alone overflow it, in which case a wholesale clear trades one + // extra emission per hot key for a hard memory bound. + if guard.len() >= MAX_SCANNER_ALERT_COOLDOWN_KEYS { + guard.retain(|_, fired_at| now.duration_since(*fired_at) < cooldown); + if guard.len() >= MAX_SCANNER_ALERT_COOLDOWN_KEYS { + guard.clear(); + } + } + match guard.get(&key) { + Some(fired_at) if now.duration_since(*fired_at) < cooldown => false, + _ => { + guard.insert(key, now); + true + } + } +} + +/// Emit a scanner alert as an S3 notification event through the standard +/// dispatch pipeline. Fire-and-forget: the notify layer owns delivery, +/// retry, and target filtering; the scanner never waits on it. +fn emit_scanner_alert_event(event_name: &str, bucket: &str, object: &str, size: i64, details: &[(&str, String)]) { + let mut req_params = HashMap::with_capacity(details.len()); + for (key, value) in details { + req_params.insert((*key).to_string(), value.clone()); + } + ecstore_send_event(EcstoreEventArgs { + event_name: event_name.to_string(), + bucket_name: bucket.to_string(), + object: crate::ScannerObjectInfo { + bucket: bucket.to_string(), + name: object.to_string(), + size, + ..Default::default() + }, + req_params, + user_agent: "Scanner".to_string(), + ..Default::default() + }); +} const MAX_PENDING_SCANNER_HEALS_PER_BUCKET: usize = 10_000; static SCANNER_INLINE_HEAL_WARN_ONCE: Once = Once::new(); @@ -1350,6 +1446,7 @@ impl ScannerItem { fn alert_excessive_versions(&self, remaining_versions: usize, cumulative_size: i64) { ensure_scanner_alert_metrics_registered(); let (too_many_versions, too_large_versions) = should_alert_excessive_versions(remaining_versions, cumulative_size); + let object_path = self.object_path(); if too_many_versions { global_metrics().record_scanner_source_executed(ScannerWorkSource::Alerts, 1); counter!( @@ -1357,13 +1454,26 @@ impl ScannerItem { "bucket" => self.bucket.clone() ) .increment(1); + if scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, &self.bucket, &object_path, scanner_alert_cooldown()) + { + emit_scanner_alert_event( + EVENT_SCANNER_MANY_VERSIONS, + &self.bucket, + &object_path, + cumulative_size, + &[ + ("versions", remaining_versions.to_string()), + ("threshold", scanner_excess_versions_threshold().to_string()), + ], + ); + } warn!( target: "rustfs::scanner::folder", event = EVENT_SCANNER_ALERT_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_FOLDER, bucket = %self.bucket, - object = %self.object_path(), + object = %object_path, versions = remaining_versions, threshold = scanner_excess_versions_threshold(), state = "excess_versions", @@ -1377,13 +1487,31 @@ impl ScannerItem { "bucket" => self.bucket.clone() ) .increment(1); + if scanner_alert_emission_allows( + ScannerAlertKind::LargeVersions, + &self.bucket, + &object_path, + scanner_alert_cooldown(), + ) { + emit_scanner_alert_event( + EVENT_SCANNER_LARGE_VERSIONS, + &self.bucket, + &object_path, + cumulative_size, + &[ + ("versions", remaining_versions.to_string()), + ("cumulativeSize", cumulative_size.to_string()), + ("threshold", scanner_excess_version_size_threshold().to_string()), + ], + ); + } warn!( target: "rustfs::scanner::folder", event = EVENT_SCANNER_ALERT_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_FOLDER, bucket = %self.bucket, - object = %self.object_path(), + object = %object_path, versions = remaining_versions, cumulative_size, threshold = scanner_excess_version_size_threshold(), @@ -1764,6 +1892,15 @@ impl FolderScanner { "root" => self.root.clone() ) .increment(1); + if scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, &self.root, folder, scanner_alert_cooldown()) { + emit_scanner_alert_event( + EVENT_SCANNER_BIG_PREFIX, + &self.root, + folder, + 0, + &[("folders", total_folders.to_string()), ("threshold", threshold.to_string())], + ); + } warn!( target: "rustfs::scanner::folder", event = EVENT_SCANNER_ALERT_STATE, @@ -3232,6 +3369,90 @@ mod tests { #[cfg(unix)] use std::os::unix::fs::{PermissionsExt, symlink}; use std::sync::Mutex; + + /// Reset the process-global alert cooldown map; test-only. + fn reset_alert_cooldowns() { + *SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = Some(ScannerAlertCooldownMap::new()); + } + + /// The emitted event-name strings must be exactly what `EventName` + /// serializes, or a bucket notification subscribed to the documented name + /// would silently never match (rustfs/backlog#1868). + #[test] + fn scanner_alert_wire_names_match_canonical_event_names() { + use rustfs_s3_types::EventName; + assert_eq!(EVENT_SCANNER_MANY_VERSIONS, EventName::ScannerManyVersions.to_string()); + assert_eq!(EVENT_SCANNER_LARGE_VERSIONS, EventName::ScannerLargeVersions.to_string()); + assert_eq!(EVENT_SCANNER_BIG_PREFIX, EventName::ScannerBigPrefix.to_string()); + } + + fn cooldown_map_len() -> usize { + SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .as_ref() + .map(|map| map.len()) + .unwrap_or(0) + } + + /// Backdate every recorded cooldown so the next check fires again. + fn expire_all_alert_cooldowns(cooldown: Duration) { + let now = Instant::now(); + let mut guard = SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if let Some(map) = guard.as_mut() { + for fired_at in map.values_mut() { + if let Some(expired) = now.checked_sub(cooldown + Duration::from_secs(1)) { + *fired_at = expired; + } + } + } + } + + /// The emission gate is the only thing standing between an over-threshold + /// object and one S3 event per scan cycle, so its edge semantics get + /// pinned directly. All scenarios share one #[test] because the cooldown + /// map is process-global and parallel tests would read each other's + /// firings. + #[test] + fn scanner_alert_emission_is_edge_held_per_key_and_bounded() { + reset_alert_cooldowns(); + let cooldown = Duration::from_secs(3600); + + // First firing allows, an immediate re-check is held. + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown)); + assert!(!scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown)); + + // Different kind, object, and bucket are independent keys. + assert!(scanner_alert_emission_allows(ScannerAlertKind::LargeVersions, "bkt", "obj", cooldown)); + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "other", cooldown)); + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "other", "obj", cooldown)); + assert_eq!(cooldown_map_len(), 4); + + // After the cooldown elapses the same key fires again. + expire_all_alert_cooldowns(cooldown); + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown)); + + // A zero cooldown degenerates to always-emit (operators may want that). + assert!(scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", "dir", Duration::ZERO)); + assert!(scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", "dir", Duration::ZERO)); + + // Hard bound: overflow the cap with zero-cooldown keys and confirm the + // map clears rather than growing past it. + reset_alert_cooldowns(); + for index in 0..=(MAX_SCANNER_ALERT_COOLDOWN_KEYS + 8) { + let _ = scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", &format!("dir-{index}"), Duration::ZERO); + } + assert!( + cooldown_map_len() <= MAX_SCANNER_ALERT_COOLDOWN_KEYS, + "cooldown map must stay bounded, got {}", + cooldown_map_len() + ); + } + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use temp_env::{with_var, with_var_unset}; use tracing_subscriber::fmt::MakeWriter; diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index e77033aa2..cbd7c6486 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -78,6 +78,7 @@ pub(crate) use rustfs_ecstore::api::disk::{ pub(crate) use rustfs_ecstore::api::error::{ Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError, }; +pub(crate) use rustfs_ecstore::api::event::{EventArgs as EcstoreEventArgs, send_event as ecstore_send_event}; #[cfg(test)] pub(crate) use rustfs_ecstore::api::layout::{ EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints, @@ -110,8 +111,8 @@ pub(crate) mod owner { ECSTORE_BUCKET_META_PREFIX, ECSTORE_RUSTFS_META_BUCKET, ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS, ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys, EcstoreDisk, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions, - EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreLcEventSrc, - EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts, + EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreEventArgs, + EcstoreLcEventSrc, EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts, EcstoreReplicationConfigurationExt, EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi, ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, @@ -121,7 +122,7 @@ pub(crate) mod owner { ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, - ecstore_save_config, scanner_replication_config_for_lifecycle_eval, + ecstore_save_config, ecstore_send_event, scanner_replication_config_for_lifecycle_eval, }; #[cfg(test)] diff --git a/docs/operations/scanner-excess-alerts.md b/docs/operations/scanner-excess-alerts.md new file mode 100644 index 000000000..8079daba6 --- /dev/null +++ b/docs/operations/scanner-excess-alerts.md @@ -0,0 +1,37 @@ +# Scanner Excess Alerts: Metrics, S3 Events, and Thresholds + +> 中文版：[scanner-excess-alerts_zh.md](scanner-excess-alerts_zh.md) + +Date: 2026-08-18 (rustfs/backlog#1868 / HS-04; includes the HS-15 threshold-delta notes) + +The background scanner detects three classes of "excess" conditions while it walks buckets and surfaces them as alerts. This page documents each alert's trigger condition, the subscribable S3 event, the cooldown semantics, and the threshold differences versus MinIO — for operators debugging alerts and for event consumers wiring up subscriptions. + +## The three alerts + +| Alert | Trigger (per scan cycle) | Metric | S3 event (RustFS wire name) | MinIO event name | +|---|---|---|---|---| +| Excess versions | Retained versions of one object ≥ `scanner:alert_excess_versions` | `rustfs_scanner_excess_object_versions_total{bucket}` | `s3:Scanner:ManyVersions` | `s3:ObjectManyVersions` | +| Excess version size | Cumulative bytes of all versions of one object ≥ `scanner:alert_excess_version_size` | `rustfs_scanner_excess_object_version_size_total{bucket}` | `s3:Scanner:LargeVersions` | `s3:ObjectLargeVersions` | +| Excess folders | Direct subfolders of one directory > `scanner:alert_excess_folders` | `rustfs_scanner_excess_folders_total{root}` | `s3:Scanner:BigPrefix` | `s3:PrefixManyFolders` | + +Subscribe like any bucket notification: configure a notification on the target bucket with the RustFS wire name above (or the `s3:Scanner:*` wildcard). Events carry `UserAgent: Scanner` as their origin marker, and `req_params` holds the observed value and the threshold (`versions` / `cumulativeSize` / `folders` / `threshold`), so consumers can judge severity directly. + +## Metrics and events fire on different cadences + +- **Metrics and structured logs are level-triggered**: as long as the object stays over the threshold, every scan cycle counts and logs it (default cycle ≈ 60s; see `scanner:speed`). +- **S3 events are edge-triggered with a cooldown**: the same (alert kind, bucket, object) emits at most once per cooldown window — 24 hours by default (`RUSTFS_SCANNER_ALERT_COOLDOWN_SECS`; set it to 0 to emit every cycle). When the window lapses and the object is still over the threshold, the event fires again. The cooldown table lives in process memory with a 4096-entry hard cap; on overflow it is cleared and rebuilt (worst case: one extra emission per still-hot key). +- A process restart resets the cooldown (every still-over-threshold object emits once more after a restart) — deliberately: restarts usually accompany incident response, and the re-emission buys visibility. + +## Threshold defaults and the MinIO deltas (HS-15) + +| Config key | ENV | RustFS default | MinIO default | Notes | +|---|---|---|---|---| +| `scanner:alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | 100 | 100 | Identical | +| `scanner:alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | 1 TiB | 1 TB | Same order of magnitude; different unit basis (TiB vs TB) | +| `scanner:alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | 65538 | 50000 | **Deliberate divergence**: 65538 tolerates the Proxmox Backup Server chunk layout (65536 chunks per directory plus the directory's own entries); MinIO's 50000 would fire continuously for PBS users. Set it to 50000 explicitly to match MinIO behavior | + +All three keys accept both env and admin config (`PUT /rustfs/admin/v3/config`, `scanner` subsystem); hot updates take effect immediately. + +## Why the event names are mapped + +RustFS's event enum (`rustfs_s3_types::EventName::ScannerManyVersions/LargeVersions/BigPrefix`) keeps the repo's established `s3:Scanner:*` wire names (literally different from MinIO's `s3:ObjectManyVersions`; the enum comments preserve the mapping). Subscribers should use the RustFS wire names in this page. If you need MinIO-literal compatibility, map the names on the console/consumer side — do not change the published wire names. diff --git a/docs/operations/scanner-excess-alerts_zh.md b/docs/operations/scanner-excess-alerts_zh.md new file mode 100644 index 000000000..d4f4995ac --- /dev/null +++ b/docs/operations/scanner-excess-alerts_zh.md @@ -0,0 +1,37 @@ +# Scanner 超限告警：指标、S3 事件与阈值 + +> English version: [scanner-excess-alerts.md](scanner-excess-alerts.md) + +日期：2026-08-18（rustfs/backlog#1868 / HS-04，含 HS-15 阈值差异说明） + +后台 scanner 在扫描过程中检测三类"超限"状态并对外告警。本文说明每类告警的触发条件、可订阅的 S3 事件、冷却语义，以及与 MinIO 的阈值差异，供运维排障与事件消费方对接。 + +## 三类告警 + +| 告警 | 触发条件（任一扫描周期） | 指标 | S3 事件（RustFS wire 名） | MinIO 对应事件名 | +|---|---|---|---|---| +| 版本数超限 | 单对象保留版本数 ≥ `scanner:alert_excess_versions` | `rustfs_scanner_excess_object_versions_total{bucket}` | `s3:Scanner:ManyVersions` | `s3:ObjectManyVersions` | +| 版本总大小超限 | 单对象全部版本累计字节 ≥ `scanner:alert_excess_version_size` | `rustfs_scanner_excess_object_version_size_total{bucket}` | `s3:Scanner:LargeVersions` | `s3:ObjectLargeVersions` | +| 子目录数超限 | 单目录直接子目录数 > `scanner:alert_excess_folders` | `rustfs_scanner_excess_folders_total{root}` | `s3:Scanner:BigPrefix` | `s3:PrefixManyFolders` | + +订阅方式与普通桶通知一致：对目标桶配置 notification，事件名填上表 RustFS wire 名（或通配 `s3:Scanner:*`）。事件以 `UserAgent: Scanner` 标记来源，`req_params` 携带实际值与阈值（`versions` / `cumulativeSize` / `folders` / `threshold`），便于消费方直接判断严重程度。 + +## 指标与事件的触发节奏不同 + +- **指标与结构化日志是电平触发**：只要对象仍在阈值之上，每个扫描周期都会计数/打日志（默认周期约 60s，见 `scanner:speed`）。 +- **S3 事件是边沿触发 + 冷却**：同一 (告警类型， 桶, 对象) 在冷却窗口内只发一次，默认 24 小时（`RUSTFS_SCANNER_ALERT_COOLDOWN_SECS`，设 0 表示每周期都发）。窗口过后对象仍超限会再次发出。冷却表在进程内有 4096 条硬顶，超限清空重建（最坏情况是每个仍超限的 key 多发一次）。 +- 进程重启会重置冷却（重启后每个仍超限的对象会再发一次）——这是有意为之：重启常伴随排障，重发提供可见性。 + +## 阈值默认值与 MinIO 差异（HS-15） + +| 配置键 | ENV | RustFS 默认 | MinIO 默认 | 差异说明 | +|---|---|---|---|---| +| `scanner:alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | 100 | 100 | 一致 | +| `scanner:alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | 1 TiB | 1 TB | 语义同量级，单位口径不同（TiB vs TB） | +| `scanner:alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | 65538 | 50000 | **有意差异**：65538 兼容 Proxmox Backup Server 的 chunk 布局（每目录 65536 个 chunk + 目录自身条目），按 MinIO 的 50000 会对 PBS 用户持续误报。如需与 MinIO 行为一致可显式配置为 50000 | + +三个键均支持 env 与 admin config（`PUT /rustfs/admin/v3/config` 的 `scanner` 子系统）双通道，热更新即时生效。 + +## 事件名映射的由来 + +RustFS 的事件枚举（`rustfs_s3_types::EventName::ScannerManyVersions/LargeVersions/BigPrefix`）沿用仓库既有 wire 名 `s3:Scanner:*`（与 MinIO 的 `s3:ObjectManyVersions` 字面不同，枚举注释中保留了映射关系）。订阅方应以本文的 RustFS wire 名为准；如需 MinIO 字面兼容，请在 console/消费侧做名称映射，不要修改已发布的 wire 名。 From 00de43528c67cf365a6e45f9f779aa46d16eebe6 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 09:55:29 +0800 Subject: [PATCH 15/51] fix(ecstore): describe peer bucket RPC failures with no details (#6190) heal_bucket, list_bucket, get_bucket_info and delete_bucket returned Error::other("") when a peer answered success=false without an error payload, so operators saw a bare "io error " after quorum reduction. Route all five bucket RPCs through peer_failure_without_details, which names the operation and bucket while staying identical across the peers of one operation so reduce_errs keeps grouping them into a single dominant error. --- .../ecstore/src/cluster/rpc/peer_s3_client.rs | 58 ++++++++++++++++--- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs index de7f0e595..02c9f75f1 100644 --- a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs @@ -214,6 +214,19 @@ fn pool_write_quorum(participant_count: usize) -> usize { (participant_count / 2) + 1 } +/// Error for a peer that reported `success = false` without an error payload. +/// +/// The message must stay identical across the peers of one operation: `reduce_errs` +/// buckets `Error::Io` by kind plus rendered message, so any per-peer detail (address, +/// timing) would split one shared failure into single-count buckets and downgrade a real +/// dominant error into `ErasureWriteQuorum`. +fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error { + match bucket { + Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")), + None => Error::other(format!("{op}: peer returned failure without error details")), + } +} + fn reduce_pool_write_quorum_errs(per_pool_errs: &[Option]) -> Option { if per_pool_errs.is_empty() { return Some(Error::ErasureWriteQuorum); @@ -1078,7 +1091,7 @@ impl PeerS3Client for RemotePeerS3Client { return if let Some(err) = response.error { Err(err.into()) } else { - Err(Error::other("")) + Err(peer_failure_without_details("heal_bucket", Some(bucket))) }; } @@ -1105,7 +1118,7 @@ impl PeerS3Client for RemotePeerS3Client { return if let Some(err) = response.error { Err(err.into()) } else { - Err(Error::other("")) + Err(peer_failure_without_details("list_bucket", None)) }; } let bucket_infos = response @@ -1136,9 +1149,7 @@ impl PeerS3Client for RemotePeerS3Client { return if let Some(err) = response.error { Err(err.into()) } else { - Err(Error::other(format!( - "make_bucket({bucket}): peer returned failure without error details" - ))) + Err(peer_failure_without_details("make_bucket", Some(bucket))) }; } @@ -1162,7 +1173,7 @@ impl PeerS3Client for RemotePeerS3Client { return if let Some(err) = response.error { Err(err.into()) } else { - Err(Error::other("")) + Err(peer_failure_without_details("get_bucket_info", Some(bucket))) }; } let bucket_info = serde_json::from_str::(&response.bucket_info)?; @@ -1190,7 +1201,7 @@ impl PeerS3Client for RemotePeerS3Client { return if let Some(err) = response.error { Err(err.into()) } else { - Err(Error::other("")) + Err(peer_failure_without_details("delete_bucket", Some(bucket))) }; } @@ -2314,4 +2325,37 @@ mod tests { .collect::>(); assert_eq!(calls, vec![1, 1, 0, 0, 0, 0, 0, 0]); } + + #[test] + fn peer_failure_without_details_names_operation_and_bucket() { + for op in ["heal_bucket", "make_bucket", "get_bucket_info", "delete_bucket"] { + let message = peer_failure_without_details(op, Some("ops-bucket")).to_string(); + assert!(message.contains(op), "{op} message must name the operation: {message}"); + assert!(message.contains("ops-bucket"), "{op} message must name the bucket: {message}"); + } + + let message = peer_failure_without_details("list_bucket", None).to_string(); + assert!(message.contains("list_bucket"), "cluster-wide message must name the operation"); + assert!(!message.trim().is_empty()); + } + + #[test] + fn peer_failure_without_details_keeps_one_reduce_errs_bucket_per_operation() { + // reduce_errs groups Io errors by kind plus rendered message: peers failing the + // same operation on the same bucket must still reach quorum as one dominant error. + let per_pool_errs = vec![ + Some(peer_failure_without_details("delete_bucket", Some("shared"))), + Some(peer_failure_without_details("delete_bucket", Some("shared"))), + Some(peer_failure_without_details("delete_bucket", Some("shared"))), + ]; + assert_eq!( + reduce_pool_write_quorum_errs(&per_pool_errs), + Some(peer_failure_without_details("delete_bucket", Some("shared"))) + ); + + assert_ne!( + peer_failure_without_details("delete_bucket", Some("shared")), + peer_failure_without_details("get_bucket_info", Some("shared")) + ); + } } From 9a2d06b370b4665116a93976377d658de65f1459 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 10:01:04 +0800 Subject: [PATCH 16/51] test(heal): lock heal vs delete/overwrite race invariants (HS-12) (#6183) * test(heal): add concurrency invariants for heal vs delete/overwrite races (HS-12) Audit conclusion for backlog#1874: RustFS does not need a persistent object-level healing marker (MinIO x-minio-healing) because every path that can touch the same (bucket, object) commit surface serializes on the same namespace write lock, and the heal lock guard spans the whole rename commit including the HEAL_RENAME_INCOMPLETE partial path. Lock the conclusion in with two race regression tests: - heal_racing_version_delete_never_resurrects_the_deleted_version: shard damage is injected on the doomed version so a Deep heal has real reconstruction work while a versioned DELETE runs concurrently; the deleted version must stay deleted and the survivor intact. - heal_racing_unversioned_overwrites_preserves_the_last_commit: unversioned overwrites (activating the post-commit tail that deletes the replaced data dir without the ns lock) race a Deep heal in a loop; the final current version must be exactly the last commit. Also adds docs/operations/heal-concurrency-safety-notes-zh.md with the full intersection matrix (17 intersections), lock-coverage argument, and the residual-window classification (commit tail races are fail-into-retry safe; bare prefix delete has zero production callers; admin no_lock is an explicit operator opt-in). Co-Authored-By: heihutu * test: remove redundant heal etag clone Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- crates/ecstore/src/set_disk/ops/heal.rs | 219 ++++++++++++++++++ .../heal-concurrency-safety-notes-zh.md | 115 +++++++++ 2 files changed, 334 insertions(+) create mode 100644 docs/operations/heal-concurrency-safety-notes-zh.md diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index de688fbdb..7c228595d 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -3297,4 +3297,223 @@ mod heal_result_report_tests { assert!(result.detail.contains("part 1")); assert!(result.detail.contains("bitrot_failure=true")); } + + // HS-12 (backlog#1874): a versioned DELETE racing an object heal must never + // resurrect the deleted version. The heal has real reconstruction work (a + // shard of the doomed version is removed), so both sides touch the same + // (bucket, object, data_dir); whichever order the ns write lock serializes + // them in, the committed delete must win. + #[tokio::test] + #[serial_test::serial] + async fn heal_racing_version_delete_never_resurrects_the_deleted_version() { + let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "heal-race-delete-no-resurrect"; + let object = "object.bin"; + set.make_bucket( + bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .expect("versioned bucket should be created"); + + let mut first_reader = PutObjReader::from_vec(vec![0x11; 1024 * 1024]); + let first_info = set + .put_object( + bucket, + object, + &mut first_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("first version should be written"); + let first_version = first_info + .version_id + .expect("versioned put should return the first version id") + .to_string(); + + let mut second_reader = PutObjReader::from_vec(vec![0x22; 1024 * 1024]); + let second_info = set + .put_object( + bucket, + object, + &mut second_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("second version should be written"); + let second_version = second_info + .version_id + .expect("versioned put should return the second version id") + .to_string(); + + // Damage one shard of the doomed version so the racing heal performs an + // actual reconstruction over its data dir instead of an early exit. + let doomed_source = disks[0] + .read_version("", bucket, object, &first_version, &ReadOptions::default()) + .await + .expect("doomed version metadata should be readable"); + let doomed_data_dir = doomed_source + .data_dir + .expect("non-inline version should have a data directory"); + tokio::fs::remove_file( + temp_dirs[1] + .path() + .join(bucket) + .join(object) + .join(doomed_data_dir.to_string()) + .join("part.1"), + ) + .await + .expect("shard damage should be injected before the race"); + + let delete_set = set.clone(); + let (delete_res, heal_res) = tokio::join!( + async { + delete_set + .delete_object( + bucket, + object, + ObjectOptions { + versioned: true, + version_id: Some(first_version.clone()), + object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new( + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await + }, + async { + set.heal_object( + bucket, + object, + "", + &HealOpts { + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + }, + ); + delete_res.expect("version delete must succeed under lock serialization"); + // The heal may legitimately report a transient failure when the version + // it was rebuilding disappears mid-flight; only the end state matters. + drop(heal_res); + + let resurrected = set + .get_object_info( + bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(first_version.clone()), + ..Default::default() + }, + ) + .await; + assert!( + matches!(&resurrected, Err(Error::FileVersionNotFound) | Err(Error::ObjectNotFound(..))), + "a racing heal must not resurrect the deleted version: {resurrected:?}" + ); + + let survivor = set + .get_object_info( + bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(second_version.clone()), + ..Default::default() + }, + ) + .await + .expect("surviving version must remain readable after the race"); + assert_eq!(survivor.size, 1024 * 1024, "survivor size must be intact"); + } + + // HS-12 (backlog#1874): unversioned overwrite commits race a Deep heal on + // the same object. The overwrite's post-commit tail deletes the replaced + // data dir without the ns lock (object.rs commit tail), which is exactly + // the intersection the audit flagged: the heal must tolerate the tail race + // (retryable outcome) and every committed overwrite must survive — the + // final current version is exactly the last payload written. + #[tokio::test] + #[serial_test::serial] + async fn heal_racing_unversioned_overwrites_preserves_the_last_commit() { + let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "heal-race-put-overwrite"; + let object = "object.bin"; + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + + const ROUNDS: usize = 8; + const PAYLOAD_SIZE: usize = 256 * 1024; + let mut last_etag = String::new(); + for round in 0..ROUNDS { + // Give the heal something to rebuild on alternating rounds: remove a + // shard of the current data dir right before the race. + if round % 2 == 1 { + let current = disks[2] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("current metadata should be readable"); + if let Some(data_dir) = current.data_dir { + let shard = temp_dirs[3] + .path() + .join(bucket) + .join(object) + .join(data_dir.to_string()) + .join("part.1"); + if shard.exists() { + tokio::fs::remove_file(&shard) + .await + .expect("shard damage should be injectable mid-race"); + } + } + } + + let payload = vec![round as u8; PAYLOAD_SIZE]; + let mut put_reader = PutObjReader::from_vec(payload); + let put_opts = ObjectOptions::default(); + let heal_opts = HealOpts { + scan_mode: HealScanMode::Deep, + ..Default::default() + }; + let (put_res, heal_res) = tokio::join!( + set.put_object(bucket, object, &mut put_reader, &put_opts), + set.heal_object(bucket, object, "", &heal_opts), + ); + let put_info = put_res.expect("overwrite must succeed under lock serialization"); + last_etag = put_info.etag.clone().unwrap_or_default(); + // Heal outcome is unconstrained (may hit the tail race and report a + // retryable error); the invariant is checked on the end state. + drop(heal_res); + } + + let final_info = set + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("object must remain readable after the race loop"); + assert_eq!( + final_info.size, PAYLOAD_SIZE as i64, + "final current version must be the last committed overwrite" + ); + assert_eq!( + final_info.etag.unwrap_or_default(), + last_etag, + "the racing heal loop must never leave a stale or resurrected current version" + ); + } } diff --git a/docs/operations/heal-concurrency-safety-notes-zh.md b/docs/operations/heal-concurrency-safety-notes-zh.md new file mode 100644 index 000000000..ffb1e90c7 --- /dev/null +++ b/docs/operations/heal-concurrency-safety-notes-zh.md @@ -0,0 +1,115 @@ +# Heal 并发安全说明（对象级 healing 标记对标审计结论） + +对应 backlog rustfs/backlog#1874（父 #1862，HS-12）。本文回答一个问题：MinIO 在 heal +期间对对象打 `x-minio-healing:true` 元数据标记以防"heal 提交与并发删除/版本清理互毁" +（cmd/xl-storage.go RenameData 的 healing 分支），RustFS 是否需要同款防御。 + +**结论：不需要。** RustFS 不存在 MinIO 用 healing 标记防御的那类竞争：所有会触达同一 +`(bucket, object)` 提交面的路径都在同一把对象级 namespace 写锁上互斥，且 heal 的锁 +guard 覆盖 rename 提交全程；MinIO 需要标记的根因（RenameData 提交内部与版本清理逻辑 +交错）在 RustFS 的提交模型中不存在。RustFS 已有一个瞬态 healing 旗标用于另一目的 +（见下文 §2），并有并发不变量回归测试锁定本结论（§5）。 + +## 1. 两个防御模型的对照 + +MinIO：heal 时对对象写 `x-minio-healing:true`（持久元数据标记），后续任何 RenameData +提交看到该标记就跳过版本清理/legacy purge 逻辑——防御发生在锁外，靠元数据让路。 + +RustFS：三层防御，全部不依赖持久对象标记： + +1. **锁内互斥**：heal 与一切前台/后台写路径的提交点在同一把 `(bucket, object)` ns 写锁 + 上串行（分布式部署为 quorum 锁 RPC，单机为进程内锁管理器；锁粒度是对象级，version + 恒为 None）。 +2. **提交模型隔离**：rename_data 提交内没有会与 heal 交错的版本清理逻辑；被替换旧版本 + 的 data_dir 物理删除被移出提交临界区（commit tail），且只删已被新提交替换的 unshared + 目录。 +3. **瞬态 healing 旗标**：`FileInfo::set_healing`（crates/filemeta/src/fileinfo.rs）在 + heal 提交的内存 FileInfo 上打 `"healing"` 内部键，rename_data 据此允许先清空 stale + 目标 data_dir 再 rename——解决 heal 复用 data_dir 做 in-place 修复时 rename(2) 无法 + 替换非空目录的文件系统语义冲突（EEXIST/ENOTEMPTY）。该键是瞬态的，不落盘 + （`is_skip_meta_key`），与 MinIO 的持久标记目的不同。非 heal 提交撞上非空目标 + data_dir 会显式失败，有测试锁定两个方向的行为。 + +## 2. 交点矩阵 + +中心路径：`heal_object_with_explicit_version_regen`（crates/ecstore/src/set_disk/ops/heal.rs， +下称 heal.rs）在入口取 `(bucket, object)` ns 写锁，guard 绑定到函数作用域末尾，覆盖 +quorum 元数据读取 → EC 重建 → 逐盘 rename 提交 → tmp 清理 → HEAL_RENAME_INCOMPLETE +部分提交返回 → 孤儿 data_dir 回收的全过程。并发侧逐交点判定： + +| # | 并发路径 | 并发侧锁 | 判定 | 关键证据 | +|---|---|---|---|---| +| 1 | PUT 对象提交 | `put_object_commit` 对象写锁，rename_data 在锁内 | 同锁串行 | ops/object.rs 提交锁段 + rename 调用点 | +| 2 | PUT 旧 data_dir tail 清理 | drop 对象锁后的 `commit_rename_data_dir`，无锁 | 无锁并发，语义安全（见 §3.1） | object.rs drop 后 tail 段；io_primitives.rs | +| 3 | DELETE 单对象/版本 | `delete_object` 对象写锁，delete_version 在锁内 | 同锁串行 | object.rs delete_object 锁段 | +| 4 | DELETE 批量 | 批量逐对象写锁（dist 走批量锁 RPC） | 同锁串行 | object.rs delete_objects 锁段 | +| 5 | CompleteMultipart | 对象写锁 + upload 路径锁双锁，rename 在锁内 | 同锁串行 | ops/multipart.rs 提交锁段 | +| 6 | CompleteMultipart tail 清理 | drop 对象锁后的旧 data_dir 删除 | 无锁并发，语义安全（见 §3.1） | multipart.rs drop 后 tail 段 | +| 7 | AbortMultipart | 仅 multipart bucket 的 upload 路径锁 | 锁 key 不相交，但资源不相交（abort 不触对象 data_dir/xl.meta）→ 无实际交点 | multipart.rs abort 锁段 | +| 8 | ILM expiry（含 DeleteAllVersions） | DeleteAllVersions 走 `delete_prefix_object=true` → 仍取对象锁；FreeVersionTask 显式取锁；noncurrent 批量走批量锁 | 同锁串行 | bucket_lifecycle_ops.rs 消费端链路 | +| 9 | 纯 prefix 删除（绕锁能力面） | `delete_prefix`-only 不取子对象锁 | 无锁并发，但生产调用方为零（见 §3.2） | object.rs delete_object 锁条件 | +| 10 | 孤儿 data_dir 回收 reclaim_orphan_data_dirs | 函数本体无锁；唯一生产调用方在 heal 锁内 | heal 流程内=锁内串行 | heal.rs 收尾调用；io_primitives.rs | +| 11 | 旧清理 receipt 对账 reconcile_old_data_cleanup_receipts | 函数本体无锁；调用点在 heal 锁内 + epoch fence 防误删 | 锁内串行 | object.rs 对账函数 | +| 12 | replication | 数据面为远端 HTTP 写（不落本地盘）；本地元数据回写走对象锁 | 同锁串行 / 无交点 | replication_resyncer.rs 链路 | +| 13 | data_movement / rebalance / decommission 源清理 | 显式取对象锁 + 版本未变复核 + guard 复用（no_lock 只是复用已持锁） | 同锁串行 | data_movement/mod.rs 源清理 | +| 14 | copy_object | 目标对象锁 / 走 put 链锁 | 同锁串行 | object.rs copy_object 锁段 | +| 15 | 另一 heal 任务（跨 HealType/force_start） | dedup key 跨类型不相交 + force_start 跳过去重 → 任务级可并发 | 最终在 ns 写锁上串行 | heal/manager.rs dedup key 构成 | +| 16 | admin `no_lock=true` heal | 客户端可控绕锁 | 无锁并发，明示运维选项（见 §3.3） | admin/handlers/heal.rs 透传 | +| 17 | stale multipart 清理 | multipart bucket 的 upload 路径锁 | 资源不相交 → 无交点 | bucket_lifecycle_ops.rs 清理链路 | + +## 3. 残留窗口定性 + +### 3.1 PUT/CompleteMultipart commit tail（交点 2/6） + +写路径提交成功、释放对象锁之后，才 best-effort 删除被替换的旧 data_dir（注释明示有意 +不阻塞下一操作）。该删除与并发 heal 对同一旧 data_dir 的读取/重建存在竞态窗口，但语义 +安全： + +- 删除目标是已被新提交替换的 unshared data_dir；heal 的 canonical 元数据来自 quorum + 仲裁（ETag/mod_time），此时 quorum 已指向新版本，heal 不会把已替换版本当作 canonical + 复活； +- 竞态最坏后果 = heal 当轮对旧版本的一次 transient 失败/空转，重试轮自然收敛；清理 + residue 会上报并重新入队 heal（`report_old_data_dir_cleanup`）； +- 换盘重建等长 heal 走 per-version 显式版本请求，quorum 元数据在锁内读取，不受 tail + 影响。 + +### 3.2 纯 prefix 删除（交点 9） + +`delete_prefix && !delete_prefix_object` 的路径不取子对象锁（对象名空间锁无法保护前缀 +递归删除），与并发 heal 存在理论复活窗口（heal 在 prefix 删除进行中依据旧 quorum 元 +数据重建某版本）。全仓库核对结论：该路径的**生产调用方为零**——所有生产 `delete_prefix: +true` 调用点均同时设置 `delete_prefix_object: true`（从而取对象锁）或在测试模块内。这 +是 API 能力面的暴露而非行为风险。若未来有调用方需要纯 prefix 删除，须在调用点证明与 +heal/scanner 的隔离（例如 bucket 级停扫围栏）。 + +### 3.3 admin `no_lock=true`（交点 16） + +admin heal 请求可透传客户端 `nolock` 参数绕过 ns 锁（与 MinIO madmin 的同名选项对齐）。 +这是运维明示选项：使用即自负与并发写的竞争责任。文档化即可，不建议收紧。 + +## 4. heal 侧自身的不变量保障 + +- dedup key 跨 HealType 不相交（object/metadata/mrf/ecdecode/prefix 各自键面）+ admin + `force_start` 可跳过去重 → 同对象可能同时存在多个 heal 任务，但它们的执行体全部在 + `heal_object` 入口的 ns 写锁上串行（生产入口均 `no_lock=false`）； +- read-repair 的本地 TTL 预留只去重自身来源，不拦截其他来源的 heal——同样由 ns 锁兜底； +- healing 旗标不落盘，故不存在"标记残留导致后续提交错误让路"的反向风险。 + +## 5. 回归测试 + +以下两个并发不变量测试随本审计加入 `crates/ecstore/src/set_disk/ops/heal.rs` 测试模块： + +- `heal_racing_version_delete_never_resurrects_the_deleted_version`：注入 doomed 版本 + shard 损坏后，版本化 DELETE 与 Deep heal 真并发（同一把锁争用），断言已删除版本不被 + 复活、存活版本完好； +- `heal_racing_unversioned_overwrites_preserves_the_last_commit`：非版本化覆盖提交（激活 + commit tail 旧 data_dir 删除）与 Deep heal 循环竞态，断言最终 current 恰为最后一次 + 提交（etag 级一致）。 + +## 6. 结论 + +MinIO 的 `x-minio-healing` 是锁外元数据防御，前提是其 RenameData 提交内部存在与 heal +交错的版本清理逻辑；RustFS 的提交模型把这类交错从根上消除（提交面锁内互斥 + 清理外 +移到 tail + tail 只删 unshared 旧目录），因此引入持久对象级 healing 标记没有对应的竞争 +可防，反而会引入 FileInfo 落盘格式变更与标记残留清理两类新成本。维持现状，本对标疑点 +关闭。 From 84bd76a3ce788c3c515b721aed689f126f87e7a3 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 12:11:53 +0800 Subject: [PATCH 17/51] chore(deps): refresh cargo dependencies (#6198) Update workspace Cargo dependency requirements and lockfile after cargo update/upgrade, including rumqttc-next 0.34.0 and MQTT API compatibility adjustments. Verification: - cargo update --verbose - cargo upgrade --verbose - cargo update -p rumqttc-next --precise 0.34.0 --verbose - cargo tree --invert rumqttc-next --locked - cargo metadata --locked --no-deps --format-version 1 - cargo fmt --all --check - cargo check -p rustfs-targets --all-targets --locked - cargo test -p rustfs-targets mqtt --locked - make pre-pr Co-authored-by: heihutu --- Cargo.lock | 207 +++++++++++++++--------------- Cargo.toml | 16 +-- crates/targets/src/target/mqtt.rs | 29 +++-- 3 files changed, 131 insertions(+), 121 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cda581ffb..82ffafc81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -964,9 +964,9 @@ dependencies = [ [[package]] name = "aws-sdk-kms" -version = "1.114.0" +version = "1.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b7d906608ee41e7ddea9983577ba82200435644d567d63dc34e822e088b453" +checksum = "d5b034f8b7ceadb873d0bc607c30bb4b0be68e09a84c837174e7c2c6878ff882" dependencies = [ "arc-swap", "aws-credential-types", @@ -990,9 +990,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.141.0" +version = "1.142.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f9420d3a2467eed22ed3635ca653653162c386a0b0f65c78189f9bd3c1379e" +checksum = "f9e15a5c55e05f4b0b7e483160b3c85cccdf77cff02c95504f3e71d460855cd2" dependencies = [ "arc-swap", "aws-credential-types", @@ -1027,9 +1027,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.105.0" +version = "1.106.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ffd0fbe7873cb548a7aa60f9573c268fff94155397fd4f14dc9f1ecaaab8516" +checksum = "2d0efcee834347b6705eca3eea2defd88242f43774f55d7326604222e3c86260" dependencies = [ "arc-swap", "aws-credential-types", @@ -1053,9 +1053,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.107.0" +version = "1.108.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175763eb222a46377df7aa257a3bca980ab3e96703fefc8f4d0b8da6ad2e254c" +checksum = "a59312a04cf19c962cfee32b64ecfee758f8786407ff6da5b30fff46ae96f201" dependencies = [ "arc-swap", "aws-credential-types", @@ -1079,9 +1079,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.110.0" +version = "1.111.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b14781dfbff48984017d57167b6ea0b6471c6920ec52b44a2677c7feb3c13" +checksum = "120e7eb63457a9e547f9986fe3b273f77c43679da4d04f46359fa881c5e19b6e" dependencies = [ "arc-swap", "aws-credential-types", @@ -1598,7 +1598,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -1617,7 +1617,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -1858,9 +1858,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" dependencies = [ "find-msvc-tools", "jobserver", @@ -1968,7 +1968,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common 0.1.6", + "crypto-common 0.1.7", "inout 0.1.4", ] @@ -2428,7 +2428,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "rand_core 0.6.4", "subtle", "zeroize", @@ -2453,11 +2453,11 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "typenum", ] @@ -3664,7 +3664,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common 0.1.6", + "crypto-common 0.1.7", "subtle", ] @@ -3924,7 +3924,7 @@ dependencies = [ "crypto-bigint 0.5.5", "digest 0.10.7", "ff 0.13.1", - "generic-array 0.14.9", + "generic-array 0.14.7", "group 0.13.0", "hkdf 0.12.4", "pem-rfc7468 0.7.0", @@ -4148,9 +4148,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "findshlibs" @@ -4369,9 +4369,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -4380,11 +4380,11 @@ dependencies = [ [[package]] name = "generic-array" -version = "1.4.4" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b" +checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "rustversion", "typenum", ] @@ -4726,9 +4726,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -5028,9 +5028,9 @@ dependencies = [ [[package]] name = "hotpath" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e810bedda5a467ef5c9b5c8a20763fefebc89b63ef36f7ee44a143085204a2" +checksum = "dce755d457a63bdd0c95e4c91511daad1b58b33209543b7f38027b676f387e5e" dependencies = [ "arc-swap", "async-channel", @@ -5062,9 +5062,9 @@ dependencies = [ [[package]] name = "hotpath-macros" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01bdc59bfc1a9984bee2ff5da63b2f6fccbaa57cd9a4119d709524632bddf341" +checksum = "a903af89a8429cb07790c3818bc15270b394f80af1bc254e5ccf9c7de2961770" dependencies = [ "proc-macro2", "quote", @@ -5073,15 +5073,15 @@ dependencies = [ [[package]] name = "hotpath-macros-meta" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9216e8a01abe1e1671c376dc8736fb1bf772d7a889538d25f9e1200120ced38" +checksum = "bcc0ab94ffbb2ee77f4a897df02b5a137a10cf24d69bda936e59aff4dd456e61" [[package]] name = "hotpath-meta" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f22a9d20435fb79511b19dae37b3607224cd98f342a410702d84657cc38fc72f" +checksum = "053481f6cec8f775a3276c7f6e2f21123111d28261e4edc15ea7421c445964bb" dependencies = [ "hotpath-macros-meta", ] @@ -5280,9 +5280,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -5294,9 +5294,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -5307,9 +5307,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -5321,16 +5321,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -5341,15 +5342,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428" dependencies = [ "displaydoc", "icu_locale_core", @@ -5417,7 +5418,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ "block-padding 0.3.3", - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -5968,9 +5969,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa" +checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a" dependencies = [ "libc", ] @@ -6033,9 +6034,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "local-ip-address" @@ -6482,9 +6483,9 @@ dependencies = [ [[package]] name = "mqttbytes-core-next" -version = "0.33.3" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff7ae19c74aba9e0ed6e4071cd52aa364e020076fa3cc6ef17e43662f756f3c" +checksum = "366b6ba2b4209ca4bc5ac731ccddf570d09831981eed07e5fbd63564cf0cf1aa" dependencies = [ "bytes", "thiserror 2.0.20", @@ -6885,7 +6886,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "chrono", "getrandom 0.2.17", "http 1.5.0", @@ -7344,9 +7345,9 @@ dependencies = [ [[package]] name = "pageant" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3a5ae18f65a85c67a77d18d42d3606c07948e3c17c1e5f74852b26589e88a5" +checksum = "3adadc44070da6f464b0918655a12f5792c156e088d8c4082d13e27d94c3e791" dependencies = [ "base16ct 1.0.0", "byteorder", @@ -7728,9 +7729,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "plotters" @@ -7836,9 +7837,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -8046,7 +8047,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -8066,7 +8067,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "petgraph 0.8.3", @@ -8087,7 +8088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -8100,7 +8101,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -8216,7 +8217,7 @@ dependencies = [ "reqwest", "serde_json", "smallvec", - "spin 0.12.2", + "spin 0.12.3", "symbolic-demangle", "tempfile", "thiserror 2.0.20", @@ -8285,9 +8286,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ "aws-lc-rs", "bytes", @@ -8553,9 +8554,9 @@ dependencies = [ [[package]] name = "redis" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3257df217f7eab0044627a268c9cc6cdb60c0c421c88f83ac41c4e31520b6b84" +checksum = "e37a4ca5c6ca42aa3e6df2fd32b987a65d32a4c2159a6f3fe0fd1df306a2658f" dependencies = [ "arc-swap", "arcstr", @@ -8567,7 +8568,7 @@ dependencies = [ "futures-channel", "futures-util", "itoa", - "num-bigint 0.4.8", + "num-bigint 0.5.1", "percent-encoding", "pin-project-lite", "rustls", @@ -8868,9 +8869,9 @@ dependencies = [ [[package]] name = "rumqttc-core-next" -version = "0.33.3" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7d9205738dd41a2546e82d27a634d07d8b303dcf7558565ff70caf3ceb0f9c" +checksum = "249896ab27ed630590971738264baa8f722f18965d2e387c706c40a3c2a572cc" dependencies = [ "async-tungstenite", "futures-io", @@ -8886,18 +8887,18 @@ dependencies = [ [[package]] name = "rumqttc-next" -version = "0.33.3" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed1bad2180ff539da671da9a996152a921bc5316eb6d8a9cc3bd441653138b08" +checksum = "477c9bbfba8f3aecc7aad31c6de2eacb75822efaa18e7aeecb8d3d8e534fbf07" dependencies = [ "rumqttc-v5-next", ] [[package]] name = "rumqttc-v5-next" -version = "0.33.3" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229576cbedfa9089f90c17c9454e9429ac1e89cdd223bac5cb39d837593f79bc" +checksum = "3dfa6ddcc7a7dd5688f9bf78d8f81cb94f367bce56c055d8d94cf81ecb0518bf" dependencies = [ "async-tungstenite", "bytes", @@ -8920,9 +8921,9 @@ dependencies = [ [[package]] name = "russh" -version = "0.62.6" +version = "0.62.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b41043523e0edcbd4e31d00903e26f12994f63b21bae9904f7405c1ed92752a5" +checksum = "9decb68e4e44e1079700e54f17c8f23806ec53d7e0db73ab1c71d9dabc666812" dependencies = [ "aes 0.9.2", "aws-lc-rs", @@ -8945,7 +8946,7 @@ dependencies = [ "enum_dispatch", "flate2", "futures", - "generic-array 1.4.4", + "generic-array 1.4.5", "getrandom 0.4.3", "ghash", "hex-literal", @@ -9491,7 +9492,7 @@ dependencies = [ "parking_lot", "rayon", "smallvec", - "spin 0.12.2", + "spin 0.12.3", ] [[package]] @@ -10833,7 +10834,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct 0.2.0", "der 0.7.10", - "generic-array 0.14.9", + "generic-array 0.14.7", "pkcs8 0.10.2", "subtle", "zeroize", @@ -11397,9 +11398,9 @@ checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" [[package]] name = "spin" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8abadc99fd9c7bbb7d0ca2b31d72a067d0c0dcd7aad25ab8cac71ba91417694b" +checksum = "0134f9043ed38b087ac4f7d4af44c79e2c9e5094421fe3164f435ce585953b10" dependencies = [ "lock_api", ] @@ -11811,7 +11812,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -11975,9 +11976,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -12651,9 +12652,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -13168,9 +13169,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "x509-cert" @@ -13350,9 +13351,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -13361,9 +13362,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8" dependencies = [ "yoke", "zerofrom", @@ -13372,13 +13373,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6bbd3acd1..ae9b497e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -228,9 +228,9 @@ atoi = "3.1.0" atomic_enum = "0.3.0" aws-config = { version = "1.10.1" } aws-credential-types = { version = "1.3.0" } -aws-sdk-kms = { default-features = false, version = "1.114.0" } -aws-sdk-s3 = { default-features = false, version = "1.141.0" } -aws-sdk-sts = { default-features = false, version = "1.110.0" } +aws-sdk-kms = { default-features = false, version = "1.115.0" } +aws-sdk-s3 = { default-features = false, version = "1.142.0" } +aws-sdk-sts = { default-features = false, version = "1.111.0" } aws-smithy-http-client = { default-features = false, version = "1.3.0" } aws-smithy-runtime-api = { version = "1.14.0" } aws-smithy-types = { version = "1.6.2" } @@ -284,8 +284,8 @@ rayon = "1.12.0" reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" } reed-solomon-simd = "3.1.0" regex = { version = "1.13.1" } -rumqttc = { package = "rumqttc-next", version = "0.33.3" } -redis = { version = "1.5.0" } +rumqttc = { package = "rumqttc-next", version = "0.34.0" } +redis = { version = "1.6.0" } rustify = { version = "0.7", default-features = false } rustix = { version = "1.1.4" } rust-embed = { version = "8.12.0" } @@ -313,7 +313,7 @@ tracing-subscriber = { version = "0.3.23" } transform-stream = "0.3.1" url = "2.5.8" urlencoding = "2.1.3" -uuid = { version = "1.24.0" } +uuid = { version = "1.24.1" } vaultrs = { version = "0.8.0" } tar = "0.4.46" walkdir = "2.5.0" @@ -341,7 +341,7 @@ libunftp = { version = "0.23.0" } unftp-core = "0.1.0" suppaftp = { version = "10.0.1" } rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] } -russh = { version = "0.62.6" } +russh = { version = "0.62.7" } russh-sftp = "2.4.0" # WebDAV @@ -350,7 +350,7 @@ dav-server = "0.11.0" # Performance Analysis and Memory Profiling mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" } libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] } -hotpath = { version = "0.23.2", default-features = false } +hotpath = { version = "0.23.3", default-features = false } # Snapshot testing for output format regression detection insta = { version = "1.48" } diff --git a/crates/targets/src/target/mqtt.rs b/crates/targets/src/target/mqtt.rs index 3120d85b1..0490c3a91 100644 --- a/crates/targets/src/target/mqtt.rs +++ b/crates/targets/src/target/mqtt.rs @@ -32,8 +32,8 @@ use arc_swap::ArcSwap; use async_trait::async_trait; use hyper_rustls::ConfigBuilderExt; use rumqttc::{ - AsyncClient, Broker, ClientError, ConnectionError, EventLoop, Incoming, MqttOptions, Outgoing, PublishNoticeError, QoS, - Transport, mqttbytes::Error as MqttBytesError, + AsyncClient, Broker, ClientError, ConnectionError, EventLoop, Incoming, MqttOptions, Outgoing, ProtocolViolation, + PublishNoticeError, PublishOptions, QoS, Transport, mqttbytes::Error as MqttBytesError, }; use rustfs_config::{ EnableState, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_WS_PATH_ALLOWLIST, @@ -791,7 +791,7 @@ where .as_ref() .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?; let notice = client - .publish_tracked(&self.args.topic, self.args.qos, false, body) + .publish_tracked(&self.args.topic, body, PublishOptions::new(self.args.qos)) .await .map_err(|error| classify_mqtt_client_error(&error))?; drop(client_guard); @@ -1145,7 +1145,7 @@ async fn run_mqtt_event_loop(mut eventloop: EventLoop, connected_status: Arc { + rumqttc::Event::Incoming(Incoming::PingResp) => { trace!(target_id = %target_id, "Received PingResp from broker. Connection is alive."); } rumqttc::Event::Incoming(Incoming::SubAck(suback)) => { @@ -1257,7 +1257,11 @@ async fn run_mqtt_event_loop(mut eventloop: EventLoop, connected_status: Arc TargetError { match err { - ClientError::Request(_) | ClientError::TryRequest(_) | ClientError::TrackingUnavailable => TargetError::NotConnected, + ClientError::RequestChannelFull(_) | ClientError::RequestChannelDisconnected(_) | ClientError::TrackingUnavailable => { + TargetError::NotConnected + } + ClientError::InvalidRequest(_) => TargetError::Request(format!("Invalid MQTT publish request: {err}")), + _ => TargetError::NotConnected, } } @@ -1270,10 +1274,14 @@ fn classify_mqtt_notice_error(err: &PublishNoticeError) -> TargetError { PublishNoticeError::Recv | PublishNoticeError::SessionReset | PublishNoticeError::Qos0NotFlushed + | PublishNoticeError::BrokerOnlySessionResume + | PublishNoticeError::SessionPersistence(_) | PublishNoticeError::TopicAliasReplayUnavailable(_) => TargetError::NotConnected, + PublishNoticeError::RetainNotSupported => TargetError::Request(format!("MQTT broker rejected publish: {err}")), PublishNoticeError::V5PubAck(_) | PublishNoticeError::V5PubRec(_) | PublishNoticeError::V5PubComp(_) => { TargetError::Request(format!("MQTT broker rejected publish: {err}")) } + _ => TargetError::NotConnected, } } @@ -1299,12 +1307,13 @@ fn is_fatal_mqtt_error(err: &ConnectionError) -> bool { | MqttBytesError::MalformedPacket // Package format error | MqttBytesError::PayloadTooLong // Too long load | MqttBytesError::PayloadSizeLimitExceeded { .. } // Load size limit exceeded - | MqttBytesError::TopicNotUtf8 // Topic Non-UTF-8 (Serious Agreement Violation) + | MqttBytesError::TopicNotUtf8 { .. } // Topic Non-UTF-8 (Serious Agreement Violation) ) } // Others that are fatal StateError variants rumqttc::StateError::InvalidState // The internal state machine is in invalid state - | rumqttc::StateError::WrongPacket // Agreement Violation: Unexpected Data Packet Received + | rumqttc::StateError::ProtocolViolation(ProtocolViolation::UnexpectedIncomingPacket(_)) // Agreement Violation: Unexpected Data Packet Received + | rumqttc::StateError::ProtocolViolation(_) // Agreement Violation | rumqttc::StateError::Unsolicited(_) // Agreement Violation: Unsolicited ACK Received | rumqttc::StateError::CollisionTimeout // Agreement Violation (if this stage occurs) | rumqttc::StateError::EmptySubscription // Agreement violation (if this stage occurs) @@ -1727,8 +1736,8 @@ where mod tests { use super::{ AsyncClient, ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTarget, MQTTTlsConfig, - MqttOptions, PublishNoticeError, QoS, QueuedPayloadMeta, classify_mqtt_client_error, classify_mqtt_notice_error, - next_reconnect_backoff, reconnect_supervisor, validate_mqtt_broker_url, + MqttOptions, PublishNoticeError, PublishOptions, QoS, QueuedPayloadMeta, classify_mqtt_client_error, + classify_mqtt_notice_error, next_reconnect_backoff, reconnect_supervisor, validate_mqtt_broker_url, }; use crate::error::TargetError; use crate::target::{REDACTED_SECRET, TargetType}; @@ -1794,7 +1803,7 @@ mod tests { .capacity(1) .build(); client - .publish("fill", QoS::AtLeastOnce, false, b"fill".as_slice()) + .publish("fill", b"fill".as_slice(), PublishOptions::new(QoS::AtLeastOnce)) .await .expect("first publish should fill the local channel"); *target.client.lock().await = Some(client); From de9145e87ab90e5c8308b52fb68d72b3b3350fb8 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 12:15:19 +0800 Subject: [PATCH 18/51] feat(storage): add default-off PUT admission gate (#6197) Add an experimental fixed-count foreground PutObject admission gate for #1882 Phase 0 validation. The gate is default-off, returns SlowDown before body ingest when saturated, and keeps the admission permit with the spawned store commit owner until store PUT returns. Co-authored-by: heihutu --- crates/config/src/constants/object.rs | 25 ++++ rustfs/src/app/object_usecase.rs | 46 +++++-- rustfs/src/app/storage_api.rs | 2 +- rustfs/src/storage/concurrency/manager.rs | 153 +++++++++++++++++++++- rustfs/src/storage/concurrency/mod.rs | 2 +- rustfs/src/storage/storage_api.rs | 2 +- 6 files changed, 211 insertions(+), 19 deletions(-) diff --git a/crates/config/src/constants/object.rs b/crates/config/src/constants/object.rs index 081308754..874933aa6 100644 --- a/crates/config/src/constants/object.rs +++ b/crates/config/src/constants/object.rs @@ -234,6 +234,31 @@ pub const ENV_OBJECT_DISK_WRITE_ABSOLUTE_CAP: &str = "RUSTFS_OBJECT_DISK_WRITE_A /// Default absolute per-object erasure write cap in seconds (`0` = disabled). pub const DEFAULT_OBJECT_DISK_WRITE_ABSOLUTE_CAP: u64 = 0; +/// Enable foreground PutObject request admission. +/// +/// This is an experimental, default-off foreground write backpressure gate for +/// strict commit tail investigations. When disabled, PUTs follow the legacy +/// path and only the existing request counters are updated. +pub const ENV_PUT_FOREGROUND_ADMISSION_ENABLE: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_ENABLE"; +pub const DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE: bool = false; + +/// Maximum foreground PutObject requests admitted concurrently per process. +/// +/// The limit is used only when [`ENV_PUT_FOREGROUND_ADMISSION_ENABLE`] is true. +/// A value of `0` disables the gate even when the enable flag is present, so a +/// partially configured rollout cannot reject every PUT. +pub const ENV_PUT_FOREGROUND_ADMISSION_LIMIT: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_LIMIT"; +pub const DEFAULT_PUT_FOREGROUND_ADMISSION_LIMIT: usize = 0; + +/// Time in milliseconds a foreground PutObject waits for an admission permit. +/// +/// Once this timeout expires the request fails before body ingest/storage +/// mutation with S3 `SlowDown`/503. `0` means fail fast when the limit is full. +pub const ENV_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS"; +pub const DEFAULT_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: u64 = 0; + +const _: () = assert!(!DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE); + /// Environment variable for minimum GetObject timeout in seconds. /// /// When dynamic timeout calculation is enabled, this is the minimum timeout diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 643b4fb6e..1bb337c69 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -56,8 +56,8 @@ use super::storage_api::object_usecase::bucket::{ }; use super::storage_api::object_usecase::compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; use super::storage_api::object_usecase::concurrency::{ - self, ConcurrencyManager, DiskReadAdmission, GetObjectGuard, PutObjectGuard, get_concurrency_aware_buffer_size, - get_concurrency_manager, get_put_concurrency_aware_buffer_size, + self, ConcurrencyManager, DiskReadAdmission, GetObjectGuard, PutObjectAdmission, PutObjectGuard, + get_concurrency_aware_buffer_size, get_concurrency_manager, get_put_concurrency_aware_buffer_size, }; #[cfg(test)] use super::storage_api::object_usecase::contract::http::HTTPPreconditions; @@ -5681,6 +5681,35 @@ impl DefaultObjectUsecase { let server_side_encryption_requested = server_side_encryption.is_some() || sse_customer_algorithm.is_some() || ssekms_key_id.is_some(); + // Resolve the store through the request-bound server context + // (backlog#1052 S6), not the process-global handle, so an embedded + // second server never writes into the first server's store. + let Some(store) = self.object_store() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + let bucket_validate_stage_start = put_stage_metrics_enabled.then(Instant::now); + validate_bucket_exists(&store, &bucket).await?; + rustfs_io_metrics::record_put_object_stage_duration_from("app_bucket_validate", bucket_validate_stage_start); + + let put_admission = match get_concurrency_manager() + .admit_put_object() + .await + .map_err(|_| s3_error!(InternalError, "foreground write admission closed"))? + { + PutObjectAdmission::Disabled => None, + PutObjectAdmission::Admitted(permit) => { + counter!("rustfs.put_object.foreground_admission.total", "result" => "admitted").increment(1); + Some(permit) + } + PutObjectAdmission::Rejected => { + counter!("rustfs.put_object.foreground_admission.total", "result" => "rejected").increment(1); + return Err(s3_error!( + SlowDown, + "foreground write concurrency limit reached, please reduce your request rate" + )); + } + }; + let mut put_request_guard = PutObjectGuard::new(); let concurrent_put_requests = PutObjectGuard::concurrent_requests(); @@ -5733,16 +5762,6 @@ impl DefaultObjectUsecase { use_large_put_concurrency_tuning, ); - // Resolve the store through the request-bound server context - // (backlog#1052 S6), not the process-global handle, so an embedded - // second server never writes into the first server's store. - let Some(store) = self.object_store() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - let bucket_validate_stage_start = put_stage_metrics_enabled.then(Instant::now); - validate_bucket_exists(&store, &bucket).await?; - rustfs_io_metrics::record_put_object_stage_duration_from("app_bucket_validate", bucket_validate_stage_start); - let sse_config_stage_start = put_stage_metrics_enabled.then(Instant::now); let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); rustfs_io_metrics::record_put_object_stage_duration_from("app_sse_config_lookup", sse_config_stage_start); @@ -6132,7 +6151,9 @@ impl DefaultObjectUsecase { let cache_adapter = cache_adapter.clone(); let request_id = request_id.clone(); let put_path = put_path.to_string(); + let put_admission = put_admission; async move { + let _put_admission = put_admission; let object_traffic_progress = object_traffic_health .as_deref() .and_then(ObjectTrafficHealth::track_write_storage); @@ -6183,6 +6204,7 @@ impl DefaultObjectUsecase { } }; rustfs_io_metrics::record_put_object_stage_duration_from("app_store_put", store_put_stage_start); + drop(_put_admission); drop(object_traffic_progress); #[cfg(test)] wait_for_put_post_store_test_hook(&bucket).await; diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 839b8a38f..c6f03f240 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -936,7 +936,7 @@ pub(crate) mod bucket { pub(crate) mod concurrency { pub(crate) use crate::storage::storage_api::concurrency_consumer::{ - ConcurrencyManager, DiskReadAdmission, GetObjectGuard, IoQueueStatus, IoStrategy, PutObjectGuard, + ConcurrencyManager, DiskReadAdmission, GetObjectGuard, IoQueueStatus, IoStrategy, PutObjectAdmission, PutObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, get_put_concurrency_aware_buffer_size, }; } diff --git a/rustfs/src/storage/concurrency/manager.rs b/rustfs/src/storage/concurrency/manager.rs index 13b509f93..5bc456148 100644 --- a/rustfs/src/storage/concurrency/manager.rs +++ b/rustfs/src/storage/concurrency/manager.rs @@ -65,6 +65,11 @@ pub struct ConcurrencyManager { bandwidth_monitor: Arc>, /// Metrics collector for I/O latency tracking (P50, P95, P99) metrics_collector: Arc, + /// Experimental fixed-count foreground PutObject admission gate. + put_admission_semaphore: Arc, + put_admission_enabled: bool, + put_admission_limit: usize, + put_admission_wait_timeout: Duration, } impl std::fmt::Debug for ConcurrencyManager { @@ -114,6 +119,18 @@ pub enum DiskReadAdmission { Rejected, } +/// Outcome of foreground PutObject request admission. +#[derive(Debug)] +pub enum PutObjectAdmission { + /// Foreground PUT admission is disabled; proceed on the legacy path. + Disabled, + /// Request is admitted and must hold the permit until the store write + /// returns or the request fails before mutation. + Admitted(tokio::sync::OwnedSemaphorePermit), + /// The fixed-count gate stayed full until the configured wait timeout. + Rejected, +} + #[allow(dead_code)] impl ConcurrencyManager { /// Create a new concurrency manager with default settings @@ -161,6 +178,18 @@ impl ConcurrencyManager { // Initialize metrics collector for I/O latency tracking // Keep 1000 samples for P95/P99 calculation let metrics_collector = Arc::new(MetricsCollector::new(performance_metrics, 1000)); + let put_admission_enabled = rustfs_utils::get_env_bool( + rustfs_config::ENV_PUT_FOREGROUND_ADMISSION_ENABLE, + rustfs_config::DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE, + ); + let put_admission_limit = rustfs_utils::get_env_usize( + rustfs_config::ENV_PUT_FOREGROUND_ADMISSION_LIMIT, + rustfs_config::DEFAULT_PUT_FOREGROUND_ADMISSION_LIMIT, + ); + let put_admission_wait_timeout = Duration::from_millis(rustfs_utils::get_env_u64( + rustfs_config::ENV_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS, + rustfs_config::DEFAULT_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS, + )); // Build queue config directly from scheduler config. let queue_config = IoPriorityQueueConfig::from_scheduler_config(&scheduler_config); @@ -176,6 +205,10 @@ impl ConcurrencyManager { pattern_detector, bandwidth_monitor, metrics_collector, + put_admission_semaphore: Arc::new(Semaphore::new(if put_admission_enabled { put_admission_limit } else { 0 })), + put_admission_enabled, + put_admission_limit, + put_admission_wait_timeout, } } @@ -199,6 +232,16 @@ impl ConcurrencyManager { self.degraded_read_semaphore.close(); } + #[cfg(test)] + pub(crate) fn with_put_admission_for_test(enabled: bool, limit: usize, wait_timeout: Duration) -> Self { + let mut manager = Self::new(); + manager.put_admission_semaphore = Arc::new(Semaphore::new(if enabled { limit } else { 0 })); + manager.put_admission_enabled = enabled; + manager.put_admission_limit = limit; + manager.put_admission_wait_timeout = wait_timeout; + manager + } + /// Track a GetObject request pub fn track_request() -> GetObjectGuard { GetObjectGuard::new() @@ -284,6 +327,32 @@ impl ConcurrencyManager { } } + /// Admit a foreground PutObject request under the experimental fixed-count gate. + /// + /// The default-off path returns [`PutObjectAdmission::Disabled`] without + /// touching the semaphore, preserving legacy behavior. When enabled, the + /// permit must be acquired before body ingest and held until the store write + /// returns, so saturated foreground writes can fail with `SlowDown` before + /// creating visible side effects. + pub async fn admit_put_object(&self) -> Result { + if !self.put_admission_enabled || self.put_admission_limit == 0 { + return Ok(PutObjectAdmission::Disabled); + } + + if self.put_admission_wait_timeout.is_zero() { + return Ok(match self.put_admission_semaphore.clone().try_acquire_owned() { + Ok(permit) => PutObjectAdmission::Admitted(permit), + Err(tokio::sync::TryAcquireError::NoPermits) => PutObjectAdmission::Rejected, + Err(tokio::sync::TryAcquireError::Closed) => PutObjectAdmission::Rejected, + }); + } + + match tokio::time::timeout(self.put_admission_wait_timeout, self.put_admission_semaphore.clone().acquire_owned()).await { + Ok(permit) => Ok(PutObjectAdmission::Admitted(permit?)), + Err(_) => Ok(PutObjectAdmission::Rejected), + } + } + // ============================================ // Adaptive I/O Strategy Methods // ============================================ @@ -692,8 +761,16 @@ impl ConcurrencyManager { /// Get a read-only workload admission snapshot for foreground writes. pub fn put_object_admission_snapshot(&self) -> WorkloadAdmissionSnapshot { - let active = PutObjectGuard::concurrent_count(); - let limit = self.scheduler_config.max_concurrent_reads; + let (active, limit, hard_gate_enabled) = if self.put_admission_enabled && self.put_admission_limit > 0 { + ( + self.put_admission_limit + .saturating_sub(self.put_admission_semaphore.available_permits()), + self.put_admission_limit, + true, + ) + } else { + (PutObjectGuard::concurrent_count(), self.scheduler_config.max_concurrent_reads, false) + }; let state = if limit == 0 { AdmissionState::Disabled } else if active >= limit { @@ -706,7 +783,10 @@ impl ConcurrencyManager { WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, state).with_counts(Some(active), None, Some(limit)); match state { - AdmissionState::Disabled => admission.with_reason("foreground write pressure tracking disabled"), + AdmissionState::Disabled => admission.with_reason("foreground write admission disabled"), + AdmissionState::Saturated if hard_gate_enabled => { + admission.with_reason("foreground write admission permits exhausted") + } AdmissionState::Saturated => admission.with_reason("foreground write concurrency reached local pressure limit"), _ => admission, } @@ -783,7 +863,7 @@ impl Default for ConcurrencyManager { mod integration_tests { use super::super::io_schedule::{IoLoadLevel, IoPriority}; use super::super::request_guard::GetObjectGuard; - use super::ConcurrencyManager; + use super::{ConcurrencyManager, PutObjectAdmission}; use crate::storage::storage_api::concurrency_consumer::PutObjectGuard; use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshotProvider, WorkloadClass}; use rustfs_io_core::io_profile::{AccessPattern, StorageMedia}; @@ -880,6 +960,71 @@ mod integration_tests { crate::storage::concurrency::reset_active_put_requests(); } + #[tokio::test] + #[serial] + async fn test_concurrency_manager_put_admission_disabled_does_not_touch_gate() { + let manager = ConcurrencyManager::with_put_admission_for_test(false, 1, Duration::ZERO); + + let admission = manager + .admit_put_object() + .await + .expect("disabled put admission must not close"); + + assert!(matches!(admission, PutObjectAdmission::Disabled)); + assert_eq!(manager.put_admission_semaphore.available_permits(), 0); + assert_eq!(manager.put_object_admission_snapshot().state, AdmissionState::Open); + } + + #[tokio::test] + #[serial] + async fn test_concurrency_manager_put_admission_rejects_when_limit_full() { + let manager = ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::ZERO); + + let first = manager.admit_put_object().await.expect("first put admission should acquire"); + assert!(matches!(first, PutObjectAdmission::Admitted(_))); + assert_eq!(manager.put_object_admission_snapshot().state, AdmissionState::Saturated); + + let second = manager + .admit_put_object() + .await + .expect("full put admission gate should reject, not close"); + assert!(matches!(second, PutObjectAdmission::Rejected)); + } + + #[tokio::test] + #[serial] + async fn test_concurrency_manager_put_admission_reuses_released_permit() { + let manager = ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::ZERO); + + let first = manager.admit_put_object().await.expect("first put admission should acquire"); + drop(first); + + let second = manager + .admit_put_object() + .await + .expect("released put admission permit should be reusable"); + assert!(matches!(second, PutObjectAdmission::Admitted(_))); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn test_concurrency_manager_put_admission_wait_timeout_rejects() { + let manager = ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::from_secs(5)); + let held = manager.admit_put_object().await.expect("first put admission should acquire"); + let waiter_manager = manager.clone(); + + let waiter = tokio::spawn(async move { waiter_manager.admit_put_object().await }); + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(5)).await; + + let admission = waiter + .await + .expect("put admission waiter task must not panic") + .expect("put admission gate must stay open"); + assert!(matches!(admission, PutObjectAdmission::Rejected)); + drop(held); + } + #[tokio::test] #[serial] async fn test_concurrency_manager_workload_admission_registry_covers_required_classes() { diff --git a/rustfs/src/storage/concurrency/mod.rs b/rustfs/src/storage/concurrency/mod.rs index 110e14cf6..9b14eebda 100644 --- a/rustfs/src/storage/concurrency/mod.rs +++ b/rustfs/src/storage/concurrency/mod.rs @@ -54,7 +54,7 @@ pub use io_schedule::{ pub use request_guard::{GetObjectGuard, PutObjectGuard}; // Concurrency manager -pub use manager::{ConcurrencyManager, DiskReadAdmission}; +pub use manager::{ConcurrencyManager, DiskReadAdmission, PutObjectAdmission}; // ============================================ // New Module Re-exports (for gradual migration) diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index d1e7c0f3a..be9548619 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -117,7 +117,7 @@ pub(crate) mod access_consumer { pub(crate) mod concurrency_consumer { pub(crate) use super::super::concurrency::{ - ConcurrencyManager, DiskReadAdmission, GetObjectGuard, IoQueueStatus, IoStrategy, PutObjectGuard, + ConcurrencyManager, DiskReadAdmission, GetObjectGuard, IoQueueStatus, IoStrategy, PutObjectAdmission, PutObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, get_put_concurrency_aware_buffer_size, }; } From b825c548505d8a074c1fa52eea72019d39258dad Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 12:21:49 +0800 Subject: [PATCH 19/51] refactor(admin): route kms management auth through shared gate (#6194) --- rustfs/src/admin/handlers/kms_management.rs | 127 ++++++++++++-------- 1 file changed, 75 insertions(+), 52 deletions(-) diff --git a/rustfs/src/admin/handlers/kms_management.rs b/rustfs/src/admin/handlers/kms_management.rs index b9ed1b4c1..796fc2319 100644 --- a/rustfs/src/admin/handlers/kms_management.rs +++ b/rustfs/src/admin/handlers/kms_management.rs @@ -16,13 +16,12 @@ use super::kms_dynamic::current_kms_config_fingerprint; use super::kms_keys::{CreateKeyHandler, DescribeKeyHandler, GenerateDataKeyHandler, ListKeysHandler}; -use crate::admin::auth::validate_admin_request; +use crate::admin::auth::authorize_admin_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{ current_kms_runtime_service_manager, current_notification_system, current_or_init_kms_runtime_service_manager, }; -use crate::auth::{check_key_valid, get_session_token}; -use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use crate::server::ADMIN_PREFIX; use hyper::{HeaderMap, Method, StatusCode}; use matchit::Params; use rustfs_kms::KmsBackend; @@ -69,6 +68,18 @@ fn kms_clear_cache_actions() -> Vec { vec![Action::KmsAction(KmsAction::ClearCacheAction)] } +/// Admin gate for the KMS management endpoints, none of which act on a key. +/// +/// The pre-check keeps these endpoints' historical missing-credentials message; +/// the shared gate reports "get cred failed". +async fn authorize_kms_management_request(req: &S3Request, actions: Vec) -> S3Result<()> { + if req.credentials.is_none() { + return Err(s3_error!(InvalidRequest, "authentication required")); + } + authorize_admin_request(req, actions).await?; + Ok(()) +} + /// Response of `POST /kms/clear-cache`. /// /// Declared rather than built inline so the shape the console already depends @@ -260,22 +271,7 @@ pub struct KmsStatusHandler {} #[async_trait::async_trait] impl Operation for KmsStatusHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - let Some(cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "authentication required")); - }; - - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - kms_service_control_actions(), - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await?; + authorize_kms_management_request(&req, kms_service_control_actions()).await?; let Some(service) = kms_encryption_service_from_context().await else { return Err(s3_error!(InternalError, "KMS service not initialized")); @@ -326,22 +322,7 @@ pub struct KmsConfigHandler {} #[async_trait::async_trait] impl Operation for KmsConfigHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - let Some(cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "authentication required")); - }; - - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - kms_configure_actions(), - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await?; + authorize_kms_management_request(&req, kms_configure_actions()).await?; let Some(service) = kms_encryption_service_from_context().await else { return Err(s3_error!(InternalError, "KMS service not initialized")); @@ -375,22 +356,7 @@ pub struct KmsClearCacheHandler {} #[async_trait::async_trait] impl Operation for KmsClearCacheHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - let Some(cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "authentication required")); - }; - - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - kms_clear_cache_actions(), - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await?; + authorize_kms_management_request(&req, kms_clear_cache_actions()).await?; let Some(service) = kms_encryption_service_from_context().await else { return Err(s3_error!(InternalError, "KMS service not initialized")); @@ -422,9 +388,14 @@ impl Operation for KmsClearCacheHandler { #[cfg(test)] mod tests { - use super::{KmsClearCacheResponse, kms_clear_cache_actions, kms_configure_actions, kms_service_control_actions}; + use super::{ + KmsClearCacheResponse, authorize_kms_management_request, kms_clear_cache_actions, kms_configure_actions, + kms_service_control_actions, + }; use crate::admin::handlers::kms_keys::stable_json_value; + use hyper::HeaderMap; use rustfs_policy::policy::action::{Action, AdminAction, KmsAction}; + use s3s::{Body, S3Request}; fn assert_has_action(actions: &[Action], action: Action) { assert!(actions.contains(&action), "expected action list to contain {action:?}"); @@ -434,6 +405,58 @@ mod tests { assert!(!actions.contains(&action), "expected action list not to contain {action:?}"); } + /// These endpoints authorize through the shared admin gate, which reports + /// "get cred failed" for a credential-less request. The pre-check keeps the + /// message these endpoints have always returned (rustfs/backlog#1829). + #[tokio::test] + async fn kms_management_gate_keeps_its_missing_credentials_message() { + let req = S3Request { + input: Body::from(String::new()), + method: http::Method::GET, + uri: "/rustfs/admin/v3/kms/status".parse().expect("uri should parse"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + let err = authorize_kms_management_request(&req, kms_service_control_actions()) + .await + .expect_err("a request without credentials must be rejected"); + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("authentication required")); + } + + /// Every management endpoint must reach the shared gate, each with its own + /// action set. The action lists are pinned above, but nothing else checks + /// which handler asks for which, and a handler that lost its gate entirely + /// would still serve its response. + #[test] + fn management_handlers_authorize_with_their_dedicated_actions() { + let src = include_str!("kms_management.rs"); + + for (handler, actions) in [ + ("KmsStatusHandler", "kms_service_control_actions()"), + ("KmsConfigHandler", "kms_configure_actions()"), + ("KmsClearCacheHandler", "kms_clear_cache_actions()"), + ] { + let block = src + .split_once(&format!("impl Operation for {handler}")) + .unwrap_or_else(|| panic!("{handler} impl should exist")) + .1; + let end = block + .find("\nimpl Operation for") + .or_else(|| block.find("\n#[cfg(test)]")) + .unwrap_or(block.len()); + assert!( + block[..end].contains(&format!("authorize_kms_management_request(&req, {actions})")), + "{handler} must authorize through the shared gate with {actions}" + ); + } + } + #[test] fn kms_management_auth_actions_use_dedicated_kms_actions() { assert_has_action(&kms_service_control_actions(), Action::KmsAction(KmsAction::ServiceControlAction)); From abffa5cf1b2d1e51f48aa6f59cec5b6e77ef9413 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 12:35:36 +0800 Subject: [PATCH 20/51] chore(storage): drop dead io-schedule metrics and helpers (#6199) --- docs/architecture/global-state-inventory.md | 2 +- rustfs/src/storage/concurrency/io_schedule.rs | 212 +----------------- rustfs/src/storage/concurrency/mod.rs | 70 +----- 3 files changed, 19 insertions(+), 265 deletions(-) diff --git a/docs/architecture/global-state-inventory.md b/docs/architecture/global-state-inventory.md index 6c06b10c8..8407ad864 100644 --- a/docs/architecture/global-state-inventory.md +++ b/docs/architecture/global-state-inventory.md @@ -107,7 +107,7 @@ inventory. Generic function-local names such as `CACHE`, `LOCK`, `INIT`, and | `AUTH_FS` | `rustfs/src/storage/access.rs` | Cache or constant / owner-local cache | Authorization tag-condition lookup keeps its filesystem helper private to the access owner. | | `LOCK_STATS` | `rustfs/src/storage/lock_optimizer.rs` | Process-global owner-local metrics | Lock optimization statistics stay private behind lock optimizer helper APIs. | | `DEADLOCK_DETECTOR` | `rustfs/src/storage/deadlock_detector.rs` | Process-global owner-local state | Deadlock detector lifecycle state stays private to the storage deadlock detector owner. | -| `CONCURRENCY_MANAGER`, `ACTIVE_GET_REQUESTS`, `ACTIVE_PUT_REQUESTS`, `IO_PRIORITY_METRICS` | `rustfs/src/storage/concurrency/*` | Process-global owner-local scheduler state | Storage concurrency manager, counters, and metrics remain inside the storage concurrency owner boundary. | +| `CONCURRENCY_MANAGER`, `ACTIVE_GET_REQUESTS`, `ACTIVE_PUT_REQUESTS` | `rustfs/src/storage/concurrency/*` | Process-global owner-local scheduler state | Storage concurrency manager and request counters remain inside the storage concurrency owner boundary. | | `GET_OBJECT_BUFFER_THRESHOLD_WARNED`, `GET_READER_STREAM_BUFFER_SIZE_OVERRIDE`, function-local `ENABLED`, `OBJECT_SEEK_SUPPORT_THRESHOLD`, `OBJECT_SEEK_SUPPORT_CONCURRENCY_THRESHOLDS` | `rustfs/src/app/object_usecase.rs` | Cache or constant / owner-local cache | Object GET/seek tuning caches and warning guards stay private to object usecase helpers. | | `SUPPORTED_HEADERS` | `rustfs/src/storage/options.rs` | Cache or constant / owner-local constant | Supported-header lookup state stays private to storage option parsing. | | `AUDIT_TARGET_SPECS`, `NOTIFICATION_TARGET_SPECS` | `rustfs/src/admin/handlers/audit.rs`, `rustfs/src/admin/handlers/event.rs`, `rustfs/src/admin/handlers/plugins_instances.rs` | Cache or constant / owner-local constant | Admin target descriptor tables stay private to their handler owners. | diff --git a/rustfs/src/storage/concurrency/io_schedule.rs b/rustfs/src/storage/concurrency/io_schedule.rs index d611dfeb2..8d202708a 100644 --- a/rustfs/src/storage/concurrency/io_schedule.rs +++ b/rustfs/src/storage/concurrency/io_schedule.rs @@ -14,21 +14,12 @@ //! I/O scheduling types for adaptive buffer sizing and load management. //! -//! # Migration Note -//! -//! This module contains types that are also available in `rustfs_io_core`. -//! For new code, prefer using types from `rustfs_io_core` directly: -//! -//! ```ignore -//! // Recommended: Use io-core types -//! use rustfs_io_core::{ -//! IoLoadLevel, IoPriority, IoSchedulerConfig, -//! calculate_optimal_buffer_size, get_buffer_size_for_media, -//! }; -//! ``` -//! -//! This module remains for backward compatibility and provides additional -//! runtime monitoring features (`IoPriorityMetrics`, `IoStrategyDebugInfo`). +//! This is the live scheduling implementation. `rustfs_io_core` supplies the +//! shared config shapes (`IoSchedulerConfig`, `IoPriorityQueueConfig`) that the +//! types here project into through `to_core_config`, plus the `io_profile` +//! storage-media model; bandwidth samples come from `rustfs_io_metrics`. +//! Same-named io-core types are those config shapes, not a backing +//! implementation this module delegates to. use rustfs_config::{KI_B, MI_B}; use rustfs_io_core::io_profile::{AccessPattern, StorageMedia, StorageProfile}; @@ -1762,169 +1753,6 @@ impl IoPriorityQueue { } } -// ============================================ -// I/O Priority Queue Metrics -// ============================================ - -/// Global metrics for I/O priority queue monitoring. -/// -/// These metrics are emitted through the shared metrics pipeline and provide -/// visibility into the priority queue behavior. -#[allow(dead_code)] -pub struct IoPriorityMetrics { - /// High priority queue depth. - pub high_queue_depth: AtomicU64, - /// Normal priority queue depth. - pub normal_queue_depth: AtomicU64, - /// Low priority queue depth. - pub low_queue_depth: AtomicU64, - /// High priority total wait time in nanoseconds. - pub high_wait_time_ns: AtomicU64, - /// Normal priority total wait time in nanoseconds. - pub normal_wait_time_ns: AtomicU64, - /// Low priority total wait time in nanoseconds. - pub low_wait_time_ns: AtomicU64, - /// Total starvation events count. - pub starvation_events: AtomicU64, - /// High priority requests processed. - pub high_processed: AtomicU64, - /// Normal priority requests processed. - pub normal_processed: AtomicU64, - /// Low priority requests processed. - pub low_processed: AtomicU64, -} - -#[allow(dead_code)] -impl Default for IoPriorityMetrics { - fn default() -> Self { - Self::new() - } -} - -#[allow(dead_code)] -impl IoPriorityMetrics { - /// Create a new metrics instance. - pub const fn new() -> Self { - Self { - high_queue_depth: AtomicU64::new(0), - normal_queue_depth: AtomicU64::new(0), - low_queue_depth: AtomicU64::new(0), - high_wait_time_ns: AtomicU64::new(0), - normal_wait_time_ns: AtomicU64::new(0), - low_wait_time_ns: AtomicU64::new(0), - starvation_events: AtomicU64::new(0), - high_processed: AtomicU64::new(0), - normal_processed: AtomicU64::new(0), - low_processed: AtomicU64::new(0), - } - } - - /// Update queue depths from status. - #[allow(dead_code)] - pub fn update_queue_depths(&self, status: &IoQueueStatus) { - self.high_queue_depth - .store(status.high_priority_waiting as u64, Ordering::Relaxed); - self.normal_queue_depth - .store(status.normal_priority_waiting as u64, Ordering::Relaxed); - self.low_queue_depth - .store(status.low_priority_waiting as u64, Ordering::Relaxed); - } - - /// Record a starvation event. - #[allow(dead_code)] - pub fn record_starvation(&self) { - self.starvation_events.fetch_add(1, Ordering::Relaxed); - } - - /// Record a processed request. - #[allow(dead_code)] - pub fn record_processed(&self, priority: IoPriority) { - match priority { - IoPriority::High => self.high_processed.fetch_add(1, Ordering::Relaxed), - IoPriority::Normal => self.normal_processed.fetch_add(1, Ordering::Relaxed), - IoPriority::Low => self.low_processed.fetch_add(1, Ordering::Relaxed), - }; - } - - /// Record wait time for a priority level. - pub fn record_wait_time(&self, priority: IoPriority, wait_ns: u64) { - match priority { - IoPriority::High => self.high_wait_time_ns.fetch_add(wait_ns, Ordering::Relaxed), - IoPriority::Normal => self.normal_wait_time_ns.fetch_add(wait_ns, Ordering::Relaxed), - IoPriority::Low => self.low_wait_time_ns.fetch_add(wait_ns, Ordering::Relaxed), - }; - } - - /// Get high priority queue depth. - pub fn get_high_queue_depth(&self) -> u64 { - self.high_queue_depth.load(Ordering::Relaxed) - } - - /// Get normal priority queue depth. - pub fn get_normal_queue_depth(&self) -> u64 { - self.normal_queue_depth.load(Ordering::Relaxed) - } - - /// Get low priority queue depth. - pub fn get_low_queue_depth(&self) -> u64 { - self.low_queue_depth.load(Ordering::Relaxed) - } - - /// Get total starvation events. - pub fn get_starvation_events(&self) -> u64 { - self.starvation_events.load(Ordering::Relaxed) - } - - /// Get metrics summary for logging/debugging. - pub fn summary(&self) -> String { - format!( - "high_queue={}, normal_queue={}, low_queue={}, starvation={}, high_proc={}, normal_proc={}, low_proc={}", - self.get_high_queue_depth(), - self.get_normal_queue_depth(), - self.get_low_queue_depth(), - self.get_starvation_events(), - self.high_processed.load(Ordering::Relaxed), - self.normal_processed.load(Ordering::Relaxed), - self.low_processed.load(Ordering::Relaxed) - ) - } -} - -/// Global I/O priority metrics instance. -#[allow(dead_code)] -pub static IO_PRIORITY_METRICS: IoPriorityMetrics = IoPriorityMetrics::new(); - -/// Get optimized buffer size for I/O operations. -/// -/// This function provides adaptive buffer sizing based on: -/// - File size (small files get smaller buffers) -/// - Concurrent request count (high concurrency gets smaller buffers) -/// - Base buffer size from configuration -/// -/// # Arguments -/// -/// * `file_size` - Size of the file being read/written (-1 for unknown) -/// -/// # Returns -/// -/// Optimal buffer size in bytes -/// -/// # Example -/// -/// ```ignore -/// let buffer_size = get_buffer_size_opt_in(1024 * 1024); // 1MB file -/// assert!(buffer_size >= 64 * 1024); // At least 64KB -/// ``` -#[allow(dead_code)] -pub fn get_buffer_size_opt_in(file_size: i64) -> usize { - // Get base buffer size from configuration - let base_buffer_size = - rustfs_utils::get_env_usize(rustfs_config::ENV_OBJECT_IO_BUFFER_SIZE, rustfs_config::DEFAULT_OBJECT_IO_BUFFER_SIZE); - - // Apply concurrency-aware adjustments - get_concurrency_aware_buffer_size(file_size, base_buffer_size) -} - // ============================================ // Unit Tests // ============================================ @@ -1933,13 +1761,12 @@ pub fn get_buffer_size_opt_in(file_size: i64) -> usize { #[allow(unused_imports)] mod tests { use super::{ - IoLoadLevel, IoPriority, IoPriorityMetrics, IoPriorityQueue, IoPriorityQueueConfig, IoSchedulerConfig, - IoSchedulingContext, IoStrategy, get_advanced_buffer_size, get_buffer_size_opt_in, get_concurrency_aware_buffer_size, + IoLoadLevel, IoPriority, IoPriorityQueue, IoPriorityQueueConfig, IoSchedulerConfig, IoSchedulingContext, IoStrategy, + get_advanced_buffer_size, get_concurrency_aware_buffer_size, }; use rustfs_io_core::io_profile::{AccessPattern, StorageMedia}; use rustfs_io_metrics::bandwidth::{BandwidthSnapshot, BandwidthTier}; use serial_test::serial; - use std::sync::atomic::Ordering; use std::time::Duration; #[tokio::test] @@ -2126,29 +1953,6 @@ mod tests { assert_eq!(config.starvation_threshold_secs, 120); } - #[tokio::test] - #[serial] - async fn test_io_priority_metrics() { - let metrics = IoPriorityMetrics::new(); - - // Test initial state - assert_eq!(metrics.get_high_queue_depth(), 0); - assert_eq!(metrics.get_normal_queue_depth(), 0); - assert_eq!(metrics.get_low_queue_depth(), 0); - assert_eq!(metrics.get_starvation_events(), 0); - - // Test recording - metrics.record_starvation(); - assert_eq!(metrics.get_starvation_events(), 1); - - metrics.record_processed(IoPriority::High); - metrics.record_processed(IoPriority::High); - metrics.record_processed(IoPriority::Normal); - - assert_eq!(metrics.high_processed.load(Ordering::Relaxed), 2); - assert_eq!(metrics.normal_processed.load(Ordering::Relaxed), 1); - } - // ============================================ // Multi-Factor Strategy Tests // ============================================ diff --git a/rustfs/src/storage/concurrency/mod.rs b/rustfs/src/storage/concurrency/mod.rs index 9b14eebda..eb47aa4ed 100644 --- a/rustfs/src/storage/concurrency/mod.rs +++ b/rustfs/src/storage/concurrency/mod.rs @@ -24,16 +24,14 @@ //! - **Concurrency Management**: Coordination of concurrent GetObject requests //! - **Request Tracking**: RAII guards for request lifecycle management //! -//! # Migration Note +//! # Relationship to the shared crates //! -//! Core algorithms have been migrated to `rustfs-io-core` and metrics to -//! `rustfs-io-metrics`. This module maintains API compatibility while -//! delegating to the new implementations. +//! The scheduling algorithm lives in [`io_schedule`], not in `rustfs-io-core`: +//! this module does not delegate to it. `rustfs-io-core` owns the shared +//! config shapes and the `io_profile` storage-media model that [`io_schedule`] +//! consumes, and `rustfs-io-metrics` owns bandwidth sampling and metric +//! recording. -// Sub-modules -// pub mod bandwidth_monitor; // Migrated to rustfs-io-metrics -// pub mod global_metrics; // Migrated to rustfs-io-metrics -// pub mod io_profile; // Migrated to rustfs-io-core pub mod io_schedule; pub mod manager; pub mod request_guard; @@ -45,9 +43,8 @@ pub mod request_guard; // I/O scheduling types (from io_schedule.rs for backward compatibility) #[allow(unused_imports)] pub use io_schedule::{ - IO_PRIORITY_METRICS, IoLoadLevel, IoPriority, IoPriorityMetrics, IoPriorityQueue, IoPriorityQueueConfig, IoQueueStatus, - IoSchedulerConfig, IoStrategy, get_advanced_buffer_size, get_buffer_size_opt_in, get_concurrency_aware_buffer_size, - get_put_concurrency_aware_buffer_size, + IoLoadLevel, IoPriority, IoPriorityQueue, IoPriorityQueueConfig, IoQueueStatus, IoSchedulerConfig, IoStrategy, + get_advanced_buffer_size, get_concurrency_aware_buffer_size, get_put_concurrency_aware_buffer_size, }; // Request tracking @@ -56,24 +53,6 @@ pub use request_guard::{GetObjectGuard, PutObjectGuard}; // Concurrency manager pub use manager::{ConcurrencyManager, DiskReadAdmission, PutObjectAdmission}; -// ============================================ -// New Module Re-exports (for gradual migration) -// ============================================ - -// Re-export types from rustfs-io-core for convenience -pub use rustfs_io_core::{ - // Backpressure types - BackpressureMonitor, - // Deadlock detection types - DeadlockDetector, - // Scheduler types - IoScheduler, - // Lock optimization types - LockOptimizer, -}; - -// Re-export types from rustfs-io-metrics for convenience - // ============================================ // Helper Functions // ============================================ @@ -83,37 +62,8 @@ pub fn get_concurrency_manager() -> &'static ConcurrencyManager { ConcurrencyManager::global() } -/// Reset the active get requests counter (for testing). -#[allow(dead_code)] -pub fn reset_active_get_requests() { - io_schedule::ACTIVE_GET_REQUESTS.store(0, std::sync::atomic::Ordering::Relaxed); -} - -#[allow(dead_code)] +/// Reset the active put requests counter (for testing). +#[cfg(test)] pub fn reset_active_put_requests() { io_schedule::ACTIVE_PUT_REQUESTS.store(0, std::sync::atomic::Ordering::Relaxed); } - -/// Create a new I/O scheduler with default configuration. -#[allow(dead_code)] -pub fn create_io_scheduler() -> IoScheduler { - IoScheduler::with_defaults() -} - -/// Create a new backpressure monitor with default configuration. -#[allow(dead_code)] -pub fn create_backpressure_monitor() -> BackpressureMonitor { - BackpressureMonitor::with_defaults() -} - -/// Create a new deadlock detector with default configuration. -#[allow(dead_code)] -pub fn create_deadlock_detector() -> DeadlockDetector { - DeadlockDetector::with_defaults() -} - -/// Create a new lock optimizer with default configuration. -#[allow(dead_code)] -pub fn create_lock_optimizer() -> LockOptimizer { - LockOptimizer::with_defaults() -} From a08de9229b69115dc1c4142d04480ec33cded2d7 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 12:43:27 +0800 Subject: [PATCH 21/51] feat(heal): wire MRF intents with durable repair journal (HS-01) (#6189) * feat(common): add MRF intent channel and Mrf request source (HS-01) Introduce the producer-facing half of the mission repair feed: a global bounded (8192) channel carrying lightweight MrfIntent values from IO error paths, plus the RUSTFS_HEAL_MRF_ENABLE delivery kill-switch and config constants for queue/journal sizing. Delivery is strictly non-blocking (try_send, drop-on-full) so it can sit on decode-failure and partial-write paths without adding latency. HealRequestSource grows a 'mrf' variant so admission accounting can attribute replayed intents. Part of backlog#1865 (option a: wire HealEvent-style intents with a durable retry ledger). Co-Authored-By: heihutu * feat(heal): add MRF queue, durable journal, and intent consumer (HS-01) Consumer half of the mission repair feed: a bounded pending queue (100k intents / 8 MiB dual ceiling, drop-newest on overflow), a durable journal at buckets/.heal/mrf/journal.bin holding the unaccepted pending snapshot, and a consumer task that batches intents off the global channel, translates them into prioritized heal requests (decode failure -> Urgent ECDecode, metadata corruption -> High Metadata, partial write -> Normal object heal), and retries full admissions with a 5s backoff and a 3-attempt ceiling. Durability: every journal record carries its own CRC32 and a format/version header, so a torn tail truncates cleanly at replay; the journal is deleted after a successful replay and when the pending set drains (mirroring MinIO's post-replay list.bin unlink). Losing the last 500 ms flush window is acceptable: replayed duplicates merge via the manager dedup key and read-repair remains the safety net. Metrics: rustfs_heal_mrf_queue_depth/_queue_bytes, _dropped_total {reason}, _replayed_total, _journal_bytes, _journal_fsync_total. The consumer is wired at heal runtime bootstrap right after manager start, honoring RUSTFS_HEAL_MRF_ENABLE (default on, rollback = off). Tests: unit tests for the dual ceiling, record roundtrip, torn-tail truncation, and the priority mapping; integration tests against a real 4-disk ECStore proving channel intents reach the manager queue as Urgent/mrf-attributed requests and journal replay arms intents, drops torn tails, and removes the file. Part of backlog#1865 (option a). Co-Authored-By: heihutu * feat(ecstore,scanner): deliver MRF intents from error paths (HS-01) Wire the three production delivery points, each a single non-blocking try_send next to the existing in-memory heal paths, which stay as the fast path: - read.rs decode-error branch: DecodeFailure intent beside the existing read-repair submit, so an Urgent ECDecode request survives restarts even when the Low-priority read-repair request was dropped or lost. - add_partial: PartialWrite intent, giving partial-write recovery a durable Normal-priority object heal across restarts. - scanner_folder metadata-corruption classification: MetadataCorruption intent beside the existing High-priority scanner heal request. All three are on error paths only: zero cost on healthy IO. Part of backlog#1865 (option a). Co-Authored-By: heihutu * fix: include mrf heal source counts Co-Authored-By: heihutu * fix: keep node heal status wire compatibility Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- crates/common/src/heal_channel.rs | 4 + crates/common/src/lib.rs | 1 + crates/common/src/mrf_channel.rs | 203 ++++++ crates/config/src/constants/heal.rs | 28 + crates/ecstore/src/set_disk/ops/object.rs | 8 + crates/ecstore/src/set_disk/read.rs | 9 + crates/heal/Cargo.toml | 2 + crates/heal/src/heal/channel.rs | 3 +- crates/heal/src/heal/manager.rs | 3 + crates/heal/src/heal/mod.rs | 1 + crates/heal/src/heal/mrf_queue.rs | 682 ++++++++++++++++++++ crates/heal/src/lib.rs | 4 + crates/heal/tests/mrf_pipeline_test.rs | 189 ++++++ crates/scanner/src/scanner_folder.rs | 9 + rustfs/src/admin/handlers/heal.rs | 2 + rustfs/src/storage/rpc/node_service/heal.rs | 1 + 16 files changed, 1148 insertions(+), 1 deletion(-) create mode 100644 crates/common/src/mrf_channel.rs create mode 100644 crates/heal/src/heal/mrf_queue.rs create mode 100644 crates/heal/tests/mrf_pipeline_test.rs diff --git a/crates/common/src/heal_channel.rs b/crates/common/src/heal_channel.rs index f0ea0530e..b5a907e4e 100644 --- a/crates/common/src/heal_channel.rs +++ b/crates/common/src/heal_channel.rs @@ -287,6 +287,9 @@ pub enum HealRequestSource { Scanner, AutoHeal, ReadRepair, + /// Mission Repair Feed: intents delivered by error paths and replayed + /// from the durable MRF journal. + Mrf, } impl HealRequestSource { @@ -297,6 +300,7 @@ impl HealRequestSource { Self::Scanner => "scanner", Self::AutoHeal => "auto_heal", Self::ReadRepair => "read_repair", + Self::Mrf => "mrf", } } } diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 1ae200f24..68f5f5c68 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -17,6 +17,7 @@ pub mod globals; pub mod heal_channel; pub mod last_minute; pub mod metrics; +pub mod mrf_channel; mod readiness; pub mod table_catalog; pub mod trace_bus; diff --git a/crates/common/src/mrf_channel.rs b/crates/common/src/mrf_channel.rs new file mode 100644 index 000000000..f0a91a238 --- /dev/null +++ b/crates/common/src/mrf_channel.rs @@ -0,0 +1,203 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Mission Repair Feed (MRF) intent channel. +//! +//! Producers on error paths (read decode failure, scanner metadata +//! corruption, partial-write recovery) hand a lightweight [`MrfIntent`] to the +//! heal crate through a global bounded channel. Delivery is strictly +//! non-blocking: `try_send_mrf_intent` never awaits and drops the intent +//! (counting it) when the channel is full or uninitialized — losing one heal +//! hint is always preferred over stalling an IO path. Durable replay of +//! unconsumed intents is the consumer's job (see `rustfs-heal` +//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`. + +use std::sync::{ + Arc, OnceLock, + atomic::{AtomicBool, Ordering}, +}; +use tokio::sync::mpsc; +use uuid::Uuid; + +/// Bounded capacity of the global MRF channel. Backpressure is resolved by +/// dropping (and counting) intents, never by blocking the producer. +const MRF_CHANNEL_CAPACITY: usize = 8192; + +/// Why an intent was produced. Drives the heal priority mapping on the +/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High, +/// PartialWrite -> Normal). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MrfKind { + /// Erasure decode failed while serving a read (read path). + DecodeFailure, + /// Scanner classified object metadata as corrupt. + MetadataCorruption, + /// A write left the object with fewer committed shards than the set size. + PartialWrite, +} + +impl MrfKind { + pub const fn as_str(self) -> &'static str { + match self { + MrfKind::DecodeFailure => "decode-failure", + MrfKind::MetadataCorruption => "metadata-corruption", + MrfKind::PartialWrite => "partial-write", + } + } +} + +/// One repair intent. Kept deliberately small so the in-memory queue and the +/// journal stay bounded; `bucket`/`object` are `Arc` so re-arming an +/// intent never re-allocates the strings. +#[derive(Clone, Debug)] +pub struct MrfIntent { + pub bucket: Arc, + pub object: Arc, + /// Version the intent targets, as raw UUID bytes. + pub version_id: Option<[u8; 16]>, + pub kind: MrfKind, + pub enqueued_at_ms: u64, + /// Times this intent has already been offered to the heal manager. + /// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`. + pub attempts: u8, +} + +/// Consumer-side retry ceiling before an intent is given up on. +pub const MRF_MAX_ATTEMPTS: u8 = 3; + +impl MrfIntent { + /// Rough in-memory footprint used by the queue's byte budget. + pub fn estimated_bytes(&self) -> usize { + // Struct + strings + version bytes; buckets and objects are usually + // far below this bound, so rounding up keeps the budget conservative. + 64 + self.bucket.len() + self.object.len() + } +} + +static GLOBAL_MRF_SENDER: OnceLock> = OnceLock::new(); + +/// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check +/// this before touching the channel so the disabled path stays allocation- and +/// sync-free. +static MRF_DELIVERY_ENABLED: AtomicBool = AtomicBool::new(true); + +/// Override delivery (used at heal-runtime startup from configuration). +pub fn set_mrf_delivery_enabled(enabled: bool) { + MRF_DELIVERY_ENABLED.store(enabled, Ordering::Relaxed); +} + +/// Whether producers currently deliver intents. +pub fn mrf_delivery_enabled() -> bool { + MRF_DELIVERY_ENABLED.load(Ordering::Relaxed) +} + +/// Create the global MRF channel and return the consumer half. Fails if the +/// channel is already initialized (the heal runtime is a singleton). +pub fn init_mrf_channel() -> Result, &'static str> { + let (sender, receiver) = mpsc::channel(MRF_CHANNEL_CAPACITY); + GLOBAL_MRF_SENDER + .set(sender) + .map_err(|_| "MRF channel sender already initialized")?; + Ok(receiver) +} + +/// Best-effort, non-blocking intent delivery from an error path. +/// +/// Returns `true` when the intent was accepted into the channel. `false` +/// means the intent was dropped (feature disabled, channel not yet +/// initialized, or channel full) — callers must not retry or await; the +/// existing read-repair / scanner heal paths remain the safety net. +/// +/// This runs on IO error paths, so it stays synchronous and cheap: one +/// bounded allocation for the two `Arc` handles plus the channel slot. +pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option) -> bool { + if !mrf_delivery_enabled() { + return false; + } + let Some(sender) = GLOBAL_MRF_SENDER.get() else { + return false; + }; + let intent = MrfIntent { + bucket: Arc::from(bucket), + object: Arc::from(object), + version_id: version_id.map(|vid| *vid.as_bytes()), + kind, + enqueued_at_ms: unix_now_ms(), + attempts: 0, + }; + sender.try_send(intent).is_ok() +} + +fn unix_now_ms() -> u64 { + // Kept trivial: the timestamp is diagnostic metadata only; wall-clock + // failure would be a bug rather than something to handle here. + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn intents_estimate_is_conservative() { + let intent = MrfIntent { + bucket: Arc::from("bucket"), + object: Arc::from("object"), + version_id: Some([0u8; 16]), + kind: MrfKind::DecodeFailure, + enqueued_at_ms: 0, + attempts: 0, + }; + assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len()); + } + + #[tokio::test] + async fn try_send_delivers_and_respects_capacity() { + let mut receiver = init_mrf_channel().expect("first initialization should succeed"); + assert!(init_mrf_channel().is_err(), "double initialization must fail"); + + assert!(try_send_mrf_intent(MrfKind::DecodeFailure, "b", "o", Some(Uuid::nil()))); + let intent = receiver.recv().await.expect("intent should arrive"); + assert_eq!(intent.kind, MrfKind::DecodeFailure); + assert_eq!(intent.bucket.as_ref(), "b"); + + // Disable delivery: producers become no-ops. + set_mrf_delivery_enabled(false); + assert!(!try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None)); + set_mrf_delivery_enabled(true); + + // Fill the bounded channel past capacity: excess intents are dropped, + // never blocking. + let mut accepted = 0; + for _ in 0..(MRF_CHANNEL_CAPACITY + 64) { + if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) { + accepted += 1; + } + } + assert_eq!(accepted, MRF_CHANNEL_CAPACITY); + } + + #[test] + fn try_send_without_channel_is_false() { + // This test may run after the tokio test above in the same process; + // the singleton semantics make a clean "uninitialized" case hard, so + // assert the flag-off behavior only. + set_mrf_delivery_enabled(false); + assert!(!try_send_mrf_intent(MrfKind::MetadataCorruption, "b", "o", None)); + set_mrf_delivery_enabled(true); + } +} diff --git a/crates/config/src/constants/heal.rs b/crates/config/src/constants/heal.rs index 647ca8533..b8cf3630a 100644 --- a/crates/config/src/constants/heal.rs +++ b/crates/config/src/constants/heal.rs @@ -177,3 +177,31 @@ pub const DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT: usize = 80; /// Default foreground pressure recheck delay for heal scheduler, in milliseconds. pub const DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS: u64 = 250; + +/// Environment variable that toggles the MRF (mission repair feed) intent +/// pipeline: error paths deliver repair intents to the heal runtime, and +/// unconsumed intents are replayed from the durable journal after a restart. +pub const ENV_HEAL_MRF_ENABLE: &str = "RUSTFS_HEAL_MRF_ENABLE"; + +/// Environment variable for the MRF in-memory queue capacity (intent count). +pub const ENV_HEAL_MRF_QUEUE_SIZE: &str = "RUSTFS_HEAL_MRF_QUEUE_SIZE"; + +/// Environment variable for the MRF journal byte budget. The journal is +/// compacted once its on-disk size crosses this bound. +pub const ENV_HEAL_MRF_JOURNAL_MAX_BYTES: &str = "RUSTFS_HEAL_MRF_JOURNAL_MAX_BYTES"; + +/// Environment variable for the MRF journal replay batch size (intents per +/// replay push round). +pub const ENV_HEAL_MRF_REPLAY_BATCH: &str = "RUSTFS_HEAL_MRF_REPLAY_BATCH"; + +/// Default behavior keeps the MRF intent pipeline enabled. +pub const DEFAULT_HEAL_MRF_ENABLE: bool = true; + +/// Default MRF queue capacity (matches MinIO's 100k MRF list ceiling). +pub const DEFAULT_HEAL_MRF_QUEUE_SIZE: usize = 100_000; + +/// Default MRF journal byte budget (8 MiB), mirroring the channel payload cap. +pub const DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES: usize = 8 * 1024 * 1024; + +/// Default MRF replay batch size. +pub const DEFAULT_HEAL_MRF_REPLAY_BATCH: usize = 256; diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index e7498ee9d..929ac4d5c 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -5845,6 +5845,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { #[tracing::instrument(skip(self))] async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { + // MRF journal intent: partial-write recovery must survive a restart + // (HS-01); the heal request below remains the in-memory fast path. + rustfs_common::mrf_channel::try_send_mrf_intent( + rustfs_common::mrf_channel::MrfKind::PartialWrite, + bucket, + object, + uuid::Uuid::try_parse(version_id).ok(), + ); let mut request = rustfs_common::heal_channel::create_heal_request_with_options( bucket.to_string(), Some(object.to_string()), diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index d59fa3e7e..07bbcf5f9 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -1077,6 +1077,15 @@ impl SetDisks { "Recoverable decode error triggered read repair" ); let version_id = fi.version_id.as_ref().map(ToString::to_string); + // MRF journal intent: keeps a durable Urgent ECDecode + // request alive across restarts even when the in-memory + // read-repair request is dropped or lost (HS-01). + rustfs_common::mrf_channel::try_send_mrf_intent( + rustfs_common::mrf_channel::MrfKind::DecodeFailure, + bucket, + object, + fi.version_id, + ); submit_read_repair_heal( bucket, object, diff --git a/crates/heal/Cargo.toml b/crates/heal/Cargo.toml index 668c4af24..4d4fb355b 100644 --- a/crates/heal/Cargo.toml +++ b/crates/heal/Cargo.toml @@ -89,6 +89,8 @@ async-trait = { workspace = true } futures = { workspace = true } metrics = { workspace = true } base64 = { workspace = true } +bytes = { workspace = true } +crc-fast = { workspace = true } [dev-dependencies] serde_json = { workspace = true, features = ["raw_value"] } diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index 23cf2f168..1e31056b2 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -612,7 +612,8 @@ impl HealChannelProcessor { HealRequestSource::Admin | HealRequestSource::AutoHeal | HealRequestSource::Internal - | HealRequestSource::ReadRepair => true, + | HealRequestSource::ReadRepair + | HealRequestSource::Mrf => true, }); // Build HealOptions with all available fields diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 66b8f637f..216e17068 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -270,6 +270,8 @@ pub struct HealSourceCounts { pub auto_heal: u64, pub internal: u64, pub read_repair: u64, + #[serde(default)] + pub mrf: u64, } impl HealSourceCounts { @@ -280,6 +282,7 @@ impl HealSourceCounts { HealRequestSource::AutoHeal => self.auto_heal += 1, HealRequestSource::Internal => self.internal += 1, HealRequestSource::ReadRepair => self.read_repair += 1, + HealRequestSource::Mrf => self.mrf += 1, } } } diff --git a/crates/heal/src/heal/mod.rs b/crates/heal/src/heal/mod.rs index 0881e5853..ff910ed70 100644 --- a/crates/heal/src/heal/mod.rs +++ b/crates/heal/src/heal/mod.rs @@ -16,6 +16,7 @@ pub mod channel; pub mod erasure_healer; pub mod event; pub mod manager; +pub mod mrf_queue; pub mod progress; pub(crate) mod replacement_readiness; pub mod resume; diff --git a/crates/heal/src/heal/mrf_queue.rs b/crates/heal/src/heal/mrf_queue.rs new file mode 100644 index 000000000..c1144435e --- /dev/null +++ b/crates/heal/src/heal/mrf_queue.rs @@ -0,0 +1,682 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Mission Repair Feed (MRF) queue, journal, and consumer. +//! +//! Intents arriving on the global channel (see `rustfs_common::mrf_channel`) +//! are buffered in a bounded in-memory queue, translated into prioritized +//! heal requests, and — while they are not yet accepted by the heal manager — +//! mirrored into a durable journal so a crash or restart can replay them. +//! This is the RustFS counterpart of MinIO's `.heal/mrf/list.bin` replay, +//! layered on top of (not replacing) read-repair and scanner heal. +//! +//! Durability model: the journal is a snapshot of the *unaccepted* pending +//! set, rewritten on a group-commit cadence (every flush interval or flush +//! threshold new intents). A rewrite is atomic at the record level only — a +//! torn tail simply truncates during replay because every record carries its +//! own CRC32. Losing the last flush window (≤500 ms) is acceptable: replayed +//! duplicates are merged by the manager's dedup key, and read-repair remains +//! the safety net. + +use super::{DiskStore, HealDiskExt as _, local_disk_map_read}; +use crate::heal::manager::HealManager; +use metrics::{counter, gauge}; +use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult}; +use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfIntent}; +use std::collections::VecDeque; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::mpsc; +use uuid::Uuid; + +use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType}; + +/// Journal location inside the metadata bucket, following the resume-state +/// layout. +pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin"; + +/// Record format tag. +const MRF_JOURNAL_FORMAT: u8 = 1; +/// Record layout version. +const MRF_JOURNAL_VERSION: u8 = 1; + +/// Fixed header size: format, version, kind, attempts, enqueued_at_ms, +/// has_version flag. +const MRF_RECORD_FIXED_HEAD: usize = 1 + 1 + 1 + 1 + 8 + 1; + +#[derive(Debug, Clone)] +pub(crate) struct MrfConsumerConfig { + /// In-memory queue capacity in intents. + pub queue_capacity: usize, + /// Journal byte budget; a pending snapshot above this bound is rejected + /// oldest-first so the journal can never grow unbounded. + pub journal_max_bytes: usize, + /// How many journal intents to re-arm per replay round. + pub replay_batch: usize, + /// Group-commit cadence for the journal snapshot. + pub flush_interval: Duration, + /// New intents between flushes that force an early snapshot. + pub flush_threshold: usize, + /// Backoff after the heal manager reports a full admission. + pub admission_backoff: Duration, +} + +impl Default for MrfConsumerConfig { + fn default() -> Self { + Self { + queue_capacity: rustfs_utils::get_env_usize( + rustfs_config::ENV_HEAL_MRF_QUEUE_SIZE, + rustfs_config::DEFAULT_HEAL_MRF_QUEUE_SIZE, + ), + journal_max_bytes: rustfs_utils::get_env_usize( + rustfs_config::ENV_HEAL_MRF_JOURNAL_MAX_BYTES, + rustfs_config::DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES, + ), + replay_batch: rustfs_utils::get_env_usize( + rustfs_config::ENV_HEAL_MRF_REPLAY_BATCH, + rustfs_config::DEFAULT_HEAL_MRF_REPLAY_BATCH, + ), + flush_interval: Duration::from_millis(500), + flush_threshold: 1000, + admission_backoff: Duration::from_secs(5), + } + } +} + +/// Bounded pending set with count and byte ceilings. Overflow drops the +/// incoming intent (never a resident one) and counts the loss. +pub(crate) struct MrfQueue { + pending: VecDeque, + bytes: usize, + capacity: usize, + byte_budget: usize, +} + +impl MrfQueue { + pub(crate) fn new(capacity: usize, byte_budget: usize) -> Self { + Self { + pending: VecDeque::new(), + bytes: 0, + capacity, + byte_budget, + } + } + + /// Returns `false` (after counting) when either ceiling would be crossed. + pub(crate) fn try_push(&mut self, intent: MrfIntent) -> bool { + let cost = intent.estimated_bytes(); + if self.pending.len() >= self.capacity || self.bytes + cost > self.byte_budget { + counter!("rustfs_heal_mrf_dropped_total", "reason" => "queue_overflow").increment(1); + return false; + } + self.bytes += cost; + self.pending.push_back(intent); + true + } + + pub(crate) fn pop_front(&mut self) -> Option { + let intent = self.pending.pop_front()?; + self.bytes = self.bytes.saturating_sub(intent.estimated_bytes()); + Some(intent) + } + + pub(crate) fn push_back(&mut self, intent: MrfIntent) { + self.bytes += intent.estimated_bytes(); + self.pending.push_back(intent); + } + + pub(crate) fn depth(&self) -> usize { + self.pending.len() + } + + pub(crate) fn bytes(&self) -> usize { + self.bytes + } + + pub(crate) fn intents(&self) -> impl Iterator { + self.pending.iter() + } +} + +// --------------------------------------------------------------------------- +// Journal record codec +// --------------------------------------------------------------------------- + +/// Append one encoded record to `out`. +pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec) { + let start = out.len(); + out.push(MRF_JOURNAL_FORMAT); + out.push(MRF_JOURNAL_VERSION); + out.push(match intent.kind { + rustfs_common::mrf_channel::MrfKind::DecodeFailure => 1, + rustfs_common::mrf_channel::MrfKind::MetadataCorruption => 2, + rustfs_common::mrf_channel::MrfKind::PartialWrite => 3, + }); + out.push(intent.attempts); + out.extend_from_slice(&intent.enqueued_at_ms.to_le_bytes()); + match intent.version_id { + Some(bytes) => { + out.push(1); + out.extend_from_slice(&bytes); + } + None => out.push(0), + } + out.extend_from_slice(&(intent.bucket.len() as u32).to_le_bytes()); + out.extend_from_slice(&(intent.object.len() as u32).to_le_bytes()); + out.extend_from_slice(intent.bucket.as_bytes()); + out.extend_from_slice(intent.object.as_bytes()); + let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); + hasher.update(&out[start..]); + out.extend_from_slice(&(hasher.finalize() as u32).to_le_bytes()); +} + +fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> { + if data.len() < MRF_RECORD_FIXED_HEAD + 8 { + return None; + } + if data[0] != MRF_JOURNAL_FORMAT || data[1] != MRF_JOURNAL_VERSION { + return None; + } + let kind = match data[2] { + 1 => rustfs_common::mrf_channel::MrfKind::DecodeFailure, + 2 => rustfs_common::mrf_channel::MrfKind::MetadataCorruption, + 3 => rustfs_common::mrf_channel::MrfKind::PartialWrite, + _ => return None, + }; + let attempts = data[3]; + let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().expect("slice length checked")); + let has_version = data[12] != 0; + let mut cursor = MRF_RECORD_FIXED_HEAD; + let version_id = if has_version { + if data.len() < cursor + 16 { + return None; + } + let bytes: [u8; 16] = data[cursor..cursor + 16].try_into().expect("slice length checked"); + cursor += 16; + Some(bytes) + } else { + None + }; + if data.len() < cursor + 8 { + return None; + } + let bucket_len = u32::from_le_bytes(data[cursor..cursor + 4].try_into().expect("slice length checked")) as usize; + let object_len = u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().expect("slice length checked")) as usize; + cursor += 8; + let body_end = cursor.checked_add(bucket_len)?.checked_add(object_len)?; + let record_end = body_end.checked_add(4)?; + if data.len() < record_end { + return None; + } + let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); + hasher.update(&data[..body_end]); + if (hasher.finalize() as u32) != u32::from_le_bytes(data[body_end..record_end].try_into().expect("slice length checked")) { + return None; + } + let bucket = std::sync::Arc::from(std::str::from_utf8(&data[cursor..cursor + bucket_len]).ok()?); + let object = std::sync::Arc::from(std::str::from_utf8(&data[cursor + bucket_len..body_end]).ok()?); + Some(( + MrfIntent { + bucket, + object, + version_id, + kind, + enqueued_at_ms, + attempts, + }, + record_end, + )) +} + +/// Decode a whole journal, stopping at the first torn or corrupt record. +/// Returns the decoded intents and the number of trailing bytes discarded. +pub(crate) fn decode_journal(data: &[u8]) -> (Vec, usize) { + let mut intents = Vec::new(); + let mut cursor = 0usize; + while cursor < data.len() { + match decode_one(&data[cursor..]) { + Some((intent, consumed)) => { + intents.push(intent); + cursor += consumed; + } + None => break, + } + } + let truncated = data.len() - cursor; + (intents, truncated) +} + +// --------------------------------------------------------------------------- +// Journal disk IO (all local disks, first successful read wins) +// --------------------------------------------------------------------------- + +async fn journal_disks() -> Vec { + let map = local_disk_map_read().await; + map.values().flatten().cloned().collect() +} + +async fn read_journal() -> Option> { + for disk in journal_disks().await { + match disk.read_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH).await { + Ok(bytes) => return Some(bytes.to_vec()), + Err(_) => continue, + } + } + None +} + +async fn write_journal(data: &[u8]) { + let payload = bytes::Bytes::copy_from_slice(data); + for disk in journal_disks().await { + if let Err(err) = disk + .write_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload.clone()) + .await + { + warn_mrf_journal_write(&err); + } + } + if !data.is_empty() { + counter!("rustfs_heal_mrf_journal_fsync_total").increment(1); + } + gauge!("rustfs_heal_mrf_journal_bytes").set(data.len() as f64); +} + +async fn delete_journal() { + for disk in journal_disks().await { + let _ = disk + .delete( + super::RUSTFS_META_BUCKET, + MRF_JOURNAL_PATH, + crate::heal::storage_api::owner::EcstoreDeleteOptions::default(), + ) + .await; + } +} + +fn warn_mrf_journal_write(err: &super::DiskError) { + tracing::warn!( + target: "rustfs::heal::mrf", + error = %err, + "MRF journal write failed; unconsumed intents may be lost on restart" + ); +} + +// --------------------------------------------------------------------------- +// Consumer +// --------------------------------------------------------------------------- + +/// Translate an intent into the prioritized heal request the issue specifies: +/// decode failures go Urgent ECDecode, metadata corruption goes High +/// Metadata, partial writes go Normal object heal. +pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest { + let bucket = intent.bucket.to_string(); + let object = intent.object.to_string(); + let version_id = intent.version_id.map(|bytes| Uuid::from_bytes(bytes).to_string()); + let (heal_type, priority) = match intent.kind { + rustfs_common::mrf_channel::MrfKind::DecodeFailure => ( + HealType::ECDecode { + bucket, + object, + version_id, + }, + HealPriority::Urgent, + ), + rustfs_common::mrf_channel::MrfKind::MetadataCorruption => (HealType::Metadata { bucket, object }, HealPriority::High), + rustfs_common::mrf_channel::MrfKind::PartialWrite => ( + HealType::Object { + bucket, + object, + version_id, + }, + HealPriority::Normal, + ), + }; + let mut request = HealRequest::new(heal_type, HealOptions::default(), priority); + request.source = rustfs_common::heal_channel::HealRequestSource::Mrf; + request +} + +struct MrfRuntime { + queue: MrfQueue, + config: MrfConsumerConfig, + new_since_flush: usize, + /// True while a journal snapshot exists on disk that no longer reflects + /// an all-consumed pending set; the next idle tick removes it (MinIO + /// deletes its `list.bin` after replay for the same reason). + journal_on_disk: bool, + /// Earliest instant a full-admission retry may proceed. + backoff_until: Option, +} + +impl MrfRuntime { + fn record_accept(&mut self) { + // Accepted intents leave the pending set; the next flush persists the + // smaller snapshot, which is the journal's compaction. + } + + fn snapshot(&self) -> Vec { + let mut buf = Vec::new(); + for intent in self.queue.intents() { + encode_intent(intent, &mut buf); + } + buf + } + + async fn flush(&mut self) { + write_journal(&self.snapshot()).await; + self.new_since_flush = 0; + self.journal_on_disk = true; + } + + /// Drain pending intents into the heal manager until it is full, the + /// queue empties, or attempts are exhausted. + async fn dispatch(&mut self, manager: &HealManager) { + if let Some(until) = self.backoff_until { + if tokio::time::Instant::now() < until { + return; + } + self.backoff_until = None; + } + while let Some(mut intent) = self.queue.pop_front() { + let request = build_heal_request(&intent); + match manager.submit_heal_request(request).await { + Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => self.record_accept(), + Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => { + intent.attempts = intent.attempts.saturating_add(1); + if intent.attempts >= MRF_MAX_ATTEMPTS { + counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1); + continue; + } + self.queue.push_back(intent); + self.backoff_until = Some(tokio::time::Instant::now() + self.config.admission_backoff); + break; + } + Ok(HealAdmissionResult::Dropped(_)) => { + counter!("rustfs_heal_mrf_dropped_total", "reason" => "admission_policy").increment(1); + } + Err(_) => { + intent.attempts = intent.attempts.saturating_add(1); + if intent.attempts >= MRF_MAX_ATTEMPTS { + counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1); + continue; + } + self.queue.push_back(intent); + self.backoff_until = Some(tokio::time::Instant::now() + self.config.admission_backoff); + break; + } + } + } + gauge!("rustfs_heal_mrf_queue_depth").set(self.queue.depth() as f64); + gauge!("rustfs_heal_mrf_queue_bytes").set(self.queue.bytes() as f64); + } +} + +/// Initialize the global MRF channel (honoring `RUSTFS_HEAL_MRF_ENABLE`) and +/// spawn the consumer task. Called once from the heal runtime bootstrap right +/// after the manager started; a disabled feature or a double call is a no-op. +/// Public for integration tests that drive the real consumer loop. +pub fn spawn_mrf_consumer(manager: Arc) { + let enabled = rustfs_utils::get_env_bool(rustfs_config::ENV_HEAL_MRF_ENABLE, rustfs_config::DEFAULT_HEAL_MRF_ENABLE); + rustfs_common::mrf_channel::set_mrf_delivery_enabled(enabled); + if !enabled { + tracing::info!( + target: "rustfs::heal::mrf", + "MRF intent pipeline disabled by configuration; producers will not deliver" + ); + return; + } + let receiver = match rustfs_common::mrf_channel::init_mrf_channel() { + Ok(receiver) => receiver, + Err(err) => { + tracing::warn!( + target: "rustfs::heal::mrf", + error = err, + "MRF channel initialization failed; intents will be dropped at producers" + ); + return; + } + }; + tokio::spawn(async move { + run_mrf_consumer(manager, receiver).await; + }); + tracing::info!(target: "rustfs::heal::mrf", "MRF intent consumer started"); +} + +/// Replay the durable journal into a fresh pending queue and submit whatever +/// it armed. Returns the number of intact intents replayed. Duplicates are +/// merged by the manager's dedup key; the journal file is removed once read +/// (torn tails truncate via the per-record CRC). Public for integration tests; +/// the live consumer invokes this through [`replay_into`] at startup. +pub async fn replay_journal_once(manager: &Arc) -> usize { + let config = MrfConsumerConfig::default(); + let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes); + let mut backoff_until: Option = None; + replay_into(manager, &mut queue, &mut backoff_until).await +} + +/// Shared replay core: read + decode + re-arm + delete, then drain what fits. +async fn replay_into( + manager: &Arc, + queue: &mut MrfQueue, + backoff_until: &mut Option, +) -> usize { + let Some(data) = read_journal().await else { + return 0; + }; + let (intents, truncated) = decode_journal(&data); + if truncated > 0 { + tracing::warn!( + target: "rustfs::heal::mrf", + truncated_bytes = truncated, + "MRF journal had a torn tail; truncated records were discarded" + ); + } + counter!("rustfs_heal_mrf_replayed_total").increment(intents.len() as u64); + let replayed = intents.len(); + for intent in intents { + queue.try_push(intent); + } + delete_journal().await; + + // Drain the replayed intents immediately; whatever the manager refuses + // stays armed in `queue` for the consumer's retry loop. + if backoff_until.is_none() { + while let Some(mut intent) = queue.pop_front() { + let request = build_heal_request(&intent); + match manager.submit_heal_request(request).await { + Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {} + Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => { + intent.attempts = intent.attempts.saturating_add(1); + if intent.attempts < MRF_MAX_ATTEMPTS { + queue.push_back(intent); + *backoff_until = Some(tokio::time::Instant::now()); + } + break; + } + Ok(HealAdmissionResult::Dropped(_)) | Err(_) => {} + } + } + } + replayed +} + +/// Replay the journal, then keep draining the channel into the heal manager +/// while persisting the pending snapshot. +async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receiver) { + let config = MrfConsumerConfig::default(); + let mut runtime = MrfRuntime { + queue: MrfQueue::new(config.queue_capacity, config.journal_max_bytes), + config: config.clone(), + new_since_flush: 0, + journal_on_disk: false, + backoff_until: None, + }; + + // Replay: read the journal, re-arm intents (duplicates are merged by the + // manager's dedup key), then drop the file so the next flush starts clean. + replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await; + + let mut flush_tick = tokio::time::interval(runtime.config.flush_interval); + flush_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut batch: Vec = Vec::with_capacity(runtime.config.replay_batch); + + loop { + tokio::select! { + received = receiver.recv_many(&mut batch, runtime.config.replay_batch) => { + if received == 0 { + // Channel closed: flush once more and stop. + runtime.flush().await; + tracing::info!( + target: "rustfs::heal::mrf", + "MRF channel closed; consumer stopped after final flush" + ); + return; + } + for intent in batch.drain(..) { + runtime.queue.try_push(intent); + runtime.new_since_flush += 1; + } + runtime.dispatch(manager.as_ref()).await; + if runtime.new_since_flush >= runtime.config.flush_threshold { + runtime.flush().await; + } + } + _ = flush_tick.tick() => { + if runtime.new_since_flush > 0 || runtime.queue.depth() > 0 { + runtime.flush().await; + runtime.dispatch(manager.as_ref()).await; + } else if runtime.journal_on_disk { + // All intents consumed: remove the journal so a restart + // replays nothing (mirrors MinIO's post-replay unlink). + delete_journal().await; + runtime.journal_on_disk = false; + gauge!("rustfs_heal_mrf_journal_bytes").set(0.0); + } + gauge!("rustfs_heal_mrf_queue_depth").set(runtime.queue.depth() as f64); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rustfs_common::mrf_channel::{MrfIntent, MrfKind}; + use std::sync::Arc as StdArc; + + fn intent(bucket: &str, object: &str, attempts: u8) -> MrfIntent { + MrfIntent { + bucket: StdArc::from(bucket), + object: StdArc::from(object), + version_id: Some([7u8; 16]), + kind: MrfKind::DecodeFailure, + enqueued_at_ms: 1_700_000_000_000, + attempts, + } + } + + #[test] + fn queue_enforces_count_and_byte_ceilings() { + let mut queue = MrfQueue::new(2, usize::MAX); + assert!(queue.try_push(intent("b", "o", 0))); + assert!(queue.try_push(intent("b", "o", 0))); + assert!(!queue.try_push(intent("b", "o", 0)), "count ceiling must drop"); + + let mut tiny = MrfQueue::new(usize::MAX, intent("bucket", "object", 0).estimated_bytes()); + assert!(tiny.try_push(intent("bucket", "object", 0))); + assert!( + !tiny.try_push(intent("bucket", "object", 0)), + "byte budget must drop before the second intent fits" + ); + } + + #[test] + fn journal_roundtrip_preserves_intents() { + let intents = vec![ + intent("bucket-a", "object/a", 0), + intent("bucket-b", "object/b", 2), + MrfIntent { + bucket: StdArc::from("bucket-c"), + object: StdArc::from("object/c"), + version_id: None, + kind: MrfKind::MetadataCorruption, + enqueued_at_ms: 5, + attempts: 1, + }, + ]; + let mut buf = Vec::new(); + for intent in &intents { + encode_intent(intent, &mut buf); + } + let (decoded, truncated) = decode_journal(&buf); + assert_eq!(truncated, 0); + assert_eq!(decoded.len(), intents.len()); + for (left, right) in decoded.iter().zip(intents.iter()) { + assert_eq!(left.bucket, right.bucket); + assert_eq!(left.object, right.object); + assert_eq!(left.version_id, right.version_id); + assert_eq!(left.kind, right.kind); + assert_eq!(left.attempts, right.attempts); + } + } + + #[test] + fn journal_torn_tail_is_truncated() { + let mut buf = Vec::new(); + encode_intent(&intent("b", "o", 0), &mut buf); + let mut torn = buf.clone(); + torn.extend_from_slice(&buf[..buf.len() / 2]); + + let (decoded, truncated) = decode_journal(&torn); + assert_eq!(decoded.len(), 1, "the intact record must survive"); + assert!(truncated > 0, "the partial tail must be discarded"); + + // A corrupted body (CRC mismatch) also truncates from that record on. + let mut corrupt = buf.clone(); + let mid = MRF_RECORD_FIXED_HEAD + 4; + corrupt[mid] ^= 0xff; + let (decoded, truncated) = decode_journal(&corrupt); + assert!(decoded.is_empty()); + assert_eq!(truncated, corrupt.len()); + } + + #[test] + fn heal_request_mapping_follows_priority_matrix() { + let decode = build_heal_request(&intent("b", "o", 0)); + assert!(matches!(decode.heal_type, HealType::ECDecode { .. })); + assert_eq!(decode.priority, HealPriority::Urgent); + + let metadata = build_heal_request(&MrfIntent { + bucket: StdArc::from("b"), + object: StdArc::from("o"), + version_id: None, + kind: MrfKind::MetadataCorruption, + enqueued_at_ms: 0, + attempts: 0, + }); + assert!(matches!(metadata.heal_type, HealType::Metadata { .. })); + assert_eq!(metadata.priority, HealPriority::High); + + let partial = build_heal_request(&MrfIntent { + bucket: StdArc::from("b"), + object: StdArc::from("o"), + version_id: None, + kind: MrfKind::PartialWrite, + enqueued_at_ms: 0, + attempts: 0, + }); + assert!(matches!(partial.heal_type, HealType::Object { .. })); + assert_eq!(partial.priority, HealPriority::Normal); + } +} diff --git a/crates/heal/src/lib.rs b/crates/heal/src/lib.rs index 3dd29b064..f1ec4cebe 100644 --- a/crates/heal/src/lib.rs +++ b/crates/heal/src/lib.rs @@ -158,6 +158,10 @@ pub async fn init_heal_manager_with_workload_provider( return Err(err); } + // Start the MRF intent consumer (error-path repair intents + durable + // journal replay) now that the manager can accept submissions. + heal::mrf_queue::spawn_mrf_consumer(heal_manager.clone()); + #[cfg(test)] test_hook_after_manager_start().await; diff --git a/crates/heal/tests/mrf_pipeline_test.rs b/crates/heal/tests/mrf_pipeline_test.rs new file mode 100644 index 000000000..36f07a5a7 --- /dev/null +++ b/crates/heal/tests/mrf_pipeline_test.rs @@ -0,0 +1,189 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! HS-01 (rustfs/backlog#1865): MRF intent pipeline integration tests. +//! +//! Drives the real consumer loop (`spawn_mrf_consumer`) against a real +//! 4-disk `ECStore` heal storage and a `HealManager` that has not started its +//! scheduler, so submitted intents stay observable in the admission queue. +//! Under `cargo nextest` each test runs in its own process, which keeps the +//! process-global MRF channel singleton safe. + +use rustfs_common::mrf_channel::{self, MrfKind}; +use rustfs_heal::heal::{ + manager::{HealConfig, HealManager}, + mrf_queue, + storage::{ECStoreHealStorage, HealStorageAPI}, +}; +use serial_test::serial; +use std::{path::Path, sync::Arc, time::Duration}; + +mod storage_api; + +use storage_api::endpoint_index::{Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks}; + +const META_BUCKET: &str = ".rustfs.sys"; +const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin"; + +async fn heal_env() -> (Vec, Arc) { + let env = rustfs_test_utils::TestECStoreEnv::builder() + .prefix("rustfs_heal_mrf_test") + .build() + .await; + let heal_storage: Arc = Arc::new(ECStoreHealStorage::new(env.ecstore.clone())); + (env.disk_paths, heal_storage) +} + +fn make_manager(storage: Arc) -> Arc { + Arc::new(HealManager::new( + storage, + Some(HealConfig { + // Keep the scheduler from draining the queue before assertions. + heal_interval: Duration::from_secs(3600), + enable_auto_heal: false, + ..Default::default() + }), + )) +} + +/// Encode one journal record independently of the implementation, so a format +/// drift between writer and this fixture fails loudly here. +fn journal_record(kind: u8, bucket: &str, object: &str, version: Option<[u8; 16]>, attempts: u8) -> Vec { + let mut body = vec![1u8, 1, kind, attempts]; + body.extend_from_slice(&1_700_000_000_000u64.to_le_bytes()); + match version { + Some(bytes) => { + body.push(1); + body.extend_from_slice(&bytes); + } + None => body.push(0), + } + body.extend_from_slice(&(bucket.len() as u32).to_le_bytes()); + body.extend_from_slice(&(object.len() as u32).to_le_bytes()); + body.extend_from_slice(bucket.as_bytes()); + body.extend_from_slice(object.as_bytes()); + let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); + hasher.update(&body); + body.extend_from_slice(&(hasher.finalize() as u32).to_le_bytes()); + body +} + +fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) { + for path in disk_paths { + let journal = path.join(META_BUCKET).join(JOURNAL_REL); + std::fs::create_dir_all(journal.parent().expect("journal parent")).expect("create journal dir"); + std::fs::write(&journal, data).expect("write journal fixture"); + } +} + +async fn wait_until(deadline: Duration, mut probe: F) -> bool +where + F: FnMut() -> Fut, + Fut: std::future::Future, +{ + let start = std::time::Instant::now(); + while start.elapsed() < deadline { + if probe().await { + return true; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + false +} + +/// A decode-failure intent delivered on the global channel must surface in the +/// heal manager as an Urgent request attributed to the MRF source. +#[tokio::test] +#[serial] +async fn decode_failure_intent_maps_to_urgent_mrf_heal_request() { + let (_disk_paths, storage) = heal_env().await; + let manager = make_manager(storage); + + mrf_queue::spawn_mrf_consumer(manager.clone()); + + assert!( + mrf_channel::try_send_mrf_intent(MrfKind::DecodeFailure, "mrf-bucket", "mrf-object", None), + "intent should be accepted while the consumer holds the channel" + ); + + let appeared = wait_until(Duration::from_secs(10), || async { + let snapshot = manager.operations_snapshot().await; + snapshot.queued_by_source.mrf >= 1 && snapshot.queued_by_priority.urgent >= 1 + }) + .await; + assert!( + appeared, + "MRF intent must reach the manager queue as an Urgent request (snapshot: {:?})", + manager.operations_snapshot().await + ); +} + +/// A journal left behind by a previous process must be replayed into the +/// manager queue and then removed, and a torn tail must not block replay of +/// the intact records. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn journal_replay_arms_intents_and_deletes_the_file() { + let (disk_paths, storage) = heal_env().await; + + // The journal reader resolves disks through the process-local disk map; + // register the environment's disks the same way server startup does. + let mut endpoints: Vec = disk_paths + .iter() + .map(|p| Endpoint::try_from(p.to_string_lossy().as_ref()).expect("endpoint from disk path")) + .collect(); + for (i, endpoint) in endpoints.iter_mut().enumerate() { + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(i); + } + let pool = PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: endpoints.len(), + endpoints: Endpoints::from(endpoints), + cmd_line: "mrf-test".to_string(), + platform: String::new(), + }; + init_local_disks(EndpointServerPools::from(vec![pool])) + .await + .expect("local disks should register"); + + let mut journal = journal_record(1, "replay-bucket", "replay-object", Some([9u8; 16]), 0); + journal.extend(journal_record(3, "replay-bucket", "partial-object", None, 1)); + // Torn tail: a third record truncated mid-way must not block the two + // intact records above. + journal.extend_from_slice(&journal_record(2, "replay-bucket", "metadata-object", None, 0)[..8]); + write_journal_to_disks(&disk_paths, &journal); + + let manager = make_manager(storage); + // Replay directly (not via the process-global channel consumer, which the + // sibling test already claimed in this process under plain `cargo test`). + let replayed = mrf_queue::replay_journal_once(&manager).await; + assert_eq!(replayed, 2, "the two intact records must be replayed"); + + let snapshot = manager.operations_snapshot().await; + assert_eq!(snapshot.queued_by_source.mrf, 2, "replayed intents must be attributed to the MRF source"); + + assert!( + disk_paths + .iter() + .all(|path| !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()), + "the journal file must be removed after a successful replay" + ); + + let snapshot = manager.operations_snapshot().await; + assert_eq!(snapshot.queued_by_priority.urgent, 1, "the decode-failure record must replay as Urgent"); + assert!(snapshot.queued_by_priority.normal >= 1, "the partial-write record must replay as Normal"); +} diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 15a01507c..030c55fc9 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -2478,6 +2478,15 @@ impl FolderScanner { } if let GetSizeFailureAction::HealMetadata { object } = failure_action { + // MRF journal intent: durable High-priority Metadata + // heal across restarts (HS-01); the scanner heal + // request below stays as the immediate path. + rustfs_common::mrf_channel::try_send_mrf_intent( + rustfs_common::mrf_channel::MrfKind::MetadataCorruption, + &item.bucket, + &object, + None, + ); self.send_required_scanner_heal_request( PendingScannerHealKind::Object, item.bucket.clone(), diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index d9e7d1f7e..52e9fc152 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -317,6 +317,7 @@ fn add_source_counts(total: &mut rustfs_heal::HealSourceCounts, next: rustfs_hea total.auto_heal = total.auto_heal.saturating_add(next.auto_heal); total.internal = total.internal.saturating_add(next.internal); total.read_repair = total.read_repair.saturating_add(next.read_repair); + total.mrf = total.mrf.saturating_add(next.mrf); } fn add_operations(total: &mut rustfs_heal::HealOperationsSnapshot, next: rustfs_heal::HealOperationsSnapshot) { @@ -2353,6 +2354,7 @@ mod tests { auto_heal: value, internal: value, read_repair: value, + mrf: value, }; let operations = |value| rustfs_heal::HealOperationsSnapshot { queue_length: value, diff --git a/rustfs/src/storage/rpc/node_service/heal.rs b/rustfs/src/storage/rpc/node_service/heal.rs index f9b990bf3..8f796726c 100644 --- a/rustfs/src/storage/rpc/node_service/heal.rs +++ b/rustfs/src/storage/rpc/node_service/heal.rs @@ -585,6 +585,7 @@ mod tests { let decoded = decode_node_heal_status(&encoded).expect("fixed v1 fixture should decode"); assert_eq!(decoded.info().bitrot_start_cycle, 9); assert_eq!(decoded.operations.queue_length, 2); + assert_eq!(decoded.operations.queued_by_source.mrf, 0); } #[test] From deb0edb7cc6b9d3d8bd6e8f6563fef4039d20b8e Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 12:45:42 +0800 Subject: [PATCH 22/51] chore: adjudicate 26 bare dead_code allows across five crates (#6187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove every bare `#[allow(dead_code)]` in io-core, object-capacity, targets, rio, and scanner. Each allow was stripped first and clippy was then asked which ones the compiler actually missed, so the verdicts rest on the diagnostic rather than on inspection. 23 were inert: they sat on `pub fn`s inside `pub mod`s, where `dead_code` does not apply, or on scanner integration-test helpers that the tests in the same file do call. The remaining 3 are in rio's private `compress_index` module and the code behind them is deleted rather than annotated. `remove_index_headers` is dead and also wrong — after skipping the 4-byte chunk header it matches against `S2_INDEX_TRAILER` where `S2_INDEX_HEADER` sits, so it returns `None` for every well-formed index; rio-v2 carries the correct equivalent that is actually in use. `restore_index_headers` is its unreachable counterpart, likewise duplicated live in rio-v2. `Index::reset` is a private method with no caller. Refs backlog#1823 --- crates/io-core/src/io_profile.rs | 6 --- .../object-capacity/src/capacity_manager.rs | 5 -- crates/rio/src/compress_index.rs | 50 ------------------- .../tests/lifecycle_integration_test.rs | 7 --- crates/targets/src/net.rs | 4 -- 5 files changed, 72 deletions(-) diff --git a/crates/io-core/src/io_profile.rs b/crates/io-core/src/io_profile.rs index 7618eb949..86cd79448 100644 --- a/crates/io-core/src/io_profile.rs +++ b/crates/io-core/src/io_profile.rs @@ -26,7 +26,6 @@ pub enum StorageMedia { } impl StorageMedia { - #[allow(dead_code)] pub fn as_str(&self) -> &'static str { match self { Self::Nvme => "nvme", @@ -60,7 +59,6 @@ pub enum AccessPattern { } impl AccessPattern { - #[allow(dead_code)] pub fn as_str(&self) -> &'static str { match self { Self::Sequential => "sequential", @@ -71,25 +69,21 @@ impl AccessPattern { } /// Check if this is a sequential access pattern. - #[allow(dead_code)] pub fn is_sequential(&self) -> bool { matches!(self, Self::Sequential) } /// Check if this is a random access pattern. - #[allow(dead_code)] pub fn is_random(&self) -> bool { matches!(self, Self::Random) } /// Check if this is a mixed access pattern. - #[allow(dead_code)] pub fn is_mixed(&self) -> bool { matches!(self, Self::Mixed) } /// Check if this pattern is unknown. - #[allow(dead_code)] pub fn is_unknown(&self) -> bool { matches!(self, Self::Unknown) } diff --git a/crates/object-capacity/src/capacity_manager.rs b/crates/object-capacity/src/capacity_manager.rs index 231d3c6a0..2d70f985c 100644 --- a/crates/object-capacity/src/capacity_manager.rs +++ b/crates/object-capacity/src/capacity_manager.rs @@ -427,7 +427,6 @@ pub enum DataSource { /// Write triggered WriteTriggered, /// Fallback value - #[allow(dead_code)] Fallback, } @@ -603,7 +602,6 @@ impl WriteRecord { /// Hybrid strategy configuration #[derive(Debug, Clone)] -#[allow(dead_code)] pub struct HybridStrategyConfig { /// Scheduled update interval pub scheduled_update_interval: Duration, @@ -998,14 +996,12 @@ impl HybridCapacityManager { } /// Get cache age - #[allow(dead_code)] pub async fn get_cache_age(&self) -> Option { let cache = self.cache.read().await; cache.as_ref().map(|c| c.last_update.elapsed()) } /// Get write frequency (writes/minute) - #[allow(dead_code)] pub async fn get_write_frequency(&self) -> usize { let record = &self.write_record; record.recent_write_count(record.monotonic_second()) @@ -1300,7 +1296,6 @@ pub fn get_capacity_manager() -> Arc { /// .update_capacity(CapacityUpdate::exact(1000, 0), DataSource::RealTime) /// .await; /// ``` -#[allow(dead_code)] pub fn create_isolated_manager(config: HybridStrategyConfig) -> Arc { Arc::new(HybridCapacityManager::new(config)) } diff --git a/crates/rio/src/compress_index.rs b/crates/rio/src/compress_index.rs index 75e415e26..c085d70c3 100644 --- a/crates/rio/src/compress_index.rs +++ b/crates/rio/src/compress_index.rs @@ -49,7 +49,6 @@ pub struct IndexInfo { pub uncompressed_offset: i64, } -#[allow(dead_code)] impl Index { pub fn new() -> Self { Self { @@ -60,14 +59,6 @@ impl Index { } } - #[allow(dead_code)] - fn reset(&mut self, max_block: usize) { - self.est_block_uncomp = max_block as i64; - self.total_compressed = -1; - self.total_uncompressed = -1; - self.info.clear(); - } - pub fn len(&self) -> usize { self.info.len() } @@ -511,47 +502,6 @@ fn read_varint(buf: &[u8]) -> io::Result<(i64, usize)> { Err(io::Error::new(io::ErrorKind::UnexpectedEof, "unexpected EOF")) } -// Helper functions for index header manipulation -#[allow(dead_code)] -pub fn remove_index_headers(b: &[u8]) -> Option<&[u8]> { - if b.len() < 4 + S2_INDEX_TRAILER.len() { - return None; - } - - // Skip size - let b = &b[4..]; - - // Check trailer - if !b.starts_with(S2_INDEX_TRAILER) { - return None; - } - - Some(&b[S2_INDEX_TRAILER.len()..]) -} - -#[allow(dead_code)] -pub fn restore_index_headers(in_data: &[u8]) -> Vec { - if in_data.is_empty() { - return Vec::new(); - } - - let mut b = Vec::with_capacity(4 + S2_INDEX_HEADER.len() + in_data.len() + S2_INDEX_TRAILER.len() + 4); - b.extend_from_slice(&[0x50, 0x2A, 0x4D, 0x18]); - b.extend_from_slice(S2_INDEX_HEADER); - b.extend_from_slice(in_data); - - let total_size = (b.len() + 4 + S2_INDEX_TRAILER.len()) as u32; - b.extend_from_slice(&total_size.to_le_bytes()); - b.extend_from_slice(S2_INDEX_TRAILER); - - let chunk_len = b.len() - 4; - b[1] = chunk_len as u8; - b[2] = (chunk_len >> 8) as u8; - b[3] = (chunk_len >> 16) as u8; - - b -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index 7b2130316..71fe961d2 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -206,7 +206,6 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec, Arc, bucket_name: &str) { (**ecstore) .make_bucket(bucket_name, &Default::default()) @@ -251,7 +250,6 @@ async fn modeled_versioned_delete_opts(bucket: &str, object: &str) -> ObjectOpti } /// Test helper: Set bucket lifecycle configuration -#[allow(dead_code)] async fn set_bucket_lifecycle(bucket_name: &str) -> Result<(), Box> { // Create a simple lifecycle configuration XML with 0 days expiry for immediate testing let lifecycle_xml = r#" @@ -274,7 +272,6 @@ async fn set_bucket_lifecycle(bucket_name: &str) -> Result<(), Box Result<(), Box> { // Create lifecycle rule that targets delete-marker cleanup only. // Keep Expiration.Days unset to avoid expiring live transitioned object versions. @@ -297,7 +294,6 @@ async fn set_bucket_lifecycle_deletemarker(bucket_name: &str) -> Result<(), Box< Ok(()) } -#[allow(dead_code)] async fn set_bucket_lifecycle_delmarker_expiration(bucket_name: &str, days: i64) -> Result<(), Box> { let lifecycle_xml = format!( r#" @@ -320,7 +316,6 @@ async fn set_bucket_lifecycle_delmarker_expiration(bucket_name: &str, days: i64) Ok(()) } -#[allow(dead_code)] async fn set_bucket_lifecycle_transition_with_tier( bucket_name: &str, storage_class: &str, @@ -368,7 +363,6 @@ async fn object_exists(ecstore: &Arc, bucket: &str, object: &str) -> bo } /// Test helper: Check if object exists -#[allow(dead_code)] async fn object_is_delete_marker(ecstore: &Arc, bucket: &str, object: &str) -> bool { if let Ok(oi) = (**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await { println!("oi: {oi:?}"); @@ -379,7 +373,6 @@ async fn object_is_delete_marker(ecstore: &Arc, bucket: &str, object: & } } -#[allow(dead_code)] async fn wait_for_object_absence(ecstore: &Arc, bucket: &str, object: &str, timeout: Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; diff --git a/crates/targets/src/net.rs b/crates/targets/src/net.rs index 812e78772..6ed2a6e41 100644 --- a/crates/targets/src/net.rs +++ b/crates/targets/src/net.rs @@ -428,7 +428,6 @@ pub fn parse_url(s: &str) -> Result { Ok(ParsedURL(uu)) } -#[allow(dead_code)] pub fn parse_http_url(s: &str) -> Result { let u = parse_url(s)?; match u.0.scheme() { @@ -437,7 +436,6 @@ pub fn parse_http_url(s: &str) -> Result { } } -#[allow(dead_code)] pub fn is_network_or_host_down(err: &std::io::Error, expect_timeouts: bool) -> bool { if err.kind() == std::io::ErrorKind::TimedOut { return !expect_timeouts; @@ -449,12 +447,10 @@ pub fn is_network_or_host_down(err: &std::io::Error, expect_timeouts: bool) -> b || err_str.contains("use of closed network connection") } -#[allow(dead_code)] pub fn is_conn_reset_err(err: &std::io::Error) -> bool { err.to_string().contains("connection reset by peer") || matches!(err.raw_os_error(), Some(libc::ECONNRESET)) } -#[allow(dead_code)] pub fn is_conn_refused_err(err: &std::io::Error) -> bool { err.to_string().contains("connection refused") || matches!(err.raw_os_error(), Some(libc::ECONNREFUSED)) } From c7a29ec0a7890e4f8217687cf04fc9582ba9b1f4 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 12:51:44 +0800 Subject: [PATCH 23/51] chore(storage): drop dead backpressure and lock-optimizer wrappers (#6195) Neither rustfs/src/storage/backpressure.rs nor rustfs/src/storage/lock_optimizer.rs had a production caller: their only non-self references were the pub mod lines in storage/mod.rs and a cfg(test) module, so the object transfer path never applied this backpressure and never took these lock shortcuts. The six removed tests in concurrent_fix_test.rs duplicated tests that lived inside the deleted files; the shared primitives they shadowed keep their own coverage in rustfs-io-core. --- crates/concurrency/src/backpressure.rs | 5 +- docs/architecture/global-state-inventory.md | 1 - rustfs/src/storage/backpressure.rs | 618 -------------------- rustfs/src/storage/concurrent_fix_test.rs | 82 +-- rustfs/src/storage/lock_optimizer.rs | 458 --------------- rustfs/src/storage/mod.rs | 2 - 6 files changed, 4 insertions(+), 1162 deletions(-) delete mode 100644 rustfs/src/storage/backpressure.rs delete mode 100644 rustfs/src/storage/lock_optimizer.rs diff --git a/crates/concurrency/src/backpressure.rs b/crates/concurrency/src/backpressure.rs index 502d4fb0f..e692d7329 100644 --- a/crates/concurrency/src/backpressure.rs +++ b/crates/concurrency/src/backpressure.rs @@ -14,9 +14,8 @@ //! Shared backpressure policy type. //! -//! The runtime backpressure implementation (byte-watermark pipes and -//! monitors) lives in `rustfs/src/storage/backpressure.rs`; this module only -//! carries the watermark policy type that implementation shares. +//! This module only carries the watermark policy; the admission primitive it +//! projects into lives in `rustfs-io-core`. use rustfs_io_core::BackpressureConfig as CoreBackpressureConfig; diff --git a/docs/architecture/global-state-inventory.md b/docs/architecture/global-state-inventory.md index 8407ad864..b110374dd 100644 --- a/docs/architecture/global-state-inventory.md +++ b/docs/architecture/global-state-inventory.md @@ -105,7 +105,6 @@ inventory. Generic function-local names such as `CACHE`, `LOCK`, `INIT`, and | `USE_STARSHARD_CACHE`, `BUCKET_CACHE_SMALL`, `BUCKET_CACHE_LARGE` | `rustfs/src/storage/ecfs_extend.rs` | Cache or constant / owner-local cache | Bucket validation cache backend selection and cache storage stay private to the ECFS extension owner. | | `GLOBAL_SSE_DEK_PROVIDER`, `SSE_TEST_LOCK` | `rustfs/src/storage/sse.rs` | Owner-local cache / test state | SSE DEK provider cache and test serialization lock stay private to the SSE owner. | | `AUTH_FS` | `rustfs/src/storage/access.rs` | Cache or constant / owner-local cache | Authorization tag-condition lookup keeps its filesystem helper private to the access owner. | -| `LOCK_STATS` | `rustfs/src/storage/lock_optimizer.rs` | Process-global owner-local metrics | Lock optimization statistics stay private behind lock optimizer helper APIs. | | `DEADLOCK_DETECTOR` | `rustfs/src/storage/deadlock_detector.rs` | Process-global owner-local state | Deadlock detector lifecycle state stays private to the storage deadlock detector owner. | | `CONCURRENCY_MANAGER`, `ACTIVE_GET_REQUESTS`, `ACTIVE_PUT_REQUESTS` | `rustfs/src/storage/concurrency/*` | Process-global owner-local scheduler state | Storage concurrency manager and request counters remain inside the storage concurrency owner boundary. | | `GET_OBJECT_BUFFER_THRESHOLD_WARNED`, `GET_READER_STREAM_BUFFER_SIZE_OVERRIDE`, function-local `ENABLED`, `OBJECT_SEEK_SUPPORT_THRESHOLD`, `OBJECT_SEEK_SUPPORT_CONCURRENCY_THRESHOLDS` | `rustfs/src/app/object_usecase.rs` | Cache or constant / owner-local cache | Object GET/seek tuning caches and warning guards stay private to object usecase helpers. | diff --git a/rustfs/src/storage/backpressure.rs b/rustfs/src/storage/backpressure.rs deleted file mode 100644 index 9585ba8a8..000000000 --- a/rustfs/src/storage/backpressure.rs +++ /dev/null @@ -1,618 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Backpressure Management for Object Data Transfer. -//! -//! This module provides backpressure-aware pipes for object data transfer, -//! preventing buffer overflow and memory exhaustion under high concurrency. - -//! # Key Features -//! -//! - Configurable buffer size with high/low watermarks -//! - Backpressure state monitoring and events -//! - Backpressure metrics emitted through the shared metrics pipeline -//! - Graceful handling of slow consumers -//! -//! # Architecture -//! -//! ```text -//! [Disk Reader] --> [BackpressurePipe] --> [HTTP Response] -//! | -//! v -//! [Buffer Monitor] -//! | -//! v -//! [High Watermark?] --> Apply Backpressure -//! ``` - -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::time::{Duration, Instant}; -use tokio::io::{DuplexStream, duplex}; -use tracing::{debug, warn}; - -use metrics::counter; -use rustfs_concurrency::PipeBackpressurePolicy; -use rustfs_io_core::BackpressureConfig as CoreBackpressureConfig; - -/// Object-transfer duplex pipe backpressure policy. -#[derive(Debug, Clone, Copy)] -pub struct ObjectPipeBackpressurePolicy { - /// Buffer size in bytes (default 4MB). - pub buffer_size: usize, - /// High watermark percentage (default 80%). - /// When buffer usage exceeds this, backpressure is applied. - pub high_watermark: u32, - /// Low watermark percentage (default 50%). - /// When buffer usage drops below this after high watermark, backpressure is released. - pub low_watermark: u32, -} - -impl Default for ObjectPipeBackpressurePolicy { - fn default() -> Self { - Self { - buffer_size: rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE, - high_watermark: rustfs_config::DEFAULT_OBJECT_BACKPRESSURE_HIGH_WATERMARK, - low_watermark: rustfs_config::DEFAULT_OBJECT_BACKPRESSURE_LOW_WATERMARK, - } - } -} - -impl ObjectPipeBackpressurePolicy { - /// Load configuration from environment variables. - pub fn from_env() -> Self { - let buffer_size = rustfs_utils::get_env_usize( - rustfs_config::ENV_OBJECT_DUPLEX_BUFFER_SIZE, - rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE, - ); - let high_watermark = rustfs_utils::get_env_u32( - rustfs_config::ENV_OBJECT_BACKPRESSURE_HIGH_WATERMARK, - rustfs_config::DEFAULT_OBJECT_BACKPRESSURE_HIGH_WATERMARK, - ); - let low_watermark = rustfs_utils::get_env_u32( - rustfs_config::ENV_OBJECT_BACKPRESSURE_LOW_WATERMARK, - rustfs_config::DEFAULT_OBJECT_BACKPRESSURE_LOW_WATERMARK, - ); - - Self { - buffer_size, - high_watermark, - low_watermark, - } - } - - /// Calculate high watermark threshold in bytes. - pub fn high_watermark_bytes(&self) -> usize { - (self.buffer_size as u64 * self.high_watermark as u64 / 100) as usize - } - - /// Calculate low watermark threshold in bytes. - pub fn low_watermark_bytes(&self) -> usize { - (self.buffer_size as u64 * self.low_watermark as u64 / 100) as usize - } - - /// Project this object-transfer policy into the shared concurrency facade policy. - pub fn to_concurrency_policy(&self) -> PipeBackpressurePolicy { - PipeBackpressurePolicy { - buffer_size: self.buffer_size, - high_watermark: self.high_watermark, - low_watermark: self.low_watermark, - } - } - - /// Project this object-transfer policy into the reusable io-core admission config. - pub fn to_core_config(&self) -> CoreBackpressureConfig { - self.to_concurrency_policy().to_core_config() - } -} - -/// Backpressure state. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum BackpressureState { - /// Normal operation, buffer usage is below high watermark. - Normal, - /// Buffer usage is above high watermark, backpressure should be applied. - HighWatermark, - /// Backpressure is actively being applied to the producer. - BackpressureApplied, -} - -impl std::fmt::Display for BackpressureState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - BackpressureState::Normal => write!(f, "normal"), - BackpressureState::HighWatermark => write!(f, "high_watermark"), - BackpressureState::BackpressureApplied => write!(f, "backpressure_applied"), - } - } -} - -/// Compact metadata snapshot for object-transfer backpressure pipes. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct BackpressurePipeMeta { - /// Buffer capacity in bytes. - pub buffer_capacity: usize, - /// Current backpressure state. - pub state: BackpressureState, - /// Age of the pipe since creation. - pub age: Duration, -} - -/// Compact metadata snapshot for the lightweight backpressure monitor. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct BackpressureMonitorMeta { - /// Buffer capacity in bytes. - pub buffer_capacity: usize, - /// Current buffer usage percentage. - pub usage_percent: f32, - /// Current backpressure state. - pub state: BackpressureState, -} - -fn calculate_usage_percent(usage: usize, capacity: usize) -> f32 { - if capacity > 0 { - (usage as f32 / capacity as f32) * 100.0 - } else { - 0.0 - } -} - -fn apply_watermark_transition( - in_high_watermark: &AtomicBool, - usage: usize, - high: usize, - low: usize, -) -> (BackpressureState, bool) { - let current = in_high_watermark.load(Ordering::Acquire); - let next_state = if usage >= high { - BackpressureState::HighWatermark - } else if usage <= low { - BackpressureState::Normal - } else if current { - BackpressureState::HighWatermark - } else { - BackpressureState::Normal - }; - let next_is_high = matches!(next_state, BackpressureState::HighWatermark); - let changed = in_high_watermark.swap(next_is_high, Ordering::AcqRel) != next_is_high; - (next_state, changed) -} - -fn saturating_sub_atomic(value: &AtomicUsize, delta: usize) { - value - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_sub(delta))) - .ok(); -} - -/// A backpressure-aware pipe wrapping tokio's duplex. -/// -/// This provides monitoring and events for backpressure conditions -/// while maintaining compatibility with the standard duplex interface. -pub struct BackpressurePipe { - /// Reader end of the duplex pipe. - reader: DuplexStream, - /// Writer end of the duplex pipe. - writer: DuplexStream, - /// Configuration. - config: ObjectPipeBackpressurePolicy, - /// Current buffer usage (approximate, updated on write). - buffer_usage: AtomicUsize, - /// Current backpressure state. - state: AtomicBool, // true = in high watermark state - /// Total bytes written. - total_written: AtomicUsize, - /// Total bytes read. - total_read: AtomicUsize, - /// Cached high watermark threshold in bytes. - high_watermark_bytes: usize, - /// Cached low watermark threshold in bytes. - low_watermark_bytes: usize, - /// Pipe creation timestamp. - created_at: Instant, -} - -impl BackpressurePipe { - /// Create a new backpressure-aware pipe with default configuration. - pub fn new() -> Self { - Self::with_config(ObjectPipeBackpressurePolicy::from_env()) - } - - /// Create a new backpressure-aware pipe with custom configuration. - pub fn with_config(config: ObjectPipeBackpressurePolicy) -> Self { - let policy = config.to_concurrency_policy(); - let (reader, writer) = duplex(policy.buffer_size); - let high_watermark_bytes = policy.high_watermark_bytes(); - let low_watermark_bytes = policy.low_watermark_bytes(); - - debug!( - buffer_size = config.buffer_size, - high_watermark = config.high_watermark, - low_watermark = config.low_watermark, - high_watermark_bytes, - low_watermark_bytes, - "Created backpressure pipe" - ); - - Self { - reader, - writer, - config, - buffer_usage: AtomicUsize::new(0), - state: AtomicBool::new(false), - total_written: AtomicUsize::new(0), - total_read: AtomicUsize::new(0), - high_watermark_bytes, - low_watermark_bytes, - created_at: Instant::now(), - } - } - - /// Take the reader end of the pipe (consumes self). - pub fn into_reader(self) -> DuplexStream { - self.reader - } - - /// Take the writer end of the pipe (consumes self). - pub fn into_writer(self) -> DuplexStream { - self.writer - } - - /// Split into reader and writer (consumes self). - pub fn split(self) -> (DuplexStream, DuplexStream) { - (self.reader, self.writer) - } - - /// Get current backpressure state. - pub fn state(&self) -> BackpressureState { - if self.state.load(Ordering::Acquire) { - BackpressureState::BackpressureApplied - } else { - BackpressureState::Normal - } - } - - /// Get a compact metadata snapshot for the pipe. - pub fn meta(&self) -> BackpressurePipeMeta { - BackpressurePipeMeta { - buffer_capacity: self.config.buffer_size, - state: self.state(), - age: self.age(), - } - } - - /// Get the age of this pipe. - pub fn age(&self) -> Duration { - self.created_at.elapsed() - } - - /// Get current buffer usage. - pub fn usage(&self) -> usize { - self.buffer_usage.load(Ordering::Acquire) - } - - /// Record bytes written (call after successful write). - pub fn record_write(&self, bytes: usize) { - self.total_written.fetch_add(bytes, Ordering::Relaxed); - self.buffer_usage.fetch_add(bytes, Ordering::Release); - self.update_watermark_state(); - } - - /// Record bytes read (call after successful read). - pub fn record_read(&self, bytes: usize) { - self.total_read.fetch_add(bytes, Ordering::Relaxed); - saturating_sub_atomic(&self.buffer_usage, bytes); - self.update_watermark_state(); - } - - /// Update watermark state and emit transition signals. - fn update_watermark_state(&self) { - let usage = self.buffer_usage.load(Ordering::Acquire); - let usage_percent = calculate_usage_percent(usage, self.config.buffer_size) as u32; - let (next_state, changed) = - apply_watermark_transition(&self.state, usage, self.high_watermark_bytes, self.low_watermark_bytes); - - if changed { - match next_state { - BackpressureState::HighWatermark => { - counter!("rustfs_backpressure_events_total", "state" => "high_watermark").increment(1); - - warn!( - buffer_usage = usage, - buffer_capacity = self.config.buffer_size, - usage_percent, - high_watermark = self.config.high_watermark, - "Backpressure: high watermark reached" - ); - } - BackpressureState::Normal => { - counter!("rustfs_backpressure_events_total", "state" => "normal").increment(1); - - debug!( - buffer_usage = usage, - buffer_capacity = self.config.buffer_size, - usage_percent, - low_watermark = self.config.low_watermark, - "Backpressure: returned to normal" - ); - } - BackpressureState::BackpressureApplied => {} - } - } - } - - /// Get total bytes written. - pub fn total_written(&self) -> usize { - self.total_written.load(Ordering::Relaxed) - } - - /// Get total bytes read. - pub fn total_read(&self) -> usize { - self.total_read.load(Ordering::Relaxed) - } - - /// Get buffer capacity. - pub fn capacity(&self) -> usize { - self.config.buffer_size - } -} - -impl Default for BackpressurePipe { - fn default() -> Self { - Self::new() - } -} - -/// A simple wrapper that provides backpressure monitoring for duplex streams. -/// -/// This is a lighter-weight alternative to `BackpressurePipe` that doesn't -/// wrap the streams but provides monitoring capabilities. -pub struct BackpressureMonitor { - /// Configuration. - config: ObjectPipeBackpressurePolicy, - /// Current buffer usage. - buffer_usage: AtomicUsize, - /// In high watermark state. - in_high_watermark: AtomicBool, - /// Cached high watermark threshold in bytes. - high_watermark_bytes: usize, - /// Cached low watermark threshold in bytes. - low_watermark_bytes: usize, -} - -impl BackpressureMonitor { - /// Create a new monitor with default configuration. - pub fn new() -> Self { - Self::with_config(ObjectPipeBackpressurePolicy::from_env()) - } - - /// Create a new monitor with custom configuration. - pub fn with_config(config: ObjectPipeBackpressurePolicy) -> Self { - let policy = config.to_concurrency_policy(); - let high_watermark_bytes = policy.high_watermark_bytes(); - let low_watermark_bytes = policy.low_watermark_bytes(); - Self { - config, - buffer_usage: AtomicUsize::new(0), - in_high_watermark: AtomicBool::new(false), - high_watermark_bytes, - low_watermark_bytes, - } - } - - /// Record bytes added to buffer. - pub fn on_write(&self, bytes: usize) -> BackpressureState { - self.buffer_usage.fetch_add(bytes, Ordering::Release); - self.update_state() - } - - /// Record bytes removed from buffer. - pub fn on_read(&self, bytes: usize) -> BackpressureState { - saturating_sub_atomic(&self.buffer_usage, bytes); - self.update_state() - } - - /// Get current state. - pub fn state(&self) -> BackpressureState { - if self.in_high_watermark.load(Ordering::Acquire) { - BackpressureState::HighWatermark - } else { - BackpressureState::Normal - } - } - - /// Get current buffer usage. - pub fn usage(&self) -> usize { - self.buffer_usage.load(Ordering::Acquire) - } - - /// Get usage percentage. - pub fn usage_percent(&self) -> f32 { - let usage = self.buffer_usage.load(Ordering::Acquire); - calculate_usage_percent(usage, self.config.buffer_size) - } - - /// Get a compact metadata snapshot for the monitor. - pub fn meta(&self) -> BackpressureMonitorMeta { - let usage = self.buffer_usage.load(Ordering::Acquire); - BackpressureMonitorMeta { - buffer_capacity: self.config.buffer_size, - usage_percent: calculate_usage_percent(usage, self.config.buffer_size), - state: self.state(), - } - } - - /// Update state based on current usage. - fn update_state(&self) -> BackpressureState { - let usage = self.buffer_usage.load(Ordering::Acquire); - let usage_percent = calculate_usage_percent(usage, self.config.buffer_size) as u32; - let (next_state, changed) = - apply_watermark_transition(&self.in_high_watermark, usage, self.high_watermark_bytes, self.low_watermark_bytes); - - if matches!(next_state, BackpressureState::HighWatermark) { - if changed { - counter!("rustfs_backpressure_events_total", "state" => "high_watermark").increment(1); - - debug!(usage_percent, "Backpressure: entered high watermark"); - } - BackpressureState::HighWatermark - } else { - if changed { - counter!("rustfs_backpressure_events_total", "state" => "normal").increment(1); - - debug!(usage_percent, "Backpressure: returned to normal"); - } - BackpressureState::Normal - } - } -} - -impl Default for BackpressureMonitor { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -#[allow(unused_imports)] -mod tests { - use super::{BackpressureMonitor, BackpressurePipe, BackpressureState, ObjectPipeBackpressurePolicy}; - - #[test] - fn test_backpressure_config_default() { - let config = ObjectPipeBackpressurePolicy::default(); - assert_eq!(config.buffer_size, 4 * 1024 * 1024); - assert_eq!(config.high_watermark, 80); - assert_eq!(config.low_watermark, 50); - } - - #[test] - fn test_backpressure_config_watermarks() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 1000, - high_watermark: 80, - low_watermark: 50, - }; - assert_eq!(config.high_watermark_bytes(), 800); - assert_eq!(config.low_watermark_bytes(), 500); - } - - #[test] - fn test_backpressure_policy_projects_to_concurrency_and_core_config() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 2000, - high_watermark: 75, - low_watermark: 40, - }; - let concurrency = config.to_concurrency_policy(); - let core = config.to_core_config(); - - assert_eq!(concurrency.buffer_size, config.buffer_size); - assert_eq!(concurrency.high_watermark, config.high_watermark); - assert_eq!(concurrency.low_watermark, config.low_watermark); - assert_eq!(core.high_water_mark, 0.75); - assert_eq!(core.low_water_mark, 0.40); - assert!(core.enabled); - } - - #[test] - fn test_backpressure_pipe_consumes_concurrency_policy_thresholds() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 2000, - high_watermark: 75, - low_watermark: 40, - }; - let concurrency = config.to_concurrency_policy(); - let pipe = BackpressurePipe::with_config(config); - - assert_eq!(pipe.capacity(), concurrency.buffer_size); - assert_eq!(pipe.high_watermark_bytes, concurrency.high_watermark_bytes()); - assert_eq!(pipe.low_watermark_bytes, concurrency.low_watermark_bytes()); - } - - #[test] - fn test_backpressure_monitor_consumes_concurrency_policy_thresholds() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 2000, - high_watermark: 75, - low_watermark: 40, - }; - let concurrency = config.to_concurrency_policy(); - let monitor = BackpressureMonitor::with_config(config); - - assert_eq!(monitor.meta().buffer_capacity, concurrency.buffer_size); - assert_eq!(monitor.high_watermark_bytes, concurrency.high_watermark_bytes()); - assert_eq!(monitor.low_watermark_bytes, concurrency.low_watermark_bytes()); - } - - #[test] - fn test_backpressure_state_display() { - assert_eq!(format!("{}", BackpressureState::Normal), "normal"); - assert_eq!(format!("{}", BackpressureState::HighWatermark), "high_watermark"); - assert_eq!(format!("{}", BackpressureState::BackpressureApplied), "backpressure_applied"); - } - - #[test] - fn test_backpressure_monitor() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 1000, - high_watermark: 80, - low_watermark: 50, - }; - let monitor = BackpressureMonitor::with_config(config); - - // Initially normal - assert_eq!(monitor.state(), BackpressureState::Normal); - assert_eq!(monitor.meta().buffer_capacity, 1000); - assert_eq!(monitor.meta().usage_percent, 0.0); - - // Write to reach high watermark - let state = monitor.on_write(850); - assert_eq!(state, BackpressureState::HighWatermark); - assert_eq!(monitor.meta().usage_percent, 85.0); - - // Read to go below low watermark - let state = monitor.on_read(400); - assert_eq!(state, BackpressureState::Normal); - assert_eq!(monitor.meta().usage_percent, 45.0); - } - - #[tokio::test] - async fn test_backpressure_pipe_creation() { - let pipe = BackpressurePipe::new(); - assert_eq!(pipe.capacity(), 4 * 1024 * 1024); - assert_eq!(pipe.state(), BackpressureState::Normal); - assert_eq!(pipe.meta().buffer_capacity, 4 * 1024 * 1024); - assert!(pipe.meta().age <= pipe.age()); - } - - #[test] - fn test_backpressure_pipe_state_transitions() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 1000, - high_watermark: 80, - low_watermark: 50, - }; - let pipe = BackpressurePipe::with_config(config); - - assert_eq!(pipe.state(), BackpressureState::Normal); - assert_eq!(pipe.meta().state, BackpressureState::Normal); - - pipe.record_write(850); - assert_eq!(pipe.state(), BackpressureState::BackpressureApplied); - assert_eq!(pipe.meta().state, BackpressureState::BackpressureApplied); - - pipe.record_read(400); - assert_eq!(pipe.state(), BackpressureState::Normal); - assert_eq!(pipe.meta().state, BackpressureState::Normal); - } -} diff --git a/rustfs/src/storage/concurrent_fix_test.rs b/rustfs/src/storage/concurrent_fix_test.rs index 2fedeced1..a80865976 100644 --- a/rustfs/src/storage/concurrent_fix_test.rs +++ b/rustfs/src/storage/concurrent_fix_test.rs @@ -14,17 +14,15 @@ //! Integration tests for concurrent request fix. //! -//! These tests verify that the timeout, backpressure, and deadlock detection -//! mechanisms work correctly under high concurrency scenarios. +//! These tests verify that the timeout and deadlock detection mechanisms work +//! correctly under high concurrency scenarios. #[cfg(test)] mod tests { - use crate::storage::backpressure::{BackpressureMonitor, BackpressureState, ObjectPipeBackpressurePolicy}; use crate::storage::concurrency::{IoLoadLevel, IoPriority}; use crate::storage::deadlock_detector::{ DeadlockDetector, LockInfo, LockType, RequestHangDetectionPolicy, RequestResourceTracker, }; - use crate::storage::lock_optimizer::{LockOptimizeConfig, LockOptimizer, LockStats}; use crate::storage::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper, TimedGetObjectResult}; use std::time::Duration; @@ -114,82 +112,6 @@ mod tests { } } - // ============================================ - // Backpressure Tests - // ============================================ - - #[test] - fn test_backpressure_config_defaults() { - let config = ObjectPipeBackpressurePolicy::default(); - assert_eq!(config.buffer_size, 4 * 1024 * 1024); // 4MB - assert_eq!(config.high_watermark, 80); - assert_eq!(config.low_watermark, 50); - } - - #[test] - fn test_backpressure_monitor_state_transitions() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 1000, - high_watermark: 80, - low_watermark: 50, - }; - let monitor = BackpressureMonitor::with_config(config); - - // Initially normal - assert_eq!(monitor.state(), BackpressureState::Normal); - - // Write to reach high watermark - let state = monitor.on_write(850); - assert_eq!(state, BackpressureState::HighWatermark); - - // Read to go below low watermark - let state = monitor.on_read(400); - assert_eq!(state, BackpressureState::Normal); - } - - #[test] - fn test_backpressure_usage_percent() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 1000, - high_watermark: 80, - low_watermark: 50, - }; - let monitor = BackpressureMonitor::with_config(config); - - monitor.on_write(500); - assert!((monitor.usage_percent() - 50.0).abs() < 1.0); - } - - // ============================================ - // Lock Optimizer Tests - // ============================================ - - #[test] - fn test_lock_optimize_config_defaults() { - let config = LockOptimizeConfig::default(); - assert!(config.enabled); - assert_eq!(config.acquire_timeout, Duration::from_secs(5)); - } - - #[test] - fn test_lock_stats_tracking() { - let stats = LockStats::new(); - - stats.record_acquire(); - stats.record_early_release(Duration::from_millis(100)); - stats.record_early_release(Duration::from_millis(200)); - - assert_eq!(stats.locks_acquired.load(std::sync::atomic::Ordering::Relaxed), 1); - assert_eq!(stats.locks_released_early.load(std::sync::atomic::Ordering::Relaxed), 2); - assert_eq!(stats.max_hold_time(), Duration::from_millis(200)); - } - - #[test] - fn test_lock_optimizer_creation() { - let optimizer = LockOptimizer::new(); - assert!(optimizer.is_enabled()); - } - // ============================================ // I/O Priority Tests // ============================================ diff --git a/rustfs/src/storage/lock_optimizer.rs b/rustfs/src/storage/lock_optimizer.rs deleted file mode 100644 index 3636c50c3..000000000 --- a/rustfs/src/storage/lock_optimizer.rs +++ /dev/null @@ -1,458 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Lock Optimization for GetObject Operations. -//! -//! This module provides optimized lock management for read operations, -//! reducing lock contention by releasing locks early (after metadata read) -//! rather than holding them for the entire data transfer duration. -//! -//! # Migration Note -//! -//! For new code, consider using `rustfs_io_core::LockOptimizer` which provides -//! the same core functionality with better separation of concerns. This module -//! remains for backward compatibility and storage-specific configuration. -//! -//! ```ignore -//! // Recommended: Use io-core directly -//! use rustfs_io_core::LockOptimizer; -//! let optimizer = LockOptimizer::with_defaults(); -//! ``` - -// Allow dead_code for public API that may be used by external modules or future features -//! # Key Features -//! -//! - Early lock release after metadata read -//! - Lock hold time monitoring -//! - Configurable optimization (can be disabled for debugging) -//! - Lock contention metrics emitted through the shared metrics pipeline -//! -//! # Architecture -//! -//! ```text -//! Traditional: [Acquire Lock] --> [Read Metadata] --> [Transfer Data] --> [Release Lock] -//! |<------------------ Lock Held ------------------>| -//! -//! Optimized: [Acquire Lock] --> [Read Metadata] --> [Release Lock] --> [Transfer Data] -//! |<- Lock Held ->| -//! ``` - -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{Duration, Instant}; -use tracing::debug; - -use metrics::histogram; - -/// Lock optimization configuration. -#[derive(Debug, Clone)] -pub struct LockOptimizeConfig { - /// Whether to enable lock optimization. - /// When enabled, read locks are released after metadata read. - /// When disabled, locks are held for the entire operation (traditional behavior). - pub enabled: bool, - /// Lock acquisition timeout. - pub acquire_timeout: Duration, -} - -impl Default for LockOptimizeConfig { - fn default() -> Self { - Self { - enabled: rustfs_config::DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE, - acquire_timeout: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT), - } - } -} - -impl LockOptimizeConfig { - /// Load configuration from environment variables. - pub fn from_env() -> Self { - let enabled = rustfs_utils::get_env_bool( - rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, - rustfs_config::DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE, - ); - let acquire_timeout = Duration::from_secs(rustfs_utils::get_env_u64( - rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, - rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT, - )); - - Self { - enabled, - acquire_timeout, - } - } -} - -/// Statistics for lock optimization monitoring. -#[derive(Debug, Default)] -pub struct LockStats { - /// Total locks acquired. - pub locks_acquired: AtomicU64, - /// Total locks released early. - pub locks_released_early: AtomicU64, - /// Total lock hold time in microseconds. - pub total_hold_time_us: AtomicU64, - /// Maximum lock hold time in microseconds. - pub max_hold_time_us: AtomicU64, -} - -impl LockStats { - /// Create new lock statistics. - pub fn new() -> Self { - Self::default() - } - - /// Record a lock acquisition. - pub fn record_acquire(&self) { - self.locks_acquired.fetch_add(1, Ordering::Relaxed); - } - - /// Record an early lock release. - pub fn record_early_release(&self, hold_time: Duration) { - self.locks_released_early.fetch_add(1, Ordering::Relaxed); - self.record_hold_time(hold_time); - } - - /// Record lock hold time. - fn record_hold_time(&self, hold_time: Duration) { - let hold_time_us = hold_time.as_micros() as u64; - self.total_hold_time_us.fetch_add(hold_time_us, Ordering::Relaxed); - - // Update max hold time - let mut current_max = self.max_hold_time_us.load(Ordering::Relaxed); - while hold_time_us > current_max { - match self - .max_hold_time_us - .compare_exchange_weak(current_max, hold_time_us, Ordering::Relaxed, Ordering::Relaxed) - { - Ok(_) => break, - Err(actual) => current_max = actual, - } - } - } - - /// Get average hold time. - pub fn avg_hold_time(&self) -> Duration { - let total = self.total_hold_time_us.load(Ordering::Relaxed); - let count = self.locks_released_early.load(Ordering::Relaxed); - total.checked_div(count).map(Duration::from_micros).unwrap_or(Duration::ZERO) - } - - /// Get maximum hold time. - pub fn max_hold_time(&self) -> Duration { - Duration::from_micros(self.max_hold_time_us.load(Ordering::Relaxed)) - } -} - -/// Global lock statistics. -static LOCK_STATS: std::sync::OnceLock> = std::sync::OnceLock::new(); - -/// Get global lock statistics. -pub fn get_lock_stats() -> Arc { - LOCK_STATS.get_or_init(|| Arc::new(LockStats::new())).clone() -} - -/// An optimized lock guard that supports early release. -/// -/// This wraps the actual lock guard and provides: -/// - Early release capability (before drop) -/// - Hold time tracking -/// - Metrics reporting -pub struct OptimizedLockGuard { - /// The underlying lock guard. - guard: Option, - /// When the lock was acquired. - acquire_time: Instant, - /// Whether the lock has been released. - released: bool, - /// Lock resource name (for logging). - resource: String, - /// Statistics reference. - stats: Arc, -} - -impl OptimizedLockGuard { - /// Create a new optimized lock guard. - pub fn new(guard: G, resource: impl Into) -> Self { - let stats = get_lock_stats(); - stats.record_acquire(); - - Self { - guard: Some(guard), - acquire_time: Instant::now(), - released: false, - resource: resource.into(), - stats, - } - } - - /// Get the lock hold time so far. - pub fn hold_time(&self) -> Duration { - self.acquire_time.elapsed() - } - - /// Check if the lock has been released. - pub fn is_released(&self) -> bool { - self.released - } - - /// Release the lock early (before drop). - /// - /// This is the key optimization: releasing the lock after - /// metadata read rather than waiting for the entire operation. - pub fn early_release(&mut self) { - if self.released { - return; - } - - let hold_time = self.hold_time(); - self.guard.take(); - self.released = true; - - self.stats.record_early_release(hold_time); - - histogram!("rustfs_lock_hold_duration_seconds").record(hold_time.as_secs_f64()); - - debug!( - resource = %self.resource, - hold_time_ms = hold_time.as_millis(), - "Lock released early (optimization active)" - ); - } - - /// Get a reference to the underlying guard. - pub fn as_ref(&self) -> Option<&G> { - if self.released { None } else { self.guard.as_ref() } - } -} - -impl Drop for OptimizedLockGuard { - fn drop(&mut self) { - if !self.released { - let hold_time = self.hold_time(); - self.guard.take(); - self.released = true; - - self.stats.record_early_release(hold_time); - - histogram!("rustfs_lock_hold_duration_seconds").record(hold_time.as_secs_f64()); - - debug!( - resource = %self.resource, - hold_time_ms = hold_time.as_millis(), - "Lock released on drop (normal release)" - ); - } - } -} - -/// A scope guard that releases a lock when it goes out of scope. -/// -/// This is a simpler version of OptimizedLockGuard for cases -/// where we just need RAII semantics without tracking. -pub struct LockScopeGuard { - guard: Option, -} - -impl LockScopeGuard { - /// Create a new scope guard. - pub fn new(guard: G) -> Self { - Self { guard: Some(guard) } - } - - /// Release the lock early. - pub fn release(&mut self) { - self.guard.take(); - } -} - -impl Drop for LockScopeGuard { - fn drop(&mut self) { - self.guard.take(); - } -} - -/// Helper for managing lock optimization in GetObject operations. -/// -/// This provides a clean interface for the common pattern: -/// 1. Acquire lock -/// 2. Read metadata -/// 3. Release lock (if optimization enabled) -/// 4. Transfer data (without lock) -pub struct LockOptimizer { - /// Configuration. - config: LockOptimizeConfig, -} - -impl LockOptimizer { - /// Create a new lock optimizer with default configuration. - pub fn new() -> Self { - Self { - config: LockOptimizeConfig::from_env(), - } - } - - /// Create a new lock optimizer with custom configuration. - pub fn with_config(config: LockOptimizeConfig) -> Self { - Self { config } - } - - /// Check if lock optimization is enabled. - pub fn is_enabled(&self) -> bool { - self.config.enabled - } - - /// Get the lock acquisition timeout. - pub fn acquire_timeout(&self) -> Duration { - self.config.acquire_timeout - } - - /// Wrap a lock guard for optimization. - pub fn wrap_guard(&self, guard: G, resource: impl Into) -> OptimizedLockGuard { - OptimizedLockGuard::new(guard, resource) - } - - /// Execute a metadata read operation with lock optimization. - /// - /// This is the main entry point for optimized lock usage: - /// - If optimization is enabled: lock is released after metadata_fn completes - /// - If optimization is disabled: lock is held until the returned guard is dropped - /// - /// # Arguments - /// - /// * `guard` - The lock guard to optimize - /// * `resource` - Resource name for logging - /// * `metadata_fn` - Function to read metadata while holding lock - /// - /// # Returns - /// - /// A tuple of (metadata result, optional guard to hold for later release) - pub async fn with_optimized_lock( - &self, - guard: G, - resource: impl Into, - metadata_fn: F, - ) -> (T, Option>) - where - F: FnOnce() -> Fut, - Fut: std::future::Future, - { - let resource = resource.into(); - let mut optimized = OptimizedLockGuard::new(guard, &resource); - - // Execute metadata read while holding lock - let result = metadata_fn().await; - - if self.config.enabled { - // Release lock early - optimized.early_release(); - (result, None) - } else { - // Keep lock for caller to release - (result, Some(optimized)) - } - } -} - -impl Default for LockOptimizer { - fn default() -> Self { - Self::new() - } -} - -/// Check if lock optimization is enabled globally. -pub fn is_lock_optimization_enabled() -> bool { - rustfs_utils::get_env_bool( - rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, - rustfs_config::DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE, - ) -} - -#[cfg(test)] -#[allow(unused_imports)] -mod tests { - use super::{LockOptimizeConfig, LockOptimizer, LockStats, OptimizedLockGuard}; - use std::sync::Mutex; - use std::sync::atomic::Ordering; - use std::time::Duration; - - #[test] - fn test_lock_optimize_config_default() { - let config = LockOptimizeConfig::default(); - assert!(config.enabled); - assert_eq!(config.acquire_timeout, Duration::from_secs(5)); - } - - #[test] - fn test_lock_stats() { - let stats = LockStats::new(); - - stats.record_acquire(); - stats.record_early_release(Duration::from_millis(100)); - stats.record_early_release(Duration::from_millis(200)); - - assert_eq!(stats.locks_acquired.load(Ordering::Relaxed), 1); - assert_eq!(stats.locks_released_early.load(Ordering::Relaxed), 2); - assert_eq!(stats.max_hold_time(), Duration::from_millis(200)); - } - - #[test] - fn test_optimized_lock_guard() { - let guard = Mutex::new(42); - let locked = guard.lock().unwrap(); - let mut optimized = OptimizedLockGuard::new(locked, "test-resource"); - - assert!(!optimized.is_released()); - assert!(optimized.hold_time() < Duration::from_secs(1)); - - optimized.early_release(); - assert!(optimized.is_released()); - } - - #[test] - fn test_lock_optimizer() { - let optimizer = LockOptimizer::new(); - assert!(optimizer.is_enabled()); - } - - #[tokio::test] - async fn test_with_optimized_lock_enabled() { - let optimizer = LockOptimizer::new(); - let guard = Mutex::new(42); - let locked = guard.lock().unwrap(); - - let (result, returned_guard) = optimizer.with_optimized_lock(locked, "test-resource", || async { 100 }).await; - - assert_eq!(result, 100); - // With optimization enabled, guard should be None (released early) - assert!(returned_guard.is_none()); - } - - #[tokio::test] - async fn test_with_optimized_lock_disabled() { - let config = LockOptimizeConfig { - enabled: false, - acquire_timeout: Duration::from_secs(5), - }; - let optimizer = LockOptimizer::with_config(config); - let guard = Mutex::new(42); - let locked = guard.lock().unwrap(); - - let (result, returned_guard) = optimizer.with_optimized_lock(locked, "test-resource", || async { 100 }).await; - - assert_eq!(result, 100); - // With optimization disabled, guard should be Some (held for later) - assert!(returned_guard.is_some()); - } -} diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index 58b6a0ce9..e0817da90 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -13,12 +13,10 @@ // limitations under the License. pub mod access; -pub mod backpressure; pub mod concurrency; pub mod deadlock_detector; pub mod ecfs; pub(crate) mod helper; -pub mod lock_optimizer; pub mod options; pub mod request_context; pub mod rpc; From 51497cb5335b5f02bdf16355bb413bad182da757 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 12:57:44 +0800 Subject: [PATCH 24/51] fix(ecstore): give peer REST failures op and bucket context (#6200) --- .../src/cluster/rpc/peer_rest_client.rs | 196 +++++++++++++++--- .../ecstore/src/cluster/rpc/peer_s3_client.rs | 2 + 2 files changed, 166 insertions(+), 32 deletions(-) diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 1426fc91f..3bac43266 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -86,6 +86,25 @@ const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30); const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024; const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024; +/// Error for a peer that reported `success = false` without an `error_info` payload. +/// +/// Same shape as `peer_s3_client::peer_failure_without_details`, over `StorageError` +/// instead of `DiskError`. The message names the operation (and the bucket, where the +/// operation has one) and nothing else, for two reasons: +/// +/// - `finalize_result` classifies failures by message substring, so any text matching +/// `message_has_network_needle` would take an answering peer offline and evict its +/// connection over a plain application-level rejection. +/// - Quorum aggregation (`reduce_errs`) buckets `Io` errors by kind plus rendered +/// message, so a per-peer detail such as the peer address would split one shared +/// failure into single-count buckets and downgrade the dominant error. +fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error { + match bucket { + Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")), + None => Error::other(format!("{op}: peer returned failure without error details")), + } +} + fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result { if !response.success { return Err(Error::other( @@ -696,7 +715,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("local_storage_info", None)); } let data = response.storage_info; @@ -719,7 +738,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("server_info", None)); } let data = response.server_properties; @@ -742,7 +761,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_cpus", None)); } let data = response.cpus; @@ -765,7 +784,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_net_info", None)); } let data = response.net_info; @@ -788,7 +807,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_partitions", None)); } let data = response.partitions; @@ -811,7 +830,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_os_info", None)); } let data = response.os_info; @@ -832,7 +851,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_se_linux_info", None)); } let data = response.sys_services; @@ -857,7 +876,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_sys_config", None)); } let data = response.sys_config; @@ -882,7 +901,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_sys_errors", None)); } let data = response.sys_errors; @@ -907,7 +926,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_mem_info", None)); } let data = response.mem_info; @@ -939,7 +958,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_metrics", None)); } let data = response.realtime_metrics; @@ -964,7 +983,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_live_events", None)); } Ok(PeerLiveEventsBatch { @@ -989,7 +1008,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_proc_info", None)); } let data = response.proc_info; @@ -1016,7 +1035,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("start_profiling", None)); } Ok(()) } @@ -1323,7 +1342,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket))); } Ok(()) } @@ -1346,7 +1365,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("delete_bucket_metadata", Some(bucket))); } Ok(()) } @@ -1369,7 +1388,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("delete_policy", None)); } Ok(()) } @@ -1392,7 +1411,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("load_policy", None)); } Ok(()) } @@ -1417,7 +1436,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("load_policy_mapping", None)); } Ok(()) } @@ -1440,7 +1459,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("delete_user", None)); } Ok(()) } @@ -1463,7 +1482,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("delete_service_account", None)); } Ok(()) } @@ -1487,7 +1506,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("load_user", None)); } Ok(()) } @@ -1510,7 +1529,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("load_service_account", None)); } Ok(()) } @@ -1533,7 +1552,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("load_group", None)); } Ok(()) } @@ -1554,7 +1573,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("reload_site_replication_config", None)); } Ok(()) } @@ -1597,7 +1616,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("signal_service", None)); } validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?; Ok(response) @@ -1667,7 +1686,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("reload_pool_meta", None)); } Ok(()) @@ -1691,7 +1710,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("stop_rebalance", None)); } Ok(()) @@ -1725,7 +1744,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("load_rebalance_meta", None)); } Ok(()) @@ -1753,7 +1772,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("start_decommission", None)); } Ok(()) @@ -1777,7 +1796,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("decommission_cancel", None)); } Ok(()) @@ -1801,7 +1820,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("clear_decommission", None)); } Ok(()) @@ -1947,6 +1966,8 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO mod tests { use super::*; use crate::config::com::STORAGE_CLASS_SUB_SYS; + use crate::disk::error::DiskError; + use crate::disk::error_reduce::reduce_errs; use crate::layout::{disks_layout::DisksLayout, endpoints::SetupType}; use rustfs_config::{ENV_KUBERNETES_SERVICE_HOST, ENV_LOCAL_ENDPOINT_HOST, ENV_STARTUP_TOPOLOGY_WAIT_MODE}; use serde_json::Value; @@ -3098,4 +3119,115 @@ mod tests { && span.get("request_id").and_then(Value::as_str) == Some("req-peer-rest") })); } + + /// Every operation name passed to `peer_failure_without_details` in this file. + const PEER_FAILURE_OPS: &[&str] = &[ + "local_storage_info", + "server_info", + "get_cpus", + "get_net_info", + "get_partitions", + "get_os_info", + "get_se_linux_info", + "get_sys_config", + "get_sys_errors", + "get_mem_info", + "get_metrics", + "get_live_events", + "get_proc_info", + "start_profiling", + "load_bucket_metadata", + "delete_bucket_metadata", + "delete_policy", + "load_policy", + "load_policy_mapping", + "delete_user", + "delete_service_account", + "load_user", + "load_service_account", + "load_group", + "reload_site_replication_config", + "signal_service", + "reload_pool_meta", + "stop_rebalance", + "load_rebalance_meta", + "start_decommission", + "decommission_cancel", + "clear_decommission", + ]; + + #[test] + fn peer_failure_without_details_names_operation_and_bucket() { + for op in PEER_FAILURE_OPS { + let message = peer_failure_without_details(op, None).to_string(); + assert!(message.contains(op), "{op} message must name the operation: {message}"); + } + + for op in ["load_bucket_metadata", "delete_bucket_metadata"] { + let message = peer_failure_without_details(op, Some("ops-bucket")).to_string(); + assert!(message.contains(op), "{op} message must name the operation: {message}"); + assert!(message.contains("ops-bucket"), "{op} message must name the bucket: {message}"); + } + } + + #[test] + fn peer_failure_without_details_keeps_one_reduce_errs_bucket_per_operation() { + // reduce_errs groups Io errors by kind plus rendered message: peers failing the + // same operation must stay a single dominant error instead of one bucket per peer. + let per_peer_errs = (0..4) + .map(|_| Some(DiskError::from(peer_failure_without_details("load_bucket_metadata", Some("shared"))))) + .collect::>(); + let (count, dominant) = reduce_errs(&per_peer_errs, &[]); + assert_eq!(count, 4, "one shared failure must not split into per-peer buckets"); + assert_eq!( + dominant, + Some(DiskError::from(peer_failure_without_details("load_bucket_metadata", Some("shared")))) + ); + + assert_ne!( + peer_failure_without_details("load_bucket_metadata", Some("shared")).to_string(), + peer_failure_without_details("delete_bucket_metadata", Some("shared")).to_string() + ); + assert_ne!( + peer_failure_without_details("load_bucket_metadata", Some("bucket-a")).to_string(), + peer_failure_without_details("load_bucket_metadata", Some("bucket-b")).to_string() + ); + } + + #[test] + fn peer_failure_without_details_never_reads_as_a_network_failure() { + // `finalize_result` marks the peer offline and evicts its connection whenever the + // message matches a network needle. A peer that answered `success = false` is alive, + // so no operation or bucket name may push this text over that classifier. + for op in PEER_FAILURE_OPS { + let err = peer_failure_without_details(op, None); + assert!( + !PeerRestClient::is_network_like_error(&err), + "{op} must not read as a transport failure: {err}" + ); + + let scoped = peer_failure_without_details(op, Some("bucket-name")); + assert!( + !PeerRestClient::is_network_like_error(&scoped), + "{op} must not read as a transport failure: {scoped}" + ); + } + + // The bucket name is caller-supplied. Every needle carries a space, which S3 bucket + // names cannot, and the name is closed by `)` before the literal text resumes, so no + // needle can straddle the boundary either. + for bucket in [ + "timed-out", + "connection-reset", + "transport-error", + "broken-pipe", + "unavailable-logs", + ] { + let err = peer_failure_without_details("load_bucket_metadata", Some(bucket)); + assert!( + !PeerRestClient::is_network_like_error(&err), + "bucket {bucket} must not push the message over the network classifier: {err}" + ); + } + } } diff --git a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs index 02c9f75f1..d44651dce 100644 --- a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs @@ -220,6 +220,8 @@ fn pool_write_quorum(participant_count: usize) -> usize { /// buckets `Error::Io` by kind plus rendered message, so any per-peer detail (address, /// timing) would split one shared failure into single-count buckets and downgrade a real /// dominant error into `ErasureWriteQuorum`. +/// +/// `peer_rest_client` carries the same helper over `StorageError` for the same response shape. fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error { match bucket { Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")), From 9ef059c9083f637e45666ec95ad6263fdaa5723b Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:18:13 +0800 Subject: [PATCH 25/51] ci(package): auto-trigger DEB/RPM packaging on releases and upload to GitHub release assets (#6202) --- .github/workflows/package.yml | 95 +++++++++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 11 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index d44205fa0..b9536d263 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -15,28 +15,35 @@ # Package Workflow - Build DEB/RPM packages # # This workflow builds DEB and RPM packages from pre-built Linux binaries -# and uploads them to Cloudflare R2. +# and uploads them to Cloudflare R2 and the GitHub release. # # Trigger: -# - release published: automatically package when a GitHub release is published -# - workflow_dispatch: manual trigger with optional tag/run_id +# - workflow_run: automatically package after "Build and Release" completes +# for a release tag (the mac/windows/linux binaries are already uploaded +# to the GitHub release before packaging starts) +# - workflow_dispatch: manual fallback (backfill / re-run) with optional tag/run_id # # Flow: -# 1. Find the Build workflow run for the release tag +# 1. Resolve the triggering Build workflow run for the release tag # 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts # 3. Build DEB packages for amd64 and arm64 # 4. Build RPM packages for x86_64 and aarch64 -# 5. Upload all packages to Cloudflare R2 +# 5. Upload all packages to Cloudflare R2 and the GitHub release name: Package DEB/RPM permissions: - contents: read + # contents: write is required to upload packages to the GitHub release + contents: write actions: read on: - release: - types: [ published ] + # Follows the same pattern as docker.yml: run after the release build + # workflow completes, so packaging is triggered only by release tags + # (e.g. 1.0.0-rc.2, 1.0.0-rc.3), never by development builds. + workflow_run: + workflows: [ "Build and Release" ] + types: [ completed ] workflow_dispatch: inputs: tag: @@ -49,13 +56,26 @@ on: type: string concurrency: - group: ${{ github.workflow }}-${{ github.event.release.tag_name || github.event.inputs.tag || github.run_id }} + group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.event.inputs.tag || github.run_id }} cancel-in-progress: true +env: + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }} + jobs: # Resolve which build run to use and extract version info resolve: name: Resolve Build + # Auto-trigger only from successful tag builds of "Build and Release". + # Tag pushes arrive as event == push with head_branch != main (a + # non-main push head_branch is the release tag name). Manual dispatch + # stays available as a fallback for backfills and re-runs. + if: >- + github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch != 'main') runs-on: ubuntu-latest timeout-minutes: 10 outputs: @@ -75,8 +95,8 @@ jobs: set -euo pipefail # Determine tag - if [[ "${{ github.event_name }}" == "release" ]]; then - TAG="${{ github.event.release.tag_name }}" + if [[ "${{ github.event_name }}" == "workflow_run" ]]; then + TAG="${HEAD_BRANCH}" elif [[ -n "$INPUT_TAG" ]]; then TAG="$INPUT_TAG" else @@ -93,6 +113,11 @@ jobs: BUILD_RUN_ID="$INPUT_RUN_ID" echo "Using explicit build run ID: $BUILD_RUN_ID" + elif [[ "${{ github.event_name }}" == "workflow_run" ]]; then + # Use the Build and Release run that triggered this workflow + BUILD_RUN_ID="${WORKFLOW_RUN_ID}" + echo "Using triggering workflow run: $BUILD_RUN_ID" + elif [[ -n "$TAG" ]]; then # Find the build run that produced this tag echo "Looking for build run for tag: $TAG" @@ -456,6 +481,54 @@ jobs: echo "✅ Latest packages updated" fi + - name: Upload packages to GitHub Release + if: needs.resolve.outputs.tag != '' + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + + TAG="${{ needs.resolve.outputs.tag }}" + DEB_FILE="${{ steps.deb.outputs.deb_file }}" + RPM_FILE="${{ steps.rpm.outputs.rpm_file }}" + + # Upload the packages, then refresh the release checksums so the new + # assets are covered, matching the binary release flow. + for f in "$DEB_FILE" "$RPM_FILE"; do + if [[ -n "$f" && -f "$f" ]]; then + echo "📤 Uploading $(basename "$f") to GitHub release ${TAG}..." + gh release upload "$TAG" "$f" --clobber + fi + done + + CHECKSUM_DIR="$(mktemp -d)" + gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \ + -D "$CHECKSUM_DIR" --clobber 2>/dev/null || true + + for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do + asset="${spec%%:*}" + checksum_cmd="${spec##*:}" + checksum_file="${CHECKSUM_DIR}/${asset}" + + touch "$checksum_file" + + for f in "$DEB_FILE" "$RPM_FILE"; do + if [[ -n "$f" && -f "$f" ]]; then + base="$(basename "$f")" + # Remove any stale entry, then append the fresh digest + grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true + mv "${checksum_file}.tmp" "$checksum_file" + (cd "$(dirname "$f")" && "$checksum_cmd" -- "$base") >> "$checksum_file" + fi + done + + echo "📤 Updating ${asset} for release ${TAG}..." + gh release upload "$TAG" "$checksum_file" --clobber + done + + echo "✅ GitHub release assets updated" + # Summary summary: name: Summary From 60eb139db9306f6ada4ed5090437a86bd41d5e95 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 13:27:38 +0800 Subject: [PATCH 26/51] refactor: import x-amz-checksum header names from the shared constants (#6193) Co-authored-by: houseme --- crates/ecstore/src/client/api_get_options.rs | 3 +- crates/ecstore/src/client/transition_api.rs | 16 ++++++---- rustfs/src/admin/router.rs | 31 ++++++++++---------- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/crates/ecstore/src/client/api_get_options.rs b/crates/ecstore/src/client/api_get_options.rs index 503b44f3a..3438453b3 100644 --- a/crates/ecstore/src/client/api_get_options.rs +++ b/crates/ecstore/src/client/api_get_options.rs @@ -20,6 +20,7 @@ #![allow(clippy::all)] use http::{HeaderMap, HeaderName, HeaderValue}; +use rustfs_utils::http::headers::AMZ_CHECKSUM_MODE; use std::collections::HashMap; use time::OffsetDateTime; use tracing::warn; @@ -76,7 +77,7 @@ impl GetObjectOptions { } } if self.checksum { - headers.insert(HeaderName::from_static("x-amz-checksum-mode"), HeaderValue::from_static("ENABLED")); + headers.insert(HeaderName::from_static(AMZ_CHECKSUM_MODE), HeaderValue::from_static("ENABLED")); } headers } diff --git a/crates/ecstore/src/client/transition_api.rs b/crates/ecstore/src/client/transition_api.rs index 6ad802418..be8e0a510 100644 --- a/crates/ecstore/src/client/transition_api.rs +++ b/crates/ecstore/src/client/transition_api.rs @@ -54,6 +54,10 @@ use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE; use rustfs_rio::HashReader; use rustfs_utils::HashAlgorithm; use rustfs_utils::{ + http::headers::{ + AMZ_CHECKSUM_CRC32, AMZ_CHECKSUM_CRC32C, AMZ_CHECKSUM_CRC64NVME, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_SHA1, + AMZ_CHECKSUM_SHA256, + }, net::get_endpoint_url, retry::{DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer}, }; @@ -1383,12 +1387,12 @@ pub(crate) fn to_object_info_for_provider( }; // Extract checksums - let checksum_crc32 = get_header("x-amz-checksum-crc32"); - let checksum_crc32c = get_header("x-amz-checksum-crc32c"); - let checksum_sha1 = get_header("x-amz-checksum-sha1"); - let checksum_sha256 = get_header("x-amz-checksum-sha256"); - let checksum_crc64nvme = get_header("x-amz-checksum-crc64nvme"); - let checksum_mode = get_header("x-amz-checksum-mode"); + let checksum_crc32 = get_header(AMZ_CHECKSUM_CRC32); + let checksum_crc32c = get_header(AMZ_CHECKSUM_CRC32C); + let checksum_sha1 = get_header(AMZ_CHECKSUM_SHA1); + let checksum_sha256 = get_header(AMZ_CHECKSUM_SHA256); + let checksum_crc64nvme = get_header(AMZ_CHECKSUM_CRC64NVME); + let checksum_mode = get_header(AMZ_CHECKSUM_MODE); // Build and return the ObjectInfo struct Ok(ObjectInfo { diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 9eb893792..717de0bee 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -67,6 +67,9 @@ use rustfs_policy::policy::action::{Action, S3Action}; use rustfs_s3_types::EventName; use rustfs_signer::pre_sign_v4; use rustfs_utils::egress::{OutboundDnsResolver, OutboundPolicy}; +use rustfs_utils::http::headers::{ + AMZ_CHECKSUM_CRC32, AMZ_CHECKSUM_CRC32C, AMZ_CHECKSUM_CRC64NVME, AMZ_CHECKSUM_SHA1, AMZ_CHECKSUM_SHA256, AMZ_CHECKSUM_TYPE, +}; use rustfs_utils::http::{ SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, get_source_scheme, insert_header, @@ -1031,28 +1034,24 @@ fn build_get_object_response_headers(output: &GetObjectOutput, base_headers: &He )?; } if let Some(checksum_crc32) = &output.checksum_crc32 { - insert_string_header(&mut headers, HeaderName::from_static("x-amz-checksum-crc32"), checksum_crc32.clone())?; + insert_string_header(&mut headers, HeaderName::from_static(AMZ_CHECKSUM_CRC32), checksum_crc32.clone())?; } if let Some(checksum_crc32c) = &output.checksum_crc32c { - insert_string_header(&mut headers, HeaderName::from_static("x-amz-checksum-crc32c"), checksum_crc32c.clone())?; + insert_string_header(&mut headers, HeaderName::from_static(AMZ_CHECKSUM_CRC32C), checksum_crc32c.clone())?; } if let Some(checksum_crc64nvme) = &output.checksum_crc64nvme { - insert_string_header( - &mut headers, - HeaderName::from_static("x-amz-checksum-crc64nvme"), - checksum_crc64nvme.clone(), - )?; + insert_string_header(&mut headers, HeaderName::from_static(AMZ_CHECKSUM_CRC64NVME), checksum_crc64nvme.clone())?; } if let Some(checksum_sha1) = &output.checksum_sha1 { - insert_string_header(&mut headers, HeaderName::from_static("x-amz-checksum-sha1"), checksum_sha1.clone())?; + insert_string_header(&mut headers, HeaderName::from_static(AMZ_CHECKSUM_SHA1), checksum_sha1.clone())?; } if let Some(checksum_sha256) = &output.checksum_sha256 { - insert_string_header(&mut headers, HeaderName::from_static("x-amz-checksum-sha256"), checksum_sha256.clone())?; + insert_string_header(&mut headers, HeaderName::from_static(AMZ_CHECKSUM_SHA256), checksum_sha256.clone())?; } if let Some(checksum_type) = &output.checksum_type { insert_string_header( &mut headers, - HeaderName::from_static("x-amz-checksum-type"), + HeaderName::from_static(AMZ_CHECKSUM_TYPE), checksum_type.as_str().to_string(), )?; } @@ -1114,12 +1113,12 @@ fn clear_object_lambda_variant_headers(headers: &mut HeaderMap) { http::header::ETAG, http::header::LAST_MODIFIED, http::header::EXPIRES, - HeaderName::from_static("x-amz-checksum-crc32"), - HeaderName::from_static("x-amz-checksum-crc32c"), - HeaderName::from_static("x-amz-checksum-crc64nvme"), - HeaderName::from_static("x-amz-checksum-sha1"), - HeaderName::from_static("x-amz-checksum-sha256"), - HeaderName::from_static("x-amz-checksum-type"), + HeaderName::from_static(AMZ_CHECKSUM_CRC32), + HeaderName::from_static(AMZ_CHECKSUM_CRC32C), + HeaderName::from_static(AMZ_CHECKSUM_CRC64NVME), + HeaderName::from_static(AMZ_CHECKSUM_SHA1), + HeaderName::from_static(AMZ_CHECKSUM_SHA256), + HeaderName::from_static(AMZ_CHECKSUM_TYPE), HeaderName::from_static("x-amz-tagging-count"), HeaderName::from_static("x-amz-request-route"), HeaderName::from_static("x-amz-request-token"), From 355c8d2e2292596bbbb034fca1fbcec998dc935a Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 13:40:10 +0800 Subject: [PATCH 27/51] fix(admin): classify missing kms config by error variant (#6196) --- .../ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs | 2 +- crates/ecstore/src/error/mod.rs | 8 ++++++++ rustfs/src/admin/handlers/kms_dynamic.rs | 8 ++++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index ec5aaf74e..326b798ee 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -1783,7 +1783,7 @@ impl TransitionState { .await; } global_metrics().record_scanner_transition_failed(1); - if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") { + if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) { error!( event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED, component = LOG_COMPONENT_ECSTORE, diff --git a/crates/ecstore/src/error/mod.rs b/crates/ecstore/src/error/mod.rs index 76dcc7a0c..f8215d908 100644 --- a/crates/ecstore/src/error/mod.rs +++ b/crates/ecstore/src/error/mod.rs @@ -1116,6 +1116,14 @@ mod tests { assert!(encoder_source.is::()); } + // The lifecycle transition worker relies on this arm alone to suppress the + // closed-connection noise (`bucket_lifecycle_ops.rs`); dropping it here would + // silently turn shutdown races back into `error!` log spam. + #[test] + fn is_network_or_host_down_covers_closed_network_connection() { + assert!(is_network_or_host_down("transition failed: use of closed network connection", false)); + } + // Regression for #952 (ECA-11): an all-`DiskNotFound` slice (every drive in // every set unreachable) must NOT be classified as "all not found", // otherwise ListObjects silently returns an empty listing and masks a full diff --git a/rustfs/src/admin/handlers/kms_dynamic.rs b/rustfs/src/admin/handlers/kms_dynamic.rs index 887ec7fad..d7ee7a0f7 100644 --- a/rustfs/src/admin/handlers/kms_dynamic.rs +++ b/rustfs/src/admin/handlers/kms_dynamic.rs @@ -22,6 +22,7 @@ use crate::admin::runtime_sources::{ current_object_store_handle_for_context, current_or_init_kms_runtime_service_manager, }; use crate::admin::storage_api::config::{read_admin_config, save_admin_config}; +use crate::admin::storage_api::error::StorageError; use crate::auth::{check_key_valid, get_session_token}; use crate::server::{ADMIN_PREFIX, RemoteAddr}; use hyper::{Method, StatusCode}; @@ -278,8 +279,11 @@ pub async fn load_kms_config() -> Option { } }, Err(e) => { - // Config not found is normal on first run - if e.to_string().contains("ConfigNotFound") || e.to_string().contains("not found") { + // Config not found is normal on first run: `read_config` maps a missing or + // empty config object to `ConfigNotFound`, so that variant is the only + // "absent" signal reaching here. Every other not-found variant (disk, + // volume, bucket) means degraded storage and must stay a warning. + if matches!(e, StorageError::ConfigNotFound) { info!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_KMS, From 0f30a75fdb808baa6fe8fa5ac2a5f91bc42d0ab9 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 18 Aug 2026 14:49:19 +0800 Subject: [PATCH 28/51] fix(ecstore): resume remote shard reads once (#6091) * fix(ecstore): preserve CopyObject producer errors * fix(ecstore): resume remote shard reads once * fix(app): resume preserved relocation I/O errors * fix(ecstore): reserve remote read recovery budget --- .../cluster/rpc/internode_data_transport.rs | 25 + crates/ecstore/src/cluster/rpc/remote_disk.rs | 1187 ++++++++++++++++- crates/rio/src/http_reader.rs | 62 +- 3 files changed, 1258 insertions(+), 16 deletions(-) diff --git a/crates/ecstore/src/cluster/rpc/internode_data_transport.rs b/crates/ecstore/src/cluster/rpc/internode_data_transport.rs index 9c33d191a..6f46fead3 100644 --- a/crates/ecstore/src/cluster/rpc/internode_data_transport.rs +++ b/crates/ecstore/src/cluster/rpc/internode_data_transport.rs @@ -233,11 +233,17 @@ pub struct NsScannerCapabilityRequest { #[async_trait] pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug { async fn open_read(&self, request: ReadStreamRequest) -> Result; + async fn open_read_fresh(&self, request: ReadStreamRequest) -> Result { + self.open_read(request).await + } /// Opens an owned-chunk stream when this transport can retain receive-buffer /// ownership. `None` preserves the established `open_read` fallback. async fn open_read_chunks(&self, _request: ReadStreamRequest) -> Result> { Ok(None) } + async fn open_read_chunks_fresh(&self, request: ReadStreamRequest) -> Result> { + self.open_read_chunks(request).await + } async fn open_write(&self, request: WriteStreamRequest) -> Result; async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result; async fn open_ns_scanner(&self, _request: NsScannerStreamRequest) -> Result { @@ -269,6 +275,15 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport { )) } + async fn open_read_fresh(&self, request: ReadStreamRequest) -> Result { + let url = build_read_file_stream_url(&request); + let mut headers = json_headers(); + build_auth_headers(&url, &Method::GET, &mut headers)?; + Ok(Box::new( + HttpReader::new_fresh_connection_with_stall_timeout(url, Method::GET, headers, None, request.stall_timeout).await?, + )) + } + async fn open_read_chunks(&self, request: ReadStreamRequest) -> Result> { let url = build_read_file_stream_url(&request); let mut headers = json_headers(); @@ -278,6 +293,16 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport { ))) } + async fn open_read_chunks_fresh(&self, request: ReadStreamRequest) -> Result> { + let url = build_read_file_stream_url(&request); + let mut headers = json_headers(); + build_auth_headers(&url, &Method::GET, &mut headers)?; + Ok(Some(Box::new( + HttpChunkReader::new_fresh_connection_with_stall_timeout(url, Method::GET, headers, None, request.stall_timeout) + .await?, + ))) + } + async fn open_write(&self, request: WriteStreamRequest) -> Result { let server_epoch = self.put_file_auth_capability(&request.endpoint).await?; let nonce = server_epoch.map(|_| Uuid::new_v4()); diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index 13e45fbd3..3ae2c8b01 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -55,18 +55,22 @@ use rustfs_protos::proto_gen::node_service::{ }; use serde::{Serialize, de::DeserializeOwned}; use std::{ + future::Future, io::Cursor, path::PathBuf, + pin::Pin, sync::{ Arc, atomic::{AtomicU32, Ordering}, }, + task::{Context, Poll}, time::Duration, }; use tokio::time; use tokio::{ - io::{self, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, + io::{self, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf}, net::TcpStream, + task::{JoinError, JoinHandle}, time::timeout, }; use tokio_util::sync::CancellationToken; @@ -84,6 +88,7 @@ const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2; const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20); const REMOTE_DISK_OPEN_READ_MAX_ATTEMPTS: usize = 2; const REMOTE_DISK_OPEN_READ_RETRY_BACKOFF: Duration = Duration::from_millis(20); +const REMOTE_READ_TIMEOUT_PARTS: u32 = 3; const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5); /// Base backoff for idempotent read-only RPC retries (grpc-optimization P3-3); doubles per attempt. const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50); @@ -214,6 +219,415 @@ where } } +fn is_retryable_remote_body_error(error: &io::Error) -> bool { + if error + .get_ref() + .and_then(|source| source.downcast_ref::()) + .is_some() + { + return true; + } + + matches!( + error.kind(), + io::ErrorKind::ConnectionReset + | io::ErrorKind::BrokenPipe + | io::ErrorKind::ConnectionAborted + | io::ErrorKind::UnexpectedEof + ) +} + +fn resumed_read_request(request: &ReadStreamRequest, emitted: usize) -> io::Result { + let offset = request + .offset + .checked_add(emitted) + .ok_or_else(|| io::Error::other("remote read resume offset overflow"))?; + let length = if request.length == 0 { + 0 + } else { + request + .length + .checked_sub(emitted) + .ok_or_else(|| io::Error::other("remote read resume offset exceeds requested length"))? + }; + Ok(ReadStreamRequest { + offset, + length, + ..request.clone() + }) +} + +#[derive(Clone, Copy)] +struct RemoteReadTimeouts { + body_stall: Option, + initial_read: Option, + recovery: Option, +} + +fn remote_read_timeouts(read_timeout: Duration) -> RemoteReadTimeouts { + let Some(recovery) = read_timeout + .checked_div(REMOTE_READ_TIMEOUT_PARTS) + .filter(|timeout| !timeout.is_zero()) + else { + return RemoteReadTimeouts { + body_stall: None, + initial_read: None, + recovery: None, + }; + }; + RemoteReadTimeouts { + body_stall: Some(recovery), + initial_read: Some(read_timeout.saturating_sub(recovery)), + recovery: Some(recovery), + } +} + +async fn with_remote_read_recovery_timeout(recovery_timeout: Option, future: F) -> Result +where + F: Future>, +{ + match recovery_timeout { + Some(recovery_timeout) => match time::timeout(recovery_timeout, future).await { + Ok(result) => result, + Err(_) => Err(DiskError::Timeout), + }, + None => future.await, + } +} + +struct AbortOnDropTask(JoinHandle); + +impl AbortOnDropTask { + fn new(handle: JoinHandle) -> Self { + Self(handle) + } +} + +impl Future for AbortOnDropTask +where + T: Send + 'static, +{ + type Output = std::result::Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Pin::new(&mut self.get_mut().0).poll(cx) + } +} + +impl Drop for AbortOnDropTask { + fn drop(&mut self) { + self.0.abort(); + } +} + +fn retry_cutoff_elapsed( + initial_read_timeout: &mut Option, + cutoff: &mut Option>>, + cx: &mut Context<'_>, +) -> bool { + if cutoff.is_none() + && let Some(timeout) = initial_read_timeout.take() + { + *cutoff = Some(Box::pin(time::sleep(timeout))); + } + cutoff.as_mut().is_some_and(|cutoff| cutoff.as_mut().poll(cx).is_ready()) +} + +type ReadResumeFuture = AbortOnDropTask>; + +struct RetryingRemoteReader { + reader: Option, + transport: Arc, + request: ReadStreamRequest, + emitted: usize, + retried: bool, + initial_read_timeout: Option, + retry_cutoff: Option>>, + recovery_timeout: Option, + resume: Option, +} + +impl RetryingRemoteReader { + fn new_with_timeouts( + reader: FileReader, + transport: Arc, + request: ReadStreamRequest, + initial_read_timeout: Option, + recovery_timeout: Option, + ) -> Self { + Self { + reader: Some(reader), + transport, + request, + emitted: 0, + retried: false, + initial_read_timeout, + retry_cutoff: None, + recovery_timeout, + resume: None, + } + } + + fn start_resume(&mut self) -> io::Result<()> { + if self.request.length != 0 && self.emitted >= self.request.length { + self.reader = None; + return Ok(()); + } + let request = resumed_read_request(&self.request, self.emitted)?; + let recovery_timeout = self.recovery_timeout; + let transport = Arc::clone(&self.transport); + self.resume = Some(AbortOnDropTask::new(tokio::spawn(async move { + with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_fresh(request)).await + }))); + Ok(()) + } + + fn retry_cutoff_elapsed(&mut self, cx: &mut Context<'_>) -> bool { + retry_cutoff_elapsed(&mut self.initial_read_timeout, &mut self.retry_cutoff, cx) + } +} + +impl AsyncRead for RetryingRemoteReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + loop { + // After the absolute cutoff, let initial progress win over a stale fresh-open. + let resume_pending = if let Some(resume) = self.resume.as_mut() { + match Pin::new(resume).poll(cx) { + Poll::Pending => true, + Poll::Ready(Ok(Ok(reader))) => { + self.resume = None; + self.reader = Some(reader); + false + } + Poll::Ready(Ok(Err(error))) => { + self.resume = None; + if self.reader.is_none() { + return Poll::Ready(Err(io::Error::other(error))); + } + continue; + } + Poll::Ready(Err(error)) => { + self.resume = None; + if self.reader.is_none() { + return Poll::Ready(Err(io::Error::other(error))); + } + continue; + } + } + } else { + false + }; + + if !self.retried && self.retry_cutoff_elapsed(cx) { + self.retried = true; + if let Err(resume_error) = self.start_resume() { + return Poll::Ready(Err(resume_error)); + } + continue; + } + + let Some(reader) = self.reader.as_mut() else { + if resume_pending { + return Poll::Pending; + } + return Poll::Ready(Ok(())); + }; + let before = buf.filled().len(); + match Pin::new(reader).poll_read(cx, buf) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Ok(())) => { + let produced = buf.filled().len() - before; + self.emitted = match self.emitted.checked_add(produced) { + Some(emitted) => emitted, + None => return Poll::Ready(Err(io::Error::other("remote read emitted byte count overflow"))), + }; + if resume_pending { + if produced == 0 && (self.request.length == 0 || self.emitted >= self.request.length) { + self.resume = None; + } else if produced == 0 { + self.reader = None; + continue; + } else { + self.resume = None; + } + } + return Poll::Ready(Ok(())); + } + Poll::Ready(Err(error)) if !self.retried && is_retryable_remote_body_error(&error) => { + self.retried = true; + self.reader = None; + if let Err(resume_error) = self.start_resume() { + return Poll::Ready(Err(resume_error)); + } + continue; + } + Poll::Ready(Err(error)) if resume_pending && is_retryable_remote_body_error(&error) => { + self.reader = None; + continue; + } + Poll::Ready(Err(error)) => return Poll::Ready(Err(error)), + } + } + } +} + +type ChunkResumeFuture = AbortOnDropTask>>; + +struct RetryingRemoteChunkReader { + reader: Option, + transport: Arc, + request: ReadStreamRequest, + emitted: usize, + retried: bool, + initial_read_timeout: Option, + retry_cutoff: Option>>, + recovery_timeout: Option, + resume: Option, +} + +impl RetryingRemoteChunkReader { + fn new_with_timeouts( + reader: rustfs_rio::ChunkReaderBox, + transport: Arc, + request: ReadStreamRequest, + initial_read_timeout: Option, + recovery_timeout: Option, + ) -> Self { + Self { + reader: Some(reader), + transport, + request, + emitted: 0, + retried: false, + initial_read_timeout, + retry_cutoff: None, + recovery_timeout, + resume: None, + } + } + + fn start_resume(&mut self) -> io::Result<()> { + if self.request.length != 0 && self.emitted >= self.request.length { + self.reader = None; + return Ok(()); + } + let request = resumed_read_request(&self.request, self.emitted)?; + let recovery_timeout = self.recovery_timeout; + let transport = Arc::clone(&self.transport); + self.resume = Some(AbortOnDropTask::new(tokio::spawn(async move { + with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_chunks_fresh(request)).await + }))); + Ok(()) + } + + fn retry_cutoff_elapsed(&mut self, cx: &mut Context<'_>) -> bool { + retry_cutoff_elapsed(&mut self.initial_read_timeout, &mut self.retry_cutoff, cx) + } +} + +impl AsyncRead for RetryingRemoteChunkReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + match rustfs_rio::ChunkReader::poll_read_chunk(self.as_mut(), cx, buf.remaining()) { + Poll::Ready(Ok(Some(chunk))) => { + buf.put_slice(&chunk); + Poll::Ready(Ok(())) + } + Poll::Ready(Ok(None)) => Poll::Ready(Ok(())), + Poll::Ready(Err(error)) => Poll::Ready(Err(error)), + Poll::Pending => Poll::Pending, + } + } +} + +impl rustfs_rio::ChunkReader for RetryingRemoteChunkReader { + fn poll_read_chunk(mut self: Pin<&mut Self>, cx: &mut Context<'_>, max: usize) -> Poll>> { + loop { + let resume_pending = if let Some(resume) = self.resume.as_mut() { + match Pin::new(resume).poll(cx) { + Poll::Pending => true, + Poll::Ready(Ok(Ok(Some(reader)))) => { + self.resume = None; + self.reader = Some(reader); + false + } + Poll::Ready(Ok(Ok(None))) => { + self.resume = None; + if self.reader.is_none() { + return Poll::Ready(Err(io::Error::other("remote resume transport did not provide a chunk reader"))); + } + continue; + } + Poll::Ready(Ok(Err(error))) => { + self.resume = None; + if self.reader.is_none() { + return Poll::Ready(Err(io::Error::other(error))); + } + continue; + } + Poll::Ready(Err(error)) => { + self.resume = None; + if self.reader.is_none() { + return Poll::Ready(Err(io::Error::other(error))); + } + continue; + } + } + } else { + false + }; + + if !self.retried && self.retry_cutoff_elapsed(cx) { + self.retried = true; + if let Err(resume_error) = self.start_resume() { + return Poll::Ready(Err(resume_error)); + } + continue; + } + + let Some(reader) = self.reader.as_mut() else { + if resume_pending { + return Poll::Pending; + } + return Poll::Ready(Ok(None)); + }; + match rustfs_rio::ChunkReader::poll_read_chunk(Pin::new(reader.as_mut()), cx, max) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Ok(Some(chunk))) => { + self.emitted = match self.emitted.checked_add(chunk.len()) { + Some(emitted) => emitted, + None => return Poll::Ready(Err(io::Error::other("remote read emitted byte count overflow"))), + }; + if resume_pending { + self.resume = None; + } + return Poll::Ready(Ok(Some(chunk))); + } + Poll::Ready(Ok(None)) if resume_pending => { + self.reader = None; + continue; + } + Poll::Ready(Ok(None)) => return Poll::Ready(Ok(None)), + Poll::Ready(Err(error)) if !self.retried && is_retryable_remote_body_error(&error) => { + self.retried = true; + self.reader = None; + if let Err(resume_error) = self.start_resume() { + return Poll::Ready(Err(resume_error)); + } + continue; + } + Poll::Ready(Err(error)) if resume_pending && is_retryable_remote_body_error(&error) => { + self.reader = None; + continue; + } + Poll::Ready(Err(error)) => return Poll::Ready(Err(error)), + } + } + } +} + #[derive(Debug)] pub struct RemoteDisk { pub id: Mutex>, @@ -2483,17 +2897,24 @@ impl DiskAPI for RemoteDisk { return Err(DiskError::FaultyDisk); } let disk = self.disk_ref().await; - let stall_timeout = get_object_disk_read_timeout(); - self.open_read_with_retry(ReadStreamRequest { + let timeouts = remote_read_timeouts(get_object_disk_read_timeout()); + let request = ReadStreamRequest { endpoint: self.endpoint.grid_host(), disk, volume: volume.to_string(), path: path.to_string(), offset, length, - stall_timeout: (!stall_timeout.is_zero()).then_some(stall_timeout), - }) - .await + stall_timeout: timeouts.body_stall, + }; + let reader = self.open_read_with_retry(request.clone()).await?; + Ok(Box::new(RetryingRemoteReader::new_with_timeouts( + reader, + Arc::clone(&self.data_transport), + request, + timeouts.initial_read, + timeouts.recovery, + ))) } async fn read_file_stream_chunks( @@ -2507,17 +2928,26 @@ impl DiskAPI for RemoteDisk { return Err(DiskError::FaultyDisk); } let disk = self.disk_ref().await; - let stall_timeout = get_object_disk_read_timeout(); - self.open_read_chunks_with_retry(ReadStreamRequest { + let timeouts = remote_read_timeouts(get_object_disk_read_timeout()); + let request = ReadStreamRequest { endpoint: self.endpoint.grid_host(), disk, volume: volume.to_string(), path: path.to_string(), offset, length, - stall_timeout: (!stall_timeout.is_zero()).then_some(stall_timeout), - }) - .await + stall_timeout: timeouts.body_stall, + }; + let reader = self.open_read_chunks_with_retry(request.clone()).await?; + Ok(reader.map(|reader| { + Box::new(RetryingRemoteChunkReader::new_with_timeouts( + reader, + Arc::clone(&self.data_transport), + request, + timeouts.initial_read, + timeouts.recovery, + )) as rustfs_rio::ChunkReaderBox + })) } /// Buffered read for remote disks. @@ -3115,12 +3545,14 @@ impl DiskAPI for RemoteDisk { mod tests { use super::*; use crate::cluster::rpc::internode_data_transport::{InternodeDataTransportCapabilities, TcpHttpInternodeDataTransport}; + use crate::erasure::coding::{BitrotReader, Erasure, decode::ParallelReader}; + use crate::io_support::bitrot::ShardReader; use crate::runtime::sources as runtime_sources; use serde_json::Value; use serial_test::serial; use std::io::{self as std_io, Write}; use std::pin::Pin; - use std::sync::{Arc, Mutex, Mutex as StdMutex, Once}; + use std::sync::{Arc, Mutex, Mutex as StdMutex, Once, atomic::AtomicUsize}; use std::task::{Context, Poll}; use tokio::io::{ReadBuf, duplex}; use tokio::net::TcpListener; @@ -4138,6 +4570,735 @@ mod tests { } } + #[derive(Debug, Clone)] + enum ResumeReadStep { + PartialThenReset(Vec), + Data(Vec), + } + + #[derive(Debug, Default)] + struct ResumeTransport { + read_steps: Mutex>, + chunk_steps: Mutex>, + read_requests: Mutex>, + chunk_requests: Mutex>, + fresh_read_requests: Mutex>, + fresh_chunk_requests: Mutex>, + } + + impl ResumeTransport { + fn with_read_steps(read_steps: Vec) -> Self { + Self { + read_steps: Mutex::new(read_steps), + ..Self::default() + } + } + + fn with_chunk_steps(chunk_steps: Vec) -> Self { + Self { + chunk_steps: Mutex::new(chunk_steps), + ..Self::default() + } + } + } + + #[derive(Debug)] + struct ChunkPartialThenErrorReader { + data: Option, + error: Option, + } + + impl rustfs_rio::ChunkReader for ChunkPartialThenErrorReader { + fn poll_read_chunk(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, max: usize) -> Poll>> { + if let Some(mut data) = self.data.take() { + let take = data.len().min(max); + let chunk = data.split_to(take); + if !data.is_empty() { + self.data = Some(data); + } + return Poll::Ready(Ok(Some(chunk))); + } + if let Some(error) = self.error.take() { + return Poll::Ready(Err(error)); + } + Poll::Ready(Ok(None)) + } + } + + impl AsyncRead for ChunkPartialThenErrorReader { + fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll> { + Poll::Ready(Err(io::Error::other("chunk reader must use chunk handoff"))) + } + } + + #[derive(Debug)] + struct BodyStallTestReader { + data: Option, + next_data: Option<(Bytes, Pin>)>, + initial_delay: Option>>, + timeout: Duration, + stall_timer: Option>>, + } + + impl BodyStallTestReader { + fn new(data: Bytes, initial_delay: Duration, timeout: Option) -> Self { + let timeout = timeout.expect("parallel resume test requires a body stall timeout"); + Self { + data: Some(data), + next_data: None, + initial_delay: Some(Box::pin(time::sleep(initial_delay))), + timeout, + stall_timer: None, + } + } + + fn with_next_data( + data: Bytes, + initial_delay: Duration, + next_data: Bytes, + next_delay: Duration, + timeout: Option, + ) -> Self { + let mut reader = Self::new(data, initial_delay, timeout); + reader.next_data = Some((next_data, Box::pin(time::sleep(next_delay)))); + reader + } + + fn poll_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll>> { + if let Some(delay) = self.initial_delay.as_mut() { + if delay.as_mut().poll(cx).is_pending() { + return Poll::Pending; + } + self.initial_delay = None; + } + if let Some(mut data) = self.data.take() { + let chunk = data.split_to(data.len().min(max)); + if !data.is_empty() { + self.data = Some(data); + } + return Poll::Ready(Ok(Some(chunk))); + } + if let Some((_, delay)) = self.next_data.as_mut() + && delay.as_mut().poll(cx).is_pending() + { + return Poll::Pending; + } + if let Some((mut data, _)) = self.next_data.take() { + let chunk = data.split_to(data.len().min(max)); + if !data.is_empty() { + self.next_data = Some((data, Box::pin(time::sleep(Duration::ZERO)))); + } + return Poll::Ready(Ok(Some(chunk))); + } + let timer = self.stall_timer.get_or_insert_with(|| Box::pin(time::sleep(self.timeout))); + match timer.as_mut().poll(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(()) => Poll::Ready(Err(io::Error::new( + std_io::ErrorKind::TimedOut, + rustfs_rio::BodyStalled { timeout: self.timeout }, + ))), + } + } + } + + impl AsyncRead for BodyStallTestReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + match self.poll_chunk(cx, buf.remaining()) { + Poll::Ready(Ok(Some(chunk))) => { + buf.put_slice(&chunk); + Poll::Ready(Ok(())) + } + Poll::Ready(Ok(None)) => Poll::Ready(Ok(())), + Poll::Ready(Err(error)) => Poll::Ready(Err(error)), + Poll::Pending => Poll::Pending, + } + } + } + + impl rustfs_rio::ChunkReader for BodyStallTestReader { + fn poll_read_chunk(mut self: Pin<&mut Self>, cx: &mut Context<'_>, max: usize) -> Poll>> { + self.poll_chunk(cx, max) + } + } + + struct CountOnDrop(Arc); + + impl Drop for CountOnDrop { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::Relaxed); + } + } + + #[derive(Debug)] + struct ParallelResumeTransport { + initial_data: Bytes, + resumed_data: Bytes, + initial_delay: Duration, + fresh_delay: Duration, + fresh_read_requests: Mutex>, + fresh_chunk_requests: Mutex>, + initial_next_data: Option<(Bytes, Duration)>, + } + + impl ParallelResumeTransport { + fn new(initial_delay: Duration, fresh_delay: Duration) -> Self { + Self { + initial_data: Bytes::from_static(b"da"), + resumed_data: Bytes::from_static(b"ta"), + initial_delay, + fresh_delay, + fresh_read_requests: Mutex::new(Vec::new()), + fresh_chunk_requests: Mutex::new(Vec::new()), + initial_next_data: None, + } + } + + fn with_initial_next_data(initial_delay: Duration, next_delay: Duration, fresh_delay: Duration) -> Self { + let mut transport = Self::new(initial_delay, fresh_delay); + transport.initial_next_data = Some((Bytes::from_static(b"ta"), next_delay)); + transport + } + } + + #[async_trait::async_trait] + impl InternodeDataTransport for ParallelResumeTransport { + async fn open_read(&self, request: ReadStreamRequest) -> Result { + let reader = match self.initial_next_data.as_ref() { + Some((next_data, next_delay)) => BodyStallTestReader::with_next_data( + self.initial_data.clone(), + self.initial_delay, + next_data.clone(), + *next_delay, + request.stall_timeout, + ), + None => BodyStallTestReader::new(self.initial_data.clone(), self.initial_delay, request.stall_timeout), + }; + Ok(Box::new(reader)) + } + + async fn open_read_fresh(&self, request: ReadStreamRequest) -> Result { + self.fresh_read_requests + .lock() + .expect("fresh read request lock should not be poisoned") + .push(request); + time::sleep(self.fresh_delay).await; + Ok(Box::new(Cursor::new(self.resumed_data.clone()))) + } + + async fn open_read_chunks(&self, request: ReadStreamRequest) -> Result> { + let reader = match self.initial_next_data.as_ref() { + Some((next_data, next_delay)) => BodyStallTestReader::with_next_data( + self.initial_data.clone(), + self.initial_delay, + next_data.clone(), + *next_delay, + request.stall_timeout, + ), + None => BodyStallTestReader::new(self.initial_data.clone(), self.initial_delay, request.stall_timeout), + }; + Ok(Some(Box::new(reader))) + } + + async fn open_read_chunks_fresh(&self, request: ReadStreamRequest) -> Result> { + self.fresh_chunk_requests + .lock() + .expect("fresh chunk request lock should not be poisoned") + .push(request); + time::sleep(self.fresh_delay).await; + Ok(Some(Box::new(ChunkPartialThenErrorReader { + data: Some(self.resumed_data.clone()), + error: None, + }))) + } + + async fn open_write(&self, _request: WriteStreamRequest) -> Result { + panic!("open_write should not be used in parallel resume tests"); + } + + async fn open_walk_dir(&self, _request: WalkDirStreamRequest) -> Result { + panic!("open_walk_dir should not be used in parallel resume tests"); + } + + fn name(&self) -> &'static str { + "parallel-resume-test" + } + + fn capabilities(&self) -> InternodeDataTransportCapabilities { + InternodeDataTransportCapabilities::tcp_http() + } + } + + #[derive(Debug, Default)] + struct PendingFreshOpenTransport { + fresh_read_drops: Arc, + fresh_chunk_drops: Arc, + } + + #[async_trait::async_trait] + impl InternodeDataTransport for PendingFreshOpenTransport { + async fn open_read(&self, _request: ReadStreamRequest) -> Result { + Ok(Box::new(PartialThenErrorReader { + cursor: Cursor::new(Vec::new()), + error: Some(io::Error::new(std_io::ErrorKind::ConnectionReset, "stream reset")), + })) + } + + async fn open_read_fresh(&self, _request: ReadStreamRequest) -> Result { + let _drop = CountOnDrop(Arc::clone(&self.fresh_read_drops)); + std::future::pending().await + } + + async fn open_read_chunks(&self, _request: ReadStreamRequest) -> Result> { + Ok(Some(Box::new(ChunkPartialThenErrorReader { + data: None, + error: Some(io::Error::new(std_io::ErrorKind::ConnectionReset, "stream reset")), + }))) + } + + async fn open_read_chunks_fresh(&self, _request: ReadStreamRequest) -> Result> { + let _drop = CountOnDrop(Arc::clone(&self.fresh_chunk_drops)); + std::future::pending().await + } + + async fn open_write(&self, _request: WriteStreamRequest) -> Result { + panic!("open_write should not be used in fresh open cancellation tests"); + } + + async fn open_walk_dir(&self, _request: WalkDirStreamRequest) -> Result { + panic!("open_walk_dir should not be used in fresh open cancellation tests"); + } + + fn name(&self) -> &'static str { + "pending-fresh-open-test" + } + + fn capabilities(&self) -> InternodeDataTransportCapabilities { + InternodeDataTransportCapabilities::tcp_http() + } + } + + fn resume_step_reader(step: ResumeReadStep) -> FileReader { + match step { + ResumeReadStep::PartialThenReset(data) => Box::new(PartialThenErrorReader { + cursor: Cursor::new(data), + error: Some(io::Error::new(std_io::ErrorKind::ConnectionReset, "stream reset")), + }), + ResumeReadStep::Data(data) => Box::new(Cursor::new(data)), + } + } + + fn resume_step_chunk_reader(step: ResumeReadStep) -> rustfs_rio::ChunkReaderBox { + match step { + ResumeReadStep::PartialThenReset(data) => Box::new(ChunkPartialThenErrorReader { + data: Some(Bytes::from(data)), + error: Some(io::Error::new(std_io::ErrorKind::ConnectionReset, "stream reset")), + }), + ResumeReadStep::Data(data) => Box::new(ChunkPartialThenErrorReader { + data: Some(Bytes::from(data)), + error: None, + }), + } + } + + #[async_trait::async_trait] + impl InternodeDataTransport for ResumeTransport { + async fn open_read(&self, request: ReadStreamRequest) -> Result { + self.read_requests + .lock() + .expect("read request lock should not be poisoned") + .push(request); + let step = self + .read_steps + .lock() + .expect("read steps lock should not be poisoned") + .remove(0); + Ok(resume_step_reader(step)) + } + + async fn open_read_fresh(&self, request: ReadStreamRequest) -> Result { + self.fresh_read_requests + .lock() + .expect("fresh read request lock should not be poisoned") + .push(request.clone()); + self.open_read(request).await + } + + async fn open_read_chunks(&self, request: ReadStreamRequest) -> Result> { + self.chunk_requests + .lock() + .expect("chunk request lock should not be poisoned") + .push(request); + let step = self + .chunk_steps + .lock() + .expect("chunk steps lock should not be poisoned") + .remove(0); + Ok(Some(resume_step_chunk_reader(step))) + } + + async fn open_read_chunks_fresh(&self, request: ReadStreamRequest) -> Result> { + self.fresh_chunk_requests + .lock() + .expect("fresh chunk request lock should not be poisoned") + .push(request.clone()); + self.open_read_chunks(request).await + } + + async fn open_write(&self, _request: WriteStreamRequest) -> Result { + panic!("open_write should not be used in remote read resume tests"); + } + + async fn open_walk_dir(&self, _request: WalkDirStreamRequest) -> Result { + panic!("open_walk_dir should not be used in remote read resume tests"); + } + + fn name(&self) -> &'static str { + "resume-test" + } + + fn capabilities(&self) -> InternodeDataTransportCapabilities { + InternodeDataTransportCapabilities::tcp_http() + } + } + + fn resume_request(length: usize) -> ReadStreamRequest { + ReadStreamRequest { + endpoint: "http://remote".to_string(), + disk: "disk".to_string(), + volume: "volume".to_string(), + path: "path".to_string(), + offset: 7, + length, + stall_timeout: None, + } + } + + #[tokio::test] + async fn remote_reader_resumes_from_emitted_bytes_without_duplicates() { + let transport = Arc::new(ResumeTransport::with_read_steps(vec![ResumeReadStep::Data(b"456789".to_vec())])); + let request = resume_request(10); + let reader = resume_step_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())); + let mut reader = RetryingRemoteReader::new_with_timeouts(reader, transport.clone(), request, None, None); + let mut output = Vec::new(); + reader + .read_to_end(&mut output) + .await + .expect("one body reset should be resumed"); + + assert_eq!(output, b"0123456789"); + let requests = transport + .read_requests + .lock() + .expect("read request lock should not be poisoned"); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].offset, 11); + assert_eq!(requests[0].length, 6); + assert_eq!( + transport + .fresh_read_requests + .lock() + .expect("fresh read request lock should not be poisoned") + .len(), + 1 + ); + } + + #[tokio::test] + async fn remote_chunk_reader_resumes_from_emitted_bytes_without_duplicates() { + let transport = Arc::new(ResumeTransport::with_chunk_steps(vec![ResumeReadStep::Data(b"456789".to_vec())])); + let request = resume_request(10); + let reader = resume_step_chunk_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())); + let mut reader = RetryingRemoteChunkReader::new_with_timeouts(reader, transport.clone(), request, None, None); + let mut output = Vec::new(); + reader + .read_to_end(&mut output) + .await + .expect("chunk body reset should be resumed"); + + assert_eq!(output, b"0123456789"); + let requests = transport + .chunk_requests + .lock() + .expect("chunk request lock should not be poisoned"); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].offset, 11); + assert_eq!(requests[0].length, 6); + assert_eq!( + transport + .fresh_chunk_requests + .lock() + .expect("fresh chunk request lock should not be poisoned") + .len(), + 1 + ); + } + + #[derive(Clone, Copy)] + enum ParallelResumePath { + Regular, + Chunk, + } + + async fn assert_parallel_resume_case( + path: ParallelResumePath, + initial_delay: Duration, + initial_next_delay: Option, + fresh_delay: Duration, + expect_success: bool, + ) { + const DATA: &[u8] = b"data"; + let transport = Arc::new(match initial_next_delay { + Some(next_delay) => ParallelResumeTransport::with_initial_next_data(initial_delay, next_delay, fresh_delay), + None => ParallelResumeTransport::new(initial_delay, fresh_delay), + }); + let remote_disk = new_remote_disk_with_transport(transport.clone()).await; + let erasure = Erasure::new(1, 1, DATA.len()); + let (buffers, errors) = match path { + ParallelResumePath::Regular => { + let reader = remote_disk + .read_file_stream("bucket", "object/part.1", 0, DATA.len()) + .await + .expect("initial remote reader should open"); + let readers = vec![ + Some(BitrotReader::new(reader, DATA.len(), rustfs_utils::HashAlgorithm::None, false)), + None, + ]; + ParallelReader::new_with_metrics_path_and_reconstruction_verification(readers, erasure, 0, DATA.len(), None) + .read() + .await + } + ParallelResumePath::Chunk => { + let reader = remote_disk + .read_file_stream_chunks("bucket", "object/part.1", 0, DATA.len()) + .await + .expect("initial remote chunk reader should open") + .expect("chunk transport should return a reader"); + let readers = vec![ + Some(BitrotReader::new( + ShardReader::Chunked(reader), + DATA.len(), + rustfs_utils::HashAlgorithm::None, + false, + )), + None, + ]; + ParallelReader::new_with_metrics_path_and_reconstruction_verification(readers, erasure, 0, DATA.len(), None) + .read() + .await + } + }; + + if expect_success { + assert_eq!(buffers[0].as_deref(), Some(DATA)); + assert!(errors[0].is_none()); + } else { + assert!(buffers[0].is_none()); + assert!(matches!(errors[0], Some(DiskError::Timeout))); + } + let requests = match path { + ParallelResumePath::Regular => transport + .fresh_read_requests + .lock() + .expect("fresh read request lock should not be poisoned"), + ParallelResumePath::Chunk => transport + .fresh_chunk_requests + .lock() + .expect("fresh chunk request lock should not be poisoned"), + }; + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].offset, 2); + assert_eq!(requests[0].length, 2); + } + + async fn assert_parallel_resume_path(path: ParallelResumePath) { + assert_parallel_resume_case(path, Duration::ZERO, None, Duration::from_millis(100), true).await; + assert_parallel_resume_case(path, Duration::from_millis(500), None, Duration::from_millis(100), true).await; + assert_parallel_resume_case(path, Duration::ZERO, None, Duration::from_millis(500), false).await; + assert_parallel_resume_case( + path, + Duration::from_millis(650), + Some(Duration::from_millis(700)), + Duration::from_millis(500), + true, + ) + .await; + } + + async fn assert_retry_drop_cancels_fresh_open(path: ParallelResumePath) { + let transport = Arc::new(PendingFreshOpenTransport::default()); + let remote_disk = new_remote_disk_with_transport(transport.clone()).await; + let mut output = [0_u8; 1]; + match path { + ParallelResumePath::Regular => { + let mut reader = remote_disk + .read_file_stream("bucket", "object/part.1", 0, 1) + .await + .expect("initial remote reader should open"); + assert!( + time::timeout(Duration::from_millis(20), reader.read(&mut output)) + .await + .is_err() + ); + drop(reader); + } + ParallelResumePath::Chunk => { + let mut reader = remote_disk + .read_file_stream_chunks("bucket", "object/part.1", 0, 1) + .await + .expect("initial remote chunk reader should open") + .expect("chunk transport should return a reader"); + assert!( + time::timeout(Duration::from_millis(20), reader.read(&mut output)) + .await + .is_err() + ); + drop(reader); + } + } + time::timeout(Duration::from_secs(1), async { + loop { + let drops = match path { + ParallelResumePath::Regular => transport.fresh_read_drops.load(Ordering::Relaxed), + ParallelResumePath::Chunk => transport.fresh_chunk_drops.load(Ordering::Relaxed), + }; + if drops == 1 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("dropping the retrying reader should cancel the pending fresh open"); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn remote_reader_recovers_body_stall_through_parallel_reader() { + temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, Some("1"))], async { + assert_parallel_resume_path(ParallelResumePath::Regular).await; + }) + .await; + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn remote_chunk_reader_recovers_body_stall_through_parallel_reader() { + temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, Some("1"))], async { + assert_parallel_resume_path(ParallelResumePath::Chunk).await; + }) + .await; + } + + #[tokio::test] + async fn remote_reader_drop_cancels_pending_fresh_open() { + assert_retry_drop_cancels_fresh_open(ParallelResumePath::Regular).await; + } + + #[tokio::test] + async fn remote_chunk_reader_drop_cancels_pending_fresh_open() { + assert_retry_drop_cancels_fresh_open(ParallelResumePath::Chunk).await; + } + + #[tokio::test] + async fn remote_reader_treats_error_after_requested_length_as_eof() { + let transport = Arc::new(ResumeTransport::default()); + let reader = resume_step_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())); + let mut reader = RetryingRemoteReader::new_with_timeouts(reader, transport.clone(), resume_request(4), None, None); + let mut output = Vec::new(); + + reader + .read_to_end(&mut output) + .await + .expect("error after the requested bytes should not trigger a redundant resume"); + assert_eq!(output, b"0123"); + assert!( + transport + .fresh_read_requests + .lock() + .expect("fresh read request lock should not be poisoned") + .is_empty() + ); + } + + #[tokio::test] + async fn remote_chunk_reader_treats_error_after_requested_length_as_eof() { + let transport = Arc::new(ResumeTransport::default()); + let reader = resume_step_chunk_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())); + let mut reader = RetryingRemoteChunkReader::new_with_timeouts(reader, transport.clone(), resume_request(4), None, None); + let mut output = Vec::new(); + + reader + .read_to_end(&mut output) + .await + .expect("error after the requested bytes should not trigger a redundant chunk resume"); + assert_eq!(output, b"0123"); + assert!( + transport + .fresh_chunk_requests + .lock() + .expect("fresh chunk request lock should not be poisoned") + .is_empty() + ); + } + + #[tokio::test] + async fn remote_reader_retries_at_most_once_and_preserves_non_retryable_errors() { + let transport = Arc::new(ResumeTransport::with_read_steps(vec![ResumeReadStep::PartialThenReset(b"456".to_vec())])); + let mut reader = RetryingRemoteReader::new_with_timeouts( + resume_step_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())), + transport.clone(), + resume_request(7), + None, + None, + ); + let error = reader + .read_to_end(&mut Vec::new()) + .await + .expect_err("second reset must not retry"); + assert_eq!(error.kind(), std_io::ErrorKind::ConnectionReset); + assert_eq!( + transport + .read_requests + .lock() + .expect("read request lock should not be poisoned") + .len(), + 1 + ); + + let transport = Arc::new(ResumeTransport::default()); + let reader = PartialThenErrorReader { + cursor: Cursor::new(b"data".to_vec()), + error: Some(io::Error::new(std_io::ErrorKind::PermissionDenied, "permission denied")), + }; + let mut reader = + RetryingRemoteReader::new_with_timeouts(Box::new(reader), transport.clone(), resume_request(4), None, None); + let error = reader + .read_to_end(&mut Vec::new()) + .await + .expect_err("non-retryable errors must not retry"); + assert_eq!(error.kind(), std_io::ErrorKind::PermissionDenied); + assert!( + transport + .read_requests + .lock() + .expect("read request lock should not be poisoned") + .is_empty() + ); + } + + #[test] + fn resumed_read_request_checks_large_offsets() { + let request = ReadStreamRequest { + offset: usize::MAX - 1, + length: 0, + ..resume_request(0) + }; + assert!(resumed_read_request(&request, 2).is_err()); + + let request = resume_request(4); + assert!(resumed_read_request(&request, 5).is_err()); + } + fn init_tracing(filter_level: Level) { INIT.call_once(|| { let _ = tracing_subscriber::fmt() @@ -4509,7 +5670,7 @@ mod tests { assert_eq!(request.path, "object/part.1"); assert_eq!(request.offset, 7); assert_eq!(request.length, 11); - assert_eq!(request.stall_timeout, Some(get_object_disk_read_timeout())); + assert_eq!(request.stall_timeout, remote_read_timeouts(get_object_disk_read_timeout()).body_stall); } other => panic!("expected read transport call, got {other:?}"), } diff --git a/crates/rio/src/http_reader.rs b/crates/rio/src/http_reader.rs index 96fd0971f..93f61a15c 100644 --- a/crates/rio/src/http_reader.rs +++ b/crates/rio/src/http_reader.rs @@ -715,6 +715,13 @@ async fn get_http_client(url: &str) -> io::Result { Ok(cached.client_for(disable_proxy)) } +async fn get_fresh_http_client(url: &str) -> io::Result { + let tuning = internode_http_client_tuning(); + let disable_proxy = should_disable_proxy_for_url(url, tuning); + let outbound_tls = crate::http_runtime_sources::outbound_tls_state().await; + build_http_client(disable_proxy, tuning, &outbound_tls).await +} + fn internode_request_context(method: &Method, url: &str, operation: Option<&'static str>) -> InternodeHttpRequestContext { let target = reqwest::Url::parse(url) .ok() @@ -962,6 +969,28 @@ impl HttpReader { Self::with_capacity_and_stall_timeout(url, method, headers, body, 0, stall_timeout).await } + pub async fn new_fresh_connection_with_stall_timeout( + url: String, + method: Method, + headers: HeaderMap, + body: Option>, + stall_timeout: Option, + ) -> io::Result { + let init = Self::open(&url, &method, &headers, body, stall_timeout, true).await?; + Ok(Self { + inner: StreamReader::new(init.stream), + url, + method, + headers, + track_internode_metrics: init.track_internode_metrics, + internode_operation: init.internode_operation, + stall_timer: None, + stall_timeout: init.stall_timeout, + request_started: init.request_started, + duration_recorded: false, + }) + } + /// Create a new HttpReader from a URL. The request is performed immediately. pub async fn with_capacity( url: String, @@ -981,7 +1010,7 @@ impl HttpReader { _read_buf_size: usize, stall_timeout: Option, ) -> io::Result { - let init = Self::open(&url, &method, &headers, body, stall_timeout).await?; + let init = Self::open(&url, &method, &headers, body, stall_timeout, false).await?; Ok(Self { inner: StreamReader::new(init.stream), url, @@ -1002,10 +1031,16 @@ impl HttpReader { headers: &HeaderMap, body: Option>, stall_timeout: Option, + force_fresh_connection: bool, ) -> io::Result { let track_internode_metrics = is_internode_rpc_url(url); let internode_operation = internode_rpc_operation(url); - let client = get_http_client(url).await.inspect_err(|_| { + let client = if force_fresh_connection { + get_fresh_http_client(url).await + } else { + get_http_client(url).await + } + .inspect_err(|_| { record_internode_error(track_internode_metrics, internode_operation); })?; let mut request: RequestBuilder = client.request(method.clone(), url).headers(headers.clone()); @@ -1121,7 +1156,28 @@ impl HttpChunkReader { body: Option>, stall_timeout: Option, ) -> io::Result { - let init = HttpReader::open(&url, &method, &headers, body, stall_timeout).await?; + let init = HttpReader::open(&url, &method, &headers, body, stall_timeout, false).await?; + Ok(Self { + inner: init.stream, + current: None, + track_internode_metrics: init.track_internode_metrics, + internode_operation: init.internode_operation, + stall_timer: None, + stall_timeout: init.stall_timeout, + request_started: init.request_started, + duration_recorded: false, + consecutive_empty_chunks: 0, + }) + } + + pub async fn new_fresh_connection_with_stall_timeout( + url: String, + method: Method, + headers: HeaderMap, + body: Option>, + stall_timeout: Option, + ) -> io::Result { + let init = HttpReader::open(&url, &method, &headers, body, stall_timeout, true).await?; Ok(Self { inner: init.stream, current: None, From 4f68f117ba485db1cff34fac79c3030f62723b69 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 18 Aug 2026 14:49:39 +0800 Subject: [PATCH 29/51] fix(lock): reap expired local lease guards (#6094) * fix(lock): reap expired local lease guards * fix(lock): reject refresh after guard expiry * test(lock): isolate expired refresh regression * style(lock): format expired refresh assertion --- crates/lock/src/client/local.rs | 351 +++++++++++++++++++++++------ crates/lock/src/namespace/tests.rs | 111 +++++++++ 2 files changed, 393 insertions(+), 69 deletions(-) diff --git a/crates/lock/src/client/local.rs b/crates/lock/src/client/local.rs index 2c65f2040..3b2a185aa 100644 --- a/crates/lock/src/client/local.rs +++ b/crates/lock/src/client/local.rs @@ -15,8 +15,10 @@ use std::collections::HashMap; use std::hash::{Hash, Hasher}; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, SystemTime}; use tokio::sync::RwLock; +use tokio::time::Instant; use crate::{ FastLockGuard, GlobalLockManager, LockClient, LockId, LockInfo, LockManager, LockMetadata, LockPriority, LockRequest, @@ -26,43 +28,51 @@ use crate::{ /// Default shard count for guard storage (must be power of 2) const DEFAULT_GUARD_SHARD_COUNT: usize = 64; +type GuardShard = Arc>>; +type GuardStorage = Arc>; + /// Local lock client using FastLock with sharded guard storage for better concurrency #[derive(Debug)] pub struct LocalClient { /// Sharded guard storage to reduce lock contention - guard_storage: Vec>>>, + guard_storage: GuardStorage, /// Mask for fast shard index calculation (shard_count - 1) shard_mask: usize, /// Optional lock manager (if None, uses global singleton) manager: Option>, + reaper_started: AtomicBool, + reaper_interval: Duration, } #[derive(Debug)] struct LocalGuardEntry { guard: FastLockGuard, expires_at: SystemTime, + deadline: Instant, ttl: Duration, - /// Owner recorded at acquire time; used only for reclaim diagnostics (#899). - owner: String, } impl LocalGuardEntry { - fn new(guard: FastLockGuard, ttl: Duration, owner: String) -> Self { + fn new(guard: FastLockGuard, ttl: Duration) -> Self { let now = SystemTime::now(); + let monotonic_now = Instant::now(); Self { guard, - expires_at: now + ttl, + expires_at: now.checked_add(ttl).unwrap_or(now), + deadline: monotonic_now.checked_add(ttl).unwrap_or(monotonic_now), ttl, - owner, } } fn is_expired(&self) -> bool { - self.expires_at <= SystemTime::now() + self.deadline <= Instant::now() } fn refresh(&mut self) { - self.expires_at = SystemTime::now() + self.ttl; + let now = SystemTime::now(); + let monotonic_now = Instant::now(); + self.expires_at = now.checked_add(self.ttl).unwrap_or(now); + self.deadline = monotonic_now.checked_add(self.ttl).unwrap_or(monotonic_now); } } @@ -77,26 +87,38 @@ impl LocalClient { pub fn with_shard_count(shard_count: usize) -> Self { assert!(shard_count.is_power_of_two(), "Shard count must be power of 2"); - let guard_storage: Vec>>> = - (0..shard_count).map(|_| Arc::new(RwLock::new(HashMap::new()))).collect(); + let guard_storage: Vec = (0..shard_count).map(|_| Arc::new(RwLock::new(HashMap::new()))).collect(); + Self::with_storage(Arc::new(guard_storage), None, crate::fast_lock::CLEANUP_INTERVAL) + } + + fn with_storage(guard_storage: GuardStorage, manager: Option>, reaper_interval: Duration) -> Self { + let shard_count = guard_storage.len(); + debug_assert!(shard_count.is_power_of_two()); Self { guard_storage, shard_mask: shard_count - 1, - manager: None, + manager, + reaper_started: AtomicBool::new(false), + reaper_interval, } } /// Create new local client with a specific lock manager /// This allows simulating multi-node environments where each node has its own lock backend pub fn with_manager(manager: Arc) -> Self { - Self { - guard_storage: (0..DEFAULT_GUARD_SHARD_COUNT) - .map(|_| Arc::new(RwLock::new(HashMap::new()))) - .collect(), - shard_mask: DEFAULT_GUARD_SHARD_COUNT - 1, - manager: Some(manager), - } + let guard_storage = (0..DEFAULT_GUARD_SHARD_COUNT) + .map(|_| Arc::new(RwLock::new(HashMap::new()))) + .collect(); + Self::with_storage(Arc::new(guard_storage), Some(manager), crate::fast_lock::CLEANUP_INTERVAL) + } + + #[cfg(test)] + pub(crate) fn with_manager_and_reaper_interval(manager: Arc, reaper_interval: Duration) -> Self { + let guard_storage = (0..DEFAULT_GUARD_SHARD_COUNT) + .map(|_| Arc::new(RwLock::new(HashMap::new()))) + .collect(); + Self::with_storage(Arc::new(guard_storage), Some(manager), reaper_interval) } /// Get the lock manager (injected manager if available, otherwise global singleton) @@ -118,52 +140,63 @@ impl LocalClient { } async fn reclaim_expired_guards_for_resource(&self, resource: &crate::ObjectKey) -> usize { - let mut reclaimed = 0usize; + let expired_entries = Self::extract_expired_guards(&self.guard_storage, Some(resource)).await; + Self::release_reclaimed_guards(expired_entries, Some(resource)) + } - for shard in &self.guard_storage { - let expired_entries = { - let mut guards = shard.write().await; - let mut retained = HashMap::with_capacity(guards.len()); - let mut expired_entries = Vec::new(); + async fn extract_expired_guards(storage: &GuardStorage, resource: Option<&crate::ObjectKey>) -> Vec { + let mut expired_entries = Vec::new(); + for shard in storage.iter() { + let mut guards = shard.write().await; + expired_entries.extend( + guards + .extract_if(|lock_id, entry| { + resource.is_none_or(|resource| &lock_id.resource == resource) && entry.is_expired() + }) + .map(|(_, entry)| entry), + ); + } + expired_entries + } - for (lock_id, entry) in std::mem::take(&mut *guards) { - if &lock_id.resource == resource && entry.is_expired() { - expired_entries.push(entry); - } else { - retained.insert(lock_id, entry); - } - } - - *guards = retained; - expired_entries - }; - - for mut entry in expired_entries { - // An expired entry whose owner never refreshed it (a dead coordinator, #698) is - // reclaimed so a live contender can re-form quorum. With guard heartbeats in place - // (#899) a live owner keeps its entry from expiring, so reaching here means the - // lease genuinely lapsed. Surface it for observability; the reclaim decision itself - // is unchanged. - let since_last_refresh = entry - .expires_at - .checked_sub(entry.ttl) - .and_then(|last_refresh| SystemTime::now().duration_since(last_refresh).ok()) - .unwrap_or(entry.ttl); - tracing::warn!( - owner = %entry.owner, - resource = %resource, - ttl_ms = entry.ttl.as_millis() as u64, - since_last_refresh_ms = since_last_refresh.as_millis() as u64, - "reclaiming expired lock guard whose lease was not refreshed" - ); - rustfs_io_metrics::record_lock_reclaimed(); - let _ = entry.guard.release(); - reclaimed = reclaimed.saturating_add(1); + fn release_reclaimed_guards( + entries: impl IntoIterator, + resource: Option<&crate::ObjectKey>, + ) -> usize { + let mut reclaimed = 0; + for mut entry in entries { + let _ = entry.guard.release(); + rustfs_io_metrics::record_lock_reclaimed(); + reclaimed += 1; + } + if reclaimed > 0 { + if let Some(resource) = resource { + tracing::debug!(event = "lock_guard_reclaimed", resource = %resource, count = reclaimed, "expired lock guards reclaimed"); + } else { + tracing::debug!(event = "lock_guard_reaper_sweep", count = reclaimed, "expired lock guards reclaimed"); } } - reclaimed } + + fn ensure_reaper(&self) { + if self.reaper_started.swap(true, Ordering::AcqRel) { + return; + } + let storage = Arc::downgrade(&self.guard_storage); + let interval = self.reaper_interval; + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + loop { + ticker.tick().await; + let Some(storage) = storage.upgrade() else { + break; + }; + let expired_entries = Self::extract_expired_guards(&storage, None).await; + Self::release_reclaimed_guards(expired_entries, None); + } + }); + } } impl Default for LocalClient { @@ -175,28 +208,36 @@ impl Default for LocalClient { #[async_trait::async_trait] impl LockClient for LocalClient { async fn acquire_lock(&self, request: &LockRequest) -> Result { + self.ensure_reaper(); let lock_manager = self.get_lock_manager(); let reclaimed_before_acquire = self.reclaim_expired_guards_for_resource(&request.resource).await; + let acquire_deadline = Instant::now() + .checked_add(request.acquire_timeout) + .unwrap_or_else(Instant::now); - let build_lock_request = || match request.lock_type { + let build_lock_request = |acquire_timeout| match request.lock_type { LockType::Exclusive => crate::ObjectLockRequest::new_write(request.resource.clone(), request.owner.clone()) - .with_acquire_timeout(request.acquire_timeout), + .with_acquire_timeout(acquire_timeout), LockType::Shared => crate::ObjectLockRequest::new_read(request.resource.clone(), request.owner.clone()) - .with_acquire_timeout(request.acquire_timeout), + .with_acquire_timeout(acquire_timeout), }; let mut retried_after_reclaim = reclaimed_before_acquire > 0; loop { - match lock_manager.acquire_lock(build_lock_request()).await { + let remaining = acquire_deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Ok(LockResponse::failure("Lock acquisition timeout", request.acquire_timeout)); + } + match lock_manager.acquire_lock(build_lock_request(remaining)).await { Ok(guard) => { let lock_id = request.lock_id.clone(); let acquired_at = SystemTime::now(); - let expires_at = acquired_at + request.ttl; + let expires_at = acquired_at.checked_add(request.ttl).unwrap_or(acquired_at); { let shard = self.get_shard(&lock_id); let mut guards = shard.write().await; - guards.insert(lock_id.clone(), LocalGuardEntry::new(guard, request.ttl, request.owner.clone())); + guards.insert(lock_id.clone(), LocalGuardEntry::new(guard, request.ttl)); } let lock_info = LockInfo { @@ -256,12 +297,24 @@ impl LockClient for LocalClient { async fn refresh(&self, lock_id: &LockId) -> Result { let shard = self.get_shard(lock_id); - let mut guards = shard.write().await; - if let Some(entry) = guards.get_mut(lock_id) { - entry.refresh(); - Ok(true) - } else { + let expired_entry = { + let mut guards = shard.write().await; + let Some(entry) = guards.get_mut(lock_id) else { + return Ok(false); + }; + if entry.is_expired() { + guards.remove(lock_id) + } else { + entry.refresh(); + None + } + }; + + if let Some(entry) = expired_entry { + Self::release_reclaimed_guards([entry], Some(&lock_id.resource)); Ok(false) + } else { + Ok(true) } } @@ -317,3 +370,163 @@ impl LockClient for LocalClient { true } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{GlobalLockManager, LockClient, LockRequest, LockType}; + + fn request(resource: crate::ObjectKey, owner: &str, ttl: Duration) -> LockRequest { + LockRequest::new(resource, LockType::Exclusive, owner) + .with_ttl(ttl) + .with_acquire_timeout(Duration::from_millis(80)) + } + + async fn wait_until_reaped(client: &LocalClient, lock_id: &LockId) { + for _ in 0..80 { + if client.check_status(lock_id).await.unwrap().is_none() { + return; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + panic!("lock guard was not reaped before test deadline"); + } + + #[tokio::test(flavor = "current_thread")] + async fn expired_guard_is_reaped_without_resource_reacquire() { + let manager = Arc::new(GlobalLockManager::new()); + let client = LocalClient::with_manager_and_reaper_interval(manager.clone(), Duration::from_millis(5)); + let request = request(crate::ObjectKey::new("bucket", "unique-chunk"), "owner-a", Duration::from_millis(10)); + let lock_id = request.lock_id.clone(); + + assert!(client.acquire_lock(&request).await.unwrap().success); + assert!(client.check_status(&lock_id).await.unwrap().is_some()); + tokio::time::sleep(Duration::from_millis(15)).await; + wait_until_reaped(&client, &lock_id).await; + + let direct = manager + .acquire_lock(crate::ObjectLockRequest::new_write(request.resource.clone(), "owner-b")) + .await; + assert!(direct.is_ok()); + } + + #[tokio::test(flavor = "current_thread")] + async fn sibling_client_cannot_reclaim_but_owner_reaper_releases_shared_lock() { + let manager = Arc::new(GlobalLockManager::new()); + let owner = LocalClient::with_manager_and_reaper_interval(manager.clone(), Duration::from_millis(5)); + let contender = LocalClient::with_manager_and_reaper_interval(manager, Duration::from_millis(5)); + let request_a = request(crate::ObjectKey::new("bucket", "shared-resource"), "owner-a", Duration::from_millis(10)); + assert!(owner.acquire_lock(&request_a).await.unwrap().success); + + let request_b = request(request_a.resource.clone(), "owner-b", Duration::from_millis(20)) + .with_acquire_timeout(Duration::from_millis(5)); + assert!(!contender.acquire_lock(&request_b).await.unwrap().success); + + tokio::time::sleep(Duration::from_millis(25)).await; + assert!(owner.check_status(&request_a.lock_id).await.unwrap().is_none()); + assert!(contender.acquire_lock(&request_b).await.unwrap().success); + } + + #[tokio::test(flavor = "current_thread")] + async fn refresh_wins_before_deadline_and_reaper_wins_after_deadline() { + let manager = Arc::new(GlobalLockManager::new()); + let client = LocalClient::with_manager_and_reaper_interval(manager, Duration::from_millis(5)); + let request = request(crate::ObjectKey::new("bucket", "refresh-race"), "owner-a", Duration::from_millis(25)); + let lock_id = request.lock_id.clone(); + assert!(client.acquire_lock(&request).await.unwrap().success); + + tokio::time::sleep(Duration::from_millis(10)).await; + assert!(client.refresh(&lock_id).await.unwrap()); + tokio::time::sleep(Duration::from_millis(15)).await; + assert!(client.check_status(&lock_id).await.unwrap().is_some()); + wait_until_reaped(&client, &lock_id).await; + } + + #[tokio::test(start_paused = true)] + async fn refresh_after_expiry_releases_guard_without_reviving_it() { + let manager = Arc::new(GlobalLockManager::new()); + let client = LocalClient::with_manager_and_reaper_interval(manager, Duration::from_secs(60)); + client.reaper_started.store(true, Ordering::Release); + let lock_request = request( + crate::ObjectKey::new("bucket", "refresh-after-expiry"), + "owner-a", + Duration::from_secs(10), + ); + let lock_id = lock_request.lock_id.clone(); + + assert!( + client + .acquire_lock(&lock_request) + .await + .expect("initial owner should acquire the lock") + .success + ); + tokio::time::advance(Duration::from_secs(11)).await; + + assert!( + !client + .refresh(&lock_id) + .await + .expect("expired refresh should return a result"), + "an expired guard must not be refreshed" + ); + assert!( + client + .check_status(&lock_id) + .await + .expect("expired guard status should be readable") + .is_none(), + "expired guard should be removed after refresh" + ); + + let contender = request( + crate::ObjectKey::new("bucket", "refresh-after-expiry"), + "owner-b", + Duration::from_secs(10), + ); + assert!( + client + .acquire_lock(&contender) + .await + .expect("contender should receive an acquisition result") + .success, + "released guard must be acquirable by a new owner" + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn zero_ttl_is_reaped_and_oversized_ttl_does_not_panic() { + let manager = Arc::new(GlobalLockManager::new()); + let client = LocalClient::with_manager_and_reaper_interval(manager, Duration::from_millis(5)); + + let zero = request(crate::ObjectKey::new("bucket", "zero-ttl"), "owner-zero", Duration::ZERO); + let zero_id = zero.lock_id.clone(); + assert!(client.acquire_lock(&zero).await.unwrap().success); + wait_until_reaped(&client, &zero_id).await; + + let huge = request(crate::ObjectKey::new("bucket", "huge-ttl"), "owner-huge", Duration::MAX); + let huge_id = huge.lock_id.clone(); + assert!(client.acquire_lock(&huge).await.unwrap().success); + wait_until_reaped(&client, &huge_id).await; + } + + #[tokio::test(flavor = "current_thread")] + async fn acquire_retry_preserves_total_deadline() { + let manager = Arc::new(GlobalLockManager::new()); + let client = LocalClient::with_manager_and_reaper_interval(manager, Duration::from_secs(60)); + let first = request(crate::ObjectKey::new("bucket", "deadline-budget"), "owner-a", Duration::from_millis(10)); + assert!(client.acquire_lock(&first).await.unwrap().success); + + let second = + request(first.resource.clone(), "owner-b", Duration::from_millis(30)).with_acquire_timeout(Duration::from_millis(60)); + let started = Instant::now(); + let response = client.acquire_lock(&second).await.unwrap(); + assert!(!response.success, "the first attempt consumed the caller's acquire budget"); + assert!( + started.elapsed() < Duration::from_millis(100), + "reclaim retry must not double the acquire budget" + ); + let recovered = client.acquire_lock(&second).await.unwrap(); + assert!(recovered.success, "the reclaimed guard must be available to the next request"); + } +} diff --git a/crates/lock/src/namespace/tests.rs b/crates/lock/src/namespace/tests.rs index 0ee5a81bb..3316828f7 100644 --- a/crates/lock/src/namespace/tests.rs +++ b/crates/lock/src/namespace/tests.rs @@ -840,6 +840,117 @@ async fn test_namespace_lock_distributed_reclaims_expired_same_resource_after_fa ); } +#[tokio::test] +async fn four_node_failed_release_converges_without_replica_repair() { + let managers = (0..4).map(|_| Arc::new(GlobalLockManager::new())).collect::>(); + let flaky_clients = managers + .iter() + .map(|manager| { + Arc::new(FlakyReleaseClient { + inner: LocalClient::with_manager_and_reaper_interval(manager.clone(), Duration::from_millis(5)), + failed_releases_remaining: AtomicUsize::new(usize::MAX), + release_attempts: AtomicUsize::new(0), + }) + }) + .collect::>(); + let clients = flaky_clients + .iter() + .map(|client| client.clone() as Arc) + .collect::>(); + let lock = NamespaceLock::Distributed(DistributedLock::new("four-node-expired-lease".to_string(), clients, 3)); + let resource = create_test_object_key("bucket", "object-four-node-expired"); + let request = LockRequest::new(resource.clone(), LockType::Exclusive, "owner-a") + .with_acquire_timeout(Duration::from_millis(300)) + .with_ttl(Duration::from_millis(40)); + + let mut guard = lock + .acquire_guard(&request) + .await + .expect("initial acquire should not error") + .expect("initial acquire should reach quorum"); + assert!(guard.release(), "release should be acknowledged while RPC cleanup is pending"); + + for _ in 0..40 { + if flaky_clients.iter().all(|client| client.release_attempts() >= 3) { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + + let deadline = tokio::time::Instant::now() + Duration::from_secs(2); + loop { + let all_reaped = + futures::future::join_all(flaky_clients.iter().map(|client| client.inner.check_status(&request.lock_id))) + .await + .into_iter() + .all(|status| status.expect("status should not error").is_none()); + if all_reaped { + break; + } + assert!(tokio::time::Instant::now() < deadline, "all four local lease entries must converge"); + tokio::time::sleep(Duration::from_millis(10)).await; + } + + for suffix in ["chunk-0", "chunk-1", ".rustfs.sys/multipart/upload-0"] { + for client in &flaky_clients { + let orphan = LockRequest::new(create_test_object_key("bucket", suffix), LockType::Exclusive, "orphan") + .with_ttl(Duration::from_millis(25)); + assert!(client.inner.acquire_lock(&orphan).await.expect("orphan acquire").success); + } + } + tokio::time::sleep(Duration::from_millis(80)).await; + + let recovered = lock + .acquire_guard( + &LockRequest::new(resource, LockType::Exclusive, "owner-b") + .with_acquire_timeout(Duration::from_millis(300)) + .with_ttl(Duration::from_millis(40)), + ) + .await + .expect("recovery acquire should not error") + .expect("four-node quorum should recover after local reapers run"); + drop(recovered); +} + +#[tokio::test] +async fn four_node_stale_quorum_contention_respects_acquire_deadline() { + let managers = (0..4).map(|_| Arc::new(GlobalLockManager::new())).collect::>(); + let node_clients = managers + .iter() + .map(|manager| Arc::new(LocalClient::with_manager_and_reaper_interval(manager.clone(), Duration::from_millis(5)))) + .collect::>(); + let resource = create_test_object_key("bucket", "stale-quorum"); + let stale = LockRequest::new(resource.clone(), LockType::Exclusive, "stale-owner").with_ttl(Duration::from_millis(180)); + for client in &node_clients { + assert!(client.acquire_lock(&stale).await.expect("stale acquire").success); + } + + let clients = node_clients + .iter() + .map(|client| client.clone() as Arc) + .collect::>(); + let lock = NamespaceLock::Distributed(DistributedLock::new("stale-quorum-deadline".to_string(), clients, 3)); + let contender = LockRequest::new(resource.clone(), LockType::Exclusive, "new-owner") + .with_acquire_timeout(Duration::from_millis(150)) + .with_ttl(Duration::from_millis(100)); + let started = tokio::time::Instant::now(); + let response = lock.acquire_guard(&contender).await.expect("contention should not error"); + assert!(response.is_none(), "unexpired leases must not be force-reclaimed"); + assert!(started.elapsed() < Duration::from_millis(350), "acquire must respect its deadline"); + + tokio::time::sleep(Duration::from_millis(80)).await; + let recovered = lock + .acquire_guard( + &LockRequest::new(resource, LockType::Exclusive, "new-owner") + .with_acquire_timeout(Duration::from_millis(300)) + .with_ttl(Duration::from_millis(100)), + ) + .await + .expect("post-expiry acquire should not error") + .expect("quorum should recover after local reapers clear stale leases"); + drop(recovered); +} + #[tokio::test] async fn test_namespace_lock_distributed_retries_transient_acquire_timeout() { let managers = (0..3).map(|_| Arc::new(GlobalLockManager::new())).collect::>(); From a4ea36b298b5e1713c29178f03f8b6bdc796a761 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 18 Aug 2026 14:49:53 +0800 Subject: [PATCH 30/51] fix(ecstore): single-flight remote disk recovery (#6096) * fix(ecstore): single-flight remote disk recovery * test(ecstore): cover remote recovery review cases * test(ecstore): exercise recovery through disk slot * test(ecstore): match format reads exactly * test(ecstore): cover recovery teardown races * style(ecstore): format recovery race tests * fix(ecstore): remove unused health snapshot helper * fix(ecstore): group recovery monitor test state * fix(protos): preserve production source in compatibility checks --- Cargo.lock | 1 + crates/ecstore/Cargo.toml | 1 + crates/ecstore/src/cluster/rpc/remote_disk.rs | 661 +++++++++++++++++- crates/ecstore/src/disk/disk_store.rs | 130 +++- crates/protos/src/lib.rs | 2 +- 5 files changed, 746 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82ffafc81..b297135af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9465,6 +9465,7 @@ dependencies = [ "tokio-stream", "tokio-util", "tonic", + "tonic-prost", "tower", "tracing", "tracing-core", diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index bdbb53e49..6525f0cfd 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -273,6 +273,7 @@ proptest = "1" rcgen.workspace = true insta = { workspace = true, features = ["yaml", "json"] } rustfs-crypto = { workspace = true } +tonic-prost = { workspace = true } [build-dependencies] shadow-rs = { workspace = true, default-features = false, features = ["build", "metadata"] } diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index 3ae2c8b01..26e9f899a 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -61,7 +61,7 @@ use std::{ pin::Pin, sync::{ Arc, - atomic::{AtomicU32, Ordering}, + atomic::{AtomicBool, AtomicU32, Ordering}, }, task::{Context, Poll}, time::Duration, @@ -630,6 +630,8 @@ impl rustfs_rio::ChunkReader for RetryingRemoteChunkReader { #[derive(Debug)] pub struct RemoteDisk { + /// Stable identity for this handle instance; replacement handles receive a new identity. + handle_id: Uuid, pub id: Mutex>, pub addr: String, endpoint: Endpoint, @@ -640,9 +642,38 @@ pub struct RemoteDisk { health: Arc, /// Cancellation token for monitoring tasks cancel_token: CancellationToken, + recovery_monitor_active: Arc, + #[cfg(test)] + recovery_monitor_start_count: Arc, + #[cfg(test)] + recovery_monitor_teardown_hook: Arc>>>, data_transport: Arc, } +struct RecoveryMonitorLease { + active: Arc, +} + +impl Drop for RecoveryMonitorLease { + fn drop(&mut self) { + self.active.store(false, Ordering::Release); + } +} + +#[cfg(test)] +#[derive(Debug, Default)] +struct RecoveryMonitorTeardownHook { + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(test)] +#[derive(Clone)] +struct RecoveryMonitorTestState { + start_count: Arc, + teardown_hook: Arc>>>, +} + // ── Connection lifecycle (grpc-optimization P3) ── /// Whether to prewarm the internode control channel in the background at construction (default off). @@ -782,14 +813,15 @@ impl RemoteDisk { .await } - fn recovery_monitor_span(addr: &str, endpoint: &Endpoint) -> tracing::Span { + fn recovery_monitor_span(addr: &str, endpoint: &Endpoint, handle_id: Uuid) -> tracing::Span { tracing::info_span!( "recovery-monitor", component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_REMOTE_DISK, kind = "remote_disk", endpoint = %endpoint, - addr = %addr + addr = %addr, + handle_id = %handle_id ) } @@ -825,6 +857,7 @@ impl RemoteDisk { rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING); let disk = Self { + handle_id: Uuid::new_v4(), id: Mutex::new(None), addr, endpoint: ep.clone(), @@ -832,6 +865,11 @@ impl RemoteDisk { health_check: opt.health_check && env_health_check, health: Arc::new(DiskHealthTracker::new()), cancel_token: CancellationToken::new(), + recovery_monitor_active: Arc::new(AtomicBool::new(false)), + #[cfg(test)] + recovery_monitor_start_count: Arc::new(AtomicU32::new(0)), + #[cfg(test)] + recovery_monitor_teardown_hook: Arc::new(tokio::sync::Mutex::new(None)), data_transport, }; record_drive_runtime_state(ep, RuntimeDriveHealthState::Online); @@ -849,6 +887,16 @@ impl RemoteDisk { self.health.runtime_state() } + #[cfg(test)] + fn recovery_monitor_is_active(&self) -> bool { + self.recovery_monitor_active.load(Ordering::Acquire) + } + + #[cfg(test)] + fn recovery_monitor_start_count(&self) -> u32 { + self.recovery_monitor_start_count.load(Ordering::Acquire) + } + pub fn offline_duration_secs(&self) -> Option { self.health.offline_duration().map(|duration| duration.as_secs()) } @@ -987,13 +1035,62 @@ impl RemoteDisk { return; } - let addr = self.addr.clone(); - let endpoint = self.endpoint.clone(); - let health = Arc::clone(&self.health); - let cancel_token = self.cancel_token.clone(); - let span = Self::recovery_monitor_span(&addr, &endpoint); + Self::schedule_recovery_monitor( + self.addr.clone(), + self.endpoint.clone(), + self.handle_id, + Arc::clone(&self.health), + self.cancel_token.clone(), + Arc::clone(&self.recovery_monitor_active), + #[cfg(test)] + RecoveryMonitorTestState { + start_count: Arc::clone(&self.recovery_monitor_start_count), + teardown_hook: Arc::clone(&self.recovery_monitor_teardown_hook), + }, + ); + } + + fn schedule_recovery_monitor( + addr: String, + endpoint: Endpoint, + handle_id: Uuid, + health: Arc, + cancel_token: CancellationToken, + active: Arc, + #[cfg(test)] test_state: RecoveryMonitorTestState, + ) { + if active + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return; + } + let span = Self::recovery_monitor_span(&addr, &endpoint, handle_id); super::spawn_background_monitor(span, async move { - Self::monitor_remote_disk_recovery(addr, endpoint, health, cancel_token).await; + #[cfg(test)] + test_state.start_count.fetch_add(1, Ordering::AcqRel); + let lease = RecoveryMonitorLease { + active: Arc::clone(&active), + }; + Self::monitor_remote_disk_recovery(addr.clone(), endpoint.clone(), Arc::clone(&health), cancel_token.clone()).await; + #[cfg(test)] + if let Some(hook) = test_state.teardown_hook.lock().await.take() { + hook.arrived.notify_one(); + hook.release.notified().await; + } + drop(lease); + if !cancel_token.is_cancelled() && health.runtime_state() != RuntimeDriveHealthState::Online { + Self::schedule_recovery_monitor( + addr, + endpoint, + handle_id, + health, + cancel_token, + active, + #[cfg(test)] + test_state, + ); + } }); } @@ -1002,7 +1099,7 @@ impl RemoteDisk { let (tx, rx) = tokio::sync::oneshot::channel(); let endpoint = self.endpoint.clone(); let addr = self.addr.clone(); - let span = Self::recovery_monitor_span(&addr, &endpoint); + let span = Self::recovery_monitor_span(&addr, &endpoint, self.handle_id); super::spawn_background_monitor(span, async move { warn!( event = EVENT_REMOTE_DISK_HEALTH, @@ -1033,9 +1130,23 @@ impl RemoteDisk { let cancel_token = self.cancel_token.clone(); let addr = self.addr.clone(); let endpoint = self.endpoint.clone(); + let handle_id = self.handle_id; + let recovery_monitor_active = Arc::clone(&self.recovery_monitor_active); + #[cfg(test)] + let recovery_monitor_teardown_hook = Arc::clone(&self.recovery_monitor_teardown_hook); tokio::spawn(async move { - Self::monitor_remote_disk_health(addr, endpoint, health, cancel_token).await; + Self::monitor_remote_disk_health( + addr, + endpoint, + handle_id, + health, + cancel_token, + recovery_monitor_active, + #[cfg(test)] + recovery_monitor_teardown_hook, + ) + .await; }); } @@ -1043,8 +1154,11 @@ impl RemoteDisk { async fn monitor_remote_disk_health( addr: String, endpoint: Endpoint, + handle_id: Uuid, health: Arc, cancel_token: CancellationToken, + recovery_monitor_active: Arc, + #[cfg(test)] recovery_monitor_teardown_hook: Arc>>>, ) { let mut interval = time::interval(get_drive_active_check_interval()); @@ -1069,11 +1183,19 @@ impl RemoteDisk { let addr_clone = addr.clone(); let endpoint_clone = endpoint.clone(); let cancel_clone = cancel_token.clone(); - let span = Self::recovery_monitor_span(&addr_clone, &endpoint_clone); - - super::spawn_background_monitor(span, async move { - Self::monitor_remote_disk_recovery(addr_clone, endpoint_clone, health_clone, cancel_clone).await; - }); + Self::schedule_recovery_monitor( + addr_clone, + endpoint_clone, + handle_id, + health_clone, + cancel_clone, + Arc::clone(&recovery_monitor_active), + #[cfg(test)] + RecoveryMonitorTestState { + start_count: Arc::new(AtomicU32::new(0)), + teardown_hook: Arc::clone(&recovery_monitor_teardown_hook), + }, + ); } loop { @@ -1132,11 +1254,19 @@ impl RemoteDisk { let addr_clone = addr.clone(); let endpoint_clone = endpoint.clone(); let cancel_clone = cancel_token.clone(); - let span = Self::recovery_monitor_span(&addr_clone, &endpoint_clone); - - super::spawn_background_monitor(span, async move { - Self::monitor_remote_disk_recovery(addr_clone, endpoint_clone, health_clone, cancel_clone).await; - }); + Self::schedule_recovery_monitor( + addr_clone, + endpoint_clone, + handle_id, + health_clone, + cancel_clone, + Arc::clone(&recovery_monitor_active), + #[cfg(test)] + RecoveryMonitorTestState { + start_count: Arc::new(AtomicU32::new(0)), + teardown_hook: Arc::clone(&recovery_monitor_teardown_hook), + }, + ); } } } @@ -1387,6 +1517,7 @@ impl RemoteDisk { subsystem = LOG_SUBSYSTEM_REMOTE_DISK, endpoint = %self.endpoint, addr = %self.addr, + handle_id = %self.handle_id, op, state = "faulty_short_circuit", "Remote disk operation short-circuited by faulty state" @@ -3548,15 +3679,23 @@ mod tests { use crate::erasure::coding::{BitrotReader, Erasure, decode::ParallelReader}; use crate::io_support::bitrot::ShardReader; use crate::runtime::sources as runtime_sources; + use rustfs_protos::proto_gen::node_service::{DiskInfoResponse, ReadAllResponse}; use serde_json::Value; use serial_test::serial; + use std::convert::Infallible; + use std::future::Future; use std::io::{self as std_io, Write}; use std::pin::Pin; use std::sync::{Arc, Mutex, Mutex as StdMutex, Once, atomic::AtomicUsize}; use std::task::{Context, Poll}; use tokio::io::{ReadBuf, duplex}; use tokio::net::TcpListener; - use tonic::transport::Endpoint as TonicEndpoint; + use tonic::transport::{Endpoint as TonicEndpoint, Server}; + use tonic::{Response, Status}; + use tonic::{ + codegen::{Body as HttpBody, BoxFuture, StdError, http}, + server::NamedService, + }; use tracing::Level; use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt}; use uuid::Uuid; @@ -3716,6 +3855,218 @@ mod tests { ns_scanner_probe_status: Arc>>, } + #[derive(Clone, Debug)] + struct AuthenticatedReadPeer { + audience: String, + disk_info_calls: Arc, + read_all_calls: Arc, + object_read_all_disks: Arc>>, + format_data: Bytes, + read_all_data: Bytes, + } + + impl AuthenticatedReadPeer { + fn new(audience: String, format_data: Bytes, read_all_data: Bytes) -> Self { + Self { + audience, + disk_info_calls: Arc::new(AtomicU32::new(0)), + read_all_calls: Arc::new(AtomicU32::new(0)), + object_read_all_disks: Arc::default(), + format_data, + read_all_data, + } + } + + fn disk_info_calls(&self) -> u32 { + self.disk_info_calls.load(Ordering::Acquire) + } + + fn read_all_calls(&self) -> u32 { + self.read_all_calls.load(Ordering::Acquire) + } + + fn object_read_all_disks(&self) -> Vec { + self.object_read_all_disks + .lock() + .expect("object read_all disk list lock poisoned") + .clone() + } + + fn verify_auth(&self, request: &Request, path: &str) -> std::result::Result<(), Status> { + let headers = request.metadata().clone().into_headers(); + crate::cluster::rpc::verify_tonic_rpc_signature(&self.audience, path, &headers) + .map_err(|err| Status::unauthenticated(err.to_string())) + } + } + + #[derive(Clone, Debug)] + struct AuthenticatedReadPeerService { + peer: AuthenticatedReadPeer, + } + + impl NamedService for AuthenticatedReadPeerService { + const NAME: &'static str = "node_service.NodeService"; + } + + impl tower::Service> for AuthenticatedReadPeerService + where + B: HttpBody + Send + 'static, + B::Error: Into + Send + 'static, + { + type Response = http::Response; + type Error = Infallible; + type Future = BoxFuture; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, request: http::Request) -> Self::Future { + match request.uri().path() { + "/node_service.NodeService/DiskInfo" => { + #[derive(Clone)] + struct DiskInfoSvc(AuthenticatedReadPeer); + + impl tonic::server::UnaryService for DiskInfoSvc { + type Response = DiskInfoResponse; + type Future = Pin, Status>> + Send>>; + + fn call(&mut self, request: Request) -> Self::Future { + let peer = self.0.clone(); + Box::pin(async move { + peer.verify_auth(&request, "/node_service.NodeService/DiskInfo")?; + let request = request.into_inner(); + let opts = serde_json::from_str::(&request.opts) + .map_err(|err| Status::invalid_argument(err.to_string()))?; + if !opts.noop { + return Err(Status::invalid_argument("recovery probe must use noop disk_info")); + } + peer.disk_info_calls.fetch_add(1, Ordering::AcqRel); + let disk_info = serde_json::to_string(&DiskInfo { + total: 1, + free: 1, + endpoint: request.disk, + ..Default::default() + }) + .map_err(|err| Status::internal(err.to_string()))?; + Ok(Response::new(DiskInfoResponse { + success: true, + disk_info, + error: None, + })) + }) + } + } + + let peer = self.peer.clone(); + Box::pin(async move { + let method = DiskInfoSvc(peer); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec); + Ok(grpc.unary(method, request).await) + }) + } + "/node_service.NodeService/ReadAll" => { + #[derive(Clone)] + struct ReadAllSvc(AuthenticatedReadPeer); + + impl tonic::server::UnaryService for ReadAllSvc { + type Response = ReadAllResponse; + type Future = Pin, Status>> + Send>>; + + fn call(&mut self, request: Request) -> Self::Future { + let peer = self.0.clone(); + Box::pin(async move { + peer.verify_auth(&request, "/node_service.NodeService/ReadAll")?; + let request = request.into_inner(); + let is_format_read = request.volume == crate::disk::RUSTFS_META_BUCKET + && request.path == crate::disk::FORMAT_CONFIG_FILE; + let disk = request.disk; + peer.read_all_calls.fetch_add(1, Ordering::AcqRel); + let data = if is_format_read { + peer.format_data.clone() + } else { + peer.object_read_all_disks + .lock() + .expect("object read_all disk list lock poisoned") + .push(disk); + peer.read_all_data.clone() + }; + Ok(Response::new(ReadAllResponse { + success: true, + data, + error: None, + })) + }) + } + } + + let peer = self.peer.clone(); + Box::pin(async move { + let method = ReadAllSvc(peer); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec); + Ok(grpc.unary(method, request).await) + }) + } + _ => Box::pin(async move { + let mut response = http::Response::new(tonic::body::Body::default()); + let headers = response.headers_mut(); + headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into()); + headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE); + Ok(response) + }), + } + } + } + + struct TestGrpcPeer { + addr: String, + peer: AuthenticatedReadPeer, + shutdown: CancellationToken, + task: tokio::task::JoinHandle<()>, + } + + impl TestGrpcPeer { + async fn spawn(format_data: Bytes, read_all_data: Bytes) -> Option { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, + Err(err) => panic!("test gRPC listener should bind: {err}"), + }; + let socket_addr = listener.local_addr().expect("listener local address should be available"); + let addr = format!("http://{socket_addr}"); + let audience = crate::cluster::rpc::normalize_tonic_rpc_audience(&socket_addr.to_string()) + .expect("test audience should normalize"); + let peer = AuthenticatedReadPeer::new(audience, format_data, read_all_data); + let service = AuthenticatedReadPeerService { peer: peer.clone() }; + let shutdown = CancellationToken::new(); + let shutdown_for_task = shutdown.clone(); + let incoming = futures_util::stream::unfold(listener, |listener| async { + Some((listener.accept().await.map(|(stream, _)| stream), listener)) + }); + let task = tokio::spawn(async move { + Server::builder() + .add_service(service) + .serve_with_incoming_shutdown(incoming, shutdown_for_task.cancelled_owned()) + .await + .expect("test gRPC peer should serve"); + }); + + Some(Self { + addr, + peer, + shutdown, + task, + }) + } + + async fn stop(self) { + self.shutdown.cancel(); + let _ = self.task.await; + } + } + impl RecordingInternodeDataTransport { fn with_ns_scanner_probe_status(status: u16) -> Self { Self { @@ -5558,6 +5909,245 @@ mod tests { accept_task.abort(); } + #[tokio::test] + async fn faulty_handle_runs_only_one_recovery_monitor() { + let endpoint = Endpoint { + url: url::Url::parse("http://remote-node:9000/data/rustfs0").expect("endpoint should parse"), + is_local: false, + pool_idx: 0, + set_idx: 0, + disk_idx: 0, + }; + let disk = RemoteDisk::new( + &endpoint, + &DiskOption { + cleanup: false, + health_check: true, + }, + Arc::new(TcpHttpInternodeDataTransport), + ) + .await + .expect("remote disk should construct"); + if !disk.health_check { + return; + } + + disk.force_runtime_state_for_test(RuntimeDriveHealthState::Offline); + disk.spawn_recovery_monitor_if_needed(); + disk.spawn_recovery_monitor_if_needed(); + tokio::time::timeout(Duration::from_secs(1), async { + while disk.recovery_monitor_start_count() == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("recovery monitor should start"); + + assert!(disk.recovery_monitor_is_active(), "only one recovery monitor should own the handle"); + assert_eq!( + disk.recovery_monitor_start_count(), + 1, + "the failed compare-exchange path must not start a second monitor" + ); + + disk.cancel_token.cancel(); + tokio::time::timeout(Duration::from_secs(1), async { + while disk.recovery_monitor_is_active() { + tokio::task::yield_now().await; + } + }) + .await + .expect("cancelled recovery monitor should release its single-flight state"); + assert!(!disk.recovery_monitor_is_active()); + } + + #[tokio::test] + #[serial(remote_disk_recovery_probe)] + async fn recovery_monitor_rearms_if_disk_fails_during_teardown() { + runtime_sources::ensure_test_rpc_secret(); + let Some(peer) = TestGrpcPeer::spawn(Bytes::new(), Bytes::new()).await else { + return; + }; + let endpoint = Endpoint { + url: url::Url::parse(&format!("{}/data/rustfs0", peer.addr)).expect("endpoint should parse"), + is_local: false, + pool_idx: 0, + set_idx: 0, + disk_idx: 0, + }; + let disk = RemoteDisk::new( + &endpoint, + &DiskOption { + cleanup: false, + health_check: true, + }, + Arc::new(TcpHttpInternodeDataTransport), + ) + .await + .expect("remote disk should construct"); + if !disk.health_check { + peer.stop().await; + return; + } + + disk.force_runtime_state_for_test(RuntimeDriveHealthState::Offline); + let hook = Arc::new(RecoveryMonitorTeardownHook::default()); + *disk.recovery_monitor_teardown_hook.lock().await = Some(Arc::clone(&hook)); + + temp_env::async_with_vars( + [ + (rustfs_config::ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS, Some("1")), + (rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD, Some("1")), + (rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, Some("1")), + ], + async { + disk.spawn_recovery_monitor_if_needed(); + tokio::time::timeout(Duration::from_secs(5), hook.arrived.notified()) + .await + .expect("first recovery monitor should reach teardown"); + assert_eq!(disk.runtime_state(), RuntimeDriveHealthState::Online); + + disk.force_runtime_state_for_test(RuntimeDriveHealthState::Offline); + hook.release.notify_one(); + tokio::time::timeout(Duration::from_secs(2), async { + while disk.recovery_monitor_start_count() < 2 { + tokio::task::yield_now().await; + } + }) + .await + .expect("teardown failure should re-arm recovery monitoring"); + assert!( + disk.recovery_monitor_is_active(), + "re-armed monitor should retain single-flight ownership" + ); + + disk.cancel_token.cancel(); + tokio::time::timeout(Duration::from_secs(2), async { + while disk.recovery_monitor_is_active() { + tokio::task::yield_now().await; + } + }) + .await + .expect("cancelled re-armed monitor should release single-flight state"); + }, + ) + .await; + + peer.stop().await; + } + + #[tokio::test] + #[serial(remote_disk_recovery_probe)] + async fn recovery_monitor_restores_online_then_real_reads_use_replacement_handle() { + runtime_sources::ensure_test_rpc_secret(); + let mut format = crate::layout::format::FormatV3::new(1, 1); + let disk_id = format.erasure.sets[0][0]; + format.erasure.this = disk_id; + let format_data = Bytes::from(format.to_json().expect("test format should serialize")); + let Some(peer) = TestGrpcPeer::spawn(format_data, Bytes::from_static(b"replacement-data")).await else { + return; + }; + let url = url::Url::parse(&format!("{}/data/rustfs0", peer.addr)).expect("endpoint should parse"); + let endpoint = Endpoint { + url, + is_local: false, + pool_idx: 0, + set_idx: 0, + disk_idx: 0, + }; + let disk = RemoteDisk::new( + &endpoint, + &DiskOption { + cleanup: false, + health_check: true, + }, + Arc::new(TcpHttpInternodeDataTransport), + ) + .await + .expect("remote disk should construct"); + disk.force_runtime_state_for_test(RuntimeDriveHealthState::Offline); + + temp_env::async_with_vars( + [ + (rustfs_config::ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS, Some("1")), + (rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD, Some("3")), + (rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, Some("1")), + ], + async { + let monitor = tokio::spawn(RemoteDisk::monitor_remote_disk_recovery( + disk.addr.clone(), + endpoint.clone(), + Arc::clone(&disk.health), + disk.cancel_token.clone(), + )); + + tokio::time::timeout(Duration::from_secs(5), async { + while disk.runtime_state() != RuntimeDriveHealthState::Online { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("three authenticated recovery probes should restore the disk online"); + monitor.await.expect("recovery monitor should exit after restoring Online"); + + assert_eq!( + peer.peer.disk_info_calls(), + 3, + "RemoteDisk recovery requires the configured three successful disk_info probes" + ); + let recovered_read = disk.read_all("bucket", "object").await.expect("recovered handle should read"); + assert_eq!(recovered_read, Bytes::from_static(b"replacement-data")); + + let old_disk = crate::disk::new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("old slot disk should construct"); + let set_disks = crate::set_disk::SetDisks::new( + "remote-recovery-test".to_string(), + Arc::new(tokio::sync::RwLock::new(vec![Some(old_disk.clone())])), + 1, + 0, + 0, + 0, + vec![endpoint.clone()], + format, + Vec::new(), + ) + .await; + set_disks.disks.write().await[0] = None; + set_disks.renew_disk(&endpoint).await; + + let slots = set_disks.disks.read().await; + let replacement = slots[0] + .as_ref() + .expect("renew_disk should publish the replacement slot") + .clone(); + drop(slots); + assert!(!Arc::ptr_eq(&replacement, &old_disk), "renew_disk must replace the stale slot handle"); + let replacement_read = replacement + .read_all("bucket", "object") + .await + .expect("production slot should route real reads through the replacement"); + assert_eq!(replacement_read, Bytes::from_static(b"replacement-data")); + let object_reads = peer.peer.object_read_all_disks(); + assert_eq!(object_reads.len(), 2, "standalone and production-slot reads should both reach the peer"); + assert_eq!(object_reads[1], disk_id.to_string(), "production slot must use the renewed disk identity"); + assert!(peer.peer.read_all_calls() >= 3, "renewal must read format metadata before the slot read"); + disk.cancel_token.cancel(); + old_disk.close().await.expect("old slot disk should close"); + replacement.close().await.expect("replacement slot disk should close"); + }, + ) + .await; + + peer.stop().await; + } + #[tokio::test] async fn test_copy_stream_with_buffer_copies_full_payload() { let payload = b"walk-dir-stream".repeat(1024); @@ -7124,6 +7714,20 @@ mod tests { ) .await .expect("remote disk should construct"); + let replacement = RemoteDisk::new( + &endpoint, + &DiskOption { + cleanup: false, + health_check: true, + }, + Arc::new(TcpHttpInternodeDataTransport), + ) + .await + .expect("replacement remote disk should construct"); + assert_ne!( + remote_disk.handle_id, replacement.handle_id, + "replacement handles need distinct log identities" + ); let span = tracing::info_span!("request-span", request_id = "req-remote-disk"); let _entered = span.enter(); @@ -7139,11 +7743,24 @@ mod tests { assert_eq!(log["span"]["name"], Value::String("recovery-monitor".to_string())); assert_eq!(log["span"]["kind"], Value::String("remote_disk".to_string())); + assert_eq!(log["span"]["handle_id"], Value::String(remote_disk.handle_id.to_string())); let spans = log["spans"].as_array().expect("spans should be present"); assert!(spans.iter().any(|span| { span.get("name").and_then(Value::as_str) == Some("request-span") && span.get("request_id").and_then(Value::as_str) == Some("req-remote-disk") })); + + remote_disk.force_runtime_state_for_test(RuntimeDriveHealthState::Offline); + remote_disk + .execute_with_timeout(|| async { Ok::<(), Error>(()) }, Duration::from_secs(1)) + .await + .expect_err("faulty handle should short-circuit"); + let faulty_log = logs + .lines() + .into_iter() + .find(|value| value.get("state").and_then(Value::as_str) == Some("faulty_short_circuit")) + .expect("expected faulty short-circuit log"); + assert_eq!(faulty_log["handle_id"], Value::String(remote_disk.handle_id.to_string())); } #[tokio::test(flavor = "current_thread")] diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 29a8037dd..5c0f6454e 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -418,6 +418,17 @@ pub struct DiskHealthTracker { pub last_capacity_free: AtomicU64, /// Last successful capacity probe timestamp pub last_capacity_probe_unix_secs: AtomicI64, + /// Authoritative atomically published runtime/status pair. + state_snapshot: AtomicU64, + transition_lock: std::sync::Mutex<()>, +} + +fn pack_health_state(runtime_state: RuntimeDriveHealthState, status: u32) -> u64 { + (u64::from(runtime_state as u32) << 32) | u64::from(status) +} + +fn unpack_health_state(snapshot: u64) -> (RuntimeDriveHealthState, u32) { + (RuntimeDriveHealthState::from_u32((snapshot >> 32) as u32), snapshot as u32) } #[derive(Debug)] @@ -739,6 +750,8 @@ impl DiskHealthTracker { last_capacity_used: AtomicU64::new(0), last_capacity_free: AtomicU64::new(0), last_capacity_probe_unix_secs: AtomicI64::new(0), + state_snapshot: AtomicU64::new(pack_health_state(RuntimeDriveHealthState::Online, DISK_HEALTH_OK)), + transition_lock: std::sync::Mutex::new(()), } } @@ -775,39 +788,52 @@ impl DiskHealthTracker { /// Check if disk is faulty pub fn is_faulty(&self) -> bool { - self.status.load(Ordering::Acquire) == DISK_HEALTH_FAULTY + unpack_health_state(self.state_snapshot.load(Ordering::Acquire)).1 == DISK_HEALTH_FAULTY + } + + fn publish_state(&self, runtime_state: RuntimeDriveHealthState, status: u32) { + self.state_snapshot + .store(pack_health_state(runtime_state, status), Ordering::Release); + self.runtime_state.store(runtime_state as u32, Ordering::Release); + self.status.store(status, Ordering::Release); } /// Set disk as faulty #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub fn set_faulty(&self) { - self.status.store(DISK_HEALTH_FAULTY, Ordering::Release); + let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + self.publish_state(RuntimeDriveHealthState::Offline, DISK_HEALTH_FAULTY); } /// Set disk as OK pub fn set_ok(&self) { - self.status.store(DISK_HEALTH_OK, Ordering::Release); + let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + self.publish_state(RuntimeDriveHealthState::Online, DISK_HEALTH_OK); } #[cfg(test)] pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) { - self.runtime_state.store(state as u32, Ordering::Release); - match state { - RuntimeDriveHealthState::Offline => self.set_faulty(), - RuntimeDriveHealthState::Online | RuntimeDriveHealthState::Suspect | RuntimeDriveHealthState::Returning => { - self.set_ok(); - } - } + let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let status = if state == RuntimeDriveHealthState::Offline { + DISK_HEALTH_FAULTY + } else { + DISK_HEALTH_OK + }; + self.publish_state(state, status); } pub fn swap_ok_to_faulty(&self) -> bool { - self.status - .compare_exchange(DISK_HEALTH_OK, DISK_HEALTH_FAULTY, Ordering::AcqRel, Ordering::Relaxed) - .is_ok() + let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let (_, status) = unpack_health_state(self.state_snapshot.load(Ordering::Acquire)); + if status != DISK_HEALTH_OK { + return false; + } + self.publish_state(RuntimeDriveHealthState::Offline, DISK_HEALTH_FAULTY); + true } pub fn runtime_state(&self) -> RuntimeDriveHealthState { - RuntimeDriveHealthState::from_u32(self.runtime_state.load(Ordering::Acquire)) + unpack_health_state(self.state_snapshot.load(Ordering::Acquire)).0 } pub fn offline_duration(&self) -> Option { @@ -823,6 +849,7 @@ impl DiskHealthTracker { } pub fn mark_failure(&self, endpoint: &Endpoint, reason: &'static str) -> bool { + let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); let current = self.runtime_state(); let now = current_unix_secs(); let next = match current { @@ -851,24 +878,19 @@ impl DiskHealthTracker { }; let became_offline = next == RuntimeDriveHealthState::Offline && current != RuntimeDriveHealthState::Offline; - if next == RuntimeDriveHealthState::Offline { - self.status.store(DISK_HEALTH_FAULTY, Ordering::Release); - } else { - self.status.store(DISK_HEALTH_OK, Ordering::Release); - } self.transition_state(endpoint, current, next, reason); became_offline } #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool { + let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); let current = self.runtime_state(); if current == RuntimeDriveHealthState::Offline { return false; } self.consecutive_successes.store(0, Ordering::Release); - self.status.store(DISK_HEALTH_FAULTY, Ordering::Release); self.transition_state(endpoint, current, RuntimeDriveHealthState::Offline, reason); true } @@ -882,11 +904,10 @@ impl DiskHealthTracker { } fn reset_for_store_init_retry_at(&self, endpoint: &Endpoint, now: Duration) { + let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); let now_nanos = unix_nanos(now); let now_secs = unix_secs_i64(now); - self.status.store(DISK_HEALTH_OK, Ordering::Release); - self.runtime_state - .store(RuntimeDriveHealthState::Online as u32, Ordering::Release); + self.publish_state(RuntimeDriveHealthState::Online, DISK_HEALTH_OK); self.consecutive_failures.store(0, Ordering::Release); self.consecutive_successes.store(0, Ordering::Release); self.offline_since_unix_secs.store(0, Ordering::Release); @@ -898,6 +919,7 @@ impl DiskHealthTracker { } pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool { + let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); let current = self.runtime_state(); let next = match current { RuntimeDriveHealthState::Online => RuntimeDriveHealthState::Online, @@ -918,7 +940,6 @@ impl DiskHealthTracker { let became_online = next == RuntimeDriveHealthState::Online; if became_online { - self.status.store(DISK_HEALTH_OK, Ordering::Release); self.consecutive_failures.store(0, Ordering::Release); self.consecutive_successes.store(0, Ordering::Release); } @@ -948,7 +969,13 @@ impl DiskHealthTracker { return; } - self.runtime_state.store(next as u32, Ordering::Release); + let current_status = unpack_health_state(self.state_snapshot.load(Ordering::Acquire)).1; + let status = match next { + RuntimeDriveHealthState::Offline => DISK_HEALTH_FAULTY, + RuntimeDriveHealthState::Returning => current_status, + RuntimeDriveHealthState::Online | RuntimeDriveHealthState::Suspect => DISK_HEALTH_OK, + }; + self.publish_state(next, status); self.last_transition_unix_secs .store(current_unix_secs() as i64, Ordering::Release); @@ -1217,7 +1244,7 @@ impl LocalDiskWrapper { return; } - if health.status.load(Ordering::Relaxed) != DISK_HEALTH_OK { + if health.is_faulty() { continue; } @@ -2909,6 +2936,57 @@ mod tests { }); } + #[test] + #[serial_test::serial] + fn concurrent_failure_and_recovery_publish_one_health_snapshot() { + temp_env::with_var(rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD, Some("2"), || { + let endpoint = Endpoint::try_from("/tmp/concurrent-health-snapshot").expect("endpoint should parse"); + let health = Arc::new(DiskHealthTracker::new()); + let transition_guard = health + .transition_lock + .lock() + .expect("health transition lock should not be poisoned"); + let start = Arc::new(std::sync::Barrier::new(3)); + let (completed_tx, completed_rx) = std::sync::mpsc::channel(); + let workers = (0..2) + .map(|_| { + let health = Arc::clone(&health); + let endpoint = endpoint.clone(); + let start = Arc::clone(&start); + let completed_tx = completed_tx.clone(); + std::thread::spawn(move || { + start.wait(); + health.mark_failure(&endpoint, "concurrent_test"); + completed_tx.send(()).expect("completion receiver should remain available"); + }) + }) + .collect::>(); + + start.wait(); + assert!( + matches!( + completed_rx.recv_timeout(Duration::from_millis(250)), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) + ), + "concurrent transitions must wait for the serialization lock" + ); + drop(transition_guard); + completed_rx + .recv_timeout(Duration::from_secs(1)) + .expect("first failure transition should complete after lock release"); + completed_rx + .recv_timeout(Duration::from_secs(1)) + .expect("second failure transition should complete after lock release"); + for worker in workers { + worker.join().expect("health transition worker should not panic"); + } + + assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline); + assert!(health.is_faulty()); + assert_eq!(health.consecutive_failures.load(Ordering::Acquire), 2); + }); + } + #[test] fn operation_success_recovers_suspect_drive_without_faulting() { let endpoint = Endpoint::try_from("/tmp/runtime-state-suspect-success").expect("endpoint should parse"); diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index a826d4a31..d23224b93 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -2557,7 +2557,7 @@ mod tests { fn production_source(source: &'static str, file_name: &str) -> &'static str { source - .split("\n#[cfg(test)]") + .split("\n#[cfg(test)]\nmod tests") .next() .unwrap_or_else(|| panic!("{file_name} should contain production source before tests")) } From a5800033bd323304d370bc81a31563b34222efdc Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 16:09:30 +0800 Subject: [PATCH 31/51] feat(heal): incremental status cursors and typed overlap policy (HS-06) (#6206) * feat(heal): incremental heal status cursors and typed overlap policy (HS-06) Incremental results: every retained result item now carries a monotonic sequence number. The status query accepts a client cursor (sinceSeq on the admin wire, Option internally) and returns only newer items, plus nextSeq (the next cursor) and minSeq (the oldest retained sequence). A cursor that fell behind the 1024-item retention window is flagged through the existing truncated signal together with minSeq so the client can restart from it. Sequencing survives task completion: the completion archive stores the seq-stamped window. None keeps the exact legacy full-snapshot behavior, so existing clients see no change. Typed overlap handling for admin starts: RUSTFS_HEAL_OVERLAP_POLICY (merge default | minio_error). Under minio_error, an admin start whose path overlaps an active or queued task rejects with typed already-running / overlapping-paths admission reasons (surfaced through reason_label in the admin error body, sharing the existing OperationAborted site because the s3s footprint ratchet forbids new s3_error! sites); an exact duplicate start rejects with already-running instead of silently merging. Scanner/autoheal/ read-repair sources never take the rejection path. forceStart semantics now match MinIO for admin requests: an admin forceStart first cancels the overlapping active admin task, then admits the replacement. Wire: the heal-control Query command grows an optional sinceSeq (defaulted and skipped when absent, so older peers stay compatible); the admin handler accepts the sinceSeq query parameter; the local channel query gains the same cursor. Tests: seq monotonicity and incremental slicing, window slide moving minSeq with lagging-cursor flags, overlap matrix (same/containing/ contained/disjoint x policy x source), forceStart cancel-then-admit, and the completion-archive window handoff. Co-Authored-By: heihutu * style: fmt after main merge --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- Cargo.lock | 2 + crates/common/src/heal_channel.rs | 23 ++ crates/config/src/constants/heal.rs | 9 + crates/heal/src/heal/channel.rs | 113 +++++- crates/heal/src/heal/manager.rs | 497 ++++++++++++++++++++++--- crates/heal/src/heal/task.rs | 140 ++++++- crates/protos/src/heal_control.rs | 76 +++- crates/scanner/src/scanner_folder.rs | 7 + rustfs/src/admin/handlers/heal.rs | 24 +- rustfs/src/storage/rpc/node_service.rs | 10 +- scripts/check_s3s_footprint.sh | 2 +- 11 files changed, 807 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b297135af..83749d574 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9539,6 +9539,8 @@ version = "1.0.0-rc.2" dependencies = [ "async-trait", "base64 0.23.1", + "bytes", + "crc-fast", "futures", "hotpath", "http 1.5.0", diff --git a/crates/common/src/heal_channel.rs b/crates/common/src/heal_channel.rs index b5a907e4e..f8162dee6 100644 --- a/crates/common/src/heal_channel.rs +++ b/crates/common/src/heal_channel.rs @@ -224,6 +224,13 @@ pub struct HealOpts { pub enum HealAdmissionDropReason { QueueFull, PolicyDropped, + /// HS-06: an admin heal start overlaps (same bucket with mutually + /// containing prefixes, or the same erasure set) an already running or + /// queued task. Only produced when RUSTFS_HEAL_OVERLAP_POLICY=minio_error. + AlreadyRunning, + /// HS-06: same as [`Self::AlreadyRunning`] but for paths that merely + /// contain (or are contained by) the active task's path. + OverlappingPaths, } impl HealAdmissionDropReason { @@ -231,6 +238,8 @@ impl HealAdmissionDropReason { match self { Self::QueueFull => "queue_full", Self::PolicyDropped => "policy_dropped", + Self::AlreadyRunning => "already_running", + Self::OverlappingPaths => "overlapping_paths", } } } @@ -317,6 +326,9 @@ pub enum HealChannelCommand { Query { heal_path: String, client_token: String, + /// Incremental result cursor (HS-06): only items with a sequence + /// greater than this are returned; `None` keeps the full snapshot. + since_seq: Option, response_tx: oneshot::Sender>, }, /// Cancel heal task @@ -522,10 +534,21 @@ async fn receive_heal_channel_response( /// Send heal query request pub async fn query_heal_status(heal_path: String, client_token: String) -> Result { + query_heal_status_since(heal_path, client_token, None).await +} + +/// Incremental heal query (HS-06): pass the client's last seen sequence +/// number to receive only newer result items. +pub async fn query_heal_status_since( + heal_path: String, + client_token: String, + since_seq: Option, +) -> Result { let (response_tx, response_rx) = oneshot::channel(); send_heal_command(HealChannelCommand::Query { heal_path, client_token, + since_seq, response_tx, }) .await?; diff --git a/crates/config/src/constants/heal.rs b/crates/config/src/constants/heal.rs index b8cf3630a..9c27a663f 100644 --- a/crates/config/src/constants/heal.rs +++ b/crates/config/src/constants/heal.rs @@ -205,3 +205,12 @@ pub const DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES: usize = 8 * 1024 * 1024; /// Default MRF replay batch size. pub const DEFAULT_HEAL_MRF_REPLAY_BATCH: usize = 256; + +/// Environment variable selecting how admin heal starts behave when the +/// requested path overlaps an already running or queued heal: `merge` +/// (default, keep today's dedup/merge semantics) or `minio_error` (return a +/// typed already-running / overlapping-paths rejection like madmin). +pub const ENV_HEAL_OVERLAP_POLICY: &str = "RUSTFS_HEAL_OVERLAP_POLICY"; + +/// Default overlap policy: merge duplicate/overlapping requests. +pub const DEFAULT_HEAL_OVERLAP_POLICY: &str = "merge"; diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index 1e31056b2..8f4ab9e8a 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -66,21 +66,37 @@ struct HealTaskStatusPayload<'a> { summary: &'a str, items: &'a [HealResultItem], truncated: bool, + /// Cursor for incremental consumption (HS-06): sequence of the next item + /// to be produced. Absent on responses without sequencing (0). + #[serde(skip_serializing_if = "u64_is_zero")] + next_seq: u64, + /// Oldest sequence still retained; with `truncated`, tells a lagging + /// client where to restart its cursor. + #[serde(skip_serializing_if = "u64_is_zero")] + min_seq: u64, #[serde(skip_serializing_if = "Option::is_none")] progress: Option<&'a HealProgress>, } +fn u64_is_zero(value: &u64) -> bool { + *value == 0 +} + fn encode_heal_task_status_payload( summary: &str, mut items: Vec, progress: Option<&HealProgress>, mut truncated: bool, + next_seq: u64, + min_seq: u64, ) -> Result<(Vec, bool)> { loop { let data = serde_json::to_vec(&HealTaskStatusPayload { summary, items: &items, truncated, + next_seq, + min_seq, progress, }) .map_err(|e| Error::Serialization(format!("failed to serialize heal task status: {e}")))?; @@ -109,8 +125,10 @@ fn encode_heal_status_response( progress: Option<&HealProgress>, detail: Option, truncated: bool, + next_seq: u64, + min_seq: u64, ) -> Result<(Vec, Option)> { - let (data, truncated) = encode_heal_task_status_payload(summary, items, progress, truncated)?; + let (data, truncated) = encode_heal_task_status_payload(summary, items, progress, truncated, next_seq, min_seq)?; Ok((data, heal_status_detail(detail, truncated))) } @@ -138,8 +156,19 @@ impl HealChannelProcessor { /// Execute a token query directly against the manager. pub async fn execute_query_request(&self, heal_path: String, client_token: String) -> Result { + self.execute_query_request_since(heal_path, client_token, None).await + } + + /// Incremental variant of [`Self::execute_query_request`] (HS-06). + pub async fn execute_query_request_since( + &self, + heal_path: String, + client_token: String, + since_seq: Option, + ) -> Result { let (response_tx, response_rx) = oneshot::channel(); - self.process_query_request(heal_path, client_token, response_tx).await?; + self.process_query_request(heal_path, client_token, since_seq, response_tx) + .await?; response_rx .await .map_err(|err| Error::other(format!("heal query channel closed: {err}")))? @@ -262,8 +291,12 @@ impl HealChannelProcessor { HealChannelCommand::Query { heal_path, client_token, + since_seq, response_tx, - } => self.process_query_request(heal_path, client_token, response_tx).await, + } => { + self.process_query_request(heal_path, client_token, since_seq, response_tx) + .await + } HealChannelCommand::Cancel { heal_path, client_token, @@ -384,6 +417,7 @@ impl HealChannelProcessor { &self, heal_path: String, client_token: String, + since_seq: Option, response_tx: oneshot::Sender>, ) -> Result<()> { debug!( @@ -398,72 +432,118 @@ impl HealChannelProcessor { ); let report = if heal_path.trim_matches('/').is_empty() { - self.heal_manager.get_task_report(&client_token).await + self.heal_manager.get_task_report_since(&client_token, since_seq).await } else { - self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await + self.heal_manager + .get_task_report_for_path_since(&heal_path, &client_token, since_seq) + .await }; - let (summary, detail, items, truncated, progress) = match report { + let (summary, detail, items, truncated, progress, next_seq, min_seq) = match report { Ok(HealTaskReport { status: HealTaskStatus::Pending | HealTaskStatus::Running, result_items, result_items_truncated, progress, - }) => ("running".to_string(), None, result_items, result_items_truncated, progress), + next_seq, + min_seq, + }) => ( + "running".to_string(), + None, + result_items, + result_items_truncated, + progress, + next_seq, + min_seq, + ), Ok(HealTaskReport { status: HealTaskStatus::Retrying { error, retry_attempt }, result_items, result_items_truncated, progress, + next_seq, + min_seq, }) => ( "running".to_string(), Some(format!("heal task retrying after recoverable failure, attempt {retry_attempt}: {error}")), result_items, result_items_truncated, progress, + next_seq, + min_seq, ), Ok(HealTaskReport { status: HealTaskStatus::Completed, result_items, result_items_truncated, progress, - }) => ("finished".to_string(), None, result_items, result_items_truncated, progress), + next_seq, + min_seq, + }) => ( + "finished".to_string(), + None, + result_items, + result_items_truncated, + progress, + next_seq, + min_seq, + ), Ok(HealTaskReport { status: HealTaskStatus::Cancelled, result_items, result_items_truncated, progress, + next_seq, + min_seq, }) => ( "stopped".to_string(), Some("heal task cancelled".to_string()), result_items, result_items_truncated, progress, + next_seq, + min_seq, ), Ok(HealTaskReport { status: HealTaskStatus::Timeout, result_items, result_items_truncated, progress, + next_seq, + min_seq, }) => ( "stopped".to_string(), Some("heal task timed out".to_string()), result_items, result_items_truncated, progress, + next_seq, + min_seq, ), Ok(HealTaskReport { status: HealTaskStatus::Failed { error }, result_items, result_items_truncated, progress, - }) => ("stopped".to_string(), Some(error), result_items, result_items_truncated, progress), + next_seq, + min_seq, + }) => ( + "stopped".to_string(), + Some(error), + result_items, + result_items_truncated, + progress, + next_seq, + min_seq, + ), Err(crate::Error::TaskNotFound { .. }) => ( "notFound".to_string(), Some("heal task not found or expired".to_string()), Vec::new(), false, None, + 0, + 0, ), Err(crate::Error::InvalidClientToken) => { let response = HealChannelResponse { @@ -490,7 +570,8 @@ impl HealChannelProcessor { } }; - let (data, detail) = encode_heal_status_response(&summary, items, progress.as_ref(), detail, truncated)?; + let (data, detail) = + encode_heal_status_response(&summary, items, progress.as_ref(), detail, truncated, next_seq, min_seq)?; let response = HealChannelResponse { request_id: client_token, @@ -805,7 +886,7 @@ mod tests { ..Default::default() }]; - let (data, detail) = encode_heal_status_response("running", items, None, None, false).unwrap(); + let (data, detail) = encode_heal_status_response("running", items, None, None, false, 0, 0).unwrap(); assert!(data.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE); let payload: serde_json::Value = serde_json::from_slice(&data).unwrap(); @@ -1575,7 +1656,7 @@ mod tests { let (tx, rx) = oneshot::channel(); processor - .process_query_request("bucket".to_string(), "completed-token".to_string(), tx) + .process_query_request("bucket".to_string(), "completed-token".to_string(), None, tx) .await .expect("query should process"); @@ -1610,7 +1691,7 @@ mod tests { let (tx, rx) = oneshot::channel(); processor - .process_query_request("bucket".to_string(), task_id.clone(), tx) + .process_query_request("bucket".to_string(), task_id.clone(), None, tx) .await .expect("query should process"); @@ -1643,7 +1724,7 @@ mod tests { let (tx, rx) = oneshot::channel(); processor - .process_query_request("bucket".to_string(), "wrong-token".to_string(), tx) + .process_query_request("bucket".to_string(), "wrong-token".to_string(), None, tx) .await .expect("query should process"); @@ -1668,7 +1749,7 @@ mod tests { let (tx, rx) = oneshot::channel(); processor - .process_query_request(String::new(), "wrong-token".to_string(), tx) + .process_query_request(String::new(), "wrong-token".to_string(), None, tx) .await .expect("query should process"); @@ -1705,7 +1786,7 @@ mod tests { let (tx, rx) = oneshot::channel(); processor - .process_query_request(String::new(), task_id.clone(), tx) + .process_query_request(String::new(), task_id.clone(), None, tx) .await .expect("query should process"); diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 216e17068..3a243c88e 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -220,6 +220,11 @@ struct CompletedHealStatus { result_items: Vec, result_items_truncated: bool, completed_at: SystemTime, + /// Sequence-stamped retained window, archived with the completion so + /// incremental consumers keep their cursor across the transition (HS-06). + seqed_items: Vec<(u64, HealResultItem)>, + next_seq: u64, + min_seq: u64, } #[derive(Debug, Clone)] @@ -240,6 +245,65 @@ pub struct HealTaskReport { pub result_items: Vec, pub result_items_truncated: bool, pub progress: Option, + /// Cursor for incremental consumption: sequence number of the next item + /// to be produced. `0` on reports from sources without sequencing. + pub next_seq: u64, + /// Oldest sequence still retained (`0` together with `next_seq` when + /// sequencing is unavailable). + pub min_seq: u64, +} + +/// Report from a live task, honoring the client's incremental cursor. +async fn active_task_report(task: &HealTask, since: Option) -> HealTaskReport { + let window = task.get_result_items_since(since).await; + HealTaskReport { + status: task.get_status().await, + result_items: window.items, + // The legacy flag stays set once anything was evicted; a lagging + // incremental cursor additionally marks this response truncated so + // the client knows to restart from `min_seq`. + result_items_truncated: task.result_items_truncated() || window.lagged, + progress: Some(task.get_progress().await), + next_seq: window.next_seq, + min_seq: window.min_seq, + } +} + +fn empty_task_report(status: HealTaskStatus) -> HealTaskReport { + HealTaskReport { + status, + result_items: Vec::new(), + result_items_truncated: false, + progress: None, + next_seq: 0, + min_seq: 0, + } +} + +fn completed_task_report(completed: &CompletedHealStatus, since: Option) -> HealTaskReport { + let mut lagged = false; + let result_items = match since { + None => completed.result_items.clone(), + Some(cursor) => { + if cursor + 1 < completed.min_seq { + lagged = true; + } + completed + .seqed_items + .iter() + .filter(|(seq, _)| *seq > cursor) + .map(|(_, item)| item.clone()) + .collect() + } + }; + HealTaskReport { + status: completed.status.clone(), + result_items, + result_items_truncated: completed.result_items_truncated || lagged, + progress: None, + next_seq: completed.next_seq, + min_seq: completed.min_seq, + } } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize, serde::Serialize)] @@ -531,6 +595,11 @@ impl PriorityHealQueue { self.dedup_keys.contains_key(&key) } + /// Iterate queued requests (used by the admin overlap check). + fn requests(&self) -> impl Iterator { + self.heap.iter().map(|item| &item.request) + } + fn contains_request_id(&self, request_id: &str) -> bool { self.heap.iter().any(|item| item.request.id == request_id) } @@ -689,6 +758,80 @@ fn recoverable_heal_retry_delay(retry_attempt: u32) -> Duration { } /// Heal config +/// HS-06 admin overlap policy. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum HealOverlapPolicy { + /// Default: overlapping admin starts merge into the existing task + /// (today's dedup semantics). + #[default] + Merge, + /// Return a typed already-running / overlapping-paths rejection like + /// madmin's ErrHealAlreadyRunning / ErrHealOverlappingPaths. + MinioError, +} + +/// Path view of a heal type for overlap comparison: a bucket plus a +/// prefix/object path inside it (`None` bucket = cluster-wide, overlaps +/// everything). +fn heal_type_path_view(heal_type: &HealType) -> (Option<&str>, &str) { + match heal_type { + HealType::Cluster => (None, ""), + HealType::Bucket { bucket } => (Some(bucket), ""), + HealType::Prefix { bucket, prefix } => (Some(bucket), prefix), + HealType::Object { bucket, object, .. } + | HealType::Metadata { bucket, object } + | HealType::ECDecode { bucket, object, .. } => (Some(bucket), object), + // MRF/MetaPath heal keys on a meta path; treat the whole set of + // buckets as one namespace so it only overlaps itself exactly. + HealType::MRF { meta_path } => (Some("\u{0}mrf"), meta_path), + // Erasure-set heal: the set id is the overlap dimension. + HealType::ErasureSet { set_disk_id, .. } => (Some("\u{0}set"), set_disk_id), + } +} + +/// How two heal paths relate for the admin overlap check (HS-06). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum OverlapVerdict { + /// Distinct targets: no conflict. + Disjoint, + /// Same target: an identical heal is already in flight. + SameTarget, + /// One target contains the other. + Overlapping, +} + +fn prefix_paths_overlap(a: &str, b: &str) -> OverlapVerdict { + if a == b { + return OverlapVerdict::SameTarget; + } + if a.is_empty() || b.is_empty() || a.starts_with(b) || b.starts_with(a) { + return OverlapVerdict::Overlapping; + } + OverlapVerdict::Disjoint +} + +fn heal_types_overlap(left: &HealType, right: &HealType) -> OverlapVerdict { + let (left_bucket, left_path) = heal_type_path_view(left); + let (right_bucket, right_path) = heal_type_path_view(right); + match (left_bucket, right_bucket) { + // Cluster-wide overlaps everything (but an exact cluster match is + // SameTarget). + (None, _) | (_, None) => { + if matches!(left, HealType::Cluster) && matches!(right, HealType::Cluster) { + OverlapVerdict::SameTarget + } else { + OverlapVerdict::Overlapping + } + } + (Some(lb), Some(rb)) => { + if lb != rb { + return OverlapVerdict::Disjoint; + } + prefix_paths_overlap(left_path, right_path) + } + } +} + #[derive(Debug, Clone)] pub struct HealConfig { /// Whether to enable auto heal @@ -709,6 +852,9 @@ pub struct HealConfig { pub low_priority_drop_when_full: bool, /// Whether notify-driven scheduler wakeups are enabled. pub event_driven_scheduler_enable: bool, + /// How admin heal starts behave on path overlap (HS-06): merge into the + /// existing task (default) or return a typed already-running rejection. + pub overlap_policy: HealOverlapPolicy, /// Whether per-set bulkhead scheduling is enabled. pub set_bulkhead_enable: bool, /// Whether erasure-set page parallelism is enabled. @@ -757,6 +903,14 @@ impl Default for HealConfig { rustfs_config::ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE, rustfs_config::DEFAULT_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE, ); + let overlap_policy = + match rustfs_utils::get_env_str(rustfs_config::ENV_HEAL_OVERLAP_POLICY, rustfs_config::DEFAULT_HEAL_OVERLAP_POLICY) + .to_lowercase() + .as_str() + { + "minio_error" => HealOverlapPolicy::MinioError, + _ => HealOverlapPolicy::Merge, + }; let set_bulkhead_enable = rustfs_utils::get_env_bool( rustfs_config::ENV_HEAL_SET_BULKHEAD_ENABLE, rustfs_config::DEFAULT_HEAL_SET_BULKHEAD_ENABLE, @@ -793,6 +947,7 @@ impl Default for HealConfig { low_priority_merge_enable, low_priority_drop_when_full, event_driven_scheduler_enable, + overlap_policy, set_bulkhead_enable, page_parallel_enable, mainline_throttle_enable, @@ -1759,6 +1914,50 @@ impl HealManager { request: HealRequest, preserve_alias: bool, ) -> Result { + // HS-06 forceStart semantics (admin only): MinIO stops the old task + // first and then starts the new one. Cancel any active admin task + // overlapping this request's path before entering admission, so the + // fresh task is never merged into the one being replaced. + if request.source == HealRequestSource::Admin && request.force_start { + let overlapping: Vec = { + let active_heals = self.active_heals.lock().await; + active_heals + .iter() + .filter(|(task_id, task)| { + task.source == HealRequestSource::Admin + && heal_types_overlap(&request.heal_type, &task.heal_type) != OverlapVerdict::Disjoint + && *task_id != &request.id + }) + .map(|(task_id, _)| task_id.clone()) + .collect() + }; + for task_id in overlapping { + match self.cancel_task(&task_id).await { + Ok(_) => info!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_QUEUE_ADMISSION, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + request_id = %request.id, + cancelled_task_id = %task_id, + result = "force_start_cancelled_overlap", + "Admin forceStart cancelled an overlapping heal task" + ), + Err(err) => warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_QUEUE_ADMISSION, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + request_id = %request.id, + cancelled_task_id = %task_id, + error = %err, + result = "force_start_cancel_failed", + "Admin forceStart failed to cancel an overlapping heal task" + ), + } + } + } + let config = self.config.read().await; let dedup_key = PriorityHealQueue::make_dedup_key(&request); @@ -1781,7 +1980,15 @@ impl HealManager { .or_else(|| retrying_heal_for_dedup_key(&retrying_heals, &dedup_key).map(|(task_id, _)| (task_id, "retrying"))) }); if let Some((merged_task_id, duplicate_state)) = duplicate.flatten() { - let admission = Self::duplicate_admission_for_request(&request, &config); + // HS-06: under the minio_error overlap policy an exact duplicate + // admin start reports the typed AlreadyRunning rejection instead + // of the silent merge (MinIO's ErrHealAlreadyRunning). + let admission = + if request.source == HealRequestSource::Admin && config.overlap_policy == HealOverlapPolicy::MinioError { + HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning) + } else { + Self::duplicate_admission_for_request(&request, &config) + }; drop(retrying_heals); drop(queue); drop(active_heals); @@ -1827,6 +2034,62 @@ impl HealManager { }); } + // HS-06 typed overlap rejection (admin only, minio_error policy): + // paths containing or contained by an active/queued task reject with + // AlreadyRunning / OverlappingPaths instead of merging. Exact + // duplicates already merged above; scanner/autoheal/read-repair + // sources never take this path. + if request.source == HealRequestSource::Admin && config.overlap_policy == HealOverlapPolicy::MinioError { + let mut rejection = None; + for (task_id, task) in active_heals.iter() { + match heal_types_overlap(&request.heal_type, &task.heal_type) { + OverlapVerdict::SameTarget => { + rejection = Some((HealAdmissionDropReason::AlreadyRunning, task_id.clone())); + break; + } + OverlapVerdict::Overlapping => { + rejection = Some((HealAdmissionDropReason::OverlappingPaths, task_id.clone())); + } + OverlapVerdict::Disjoint => {} + } + } + if rejection.is_none() { + for queued in queue.requests() { + match heal_types_overlap(&request.heal_type, &queued.heal_type) { + OverlapVerdict::SameTarget => { + rejection = Some((HealAdmissionDropReason::AlreadyRunning, queued.id.clone())); + break; + } + OverlapVerdict::Overlapping => { + rejection = Some((HealAdmissionDropReason::OverlappingPaths, queued.id.clone())); + } + OverlapVerdict::Disjoint => {} + } + } + } + if let Some((reason, overlap_task_id)) = rejection { + drop(retrying_heals); + drop(queue); + drop(active_heals); + Self::record_admission_metric(request.source, HealAdmissionResult::Dropped(reason), "overlap_rejected"); + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_QUEUE_ADMISSION, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + request_id = %request.id, + overlap_task_id = %overlap_task_id, + reason = reason.as_str(), + result = "overlap_rejected", + "Admin heal start rejected by overlap policy" + ); + return Ok(HealAdmissionReceipt { + result: HealAdmissionResult::Dropped(reason), + task_id: overlap_task_id, + }); + } + } + let mut task_id = request.id.clone(); let admission = Self::admit_request_to_queue(&mut queue, request, &config, "submit"); if admission == HealAdmissionResult::Merged @@ -1899,28 +2162,25 @@ impl HealManager { } pub async fn get_task_report(&self, task_id: &str) -> Result { + self.get_task_report_since(task_id, None).await + } + + /// Incremental variant of [`Self::get_task_report`] (HS-06): `since` is + /// the client's last seen sequence number; `None` keeps the legacy + /// full-snapshot semantics. + pub async fn get_task_report_since(&self, task_id: &str, since: Option) -> Result { let canonical_task_id = self.canonical_task_id(task_id).await; { let active_heals = self.active_heals.lock().await; if let Some(task) = active_heals.get(&canonical_task_id) { - return Ok(HealTaskReport { - status: task.get_status().await, - result_items: task.get_result_items().await, - result_items_truncated: task.result_items_truncated(), - progress: Some(task.get_progress().await), - }); + return Ok(active_task_report(task, since).await); } } { let retrying_heals = self.retrying_heals.lock().await; if let Some(retrying) = retrying_heals.get(&canonical_task_id) { - return Ok(HealTaskReport { - status: retrying.status(), - result_items: Vec::new(), - result_items_truncated: false, - progress: None, - }); + return Ok(empty_task_report(retrying.status())); } } @@ -1930,36 +2190,21 @@ impl HealManager { if let Some(completed) = completed_heals.get(&canonical_task_id) && completed_status_is_retrying(&completed.status) { - return Ok(HealTaskReport { - status: completed.status.clone(), - result_items: completed.result_items.clone(), - result_items_truncated: completed.result_items_truncated, - progress: None, - }); + return Ok(completed_task_report(completed, since)); } } { let queue = self.heal_queue.lock().await; if queue.contains_request_id(&canonical_task_id) { - return Ok(HealTaskReport { - status: HealTaskStatus::Pending, - result_items: Vec::new(), - result_items_truncated: false, - progress: None, - }); + return Ok(empty_task_report(HealTaskStatus::Pending)); } } let mut completed_heals = self.completed_heals.lock().await; prune_completed_heal_statuses(&mut completed_heals); if let Some(completed) = completed_heals.get(&canonical_task_id) { - return Ok(HealTaskReport { - status: completed.status.clone(), - result_items: completed.result_items.clone(), - result_items_truncated: completed.result_items_truncated, - progress: None, - }); + return Ok(completed_task_report(completed, since)); } Err(Error::TaskNotFound { @@ -1968,18 +2213,23 @@ impl HealManager { } pub async fn get_task_report_for_path(&self, heal_path: &str, task_id: &str) -> Result { + self.get_task_report_for_path_since(heal_path, task_id, None).await + } + + /// Incremental variant of [`Self::get_task_report_for_path`] (HS-06). + pub async fn get_task_report_for_path_since( + &self, + heal_path: &str, + task_id: &str, + since: Option, + ) -> Result { let canonical_task_id = self.canonical_task_id(task_id).await; { let active_heals = self.active_heals.lock().await; if let Some(task) = active_heals.get(&canonical_task_id) && heal_type_matches_path(&task.heal_type, heal_path) { - return Ok(HealTaskReport { - status: task.get_status().await, - result_items: task.get_result_items().await, - result_items_truncated: task.result_items_truncated(), - progress: Some(task.get_progress().await), - }); + return Ok(active_task_report(task, since).await); } } @@ -1988,12 +2238,7 @@ impl HealManager { if let Some(retrying) = retrying_heals.get(&canonical_task_id) && heal_type_matches_path(&retrying.request.heal_type, heal_path) { - return Ok(HealTaskReport { - status: retrying.status(), - result_items: Vec::new(), - result_items_truncated: false, - progress: None, - }); + return Ok(empty_task_report(retrying.status())); } } @@ -2004,24 +2249,14 @@ impl HealManager { && heal_type_matches_path(&completed.heal_type, heal_path) && completed_status_is_retrying(&completed.status) { - return Ok(HealTaskReport { - status: completed.status.clone(), - result_items: completed.result_items.clone(), - result_items_truncated: completed.result_items_truncated, - progress: None, - }); + return Ok(completed_task_report(completed, since)); } } { let queue = self.heal_queue.lock().await; if queue.contains_request_id_matching_path(&canonical_task_id, heal_path) { - return Ok(HealTaskReport { - status: HealTaskStatus::Pending, - result_items: Vec::new(), - result_items_truncated: false, - progress: None, - }); + return Ok(empty_task_report(HealTaskStatus::Pending)); } } @@ -2031,12 +2266,7 @@ impl HealManager { if let Some(completed) = completed_heals.get(&canonical_task_id) && heal_type_matches_path(&completed.heal_type, heal_path) { - return Ok(HealTaskReport { - status: completed.status.clone(), - result_items: completed.result_items.clone(), - result_items_truncated: completed.result_items_truncated, - progress: None, - }); + return Ok(completed_task_report(completed, since)); } } @@ -3231,12 +3461,16 @@ impl HealManager { completed_task.get_status().await }; let completed_progress = completed_task.get_progress().await; + let final_window = completed_task.get_result_items_since(None).await; let completed_status_entry = CompletedHealStatus { heal_type: completed_task.heal_type.clone(), status: completed_status.clone(), - result_items: completed_task.get_result_items().await, + result_items: final_window.items.clone(), result_items_truncated: completed_task.result_items_truncated(), completed_at: SystemTime::now(), + seqed_items: completed_task.get_seqed_result_items().await, + next_seq: final_window.next_seq, + min_seq: final_window.min_seq, }; let mut completed_heals_guard = completed_heals_clone.lock().await; prune_completed_heal_statuses(&mut completed_heals_guard); @@ -5008,6 +5242,9 @@ mod tests { }, result_items: Vec::new(), result_items_truncated: false, + seqed_items: Vec::new(), + next_seq: 0, + min_seq: 0, completed_at: SystemTime::now(), }, ); @@ -5289,6 +5526,136 @@ mod tests { assert_eq!(snapshot.queued_by_source.internal, 0); } + // HS-06 (backlog#1870): overlap policy + forceStart semantics. + fn manager_with_policy(policy: HealOverlapPolicy) -> HealManager { + let storage: Arc = Arc::new(MockStorage); + HealManager::new( + storage, + Some(HealConfig { + overlap_policy: policy, + ..Default::default() + }), + ) + } + + fn admin_prefix_request(bucket: &str, prefix: &str) -> HealRequest { + let mut request = HealRequest::new( + HealType::Prefix { + bucket: bucket.to_string(), + prefix: prefix.to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + request.source = HealRequestSource::Admin; + request + } + + async fn insert_active_task(manager: &HealManager, request: HealRequest) -> String { + let task = Arc::new(HealTask::from_request(request, manager.storage.clone())); + let task_id = task.id.clone(); + manager.active_heals.lock().await.insert(task_id.clone(), task); + task_id + } + + #[tokio::test] + async fn overlap_policy_minio_error_rejects_same_and_containing_paths() { + let manager = manager_with_policy(HealOverlapPolicy::MinioError); + insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await; + + // Same target: typed AlreadyRunning. + let same = manager + .submit_heal_request(admin_prefix_request("bucket-a", "logs/")) + .await + .expect("admission must decide"); + assert_eq!( + same, + HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning), + "an identical target must reject with already-running" + ); + + // Contained path: typed OverlappingPaths. + let nested = manager + .submit_heal_request(admin_prefix_request("bucket-a", "logs/app/")) + .await + .expect("admission must decide"); + assert_eq!( + nested, + HealAdmissionResult::Dropped(HealAdmissionDropReason::OverlappingPaths), + "a path inside the active task's path must reject with overlapping-paths" + ); + + // Containing path (bucket-wide vs nested active): also overlapping. + let wide = manager + .submit_heal_request(admin_prefix_request("bucket-a", "")) + .await + .expect("admission must decide"); + assert_eq!( + wide, + HealAdmissionResult::Dropped(HealAdmissionDropReason::OverlappingPaths), + "a bucket-wide start overlapping a nested active heal must reject" + ); + + // Disjoint bucket: unaffected. + let disjoint = manager + .submit_heal_request(admin_prefix_request("bucket-b", "logs/")) + .await + .expect("admission must decide"); + assert_eq!(disjoint, HealAdmissionResult::Accepted); + } + + #[tokio::test] + async fn overlap_policy_default_merge_keeps_today_semantics() { + let manager = manager_with_policy(HealOverlapPolicy::Merge); + insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await; + + // Different-dedup-key overlap still merges under the default policy: + // the nested path dedups to its own key but nothing rejects it. + let nested = manager + .submit_heal_request(admin_prefix_request("bucket-a", "logs/app/")) + .await + .expect("admission must decide"); + assert_eq!(nested, HealAdmissionResult::Accepted, "default policy must not reject overlaps"); + + // Non-admin sources never get overlap rejections even under minio_error. + let manager = manager_with_policy(HealOverlapPolicy::MinioError); + insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await; + let mut scanner_request = admin_prefix_request("bucket-a", "logs/app/"); + scanner_request.source = HealRequestSource::Scanner; + let admitted = manager + .submit_heal_request(scanner_request) + .await + .expect("admission must decide"); + assert_eq!(admitted, HealAdmissionResult::Accepted, "scanner sources must never be overlap-rejected"); + } + + #[tokio::test] + async fn admin_force_start_cancels_overlapping_active_task_first() { + let manager = manager_with_policy(HealOverlapPolicy::Merge); + let old_id = insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await; + + let mut replacement = admin_prefix_request("bucket-a", "logs/"); + replacement.force_start = true; + let receipt = manager + .submit_heal_request_with_receipt(replacement) + .await + .expect("force-start submission must decide"); + + assert!(receipt.result.is_admitted(), "the new task must be admitted (Accepted or Merged)"); + let old_task_gone = { + let active_heals = manager.active_heals.lock().await; + !active_heals.contains_key(&old_id) + }; + assert!( + old_task_gone, + "the overlapping admin task must be cancelled (removed from the active table) before the new one starts" + ); + assert!( + matches!(manager.get_task_status(&old_id).await, Err(Error::TaskNotFound { .. })), + "a cancelled task must no longer resolve as an active heal" + ); + } + #[tokio::test] async fn test_operations_snapshot_counts_active_by_source_and_priority() { let storage: Arc = Arc::new(MockStorage); @@ -5591,6 +5958,9 @@ mod tests { status: HealTaskStatus::Completed, result_items: Vec::new(), result_items_truncated: false, + seqed_items: Vec::new(), + next_seq: 0, + min_seq: 0, completed_at: SystemTime::now(), }, ); @@ -5625,6 +5995,9 @@ mod tests { ..Default::default() }], result_items_truncated: true, + seqed_items: Vec::new(), + next_seq: 0, + min_seq: 0, completed_at: SystemTime::now(), }, ); diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index f6472eb65..2c0626d8f 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -32,7 +32,7 @@ use std::{ future::Future, sync::{ Arc, - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, }, time::{Duration, Instant, SystemTime}, }; @@ -351,6 +351,20 @@ impl HealRequest { } /// Heal task +/// Incremental view over a task's retained result items (HS-06). +/// +/// `next_seq` is the cursor a client should pass on its next poll; `min_seq` +/// is the oldest sequence still retained; `lagged` means the client's cursor +/// fell behind `min_seq` and items were skipped — the client should restart +/// from `min_seq`. +#[derive(Debug, Clone)] +pub struct HealResultWindow { + pub items: Vec, + pub next_seq: u64, + pub min_seq: u64, + pub lagged: bool, +} + pub struct HealTask { /// Task ID pub id: String, @@ -373,8 +387,16 @@ pub struct HealTask { pub status: Arc>, /// Progress tracking pub progress: Arc>, - /// Result items collected from storage heal calls. - pub result_items: Arc>>, + /// Result items collected from storage heal calls, each stamped with a + /// monotonically increasing sequence number for incremental consumption + /// (the client passes the last seen seq back and receives only newer + /// items; see `get_result_items_since`). + pub result_items: Arc>>, + /// Next sequence number to assign; starts at 1. + next_item_seq: Arc, + /// Sequence number of the oldest item still inside the retention window; + /// equals `next_item_seq` while the window is empty. + min_available_seq: Arc, result_items_truncated: Arc, batch_failure: Arc>>, batch_failure_recorded: Arc, @@ -426,6 +448,8 @@ impl HealTask { status: Arc::new(RwLock::new(HealTaskStatus::Pending)), progress: Arc::new(RwLock::new(HealProgress::new())), result_items: Arc::new(RwLock::new(Vec::new())), + next_item_seq: Arc::new(AtomicU64::new(1)), + min_available_seq: Arc::new(AtomicU64::new(1)), result_items_truncated: Arc::new(AtomicBool::new(false)), batch_failure: Arc::new(RwLock::new(None)), batch_failure_recorded: Arc::new(AtomicBool::new(false)), @@ -911,18 +935,63 @@ impl HealTask { } pub async fn get_result_items(&self) -> Vec { + self.result_items.read().await.iter().map(|(_, item)| item.clone()).collect() + } + + /// Sequence-stamped retained window, used when archiving a completed + /// task so incremental cursors survive the transition (HS-06). + pub async fn get_seqed_result_items(&self) -> Vec<(u64, HealResultItem)> { self.result_items.read().await.clone() } + /// Incremental result window (HS-06): `since = None` returns the full + /// retained window (legacy snapshot semantics); `since = Some(seq)` + /// returns only items stamped with a sequence greater than `seq`. + /// `lagged` warns that the caller's cursor fell behind the window start + /// and items were skipped (the response carries `min_seq` as the catch-up + /// cursor). + pub async fn get_result_items_since(&self, since: Option) -> HealResultWindow { + let result_items = self.result_items.read().await; + let next_seq = self.next_item_seq.load(Ordering::Relaxed); + let min_seq = self.min_available_seq.load(Ordering::Relaxed); + let mut lagged = false; + let items = match since { + None => result_items.iter().map(|(_, item)| item.clone()).collect::>(), + Some(cursor) => { + if cursor + 1 < min_seq { + lagged = true; + } + result_items + .iter() + .filter(|(seq, _)| *seq > cursor) + .map(|(_, item)| item.clone()) + .collect::>() + } + }; + HealResultWindow { + items, + next_seq, + min_seq, + lagged, + } + } + pub fn result_items_truncated(&self) -> bool { self.result_items_truncated.load(Ordering::Relaxed) } async fn record_result_item(&self, result: HealResultItem) { + let seq = self.next_item_seq.fetch_add(1, Ordering::Relaxed); let mut result_items = self.result_items.write().await; if result_items.len() < MAX_RETAINED_HEAL_RESULT_ITEMS { - result_items.push(result); + result_items.push((seq, result)); } else { + // Slide the window: the oldest item leaves and the cursor for the + // oldest still-available item moves forward with it. + result_items.remove(0); + self.min_available_seq + .store(result_items.first().map_or(seq, |(oldest, _)| *oldest), Ordering::Relaxed); + result_items.push((seq, result)); self.result_items_truncated.store(true, Ordering::Relaxed); } } @@ -3880,6 +3949,69 @@ mod tests { assert!(task.result_items_truncated()); } + // HS-06 (backlog#1870): incremental result windows. + #[tokio::test] + async fn result_items_seq_is_monotonic_and_incremental_slices_work() { + let storage = Arc::new(MockStorage::default()); + let task = HealTask::from_request(HealRequest::bucket("bucket-a".to_string()), storage); + + for round in 0..5u64 { + let item = HealResultItem { + object_size: round as usize, + ..Default::default() + }; + task.record_result_item(item).await; + } + + let full = task.get_result_items_since(None).await; + assert_eq!(full.items.len(), 5, "None keeps the full-snapshot semantics"); + assert_eq!(full.next_seq, 6, "next_seq is one past the last assigned"); + assert_eq!(full.min_seq, 1, "nothing was evicted yet"); + assert!(!full.lagged); + + // Incremental: only items newer than the cursor. + let incremental = task.get_result_items_since(Some(3)).await; + assert_eq!( + incremental.items.iter().map(|item| item.object_size).collect::>(), + vec![3, 4], + "only sequences greater than the cursor are returned" + ); + assert_eq!(incremental.next_seq, 6); + + // A cursor at the head is not lagging. + assert!(!task.get_result_items_since(Some(0)).await.lagged); + } + + #[tokio::test] + async fn result_items_window_slide_moves_min_seq_and_flags_lagging_cursors() { + let storage = Arc::new(MockStorage::default()); + let task = HealTask::from_request(HealRequest::bucket("bucket-a".to_string()), storage); + + // Fill the window completely, then push two more items: seq 1 and 2 + // are evicted by the slide. + for _ in 0..(MAX_RETAINED_HEAL_RESULT_ITEMS + 2) { + task.record_result_item(HealResultItem::default()).await; + } + + let full = task.get_result_items_since(None).await; + assert_eq!(full.items.len(), MAX_RETAINED_HEAL_RESULT_ITEMS); + assert_eq!(full.min_seq, 3, "each evicted head item moved the oldest-available cursor"); + assert!(task.result_items_truncated()); + + // A client still polling from before the eviction is lagging. + let lagging = task.get_result_items_since(Some(0)).await; + assert!(lagging.lagged, "a cursor behind min_seq must be flagged"); + assert_eq!(lagging.min_seq, 3, "the response tells the client where to restart"); + + // A cursor inside the window is fine. + assert!(!task.get_result_items_since(Some(3)).await.lagged); + + // The lagging client restarts from min_seq and gets the full window. + let catch_up = task.get_result_items_since(Some(3)).await; + assert_eq!(catch_up.items.len(), MAX_RETAINED_HEAL_RESULT_ITEMS - 1); + assert!(!catch_up.lagged); + } + #[tokio::test] async fn test_recursive_bucket_heal_skips_object_dir_candidates() { let storage = Arc::new(MockStorage { diff --git a/crates/protos/src/heal_control.rs b/crates/protos/src/heal_control.rs index df89126d2..515c0db75 100644 --- a/crates/protos/src/heal_control.rs +++ b/crates/protos/src/heal_control.rs @@ -177,16 +177,38 @@ impl StartCommand { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "action", rename_all = "snake_case", deny_unknown_fields)] pub enum Command { - Start { request: StartCommand }, - Query { heal_path: String, client_token: String }, - Cancel { heal_path: String, client_token: String }, + Start { + request: StartCommand, + }, + Query { + heal_path: String, + client_token: String, + /// Incremental result cursor (HS-06): only items with a sequence + /// greater than this are returned. Absent = legacy full snapshot. + /// Optional + defaulted so older peers stay wire-compatible. + #[serde(default, skip_serializing_if = "Option::is_none")] + since_seq: Option, + }, + Cancel { + heal_path: String, + client_token: String, + }, } #[derive(Debug)] pub enum ExecutableCommand { - Start { request: HealChannelRequest }, - Query { heal_path: String, client_token: String }, - Cancel { heal_path: String, client_token: String }, + Start { + request: HealChannelRequest, + }, + Query { + heal_path: String, + client_token: String, + since_seq: Option, + }, + Cancel { + heal_path: String, + client_token: String, + }, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -227,8 +249,22 @@ impl Envelope { ) } - pub fn query(request_id: String, metadata: RequestMetadata, heal_path: String, client_token: String) -> Result { - Self::new(request_id, metadata, Command::Query { heal_path, client_token }) + pub fn query( + request_id: String, + metadata: RequestMetadata, + heal_path: String, + client_token: String, + since_seq: Option, + ) -> Result { + Self::new( + request_id, + metadata, + Command::Query { + heal_path, + client_token, + since_seq, + }, + ) } pub fn cancel( @@ -286,7 +322,15 @@ impl Envelope { Command::Start { request } => ExecutableCommand::Start { request: request.into_channel_request(self.request_id.clone())?, }, - Command::Query { heal_path, client_token } => ExecutableCommand::Query { heal_path, client_token }, + Command::Query { + heal_path, + client_token, + since_seq, + } => ExecutableCommand::Query { + heal_path, + client_token, + since_seq, + }, Command::Cancel { heal_path, client_token } => ExecutableCommand::Cancel { heal_path, client_token }, }; Ok((self.request_id, self.coordinator_epoch, command)) @@ -305,6 +349,11 @@ pub enum Admission { Full, DroppedQueueFull, DroppedPolicy, + /// HS-06: admin start rejected because the same target is already being + /// healed (RUSTFS_HEAL_OVERLAP_POLICY=minio_error only). + DroppedAlreadyRunning, + /// HS-06: admin start rejected because its path overlaps an active heal. + DroppedOverlappingPaths, } impl From for Admission { @@ -315,6 +364,8 @@ impl From for Admission { HealAdmissionResult::Full => Self::Full, HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull) => Self::DroppedQueueFull, HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped) => Self::DroppedPolicy, + HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning) => Self::DroppedAlreadyRunning, + HealAdmissionResult::Dropped(HealAdmissionDropReason::OverlappingPaths) => Self::DroppedOverlappingPaths, } } } @@ -331,6 +382,8 @@ impl Admission { Self::Full => HealAdmissionResult::Full, Self::DroppedQueueFull => HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull), Self::DroppedPolicy => HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped), + Self::DroppedAlreadyRunning => HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning), + Self::DroppedOverlappingPaths => HealAdmissionResult::Dropped(HealAdmissionDropReason::OverlappingPaths), } } } @@ -592,6 +645,7 @@ mod tests { metadata(2, 7), "bucket/prefix".to_string(), "token".to_string(), + None, ) .unwrap(); let cancel = Envelope::cancel( @@ -667,6 +721,7 @@ mod tests { RequestMetadata::new([0x11; 16], 1_700_000_000_000, 1_700_000_030_000, 9), "bucket/prefix".to_string(), "client-token".to_string(), + None, ) .unwrap(); let cancel = Envelope::cancel( @@ -749,7 +804,7 @@ mod tests { assert!(Envelope::start(test_request(request_id.clone()), metadata(0, 7)).is_err()); assert!(Envelope::start(test_request(request_id.clone()), metadata(1, 0)).is_err()); assert!(Envelope::start(test_request(request_id.clone()), RequestMetadata::new([1; 16], 1_000, 31_001, 7),).is_err()); - assert!(Envelope::query(request_id.clone(), metadata(1, 7), String::new(), String::new()).is_err()); + assert!(Envelope::query(request_id.clone(), metadata(1, 7), String::new(), String::new(), None).is_err()); assert!(Envelope::cancel(request_id.clone(), metadata(1, 7), String::new(), String::new()).is_ok()); let mut noncanonical_request = test_request(request_id.to_uppercase()); @@ -782,6 +837,7 @@ mod tests { metadata(1, 7), "x".repeat(ENVELOPE_MAX_SIZE), "token".to_string(), + None, ) .unwrap(); let error = super::encode_envelope(&oversized).unwrap_err(); diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 030c55fc9..5766385e8 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -1770,6 +1770,13 @@ impl FolderScanner { HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped) => { self.clear_pending_scanner_heal(kind, bucket, object, version_id); } + // Admin-only overlap rejections (HS-06); the scanner never sees + // them, but if it ever does, treat them as terminal like any + // other policy drop rather than endlessly retrying. + HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning) + | HealAdmissionResult::Dropped(HealAdmissionDropReason::OverlappingPaths) => { + self.clear_pending_scanner_heal(kind, bucket, object, version_id); + } } } diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 52e9fc152..3d98e966a 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -66,6 +66,9 @@ struct HealInitParams { client_token: String, force_start: bool, force_stop: bool, + /// Incremental result cursor (HS-06): only result items with a sequence + /// greater than this are returned; absent means full snapshot. + since_seq: Option, } fn extract_heal_init_params(body: &Bytes, uri: &Uri, params: Params<'_, '_>) -> S3Result { @@ -98,6 +101,16 @@ fn extract_heal_init_params(body: &Bytes, uri: &Uri, params: Params<'_, '_>) -> } hip.force_stop = parse_heal_query_bool(value.as_ref())?; } + "sinceSeq" => { + if !seen.insert("sinceSeq") { + return Err(s3_error!(InvalidArgument, "duplicate heal query parameter")); + } + hip.since_seq = Some( + value + .parse::() + .map_err(|_| s3_error!(InvalidArgument, "sinceSeq must be a non-negative integer"))?, + ); + } _ => return Err(s3_error!(InvalidArgument, "unknown heal query parameter")), } } @@ -978,7 +991,15 @@ fn reject_heal_admission(result: rustfs_common::heal_channel::HealAdmissionResul result.result_label(), result.reason_label() ), - HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped) => s3_error!( + // Overlap rejections (HS-06) share this arm: the s3s footprint + // ratchet forbids new s3_error! sites, and the typed reason is + // preserved through reason_label() ("already_running" / + // "overlapping_paths") so madmin-style clients can distinguish. + HealAdmissionResult::Dropped( + HealAdmissionDropReason::PolicyDropped + | HealAdmissionDropReason::AlreadyRunning + | HealAdmissionDropReason::OverlappingPaths, + ) => s3_error!( OperationAborted, "heal request not admitted: admission={}, reason={}", result.result_label(), @@ -1403,6 +1424,7 @@ impl Operation for HealHandler { new_heal_control_metadata(&route)?, heal_path, client_token.clone(), + hip.since_seq, ) .map_err(|err| s3_error!(InternalError, "encode heal control query failed: {err}"))?; let response = submit_cluster_heal_channel_command(context, route, envelope, &request_id, client_token).await?; diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index a39ca6c3a..352bfbd58 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -571,8 +571,12 @@ async fn execute_heal_control_envelope_with_manager( admission: receipt.result.into(), } } - rustfs_protos::heal_control::ExecutableCommand::Query { heal_path, client_token } => { - let response = timeout(remaining, processor.execute_query_request(heal_path, client_token)) + rustfs_protos::heal_control::ExecutableCommand::Query { + heal_path, + client_token, + since_seq, + } => { + let response = timeout(remaining, processor.execute_query_request_since(heal_path, client_token, since_seq)) .await .map_err(|_| Status::deadline_exceeded("heal control query expired before execution"))? .map_err(|_| Status::internal("heal control query failed"))?; @@ -2517,6 +2521,7 @@ mod tests { metadata(), "bucket/prefix".to_string(), canonical_token.clone(), + None, ) .unwrap(); let query_result = execute_heal_control_envelope_with_manager(query, coordinator_epoch, Some(Arc::clone(&manager))) @@ -2555,6 +2560,7 @@ mod tests { metadata(), "bucket/prefix".to_string(), canonical_token, + None, ) .unwrap(); let stopped_result = execute_heal_control_envelope_with_manager(stopped_query, coordinator_epoch, Some(manager)) diff --git a/scripts/check_s3s_footprint.sh b/scripts/check_s3s_footprint.sh index 56fb60d35..1ebd59f5b 100755 --- a/scripts/check_s3s_footprint.sh +++ b/scripts/check_s3s_footprint.sh @@ -25,7 +25,7 @@ cd "$(dirname "$0")/.." # Baselines verified on 2026-08-11. Lower-only; see header. # Excludes crates/e2e_test/ — test infrastructure legitimately uses s3s # to verify S3 behavior and does not widen the production s3s surface. -S3S_IMPORT_FILES_BASELINE=213 +S3S_IMPORT_FILES_BASELINE=211 S3_ERROR_LINES_BASELINE=1620 S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::' E2E_TEST_GLOB='--glob=!crates/e2e_test/**' From bd296eff9e69d19ae8895e371f443addd5671ec9 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 16:14:05 +0800 Subject: [PATCH 32/51] chore(io-core): drop eight zero-consumer modules (#6201) --- ARCHITECTURE.md | 8 +- Cargo.lock | 1 - crates/io-core/CHANGELOG.md | 3 + crates/io-core/Cargo.toml | 5 +- crates/io-core/README.md | 138 +-- crates/io-core/README_zh.md | 149 +--- crates/io-core/examples/scheduler_example.rs | 190 ---- crates/io-core/src/bufreader_optimizer.rs | 227 ----- crates/io-core/src/direct_io.rs | 332 ------- crates/io-core/src/io_priority_queue.rs | 381 -------- crates/io-core/src/lib.rs | 73 +- crates/io-core/src/progress.rs | 138 +++ crates/io-core/src/reader.rs | 412 --------- crates/io-core/src/scheduler.rs | 882 ------------------- crates/io-core/src/shared_memory.rs | 320 ------- crates/io-core/src/timeout_wrapper.rs | 501 ----------- crates/io-core/src/writer.rs | 443 ---------- 17 files changed, 195 insertions(+), 4008 deletions(-) delete mode 100644 crates/io-core/examples/scheduler_example.rs delete mode 100644 crates/io-core/src/bufreader_optimizer.rs delete mode 100644 crates/io-core/src/direct_io.rs delete mode 100644 crates/io-core/src/io_priority_queue.rs create mode 100644 crates/io-core/src/progress.rs delete mode 100644 crates/io-core/src/reader.rs delete mode 100644 crates/io-core/src/scheduler.rs delete mode 100644 crates/io-core/src/shared_memory.rs delete mode 100644 crates/io-core/src/timeout_wrapper.rs delete mode 100644 crates/io-core/src/writer.rs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 809d0a3f5..4475c2997 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -31,7 +31,7 @@ HTTP request → storage/ecfs (erasure coding, encryption, checksums) → ecstore (disk pool selection, data distribution) → rio (reader pipeline: encrypt → compress → hash → write) - → io-core (zero-copy I/O, buffer pool, direct I/O) + → io-core (buffer pool, storage profiling, admission control) → local disk / remote disk via RPC ``` @@ -55,7 +55,7 @@ rustfs/ # Workspace root (virtual manifest) ├── crates/ # library crates (authoritative list: Cargo.toml [workspace].members) │ ├── ecstore/ # Erasure-coded storage engine │ ├── rio/ # Reader I/O pipeline (encrypt, compress, hash) -│ ├── io-core/ # Zero-copy I/O, scheduling, buffer pool +│ ├── io-core/ # Buffer pool, storage profiling, admission control │ ├── io-metrics/ # I/O metrics collection │ ├── common/ # Shared runtime state, globals, data usage types │ ├── config/ # Configuration types and parsing @@ -302,7 +302,7 @@ The binary (`main.rs`) boots in this order: │ │ │ ┌─────▼──────┐ ┌──────▼──────┐ ┌──────▼──────┐ │ ecstore │ │ rio │ │ io-core │ - │ (core) │ │ (readers) │ │ (zero-copy) │ + │ (core) │ │ (readers) │ │ (buffers) │ └─────┬──────┘ └─────────────┘ └─────────────┘ │ ┌─────┬──┼──┬─────┬──────┐ @@ -314,7 +314,7 @@ The binary (`main.rs`) boots in this order: - **"Where does S3 PutObject go?"** `server/` routes → `app/object_usecase` validates → `storage/ecfs` encodes → - `ecstore` distributes → `rio` encrypts/compresses → `io-core` writes + `ecstore` distributes → `rio` encrypts/compresses → `io-core` supplies buffers - **"Where are bucket policies enforced?"** `app/bucket_usecase` calls into `crates/policy/` diff --git a/Cargo.lock b/Cargo.lock index 83749d574..0fb09e78e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9613,7 +9613,6 @@ version = "1.0.0-rc.2" dependencies = [ "bytes", "hotpath", - "memmap2", "rustfs-io-metrics", "thiserror 2.0.20", "tokio", diff --git a/crates/io-core/CHANGELOG.md b/crates/io-core/CHANGELOG.md index 2ed1235c4..b60a494a1 100644 --- a/crates/io-core/CHANGELOG.md +++ b/crates/io-core/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed +#### rustfs-io-core +- **Zero-consumer modules** (added in 0.0.5): `reader`, `writer`, `bufreader_optimizer`, `shared_memory`, `direct_io`, `timeout_wrapper`, `io_priority_queue`, and `scheduler` had no caller in the workspace and were removed (rustfs/backlog#1824). The scheduling algorithm and the request timeout wrapper that RustFS actually runs live in `rustfs/src/storage/`; this crate keeps the config shapes they project into. `OperationProgress` moved to the new `progress` module and is still exported as `rustfs_io_core::OperationProgress`. + #### rustfs-io-metrics - **Unified configuration** (added in 0.0.5): the zero-consumer `IoConfig`, `CacheSettings`, `IoSchedulerSettings`, `BackpressureSettings`, `TimeoutSettings`, `DeadlockDetectionSettings` types and their `DEFAULT_*` constants were removed (rustfs/rustfs#6008); rustfs-io-core's `IoSchedulerConfig`/`BackpressureConfig` remain the canonical configuration types. diff --git a/crates/io-core/Cargo.toml b/crates/io-core/Cargo.toml index 9c309633b..f3118c19a 100644 --- a/crates/io-core/Cargo.toml +++ b/crates/io-core/Cargo.toml @@ -20,8 +20,8 @@ license.workspace = true repository.workspace = true rust-version.workspace = true homepage.workspace = true -description = "Buffered I/O reader and writer implementations for RustFS (mmap-then-copy, aligned pread)" -keywords = ["io", "reader", "writer", "rustfs", "mmap"] +description = "Shared I/O primitives for RustFS (buffer pool, storage profiling, backpressure, deadlock detection)" +keywords = ["io", "buffer", "pool", "rustfs", "backpressure"] categories = ["development-tools", "filesystem"] [lints] @@ -38,7 +38,6 @@ hotpath.workspace = true bytes = { workspace = true, features = ["serde"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["io-util", "fs", "sync", "rt-multi-thread"] } -memmap2 = { workspace = true } rustfs-io-metrics = { workspace = true } tracing = { workspace = true } diff --git a/crates/io-core/README.md b/crates/io-core/README.md index ee5f7dd50..a372dab1b 100644 --- a/crates/io-core/README.md +++ b/crates/io-core/README.md @@ -23,67 +23,20 @@ ## Overview -**rustfs-io-core** is the core I/O scheduling module for [RustFS](https://rustfs.com), a distributed object storage system. It provides: +**rustfs-io-core** holds the shared I/O primitives for [RustFS](https://rustfs.com), a distributed object storage system. It provides: -- **I/O Scheduler**: Adaptive buffer size calculation and load management -- **Priority Queue**: Request priority scheduling with starvation prevention +- **Buffer Pool**: Tiered `BytesPool` for buffer reuse +- **Storage Profiling**: Storage-media and access-pattern model (`io_profile`) +- **Scheduler Configuration**: The `IoSchedulerConfig` / `IoPriorityQueueConfig` shapes the storage layer projects into - **Backpressure Control**: System overload protection with graceful degradation - **Deadlock Detection**: Wait-for graph based deadlock detection algorithm - **Lock Optimizer**: Adaptive spin lock optimization -- **Timeout Wrapper**: Dynamic timeout calculation and operation progress tracking +- **Progress Tracking**: Byte progress and staleness for long-running operations + +The scheduling algorithm itself lives in `rustfs/src/storage/concurrency/io_schedule.rs`; this crate carries the configuration shapes it projects into, not a second implementation. ## Features -### I/O Scheduler - -Adaptive I/O scheduling with dynamic buffer size calculation based on file size, access pattern, and system load: - -```rust -use rustfs_io_core::{IoScheduler, IoSchedulerConfig, IoLoadLevel}; -use rustfs_io_core::io_profile::{StorageMedia, AccessPattern}; - -// Create scheduler -let config = IoSchedulerConfig { - max_concurrent_reads: 64, - base_buffer_size: 64 * 1024, // 64 KB - max_buffer_size: 1024 * 1024, // 1 MB - ..Default::default() -}; -let scheduler = IoScheduler::new(config); - -// Calculate optimal buffer size -let buffer_size = calculate_optimal_buffer_size( - 10 * 1024 * 1024, // 10 MB file - 64 * 1024, // base buffer - true, // sequential access - 4, // concurrent requests - StorageMedia::Ssd, - IoLoadLevel::Low, -); -``` - -### Priority Queue - -Priority queue with starvation prevention: - -```rust -use rustfs_io_core::{IoPriorityQueue, IoPriority, IoQueueStatus}; - -let queue = IoPriorityQueue::<()>::new(100); - -// Enqueue request -let request_id = queue.enqueue(IoPriority::High, (), 1024); - -// Dequeue request -if let Some((priority, data)) = queue.dequeue() { - println!("Processing priority {:?} request", priority); -} - -// Check queue status -let status = queue.status(); -println!("High priority waiting: {}", status.high_priority_waiting); -``` - ### Backpressure Control System overload protection: @@ -148,71 +101,23 @@ let stats = optimizer.stats(); println!("Locks acquired: {}", stats.total_acquired()); ``` -### Timeout Wrapper +### Progress Tracking -Dynamic timeout calculation: +Byte progress and staleness for long-running operations: ```rust -use rustfs_io_core::{RequestTimeoutWrapper, TimeoutConfig}; +use rustfs_io_core::OperationProgress; use std::time::Duration; -let config = TimeoutConfig { - base_timeout: Duration::from_secs(5), - timeout_per_mb: Duration::from_millis(100), - max_timeout: Duration::from_secs(300), - ..Default::default() -}; -let wrapper = RequestTimeoutWrapper::new(config); +let progress = OperationProgress::new(Some(1000), Duration::from_secs(5)); -// Calculate operation timeout -let timeout = wrapper.calculate_timeout(10 * 1024 * 1024); // 10 MB -``` - -## Buffer Size Calculation - -Multiple buffer size calculation functions are provided: - -```rust -use rustfs_io_core::{ - get_concurrency_aware_buffer_size, - get_advanced_buffer_size, - get_buffer_size_for_media, - calculate_optimal_buffer_size, - KI_B, MI_B, -}; -use rustfs_io_core::io_profile::StorageMedia; - -// Basic calculation -let size1 = get_concurrency_aware_buffer_size(1024 * 1024, 64 * 1024); - -// Advanced calculation (considering access pattern) -let size2 = get_advanced_buffer_size(10 * 1024 * 1024, 64 * 1024, true); - -// Media type optimization -let size3 = get_buffer_size_for_media(64 * 1024, StorageMedia::Ssd); - -// Comprehensive calculation -let size4 = calculate_optimal_buffer_size( - 100 * 1024 * 1024, // 100 MB file - 64 * 1024, // base buffer - true, // sequential access - 4, // concurrent requests - StorageMedia::Nvme, - IoLoadLevel::Low, -); +progress.update(500); +assert_eq!(progress.progress_percent(), Some(50.0)); +assert!(!progress.is_stale()); ``` ## Configuration -### Environment Variables - -| Variable | Description | Default | -|----------|-------------|---------| -| `RUSTFS_MAX_CONCURRENT_READS` | Max concurrent reads | 64 | -| `RUSTFS_BASE_BUFFER_SIZE` | Base buffer size | 65536 | -| `RUSTFS_MAX_BUFFER_SIZE` | Max buffer size | 1048576 | -| `RUSTFS_IO_TIMEOUT_SECS` | I/O timeout seconds | 30 | - ### Code Configuration ```rust @@ -240,12 +145,11 @@ rustfs-io-core/ ├── src/ │ ├── lib.rs # Module entry │ ├── config.rs # Configuration types -│ ├── scheduler.rs # I/O scheduler -│ ├── io_priority_queue.rs # Priority queue +│ ├── pool.rs # Tiered buffer pool │ ├── backpressure.rs # Backpressure control │ ├── deadlock_detector.rs # Deadlock detection │ ├── lock_optimizer.rs # Lock optimization -│ ├── timeout_wrapper.rs # Timeout wrapper +│ ├── progress.rs # Operation progress tracking │ └── io_profile.rs # I/O profile └── Cargo.toml ``` @@ -254,21 +158,15 @@ rustfs-io-core/ ```bash # Run all tests -cargo test --package rustfs-io-core +cargo nextest run --package rustfs-io-core # Run specific tests -cargo test --package rustfs-io-core --lib scheduler - -# Run benchmarks -cargo bench --package rustfs-io-core +cargo nextest run --package rustfs-io-core -E 'test(backpressure)' ``` ## Documentation - [API Documentation](https://docs.rs/rustfs-io-core) -- [I/O Scheduler Design](./docs/scheduler-design.md) -- [Backpressure Control Design](./docs/backpressure-design.md) -- [Deadlock Detection Algorithm](./docs/deadlock-detection.md) ## Related Modules diff --git a/crates/io-core/README_zh.md b/crates/io-core/README_zh.md index 7471d86c5..9840c1ecd 100644 --- a/crates/io-core/README_zh.md +++ b/crates/io-core/README_zh.md @@ -23,71 +23,20 @@ ## 📖 概述 -**rustfs-io-core** 是 [RustFS](https://rustfs.com) 分布式对象存储系统的核心 I/O 调度模块。它提供了： +**rustfs-io-core** 是 [RustFS](https://rustfs.com) 分布式对象存储系统的共享 I/O 基础组件。它提供了： -- **I/O 调度器**：自适应缓冲区大小计算和负载管理 -- **优先级队列**：支持饥饿预防的请求优先级调度 +- **缓冲池**：分级复用的 `BytesPool` +- **存储画像**：存储介质与访问模式模型（`io_profile`） +- **调度配置**：存储层投影使用的 `IoSchedulerConfig` / `IoPriorityQueueConfig` - **背压控制**：系统过载保护和优雅降级 - **死锁检测**：基于等待图的死锁检测算法 - **锁优化**：自适应自旋锁优化 -- **超时包装器**：动态超时计算和操作进度追踪 +- **进度追踪**：长耗时操作的字节进度与停滞判定 + +调度算法本身位于 `rustfs/src/storage/concurrency/io_schedule.rs`；本 crate 只承载它投影使用的配置形状，不是第二套实现。 ## ✨ 核心功能 -### I/O 调度器 (IoScheduler) - -自适应 I/O 调度，根据文件大小、访问模式和系统负载动态调整缓冲区大小： - -```rust -use rustfs_io_core::{IoScheduler, IoSchedulerConfig, IoLoadLevel}; -use rustfs_io_core::io_profile::{StorageMedia, AccessPattern}; - -// 创建调度器 -let config = IoSchedulerConfig { - max_concurrent_reads: 64, - base_buffer_size: 64 * 1024, // 64 KB - max_buffer_size: 1024 * 1024, // 1 MB - ..Default::default() -}; -let scheduler = IoScheduler::new(config); - -// 计算最优缓冲区大小 -let buffer_size = scheduler.calculate_buffer_size( - 10 * 1024 * 1024, // 10 MB 文件 - true, // 顺序访问 - StorageMedia::Ssd, - IoLoadLevel::Low, -); -println!("缓冲区大小: {} bytes", buffer_size); -``` - -### 优先级队列 (IoPriorityQueue) - -支持饥饿预防的优先级队列： - -```rust -use rustfs_io_core::{IoPriorityQueue, IoPriority, IoQueueStatus}; - -let queue = IoPriorityQueue::<()>::new(100); - -// 入队请求 -let request_id = queue.enqueue( - IoPriority::High, - (), // 请求数据 - 1024, // 请求大小 -); - -// 出队请求 -if let Some((priority, data)) = queue.dequeue() { - println!("处理优先级 {:?} 的请求", priority); -} - -// 检查队列状态 -let status = queue.status(); -println!("高优先级等待: {}", status.high_priority_waiting); -println!("低优先级等待: {}", status.low_priority_waiting); -``` - ### 背压控制 (BackpressureMonitor) 系统过载保护： @@ -165,78 +114,23 @@ let stats = optimizer.stats(); println!("获取锁次数: {}", stats.locks_acquired.load(std::sync::atomic::Ordering::Relaxed)); ``` -### 超时包装器 (RequestTimeoutWrapper) +### 进度追踪 (OperationProgress) -动态超时计算： +长耗时操作的字节进度与停滞判定： ```rust -use rustfs_io_core::{RequestTimeoutWrapper, TimeoutConfig}; +use rustfs_io_core::OperationProgress; use std::time::Duration; -let config = TimeoutConfig { - base_timeout: Duration::from_secs(5), - timeout_per_mb: Duration::from_millis(100), - max_timeout: Duration::from_secs(300), - ..Default::default() -}; -let wrapper = RequestTimeoutWrapper::new(config); +let progress = OperationProgress::new(Some(1000), Duration::from_secs(5)); -// 计算操作超时 -let timeout = wrapper.calculate_timeout(10 * 1024 * 1024); // 10 MB -println!("超时时间: {:?}", timeout); - -// 执行带超时的操作 -let result = wrapper.execute_with_timeout(async { - // 异步操作 - Ok::<_, std::io::Error>(()) -}, timeout).await; -``` - -## 📊 缓冲区大小计算 - -模块提供了多种缓冲区大小计算函数： - -```rust -use rustfs_io_core::{ - get_concurrency_aware_buffer_size, - get_advanced_buffer_size, - get_buffer_size_for_media, - calculate_optimal_buffer_size, - KI_B, MI_B, -}; -use rustfs_io_core::io_profile::StorageMedia; - -// 基础计算 -let size1 = get_concurrency_aware_buffer_size(1024 * 1024, 64 * 1024); - -// 高级计算（考虑访问模式） -let size2 = get_advanced_buffer_size(10 * 1024 * 1024, 64 * 1024, true); - -// 媒体类型优化 -let size3 = get_buffer_size_for_media(64 * 1024, StorageMedia::Ssd); - -// 综合计算 -let size4 = calculate_optimal_buffer_size( - 100 * 1024 * 1024, // 100 MB 文件 - 64 * 1024, // 基础缓冲区 - true, // 顺序访问 - 4, // 并发请求数 - StorageMedia::Nvme, - IoLoadLevel::Low, -); +progress.update(500); +assert_eq!(progress.progress_percent(), Some(50.0)); +assert!(!progress.is_stale()); ``` ## 🔧 配置 -### 环境变量 - -| 变量名 | 描述 | 默认值 | -|--------|------|--------| -| `RUSTFS_MAX_CONCURRENT_READS` | 最大并发读数 | 64 | -| `RUSTFS_BASE_BUFFER_SIZE` | 基础缓冲区大小 | 65536 | -| `RUSTFS_MAX_BUFFER_SIZE` | 最大缓冲区大小 | 1048576 | -| `RUSTFS_IO_TIMEOUT_SECS` | I/O 超时秒数 | 30 | - ### 代码配置 ```rust @@ -264,12 +158,11 @@ rustfs-io-core/ ├── src/ │ ├── lib.rs # 模块入口 │ ├── config.rs # 配置类型 -│ ├── scheduler.rs # I/O 调度器 -│ ├── io_priority_queue.rs # 优先级队列 +│ ├── pool.rs # 分级缓冲池 │ ├── backpressure.rs # 背压控制 │ ├── deadlock_detector.rs # 死锁检测 │ ├── lock_optimizer.rs # 锁优化 -│ ├── timeout_wrapper.rs # 超时包装器 +│ ├── progress.rs # 操作进度追踪 │ └── io_profile.rs # I/O 配置文件 └── Cargo.toml ``` @@ -278,21 +171,15 @@ rustfs-io-core/ ```bash # 运行所有测试 -cargo test --package rustfs-io-core +cargo nextest run --package rustfs-io-core # 运行特定测试 -cargo test --package rustfs-io-core --lib scheduler - -# 运行基准测试 -cargo bench --package rustfs-io-core +cargo nextest run --package rustfs-io-core -E 'test(backpressure)' ``` ## 📚 文档 - [API 文档](https://docs.rs/rustfs-io-core) -- [I/O 调度器设计](./docs/scheduler-design.md) -- [背压控制原理](./docs/backpressure-design.md) -- [死锁检测算法](./docs/deadlock-detection.md) ## 🔗 相关模块 diff --git a/crates/io-core/examples/scheduler_example.rs b/crates/io-core/examples/scheduler_example.rs deleted file mode 100644 index 00fa560ae..000000000 --- a/crates/io-core/examples/scheduler_example.rs +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Example demonstrating I/O scheduler usage. - -use rustfs_io_core::io_profile::StorageMedia; -use rustfs_io_core::{ - BackpressureMonitor, BackpressureState, DeadlockDetector, IoLoadLevel, IoScheduler, IoSchedulerConfig, KI_B, LockOptimizer, - LockType, MI_B, calculate_optimal_buffer_size, get_buffer_size_for_media, -}; -use std::time::Duration; - -fn main() { - println!("=== rustfs-io-core Example ===\n"); - - // 1. I/O scheduler example - io_scheduler_example(); - - // 2. Buffer size calculation example - buffer_size_example(); - - // 3. Backpressure control example - backpressure_example(); - - // 4. Deadlock detection example - deadlock_detection_example(); - - // 5. Lock optimizer example - lock_optimizer_example(); -} - -fn io_scheduler_example() { - println!("--- I/O Scheduler ---"); - - // Create scheduler with configuration - let config = IoSchedulerConfig { - max_concurrent_reads: 64, - base_buffer_size: 64 * KI_B, - max_buffer_size: MI_B, - ..Default::default() - }; - let scheduler = IoScheduler::new(config); - - println!(" Max concurrent reads: {}", scheduler.config().max_concurrent_reads); - println!(" Base buffer size: {} KB", scheduler.config().base_buffer_size / KI_B); - println!(" Max buffer size: {} KB", scheduler.config().max_buffer_size / KI_B); - - // Calculate buffer sizes for different scenarios - let scenarios = [ - ("Small file", 10 * KI_B as i64, true, StorageMedia::Ssd), - ("Medium file", MI_B as i64, true, StorageMedia::Ssd), - ("Large sequential", 100 * MI_B as i64, true, StorageMedia::Ssd), - ("Large random", 100 * MI_B as i64, false, StorageMedia::Ssd), - ("NVMe large", 100 * MI_B as i64, true, StorageMedia::Nvme), - ("HDD large", 100 * MI_B as i64, true, StorageMedia::Hdd), - ]; - - for (name, size, sequential, media) in scenarios { - let buffer = calculate_optimal_buffer_size(size, 64 * KI_B, sequential, 4, media, IoLoadLevel::Low); - println!(" {}: {} bytes ({} KB)", name, buffer, buffer / KI_B); - } - - println!(); -} - -fn buffer_size_example() { - println!("--- Buffer Size Calculation ---"); - - // Comprehensive calculation - let size1 = calculate_optimal_buffer_size(10 * MI_B as i64, 64 * KI_B, true, 4, StorageMedia::Ssd, IoLoadLevel::Low); - println!(" Comprehensive (10MB, sequential, SSD): {} KB", size1 / KI_B); - - // Media type optimization - let media_types = [ - StorageMedia::Nvme, - StorageMedia::Ssd, - StorageMedia::Hdd, - StorageMedia::Unknown, - ]; - for media in media_types { - let size = get_buffer_size_for_media(64 * KI_B, media); - println!(" {} optimized: {} KB", media.as_str(), size / KI_B); - } - - println!(); -} - -fn backpressure_example() { - println!("--- Backpressure Control ---"); - - let monitor = BackpressureMonitor::with_defaults(); - - // Check initial state - let state = monitor.state(); - let state_str = match state { - BackpressureState::Normal => "Normal", - BackpressureState::Warning => "Warning", - BackpressureState::Critical => "Critical", - }; - println!(" Initial state: {}", state_str); - - // Check if active - let is_active = monitor.is_active(); - println!(" Backpressure active: {}", is_active); - - // Try to acquire permit - if monitor.try_acquire() { - println!(" Successfully acquired permit"); - monitor.release(); - println!(" Released permit"); - } - - // View statistics - println!(" Total processed: {}", monitor.total_processed()); - println!(" Total rejected: {}", monitor.total_rejected()); - - println!(); -} - -fn deadlock_detection_example() { - println!("--- Deadlock Detection ---"); - - let detector = DeadlockDetector::with_defaults(); - - // Register locks - let mutex1 = detector.register_lock(LockType::Mutex); - let mutex2 = detector.register_lock(LockType::Mutex); - println!(" Registered locks: mutex1={}, mutex2={}", mutex1, mutex2); - - // Simulate normal operation - detector.record_acquire(mutex1, 1); // Thread 1 acquires mutex1 - detector.record_acquire(mutex2, 2); // Thread 2 acquires mutex2 - println!(" Normal operation: no deadlock"); - - // Detect deadlock - if detector.detect_deadlock().is_none() { - println!(" Detection result: no deadlock"); - } - - // Simulate deadlock scenario - detector.record_wait(mutex2, 1); // Thread 1 waits for mutex2 - detector.record_wait(mutex1, 2); // Thread 2 waits for mutex1 - - // Detect deadlock - if let Some(deadlock) = detector.detect_deadlock() { - println!(" Detection result: deadlock found {:?}", deadlock); - } - - // Cleanup - detector.unregister_lock(mutex1); - detector.unregister_lock(mutex2); - println!(); -} - -fn lock_optimizer_example() { - println!("--- Lock Optimizer ---"); - - let optimizer = LockOptimizer::with_defaults(); - - // Simulate lock operations - for _i in 0..5 { - optimizer.on_acquire(); - // Simulate work - std::thread::sleep(Duration::from_millis(10)); - optimizer.on_release(Duration::from_millis(10)); - } - - // View statistics - let stats = optimizer.stats(); - let acquired = stats.total_acquired(); - let avg_hold = stats.avg_hold_time(); - let contention = stats.contention_rate(); - - println!(" Locks acquired: {}", acquired); - println!(" Average hold time: {:?}", avg_hold); - println!(" Contention rate: {:.2}%", contention * 100.0); - - println!(); -} diff --git a/crates/io-core/src/bufreader_optimizer.rs b/crates/io-core/src/bufreader_optimizer.rs deleted file mode 100644 index 26ec968d5..000000000 --- a/crates/io-core/src/bufreader_optimizer.rs +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! BufReader layer optimizer for minimizing redundant buffering layers. -//! -//! This module provides optimization for BufReader usage in data paths, -//! including layer count limiting and dynamic buffer size adjustment. - -use std::sync::atomic::{AtomicU64, Ordering}; - -/// BufReader optimization configuration. -#[derive(Debug, Clone)] -pub struct BufReaderConfig { - /// Maximum number of nested BufReader layers (default: 2) - pub max_layers: u32, - - /// Buffer size for small files (default: 8KB) - pub small_file_buffer: usize, - - /// Buffer size for large files (default: 64KB) - pub large_file_buffer: usize, - - /// Threshold for large file classification (default: 1MB) - pub large_file_threshold: usize, -} - -impl Default for BufReaderConfig { - fn default() -> Self { - Self { - max_layers: 2, - small_file_buffer: 8 * 1024, // 8KB - large_file_buffer: 64 * 1024, // 64KB - large_file_threshold: 1024 * 1024, // 1MB - } - } -} - -/// BufReader optimization statistics. -#[derive(Debug, Default)] -pub struct BufReaderStats { - /// Total number of readers created - pub total_readers: AtomicU64, - - /// Number of redundant layers eliminated - pub eliminated_layers: AtomicU64, - - /// Number of buffer size adjustments - pub buffer_size_adjustments: AtomicU64, -} - -/// BufReader layer optimizer. -/// -/// Analyzes and optimizes BufReader nesting in data paths, -/// dynamically adjusting buffer sizes based on data characteristics. -pub struct BufReaderOptimizer { - config: BufReaderConfig, - stats: BufReaderStats, -} - -impl BufReaderOptimizer { - /// Create a new BufReader optimizer with the given configuration. - pub fn new(config: BufReaderConfig) -> Self { - Self { - config, - stats: BufReaderStats::default(), - } - } - - /// Create a new BufReader optimizer with default configuration. - pub fn with_defaults() -> Self { - Self::new(BufReaderConfig::default()) - } - - /// Calculate the optimal buffer size based on data size. - /// - /// Returns the appropriate buffer size based on whether the data - /// is classified as a small or large file. - pub fn optimal_buffer_size(&self, data_size: Option) -> usize { - match data_size { - Some(size) if size >= self.config.large_file_threshold => self.config.large_file_buffer, - Some(_) => self.config.small_file_buffer, - None => self.config.small_file_buffer, - } - } - - /// Optimize a reader by wrapping it with an appropriately sized BufReader. - /// - /// This method applies the optimal buffer size based on the expected - /// data size and tracks statistics. - pub fn optimize(&self, reader: R, data_size: Option) -> tokio::io::BufReader { - let buffer_size = self.optimal_buffer_size(data_size); - self.stats.total_readers.fetch_add(1, Ordering::Relaxed); - tokio::io::BufReader::with_capacity(buffer_size, reader) - } - - /// Get the statistics for this optimizer. - pub fn stats(&self) -> &BufReaderStats { - &self.stats - } - - /// Get the configuration for this optimizer. - pub fn config(&self) -> &BufReaderConfig { - &self.config - } -} - -/// Marker trait for buffered sources. -/// -/// Types implementing this trait are considered already buffered -/// and should not be wrapped with additional BufReader layers. -pub trait BufferedSource: tokio::io::AsyncRead {} - -impl BufReaderOptimizer { - /// Check if a reader is already a buffered source. - /// - /// Returns true if the reader implements `BufferedSource`, - /// indicating it should not be wrapped with BufReader. - pub fn is_buffered_source(&self, _reader: &R) -> bool { - true - } - - /// Eliminate redundant BufReader layers if possible. - /// - /// This method attempts to reduce the nesting depth of BufReader - /// layers to improve performance. - pub fn eliminate_redundant_layers(&self, reader: R) -> R { - // For now, just return the reader as-is - // Future implementation could detect and unwrap nested BufReaders - self.stats.eliminated_layers.fetch_add(0, Ordering::Relaxed); - reader - } -} - -#[cfg(test)] -mod tests { - use super::*; - use tokio::io::AsyncReadExt; - - #[test] - fn test_default_config() { - let config = BufReaderConfig::default(); - assert_eq!(config.max_layers, 2); - assert_eq!(config.small_file_buffer, 8 * 1024); - assert_eq!(config.large_file_buffer, 64 * 1024); - assert_eq!(config.large_file_threshold, 1024 * 1024); - } - - #[test] - fn test_optimal_buffer_size_small_file() { - let optimizer = BufReaderOptimizer::with_defaults(); - - // Small file (< 1MB) - assert_eq!(optimizer.optimal_buffer_size(Some(100)), 8 * 1024); - assert_eq!(optimizer.optimal_buffer_size(Some(1024)), 8 * 1024); - assert_eq!(optimizer.optimal_buffer_size(Some(512 * 1024)), 8 * 1024); - } - - #[test] - fn test_optimal_buffer_size_large_file() { - let optimizer = BufReaderOptimizer::with_defaults(); - - // Large file (>= 1MB) - assert_eq!(optimizer.optimal_buffer_size(Some(1024 * 1024)), 64 * 1024); - assert_eq!(optimizer.optimal_buffer_size(Some(10 * 1024 * 1024)), 64 * 1024); - } - - #[test] - fn test_optimal_buffer_size_unknown() { - let optimizer = BufReaderOptimizer::with_defaults(); - - // Unknown size - assert_eq!(optimizer.optimal_buffer_size(None), 8 * 1024); - } - - #[tokio::test] - async fn test_optimize_creates_bufreader() { - let optimizer = BufReaderOptimizer::with_defaults(); - let data = vec![1u8, 2, 3, 4, 5]; - let cursor = std::io::Cursor::new(data.clone()); - - let mut reader = optimizer.optimize(cursor, Some(5)); - - let mut buf = vec![0u8; 5]; - let n = reader.read(&mut buf).await.unwrap(); - - assert_eq!(n, 5); - assert_eq!(buf, data); - } - - #[test] - fn test_stats_tracking() { - let optimizer = BufReaderOptimizer::with_defaults(); - - assert_eq!(optimizer.stats().total_readers.load(Ordering::Relaxed), 0); - - let cursor = std::io::Cursor::new(vec![1u8, 2, 3]); - let _reader = optimizer.optimize(cursor, Some(3)); - - assert_eq!(optimizer.stats().total_readers.load(Ordering::Relaxed), 1); - } - - #[test] - fn test_custom_config() { - let config = BufReaderConfig { - max_layers: 3, - small_file_buffer: 4 * 1024, - large_file_buffer: 128 * 1024, - large_file_threshold: 2 * 1024 * 1024, - }; - - let optimizer = BufReaderOptimizer::new(config); - - assert_eq!(optimizer.optimal_buffer_size(Some(1024 * 1024)), 4 * 1024); - assert_eq!(optimizer.optimal_buffer_size(Some(3 * 1024 * 1024)), 128 * 1024); - } -} diff --git a/crates/io-core/src/direct_io.rs b/crates/io-core/src/direct_io.rs deleted file mode 100644 index 62e9834c8..000000000 --- a/crates/io-core/src/direct_io.rs +++ /dev/null @@ -1,332 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Aligned pread-based file reader. -//! -//! This module provides an aligned, position-based file reader that uses -//! `pread`/`FileExt::read_at` for I/O operations. It performs reads at -//! 512-byte-aligned offsets and sizes, making it suitable as a foundation -//! for workloads where alignment matters. -//! -//! Note: This reader does **not** set the `O_DIRECT` flag and therefore does -//! not bypass the OS page cache. It is an aligned `pread`-based reader, not -//! true Direct I/O. To implement true O_DIRECT on Linux, the file must be -//! opened with `O_DIRECT` via `libc::open`. -//! -//! # Platform Support -//! -//! The `read_at` implementation is only available on Unix-like platforms. -//! On other platforms, this reader will return an error. - -use std::io::{self}; -use std::pin::Pin; -use std::task::{Context, Poll}; -use tokio::io::{AsyncRead, ReadBuf}; - -/// Errors that can occur during aligned pread operations. -#[derive(Debug, Clone)] -pub enum AlignedPreadError { - /// Platform doesn't support `read_at`-based I/O - UnsupportedPlatform, - /// File descriptor doesn't support this reader - UnsupportedFile, - /// I/O error occurred - Io(String), - /// Invalid alignment (reads require 512-byte-aligned offset and size) - AlignmentError { offset: u64, size: usize }, -} - -impl std::fmt::Display for AlignedPreadError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::UnsupportedPlatform => write!(f, "Aligned pread not supported on this platform"), - Self::UnsupportedFile => write!(f, "File doesn't support this reader"), - Self::Io(msg) => write!(f, "I/O error: {}", msg), - Self::AlignmentError { offset, size } => { - write!(f, "Alignment error: offset={}, size={}", offset, size) - } - } - } -} - -impl std::error::Error for AlignedPreadError {} - -impl From for AlignedPreadError { - fn from(err: io::Error) -> Self { - Self::Io(err.to_string()) - } -} - -/// Aligned pread-based file reader for Unix platforms. -/// -/// This reader performs I/O using `pread`/`FileExt::read_at` at -/// 512-byte-aligned offsets and sizes, without modifying the file's -/// current position. -/// -/// **Note:** This reader does **not** set the `O_DIRECT` flag and therefore -/// does **not** bypass the OS page cache. It is an aligned `pread`-based -/// reader. To implement true O_DIRECT, the file must be opened with -/// `O_DIRECT` via `libc::open`. -/// -/// # Platform Support -/// -/// Only available on Linux (uses `FileExt::read_at`). On other platforms, -/// use `BytesBufferedReader` instead. -/// -/// # Alignment Requirements -/// -/// Reads have strict alignment requirements: -/// - File offset must be aligned to 512 bytes -/// - Buffer size must be a multiple of 512 bytes -/// - Buffer address must be aligned (handled internally) -/// -/// # Example -/// -/// ```ignore -/// use rustfs_io_core::AlignedPreadReader; -/// -/// // Linux only -/// #[cfg(target_os = "linux")] -/// let reader = AlignedPreadReader::new(file, offset, size)?; -/// ``` -#[cfg(target_os = "linux")] -pub struct AlignedPreadReader { - /// Underlying file handle used for aligned pread I/O - file: std::fs::File, - /// Current read position - pos: u64, - /// Remaining bytes to read - remaining: usize, - /// Buffer for aligned reads - buffer: Vec, - /// Current position in the buffer - buffer_pos: usize, - /// Amount of data in the buffer - buffer_len: usize, -} - -#[cfg(target_os = "linux")] -impl AlignedPreadReader { - /// Alignment requirement for reads (512 bytes for most systems) - pub const ALIGNMENT: usize = 512; - - /// Create a new aligned pread-based reader. - /// - /// # Arguments - /// - /// * `file` - File to read from - /// * `offset` - Starting offset in the file (must be 512-byte aligned) - /// * `size` - Number of bytes to read (must be 512-byte aligned) - /// - /// # Returns - /// - /// An `AlignedPreadReader` that reads the file at the given offset. - /// - /// # Errors - /// - /// Returns an error if offset or size are not 512-byte aligned. - pub fn new(file: std::fs::File, offset: u64, size: usize) -> Result { - // Check alignment - if !offset.is_multiple_of(Self::ALIGNMENT as u64) { - return Err(AlignedPreadError::AlignmentError { offset, size }); - } - if !size.is_multiple_of(Self::ALIGNMENT) { - return Err(AlignedPreadError::AlignmentError { offset, size }); - } - - Ok(Self { - file, - pos: offset, - remaining: size, - buffer: Vec::new(), - buffer_pos: 0, - buffer_len: 0, - }) - } - - /// Read a chunk of data using aligned pread. - /// - /// This method performs aligned reads and handles the buffering required - /// by this aligned pread implementation. It does not use `O_DIRECT`. - fn read_chunk(&mut self, buf: &mut [u8]) -> io::Result { - // If buffer is exhausted, read more data - if self.buffer_pos >= self.buffer_len { - if self.remaining == 0 { - return Ok(0); - } - - // Allocate aligned buffer - let chunk_size = (self.remaining).min(64 * 1024); // 64KB chunks - let aligned_size = chunk_size.div_ceil(Self::ALIGNMENT) * Self::ALIGNMENT; - - self.buffer = vec![0u8; aligned_size]; - - // Use pread for atomic read at position (no file offset modification) - use std::os::unix::fs::FileExt; - let n = self.file.read_at(&mut self.buffer, self.pos)?; - - self.buffer_pos = 0; - self.buffer_len = n; - self.pos += n as u64; - self.remaining -= n; - - if n == 0 { - return Ok(0); - } - } - - // Copy from buffer to user buffer - let available = self.buffer_len - self.buffer_pos; - let to_copy = buf.len().min(available); - buf[..to_copy].copy_from_slice(&self.buffer[self.buffer_pos..self.buffer_pos + to_copy]); - self.buffer_pos += to_copy; - - Ok(to_copy) - } -} - -#[cfg(target_os = "linux")] -impl AsyncRead for AlignedPreadReader { - fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { - let filled = buf.filled().len(); - let mut remaining = buf.initialize_unfilled(); - - while !remaining.is_empty() { - match self.read_chunk(remaining) { - Ok(0) => break, - Ok(n) => { - remaining = &mut remaining[n..]; - } - Err(e) => return Poll::Ready(Err(e)), - } - } - - let _n_read = buf.filled().len() - filled; - Poll::Ready(Ok(())) - } -} - -/// Aligned pread reader stub for non-Linux platforms. -/// -/// On non-Linux platforms, `read_at`-based I/O is not available through this -/// type. This stub exists to provide a consistent API across platforms. -#[cfg(not(target_os = "linux"))] -pub struct AlignedPreadReader { - _priv: (), -} - -#[cfg(not(target_os = "linux"))] -impl AlignedPreadReader { - /// Create a new aligned pread reader (not supported on this platform). - /// - /// Always returns an error on non-Linux platforms. - pub fn new(_file: std::fs::File, _offset: u64, _size: usize) -> Result { - Err(AlignedPreadError::UnsupportedPlatform) - } -} - -#[cfg(not(target_os = "linux"))] -impl AsyncRead for AlignedPreadReader { - fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll> { - Poll::Ready(Err(io::Error::new( - io::ErrorKind::Unsupported, - "Aligned pread-based I/O not supported on this platform", - ))) - } -} - -impl std::fmt::Debug for AlignedPreadReader { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - #[cfg(target_os = "linux")] - { - f.debug_struct("AlignedPreadReader") - .field("pos", &self.pos) - .field("remaining", &self.remaining) - .field("buffer_len", &self.buffer_len) - .finish() - } - #[cfg(not(target_os = "linux"))] - { - f.debug_struct("AlignedPreadReader") - .field("platform", &"unsupported") - .finish() - } - } -} - -/// Historical name for aligned pread errors. -#[deprecated(since = "1.0.0-beta.8", note = "use AlignedPreadError; this reader does not set O_DIRECT")] -pub type DirectIoError = AlignedPreadError; - -/// Historical name for the aligned pread-based reader. -#[deprecated(since = "1.0.0-beta.8", note = "use AlignedPreadReader; this reader does not set O_DIRECT")] -pub type DirectIoReader = AlignedPreadReader; - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_alignment_check() { - #[cfg(target_os = "linux")] - { - // Valid alignment - let file = std::fs::File::open("/dev/zero").unwrap(); - assert!( - AlignedPreadReader::new(file, 0, 512).is_ok(), - "Should succeed with aligned offset and size" - ); - - let file = std::fs::File::open("/dev/zero").expect("open /dev/zero for alias"); - assert!( - AlignedPreadReader::new(file, 0, 512).is_ok(), - "Should succeed through aligned pread alias" - ); - - // Invalid offset - let file = std::fs::File::open("/dev/zero").unwrap(); - assert!(AlignedPreadReader::new(file, 1, 512).is_err(), "Should fail with unaligned offset"); - - // Invalid size - let file = std::fs::File::open("/dev/zero").unwrap(); - assert!(AlignedPreadReader::new(file, 0, 511).is_err(), "Should fail with unaligned size"); - } - - #[cfg(not(target_os = "linux"))] - { - // Non-Linux should return UnsupportedPlatform - let file = std::fs::File::open(std::env::current_exe().unwrap()).unwrap(); - assert!(matches!( - AlignedPreadReader::new(file, 0, 512), - Err(AlignedPreadError::UnsupportedPlatform) - )); - } - } - - #[test] - #[allow(deprecated)] - fn test_legacy_direct_io_alias() { - #[cfg(target_os = "linux")] - { - let file = std::fs::File::open("/dev/zero").unwrap(); - assert!(DirectIoReader::new(file, 0, 512).is_ok()); - } - - #[cfg(not(target_os = "linux"))] - { - let file = std::fs::File::open(std::env::current_exe().unwrap()).unwrap(); - assert!(matches!(DirectIoReader::new(file, 0, 512), Err(AlignedPreadError::UnsupportedPlatform))); - } - } -} diff --git a/crates/io-core/src/io_priority_queue.rs b/crates/io-core/src/io_priority_queue.rs deleted file mode 100644 index bb7b4cf45..000000000 --- a/crates/io-core/src/io_priority_queue.rs +++ /dev/null @@ -1,381 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! I/O priority queue for scheduling I/O operations. -//! -//! This module provides a priority queue implementation for I/O operations -//! with support for starvation prevention and fair scheduling. - -use crate::config::IoPriorityQueueConfig; -use crate::scheduler::IoPriority; -use std::collections::VecDeque; -use std::time::{Duration, Instant}; - -/// A queued I/O request. -#[derive(Debug, Clone)] -pub struct IoRequest { - /// Request ID. - pub id: u64, - /// Request priority. - pub priority: IoPriority, - /// Request size in bytes. - pub size: usize, - /// Queue time. - pub queued_at: Instant, - /// Whether this is a sequential read. - pub is_sequential: bool, -} - -impl IoRequest { - /// Create a new I/O request. - pub fn new(id: u64, priority: IoPriority, size: usize, is_sequential: bool) -> Self { - Self { - id, - priority, - size, - queued_at: Instant::now(), - is_sequential, - } - } - - /// Get the wait time in the queue. - pub fn wait_time(&self) -> Duration { - self.queued_at.elapsed() - } -} - -/// Queue status for a priority level. -#[derive(Debug, Clone, Default)] -pub struct IoQueueStatus { - /// Number of requests in the queue. - pub count: usize, - /// Total size of all requests. - pub total_size: usize, - /// Oldest request wait time. - pub oldest_wait: Option, - /// Number of requests processed. - pub processed: u64, -} - -impl IoQueueStatus { - /// Create new queue status. - pub fn new() -> Self { - Self::default() - } -} - -/// I/O priority queue. -pub struct IoPriorityQueue { - /// Queue configuration. - config: IoPriorityQueueConfig, - /// High priority queue. - high: VecDeque, - /// Normal priority queue. - normal: VecDeque, - /// Low priority queue. - low: VecDeque, - /// Next request ID. - next_id: u64, - /// Last dequeue time for each priority (for starvation prevention). - last_dequeue: [Option; 3], - /// Statistics for each queue. - stats: [IoQueueStatus; 3], -} - -impl IoPriorityQueue { - /// Create a new priority queue with the given configuration. - pub fn new(config: IoPriorityQueueConfig) -> Self { - Self { - config, - high: VecDeque::with_capacity(100), - normal: VecDeque::with_capacity(500), - low: VecDeque::with_capacity(200), - next_id: 0, - last_dequeue: [None, None, None], - stats: [IoQueueStatus::new(), IoQueueStatus::new(), IoQueueStatus::new()], - } - } - - /// Create with default configuration. - pub fn with_defaults() -> Self { - Self::new(IoPriorityQueueConfig::default()) - } - - /// Get the configuration. - pub fn config(&self) -> &IoPriorityQueueConfig { - &self.config - } - - /// Enqueue a request. - pub fn enqueue(&mut self, priority: IoPriority, size: usize, is_sequential: bool) -> u64 { - let id = self.next_id; - self.next_id += 1; - - let request = IoRequest::new(id, priority, size, is_sequential); - - match priority { - IoPriority::High => { - if self.high.len() < self.config.high_capacity { - self.high.push_back(request); - } - } - IoPriority::Normal => { - if self.normal.len() < self.config.normal_capacity { - self.normal.push_back(request); - } - } - IoPriority::Low => { - if self.low.len() < self.config.low_capacity { - self.low.push_back(request); - } - } - } - - id - } - - /// Dequeue the next request. - /// - /// Uses weighted fair queuing with starvation prevention. - pub fn dequeue(&mut self) -> Option { - let now = Instant::now(); - - // Check for starvation: if a lower priority queue hasn't been served in a while, - // give it priority - let normal_starved = self.is_starved(IoPriority::Normal, now); - let low_starved = self.is_starved(IoPriority::Low, now); - - // Priority order with starvation consideration - // Check conditions first, then dequeue - let dequeue_high = !self.high.is_empty() && !low_starved && !normal_starved; - let dequeue_normal = !self.normal.is_empty() && !low_starved; - let dequeue_low = !self.low.is_empty(); - let dequeue_high_fallback = !self.high.is_empty(); - let dequeue_normal_fallback = !self.normal.is_empty(); - - if dequeue_high { - let request = self.high.pop_front(); - if request.is_some() { - self.last_dequeue[0] = Some(Instant::now()); - self.stats[0].processed += 1; - } - request - } else if dequeue_normal { - let request = self.normal.pop_front(); - if request.is_some() { - self.last_dequeue[1] = Some(Instant::now()); - self.stats[1].processed += 1; - } - request - } else if dequeue_low { - let request = self.low.pop_front(); - if request.is_some() { - self.last_dequeue[2] = Some(Instant::now()); - self.stats[2].processed += 1; - } - request - } else if dequeue_high_fallback { - let request = self.high.pop_front(); - if request.is_some() { - self.last_dequeue[0] = Some(Instant::now()); - self.stats[0].processed += 1; - } - request - } else if dequeue_normal_fallback { - let request = self.normal.pop_front(); - if request.is_some() { - self.last_dequeue[1] = Some(Instant::now()); - self.stats[1].processed += 1; - } - request - } else { - None - } - } - - /// Check if a priority level is starved. - fn is_starved(&self, priority: IoPriority, now: Instant) -> bool { - let idx = match priority { - IoPriority::High => 0, - IoPriority::Normal => 1, - IoPriority::Low => 2, - }; - - if let Some(last) = self.last_dequeue[idx] { - now.duration_since(last) > self.config.starvation_threshold - } else { - false - } - } - - /// Get the total number of queued requests. - pub fn len(&self) -> usize { - self.high.len() + self.normal.len() + self.low.len() - } - - /// Check if the queue is empty. - pub fn is_empty(&self) -> bool { - self.high.is_empty() && self.normal.is_empty() && self.low.is_empty() - } - - /// Get queue status for a priority level. - pub fn status(&self, priority: IoPriority) -> IoQueueStatus { - let (queue, idx) = match priority { - IoPriority::High => (&self.high, 0), - IoPriority::Normal => (&self.normal, 1), - IoPriority::Low => (&self.low, 2), - }; - - let mut status = self.stats[idx].clone(); - status.count = queue.len(); - status.total_size = queue.iter().map(|r| r.size).sum(); - status.oldest_wait = queue.front().map(|r| r.wait_time()); - status - } - - /// Get the total queue status. - pub fn total_status(&self) -> IoQueueStatus { - let mut total = IoQueueStatus::new(); - total.count = self.len(); - total.total_size = self - .high - .iter() - .chain(self.normal.iter()) - .chain(self.low.iter()) - .map(|r| r.size) - .sum(); - total.processed = self.stats.iter().map(|s| s.processed).sum(); - total.oldest_wait = self - .high - .front() - .map(|r| r.wait_time()) - .or_else(|| self.normal.front().map(|r| r.wait_time())) - .or_else(|| self.low.front().map(|r| r.wait_time())); - total - } - - /// Clear all queues. - pub fn clear(&mut self) { - self.high.clear(); - self.normal.clear(); - self.low.clear(); - } - - /// Peek at the next request without removing it. - pub fn peek(&self) -> Option<&IoRequest> { - if !self.high.is_empty() { - self.high.front() - } else if !self.normal.is_empty() { - self.normal.front() - } else { - self.low.front() - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_enqueue_dequeue() { - let mut queue = IoPriorityQueue::with_defaults(); - - let id1 = queue.enqueue(IoPriority::High, 1024, true); - let id2 = queue.enqueue(IoPriority::Normal, 2048, false); - let id3 = queue.enqueue(IoPriority::Low, 4096, true); - - assert_eq!(queue.len(), 3); - - // High priority should be dequeued first - let req1 = queue.dequeue().unwrap(); - assert_eq!(req1.id, id1); - assert_eq!(req1.priority, IoPriority::High); - - let req2 = queue.dequeue().unwrap(); - assert_eq!(req2.id, id2); - assert_eq!(req2.priority, IoPriority::Normal); - - let req3 = queue.dequeue().unwrap(); - assert_eq!(req3.id, id3); - assert_eq!(req3.priority, IoPriority::Low); - - assert!(queue.is_empty()); - } - - #[test] - fn test_queue_status() { - let mut queue = IoPriorityQueue::with_defaults(); - - queue.enqueue(IoPriority::High, 1024, true); - queue.enqueue(IoPriority::High, 2048, true); - queue.enqueue(IoPriority::Normal, 4096, false); - - let high_status = queue.status(IoPriority::High); - assert_eq!(high_status.count, 2); - assert_eq!(high_status.total_size, 3072); - - let normal_status = queue.status(IoPriority::Normal); - assert_eq!(normal_status.count, 1); - assert_eq!(normal_status.total_size, 4096); - - let total = queue.total_status(); - assert_eq!(total.count, 3); - assert_eq!(total.total_size, 7168); - } - - #[test] - fn test_queue_capacity() { - let config = IoPriorityQueueConfig { - high_capacity: 2, - normal_capacity: 2, - low_capacity: 2, - ..Default::default() - }; - let mut queue = IoPriorityQueue::new(config); - - queue.enqueue(IoPriority::High, 1024, true); - queue.enqueue(IoPriority::High, 1024, true); - queue.enqueue(IoPriority::High, 1024, true); // Should be dropped - - assert_eq!(queue.status(IoPriority::High).count, 2); - } - - #[test] - fn test_clear() { - let mut queue = IoPriorityQueue::with_defaults(); - - queue.enqueue(IoPriority::High, 1024, true); - queue.enqueue(IoPriority::Normal, 2048, false); - queue.enqueue(IoPriority::Low, 4096, true); - - assert_eq!(queue.len(), 3); - queue.clear(); - assert!(queue.is_empty()); - } - - #[test] - fn test_peek() { - let mut queue = IoPriorityQueue::with_defaults(); - - queue.enqueue(IoPriority::Normal, 2048, false); - queue.enqueue(IoPriority::High, 1024, true); - - let peeked = queue.peek().unwrap(); - assert_eq!(peeked.priority, IoPriority::High); - - // Peek shouldn't remove the item - assert_eq!(queue.len(), 2); - } -} diff --git a/crates/io-core/src/lib.rs b/crates/io-core/src/lib.rs index 67a374ea7..ca5f20c53 100644 --- a/crates/io-core/src/lib.rs +++ b/crates/io-core/src/lib.rs @@ -12,85 +12,39 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Buffered I/O reader and writer implementations for RustFS. +//! Shared I/O primitives for RustFS. //! -//! This crate provides buffered readers and writers for I/O operations. -//! Prefer `BytesBufferedReader`, `BytesMutWriter`, and `AlignedPreadReader` -//! for new code. Historical `ZeroCopy*` and `DirectIo*` names remain exported -//! for backward compatibility. +//! This crate holds the buffer pool and the concurrency-control primitives +//! that the storage layer builds on: //! -//! # Features -//! -//! - Memory-mapped file reading (mmap-then-copy) on Unix platforms -//! - Bytes-based buffered wrapping -//! - AsyncRead trait implementations -//! - Tiered BytesPool for buffer management -//! - Aligned pread-based reader (NOT true Direct I/O / O_DIRECT) +//! - Tiered `BytesPool` for buffer management +//! - Storage-media and access-pattern profiling (`io_profile`) +//! - Scheduler and priority-queue configuration shapes +//! - Backpressure admission, deadlock detection, lock optimization +//! - Progress tracking for long-running operations //! //! # Example //! //! ```ignore -//! use rustfs_io_core::{BytesBufferedReader, BytesPool}; -//! use bytes::Bytes; +//! use rustfs_io_core::BytesPool; //! -//! // Create from existing bytes (zero-copy) -//! let data = Bytes::from("hello world"); -//! let reader = BytesBufferedReader::from_bytes(data); -//! -//! // Create from file using buffered reads -//! let reader = BytesBufferedReader::from_file_read(&file, 0, 1024).await?; -//! -//! // Use BytesPool //! let pool = BytesPool::new_tiered(); //! let mut buffer = pool.acquire_buffer(8192).await; //! ``` pub mod backpressure; -pub mod bufreader_optimizer; pub mod config; pub mod deadlock_detector; -pub mod direct_io; -pub mod io_priority_queue; pub mod io_profile; pub mod lock_optimizer; pub mod pool; -pub mod reader; -pub mod scheduler; -pub mod shared_memory; -pub mod timeout_wrapper; -pub mod writer; +pub mod progress; -#[cfg(target_os = "linux")] -pub use direct_io::{AlignedPreadError, AlignedPreadReader}; -#[cfg(target_os = "linux")] -#[allow(deprecated)] -pub use direct_io::{DirectIoError, DirectIoReader}; pub use pool::{BytesPool, BytesPoolConfig, BytesPoolMetrics, PooledBuffer}; -#[allow(deprecated)] -pub use reader::ZeroCopyObjectReader; -pub use reader::{BytesBufferedReader, ZeroCopyReadError}; -#[allow(deprecated)] -pub use writer::ZeroCopyObjectWriter; -pub use writer::{BytesMutWriter, ZeroCopyWriteError}; - -// BufReader optimizer exports -pub use bufreader_optimizer::{BufReaderConfig, BufReaderOptimizer, BufReaderStats, BufferedSource}; - -// Shared memory exports -pub use shared_memory::{ArcData, ArcMetadata, SharedMemoryConfig, SharedMemoryPool, SharedMemoryStats}; // Config exports pub use config::{ConfigError, IoPriorityQueueConfig, IoSchedulerConfig}; -// Scheduler exports -pub use scheduler::{ - BandwidthTier, IoLoadLevel, IoLoadMetrics, IoPriority, IoScheduler, IoSchedulingContext, IoStrategy, KI_B, MI_B, - calculate_optimal_buffer_size, get_advanced_buffer_size, get_buffer_size_for_media, get_concurrency_aware_buffer_size, -}; - -// Priority queue exports -pub use io_priority_queue::{IoPriorityQueue, IoQueueStatus, IoRequest}; - // Backpressure exports pub use backpressure::{BackpressureConfig, BackpressureError, BackpressureMonitor, BackpressureState}; @@ -100,8 +54,5 @@ pub use deadlock_detector::{DeadlockDetector, DeadlockDetectorConfig, LockInfo, // Lock optimizer exports pub use lock_optimizer::{LockGuard, LockOptimizeConfig, LockOptimizer, LockStats}; -// Timeout wrapper exports -pub use timeout_wrapper::{ - OperationProgress, RequestTimeoutWrapper, TimeoutConfig, TimeoutError, TimeoutStats, calculate_adaptive_timeout, - estimate_bytes_per_second, -}; +// Progress tracking exports +pub use progress::OperationProgress; diff --git a/crates/io-core/src/progress.rs b/crates/io-core/src/progress.rs new file mode 100644 index 000000000..44e4460dd --- /dev/null +++ b/crates/io-core/src/progress.rs @@ -0,0 +1,138 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Progress tracking for long-running I/O operations. +//! +//! Re-exported as `rustfs_concurrency::OperationProgress` for the storage +//! timeout implementation, which uses `is_stale` to tell a slow transfer +//! apart from a stalled one. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +/// Operation progress tracker. +#[derive(Debug)] +pub struct OperationProgress { + /// Total size (if known). + pub total_size: Option, + /// Bytes processed. + bytes_processed: AtomicU64, + /// Last update time. + last_update: std::sync::Mutex, + /// Stale timeout. + stale_timeout: Duration, + /// Start time for transfer rate calculation. + start_time: Instant, +} + +impl OperationProgress { + /// Create new operation progress. + pub fn new(total_size: Option, stale_timeout: Duration) -> Self { + Self { + total_size, + bytes_processed: AtomicU64::new(0), + last_update: std::sync::Mutex::new(Instant::now()), + stale_timeout, + start_time: Instant::now(), + } + } + + /// Update progress. + pub fn update(&self, bytes: u64) { + self.bytes_processed.store(bytes, Ordering::Relaxed); + if let Ok(mut last) = self.last_update.lock() { + *last = Instant::now(); + } + } + + /// Add to progress. + pub fn add(&self, bytes: u64) { + self.bytes_processed.fetch_add(bytes, Ordering::Relaxed); + if let Ok(mut last) = self.last_update.lock() { + *last = Instant::now(); + } + } + + /// Get current progress. + pub fn current(&self) -> u64 { + self.bytes_processed.load(Ordering::Relaxed) + } + + /// Check if progress is stale. + pub fn is_stale(&self) -> bool { + if let Ok(last) = self.last_update.lock() { + last.elapsed() > self.stale_timeout + } else { + false + } + } + + /// Get progress percentage. + pub fn progress_percent(&self) -> Option { + self.total_size.map(|total| { + if total == 0 { + 100.0 + } else { + let processed = self.bytes_processed.load(Ordering::Relaxed); + (processed as f64 / total as f64 * 100.0).min(100.0) + } + }) + } + + /// Get remaining bytes. + pub fn remaining(&self) -> Option { + self.total_size.map(|total| { + let processed = self.bytes_processed.load(Ordering::Relaxed); + total.saturating_sub(processed) + }) + } + + /// Calculate transfer rate in bytes per second. + /// + /// Returns 0 if no time has elapsed or no data transferred. + pub fn transfer_rate(&self) -> u64 { + let processed = self.bytes_processed.load(Ordering::Relaxed); + if processed == 0 { + return 0; + } + + let elapsed = self.start_time.elapsed().as_secs_f64(); + if elapsed > 0.0 { + (processed as f64 / elapsed) as u64 + } else { + 0 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_operation_progress() { + let progress = OperationProgress::new(Some(1000), Duration::from_secs(5)); + + assert_eq!(progress.current(), 0); + assert_eq!(progress.progress_percent(), Some(0.0)); + + progress.update(500); + assert_eq!(progress.current(), 500); + assert_eq!(progress.progress_percent(), Some(50.0)); + + progress.add(300); + assert_eq!(progress.current(), 800); + assert_eq!(progress.remaining(), Some(200)); + } +} diff --git a/crates/io-core/src/reader.rs b/crates/io-core/src/reader.rs deleted file mode 100644 index 49433af73..000000000 --- a/crates/io-core/src/reader.rs +++ /dev/null @@ -1,412 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Bytes-backed object reader implementation. - -use bytes::Bytes; -use std::io; -use std::pin::Pin; -use std::task::{Context, Poll}; -use tokio::io::{AsyncRead, ReadBuf}; - -/// Errors that can occur during Bytes-backed read operations. -#[derive(Debug, Clone)] -pub enum ZeroCopyReadError { - /// I/O error occurred. - Io(String), - /// Memory mapping error. - Mmap(String), - /// Invalid offset or size. - InvalidRange, -} - -impl std::fmt::Display for ZeroCopyReadError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Io(msg) => write!(f, "I/O error: {}", msg), - Self::Mmap(msg) => write!(f, "Mmap error: {}", msg), - Self::InvalidRange => write!(f, "Invalid offset or size"), - } - } -} - -impl std::error::Error for ZeroCopyReadError {} - -impl From for ZeroCopyReadError { - fn from(err: io::Error) -> Self { - Self::Io(err.to_string()) - } -} - -/// Bytes-backed object reader. -/// -/// `from_bytes` wraps existing `Bytes` without copying, but file constructors -/// copy file data into owned `Bytes` after mmap or normal reads. -/// -/// # Example -/// -/// ```ignore -/// use bytes::Bytes; -/// use rustfs_io_core::BytesBufferedReader; -/// -/// // Create from bytes without copying the `Bytes` buffer -/// let data = Bytes::from("hello world"); -/// let reader = BytesBufferedReader::from_bytes(data); -/// -/// // Read using AsyncRead trait -/// let mut buf = vec![0u8; 1024]; -/// let n = reader.read(&mut buf[..]).await?; -/// ``` -pub struct BytesBufferedReader { - /// Internal data source (could be mmap or owned bytes) - data: Bytes, - /// Current read position - pos: usize, -} - -/// Historical name for the bytes-backed object reader. -#[deprecated( - since = "1.0.0-beta.8", - note = "use BytesBufferedReader; file constructors copy into owned Bytes" -)] -pub type ZeroCopyObjectReader = BytesBufferedReader; - -impl BytesBufferedReader { - /// Create a reader from existing bytes. - /// - /// This is a true zero-copy operation - the Bytes are wrapped - /// without any allocation or copying. - /// - /// # Arguments - /// - /// * `data` - Bytes to wrap - /// - /// # Example - /// - /// ```ignore - /// let data = Bytes::from("hello world"); - /// let reader = BytesBufferedReader::from_bytes(data); - /// ``` - pub fn from_bytes(data: Bytes) -> Self { - Self { data, pos: 0 } - } - - /// Create a Bytes-backed reader from a file using mmap-then-copy. - /// - /// This maps the requested file range and copies it into owned `Bytes` - /// before returning. It does not expose the mmap as a zero-copy buffer. - /// - /// # Arguments - /// - /// * `path` - Path to the file to memory map - /// * `offset` - Offset within the file to start reading - /// * `size` - Number of bytes to read - /// - /// # Returns - /// - /// A reader backed by copied file data. - /// - /// # Errors - /// - /// Returns an error if the file cannot be memory mapped. - /// - /// # Example - /// - /// ```ignore - /// let reader = BytesBufferedReader::from_file_mmap_path("large_file.bin", 0, 1024).await?; - /// ``` - #[cfg(unix)] - // SAFETY: The mmap is created from a read-only file handle for the - // caller-provided range, then copied into owned `Bytes` before the file and - // mapping are dropped. - #[allow(unsafe_code)] - pub async fn from_file_mmap_path(path: &std::path::Path, offset: u64, size: usize) -> Result { - use memmap2::MmapOptions; - - let path = path.to_path_buf(); - let (offset, size) = (offset, size); - - tokio::task::spawn_blocking(move || { - // Open the file in sync context - let std_file = std::fs::File::open(&path).map_err(|e| ZeroCopyReadError::Io(e.to_string()))?; - - // SAFETY: `std_file` remains open while the mapping is created and - // copied, and the mapped bytes are not exposed beyond this closure. - let mmap = unsafe { MmapOptions::new().offset(offset).len(size).map(&std_file) } - .map_err(|e| ZeroCopyReadError::Mmap(e.to_string()))?; - - // Convert to Bytes (this is a copy, but only done once) - Ok(Self { - data: Bytes::copy_from_slice(&mmap), - pos: 0, - }) - }) - .await - .map_err(|e| ZeroCopyReadError::Io(e.to_string()))? - } - - /// Create a Bytes-backed reader from a file using normal reads. - /// - /// This path reads the requested range into an owned buffer and wraps it in - /// `Bytes`. It does not perform mmap or zero-copy file I/O. - /// - /// # Arguments - /// - /// * `file` - File to read from - /// * `offset` - Offset within the file to start reading - /// * `size` - Number of bytes to map - /// - /// # Returns - /// - /// A reader backed by copied file data. - /// - /// # Errors - /// - /// Returns an error if the file cannot be read. - /// - /// # Example - /// - /// ```ignore - /// let file = tokio::fs::File::open("large_file.bin").await?; - /// let reader = BytesBufferedReader::from_file_read(&file, 0, 1024).await?; - /// ``` - #[cfg(unix)] - pub async fn from_file_read(file: &tokio::fs::File, offset: u64, size: usize) -> Result { - use tokio::io::{AsyncReadExt, AsyncSeekExt, SeekFrom}; - - let mut cloned = file.try_clone().await?; - cloned.seek(SeekFrom::Start(offset)).await?; - - let mut buffer = vec![0u8; size]; - cloned.read_exact(&mut buffer).await?; - - Ok(Self { - data: Bytes::from(buffer), - pos: 0, - }) - } - - /// Create a Bytes-backed reader from a file (non-Unix fallback). - /// - /// On platforms that don't support mmap, this falls back to regular file I/O. - #[cfg(not(unix))] - pub async fn from_file_read(file: &tokio::fs::File, offset: u64, size: usize) -> Result { - use tokio::io::{AsyncReadExt, AsyncSeekExt, SeekFrom}; - - let mut cloned = file.try_clone().await?; - cloned.seek(SeekFrom::Start(offset)).await?; - - let mut buffer = vec![0u8; size]; - cloned.read_exact(&mut buffer).await?; - - Ok(Self { - data: Bytes::from(buffer), - pos: 0, - }) - } - - /// Historical name for `from_file_read`. - #[deprecated( - since = "1.0.0-beta.8", - note = "use from_file_read; this method performs normal reads into owned Bytes" - )] - pub async fn from_file_mmap(file: &tokio::fs::File, offset: u64, size: usize) -> Result { - Self::from_file_read(file, offset, size).await - } - - /// Get the remaining data as Bytes (zero-copy). - /// - /// This returns a slice of the remaining data without copying. - /// The returned Bytes shares the underlying memory with this reader. - /// - /// # Example - /// - /// ```ignore - /// let remaining = reader.remaining_bytes(); - /// println!("Remaining: {} bytes", remaining.len()); - /// ``` - pub fn remaining_bytes(&self) -> Bytes { - self.data.slice(self.pos..) - } - - /// Get the total length of the data. - pub fn len(&self) -> usize { - self.data.len() - } - - /// Check if the reader has reached the end. - pub fn is_empty(&self) -> bool { - self.pos >= self.data.len() - } - - /// Get the current read position. - pub fn position(&self) -> usize { - self.pos - } -} - -impl AsyncRead for BytesBufferedReader { - fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { - let remaining = self.data.len() - self.pos; - if remaining == 0 { - return Poll::Ready(Ok(())); - } - - let to_read = std::cmp::min(remaining, buf.remaining()); - let slice = &self.data[self.pos..self.pos + to_read]; - buf.put_slice(slice); - self.pos += to_read; - - Poll::Ready(Ok(())) - } -} - -impl std::fmt::Debug for BytesBufferedReader { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("BytesBufferedReader") - .field("data_len", &self.data.len()) - .field("pos", &self.pos) - .field("remaining", &(self.data.len() - self.pos)) - .finish() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::path::PathBuf; - use tokio::io::AsyncReadExt; - - fn temp_file_path(test_name: &str) -> PathBuf { - let nonce = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("system time should be after unix epoch") - .as_nanos(); - std::env::temp_dir().join(format!("rustfs-io-core-{test_name}-{}-{nonce}", std::process::id())) - } - - #[tokio::test] - async fn test_from_bytes() { - let data = Bytes::from("hello world"); - let mut reader = BytesBufferedReader::from_bytes(data.clone()); - - let mut buf = [0u8; 11]; - let n = reader.read(&mut buf[..]).await.unwrap(); - - assert_eq!(n, 11); - assert_eq!(&buf[..n], b"hello world"); - } - - #[tokio::test] - async fn test_preferred_reader_alias() { - let data = Bytes::from("hello world"); - let mut reader = BytesBufferedReader::from_bytes(data); - - let mut buf = [0u8; 5]; - let n = reader.read(&mut buf[..]).await.expect("read bytes from alias"); - - assert_eq!(n, 5); - assert_eq!(&buf[..n], b"hello"); - } - - #[tokio::test] - async fn test_from_file_read_reads_requested_range() { - let path = temp_file_path("from-file-read"); - tokio::fs::write(&path, b"hello world") - .await - .expect("write temp file for reader test"); - - let file = tokio::fs::File::open(&path).await.expect("open temp file for reader test"); - let mut reader = BytesBufferedReader::from_file_read(&file, 6, 5) - .await - .expect("read requested range into Bytes"); - - let mut output = Vec::new(); - reader.read_to_end(&mut output).await.expect("drain reader output"); - - assert_eq!(output, b"world"); - - let _ = tokio::fs::remove_file(path).await; - } - - #[tokio::test] - #[allow(deprecated)] - async fn test_from_file_mmap_legacy_alias_reads_requested_range() { - let path = temp_file_path("from-file-mmap"); - tokio::fs::write(&path, b"hello world") - .await - .expect("write temp file for legacy reader test"); - - let file = tokio::fs::File::open(&path) - .await - .expect("open temp file for legacy reader test"); - let mut reader = BytesBufferedReader::from_file_mmap(&file, 0, 5) - .await - .expect("read requested range through legacy alias"); - - let mut output = Vec::new(); - reader.read_to_end(&mut output).await.expect("drain legacy reader output"); - - assert_eq!(output, b"hello"); - - let _ = tokio::fs::remove_file(path).await; - } - - #[tokio::test] - async fn test_remaining_bytes() { - let data = Bytes::from("hello world"); - let reader = BytesBufferedReader::from_bytes(data); - - let remaining = reader.remaining_bytes(); - assert_eq!(remaining.len(), 11); - assert_eq!(&remaining[..], b"hello world"); - } - - #[tokio::test] - async fn test_position() { - let data = Bytes::from("hello world"); - let mut reader = BytesBufferedReader::from_bytes(data); - - assert_eq!(reader.position(), 0); - - let mut buf = [0u8; 5]; - reader.read_exact(&mut buf[..]).await.unwrap(); - - assert_eq!(reader.position(), 5); - } - - #[tokio::test] - async fn test_is_empty() { - let data = Bytes::from(""); - let reader = BytesBufferedReader::from_bytes(data); - assert!(reader.is_empty()); - - let data = Bytes::from("hello"); - let reader = BytesBufferedReader::from_bytes(data); - assert!(!reader.is_empty()); - } - - #[tokio::test] - #[allow(deprecated)] - async fn test_legacy_reader_alias() { - let data = Bytes::from("hello world"); - let mut reader = ZeroCopyObjectReader::from_bytes(data); - - let mut buf = [0u8; 5]; - let n = reader.read(&mut buf[..]).await.expect("read bytes through legacy alias"); - - assert_eq!(n, 5); - assert_eq!(&buf[..n], b"hello"); - } -} diff --git a/crates/io-core/src/scheduler.rs b/crates/io-core/src/scheduler.rs deleted file mode 100644 index 76afa58cf..000000000 --- a/crates/io-core/src/scheduler.rs +++ /dev/null @@ -1,882 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! I/O scheduler for adaptive buffer sizing and load management. -//! -//! This module provides the core I/O scheduling logic that determines -//! optimal buffer sizes, I/O strategies, and load management decisions. - -use crate::config::IoSchedulerConfig; -use crate::io_profile::{AccessPattern, StorageMedia, StorageProfile}; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::time::Duration; - -/// I/O priority levels. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum IoPriority { - /// High priority for small, latency-sensitive operations. - High, - /// Normal priority for standard operations. - #[default] - Normal, - /// Low priority for large, throughput-oriented operations. - Low, -} - -impl IoPriority { - /// Determine priority based on request size. - /// - /// A negative `size` means the size is unknown (-1 by convention) and maps - /// to `Normal`; casting it to `usize` would wrap to a huge value and - /// misclassify the request as `Low`. - pub fn from_size(size: i64, high_threshold: usize, low_threshold: usize) -> Self { - if size < 0 { - return IoPriority::Normal; - } - let size = size as usize; - if size < high_threshold { - IoPriority::High - } else if size > low_threshold { - IoPriority::Low - } else { - IoPriority::Normal - } - } - - /// Get the priority as a string for metrics labels. - pub fn as_str(&self) -> &'static str { - match self { - IoPriority::High => "high", - IoPriority::Normal => "normal", - IoPriority::Low => "low", - } - } - - /// Check if this is high priority. - pub fn is_high(&self) -> bool { - matches!(self, IoPriority::High) - } - - /// Check if this is normal priority. - pub fn is_normal(&self) -> bool { - matches!(self, IoPriority::Normal) - } - - /// Check if this is low priority. - pub fn is_low(&self) -> bool { - matches!(self, IoPriority::Low) - } -} - -impl std::fmt::Display for IoPriority { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.as_str()) - } -} - -/// I/O load level. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Default)] -pub enum IoLoadLevel { - /// Low load - system is underutilized. - Low, - /// Medium load - system is moderately utilized. - #[default] - Medium, - /// High load - system is heavily utilized. - High, - /// Critical load - system is overloaded. - Critical, -} - -impl IoLoadLevel { - /// Get the load level as a string for metrics labels. - pub fn as_str(&self) -> &'static str { - match self { - IoLoadLevel::Low => "low", - IoLoadLevel::Medium => "medium", - IoLoadLevel::High => "high", - IoLoadLevel::Critical => "critical", - } - } - - /// Determine load level from wait time. - pub fn from_wait_time(wait_time: Duration, low_threshold: Duration, high_threshold: Duration) -> Self { - if wait_time <= low_threshold { - IoLoadLevel::Low - } else if wait_time <= high_threshold { - IoLoadLevel::Medium - } else if wait_time <= high_threshold * 2 { - IoLoadLevel::High - } else { - IoLoadLevel::Critical - } - } -} - -impl std::fmt::Display for IoLoadLevel { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.as_str()) - } -} - -/// Bandwidth tier classification. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum BandwidthTier { - /// Low bandwidth (< 100 MB/s). - Low, - /// Medium bandwidth (100-500 MB/s). - #[default] - Medium, - /// High bandwidth (> 500 MB/s). - High, - /// Unknown bandwidth. - Unknown, -} - -impl BandwidthTier { - /// Determine bandwidth tier from bytes per second. - pub fn from_bps(bps: u64) -> Self { - const MB: u64 = 1024 * 1024; - if bps < 100 * MB { - BandwidthTier::Low - } else if bps < 500 * MB { - BandwidthTier::Medium - } else { - BandwidthTier::High - } - } - - /// Get the tier as a string for metrics labels. - pub fn as_str(&self) -> &'static str { - match self { - BandwidthTier::Low => "low", - BandwidthTier::Medium => "medium", - BandwidthTier::High => "high", - BandwidthTier::Unknown => "unknown", - } - } -} - -/// I/O strategy decision. -#[derive(Debug, Clone)] -pub struct IoStrategy { - /// Buffer size to use for I/O operations. - pub buffer_size: usize, - /// Buffer multiplier based on storage media. - pub buffer_multiplier: f64, - /// Whether to enable readahead. - pub enable_readahead: bool, - /// Whether to use buffered I/O. - pub use_buffered_io: bool, - - // Performance state - /// Current number of concurrent requests. - pub concurrent_requests: usize, - /// Observed bandwidth in bytes per second. - pub observed_bandwidth_bps: Option, - /// Bandwidth tier classification. - pub bandwidth_tier: BandwidthTier, - /// Current load level. - pub load_level: IoLoadLevel, - - // Priority - /// I/O priority for this operation. - pub priority: IoPriority, - - // Decision flags - /// Whether to throttle random I/O. - pub should_throttle_random_io: bool, - /// Whether to expand buffer for sequential access. - pub should_expand_for_sequential: bool, - /// Whether to reduce buffer due to concurrency. - pub should_reduce_for_concurrency: bool, - /// Whether to reduce buffer due to low bandwidth. - pub should_reduce_for_bandwidth: bool, -} - -impl Default for IoStrategy { - fn default() -> Self { - Self { - buffer_size: 128 * 1024, - buffer_multiplier: 1.0, - enable_readahead: true, - use_buffered_io: true, - concurrent_requests: 0, - observed_bandwidth_bps: None, - bandwidth_tier: BandwidthTier::Medium, - load_level: IoLoadLevel::Low, - priority: IoPriority::Normal, - should_throttle_random_io: false, - should_expand_for_sequential: false, - should_reduce_for_concurrency: false, - should_reduce_for_bandwidth: false, - } - } -} - -impl IoStrategy { - /// Create a new strategy with default values. - pub fn new() -> Self { - Self::default() - } - - /// Create a strategy for sequential access. - pub fn sequential(buffer_size: usize) -> Self { - Self { - buffer_size, - enable_readahead: true, - should_expand_for_sequential: true, - ..Self::default() - } - } - - /// Create a strategy for random access. - pub fn random(buffer_size: usize) -> Self { - Self { - buffer_size, - enable_readahead: false, - should_throttle_random_io: true, - ..Self::default() - } - } -} - -/// I/O load metrics. -#[derive(Debug, Clone, Default)] -pub struct IoLoadMetrics { - /// Number of samples in the current window. - pub sample_count: usize, - /// Total wait time in the window. - pub total_wait_time: Duration, - /// Maximum wait time in the window. - pub max_wait_time: Duration, - /// Average wait time. - pub avg_wait_time: Duration, - /// Current load level. - pub load_level: IoLoadLevel, -} - -impl IoLoadMetrics { - /// Create new load metrics. - pub fn new() -> Self { - Self::default() - } - - /// Add a wait time sample. - pub fn add_sample(&mut self, wait_time: Duration) { - self.sample_count += 1; - self.total_wait_time += wait_time; - if wait_time > self.max_wait_time { - self.max_wait_time = wait_time; - } - self.avg_wait_time = if self.sample_count > 0 { - self.total_wait_time / self.sample_count as u32 - } else { - Duration::ZERO - }; - } - - /// Update load level based on thresholds. - pub fn update_load_level(&mut self, low_threshold: Duration, high_threshold: Duration) { - self.load_level = IoLoadLevel::from_wait_time(self.avg_wait_time, low_threshold, high_threshold); - } - - /// Reset the metrics. - pub fn reset(&mut self) { - *self = Self::default(); - } -} - -/// I/O scheduler. -pub struct IoScheduler { - /// Scheduler configuration. - config: IoSchedulerConfig, - /// Active request counter. - active_requests: AtomicUsize, - /// Load metrics. - load_metrics: std::sync::Mutex, -} - -impl IoScheduler { - /// Create a new I/O scheduler with the given configuration. - pub fn new(config: IoSchedulerConfig) -> Self { - Self { - config, - active_requests: AtomicUsize::new(0), - load_metrics: std::sync::Mutex::new(IoLoadMetrics::new()), - } - } - - /// Create a new I/O scheduler with default configuration. - pub fn with_defaults() -> Self { - Self::new(IoSchedulerConfig::default()) - } - - /// Get the scheduler configuration. - pub fn config(&self) -> &IoSchedulerConfig { - &self.config - } - - /// Get the current number of active requests. - pub fn active_requests(&self) -> usize { - self.active_requests.load(Ordering::Relaxed) - } - - /// Increment the active request count. - pub fn increment_requests(&self) { - self.active_requests.fetch_add(1, Ordering::Relaxed); - } - - /// Decrement the active request count. - pub fn decrement_requests(&self) { - self.active_requests.fetch_sub(1, Ordering::Relaxed); - } - - /// Calculate I/O strategy for a request. - pub fn calculate_strategy(&self, file_size: i64, permit_wait_time: Duration, is_sequential: bool) -> IoStrategy { - let concurrent_requests = self.active_requests.load(Ordering::Relaxed); - - // Determine priority based on file size - let priority = IoPriority::from_size( - file_size, - self.config.high_priority_size_threshold, - self.config.low_priority_size_threshold, - ); - - // Determine load level - let load_level = - IoLoadLevel::from_wait_time(permit_wait_time, self.config.load_low_threshold(), self.config.load_high_threshold()); - - // Calculate base buffer size - let base_buffer = self.config.base_buffer_size; - - // Adjust for concurrency - let concurrency_factor = match concurrent_requests { - 0..=2 => 1.0, - 3..=4 => 0.75, - 5..=8 => 0.5, - _ => 0.4, - }; - - // Adjust for load level - let load_factor = match load_level { - IoLoadLevel::Low => 1.2, - IoLoadLevel::Medium => 1.0, - IoLoadLevel::High => 0.7, - IoLoadLevel::Critical => 0.5, - }; - - // Adjust for access pattern - let sequential_factor = if is_sequential { 1.5 } else { 1.0 }; - - // Calculate final buffer size - let buffer_size = (base_buffer as f64 * concurrency_factor * load_factor * sequential_factor) as usize; - let buffer_size = buffer_size.clamp(self.config.min_buffer_size, self.config.max_buffer_size); - - IoStrategy { - buffer_size, - buffer_multiplier: concurrency_factor * load_factor * sequential_factor, - enable_readahead: is_sequential && load_level != IoLoadLevel::Critical, - use_buffered_io: true, - concurrent_requests, - observed_bandwidth_bps: None, - bandwidth_tier: BandwidthTier::Unknown, - load_level, - priority, - should_throttle_random_io: !is_sequential && load_level >= IoLoadLevel::High, - should_expand_for_sequential: is_sequential && load_level <= IoLoadLevel::Medium, - should_reduce_for_concurrency: concurrent_requests > 4, - should_reduce_for_bandwidth: false, - } - } - - /// Calculate multi-factor I/O strategy. - pub fn calculate_multi_factor_strategy( - &self, - file_size: i64, - permit_wait_time: Duration, - is_sequential: bool, - storage_profile: Option<&StorageProfile>, - ) -> IoStrategy { - let mut strategy = self.calculate_strategy(file_size, permit_wait_time, is_sequential); - - // Apply storage profile adjustments - if let Some(profile) = storage_profile { - // Adjust buffer size based on storage media - let media_factor = match profile.media { - StorageMedia::Nvme => 1.5, - StorageMedia::Ssd => 1.2, - StorageMedia::Hdd => 0.8, - StorageMedia::Unknown => 1.0, - }; - - strategy.buffer_size = (strategy.buffer_size as f64 * media_factor).min(self.config.max_buffer_size as f64) as usize; - - // Apply sequential boost if applicable - if is_sequential { - strategy.buffer_size = (strategy.buffer_size as f64 * profile.sequential_boost_multiplier) - .min(self.config.max_buffer_size as f64) as usize; - } - - // Apply random penalty if applicable - if !is_sequential { - strategy.buffer_size = (strategy.buffer_size as f64 * profile.random_penalty_multiplier) - .max(self.config.min_buffer_size as f64) as usize; - } - - // Update readahead preference - strategy.enable_readahead = strategy.enable_readahead && profile.prefers_readahead; - } - - strategy - } - - /// Record a wait time sample for load tracking. - pub fn record_wait_time(&self, wait_time: Duration) { - if let Ok(mut metrics) = self.load_metrics.lock() { - metrics.add_sample(wait_time); - metrics.update_load_level(self.config.load_low_threshold(), self.config.load_high_threshold()); - } - } - - /// Get current load metrics. - pub fn load_metrics(&self) -> IoLoadMetrics { - if let Ok(metrics) = self.load_metrics.lock() { - metrics.clone() - } else { - IoLoadMetrics::default() - } - } -} - -impl Default for IoScheduler { - fn default() -> Self { - Self::with_defaults() - } -} - -// ============================================================================ -// Buffer Size Calculation Functions -// ============================================================================ - -/// Constants for buffer size calculations. -pub const KI_B: usize = 1024; -pub const MI_B: usize = 1024 * 1024; - -/// Get concurrency-aware buffer size. -/// -/// Adjusts buffer size based on the current level of concurrent requests. -/// Higher concurrency leads to smaller buffers to reduce memory pressure. -/// -/// # Arguments -/// -/// * `file_size` - Size of the file being read (-1 if unknown) -/// * `base_buffer_size` - Base buffer size from workload profile -/// -/// # Returns -/// -/// Adjusted buffer size in bytes -pub fn get_concurrency_aware_buffer_size(file_size: i64, base_buffer_size: usize) -> usize { - // Get current concurrency level from global counter - let concurrent_requests = 1; // Default to 1 if no global counter available - - // Define concurrency thresholds - let medium_threshold = 4; - let high_threshold = 8; - - // Calculate adaptive multiplier based on concurrency - let adaptive_multiplier = if concurrent_requests <= 2 { - // Low concurrency (1-2): use full buffer size - 1.0 - } else if concurrent_requests <= medium_threshold { - // Medium concurrency (3-4): slightly reduce buffer size (75% of base) - 0.75 - } else if concurrent_requests <= high_threshold { - // Higher concurrency (5-8): more aggressive reduction (50% of base) - 0.5 - } else { - // Very high concurrency (>8): minimize memory per request (40% of base) - 0.4 - }; - - // Calculate the adjusted buffer size - let adjusted_size = (base_buffer_size as f64 * adaptive_multiplier) as usize; - - // Ensure we stay within reasonable bounds - let min_buffer = if file_size > 0 && file_size < 100 * KI_B as i64 { - 32 * KI_B // For very small files, use minimum buffer - } else { - 64 * KI_B // Standard minimum buffer size - }; - - let max_buffer = if concurrent_requests > high_threshold { - 256 * KI_B // Cap at 256KB for high concurrency - } else { - MI_B // Cap at 1MB for lower concurrency - }; - - adjusted_size.clamp(min_buffer, max_buffer) -} - -/// Advanced concurrency-aware buffer sizing with file size optimization. -/// -/// This enhanced version considers both concurrency level and file size patterns -/// to provide even better performance characteristics. -/// -/// # Arguments -/// -/// * `file_size` - Size of the file being read (-1 if unknown) -/// * `base_buffer_size` - Baseline buffer size from workload profile -/// * `is_sequential` - Whether this is a sequential read (hint for optimization) -/// * `concurrent_requests` - Current number of concurrent requests -/// -/// # Returns -/// -/// Optimized buffer size in bytes -pub fn get_advanced_buffer_size( - file_size: i64, - base_buffer_size: usize, - is_sequential: bool, - concurrent_requests: usize, -) -> usize { - // For very small files, use smaller buffers regardless of concurrency - if file_size > 0 && file_size < 256 * KI_B as i64 { - return (file_size as usize / 4).clamp(16 * KI_B, 64 * KI_B); - } - - // Base calculation from standard function - let standard_size = get_concurrency_aware_buffer_size(file_size, base_buffer_size); - - let medium_threshold = 4; - let high_threshold = 8; - - // For sequential reads, we can be more aggressive with buffer sizes - if is_sequential && concurrent_requests <= medium_threshold { - // Boost buffer size for sequential reads under low concurrency - let boosted = (standard_size as f64 * 1.5) as usize; - return boosted.min(MI_B); - } - - // For random reads under high concurrency, reduce buffer size - if !is_sequential && concurrent_requests > high_threshold { - let reduced = (standard_size as f64 * 0.7) as usize; - return reduced.max(32 * KI_B); - } - - standard_size -} - -/// Get buffer size with storage media optimization. -/// -/// Adjusts buffer size based on storage media characteristics. -/// -/// # Arguments -/// -/// * `base_size` - Base buffer size -/// * `media` - Storage media type -/// -/// # Returns -/// -/// Optimized buffer size for the storage media -pub fn get_buffer_size_for_media(base_size: usize, media: StorageMedia) -> usize { - let multiplier = match media { - StorageMedia::Nvme => 1.5, // NVMe can handle larger buffers - StorageMedia::Ssd => 1.2, // SSD benefits from moderate buffers - StorageMedia::Hdd => 0.8, // HDD prefers smaller buffers to reduce seek overhead - StorageMedia::Unknown => 1.0, - }; - - (base_size as f64 * multiplier).min(MI_B as f64) as usize -} - -/// Calculate optimal buffer size using multi-factor analysis. -/// -/// This is the main entry point for buffer size calculation, considering -/// all factors: concurrency, storage media, access pattern, and load. -/// -/// # Arguments -/// -/// * `file_size` - Size of the file being read -/// * `base_buffer_size` - Base buffer size -/// * `is_sequential` - Whether access is sequential -/// * `concurrent_requests` - Current concurrency level -/// * `media` - Storage media type -/// * `load_level` - Current I/O load level -/// -/// # Returns -/// -/// Optimally calculated buffer size -pub fn calculate_optimal_buffer_size( - file_size: i64, - base_buffer_size: usize, - is_sequential: bool, - concurrent_requests: usize, - media: StorageMedia, - load_level: IoLoadLevel, -) -> usize { - // Start with advanced buffer size calculation - let mut buffer_size = get_advanced_buffer_size(file_size, base_buffer_size, is_sequential, concurrent_requests); - - // Apply storage media optimization - buffer_size = get_buffer_size_for_media(buffer_size, media); - - // Apply load-based adjustment - let load_multiplier = match load_level { - IoLoadLevel::Low => 1.2, - IoLoadLevel::Medium => 1.0, - IoLoadLevel::High => 0.7, - IoLoadLevel::Critical => 0.5, - }; - - buffer_size = (buffer_size as f64 * load_multiplier) as usize; - - // Final bounds check - buffer_size.clamp(32 * KI_B, MI_B) -} - -/// I/O scheduling context for multi-factor strategy calculation. -#[derive(Debug, Clone)] -pub struct IoSchedulingContext { - /// File size in bytes (-1 if unknown). - pub file_size: i64, - /// Base buffer size from configuration. - pub base_buffer_size: usize, - /// Time spent waiting for permit. - pub permit_wait_duration: Duration, - /// Whether access is sequential. - pub is_sequential_hint: bool, - /// Detected access pattern. - pub access_pattern: AccessPattern, - /// Detected storage media. - pub storage_media: StorageMedia, - /// Observed bandwidth in bytes per second. - pub observed_bandwidth_bps: Option, - /// Current concurrent request count. - pub concurrent_requests: usize, -} - -impl Default for IoSchedulingContext { - fn default() -> Self { - Self { - file_size: -1, - base_buffer_size: 128 * KI_B, - permit_wait_duration: Duration::ZERO, - is_sequential_hint: true, - access_pattern: AccessPattern::Unknown, - storage_media: StorageMedia::Unknown, - observed_bandwidth_bps: None, - concurrent_requests: 1, - } - } -} - -impl IoSchedulingContext { - /// Create a new scheduling context. - pub fn new(file_size: i64, base_buffer_size: usize) -> Self { - Self { - file_size, - base_buffer_size, - ..Self::default() - } - } - - /// Builder pattern: set sequential hint. - pub fn with_sequential(mut self, is_sequential: bool) -> Self { - self.is_sequential_hint = is_sequential; - self.access_pattern = if is_sequential { - AccessPattern::Sequential - } else { - AccessPattern::Random - }; - self - } - - /// Builder pattern: set storage media. - pub fn with_media(mut self, media: StorageMedia) -> Self { - self.storage_media = media; - self - } - - /// Builder pattern: set bandwidth. - pub fn with_bandwidth(mut self, bps: u64) -> Self { - self.observed_bandwidth_bps = Some(bps); - self - } - - /// Builder pattern: set concurrency. - pub fn with_concurrency(mut self, count: usize) -> Self { - self.concurrent_requests = count; - self - } - - /// Builder pattern: set wait duration. - pub fn with_wait_duration(mut self, duration: Duration) -> Self { - self.permit_wait_duration = duration; - self - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_io_priority() { - assert_eq!(IoPriority::from_size(1024, 64 * 1024, 4 * 1024 * 1024), IoPriority::High); - assert_eq!(IoPriority::from_size(1024 * 1024, 64 * 1024, 4 * 1024 * 1024), IoPriority::Normal); - assert_eq!(IoPriority::from_size(10 * 1024 * 1024, 64 * 1024, 4 * 1024 * 1024), IoPriority::Low); - } - - #[test] - fn test_io_priority_unknown_size_is_normal() { - // -1 means "size unknown" and must not wrap to usize::MAX (=> Low). - assert_eq!(IoPriority::from_size(-1, 64 * 1024, 4 * 1024 * 1024), IoPriority::Normal); - assert_eq!(IoPriority::from_size(i64::MIN, 64 * 1024, 4 * 1024 * 1024), IoPriority::Normal); - } - - #[test] - fn test_io_load_level() { - let low = Duration::from_millis(5); - let high = Duration::from_millis(50); - - assert_eq!(IoLoadLevel::from_wait_time(Duration::from_millis(1), low, high), IoLoadLevel::Low); - assert_eq!(IoLoadLevel::from_wait_time(Duration::from_millis(20), low, high), IoLoadLevel::Medium); - assert_eq!(IoLoadLevel::from_wait_time(Duration::from_millis(60), low, high), IoLoadLevel::High); - assert_eq!(IoLoadLevel::from_wait_time(Duration::from_millis(150), low, high), IoLoadLevel::Critical); - } - - #[test] - fn test_bandwidth_tier() { - assert_eq!(BandwidthTier::from_bps(50 * 1024 * 1024), BandwidthTier::Low); - assert_eq!(BandwidthTier::from_bps(200 * 1024 * 1024), BandwidthTier::Medium); - assert_eq!(BandwidthTier::from_bps(600 * 1024 * 1024), BandwidthTier::High); - } - - #[test] - fn test_io_strategy_default() { - let strategy = IoStrategy::default(); - assert!(strategy.buffer_size > 0); - assert!(strategy.enable_readahead); - } - - #[test] - fn test_io_scheduler() { - let scheduler = IoScheduler::with_defaults(); - - let strategy = scheduler.calculate_strategy(1024 * 1024, Duration::from_millis(5), true); - assert!(strategy.buffer_size > 0); - assert!(strategy.enable_readahead); - assert_eq!(strategy.load_level, IoLoadLevel::Low); - } - - #[test] - fn test_io_scheduler_with_concurrency() { - let scheduler = IoScheduler::with_defaults(); - - // Simulate concurrent requests - scheduler.increment_requests(); - scheduler.increment_requests(); - scheduler.increment_requests(); - - let strategy = scheduler.calculate_strategy(1024 * 1024, Duration::from_millis(5), true); - assert_eq!(strategy.concurrent_requests, 3); - } - - #[test] - fn test_load_metrics() { - let mut metrics = IoLoadMetrics::new(); - - metrics.add_sample(Duration::from_millis(10)); - metrics.add_sample(Duration::from_millis(20)); - metrics.add_sample(Duration::from_millis(30)); - - assert_eq!(metrics.sample_count, 3); - assert_eq!(metrics.avg_wait_time, Duration::from_millis(20)); - assert_eq!(metrics.max_wait_time, Duration::from_millis(30)); - } - - #[test] - fn test_get_concurrency_aware_buffer_size() { - // Test with default concurrency (1) - let size = get_concurrency_aware_buffer_size(1024 * 1024, 128 * KI_B); - assert!(size >= 64 * KI_B); - assert!(size <= MI_B); - - // Test with small file - let size = get_concurrency_aware_buffer_size(50 * KI_B as i64, 128 * KI_B); - assert!(size >= 32 * KI_B); - } - - #[test] - fn test_get_advanced_buffer_size() { - // Sequential read with low concurrency - let size = get_advanced_buffer_size(10 * MI_B as i64, 128 * KI_B, true, 2); - assert!(size >= 128 * KI_B); - - // Random read with high concurrency - let size = get_advanced_buffer_size(10 * MI_B as i64, 128 * KI_B, false, 10); - assert!(size >= 32 * KI_B); - - // Very small file - let size = get_advanced_buffer_size(100 * KI_B as i64, 128 * KI_B, true, 1); - assert!(size <= 64 * KI_B); - } - - #[test] - fn test_get_buffer_size_for_media() { - let base = 128 * KI_B; - - // NVMe should get larger buffers - let nvme_size = get_buffer_size_for_media(base, StorageMedia::Nvme); - assert!(nvme_size > base); - - // SSD should get slightly larger buffers - let ssd_size = get_buffer_size_for_media(base, StorageMedia::Ssd); - assert!(ssd_size > base); - - // HDD should get smaller buffers - let hdd_size = get_buffer_size_for_media(base, StorageMedia::Hdd); - assert!(hdd_size < base); - } - - #[test] - fn test_calculate_optimal_buffer_size() { - // Low load, sequential, NVMe - let size = calculate_optimal_buffer_size(10 * MI_B as i64, 128 * KI_B, true, 2, StorageMedia::Nvme, IoLoadLevel::Low); - assert!(size >= 32 * KI_B); - assert!(size <= MI_B); - - // Critical load, random, HDD - let size = - calculate_optimal_buffer_size(10 * MI_B as i64, 128 * KI_B, false, 10, StorageMedia::Hdd, IoLoadLevel::Critical); - assert!(size >= 32 * KI_B); - assert!(size <= MI_B); - } - - #[test] - fn test_io_scheduling_context() { - let ctx = IoSchedulingContext::new(10 * MI_B as i64, 256 * KI_B) - .with_sequential(true) - .with_media(StorageMedia::Nvme) - .with_bandwidth(500 * MI_B as u64) - .with_concurrency(4); - - assert_eq!(ctx.file_size, 10 * MI_B as i64); - assert_eq!(ctx.base_buffer_size, 256 * KI_B); - assert!(ctx.is_sequential_hint); - assert_eq!(ctx.storage_media, StorageMedia::Nvme); - assert_eq!(ctx.observed_bandwidth_bps, Some(500 * MI_B as u64)); - assert_eq!(ctx.concurrent_requests, 4); - } -} diff --git a/crates/io-core/src/shared_memory.rs b/crates/io-core/src/shared_memory.rs deleted file mode 100644 index e21781beb..000000000 --- a/crates/io-core/src/shared_memory.rs +++ /dev/null @@ -1,320 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Shared memory pool for zero-copy data sharing. -//! -//! This module provides Arc-based shared memory management for -//! efficient cross-task data passing without serialization. - -use std::convert::AsRef; -use std::ops::Deref; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Instant; - -/// Shared memory pool configuration. -#[derive(Debug, Clone)] -pub struct SharedMemoryConfig { - /// Whether shared memory is enabled - pub enabled: bool, - - /// Maximum pool size in bytes - pub max_pool_size: usize, - - /// Maximum object size in bytes - pub max_object_size: usize, -} - -impl Default for SharedMemoryConfig { - fn default() -> Self { - Self { - enabled: true, - max_pool_size: 100 * 1024 * 1024, // 100MB - max_object_size: 10 * 1024 * 1024, // 10MB - } - } -} - -/// Shared memory pool statistics. -#[derive(Debug, Default)] -pub struct SharedMemoryStats { - /// Total number of objects created - pub total_objects: AtomicU64, - - /// Total number of shared references - pub total_shared_refs: AtomicU64, - - /// Current memory usage in bytes - pub current_memory: AtomicU64, - - /// Peak memory usage in bytes - pub peak_memory: AtomicU64, -} - -/// Arc data metadata. -#[derive(Clone, Debug)] -pub struct ArcMetadata { - /// Size of the data (if measurable) - pub size: Option, - - /// Creation timestamp - pub created_at: Instant, -} - -/// Arc-based data wrapper for zero-copy sharing. -/// -/// This wrapper uses Arc to enable shared ownership of data -/// across multiple tasks without copying. -pub struct ArcData { - /// The wrapped data - inner: Arc, - - /// Metadata about the data - metadata: ArcMetadata, -} - -impl Clone for ArcData { - fn clone(&self) -> Self { - Self { - inner: Arc::clone(&self.inner), - metadata: self.metadata.clone(), - } - } -} - -impl ArcData { - /// Create a new ArcData wrapper. - pub fn new(data: T) -> Self { - ArcData { - inner: Arc::new(data), - metadata: ArcMetadata { - size: None, - created_at: Instant::now(), - }, - } - } - - /// Create a new ArcData wrapper with known size. - pub fn with_size(data: T, size: usize) -> Self { - ArcData { - inner: Arc::new(data), - metadata: ArcMetadata { - size: Some(size), - created_at: Instant::now(), - }, - } - } - - /// Get the reference count. - pub fn ref_count(&self) -> usize { - Arc::strong_count(&self.inner) - } - - /// Convert into the underlying Arc. - pub fn into_arc(self) -> Arc { - self.inner - } - - /// Get the metadata. - pub fn metadata(&self) -> &ArcMetadata { - &self.metadata - } - - /// Get the size if known. - pub fn size(&self) -> Option { - self.metadata.size - } -} - -impl AsRef for ArcData { - fn as_ref(&self) -> &T { - &self.inner - } -} - -impl Deref for ArcData { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -impl std::fmt::Debug for ArcData { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ArcData") - .field("ref_count", &self.ref_count()) - .field("metadata", &self.metadata) - .finish() - } -} - -/// Shared memory pool for managing Arc-based shared data. -pub struct SharedMemoryPool { - config: SharedMemoryConfig, - stats: SharedMemoryStats, -} - -impl SharedMemoryPool { - /// Create a new shared memory pool with the given configuration. - pub fn new(config: SharedMemoryConfig) -> Self { - Self { - config, - stats: SharedMemoryStats::default(), - } - } - - /// Create a new shared memory pool with default configuration. - pub fn with_defaults() -> Self { - Self::new(SharedMemoryConfig::default()) - } - - /// Create shared data. - /// - /// This method wraps the data in an ArcData for zero-copy sharing. - pub fn create(&self, data: T) -> ArcData { - self.stats.total_objects.fetch_add(1, Ordering::Relaxed); - ArcData::new(data) - } - - /// Create shared data with known size. - /// - /// This method tracks memory usage for statistics. - pub fn create_with_size(&self, data: T, size: usize) -> ArcData { - self.stats.total_objects.fetch_add(1, Ordering::Relaxed); - - // Update memory statistics - self.stats.current_memory.fetch_add(size as u64, Ordering::Relaxed); - - // Update peak memory - let current = self.stats.current_memory.load(Ordering::Relaxed); - let mut peak = self.stats.peak_memory.load(Ordering::Relaxed); - if current > peak { - peak = current; - self.stats.peak_memory.store(peak, Ordering::Relaxed); - } - - ArcData::with_size(data, size) - } - - /// Share data by increasing reference count. - /// - /// This method creates a new ArcData that shares the underlying data - /// without copying. - pub fn share(&self, data: &ArcData) -> ArcData { - self.stats.total_shared_refs.fetch_add(1, Ordering::Relaxed); - data.clone() - } - - /// Get the statistics for this pool. - pub fn stats(&self) -> &SharedMemoryStats { - &self.stats - } - - /// Get the configuration for this pool. - pub fn config(&self) -> &SharedMemoryConfig { - &self.config - } - - /// Check if the pool is enabled. - pub fn is_enabled(&self) -> bool { - self.config.enabled - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_arc_data_new() { - let data = vec![1u8, 2, 3, 4, 5]; - let arc_data = ArcData::new(data.clone()); - - assert_eq!(arc_data.as_ref(), &data); - assert_eq!(arc_data.ref_count(), 1); - } - - #[test] - fn test_arc_data_clone() { - let data = vec![1u8, 2, 3, 4, 5]; - let arc_data = ArcData::new(data); - - assert_eq!(arc_data.ref_count(), 1); - - let arc_data2 = arc_data.clone(); - assert_eq!(arc_data.ref_count(), 2); - assert_eq!(arc_data2.ref_count(), 2); - - let arc_data3 = arc_data.clone(); - assert_eq!(arc_data.ref_count(), 3); - assert_eq!(arc_data2.ref_count(), 3); - assert_eq!(arc_data3.ref_count(), 3); - } - - #[test] - fn test_arc_data_deref() { - let data = vec![1u8, 2, 3, 4, 5]; - let arc_data = ArcData::new(data); - - // Test Deref trait - assert_eq!(arc_data.len(), 5); - assert_eq!(arc_data[0], 1); - } - - #[test] - fn test_shared_memory_pool_create() { - let pool = SharedMemoryPool::with_defaults(); - let data = vec![1u8, 2, 3, 4, 5]; - - let arc_data = pool.create(data.clone()); - - assert_eq!(arc_data.as_ref(), &data); - assert_eq!(pool.stats().total_objects.load(Ordering::Relaxed), 1); - } - - #[test] - fn test_shared_memory_pool_share() { - let pool = SharedMemoryPool::with_defaults(); - let data = vec![1u8, 2, 3, 4, 5]; - - let arc_data = pool.create(data); - assert_eq!(arc_data.ref_count(), 1); - - let shared = pool.share(&arc_data); - assert_eq!(arc_data.ref_count(), 2); - assert_eq!(shared.ref_count(), 2); - assert_eq!(pool.stats().total_shared_refs.load(Ordering::Relaxed), 1); - } - - #[test] - fn test_shared_memory_pool_with_size() { - let pool = SharedMemoryPool::with_defaults(); - let data = vec![1u8; 1024]; - - let arc_data = pool.create_with_size(data, 1024); - - assert_eq!(arc_data.size(), Some(1024)); - assert_eq!(pool.stats().current_memory.load(Ordering::Relaxed), 1024); - } - - #[test] - fn test_default_config() { - let config = SharedMemoryConfig::default(); - - assert!(config.enabled); - assert_eq!(config.max_pool_size, 100 * 1024 * 1024); - assert_eq!(config.max_object_size, 10 * 1024 * 1024); - } -} diff --git a/crates/io-core/src/timeout_wrapper.rs b/crates/io-core/src/timeout_wrapper.rs deleted file mode 100644 index 9d9ce4b22..000000000 --- a/crates/io-core/src/timeout_wrapper.rs +++ /dev/null @@ -1,501 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Timeout wrapper for I/O operations. -//! -//! This module provides timeout management for I/O operations with -//! dynamic timeout calculation based on operation size. - -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{Duration, Instant}; - -/// Timeout configuration. -#[derive(Debug, Clone)] -pub struct TimeoutConfig { - /// Base timeout for small operations. - pub base_timeout: Duration, - /// Timeout per MB of data. - pub timeout_per_mb: Duration, - /// Maximum timeout. - pub max_timeout: Duration, - /// Minimum timeout. - pub min_timeout: Duration, - /// GetObject operation timeout. - pub get_object_timeout: Duration, - /// PutObject operation timeout. - pub put_object_timeout: Duration, - /// ListObjects operation timeout. - pub list_objects_timeout: Duration, - /// Whether dynamic timeout is enabled. - pub enable_dynamic_timeout: bool, -} - -impl Default for TimeoutConfig { - fn default() -> Self { - Self { - base_timeout: Duration::from_secs(5), - timeout_per_mb: Duration::from_millis(100), - max_timeout: Duration::from_secs(300), - min_timeout: Duration::from_secs(1), - get_object_timeout: Duration::from_secs(30), - put_object_timeout: Duration::from_secs(60), - list_objects_timeout: Duration::from_secs(10), - enable_dynamic_timeout: true, - } - } -} - -impl TimeoutConfig { - /// Create new timeout configuration. - pub fn new() -> Self { - Self::default() - } - - /// Calculate dynamic timeout based on size. - pub fn calculate_timeout(&self, size_bytes: u64) -> Duration { - if !self.enable_dynamic_timeout { - return self.base_timeout; - } - - let mb = size_bytes as f64 / (1024.0 * 1024.0); - let timeout = self.base_timeout + self.timeout_per_mb.mul_f64(mb); - timeout.clamp(self.min_timeout, self.max_timeout) - } - - /// Validate the configuration. - pub fn validate(&self) -> Result<(), TimeoutError> { - if self.min_timeout > self.max_timeout { - return Err(TimeoutError::InvalidConfig("min_timeout must be <= max_timeout".to_string())); - } - if self.base_timeout < self.min_timeout || self.base_timeout > self.max_timeout { - return Err(TimeoutError::InvalidConfig( - "base_timeout must be between min_timeout and max_timeout".to_string(), - )); - } - Ok(()) - } -} - -/// Timeout error. -#[derive(Debug, Clone, thiserror::Error)] -pub enum TimeoutError { - /// Operation timed out. - #[error("Operation timed out after {0:?}")] - TimedOut(Duration), - /// Invalid configuration. - #[error("Invalid timeout config: {0}")] - InvalidConfig(String), -} - -/// Operation progress tracker. -#[derive(Debug)] -pub struct OperationProgress { - /// Total size (if known). - pub total_size: Option, - /// Bytes processed. - bytes_processed: AtomicU64, - /// Last update time. - last_update: std::sync::Mutex, - /// Stale timeout. - stale_timeout: Duration, - /// Start time for transfer rate calculation. - start_time: Instant, -} - -impl OperationProgress { - /// Create new operation progress. - pub fn new(total_size: Option, stale_timeout: Duration) -> Self { - Self { - total_size, - bytes_processed: AtomicU64::new(0), - last_update: std::sync::Mutex::new(Instant::now()), - stale_timeout, - start_time: Instant::now(), - } - } - - /// Update progress. - pub fn update(&self, bytes: u64) { - self.bytes_processed.store(bytes, Ordering::Relaxed); - if let Ok(mut last) = self.last_update.lock() { - *last = Instant::now(); - } - } - - /// Add to progress. - pub fn add(&self, bytes: u64) { - self.bytes_processed.fetch_add(bytes, Ordering::Relaxed); - if let Ok(mut last) = self.last_update.lock() { - *last = Instant::now(); - } - } - - /// Get current progress. - pub fn current(&self) -> u64 { - self.bytes_processed.load(Ordering::Relaxed) - } - - /// Check if progress is stale. - pub fn is_stale(&self) -> bool { - if let Ok(last) = self.last_update.lock() { - last.elapsed() > self.stale_timeout - } else { - false - } - } - - /// Get progress percentage. - pub fn progress_percent(&self) -> Option { - self.total_size.map(|total| { - if total == 0 { - 100.0 - } else { - let processed = self.bytes_processed.load(Ordering::Relaxed); - (processed as f64 / total as f64 * 100.0).min(100.0) - } - }) - } - - /// Get remaining bytes. - pub fn remaining(&self) -> Option { - self.total_size.map(|total| { - let processed = self.bytes_processed.load(Ordering::Relaxed); - total.saturating_sub(processed) - }) - } - - /// Calculate transfer rate in bytes per second. - /// - /// Returns 0 if no time has elapsed or no data transferred. - pub fn transfer_rate(&self) -> u64 { - let processed = self.bytes_processed.load(Ordering::Relaxed); - if processed == 0 { - return 0; - } - - let elapsed = self.start_time.elapsed().as_secs_f64(); - if elapsed > 0.0 { - (processed as f64 / elapsed) as u64 - } else { - 0 - } - } -} - -/// Request timeout wrapper. -pub struct RequestTimeoutWrapper { - /// Configuration. - config: TimeoutConfig, - /// Start time. - start_time: Instant, - /// Operation progress. - progress: Option, -} - -impl RequestTimeoutWrapper { - /// Create a new timeout wrapper. - pub fn new(config: TimeoutConfig) -> Self { - Self { - config, - start_time: Instant::now(), - progress: None, - } - } - - /// Create with progress tracking. - pub fn with_progress(config: TimeoutConfig, total_size: Option, stale_timeout: Duration) -> Self { - Self { - config, - start_time: Instant::now(), - progress: Some(OperationProgress::new(total_size, stale_timeout)), - } - } - - /// Get the configuration. - pub fn config(&self) -> &TimeoutConfig { - &self.config - } - - /// Get elapsed time. - pub fn elapsed(&self) -> Duration { - self.start_time.elapsed() - } - - /// Get remaining time. - pub fn remaining(&self, timeout: Duration) -> Option { - let elapsed = self.elapsed(); - if elapsed >= timeout { None } else { Some(timeout - elapsed) } - } - - /// Check if timed out. - pub fn is_timed_out(&self, size: Option) -> bool { - let timeout = self.get_timeout(size); - self.elapsed() > timeout - } - - /// Get the timeout for a given size. - pub fn get_timeout(&self, size: Option) -> Duration { - if self.config.enable_dynamic_timeout { - if let Some(s) = size { - self.config.calculate_timeout(s) - } else { - self.config.base_timeout - } - } else { - self.config.base_timeout - } - } - - /// Check if timed out and return error if so. - pub fn check_timeout(&self, size: Option) -> Result<(), TimeoutError> { - if self.is_timed_out(size) { - Err(TimeoutError::TimedOut(self.get_timeout(size))) - } else { - Ok(()) - } - } - - /// Get progress. - pub fn progress(&self) -> Option<&OperationProgress> { - self.progress.as_ref() - } - - /// Update progress. - pub fn update_progress(&self, bytes: u64) { - if let Some(ref progress) = self.progress { - progress.update(bytes); - } - } - - /// Check if operation is stalled (no progress for a while). - pub fn is_stalled(&self) -> bool { - self.progress.as_ref().is_some_and(|p| p.is_stale()) - } - - /// Get progress percentage. - pub fn progress_percent(&self) -> Option { - self.progress.as_ref().and_then(|p| p.progress_percent()) - } -} - -/// Timeout statistics. -#[derive(Debug, Default)] -pub struct TimeoutStats { - /// Total operations. - pub total_operations: AtomicU64, - /// Timed out operations. - pub timed_out: AtomicU64, - /// Total wait time in nanoseconds. - pub total_wait_time_ns: AtomicU64, - /// Maximum wait time in nanoseconds. - pub max_wait_time_ns: AtomicU64, -} - -impl TimeoutStats { - /// Create new timeout statistics. - pub fn new() -> Self { - Self::default() - } - - /// Record an operation. - pub fn record_operation(&self, wait_time: Duration) { - self.total_operations.fetch_add(1, Ordering::Relaxed); - let ns = wait_time.as_nanos() as u64; - self.total_wait_time_ns.fetch_add(ns, Ordering::Relaxed); - - let mut current = self.max_wait_time_ns.load(Ordering::Relaxed); - while ns > current { - match self - .max_wait_time_ns - .compare_exchange_weak(current, ns, Ordering::Relaxed, Ordering::Relaxed) - { - Ok(_) => break, - Err(actual) => current = actual, - } - } - } - - /// Record a timeout. - pub fn record_timeout(&self) { - self.timed_out.fetch_add(1, Ordering::Relaxed); - } - - /// Get timeout rate. - pub fn timeout_rate(&self) -> f64 { - let total = self.total_operations.load(Ordering::Relaxed); - let timed_out = self.timed_out.load(Ordering::Relaxed); - if total == 0 { 0.0 } else { timed_out as f64 / total as f64 } - } - - /// Get average wait time. - pub fn avg_wait_time(&self) -> Duration { - let total = self.total_wait_time_ns.load(Ordering::Relaxed); - let count = self.total_operations.load(Ordering::Relaxed); - total.checked_div(count).map(Duration::from_nanos).unwrap_or(Duration::ZERO) - } - - /// Reset statistics. - pub fn reset(&self) { - self.total_operations.store(0, Ordering::Relaxed); - self.timed_out.store(0, Ordering::Relaxed); - self.total_wait_time_ns.store(0, Ordering::Relaxed); - self.max_wait_time_ns.store(0, Ordering::Relaxed); - } -} - -/// Calculate adaptive timeout based on historical data and current conditions. -/// -/// This function adjusts the timeout based on: -/// - Historical transfer rate -/// - Recent timeout count -/// - Object size -pub fn calculate_adaptive_timeout( - base_timeout: Duration, - historical_rate_bps: Option, - recent_timeout_count: u32, - object_size: u64, -) -> Duration { - // If we have recent timeouts, increase timeout - let timeout_multiplier = if recent_timeout_count > 3 { - 2.0 // Double timeout if many recent timeouts - } else if recent_timeout_count > 1 { - 1.5 // 50% increase if some timeouts - } else { - 1.0 // No adjustment - }; - - // Adaptive timeout bounds: 5 seconds minimum, 10 minutes maximum. - const MIN_SECS: f64 = 5.0; - const MAX_SECS: f64 = 600.0; - - // If we have historical rate data, use it for estimation - let estimated_secs = match historical_rate_bps { - Some(rate) if rate > 0 => (object_size as f64 / rate as f64) * 1.2, // 20% buffer - _ => base_timeout.as_secs_f64(), - }; - - // Clamp BEFORE constructing the Duration: `from_secs_f64` panics when the - // estimate overflows Duration (huge object_size with a tiny historical rate). - Duration::from_secs_f64((estimated_secs * timeout_multiplier).clamp(MIN_SECS, MAX_SECS)) -} - -/// Estimate bytes per second transfer rate. -/// -/// This is used for adaptive timeout calculation. -pub fn estimate_bytes_per_second(object_size: u64, expected_duration: Duration) -> u64 { - let secs = expected_duration.as_secs_f64(); - if secs > 0.0 { - (object_size as f64 / secs) as u64 - } else { - // Return a reasonable default (1 MB/s) - 1024 * 1024 - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_timeout_config() { - let config = TimeoutConfig::default(); - assert!(config.validate().is_ok()); - - // Small file - let timeout = config.calculate_timeout(1024); - assert!(timeout >= config.min_timeout); - - // Large file - let timeout = config.calculate_timeout(100 * 1024 * 1024); - assert!(timeout <= config.max_timeout); - } - - #[test] - fn test_timeout_config_validation() { - let config = TimeoutConfig { - min_timeout: Duration::from_secs(10), - max_timeout: Duration::from_secs(5), - ..Default::default() - }; - assert!(config.validate().is_err()); - } - - #[test] - fn test_adaptive_timeout_extreme_estimate_does_not_panic() { - // A huge object with a tiny historical rate used to overflow - // Duration::from_secs_f64 and panic; it must clamp to the upper bound. - let timeout = calculate_adaptive_timeout(Duration::from_secs(30), Some(1), 0, u64::MAX); - assert_eq!(timeout, Duration::from_secs(600)); - - // Tiny estimates clamp to the lower bound. - let timeout = calculate_adaptive_timeout(Duration::from_secs(30), Some(u64::MAX), 0, 1); - assert_eq!(timeout, Duration::from_secs(5)); - } - - #[test] - fn test_operation_progress() { - let progress = OperationProgress::new(Some(1000), Duration::from_secs(5)); - - assert_eq!(progress.current(), 0); - assert_eq!(progress.progress_percent(), Some(0.0)); - - progress.update(500); - assert_eq!(progress.current(), 500); - assert_eq!(progress.progress_percent(), Some(50.0)); - - progress.add(300); - assert_eq!(progress.current(), 800); - assert_eq!(progress.remaining(), Some(200)); - } - - #[test] - fn test_request_timeout_wrapper() { - let config = TimeoutConfig { - base_timeout: Duration::from_millis(100), - enable_dynamic_timeout: false, - ..Default::default() - }; - let wrapper = RequestTimeoutWrapper::new(config); - - assert!(!wrapper.is_timed_out(None)); - - std::thread::sleep(Duration::from_millis(150)); - - assert!(wrapper.is_timed_out(None)); - assert!(wrapper.check_timeout(None).is_err()); - } - - #[test] - fn test_timeout_stats() { - let stats = TimeoutStats::new(); - - stats.record_operation(Duration::from_millis(10)); - stats.record_operation(Duration::from_millis(20)); - stats.record_timeout(); - - assert_eq!(stats.total_operations.load(Ordering::Relaxed), 2); - assert_eq!(stats.timed_out.load(Ordering::Relaxed), 1); - assert!((stats.timeout_rate() - 0.5).abs() < 0.01); - } - - #[test] - fn test_progress_tracking() { - let config = TimeoutConfig::default(); - let wrapper = RequestTimeoutWrapper::with_progress(config, Some(1000), Duration::from_secs(1)); - - wrapper.update_progress(500); - assert_eq!(wrapper.progress_percent(), Some(50.0)); - assert!(!wrapper.is_stalled()); - } -} diff --git a/crates/io-core/src/writer.rs b/crates/io-core/src/writer.rs deleted file mode 100644 index 2d1f1a754..000000000 --- a/crates/io-core/src/writer.rs +++ /dev/null @@ -1,443 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! BytesMut-backed object writer for optimized write operations. -//! -//! It uses `BytesMut` for efficient buffering; writes into that buffer may -//! still copy input bytes. The historical `ZeroCopyObjectWriter` name remains -//! available as a deprecated compatibility alias. - -use bytes::{BufMut, Bytes, BytesMut}; -use std::pin::Pin; -use std::task::{Context, Poll}; -use tokio::io::AsyncWrite; - -/// BytesMut-backed object writer for optimized write operations. -/// -/// This writer minimizes memory allocations by: -/// - Using BytesMut for efficient buffer growth -/// - Accepting `Bytes` inputs for efficient buffer handling -/// - Optional integration with BytesPool for buffer reuse -/// -/// # Example -/// -/// ```ignore -/// use rustfs_io_core::BytesMutWriter; -/// use bytes::Bytes; -/// -/// #[tokio::main] -/// async fn main() -> Result<(), Box> { -/// let mut writer = BytesMutWriter::new(); -/// -/// // Write into the internal BytesMut buffer -/// let data = Bytes::from("hello world"); -/// writer.write_buffered(data).await?; -/// -/// // Get the result as Bytes (zero-copy conversion) -/// let result = writer.into_bytes(); -/// -/// Ok(()) -/// } -/// ``` -pub struct BytesMutWriter { - /// Internal buffer using BytesMut for efficient growth - buffer: BytesMut, - /// Total bytes written - bytes_written: usize, - /// Whether the writer has been finalized - finalized: bool, -} - -/// Historical name for the BytesMut-backed object writer. -#[deprecated(since = "1.0.0-beta.8", note = "use BytesMutWriter; writes append into a BytesMut buffer")] -pub type ZeroCopyObjectWriter = BytesMutWriter; - -impl BytesMutWriter { - /// Create a new bytes-backed object writer with default capacity (8KB). - /// - /// # Example - /// - /// ```ignore - /// let writer = BytesMutWriter::new(); - /// ``` - pub fn new() -> Self { - Self::with_capacity(8 * 1024) - } - - /// Create a new bytes-backed object writer with specified capacity. - /// - /// # Arguments - /// - /// * `capacity` - Initial buffer capacity in bytes - /// - /// # Example - /// - /// ```ignore - /// let writer = BytesMutWriter::with_capacity(64 * 1024); - /// ``` - pub fn with_capacity(capacity: usize) -> Self { - Self { - buffer: BytesMut::with_capacity(capacity), - bytes_written: 0, - finalized: false, - } - } - - /// Write data into the internal buffer. - /// - /// This method accepts `Bytes` for API compatibility, then appends the - /// bytes into the internal `BytesMut` buffer. - /// - /// # Arguments - /// - /// * `data` - Data to append to the internal buffer - /// - /// # Returns - /// - /// * `Ok(usize)` - Number of bytes written - /// * `Err(ZeroCopyWriteError)` - Write error - /// - /// # Example - /// - /// ```ignore - /// let data = Bytes::from("hello world"); - /// let written = writer.write_buffered(data).await?; - /// ``` - pub async fn write_buffered(&mut self, data: Bytes) -> Result { - if self.finalized { - return Err(ZeroCopyWriteError::Finalized("Cannot write to finalized writer".to_string())); - } - - let len = data.len(); - self.buffer.put(data); - - self.bytes_written += len; - Ok(len) - } - - /// Historical name for `write_buffered`. - #[deprecated( - since = "1.0.0-beta.8", - note = "use write_buffered; this method appends bytes into an internal buffer" - )] - pub async fn write_zero_copy(&mut self, data: Bytes) -> Result { - self.write_buffered(data).await - } - - /// Write a slice of data. - /// - /// # Arguments - /// - /// * `data` - Data slice to write - /// - /// # Returns - /// - /// * `Ok(usize)` - Number of bytes written - /// * `Err(ZeroCopyWriteError)` - Write error - pub async fn write_slice(&mut self, data: &[u8]) -> Result { - if self.finalized { - return Err(ZeroCopyWriteError::Finalized("Cannot write to finalized writer".to_string())); - } - - let len = data.len(); - self.buffer.put_slice(data); - self.bytes_written += len; - Ok(len) - } - - /// Finalize the writer and consume it, returning the written data as Bytes. - /// - /// This converts the internal BytesMut to Bytes, which is a zero-copy - /// operation that freezes the buffer. - /// - /// # Returns - /// - /// The written data as Bytes - /// - /// # Example - /// - /// ```ignore - /// let result = writer.into_bytes(); - /// ``` - pub fn into_bytes(mut self) -> Bytes { - self.finalized = true; - self.buffer.freeze() - } - - /// Get the current buffer as a slice (without consuming). - /// - /// # Returns - /// - /// Slice of the current buffer content - pub fn as_slice(&self) -> &[u8] { - &self.buffer[..] - } - - /// Get the total number of bytes written. - /// - /// # Returns - /// - /// Number of bytes written - pub fn bytes_written(&self) -> usize { - self.bytes_written - } - - /// Get the current buffer capacity. - /// - /// # Returns - /// - /// Current buffer capacity in bytes - pub fn capacity(&self) -> usize { - self.buffer.capacity() - } - - /// Get the current buffer length. - /// - /// # Returns - /// - /// Current buffer length in bytes - pub fn len(&self) -> usize { - self.buffer.len() - } - - /// Check if the buffer is empty. - /// - /// # Returns - /// - /// `true` if buffer is empty, `false` otherwise - pub fn is_empty(&self) -> bool { - self.buffer.is_empty() - } - - /// Clear the buffer, resetting it to empty. - /// - /// This does not change the capacity, just resets the length to 0. - pub fn clear(&mut self) { - self.buffer.clear(); - self.bytes_written = 0; - self.finalized = false; - } - - /// Reserve additional capacity in the buffer. - /// - /// # Arguments - /// - /// * `additional` - Additional capacity to reserve - pub fn reserve(&mut self, additional: usize) { - self.buffer.reserve(additional); - } -} - -impl Default for BytesMutWriter { - fn default() -> Self { - Self::new() - } -} - -impl std::fmt::Debug for BytesMutWriter { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("BytesMutWriter") - .field("buffer_len", &self.buffer.len()) - .field("buffer_capacity", &self.buffer.capacity()) - .field("bytes_written", &self.bytes_written) - .field("finalized", &self.finalized) - .finish() - } -} - -/// AsyncWrite implementation for BytesMutWriter. -/// -/// This allows the writer to be used with tokio's async I/O utilities. -impl AsyncWrite for BytesMutWriter { - fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll> { - if self.finalized { - return Poll::Ready(Err(tokio::io::Error::new( - tokio::io::ErrorKind::WriteZero, - "Cannot write to finalized writer", - ))); - } - - let len = buf.len(); - self.buffer.put_slice(buf); - self.bytes_written += len; - Poll::Ready(Ok(len)) - } - - fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - // Nothing to flush for in-memory buffer - Poll::Ready(Ok(())) - } - - fn poll_shutdown(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - self.finalized = true; - Poll::Ready(Ok(())) - } -} - -/// Zero-copy write error types. -#[derive(Debug, thiserror::Error)] -pub enum ZeroCopyWriteError { - /// I/O error occurred - #[error("I/O error: {0}")] - Io(#[from] tokio::io::Error), - - /// Writer has been finalized and cannot accept more writes - #[error("Writer finalized: {0}")] - Finalized(String), - - /// Invalid input provided - #[error("Invalid input: {0}")] - InvalidInput(String), -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_new_writer() { - let writer = BytesMutWriter::new(); - assert!(writer.is_empty()); - assert_eq!(writer.bytes_written(), 0); - assert!(writer.capacity() >= 8 * 1024); - } - - #[tokio::test] - async fn test_write_buffered() { - let mut writer = BytesMutWriter::new(); - let data = Bytes::from("hello world"); - - let written = writer.write_buffered(data).await.unwrap(); - assert_eq!(written, 11); - assert_eq!(writer.bytes_written(), 11); - assert_eq!(writer.as_slice(), b"hello world"); - } - - #[tokio::test] - async fn test_preferred_writer_alias() { - let mut writer = BytesMutWriter::new(); - let written = writer - .write_buffered(Bytes::from("hello world")) - .await - .expect("write bytes through alias"); - - assert_eq!(written, 11); - assert_eq!(writer.as_slice(), b"hello world"); - } - - #[tokio::test] - async fn test_write_slice() { - let mut writer = BytesMutWriter::new(); - let data = b"hello world"; - - let written = writer.write_slice(data).await.unwrap(); - assert_eq!(written, 11); - assert_eq!(writer.bytes_written(), 11); - assert_eq!(writer.as_slice(), b"hello world"); - } - - #[tokio::test] - async fn test_into_bytes() { - let mut writer = BytesMutWriter::new(); - let data = Bytes::from("hello world"); - - writer.write_buffered(data).await.unwrap(); - let result = writer.into_bytes(); - - assert_eq!(result.as_ref(), b"hello world"); - } - - #[tokio::test] - async fn test_write_after_finalize() { - let mut writer = BytesMutWriter::new(); - let data = Bytes::from("hello"); - - writer.write_buffered(data).await.unwrap(); - let _result = writer.into_bytes(); - - // Create new writer and try to write after finalize - let mut writer2 = BytesMutWriter::new(); - writer2.write_buffered(Bytes::from("test")).await.unwrap(); - let _ = writer2.into_bytes(); - - // Writing to a consumed writer should work via new writer - let mut writer3 = BytesMutWriter::new(); - let result = writer3.write_buffered(Bytes::from("final")).await; - assert!(result.is_ok()); - } - - #[tokio::test] - async fn test_clear() { - let mut writer = BytesMutWriter::new(); - writer.write_slice(b"hello").await.unwrap(); - - writer.clear(); - assert!(writer.is_empty()); - assert_eq!(writer.bytes_written(), 0); - // Capacity should remain - assert!(writer.capacity() > 0); - } - - #[tokio::test] - async fn test_reserve() { - let mut writer = BytesMutWriter::with_capacity(10); - let initial_capacity = writer.capacity(); - - writer.reserve(1000); - // Reserve ensures at least the additional capacity can be added - // but may allocate more than requested - assert!(writer.capacity() >= initial_capacity); - } - - #[tokio::test] - async fn test_multiple_writes() { - let mut writer = BytesMutWriter::new(); - - writer.write_buffered(Bytes::from("hello ")).await.unwrap(); - writer.write_slice(b"world").await.unwrap(); - - assert_eq!(writer.as_slice(), b"hello world"); - assert_eq!(writer.bytes_written(), 11); - } - - #[tokio::test] - async fn test_async_write() { - use tokio::io::AsyncWriteExt; - - let mut writer = BytesMutWriter::new(); - let data = b"hello world"; - - let written = writer.write(data).await.unwrap(); - assert_eq!(written, 11); - assert_eq!(writer.as_slice(), b"hello world"); - } - - #[tokio::test] - async fn test_debug() { - let writer = BytesMutWriter::new(); - let debug_str = format!("{:?}", writer); - assert!(debug_str.contains("BytesMutWriter")); - assert!(debug_str.contains("buffer_len")); - } - - #[tokio::test] - #[allow(deprecated)] - async fn test_legacy_writer_alias() { - let mut writer = ZeroCopyObjectWriter::new(); - let written = writer.write_zero_copy(Bytes::from("hello")).await.unwrap(); - - assert_eq!(written, 5); - assert_eq!(writer.as_slice(), b"hello"); - } -} From f06a9c9cbacb826ef4ce58a1e8b2da954e6c1044 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 17:04:55 +0800 Subject: [PATCH 33/51] fix(deps): record heal's bytes and crc-fast in the lockfile (#6211) From 68547ed7ea4a325e4b148778dfbdf188269700e0 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 17:05:30 +0800 Subject: [PATCH 34/51] test(e2e): drop 187 no-op serial markers from the three densest e2e suites (#6209) test(e2e): drop no-op serial markers from the three densest e2e suites serial_test's #[serial] is an in-process mutex. cargo-nextest, this repo's authoritative runner, executes every test in its own process, so the mutex is never contended and the attribute is a documented no-op -- see the "Serial execution & nextest profiles" section of docs/testing/README.md and the header of .config/nextest.toml. Cross-process serialization is provided only by a [test-groups] entry with max-threads = 1. Remove 187 such markers (plus 3 now-unused imports) from the three marker-densest modules of crates/e2e_test: multipart_auth_test.rs 85 replication_extension_test.rs 68 object_lock/object_lock_test.rs 34 None of these modules is covered by any [test-groups] entry, so the markers were carrying no isolation for any lane. Every test in all three files builds its own server via RustFSTestEnvironment::new(), which gives a UUID temp dir and a uniquely allocated port -- the .config/nextest.toml comment on replication_extension_test already states this explicitly ("parallel-safe by construction"). No test mutates process env, binds a fixed port, or touches process-global state, so nothing here needed temp_env or a test-group instead. Pure deletion: 190 lines removed, 0 added, no test renamed, no behaviour changed. serial_test stays in Cargo.toml -- 338 markers across 90 other files in the crate still use it. Refs: backlog#1846 (T1). --- crates/e2e_test/src/multipart_auth_test.rs | 86 ------------------- .../src/object_lock/object_lock_test.rs | 35 -------- .../src/replication_extension_test.rs | 69 --------------- 3 files changed, 190 deletions(-) diff --git a/crates/e2e_test/src/multipart_auth_test.rs b/crates/e2e_test/src/multipart_auth_test.rs index 81613bf97..dc0d6dd33 100644 --- a/crates/e2e_test/src/multipart_auth_test.rs +++ b/crates/e2e_test/src/multipart_auth_test.rs @@ -30,7 +30,6 @@ use md5::{Digest as Md5Digest, Md5}; use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; -use serial_test::serial; use std::collections::HashMap; use std::error::Error; use std::io::Cursor; @@ -356,7 +355,6 @@ async fn run_post_object_policy_case( /// smuggles one extra field the policy never declared, and the upload must be /// rejected with 403 AccessDenied naming the offending field. #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_fields_missing_from_policy_conditions() -> Result<(), Box> { init_logging(); @@ -484,7 +482,6 @@ async fn test_anonymous_post_object_rejects_fields_missing_from_policy_condition /// sends a different one, and the upload must be rejected with 400 /// InvalidPolicyDocument naming the field. #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_exact_condition_policy_mismatches() -> Result<(), Box> { init_logging(); @@ -689,7 +686,6 @@ async fn test_anonymous_post_object_rejects_exact_condition_policy_mismatches() /// one of them with a different value, and the upload must be rejected with /// 400 InvalidPolicyDocument naming the mismatched field. #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_object_lock_policy_mismatches() -> Result<(), Box> { init_logging(); @@ -757,7 +753,6 @@ async fn test_anonymous_post_object_rejects_object_lock_policy_mismatches() -> R /// exact values, the form sends a different parameter value, and the upload /// must be rejected with 400 InvalidPolicyDocument naming the parameter. #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_sse_kms_policy_mismatches() -> Result<(), Box> { init_logging(); @@ -839,7 +834,6 @@ async fn test_anonymous_post_object_rejects_sse_kms_policy_mismatches() -> Resul /// NotImplemented (SSE-KMS POST uploads are not implemented), not with a /// policy error. #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_sse_kms_params_outside_policy_conditions() -> Result<(), Box> { init_logging(); @@ -894,7 +888,6 @@ async fn test_anonymous_post_object_rejects_sse_kms_params_outside_policy_condit } #[tokio::test] -#[serial] async fn test_anonymous_multipart_control_apis_require_auth() -> Result<(), Box> { init_logging(); @@ -968,7 +961,6 @@ async fn test_anonymous_multipart_control_apis_require_auth() -> Result<(), Box< } #[tokio::test] -#[serial] async fn test_anonymous_post_object_requires_auth() -> Result<(), Box> { init_logging(); @@ -1002,7 +994,6 @@ async fn test_anonymous_post_object_requires_auth() -> Result<(), Box Result<(), Box> { init_logging(); @@ -1066,7 +1057,6 @@ async fn test_anonymous_post_object_honors_success_action_status() -> Result<(), } #[tokio::test] -#[serial] async fn test_anonymous_post_object_honors_success_action_redirect() -> Result<(), Box> { init_logging(); @@ -1139,7 +1129,6 @@ async fn test_anonymous_post_object_honors_success_action_redirect() -> Result<( } #[tokio::test] -#[serial] async fn test_anonymous_post_object_defaults_to_no_content() -> Result<(), Box> { init_logging(); @@ -1185,7 +1174,6 @@ async fn test_anonymous_post_object_defaults_to_no_content() -> Result<(), Box Result<(), Box> { init_logging(); @@ -1232,7 +1220,6 @@ async fn test_anonymous_post_object_rejects_sse_kms() -> Result<(), Box Result<(), Box> { init_logging(); @@ -1290,7 +1277,6 @@ async fn test_anonymous_post_object_accepts_sse_s3() -> Result<(), Box Result<(), Box> { init_logging(); @@ -1363,7 +1349,6 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_s3() -> Result<(), B } #[tokio::test] -#[serial] async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(), Box> { init_logging(); @@ -1437,7 +1422,6 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(), } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_sse_s3_policy_mismatch() -> Result<(), Box> { init_logging(); @@ -1488,7 +1472,6 @@ async fn test_anonymous_post_object_rejects_sse_s3_policy_mismatch() -> Result<( } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_sse_s3_missing_from_policy_conditions() -> Result<(), Box> { init_logging(); @@ -1552,7 +1535,6 @@ async fn test_anonymous_post_object_accepts_sse_s3_missing_from_policy_condition } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_storage_class_exact_policy_match() -> Result<(), Box> { init_logging(); @@ -1606,7 +1588,6 @@ async fn test_anonymous_post_object_accepts_storage_class_exact_policy_match() } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_storage_class_missing_from_policy_conditions() -> Result<(), Box> { init_logging(); @@ -1657,7 +1638,6 @@ async fn test_anonymous_post_object_rejects_storage_class_missing_from_policy_co } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_invalid_storage_class_value() -> Result<(), Box> { init_logging(); @@ -1709,7 +1689,6 @@ async fn test_anonymous_post_object_rejects_invalid_storage_class_value() -> Res } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_checksum_algorithm_missing_from_policy_conditions() -> Result<(), Box> { init_logging(); @@ -1765,7 +1744,6 @@ async fn test_anonymous_post_object_rejects_checksum_algorithm_missing_from_poli } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_checksum_algorithm_policy_mismatch() -> Result<(), Box> { init_logging(); @@ -1822,7 +1800,6 @@ async fn test_anonymous_post_object_rejects_checksum_algorithm_policy_mismatch() } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_checksum_auxiliary_fields_missing_from_policy_conditions() -> Result<(), Box> { init_logging(); @@ -1886,7 +1863,6 @@ async fn test_anonymous_post_object_rejects_checksum_auxiliary_fields_missing_fr } #[tokio::test] -#[serial] async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_conditions() -> Result<(), Box> { init_logging(); @@ -1963,7 +1939,6 @@ async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_condition } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_sse_c_exact_policy_mismatch() -> Result<(), Box> { init_logging(); @@ -2022,7 +1997,6 @@ async fn test_anonymous_post_object_rejects_sse_c_exact_policy_mismatch() -> Res } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_duplicate_key_form_values() -> Result<(), Box> { init_logging(); @@ -2072,7 +2046,6 @@ async fn test_anonymous_post_object_rejects_duplicate_key_form_values() -> Resul } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_invalid_success_action_status() -> Result<(), Box> { init_logging(); @@ -2120,7 +2093,6 @@ async fn test_anonymous_post_object_rejects_invalid_success_action_status() -> R } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_invalid_success_action_redirect() -> Result<(), Box> { init_logging(); @@ -2168,7 +2140,6 @@ async fn test_anonymous_post_object_rejects_invalid_success_action_redirect() } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_form_fields_missing_from_policy_conditions() -> Result<(), Box> { init_logging(); @@ -2223,7 +2194,6 @@ async fn test_anonymous_post_object_rejects_form_fields_missing_from_policy_cond } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_form_fields_covered_by_policy_conditions() -> Result<(), Box> { init_logging(); @@ -2280,7 +2250,6 @@ async fn test_anonymous_post_object_accepts_form_fields_covered_by_policy_condit } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_starts_with_policy_mismatch() -> Result<(), Box> { init_logging(); @@ -2335,7 +2304,6 @@ async fn test_anonymous_post_object_rejects_starts_with_policy_mismatch() -> Res } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_content_length_range_violation() -> Result<(), Box> { init_logging(); @@ -2388,7 +2356,6 @@ async fn test_anonymous_post_object_rejects_content_length_range_violation() } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_success_action_status_exact_policy_match() -> Result<(), Box> { init_logging(); @@ -2445,7 +2412,6 @@ async fn test_anonymous_post_object_accepts_success_action_status_exact_policy_m } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_success_action_redirect_policy_mismatch() -> Result<(), Box> { init_logging(); @@ -2502,7 +2468,6 @@ async fn test_anonymous_post_object_rejects_success_action_redirect_policy_misma } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_success_action_redirect_exact_policy_match() -> Result<(), Box> { init_logging(); @@ -2568,7 +2533,6 @@ async fn test_anonymous_post_object_accepts_success_action_redirect_exact_policy } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_success_action_redirect_missing_from_policy_conditions() -> Result<(), Box> { init_logging(); @@ -2621,7 +2585,6 @@ async fn test_anonymous_post_object_rejects_success_action_redirect_missing_from } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_metadata_field_covered_by_starts_with() -> Result<(), Box> { init_logging(); @@ -2676,7 +2639,6 @@ async fn test_anonymous_post_object_accepts_metadata_field_covered_by_starts_wit } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_content_type_field_exact_policy_match() -> Result<(), Box> { init_logging(); @@ -2734,7 +2696,6 @@ async fn test_anonymous_post_object_accepts_content_type_field_exact_policy_matc } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_content_type_field_covered_by_starts_with() -> Result<(), Box> { init_logging(); @@ -2792,7 +2753,6 @@ async fn test_anonymous_post_object_accepts_content_type_field_covered_by_starts } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_content_disposition_field_exact_policy_match() -> Result<(), Box> { init_logging(); @@ -2850,7 +2810,6 @@ async fn test_anonymous_post_object_accepts_content_disposition_field_exact_poli } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_cache_control_field_exact_policy_match() -> Result<(), Box> { init_logging(); @@ -2908,7 +2867,6 @@ async fn test_anonymous_post_object_accepts_cache_control_field_exact_policy_mat } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_content_language_field_exact_policy_match() -> Result<(), Box> { init_logging(); @@ -2966,7 +2924,6 @@ async fn test_anonymous_post_object_accepts_content_language_field_exact_policy_ } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_content_encoding_field_exact_policy_match() -> Result<(), Box> { init_logging(); @@ -3024,7 +2981,6 @@ async fn test_anonymous_post_object_accepts_content_encoding_field_exact_policy_ } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_website_redirect_location_exact_policy_match() -> Result<(), Box> { init_logging(); @@ -3082,7 +3038,6 @@ async fn test_anonymous_post_object_accepts_website_redirect_location_exact_poli } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_expires_field_exact_policy_match() -> Result<(), Box> { init_logging(); @@ -3140,7 +3095,6 @@ async fn test_anonymous_post_object_accepts_expires_field_exact_policy_match() } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_object_lock_retention_without_permission() -> Result<(), Box> { init_logging(); @@ -3196,7 +3150,6 @@ async fn test_anonymous_post_object_rejects_object_lock_retention_without_permis } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_object_lock_retention_missing_from_policy_conditions() -> Result<(), Box> { init_logging(); @@ -3256,7 +3209,6 @@ async fn test_anonymous_post_object_rejects_object_lock_retention_missing_from_p } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_object_lock_legal_hold_without_permission() -> Result<(), Box> { init_logging(); @@ -3309,7 +3261,6 @@ async fn test_anonymous_post_object_rejects_object_lock_legal_hold_without_permi } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_object_lock_legal_hold_policy_mismatch() -> Result<(), Box> { init_logging(); @@ -3368,7 +3319,6 @@ async fn test_anonymous_post_object_rejects_object_lock_legal_hold_policy_mismat } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_object_lock_legal_hold_missing_from_policy_conditions() -> Result<(), Box> { init_logging(); @@ -3426,7 +3376,6 @@ async fn test_anonymous_post_object_rejects_object_lock_legal_hold_missing_from_ } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_tagging_field_exact_policy_match() -> Result<(), Box> { init_logging(); @@ -3492,7 +3441,6 @@ async fn test_anonymous_post_object_accepts_tagging_field_exact_policy_match() } #[tokio::test] -#[serial] async fn test_anonymous_post_object_accepts_metadata_field_exact_policy_match() -> Result<(), Box> { init_logging(); @@ -3551,7 +3499,6 @@ async fn test_anonymous_post_object_accepts_metadata_field_exact_policy_match() } #[tokio::test] -#[serial] async fn test_anonymous_post_object_allows_x_ignore_fields_outside_policy_conditions() -> Result<(), Box> { init_logging(); @@ -3604,7 +3551,6 @@ async fn test_anonymous_post_object_allows_x_ignore_fields_outside_policy_condit } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_sigv4_date_policy_mismatch() -> Result<(), Box> { init_logging(); @@ -3657,7 +3603,6 @@ async fn test_anonymous_post_object_rejects_sigv4_date_policy_mismatch() -> Resu } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_mismatched_bucket_form_field() -> Result<(), Box> { init_logging(); @@ -3712,7 +3657,6 @@ async fn test_anonymous_post_object_rejects_mismatched_bucket_form_field() -> Re } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_multiple_bucket_values() -> Result<(), Box> { init_logging(); @@ -3764,7 +3708,6 @@ async fn test_anonymous_post_object_rejects_multiple_bucket_values() -> Result<( } #[tokio::test] -#[serial] async fn test_anonymous_post_object_rejects_extra_content_disposition_field() -> Result<(), Box> { init_logging(); @@ -3820,7 +3763,6 @@ async fn test_anonymous_post_object_rejects_extra_content_disposition_field() } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_expands_tar_entries_with_prefix_headers() -> Result<(), Box> { init_logging(); @@ -3891,7 +3833,6 @@ async fn test_signed_put_object_extract_expands_tar_entries_with_prefix_headers( } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_preserves_request_metadata_on_extracted_objects() -> Result<(), Box> { init_logging(); @@ -3956,7 +3897,6 @@ async fn test_signed_put_object_extract_preserves_request_metadata_on_extracted_ } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_preserves_sse_s3_and_redirect() -> Result<(), Box> { init_logging(); @@ -4004,7 +3944,6 @@ async fn test_signed_put_object_extract_preserves_sse_s3_and_redirect() -> Resul } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_preserves_storage_class() -> Result<(), Box> { init_logging(); @@ -4047,7 +3986,6 @@ async fn test_signed_put_object_extract_preserves_storage_class() -> Result<(), } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_rejects_invalid_storage_class() -> Result<(), Box> { init_logging(); @@ -4083,7 +4021,6 @@ async fn test_signed_put_object_extract_rejects_invalid_storage_class() -> Resul } #[tokio::test] -#[serial] async fn test_signed_put_object_rejects_write_offset_bytes_header() -> Result<(), Box> { init_logging(); @@ -4137,7 +4074,6 @@ async fn test_signed_put_object_rejects_write_offset_bytes_header() -> Result<() } #[tokio::test] -#[serial] async fn test_raw_signed_put_object_write_offset_bytes_returns_minio_compatible_error_body() -> Result<(), Box> { init_logging(); @@ -4176,7 +4112,6 @@ async fn test_raw_signed_put_object_write_offset_bytes_returns_minio_compatible_ } #[tokio::test] -#[serial] async fn test_anonymous_put_object_write_offset_bytes_returns_minio_compatible_error_body() -> Result<(), Box> { init_logging(); @@ -4235,7 +4170,6 @@ async fn test_anonymous_put_object_write_offset_bytes_returns_minio_compatible_e } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_uses_bucket_default_sse_s3() -> Result<(), Box> { init_logging(); @@ -4300,7 +4234,6 @@ async fn test_signed_put_object_extract_uses_bucket_default_sse_s3() -> Result<( } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_rejects_bucket_default_sse_kms() -> Result<(), Box> { init_logging(); @@ -4356,7 +4289,6 @@ async fn test_signed_put_object_extract_rejects_bucket_default_sse_kms() -> Resu } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_preserves_sse_c() -> Result<(), Box> { init_logging(); @@ -4421,7 +4353,6 @@ async fn test_signed_put_object_extract_preserves_sse_c() -> Result<(), Box Result<(), Box> { init_logging(); @@ -4476,7 +4407,6 @@ async fn test_signed_put_object_extract_preserves_object_lock_legal_hold() -> Re } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_preserves_object_lock_retention() -> Result<(), Box> { init_logging(); @@ -4536,7 +4466,6 @@ async fn test_signed_put_object_extract_preserves_object_lock_retention() -> Res } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_pax_retention_overrides_request_retention() -> Result<(), Box> { init_logging(); @@ -4600,7 +4529,6 @@ async fn test_signed_put_object_extract_pax_retention_overrides_request_retentio } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box> { init_logging(); @@ -4634,7 +4562,6 @@ async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_preserves_entry_mtime() -> Result<(), Box> { init_logging(); @@ -4670,7 +4597,6 @@ async fn test_signed_put_object_extract_preserves_entry_mtime() -> Result<(), Bo } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id() -> Result<(), Box> { init_logging(); @@ -4724,7 +4650,6 @@ async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id() } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retention_conditions() -> Result<(), Box> { init_logging(); @@ -5034,7 +4959,6 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_accepts_compat_header() -> Result<(), Box> { init_logging(); @@ -5076,7 +5000,6 @@ async fn test_signed_put_object_extract_accepts_compat_header() -> Result<(), Bo } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_preserves_directory_markers_by_default() -> Result<(), Box> { init_logging(); @@ -5137,7 +5060,6 @@ async fn test_signed_put_object_extract_preserves_directory_markers_by_default() } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_expands_tar_gz_archive() -> Result<(), Box> { init_logging(); @@ -5189,7 +5111,6 @@ async fn test_signed_put_object_extract_expands_tar_gz_archive() -> Result<(), B } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_expands_tgz_archive() -> Result<(), Box> { init_logging(); @@ -5241,7 +5162,6 @@ async fn test_signed_put_object_extract_expands_tgz_archive() -> Result<(), Box< } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_expands_tbz2_archive() -> Result<(), Box> { init_logging(); @@ -5293,7 +5213,6 @@ async fn test_signed_put_object_extract_expands_tbz2_archive() -> Result<(), Box } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_expands_txz_archive() -> Result<(), Box> { init_logging(); @@ -5345,7 +5264,6 @@ async fn test_signed_put_object_extract_expands_txz_archive() -> Result<(), Box< } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled() -> Result<(), Box> { init_logging(); @@ -5419,7 +5337,6 @@ async fn test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_e } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_normalizes_prefix_header_value() -> Result<(), Box> { init_logging(); @@ -5462,7 +5379,6 @@ async fn test_signed_put_object_extract_normalizes_prefix_header_value() -> Resu } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_expands_tzst_archive() -> Result<(), Box> { init_logging(); @@ -5514,7 +5430,6 @@ async fn test_signed_put_object_extract_expands_tzst_archive() -> Result<(), Box } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_rejects_missing_archive_extension() -> Result<(), Box> { init_logging(); @@ -5548,7 +5463,6 @@ async fn test_signed_put_object_extract_rejects_missing_archive_extension() -> R } #[tokio::test] -#[serial] async fn test_signed_put_object_extract_rejects_invalid_tar_gz_payload() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/object_lock/object_lock_test.rs b/crates/e2e_test/src/object_lock/object_lock_test.rs index 0748ad499..7a9d6f9bc 100644 --- a/crates/e2e_test/src/object_lock/object_lock_test.rs +++ b/crates/e2e_test/src/object_lock/object_lock_test.rs @@ -33,7 +33,6 @@ use aws_sdk_s3::types::{ ObjectLockMode, ObjectLockRetentionMode, }; use chrono::{DateTime, Duration, Utc}; -use serial_test::serial; use tracing::info; /// Initialize test logging @@ -107,7 +106,6 @@ fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> DateTime // ============================================================================ #[tokio::test] -#[serial] async fn test_delete_object_blocked_by_compliance_retention() { init_logging(); info!("🧪 Test: DeleteObject blocked by COMPLIANCE retention"); @@ -145,7 +143,6 @@ async fn test_delete_object_blocked_by_compliance_retention() { } #[tokio::test] -#[serial] async fn test_delete_object_blocked_by_governance_without_bypass() { init_logging(); info!("🧪 Test: DeleteObject blocked by GOVERNANCE retention without bypass"); @@ -175,7 +172,6 @@ async fn test_delete_object_blocked_by_governance_without_bypass() { } #[tokio::test] -#[serial] async fn test_delete_object_allowed_by_governance_with_bypass() { init_logging(); info!("🧪 Test: DeleteObject allowed by GOVERNANCE retention with bypass"); @@ -215,7 +211,6 @@ async fn test_delete_object_allowed_by_governance_with_bypass() { } #[tokio::test] -#[serial] async fn test_delete_object_creates_delete_marker_for_retained_current_version() { init_logging(); info!("🧪 Test: DeleteObject creates delete marker for retained current version"); @@ -266,7 +261,6 @@ async fn test_delete_object_creates_delete_marker_for_retained_current_version() } #[tokio::test] -#[serial] async fn test_delete_object_blocked_by_legal_hold() { init_logging(); info!("🧪 Test: DeleteObject blocked by Legal Hold"); @@ -299,7 +293,6 @@ async fn test_delete_object_blocked_by_legal_hold() { } #[tokio::test] -#[serial] async fn test_delete_object_allowed_with_legal_hold_off() { init_logging(); info!("🧪 Test: DeleteObject allowed with Legal Hold OFF"); @@ -335,7 +328,6 @@ async fn test_delete_object_allowed_with_legal_hold_off() { } #[tokio::test] -#[serial] async fn test_delete_object_after_legal_hold_removed() { init_logging(); info!("🧪 Test: DeleteObject succeeds after Legal Hold is removed"); @@ -369,7 +361,6 @@ async fn test_delete_object_after_legal_hold_removed() { } #[tokio::test] -#[serial] async fn test_get_object_legal_hold_returns_updated_status() { init_logging(); info!("🧪 Test: GetObjectLegalHold returns updated status"); @@ -425,7 +416,6 @@ async fn test_get_object_legal_hold_returns_updated_status() { } #[tokio::test] -#[serial] async fn test_get_object_retention_returns_configured_values() { init_logging(); info!("🧪 Test: GetObjectRetention returns configured values"); @@ -476,7 +466,6 @@ async fn test_get_object_retention_returns_configured_values() { // creating a new current version. The lock protects the existing version // from deletion; it never blocks new versions. #[tokio::test] -#[serial] async fn test_put_object_overwrite_creates_new_version_under_legal_hold() { init_logging(); info!("🧪 Test: PutObject overwrite of a legal-hold version creates a new version"); @@ -561,7 +550,6 @@ async fn test_put_object_overwrite_creates_new_version_under_legal_hold() { } #[tokio::test] -#[serial] async fn test_copy_object_applies_requested_legal_hold() { init_logging(); info!("🧪 Test: CopyObject applies requested Legal Hold"); @@ -613,7 +601,6 @@ async fn test_copy_object_applies_requested_legal_hold() { } #[tokio::test] -#[serial] async fn test_copy_object_does_not_inherit_source_legal_hold() { init_logging(); info!("🧪 Test: CopyObject does not inherit source Legal Hold"); @@ -707,7 +694,6 @@ async fn test_copy_object_does_not_inherit_source_legal_hold() { } #[tokio::test] -#[serial] async fn test_copy_object_overwrite_creates_new_version_under_legal_hold() { init_logging(); info!("🧪 Test: CopyObject overwrite of a legal-hold destination creates a new version"); @@ -787,7 +773,6 @@ async fn test_copy_object_overwrite_creates_new_version_under_legal_hold() { } #[tokio::test] -#[serial] async fn test_create_multipart_upload_applies_requested_legal_hold() { init_logging(); info!("🧪 Test: CreateMultipartUpload applies requested Legal Hold"); @@ -853,7 +838,6 @@ async fn test_create_multipart_upload_applies_requested_legal_hold() { } #[tokio::test] -#[serial] async fn test_create_multipart_upload_creates_new_version_under_compliance_retention() { init_logging(); info!("🧪 Test: CreateMultipartUpload over a COMPLIANCE-retained key creates a new version"); @@ -933,7 +917,6 @@ async fn test_create_multipart_upload_creates_new_version_under_compliance_reten } #[tokio::test] -#[serial] async fn test_delete_completed_multipart_object_blocked_by_legal_hold() { init_logging(); info!("🧪 Test: Delete completed multipart object blocked by Legal Hold"); @@ -993,7 +976,6 @@ async fn test_delete_completed_multipart_object_blocked_by_legal_hold() { } #[tokio::test] -#[serial] async fn test_delete_completed_multipart_object_blocked_by_retention() { init_logging(); info!("🧪 Test: Delete completed multipart object blocked by retention"); @@ -1055,7 +1037,6 @@ async fn test_delete_completed_multipart_object_blocked_by_retention() { } #[tokio::test] -#[serial] async fn test_complete_multipart_upload_creates_new_version_under_legal_hold() { init_logging(); info!("🧪 Test: CompleteMultipartUpload creates a new version when the current version is under Legal Hold"); @@ -1135,7 +1116,6 @@ async fn test_complete_multipart_upload_creates_new_version_under_legal_hold() { } #[tokio::test] -#[serial] async fn test_complete_multipart_upload_creates_new_version_under_compliance_retention() { init_logging(); info!("🧪 Test: CompleteMultipartUpload creates a new version when the current version is under COMPLIANCE retention"); @@ -1209,7 +1189,6 @@ async fn test_complete_multipart_upload_creates_new_version_under_compliance_ret } #[tokio::test] -#[serial] async fn test_write_paths_require_put_object_legal_hold_permission() { init_logging(); info!("🧪 Test: write paths require PutObjectLegalHold permission"); @@ -1273,7 +1252,6 @@ async fn test_write_paths_require_put_object_legal_hold_permission() { } #[tokio::test] -#[serial] async fn test_write_paths_require_put_object_retention_permission() { init_logging(); info!("🧪 Test: write paths require PutObjectRetention permission"); @@ -1345,7 +1323,6 @@ async fn test_write_paths_require_put_object_retention_permission() { // ============================================================================ #[tokio::test] -#[serial] async fn test_delete_objects_mixed_locked_unlocked() { init_logging(); info!("🧪 Test: DeleteObjects with mixed locked and unlocked objects"); @@ -1427,7 +1404,6 @@ async fn test_delete_objects_mixed_locked_unlocked() { // ============================================================================ #[tokio::test] -#[serial] async fn test_put_retention_compliance_cannot_shorten() { init_logging(); info!("🧪 Test: PutObjectRetention cannot shorten COMPLIANCE retention"); @@ -1468,7 +1444,6 @@ async fn test_put_retention_compliance_cannot_shorten() { } #[tokio::test] -#[serial] async fn test_put_retention_compliance_can_extend() { init_logging(); info!("🧪 Test: PutObjectRetention can extend COMPLIANCE retention"); @@ -1509,7 +1484,6 @@ async fn test_put_retention_compliance_can_extend() { } #[tokio::test] -#[serial] async fn test_put_retention_governance_extend_without_bypass() { init_logging(); info!("🧪 Test: PutObjectRetention on GOVERNANCE can extend without bypass"); @@ -1553,7 +1527,6 @@ async fn test_put_retention_governance_extend_without_bypass() { } #[tokio::test] -#[serial] async fn test_put_retention_governance_shorten_requires_bypass() { init_logging(); info!("🧪 Test: PutObjectRetention on GOVERNANCE requires bypass to shorten"); @@ -1615,7 +1588,6 @@ async fn test_put_retention_governance_shorten_requires_bypass() { // ============================================================================ #[tokio::test] -#[serial] async fn test_default_retention_applied_to_new_objects() { init_logging(); info!("🧪 Test: Default retention is applied to new objects"); @@ -1685,7 +1657,6 @@ async fn test_default_retention_applied_to_new_objects() { } #[tokio::test] -#[serial] async fn test_delete_object_creates_delete_marker_for_default_retained_current_version() { init_logging(); info!("🧪 Test: DeleteObject creates delete marker for default-retained current version"); @@ -1770,7 +1741,6 @@ async fn test_delete_object_creates_delete_marker_for_default_retained_current_v } #[tokio::test] -#[serial] async fn test_put_copy_and_multipart_reject_incomplete_retention_headers() { init_logging(); info!("🧪 Test: write paths reject incomplete Object Lock retention headers"); @@ -1869,7 +1839,6 @@ async fn test_put_copy_and_multipart_reject_incomplete_retention_headers() { } #[tokio::test] -#[serial] async fn test_copy_object_retention_uses_destination_policy() { init_logging(); info!("🧪 Test: CopyObject retention follows destination policy"); @@ -2051,7 +2020,6 @@ async fn test_copy_object_retention_uses_destination_policy() { } #[tokio::test] -#[serial] async fn test_multipart_default_retention_fixed_at_create() { init_logging(); info!("🧪 Test: multipart default retention is fixed at CreateMultipartUpload"); @@ -2122,7 +2090,6 @@ async fn test_multipart_default_retention_fixed_at_create() { // ============================================================================ #[tokio::test] -#[serial] async fn test_unretained_object_lock_object_delete_and_bucket_cleanup() { init_logging(); info!("🧪 Test: Unretained Object Lock object delete and bucket cleanup (Issue #5339)"); @@ -2243,7 +2210,6 @@ async fn test_unretained_object_lock_object_delete_and_bucket_cleanup() { } #[tokio::test] -#[serial] async fn test_versioning_auto_enabled_with_object_lock() { init_logging(); info!("🧪 Test: Versioning is auto-enabled when Object Lock is configured"); @@ -2302,7 +2268,6 @@ async fn test_versioning_auto_enabled_with_object_lock() { // ============================================================================ #[tokio::test] -#[serial] async fn test_error_message_distinguishes_legal_hold_from_retention() { init_logging(); info!("🧪 Test: Error messages distinguish Legal Hold from Retention"); diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index 1941bb105..ec847bfb7 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -60,7 +60,6 @@ use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; use s3s::header::X_AMZ_REPLICATION_STATUS; -use serial_test::serial; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; use std::convert::Infallible; @@ -2506,7 +2505,6 @@ async fn build_replication_pair( /// metadata was inherited wholesale from the source, so the scanner heal pass /// skipped it too — no PENDING/FAILED marker meant nothing to re-drive). #[tokio::test] -#[serial] async fn test_copy_object_replicates_to_target() -> TestResult { init_logging(); @@ -2555,7 +2553,6 @@ async fn test_copy_object_replicates_to_target() -> TestResult { /// independent object; every member must replicate to the remote target like a /// regular PUT (MinIO PutObjectExtract parity). #[tokio::test] -#[serial] async fn test_snowball_extract_replicates_members_to_target() -> TestResult { init_logging(); @@ -2601,7 +2598,6 @@ async fn test_snowball_extract_replicates_members_to_target() -> TestResult { } #[tokio::test] -#[serial] async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box> { init_logging(); @@ -2638,7 +2634,6 @@ async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box< } #[tokio::test] -#[serial] async fn test_replication_check_rejects_target_without_object_lock() -> Result<(), Box> { init_logging(); @@ -2692,7 +2687,6 @@ async fn test_replication_check_rejects_target_without_object_lock() -> Result<( } #[tokio::test] -#[serial] async fn test_set_remote_target_rejects_unversioned_source_bucket() -> Result<(), Box> { init_logging(); @@ -2731,7 +2725,6 @@ async fn test_set_remote_target_rejects_unversioned_source_bucket() -> Result<() } #[tokio::test] -#[serial] async fn test_replication_check_rejects_unversioned_source_bucket() -> Result<(), Box> { init_logging(); @@ -2755,7 +2748,6 @@ async fn test_replication_check_rejects_unversioned_source_bucket() -> Result<() } #[tokio::test] -#[serial] async fn test_replication_check_rejects_missing_replication_config() -> Result<(), Box> { init_logging(); @@ -2779,7 +2771,6 @@ async fn test_replication_check_rejects_missing_replication_config() -> Result<( } #[tokio::test] -#[serial] async fn test_replication_check_rejects_invalid_bucket() -> Result<(), Box> { init_logging(); @@ -2798,7 +2789,6 @@ async fn test_replication_check_rejects_invalid_bucket() -> Result<(), Box Result<(), Box> { init_logging(); @@ -2842,7 +2832,6 @@ async fn test_set_remote_target_rejects_same_bucket_on_same_deployment() -> Resu } #[tokio::test] -#[serial] async fn test_set_remote_target_rejects_unversioned_target_bucket() -> Result<(), Box> { init_logging(); @@ -2876,7 +2865,6 @@ async fn test_set_remote_target_rejects_unversioned_target_bucket() -> Result<() } #[tokio::test] -#[serial] async fn test_set_remote_target_update_requires_arn() -> Result<(), Box> { init_logging(); @@ -2928,7 +2916,6 @@ async fn test_set_remote_target_update_requires_arn() -> Result<(), Box Result<(), Box> { init_logging(); @@ -3017,7 +3004,6 @@ async fn fetch_single_target( } #[tokio::test] -#[serial] async fn test_set_remote_target_partial_update_preserves_credentials() -> Result<(), Box> { init_logging(); @@ -3103,7 +3089,6 @@ async fn test_set_remote_target_partial_update_preserves_credentials() -> Result } #[tokio::test] -#[serial] async fn test_set_remote_target_rejects_invalid_target_url() -> Result<(), Box> { init_logging(); @@ -3145,7 +3130,6 @@ async fn test_set_remote_target_rejects_invalid_target_url() -> Result<(), Box Result<(), Box> { init_logging(); @@ -3230,7 +3214,6 @@ async fn test_set_remote_target_rejects_self_signed_https_target_without_skip_tl } #[tokio::test] -#[serial] async fn test_set_remote_target_allows_self_signed_https_target_with_skip_tls_verify() -> Result<(), Box> { init_logging(); @@ -3342,7 +3325,6 @@ async fn test_set_remote_target_allows_self_signed_https_target_with_skip_tls_ve } #[tokio::test] -#[serial] async fn test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem() -> Result<(), Box> { init_logging(); @@ -3427,7 +3409,6 @@ async fn test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_ } #[tokio::test] -#[serial] async fn test_set_remote_target_allows_private_ca_https_target_with_ca_cert_pem() -> Result<(), Box> { init_logging(); @@ -3518,7 +3499,6 @@ async fn test_set_remote_target_allows_private_ca_https_target_with_ca_cert_pem( } #[tokio::test] -#[serial] async fn test_list_remote_targets_rejects_empty_bucket() -> Result<(), Box> { init_logging(); @@ -3538,7 +3518,6 @@ async fn test_list_remote_targets_rejects_empty_bucket() -> Result<(), Box Result<(), Box> { init_logging(); @@ -3557,7 +3536,6 @@ async fn test_list_remote_targets_rejects_invalid_bucket() -> Result<(), Box Result<(), Box> { init_logging(); @@ -3598,7 +3576,6 @@ async fn test_remove_remote_target_rejects_missing_target() -> Result<(), Box Result<(), Box> { init_logging(); @@ -3623,7 +3600,6 @@ async fn test_remove_remote_target_rejects_missing_arn() -> Result<(), Box Result<(), Box> { init_logging(); @@ -3647,7 +3623,6 @@ async fn test_remove_remote_target_rejects_invalid_bucket() -> Result<(), Box Result<(), Box> { init_logging(); @@ -3687,7 +3662,6 @@ async fn test_remove_remote_target_rejects_target_used_by_replication() -> Resul } #[tokio::test] -#[serial] async fn test_delete_bucket_replication_removes_remote_target() -> Result<(), Box> { init_logging(); @@ -3737,7 +3711,6 @@ async fn test_delete_bucket_replication_removes_remote_target() -> Result<(), Bo } #[tokio::test] -#[serial] async fn test_bucket_replication_replicates_put_object_issue_2539() -> Result<(), Box> { init_logging(); @@ -3779,7 +3752,6 @@ async fn test_bucket_replication_replicates_put_object_issue_2539() -> Result<() } #[tokio::test] -#[serial] async fn test_bucket_replication_converges_delete_marker_and_version_purge() -> TestResult { init_logging(); @@ -3878,7 +3850,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() -> } #[tokio::test] -#[serial] async fn test_bucket_replication_disabled_delete_marker_does_not_propagate() -> TestResult { init_logging(); @@ -3965,7 +3936,6 @@ async fn test_bucket_replication_disabled_delete_marker_does_not_propagate() -> /// interoperability profile for a runner that provisions MinIO credentials /// and a reachable endpoint. #[tokio::test] -#[serial] async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestResult { init_logging(); @@ -4293,7 +4263,6 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR } #[tokio::test] -#[serial] async fn test_single_bucket_multipart_replication_fans_out_to_multiple_targets() -> Result<(), Box> { init_logging(); @@ -4462,7 +4431,6 @@ async fn test_repl17_failure_observation_helpers() -> TestResult { /// the replica is decryptable only with the original customer key. The /// backlog#1291 property still holds: never a silent plaintext replica. #[tokio::test] -#[serial] async fn test_bucket_replication_sse_c_contract() -> TestResult { init_logging(); @@ -4540,7 +4508,6 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult { /// part — part boundaries and the encrypted-multipart marker survive so the /// replica decrypts each part with its part-derived nonce. #[tokio::test] -#[serial] async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult { init_logging(); @@ -4657,7 +4624,6 @@ async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult { /// (independent KMS, so success proves target-owned envelopes), preserved /// source ETag, and a version that stays stable across scanner cycles. #[tokio::test] -#[serial] async fn test_bucket_replication_sse_s3_contract() -> TestResult { init_logging(); assert_managed_sse_replicates_and_reencrypts("sse-s3", false).await @@ -4667,7 +4633,6 @@ async fn test_bucket_replication_sse_s3_contract() -> TestResult { /// fail closed — replication FAILED, and no plaintext (or any) replica ever /// materializes on the target. #[tokio::test] -#[serial] async fn test_bucket_replication_sse_s3_fails_closed_without_target_kms() -> TestResult { init_logging(); @@ -4711,7 +4676,6 @@ async fn test_bucket_replication_sse_s3_fails_closed_without_target_kms() -> Tes /// the ETag comparison sees the preserved source ETag on the replica and does /// not rewrite it, so the replica's version stays stable through the resync. #[tokio::test] -#[serial] async fn test_bucket_replication_sse_s3_resync_converges() -> TestResult { init_logging(); @@ -4768,7 +4732,6 @@ async fn test_bucket_replication_sse_s3_resync_converges() -> TestResult { /// re-encrypts under its own default key. The independent-KMS pair proves the /// replica's envelope is target-owned. #[tokio::test] -#[serial] async fn test_bucket_replication_sse_kms_contract() -> TestResult { init_logging(); assert_managed_sse_replicates_and_reencrypts("sse-kms", true).await @@ -4779,7 +4742,6 @@ async fn test_bucket_replication_sse_kms_contract() -> TestResult { /// carries the full header set (SSE intent, content-type, user metadata) and /// the completed replica preserves the source's multipart ETag. #[tokio::test] -#[serial] async fn test_bucket_replication_sse_s3_multipart_reencrypts() -> TestResult { init_logging(); @@ -4873,7 +4835,6 @@ async fn test_bucket_replication_sse_s3_multipart_reencrypts() -> TestResult { /// still-running source's data scanner (short cycle via [`FAST_SCANNER_ENV`]) /// re-drives the failed objects once the target is reachable again. #[tokio::test] -#[serial] async fn test_bucket_replication_recovers_after_target_outage() -> TestResult { init_logging(); @@ -4953,7 +4914,6 @@ async fn test_bucket_replication_recovers_after_target_outage() -> TestResult { /// must settle back to zero even though the historical failed counter remains /// non-zero. #[tokio::test] -#[serial] async fn test_bucket_replication_backlog_metrics_observe_outage_and_recovery() -> TestResult { init_logging(); @@ -5087,7 +5047,6 @@ async fn test_bucket_replication_backlog_metrics_observe_outage_and_recovery() - /// must converge every persisted failure, including the replayed delete marker /// (whose replication decision is re-derived from the live config). #[tokio::test] -#[serial] async fn test_bucket_replication_replays_failed_entries_after_source_restart() -> TestResult { init_logging(); @@ -5179,7 +5138,6 @@ async fn test_bucket_replication_replays_failed_entries_after_source_restart() - } #[tokio::test] -#[serial] async fn test_bucket_replication_replayed_delete_marker_preserves_source_mtime_without_source_restart() -> TestResult { init_logging(); @@ -5249,7 +5207,6 @@ async fn test_bucket_replication_replayed_delete_marker_preserves_source_mtime_w } #[tokio::test] -#[serial] async fn test_sequential_bucket_replication_succeeds_for_multiple_buckets() -> Result<(), Box> { init_logging(); @@ -5293,7 +5250,6 @@ async fn test_sequential_bucket_replication_succeeds_for_multiple_buckets() -> R } #[tokio::test] -#[serial] async fn test_replication_recovers_after_runtime_target_cache_is_cleared() -> Result<(), Box> { init_logging(); @@ -5337,7 +5293,6 @@ async fn test_replication_recovers_after_runtime_target_cache_is_cleared() -> Re } #[tokio::test] -#[serial] async fn test_site_replication_allows_self_signed_https_with_skip_tls_verify_real_dual_node() -> TestResult { init_logging(); @@ -5416,7 +5371,6 @@ async fn test_site_replication_allows_self_signed_https_with_skip_tls_verify_rea } #[tokio::test] -#[serial] async fn test_site_replication_allows_private_ca_https_with_ca_cert_pem_real_dual_node() -> TestResult { init_logging(); @@ -5495,7 +5449,6 @@ async fn test_site_replication_allows_private_ca_https_with_ca_cert_pem_real_dua } #[tokio::test] -#[serial] async fn test_site_replication_resync_lifecycle_survives_real_server_restart() -> Result<(), Box> { init_logging(); let resync_process_env = [ @@ -5715,7 +5668,6 @@ async fn test_site_replication_resync_lifecycle_survives_real_server_restart() - } #[tokio::test] -#[serial] async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> Result<(), Box> { init_logging(); @@ -5964,7 +5916,6 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R } #[tokio::test] -#[serial] async fn test_site_replication_remove_all_real_dual_node() -> Result<(), Box> { init_logging(); @@ -6084,7 +6035,6 @@ async fn test_site_replication_remove_all_real_dual_node() -> Result<(), Box Result<(), Box> { init_logging(); @@ -6193,7 +6143,6 @@ async fn test_site_replication_state_edit_fresh_and_stale_real_dual_node() -> Re } #[tokio::test] -#[serial] async fn test_site_replication_replicates_object_with_bucket_versioning_real_dual_node() -> TestResult { init_logging(); @@ -6284,7 +6233,6 @@ async fn test_site_replication_replicates_object_with_bucket_versioning_real_dua /// receiver was dropped with only a debug line, while `replicate status` still reported /// "1/1 Buckets in sync" because both configs were byte-identical. #[tokio::test] -#[serial] async fn test_site_replication_config_broadcast_keeps_reverse_direction_real_dual_node() -> TestResult { init_logging(); @@ -6423,7 +6371,6 @@ async fn wait_for_site_replication_rule( } #[tokio::test] -#[serial] async fn test_site_replication_active_active_converges_without_loops_real_dual_node() -> TestResult { init_logging(); @@ -6741,7 +6688,6 @@ async fn test_site_replication_active_active_converges_without_loops_real_dual_n } #[tokio::test] -#[serial] async fn test_site_replication_replicates_policy_backed_user_access_real_dual_node() -> Result<(), Box> { init_logging(); @@ -6829,7 +6775,6 @@ async fn test_site_replication_replicates_policy_backed_user_access_real_dual_no } #[tokio::test] -#[serial] async fn test_site_replication_replicates_group_policy_backed_access_real_dual_node() -> Result<(), Box> { init_logging(); @@ -6920,7 +6865,6 @@ async fn test_site_replication_replicates_group_policy_backed_access_real_dual_n } #[tokio::test] -#[serial] async fn test_service_account_policy_from_accountinfo_round_trips_real_single_node() -> TestResult { init_logging(); @@ -6972,7 +6916,6 @@ async fn test_service_account_policy_from_accountinfo_round_trips_real_single_no } #[tokio::test] -#[serial] async fn test_site_replication_replicates_multiple_service_accounts_real_dual_node() -> Result<(), Box> { init_logging(); @@ -7073,7 +7016,6 @@ async fn test_site_replication_replicates_multiple_service_accounts_real_dual_no } #[tokio::test] -#[serial] async fn test_site_replication_replicates_service_accounts_created_from_sts_session_real_dual_node() -> TestResult { init_logging(); @@ -7214,7 +7156,6 @@ async fn wait_for_target_request_version_id( } #[tokio::test] -#[serial] async fn test_bucket_resync_restart_revisits_objects_before_out_of_order_checkpoint() -> TestResult { init_logging(); @@ -7333,7 +7274,6 @@ async fn test_bucket_resync_restart_revisits_objects_before_out_of_order_checkpo /// CreateMultipartUpload (the version is decided at initiate time) must both /// carry the source version as `?versionId=`. #[tokio::test] -#[serial] async fn test_replication_put_and_create_multipart_carry_source_version_id_query() -> TestResult { init_logging(); @@ -7448,7 +7388,6 @@ async fn test_replication_put_and_create_multipart_carry_source_version_id_query /// flow to the onward bucket, proving B's outbound replication and scanner /// are live. #[tokio::test] -#[serial] async fn test_scanner_never_cascades_inbound_replicas() -> TestResult { init_logging(); @@ -7529,7 +7468,6 @@ async fn test_scanner_never_cascades_inbound_replicas() -> TestResult { /// version ids and still mint its own there — the check must not report OK /// while multipart deletes and heals would silently miss. #[tokio::test] -#[serial] async fn test_replication_check_flags_multipart_only_version_minting_target() -> TestResult { init_logging(); @@ -7602,7 +7540,6 @@ async fn test_replication_check_flags_multipart_only_version_minting_target() -> } #[tokio::test] -#[serial] async fn test_replication_check_aborts_failed_multipart_probes() -> TestResult { init_logging(); @@ -7765,7 +7702,6 @@ async fn test_replication_check_aborts_failed_multipart_probes() -> TestResult { /// BucketRemoteTargetVersionMismatch — while still cleaning up the probe /// object via the version id the target actually assigned. #[tokio::test] -#[serial] async fn test_replication_check_flags_version_minting_target() -> TestResult { init_logging(); @@ -8006,7 +7942,6 @@ async fn wait_for_target_marker_purged( /// the target forever. Contract under test: a failed purge attempt is retried /// within the watch window and converges once the fault clears. #[tokio::test] -#[serial] async fn test_delayed_delete_marker_purge_retries_after_transient_target_failure() -> TestResult { init_logging(); let source_bucket = "delayed-purge-retry-src"; @@ -8066,7 +8001,6 @@ async fn test_delayed_delete_marker_purge_retries_after_transient_target_failure /// with an idempotent 204, which used to look like success and strand the /// real marker on the target forever. #[tokio::test] -#[serial] async fn test_delayed_delete_marker_purge_uses_target_assigned_version() -> TestResult { init_logging(); let source_bucket = "delayed-purge-mint-src"; @@ -8099,7 +8033,6 @@ async fn test_delayed_delete_marker_purge_uses_target_assigned_version() -> Test /// replayed purge succeeds, the entry must be acknowledged instead of being /// retained as Missed forever. #[tokio::test] -#[serial] async fn test_delayed_delete_marker_purge_exhaustion_persists_to_mrf_and_replays_on_restart() -> TestResult { init_logging(); let source_bucket = "delayed-purge-mrf-src"; @@ -8236,7 +8169,6 @@ async fn build_scanner_compensation_pair( /// nil-version objects entirely (`scanner_folder.rs` heal_replication), so it /// must NEVER be compensated. #[tokio::test] -#[serial] async fn test_scanner_compensates_existing_objects_across_write_paths() -> TestResult { init_logging(); let source_bucket = "scanner-comp-src"; @@ -8352,7 +8284,6 @@ async fn test_scanner_compensates_existing_objects_across_write_paths() -> TestR /// written after the rule replicate normally (the setting only gates the /// existing-object resync path). #[tokio::test] -#[serial] async fn test_scanner_never_compensates_when_existing_object_replication_disabled() -> TestResult { init_logging(); let source_bucket = "scanner-disabled-src"; From 382ae9529ec447e10010d11cbb4c095cf0e1d237 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 17:20:55 +0800 Subject: [PATCH 35/51] feat(ecstore): instrument rename sync tail metrics (#6205) Add default-off PUT stage attribution for the rename_data sync tail so strict durability probes can split queue wait, fdatasync, directory fsync, rename, per-disk wait, and quorum wait without changing commit ordering or S3-visible behavior. Co-authored-by: heihutu --- crates/ecstore/src/disk/local.rs | 82 +++++++++++--- crates/ecstore/src/disk/os.rs | 101 ++++++++++++++++-- .../src/set_disk/core/io_primitives.rs | 19 +++- crates/io-metrics/src/lib.rs | 69 ++++++++++++ 4 files changed, 246 insertions(+), 25 deletions(-) diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 1932744e8..a71f2c55b 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -9264,17 +9264,27 @@ impl DiskAPI for LocalDisk { // accept that window (documented in docs/operations/durability-modes.md). if durability.syncs_commit_metadata() && let Some(parent) = dst_file_path.parent() - && let Err(err) = os::fsync_dir(parent).await { - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; - // The commit rename changed the dst part inodes before this fsync - // failed and rolled them back; drop any fd cached during that - // window so readers re-open the restored inode (rustfs/backlog#1177). - for part_path in &invalidate_part_paths { - self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = os::fsync_dir(parent).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) + .map_err(to_file_error)?; + // The commit rename changed the dst part inodes before this fsync + // failed and rolled them back; drop any fd cached during that + // window so readers re-open the restored inode (rustfs/backlog#1177). + for part_path in &invalidate_part_paths { + self.io_backend.invalidate_cached_fd(dst_volume, part_path).await; + } + return Err(to_file_error(err).into()); } - return Err(to_file_error(err).into()); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); } // First PUT of an object creates its directory (and any missing prefix @@ -9293,7 +9303,12 @@ impl DiskAPI for LocalDisk { if !dir.starts_with(&dst_volume_dir) { break; } + let fsync_started = rustfs_io_metrics::put_stage_timer(); if let Err(err) = os::fsync_dir(dir).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) .map_err(to_file_error)?; // Same post-commit rollback window as above — drop cached @@ -9304,6 +9319,10 @@ impl DiskAPI for LocalDisk { } return Err(to_file_error(err).into()); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); if dir == dst_volume_dir.as_path() { break; } @@ -9532,10 +9551,21 @@ impl DiskAPI for LocalDisk { } if let Some(admission) = file_sync_admission.as_ref() && let Some(backup_parent) = backup_path.parent() - && let Err(err) = - os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await { - return Err(DiskError::from(to_file_error(err))); + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + fsync_started, + ); + return Err(DiskError::from(to_file_error(err))); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + fsync_started, + ); } local_rollback_path = None; } @@ -9573,11 +9603,22 @@ impl DiskAPI for LocalDisk { // Persist the commit rename's directory entry across power loss. if let Some(admission) = file_sync_admission.as_ref() && let Some(dst_parent) = dst_file_path.parent() - && let Err(err) = - os::fsync_dir_with_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission).await { - rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; - return Err(err); + let fsync_started = rustfs_io_metrics::put_stage_timer(); + if let Err(err) = + os::fsync_dir_with_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission).await + { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); + rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + fsync_started, + ); } // Same power-loss gap as the non-inline path (rustfs/backlog#922 @@ -9595,9 +9636,14 @@ impl DiskAPI for LocalDisk { if !ancestor_dir.starts_with(&dst_volume_dir) { break; } + let fsync_started = rustfs_io_metrics::put_stage_timer(); if let Err(err) = os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); rollback_inline_metadata_commit_std( &dst_file_path, rollback_data_dir, @@ -9605,6 +9651,10 @@ impl DiskAPI for LocalDisk { )?; return Err(err); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + fsync_started, + ); if ancestor_dir == dst_volume_dir.as_path() { break; } diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index f5b1c5f66..decb36964 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -343,6 +343,7 @@ pub(crate) async fn acquire_rename_data_mutation_lease( /// this order uniform prevents one slow disk from reserving global capacity /// while it waits for its own concurrency slot. async fn acquire_file_sync_permits(disk_permits: Arc) -> io::Result<(OwnedSemaphorePermit, SemaphorePermit<'static>)> { + let wait_started = rustfs_io_metrics::put_stage_timer(); let disk_permit = disk_permits .acquire_owned() .await @@ -351,6 +352,10 @@ async fn acquire_file_sync_permits(disk_permits: Arc) -> io::Result<( .acquire() .await .map_err(|_| io::Error::other("global file sync concurrency limiter closed"))?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT, + wait_started, + ); Ok((disk_permit, global_permit)) } @@ -551,9 +556,19 @@ pub(crate) fn sync_file(path: &Path) -> io::Result<()> { file.sync_data() } +fn sync_file_with_put_stage_metric(path: &Path) -> io::Result<()> { + let sync_started = rustfs_io_metrics::put_stage_timer(); + let result = sync_file(path); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC, + sync_started, + ); + result +} + fn sync_files(paths: &[PathBuf]) -> io::Result<()> { for path in paths { - sync_file(path)?; + sync_file_with_put_stage_metric(path)?; } Ok(()) } @@ -599,7 +614,13 @@ pub(crate) async fn sync_dir_files_with_limiter(dir: impl AsRef, disk_perm let files = regular_files(&scan_dir)?; if files.len() < PARALLEL_FILE_SYNC_THRESHOLD { sync_files(&files)?; - fsync_dir_std(scan_dir)?; + let fsync_started = rustfs_io_metrics::put_stage_timer(); + let result = fsync_dir_std(scan_dir); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_SRC_DIR_FSYNC, + fsync_started, + ); + result?; return Ok(None); } Ok::<_, io::Error>(Some(files)) @@ -612,10 +633,19 @@ pub(crate) async fn sync_dir_files_with_limiter(dir: impl AsRef, disk_perm futures::stream::iter(files.into_iter().map(Ok::<_, io::Error>)) .try_for_each_concurrent(MAX_PARALLEL_FILE_SYNCS, |path| { let disk_permits = disk_permits.clone(); - async move { run_file_sync_blocking(disk_permits, move || sync_file(&path)).await } + async move { run_file_sync_blocking(disk_permits, move || sync_file_with_put_stage_metric(&path)).await } }) .await?; - run_file_sync_blocking(disk_permits, move || fsync_dir_std(dir)).await + run_file_sync_blocking(disk_permits, move || { + let fsync_started = rustfs_io_metrics::put_stage_timer(); + let result = fsync_dir_std(dir); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_SRC_DIR_FSYNC, + fsync_started, + ); + result + }) + .await } /// Check if the given disk path is the root disk. @@ -1174,10 +1204,15 @@ pub(crate) struct FileSyncAdmission { } pub(crate) async fn acquire_file_sync_admission(disk_permits: Arc) -> io::Result { + let wait_started = rustfs_io_metrics::put_stage_timer(); let disk_permit = disk_permits .acquire_owned() .await .map_err(|_| io::Error::other("disk file sync concurrency limiter closed"))?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT, + wait_started, + ); Ok(FileSyncAdmission { disk_permit: Arc::new(disk_permit), }) @@ -1200,10 +1235,15 @@ async fn run_blocking_namespace_file_sync_operation_with_global io::Result + Send + 'static, ) -> io::Result { + let wait_started = rustfs_io_metrics::put_stage_timer(); let global_permit = global_permits .acquire() .await .map_err(|_| io::Error::other("global file sync concurrency limiter closed"))?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT, + wait_started, + ); let disk_permit = admission.disk_permit.clone(); let result = tokio::task::spawn_blocking(move || { let _lease = lease; @@ -1420,7 +1460,13 @@ fn rename_into_existing_parent( use rustix::fs::{Mode, OFlags, open, renameat}; let Some(parent_guard) = parent_guard else { - return super::fs::rename_std(src_file_path, dst_file_path); + let rename_started = rustfs_io_metrics::put_stage_timer(); + let result = super::fs::rename_std(src_file_path, dst_file_path); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL, + rename_started, + ); + return result; }; let src_parent = src_file_path .parent() @@ -1441,7 +1487,13 @@ fn rename_into_existing_parent( .last() .ok_or_else(|| io::Error::other("rename destination parent guard is empty"))?; - renameat(&src_parent, src_name, dst_parent, dst_name).map_err(io::Error::from) + let rename_started = rustfs_io_metrics::put_stage_timer(); + let result = renameat(&src_parent, src_name, dst_parent, dst_name).map_err(io::Error::from); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL, + rename_started, + ); + result } #[cfg(windows)] @@ -2890,6 +2942,7 @@ pub fn is_dir_not_empty_error(err: &io::Error) -> bool { #[cfg(test)] mod tests { use super::*; + use crate::test_metrics::CapturingRecorder; use std::sync::Mutex; use std::time::Duration; use tempfile::tempdir; @@ -2910,6 +2963,42 @@ mod tests { PublicationRoot::new(&common).expect("test publication root should open") } + #[test] + #[serial_test::serial(file_sync_metrics)] + fn sync_file_with_put_stage_metric_records_fdatasync_only_when_enabled() { + let previous_gate = rustfs_io_metrics::put_stage_metrics_enabled(); + rustfs_io_metrics::set_put_stage_metrics_enabled(false); + let dir = tempdir().expect("temp dir should be created"); + let path = dir.path().join("part.1"); + std::fs::write(&path, b"payload").expect("test file should be written"); + let recorder = CapturingRecorder::default(); + + metrics::with_local_recorder(&recorder, || { + sync_file_with_put_stage_metric(&path).expect("disabled metric sync_file should succeed"); + assert_eq!( + recorder.histogram_sample_count("rustfs_s3_put_object_stage_duration_ms"), + 0, + "disabled PUT stage metrics must not emit fdatasync samples" + ); + + rustfs_io_metrics::set_put_stage_metrics_enabled(true); + sync_file_with_put_stage_metric(&path).expect("enabled metric sync_file should succeed"); + rustfs_io_metrics::set_put_stage_metrics_enabled(false); + }); + + assert_eq!( + recorder + .histogram_values( + "rustfs_s3_put_object_stage_duration_ms", + &[("stage", rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC)] + ) + .len(), + 1, + "enabled PUT stage metrics must emit one fdatasync sample" + ); + rustfs_io_metrics::set_put_stage_metrics_enabled(previous_gate); + } + async fn rename_all( src_file_path: impl AsRef, dst_file_path: impl AsRef, diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 7af25a64c..dcf5d11e3 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -3389,8 +3389,15 @@ impl SetDisks { // A no-op immediately-ready future in production. Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await; - disk.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object) - .await + let disk_wait_started = rustfs_io_metrics::put_stage_timer(); + let result = disk + .rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object) + .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT, + disk_wait_started, + ); + result }) .catch_unwind() }); @@ -3403,7 +3410,13 @@ impl SetDisks { let mut cleanup_data_dirs = vec![None; disk_count]; let mut old_current_sizes = vec![None; disk_count]; - let (results, mut file_infos) = fanout.await.map_err(|_| DiskError::Unexpected)?; + let quorum_wait_started = rustfs_io_metrics::put_stage_timer(); + let fanout_result = fanout.await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT, + quorum_wait_started, + ); + let (results, mut file_infos) = fanout_result.map_err(|_| DiskError::Unexpected)?; for (idx, result) in results.iter().enumerate() { match result { diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index 3f021772d..86b38ed5a 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -109,6 +109,17 @@ pub fn put_stage_timer() -> Option { put_stage_metrics_enabled().then(std::time::Instant::now) } +pub const PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT: &str = "set_disk_rename_quorum_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT: &str = "set_disk_rename_disk_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_file_sync_permit_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_global_file_sync_permit_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC: &str = "set_disk_rename_file_fdatasync"; +pub const PUT_STAGE_SET_DISK_RENAME_SRC_DIR_FSYNC: &str = "set_disk_rename_src_dir_fsync"; +pub const PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC: &str = "set_disk_rename_dst_dir_fsync"; +pub const PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC: &str = "set_disk_rename_backup_dir_fsync"; +pub const PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC: &str = "set_disk_rename_ancestor_dir_fsync"; +pub const PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL: &str = "set_disk_rename_rename_syscall"; + #[inline(always)] pub fn get_stage_metrics_enabled() -> bool { GET_STAGE_METRICS_ENABLED.load(Ordering::Relaxed) @@ -2618,6 +2629,7 @@ mod tests { use super::*; use metrics_util::MetricKind; use metrics_util::debugging::{DebugValue, DebuggingRecorder}; + use std::collections::HashSet; use std::sync::{Arc, Barrier, Mutex}; // Serialize tests that mutate the process-global PUT_STAGE_METRICS_ENABLED flag. @@ -2861,6 +2873,63 @@ mod tests { set_put_stage_metrics_enabled(false); } + #[test] + fn put_stage_sync_tail_labels_are_static_and_gated() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let stages = [ + PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT, + PUT_STAGE_SET_DISK_RENAME_DISK_WAIT, + PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT, + PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT, + PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC, + PUT_STAGE_SET_DISK_RENAME_SRC_DIR_FSYNC, + PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, + PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, + PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, + PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL, + ]; + let unique = stages.iter().copied().collect::>(); + assert_eq!(unique.len(), stages.len()); + assert!( + stages + .iter() + .all(|stage| stage.starts_with("set_disk_rename_") && !stage.contains('/') && !stage.contains('{')) + ); + + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + set_put_stage_metrics_enabled(false); + for stage in stages { + record_put_object_stage_duration(stage, 1.0); + } + set_put_stage_metrics_enabled(true); + for stage in stages { + record_put_object_stage_duration(stage, 1.0); + } + set_put_stage_metrics_enabled(false); + }); + + let recorded = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter(|(composite, _, _, _)| { + composite.kind() == MetricKind::Histogram && composite.key().name() == "rustfs_s3_put_object_stage_duration_ms" + }) + .flat_map(|(composite, _, _, _)| { + composite + .key() + .labels() + .filter(|label| label.key() == "stage") + .map(|label| label.value().to_string()) + .collect::>() + }) + .collect::>(); + assert_eq!(recorded.len(), stages.len()); + assert!(stages.iter().all(|stage| recorded.contains(*stage))); + } + #[test] fn test_put_object_diagnostic_buckets() { assert_eq!(put_object_size_bucket(0), "unknown"); From a38743caf5a54c18a6c99bbc73d5ccbe2bc42c2b Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 18:27:01 +0800 Subject: [PATCH 36/51] chore(zip): trim the unused extract and create surface (#6210) rustfs-zip has one workspace consumer, and it uses only CompressionFormat::{from_extension, extension, get_decoder} and ArchiveLimits. Remove the tar/zip extract, zip create, and in-memory compress helpers together with the types and dependencies that only served them. Trimming public API is semver-major once the stable tag is cut, so it costs least now. --- Cargo.lock | 5 - crates/zip/Cargo.toml | 16 +- crates/zip/README.md | 54 +- crates/zip/benches/zip_benchmark.rs | 416 ------- crates/zip/src/lib.rs | 1685 +-------------------------- 5 files changed, 58 insertions(+), 2118 deletions(-) delete mode 100644 crates/zip/benches/zip_benchmark.rs diff --git a/Cargo.lock b/Cargo.lock index 0fb09e78e..82f66641e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10489,15 +10489,10 @@ dependencies = [ name = "rustfs-zip" version = "1.0.0-rc.2" dependencies = [ - "astral-tokio-tar", "async-compression", - "criterion", "hotpath", - "tempfile", "thiserror 2.0.20", "tokio", - "tokio-stream", - "zip", ] [[package]] diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml index 54cdd9547..01ec5d84e 100644 --- a/crates/zip/Cargo.toml +++ b/crates/zip/Cargo.toml @@ -20,7 +20,7 @@ repository.workspace = true rust-version.workspace = true version.workspace = true homepage.workspace = true -description = "ZIP file handling for RustFS, providing support for reading and writing ZIP archives." +description = "Archive format detection and async stream decoders for RustFS." keywords = ["zip", "compression", "rustfs", "Minio"] categories = ["web-programming", "development-tools", "compression"] documentation = "https://docs.rs/rustfs-zip/latest/rustfs_zip/" @@ -28,10 +28,6 @@ documentation = "https://docs.rs/rustfs-zip/latest/rustfs_zip/" [lib] doctest = false -[[bench]] -name = "zip_benchmark" -harness = false - [features] default = [] hotpath = ["hotpath/hotpath", "hotpath/tokio"] @@ -48,16 +44,8 @@ async-compression = { workspace = true, features = [ "zstd", "xz", ] } -tokio = { workspace = true, features = ["fs", "io-util", "macros", "rt-multi-thread"] } -tokio-stream = { workspace = true } -astral-tokio-tar = { workspace = true } +tokio = { workspace = true, features = ["io-util", "macros", "rt"] } thiserror = { workspace = true } -zip = { workspace = true } - -[dev-dependencies] -criterion = { workspace = true, features = ["html_reports"] } -tempfile = { workspace = true } - [lints] workspace = true diff --git a/crates/zip/README.md b/crates/zip/README.md index 78e105f5f..a9b7865d5 100644 --- a/crates/zip/README.md +++ b/crates/zip/README.md @@ -1,9 +1,9 @@ [![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com) -# RustFS Zip - Archive And Compression Primitives +# RustFS Zip - Archive Format Detection And Stream Decoding

- High-performance compression and archiving for RustFS object storage + Archive format detection and async stream decoders for RustFS object storage

@@ -17,53 +17,23 @@ ## 📖 Overview -**RustFS Zip** provides archive and compression primitives for the [RustFS](https://rustfs.com) distributed object storage system. Today it is primarily used by RustFS archive extract flows to: +**RustFS Zip** provides the archive primitives used by the [RustFS](https://rustfs.com) archive extract flow: -- identify archive/compression formats by extension -- stream tar and tar+compression inputs through async decoders -- provide small ZIP read/write helpers for local archive workflows +- identify a compression format from an archive extension +- wrap an async reader in the matching stream decoder +- carry the shared default archive guardrails ## Current Features -- A clearer type model with: - - `CompressionCodec` for stream codecs - - `ArchiveKind` for container families - - `ArchiveFormat` for concrete archive/container combinations -- Async stream codecs for `gzip`, `bzip2`, `zlib`, `xz`, and `zstd` -- Tar archive iteration over async readers through `read_archive_entries()` / `extract_tar_entries()` -- Archive guardrails through `ArchiveLimits` for entry count, entry size, total unpacked size, and path length -- In-memory compression helpers for payload round-trip workflows -- Blocking ZIP create/extract helpers for local archive files -- ZIP helper metadata via `ZipEntry`, including: - - `compression_method` - - `archive_kind` - - `format` - - `unix_mode` -- ZIP helper options via `ZipWriteOptions`, including: - - `compression_level` - - `create_directory_entries` - -## Compatibility - -- `CompressionFormat` is retained as a compatibility layer for existing callers -- New code should prefer `ArchiveFormat`, `ArchiveKind`, and `CompressionCodec` when expressing archive semantics - -## ZIP Helper Scope - -The file-based ZIP helper APIs are best suited for: - -- local archive import/export flows -- admin-side packaging helpers -- test fixtures and tooling - -They are not intended to be a remote streaming ZIP access engine. +- `CompressionFormat::from_extension()` for extension-based format detection, including tar-family suffixes such as `tgz`, `tbz2`, `txz`, and `tzst` +- `CompressionFormat::get_decoder()` for async stream decoding of `gzip`, `bzip2`, `zlib`, `xz`, and `zstd`, plus a pass-through reader for plain `tar` +- `ArchiveLimits` with the default entry count, entry size, total unpacked size, and path length guardrails ## Current Boundaries -- ZIP is supported via file-based helper APIs, not the tar-family async stream APIs -- Tar-family stream APIs are intended for `tar`, `tar.gz`, `tar.bz2`, `tar.xz`, `tar.zst`, and similar compressed tar flows -- Default archive guardrails are intentionally conservative and do not replace higher-level RustFS object-path validation -- This crate does not currently implement a general-purpose parallel archive engine +- ZIP has no stream decoder: `get_decoder()` rejects `CompressionFormat::Zip`, because ZIP needs central-directory semantics that a forward-only stream cannot provide +- This crate detects formats and hands back decoders; archive iteration, entry writing, and extraction to disk belong to the caller +- `ArchiveLimits` carries the values only; enforcement and the resulting protocol error belong to the caller - Archive extraction safety policy remains the responsibility of the RustFS caller for object-store flows ## 📚 Documentation diff --git a/crates/zip/benches/zip_benchmark.rs b/crates/zip/benches/zip_benchmark.rs deleted file mode 100644 index c7047918b..000000000 --- a/crates/zip/benches/zip_benchmark.rs +++ /dev/null @@ -1,416 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; -use rustfs_zip::{ - ArchiveLimits, CompressionFormat, CompressionLevel, ZipWriteOptions, create_zip_with_options, extract_tar_entries, - extract_zip_to_path_with_limits, extract_zip_with_limits, -}; -use std::hint::black_box; -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; -use tempfile::tempdir; -use tokio::runtime::Builder; -use tokio_tar::{Builder as TarBuilder, Header}; -use zip::ZipArchive; - -fn build_runtime() -> tokio::runtime::Runtime { - Builder::new_current_thread() - .enable_all() - .build() - .expect("build tokio runtime for rustfs-zip benchmarks") -} - -async fn build_tar_payload(entry_count: usize, payload_size: usize) -> Vec { - let sink = tokio::io::duplex(64 * 1024); - let (writer, mut reader) = sink; - let write_task = tokio::spawn(async move { - let mut builder = TarBuilder::new(writer); - let payload = vec![b'a'; payload_size]; - for index in 0..entry_count { - let mut header = Header::new_gnu(); - header.set_size(payload.len() as u64); - header.set_mode(0o644); - header.set_cksum(); - builder - .append_data(&mut header, format!("entry-{index}.txt"), &payload[..]) - .await - .expect("append tar benchmark entry"); - } - builder.finish().await.expect("finish tar benchmark archive"); - }); - - let mut output = Vec::new(); - tokio::io::copy(&mut reader, &mut output) - .await - .expect("read tar benchmark archive"); - write_task.await.expect("join tar writer task"); - output -} - -async fn build_compressed_tar_payload(format: CompressionFormat, entry_count: usize, payload_size: usize) -> Vec { - let tar_payload = build_tar_payload(entry_count, payload_size).await; - rustfs_zip::Compressor::new(format) - .compress(&tar_payload) - .await - .expect("compress tar benchmark payload") -} - -fn bench_tar_family_extract(c: &mut Criterion) { - let runtime = build_runtime(); - let mut group = c.benchmark_group("zip_tar_family_extract"); - - for (name, format, entry_count, payload_size) in [ - ("tar_gzip_small_many", CompressionFormat::Gzip, 64usize, 256usize), - ("tar_zstd_medium", CompressionFormat::Zstd, 16usize, 16 * 1024usize), - ] { - let payload = runtime.block_on(build_compressed_tar_payload(format, entry_count, payload_size)); - group.throughput(Throughput::Bytes(payload.len() as u64)); - group.bench_with_input(BenchmarkId::new(name, payload.len()), &payload, |b, payload| { - b.iter(|| { - runtime.block_on(async { - let seen = Arc::new(AtomicUsize::new(0)); - let seen_ref = Arc::clone(&seen); - extract_tar_entries(std::io::Cursor::new(payload.clone()), format, move |_entry| { - let seen_ref = Arc::clone(&seen_ref); - async move { - seen_ref.fetch_add(1, Ordering::Relaxed); - Ok(()) - } - }) - .await - .expect("extract tar benchmark payload"); - black_box(seen.load(Ordering::Relaxed)); - }); - }); - }); - } - - group.finish(); -} - -fn bench_zip_helper_round_trip(c: &mut Criterion) { - let runtime = build_runtime(); - let mut group = c.benchmark_group("zip_helper_round_trip"); - - let zip_matrix = [ - ("stored_flat_32x128", CompressionLevel::Fastest, 32usize, 128usize, "flat"), - ("stored_nested_32x256", CompressionLevel::Fastest, 32usize, 256usize, "nested"), - ("stored_flat_256x128", CompressionLevel::Fastest, 256usize, 128usize, "flat"), - ("deflated_flat_32x1k", CompressionLevel::Best, 32usize, 1024usize, "flat"), - ("deflated_nested_256x1k", CompressionLevel::Best, 256usize, 1024usize, "nested"), - ("deflated_deep_1024x4k", CompressionLevel::Best, 1024usize, 4 * 1024usize, "deep"), - ]; - - for (name, compression_level, file_count, payload_size, layout) in zip_matrix { - let files = (0..file_count) - .map(|index| { - let path = match layout { - "flat" => format!("file-{index}.txt"), - "nested" => format!("batch-{}/file-{index}.txt", index % 8), - "deep" => format!("lvl1/lvl2-{}/lvl3-{}/file-{index}.txt", index % 16, index % 32), - _ => format!("file-{index}.txt"), - }; - (path, vec![b'b'; payload_size]) - }) - .collect::>(); - let total_bytes = (file_count * payload_size) as u64; - group.throughput(Throughput::Bytes(total_bytes)); - - group.bench_with_input(BenchmarkId::new(name, total_bytes), &files, |b, files| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - let extract_path = temp.path().join("extract"); - runtime.block_on(async { - create_zip_with_options( - &zip_path, - files.clone(), - ZipWriteOptions { - compression_level, - create_directory_entries: true, - }, - ) - .await - .expect("create zip benchmark archive"); - - let entries = extract_zip_with_limits(&zip_path, &extract_path, ArchiveLimits::default()) - .await - .expect("extract zip benchmark archive"); - black_box(entries.len()); - }); - }); - }); - } - - group.finish(); -} - -fn bench_zip_helper_hotspot_breakdown(c: &mut Criterion) { - let runtime = build_runtime(); - let mut group = c.benchmark_group("zip_helper_hotspot_breakdown"); - let files = (0..32) - .map(|index| (format!("batch/file-{index}.txt"), vec![b'c'; 256])) - .collect::>(); - let total_bytes = (32 * 256) as u64; - group.throughput(Throughput::Bytes(total_bytes)); - - group.bench_function("fs_setup_cleanup_only", |b| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - let extract_path = temp.path().join("extract"); - black_box((zip_path, extract_path)); - }); - }); - - group.bench_function("zip_create_only_stored_small", |b| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - runtime.block_on(async { - create_zip_with_options( - &zip_path, - files.clone(), - ZipWriteOptions { - compression_level: CompressionLevel::Fastest, - create_directory_entries: true, - }, - ) - .await - .expect("create zip benchmark archive"); - }); - }); - }); - - let payload_for_extract = { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - runtime.block_on(async { - create_zip_with_options( - &zip_path, - files.clone(), - ZipWriteOptions { - compression_level: CompressionLevel::Fastest, - create_directory_entries: true, - }, - ) - .await - .expect("prepare zip benchmark extract payload"); - }); - std::fs::read(&zip_path).expect("read benchmark zip payload") - }; - - group.bench_function("zip_extract_only_stored_small", |b| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - let extract_path = temp.path().join("extract"); - std::fs::write(&zip_path, &payload_for_extract).expect("write benchmark zip payload"); - runtime.block_on(async { - let entries = extract_zip_with_limits(&zip_path, &extract_path, ArchiveLimits::default()) - .await - .expect("extract zip benchmark archive"); - black_box(entries.len()); - }); - }); - }); - - group.bench_function("zip_extract_only_stored_small_summary_only", |b| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - let extract_path = temp.path().join("extract"); - std::fs::write(&zip_path, &payload_for_extract).expect("write benchmark zip payload"); - runtime.block_on(async { - let summary = extract_zip_to_path_with_limits(&zip_path, &extract_path, ArchiveLimits::default()) - .await - .expect("extract zip benchmark summary path"); - black_box(summary.entry_count); - }); - }); - }); - - group.bench_function("zip_reader_only_stored_small", |b| { - b.iter(|| { - let cursor = std::io::Cursor::new(payload_for_extract.clone()); - let mut archive = ZipArchive::new(cursor).expect("open zip archive for reader-only benchmark"); - let mut total_bytes = 0usize; - for index in 0..archive.len() { - let mut zip_file = archive.by_index(index).expect("access zip entry by index"); - let enclosed_name = zip_file - .enclosed_name() - .expect("resolve enclosed zip entry name") - .to_string_lossy() - .replace('\\', "/"); - let size = zip_file.size(); - assert!(!enclosed_name.is_empty(), "zip reader-only benchmark expects non-empty names"); - assert!( - size <= ArchiveLimits::default().max_entry_size, - "zip reader-only benchmark expects small entries" - ); - if !zip_file.is_dir() { - let mut sink = [0_u8; 256]; - let bytes_read = - std::io::Read::read(&mut zip_file, &mut sink).expect("read zip entry payload for reader-only benchmark"); - total_bytes += bytes_read; - } - } - black_box(total_bytes); - }); - }); - - group.bench_function("file_write_only_stored_small", |b| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let extract_path = temp.path().join("extract"); - std::fs::create_dir_all(&extract_path).expect("create extract dir for file-write-only benchmark"); - let mut total_bytes = 0usize; - for index in 0..32 { - let path = extract_path.join(format!("file-{index}.txt")); - std::fs::write(&path, [b'c'; 256]).expect("write small file for file-write-only benchmark"); - total_bytes += 256; - } - black_box(total_bytes); - }); - }); - - group.finish(); -} - -fn build_object_archive_files( - metadata_count: usize, - metadata_size: usize, - payload_count: usize, - payload_size: usize, -) -> Vec<(String, Vec)> { - let mut files = Vec::with_capacity(metadata_count * 2 + payload_count); - - for index in 0..metadata_count { - let key_prefix = format!( - "bucket-a/shard-{}/tenant-{}/dataset-{}/object-{index:04}", - index % 8, - index % 16, - index % 32 - ); - files.push(( - format!("{key_prefix}/meta.json"), - format!( - "{{\"key\":\"object-{index:04}\",\"etag\":\"{:032x}\",\"size\":{},\"content_type\":\"application/octet-stream\"}}", - index, - payload_size - ) - .into_bytes(), - )); - files.push((format!("{key_prefix}/tags.txt"), vec![b'm'; metadata_size])); - } - - for index in 0..payload_count { - let payload_prefix = format!( - "bucket-a/shard-{}/tenant-{}/dataset-{}/object-{index:04}", - index % 8, - index % 16, - index % 32 - ); - files.push((format!("{payload_prefix}/part-00000.bin"), vec![b'p'; payload_size])); - } - - files -} - -fn bench_zip_object_archive_extract(c: &mut Criterion) { - let runtime = build_runtime(); - let mut group = c.benchmark_group("zip_object_archive_extract"); - - for (name, compression_level, metadata_count, metadata_size, payload_count, payload_size) in [ - ( - "stored_metadata_heavy_384m_24p", - CompressionLevel::Fastest, - 384usize, - 192usize, - 24usize, - 32 * 1024usize, - ), - ( - "deflated_mixed_192m_32p", - CompressionLevel::Best, - 192usize, - 256usize, - 32usize, - 64 * 1024usize, - ), - ] { - let files = build_object_archive_files(metadata_count, metadata_size, payload_count, payload_size); - let total_bytes = files.iter().map(|(_, payload)| payload.len() as u64).sum::(); - let payload = { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("object-archive.zip"); - runtime.block_on(async { - create_zip_with_options( - &zip_path, - files.clone(), - ZipWriteOptions { - compression_level, - create_directory_entries: true, - }, - ) - .await - .expect("create object archive benchmark payload"); - }); - std::fs::read(&zip_path).expect("read object archive benchmark payload") - }; - - group.throughput(Throughput::Bytes(total_bytes)); - group.bench_function(BenchmarkId::new("extract_full", name), |b| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - let extract_path = temp.path().join("extract"); - std::fs::write(&zip_path, &payload).expect("write object archive benchmark payload"); - runtime.block_on(async { - let entries = extract_zip_with_limits(&zip_path, &extract_path, ArchiveLimits::default()) - .await - .expect("extract object archive benchmark payload"); - black_box(entries.len()); - }); - }); - }); - - group.bench_function(BenchmarkId::new("extract_summary_only", name), |b| { - b.iter(|| { - let temp = tempdir().expect("create benchmark tempdir"); - let zip_path = temp.path().join("archive.zip"); - let extract_path = temp.path().join("extract"); - std::fs::write(&zip_path, &payload).expect("write object archive benchmark payload"); - runtime.block_on(async { - let summary = extract_zip_to_path_with_limits(&zip_path, &extract_path, ArchiveLimits::default()) - .await - .expect("extract object archive benchmark summary"); - black_box(summary.file_count); - }); - }); - }); - } - - group.finish(); -} - -criterion_group!( - benches, - bench_tar_family_extract, - bench_zip_helper_round_trip, - bench_zip_helper_hotspot_breakdown, - bench_zip_object_archive_extract -); -criterion_main!(benches); diff --git a/crates/zip/src/lib.rs b/crates/zip/src/lib.rs index d61968fbb..82aa8867a 100644 --- a/crates/zip/src/lib.rs +++ b/crates/zip/src/lib.rs @@ -13,21 +13,8 @@ // limitations under the License. use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZlibDecoder, ZstdDecoder}; -use async_compression::tokio::write::{BzEncoder, GzipEncoder, XzEncoder, ZlibEncoder, ZstdEncoder}; -use std::collections::HashSet; -use std::future::Future; -use std::io::{Read, Write}; -use std::path::{Component, Path, PathBuf}; -use std::pin::Pin; -use std::sync::{Arc, Mutex}; -use std::task::{Context, Poll}; use thiserror::Error; -use tokio::fs::File; -use tokio::io::{self, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader, BufWriter}; -use tokio::task::spawn_blocking; -use tokio_stream::StreamExt; -use tokio_tar::Archive; -use zip::{CompressionMethod, ZipArchive, ZipWriter, write::SimpleFileOptions}; +use tokio::io::{AsyncRead, BufReader}; pub type Result = std::result::Result; @@ -38,51 +25,6 @@ pub enum ZipError { format: CompressionFormat, operation: &'static str, }, - #[error("invalid compression level {0}: value exceeds i32::MAX")] - InvalidCompressionLevel(u32), - #[error("unsafe archive entry path: {0}")] - UnsafeEntryPath(String), - #[error("archive entry path length {length} exceeds limit {limit}: {path}")] - EntryPathTooLong { path: String, length: usize, limit: usize }, - #[error("archive entry count {count} exceeds limit {limit}")] - EntryCountLimitExceeded { count: usize, limit: usize }, - #[error("archive entry '{path}' size {size} exceeds limit {limit}")] - EntrySizeLimitExceeded { path: String, size: u64, limit: u64 }, - #[error("archive total unpacked size {size} exceeds limit {limit}")] - TotalUnpackedSizeLimitExceeded { size: u64, limit: u64 }, - #[error(transparent)] - Io(#[from] io::Error), - #[error(transparent)] - Zip(#[from] zip::result::ZipError), - #[error(transparent)] - Join(#[from] tokio::task::JoinError), -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum CompressionCodec { - Gzip, - Bzip2, - Xz, - Zlib, - Zstd, -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum ArchiveKind { - Tar, - Zip, -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum ArchiveFormat { - Tar, - TarGzip, - TarBzip2, - TarXz, - TarZlib, - TarZstd, - Zip, - Unknown, } #[derive(Debug, PartialEq, Eq, Clone, Copy)] @@ -97,35 +39,9 @@ pub enum CompressionFormat { Unknown, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum CompressionLevel { - Fastest, - Best, - #[default] - Default, - Level(u32), -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ZipEntry { - pub name: String, - pub size: u64, - pub compressed_size: u64, - pub is_dir: bool, - pub compression_method: String, - pub archive_kind: ArchiveKind, - pub format: ArchiveFormat, - pub unix_mode: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct ZipExtractSummary { - pub entry_count: usize, - pub directory_count: usize, - pub file_count: usize, - pub total_unpacked_size: u64, -} - +/// Archive guardrails. The values are carried here so every archive caller +/// shares one default policy; enforcement belongs to the caller, which maps a +/// breach onto its own protocol error. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ArchiveLimits { pub max_entries: usize, @@ -147,103 +63,23 @@ impl Default for ArchiveLimits { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ZipWriteOptions { - pub compression_level: CompressionLevel, - pub create_directory_entries: bool, -} - -impl Default for ZipWriteOptions { - fn default() -> Self { - Self { - compression_level: CompressionLevel::Default, - create_directory_entries: false, - } - } -} - -const SMALL_ZIP_EXTRACT_FAST_PATH_LIMIT: u64 = 8 * 1024; - -#[derive(Clone, Default)] -struct SharedBuffer { - inner: Arc>>, -} - -impl SharedBuffer { - fn into_vec(self) -> Vec { - self.inner.lock().expect("shared in-memory writer lock poisoned").clone() - } -} - -impl AsyncWrite for SharedBuffer { - fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll> { - let mut inner = self - .inner - .lock() - .map_err(|_| io::Error::other("shared in-memory writer lock poisoned"))?; - inner.extend_from_slice(buf); - Poll::Ready(Ok(buf.len())) - } - - fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } -} - impl CompressionFormat { + /// Map an archive extension onto the stream codec needed to read it. + /// Tar-family suffixes (`tgz`, `tbz2`, `txz`, `tzst`, ...) resolve to their + /// codec because the tar container itself is read from the decoded stream. pub fn from_extension(ext: &str) -> Self { - Self::from_archive_format(ArchiveFormat::from_extension(ext)) - } - - pub fn from_archive_format(format: ArchiveFormat) -> Self { - match format { - ArchiveFormat::TarGzip => CompressionFormat::Gzip, - ArchiveFormat::TarBzip2 => CompressionFormat::Bzip2, - ArchiveFormat::TarXz => CompressionFormat::Xz, - ArchiveFormat::TarZlib => CompressionFormat::Zlib, - ArchiveFormat::TarZstd => CompressionFormat::Zstd, - ArchiveFormat::Tar => CompressionFormat::Tar, - ArchiveFormat::Zip => CompressionFormat::Zip, - ArchiveFormat::Unknown => CompressionFormat::Unknown, + match ext.to_ascii_lowercase().as_str() { + "gz" | "gzip" | "tgz" => CompressionFormat::Gzip, + "bz2" | "bzip2" | "tbz" | "tbz2" => CompressionFormat::Bzip2, + "xz" | "txz" => CompressionFormat::Xz, + "zlib" | "zz" => CompressionFormat::Zlib, + "zst" | "zstd" | "tzst" => CompressionFormat::Zstd, + "tar" => CompressionFormat::Tar, + "zip" => CompressionFormat::Zip, + _ => CompressionFormat::Unknown, } } - pub fn archive_format_from_path>(path: P) -> ArchiveFormat { - ArchiveFormat::from_path(path) - } - - pub fn archive_kind(&self) -> Option { - match self { - CompressionFormat::Tar => Some(ArchiveKind::Tar), - CompressionFormat::Zip => Some(ArchiveKind::Zip), - CompressionFormat::Gzip - | CompressionFormat::Bzip2 - | CompressionFormat::Xz - | CompressionFormat::Zlib - | CompressionFormat::Zstd - | CompressionFormat::Unknown => None, - } - } - - pub fn compression_codec(&self) -> Option { - match self { - CompressionFormat::Gzip => Some(CompressionCodec::Gzip), - CompressionFormat::Bzip2 => Some(CompressionCodec::Bzip2), - CompressionFormat::Xz => Some(CompressionCodec::Xz), - CompressionFormat::Zlib => Some(CompressionCodec::Zlib), - CompressionFormat::Zstd => Some(CompressionCodec::Zstd), - CompressionFormat::Tar | CompressionFormat::Zip | CompressionFormat::Unknown => None, - } - } - - pub fn from_path>(path: P) -> Self { - Self::from_archive_format(ArchiveFormat::from_path(path)) - } - pub fn extension(&self) -> &'static str { match self { CompressionFormat::Gzip => "gz", @@ -257,10 +93,6 @@ impl CompressionFormat { } } - pub fn is_supported(&self) -> bool { - !matches!(self, CompressionFormat::Unknown) - } - pub fn get_decoder(&self, input: R) -> Result> where R: AsyncRead + Send + Unpin + 'static, @@ -290,625 +122,14 @@ impl CompressionFormat { Ok(decoder) } - - fn convert_level(level: CompressionLevel) -> Result { - match level { - CompressionLevel::Fastest => Ok(async_compression::Level::Fastest), - CompressionLevel::Best => Ok(async_compression::Level::Best), - CompressionLevel::Default => Ok(async_compression::Level::Default), - CompressionLevel::Level(n) => { - let level = i32::try_from(n).map_err(|_| ZipError::InvalidCompressionLevel(n))?; - Ok(async_compression::Level::Precise(level)) - } - } - } - - pub fn get_encoder(&self, output: W, level: CompressionLevel) -> Result> - where - W: AsyncWrite + Send + Unpin + 'static, - { - let writer = BufWriter::new(output); - - let encoder: Box = match self { - CompressionFormat::Gzip => Box::new(GzipEncoder::with_quality(writer, Self::convert_level(level)?)), - CompressionFormat::Bzip2 => Box::new(BzEncoder::with_quality(writer, Self::convert_level(level)?)), - CompressionFormat::Zlib => Box::new(ZlibEncoder::with_quality(writer, Self::convert_level(level)?)), - CompressionFormat::Xz => Box::new(XzEncoder::with_quality(writer, Self::convert_level(level)?)), - CompressionFormat::Zstd => Box::new(ZstdEncoder::with_quality(writer, Self::convert_level(level)?)), - CompressionFormat::Tar => Box::new(writer), - CompressionFormat::Zip => { - return Err(ZipError::UnsupportedFormat { - format: *self, - operation: "stream encoding", - }); - } - CompressionFormat::Unknown => { - return Err(ZipError::UnsupportedFormat { - format: *self, - operation: "encoding", - }); - } - }; - - Ok(encoder) - } -} - -impl ArchiveFormat { - pub fn from_extension(ext: &str) -> Self { - match ext.to_ascii_lowercase().as_str() { - "gz" | "gzip" | "tgz" => ArchiveFormat::TarGzip, - "bz2" | "bzip2" | "tbz" | "tbz2" => ArchiveFormat::TarBzip2, - "xz" | "txz" => ArchiveFormat::TarXz, - "zlib" | "zz" => ArchiveFormat::TarZlib, - "zst" | "zstd" | "tzst" => ArchiveFormat::TarZstd, - "tar" => ArchiveFormat::Tar, - "zip" => ArchiveFormat::Zip, - _ => ArchiveFormat::Unknown, - } - } - - pub fn from_path>(path: P) -> Self { - let path = path.as_ref(); - let lower_name = path.file_name().and_then(|name| name.to_str()).map(str::to_ascii_lowercase); - - if let Some(name) = lower_name { - if name.ends_with(".tar.gz") || name.ends_with(".tgz") { - return ArchiveFormat::TarGzip; - } - if name.ends_with(".tar.bz2") || name.ends_with(".tbz") || name.ends_with(".tbz2") { - return ArchiveFormat::TarBzip2; - } - if name.ends_with(".tar.xz") || name.ends_with(".txz") { - return ArchiveFormat::TarXz; - } - if name.ends_with(".tar.zst") || name.ends_with(".tzst") { - return ArchiveFormat::TarZstd; - } - if name.ends_with(".tar.zlib") { - return ArchiveFormat::TarZlib; - } - } - - path.extension() - .and_then(|s| s.to_str()) - .map(Self::from_extension) - .unwrap_or(ArchiveFormat::Unknown) - } - - pub fn archive_kind(&self) -> Option { - match self { - ArchiveFormat::Tar - | ArchiveFormat::TarGzip - | ArchiveFormat::TarBzip2 - | ArchiveFormat::TarXz - | ArchiveFormat::TarZlib - | ArchiveFormat::TarZstd => Some(ArchiveKind::Tar), - ArchiveFormat::Zip => Some(ArchiveKind::Zip), - ArchiveFormat::Unknown => None, - } - } - - pub fn compression_codec(&self) -> Option { - match self { - ArchiveFormat::TarGzip => Some(CompressionCodec::Gzip), - ArchiveFormat::TarBzip2 => Some(CompressionCodec::Bzip2), - ArchiveFormat::TarXz => Some(CompressionCodec::Xz), - ArchiveFormat::TarZlib => Some(CompressionCodec::Zlib), - ArchiveFormat::TarZstd => Some(CompressionCodec::Zstd), - ArchiveFormat::Tar | ArchiveFormat::Zip | ArchiveFormat::Unknown => None, - } - } - - pub fn extension(&self) -> &'static str { - match self { - ArchiveFormat::Tar => "tar", - ArchiveFormat::TarGzip => "tar.gz", - ArchiveFormat::TarBzip2 => "tar.bz2", - ArchiveFormat::TarXz => "tar.xz", - ArchiveFormat::TarZlib => "tar.zlib", - ArchiveFormat::TarZstd => "tar.zst", - ArchiveFormat::Zip => "zip", - ArchiveFormat::Unknown => "", - } - } -} - -/// Read entries from a tar-family archive stream. -/// -/// Supported formats are: -/// - `CompressionFormat::Tar` -/// - `CompressionFormat::Gzip` -/// - `CompressionFormat::Bzip2` -/// - `CompressionFormat::Xz` -/// - `CompressionFormat::Zlib` -/// - `CompressionFormat::Zstd` -/// -/// `CompressionFormat::Zip` is intentionally not supported here because ZIP -/// requires central-directory semantics and is handled through file-based -/// helper APIs. -pub async fn read_archive_entries(input: R, format: CompressionFormat, callback: F) -> Result<()> -where - R: AsyncRead + Send + Unpin + 'static, - F: FnMut(tokio_tar::Entry>>) -> Fut + Send + 'static, - Fut: Future> + Send, -{ - read_archive_entries_with_limits(input, format, ArchiveLimits::default(), callback).await -} - -pub async fn read_archive_entries_with_limits( - input: R, - format: CompressionFormat, - limits: ArchiveLimits, - mut callback: F, -) -> Result<()> -where - R: AsyncRead + Send + Unpin + 'static, - F: FnMut(tokio_tar::Entry>>) -> Fut + Send + 'static, - Fut: Future> + Send, -{ - let decoder = format.get_decoder(input)?; - let mut ar = Archive::new(decoder); - let mut entries = ar.entries()?; - let mut entry_count = 0_usize; - let mut total_unpacked_size = 0_u64; - - while let Some(entry) = entries.next().await { - let entry = entry?; - entry_count += 1; - validate_archive_entry_count(entry_count, limits)?; - - let entry_path = entry.path()?.to_string_lossy().into_owned(); - validate_archive_entry_name(&entry_path, limits)?; - - let entry_size = entry.header().size()?; - validate_archive_entry_size(&entry_path, entry_size, limits)?; - total_unpacked_size = total_unpacked_size.saturating_add(entry_size); - validate_archive_total_size(total_unpacked_size, limits)?; - - callback(entry).await?; - } - - Ok(()) -} - -/// Backward-compatible wrapper for archive entry iteration. -pub async fn decompress(input: R, format: CompressionFormat, callback: F) -> Result<()> -where - R: AsyncRead + Send + Unpin + 'static, - F: FnMut(tokio_tar::Entry>>) -> Fut + Send + 'static, - Fut: Future> + Send, -{ - read_archive_entries(input, format, callback).await -} - -/// Explicit tar-family alias for callers that want a clearer name than -/// `decompress()`. -pub async fn extract_tar_entries(input: R, format: CompressionFormat, callback: F) -> Result<()> -where - R: AsyncRead + Send + Unpin + 'static, - F: FnMut(tokio_tar::Entry>>) -> Fut + Send + 'static, - Fut: Future> + Send, -{ - read_archive_entries(input, format, callback).await -} - -fn normalize_zip_entry_name(name: &str) -> Result { - let path = Path::new(name); - let mut normalized = PathBuf::new(); - - for component in path.components() { - match component { - Component::Normal(part) => normalized.push(part), - Component::CurDir => {} - Component::ParentDir | Component::RootDir | Component::Prefix(_) => { - return Err(ZipError::UnsafeEntryPath(name.to_string())); - } - } - } - - let normalized = normalized.to_string_lossy().replace('\\', "/"); - if normalized.is_empty() { - return Err(ZipError::UnsafeEntryPath(name.to_string())); - } - - Ok(normalized) -} - -fn validate_archive_entry_name(name: &str, limits: ArchiveLimits) -> Result<()> { - if !limits.validate_entry_paths { - return Ok(()); - } - - let normalized = normalize_zip_entry_name(name)?; - let length = normalized.len(); - if length > limits.max_path_length { - return Err(ZipError::EntryPathTooLong { - path: normalized, - length, - limit: limits.max_path_length, - }); - } - - Ok(()) -} - -fn validate_archive_entry_size(path: &str, size: u64, limits: ArchiveLimits) -> Result<()> { - if size > limits.max_entry_size { - return Err(ZipError::EntrySizeLimitExceeded { - path: path.to_string(), - size, - limit: limits.max_entry_size, - }); - } - - Ok(()) -} - -fn validate_archive_entry_count(count: usize, limits: ArchiveLimits) -> Result<()> { - if count > limits.max_entries { - return Err(ZipError::EntryCountLimitExceeded { - count, - limit: limits.max_entries, - }); - } - - Ok(()) -} - -fn validate_archive_total_size(total_size: u64, limits: ArchiveLimits) -> Result<()> { - if total_size > limits.max_total_unpacked_size { - return Err(ZipError::TotalUnpackedSizeLimitExceeded { - size: total_size, - limit: limits.max_total_unpacked_size, - }); - } - - Ok(()) -} - -fn zip_method_for_level(level: CompressionLevel) -> CompressionMethod { - match level { - CompressionLevel::Fastest => CompressionMethod::Stored, - CompressionLevel::Best | CompressionLevel::Default | CompressionLevel::Level(_) => CompressionMethod::Deflated, - } -} - -fn parent_directories_for(path: &str) -> Vec { - let path = Path::new(path); - let mut current = PathBuf::new(); - let mut directories = Vec::new(); - - if let Some(parent) = path.parent() { - for component in parent.components() { - if let Component::Normal(part) = component { - current.push(part); - directories.push(format!("{}/", current.to_string_lossy().replace('\\', "/"))); - } - } - } - - directories -} - -fn ensure_directory(path: &Path, created_directories: &mut HashSet) -> Result<()> { - let path = path.to_path_buf(); - if created_directories.insert(path.clone()) { - std::fs::create_dir_all(&path)?; - } - - Ok(()) -} - -fn write_small_zip_entry(reader: &mut R, output_path: &Path, size: u64) -> Result<()> { - let size = usize::try_from(size).map_err(|_| io::Error::other("small zip entry size overflow"))?; - let mut buffer = [0_u8; SMALL_ZIP_EXTRACT_FAST_PATH_LIMIT as usize]; - reader.read_exact(&mut buffer[..size])?; - // `read_exact` stops as soon as the declared bytes are read and never performs the - // terminal zero-length read that the zip crate's `Crc32Reader` uses to validate the - // entry checksum. Force one extra read to EOF so a corrupted small entry is rejected - // here, matching the large-entry `io::copy` path which already reads through EOF. - let mut trailing = [0_u8; 1]; - if reader.read(&mut trailing)? != 0 { - return Err(io::Error::other("small zip entry produced more data than its declared size").into()); - } - std::fs::write(output_path, &buffer[..size])?; - Ok(()) -} - -pub async fn extract_zip_simple, Q: AsRef>(zip_path: P, extract_to: Q) -> Result> { - extract_zip_with_limits(zip_path, extract_to, ArchiveLimits::default()).await -} - -pub async fn extract_zip_to_path_with_limits, Q: AsRef>( - zip_path: P, - extract_to: Q, - limits: ArchiveLimits, -) -> Result { - let zip_path = zip_path.as_ref().to_path_buf(); - let extract_to = extract_to.as_ref().to_path_buf(); - - spawn_blocking(move || extract_zip_impl(zip_path, extract_to, limits, false).map(|(_, summary)| summary)).await? -} - -pub async fn extract_zip_with_limits, Q: AsRef>( - zip_path: P, - extract_to: Q, - limits: ArchiveLimits, -) -> Result> { - let zip_path = zip_path.as_ref().to_path_buf(); - let extract_to = extract_to.as_ref().to_path_buf(); - - spawn_blocking(move || extract_zip_impl(zip_path, extract_to, limits, true).map(|(entries, _)| entries.unwrap_or_default())) - .await? -} - -fn extract_zip_impl( - zip_path: PathBuf, - extract_to: PathBuf, - limits: ArchiveLimits, - collect_entries: bool, -) -> Result<(Option>, ZipExtractSummary)> { - let file = std::fs::File::open(&zip_path)?; - let mut archive = ZipArchive::new(file)?; - std::fs::create_dir_all(&extract_to)?; - let mut created_directories = HashSet::from([extract_to.clone()]); - - let mut entries = collect_entries.then(|| Vec::with_capacity(archive.len())); - let mut summary = ZipExtractSummary::default(); - for index in 0..archive.len() { - validate_archive_entry_count(index + 1, limits)?; - let mut zip_file = archive.by_index(index)?; - let enclosed_name = zip_file - .enclosed_name() - .ok_or_else(|| ZipError::UnsafeEntryPath(zip_file.name().to_string()))?; - let entry_name = enclosed_name.to_string_lossy().replace('\\', "/"); - let is_dir = zip_file.is_dir(); - let size = zip_file.size(); - validate_archive_entry_name(&entry_name, limits)?; - validate_archive_entry_size(&entry_name, size, limits)?; - summary.total_unpacked_size = summary.total_unpacked_size.saturating_add(size); - validate_archive_total_size(summary.total_unpacked_size, limits)?; - let output_path = extract_to.join(&enclosed_name); - - if is_dir { - ensure_directory(&output_path, &mut created_directories)?; - summary.directory_count += 1; - } else { - if let Some(parent) = output_path.parent() { - ensure_directory(parent, &mut created_directories)?; - } - if size <= SMALL_ZIP_EXTRACT_FAST_PATH_LIMIT { - write_small_zip_entry(&mut zip_file, &output_path, size)?; - } else { - let mut output = std::fs::File::create(&output_path)?; - std::io::copy(&mut zip_file, &mut output)?; - } - summary.file_count += 1; - } - summary.entry_count += 1; - - if let Some(ref mut entries) = entries { - entries.push(ZipEntry { - name: entry_name, - size, - compressed_size: zip_file.compressed_size(), - is_dir, - compression_method: format!("{:?}", zip_file.compression()), - archive_kind: ArchiveKind::Zip, - format: ArchiveFormat::Zip, - unix_mode: zip_file.unix_mode(), - }); - } - } - - Ok((entries, summary)) -} - -pub async fn create_zip_simple>( - zip_path: P, - files: Vec<(String, Vec)>, - compression_level: CompressionLevel, -) -> Result<()> { - create_zip_with_options( - zip_path, - files, - ZipWriteOptions { - compression_level, - ..ZipWriteOptions::default() - }, - ) - .await -} - -pub async fn create_zip_with_options>( - zip_path: P, - files: Vec<(String, Vec)>, - options: ZipWriteOptions, -) -> Result<()> { - let zip_path = zip_path.as_ref().to_path_buf(); - - spawn_blocking(move || -> Result<()> { - if let Some(parent) = zip_path.parent() { - std::fs::create_dir_all(parent)?; - } - - let file = std::fs::File::create(&zip_path)?; - let mut writer = ZipWriter::new(file); - let file_options = SimpleFileOptions::default().compression_method(zip_method_for_level(options.compression_level)); - let explicit_directories = files - .iter() - .filter(|(name, _)| name.ends_with('/')) - .map(|(name, _)| normalize_zip_entry_name(name)) - .collect::>>()?; - let mut written_directories = HashSet::new(); - - for (name, contents) in files { - let entry_name = normalize_zip_entry_name(&name)?; - if name.ends_with('/') { - if written_directories.insert(entry_name.clone()) { - writer.add_directory(entry_name, file_options)?; - } - } else { - if options.create_directory_entries { - for directory in parent_directories_for(&entry_name) { - if !explicit_directories.contains(&directory) && written_directories.insert(directory.clone()) { - writer.add_directory(directory, file_options)?; - } - } - } - writer.start_file(entry_name, file_options)?; - writer.write_all(&contents)?; - } - } - - writer.finish()?; - Ok(()) - }) - .await? -} - -pub struct Compressor { - format: CompressionFormat, - level: CompressionLevel, -} - -impl Compressor { - pub fn new(format: CompressionFormat) -> Self { - Self { - format, - level: CompressionLevel::Default, - } - } - - pub fn with_level(mut self, level: CompressionLevel) -> Self { - self.level = level; - self - } - - pub async fn compress(&self, input: &[u8]) -> Result> { - let sink = SharedBuffer::default(); - let mut encoder = self.format.get_encoder(sink.clone(), self.level)?; - let mut reader = input; - - io::copy(&mut reader, &mut encoder).await?; - encoder.shutdown().await?; - drop(encoder); - - Ok(sink.into_vec()) - } - - pub async fn decompress(&self, input: Vec) -> Result> { - let mut output = Vec::new(); - let cursor = std::io::Cursor::new(input); - let mut decoder = self.format.get_decoder(cursor)?; - - decoder.read_to_end(&mut output).await?; - Ok(output) - } -} - -pub struct Decompressor { - format: CompressionFormat, -} - -impl Decompressor { - pub fn new(format: CompressionFormat) -> Self { - Self { format } - } - - pub fn auto_detect>(path: P) -> Self { - Self { - format: CompressionFormat::from_path(path), - } - } - - pub async fn decompress_file>(&self, input_path: P, output_path: P) -> Result<()> { - let input_file = File::open(&input_path).await?; - let output_file = File::create(&output_path).await?; - - let mut decoder = self.format.get_decoder(input_file)?; - let mut writer = BufWriter::new(output_file); - - io::copy(&mut decoder, &mut writer).await?; - writer.shutdown().await?; - - Ok(()) - } } #[cfg(test)] mod tests { use super::*; + use async_compression::tokio::write::GzipEncoder; use std::mem::size_of; - use tempfile::tempdir; - use tokio::fs; - use tokio::io::AsyncReadExt; - use tokio_tar::{Builder, Header}; - use zip::write::FileOptions; - - async fn build_tar_bytes(files: &[(&str, &[u8])]) -> io::Result> { - let sink = SharedBuffer::default(); - let handle = sink.clone(); - let mut builder = Builder::new(sink); - - for (path, content) in files { - let mut header = Header::new_gnu(); - header.set_size(content.len() as u64); - header.set_mode(0o644); - header.set_cksum(); - builder.append_data(&mut header, *path, &content[..]).await?; - } - - builder.finish().await?; - Ok(handle.into_vec()) - } - - async fn build_compressed_tar_bytes(format: CompressionFormat, files: &[(&str, &[u8])]) -> Result> { - let tar_bytes = build_tar_bytes(files).await?; - Compressor::new(format).compress(&tar_bytes).await - } - - async fn build_zip_file_with_entries(path: &Path, files: &[(&str, &[u8])]) -> Result<()> { - let path = path.to_path_buf(); - let files = files - .iter() - .map(|(name, content)| ((*name).to_string(), content.to_vec())) - .collect::>(); - spawn_blocking(move || -> Result<()> { - let file = std::fs::File::create(path)?; - let mut writer = ZipWriter::new(file); - let options: FileOptions<'_, ()> = FileOptions::default().compression_method(CompressionMethod::Stored); - for (name, content) in files { - writer.start_file(name, options)?; - writer.write_all(&content)?; - } - writer.finish()?; - Ok(()) - }) - .await??; - Ok(()) - } - - async fn collect_archive_entries(payload: Vec, format: CompressionFormat) -> Result)>> { - let seen = Arc::new(Mutex::new(Vec::<(String, Vec)>::new())); - let seen_ref = Arc::clone(&seen); - let cursor = std::io::Cursor::new(payload); - - read_archive_entries(cursor, format, move |mut entry| { - let seen_ref = Arc::clone(&seen_ref); - async move { - let path = entry.path()?.to_string_lossy().into_owned(); - let mut content = Vec::new(); - entry.read_to_end(&mut content).await?; - seen_ref.lock().expect("seen collection lock poisoned").push((path, content)); - Ok(()) - } - }) - .await?; - - Ok(seen.lock().expect("seen collection lock poisoned").clone()) - } + use tokio::io::{AsyncReadExt, AsyncWriteExt}; #[test] fn test_compression_format_from_extension() { @@ -918,869 +139,51 @@ mod tests { assert_eq!(CompressionFormat::from_extension("txt"), CompressionFormat::Unknown); } - #[test] - fn test_archive_format_from_extension() { - assert_eq!(ArchiveFormat::from_extension("gz"), ArchiveFormat::TarGzip); - assert_eq!(ArchiveFormat::from_extension("tbz2"), ArchiveFormat::TarBzip2); - assert_eq!(ArchiveFormat::from_extension("txz"), ArchiveFormat::TarXz); - assert_eq!(ArchiveFormat::from_extension("zip"), ArchiveFormat::Zip); - assert_eq!(ArchiveFormat::from_extension("txt"), ArchiveFormat::Unknown); - } - - #[test] - fn test_compression_format_from_path_handles_compound_suffixes() { - assert_eq!(CompressionFormat::from_path("archive.tar.gz"), CompressionFormat::Gzip); - assert_eq!(CompressionFormat::from_path("archive.tgz"), CompressionFormat::Gzip); - assert_eq!(CompressionFormat::from_path("archive.tar.bz2"), CompressionFormat::Bzip2); - assert_eq!(CompressionFormat::from_path("archive.zip"), CompressionFormat::Zip); - assert_eq!(CompressionFormat::from_path("archive"), CompressionFormat::Unknown); - } - - #[test] - fn test_archive_format_from_path_handles_compound_suffixes() { - assert_eq!(ArchiveFormat::from_path("archive.tar.gz"), ArchiveFormat::TarGzip); - assert_eq!(ArchiveFormat::from_path("archive.tar.bz2"), ArchiveFormat::TarBzip2); - assert_eq!(ArchiveFormat::from_path("archive.tar.xz"), ArchiveFormat::TarXz); - assert_eq!(ArchiveFormat::from_path("archive.tar.zst"), ArchiveFormat::TarZstd); - assert_eq!(ArchiveFormat::from_path("archive.zip"), ArchiveFormat::Zip); - assert_eq!(ArchiveFormat::from_path("archive"), ArchiveFormat::Unknown); - } - - #[test] - fn test_archive_format_and_legacy_compression_format_are_compatible() { - assert_eq!(CompressionFormat::from_archive_format(ArchiveFormat::TarGzip), CompressionFormat::Gzip); - assert_eq!(CompressionFormat::from_archive_format(ArchiveFormat::Tar), CompressionFormat::Tar); - assert_eq!(CompressionFormat::from_archive_format(ArchiveFormat::Zip), CompressionFormat::Zip); - } - - #[test] - fn test_archive_format_exposes_archive_kind_and_codec() { - assert_eq!(ArchiveFormat::TarGzip.archive_kind(), Some(ArchiveKind::Tar)); - assert_eq!(ArchiveFormat::TarGzip.compression_codec(), Some(CompressionCodec::Gzip)); - assert_eq!(ArchiveFormat::Tar.archive_kind(), Some(ArchiveKind::Tar)); - assert_eq!(ArchiveFormat::Tar.compression_codec(), None); - assert_eq!(ArchiveFormat::Zip.archive_kind(), Some(ArchiveKind::Zip)); - assert_eq!(ArchiveFormat::Zip.compression_codec(), None); - } - - #[test] - fn test_legacy_compression_format_exposes_kind_and_codec() { - assert_eq!(CompressionFormat::Gzip.archive_kind(), None); - assert_eq!(CompressionFormat::Gzip.compression_codec(), Some(CompressionCodec::Gzip)); - assert_eq!(CompressionFormat::Tar.archive_kind(), Some(ArchiveKind::Tar)); - assert_eq!(CompressionFormat::Tar.compression_codec(), None); - assert_eq!(CompressionFormat::Zip.archive_kind(), Some(ArchiveKind::Zip)); - } - #[test] fn test_compression_format_size_is_small() { assert!(size_of::() <= 8); assert!(size_of::>() <= 16); } - #[test] - fn test_convert_level_rejects_overflow() { - let err = match CompressionFormat::Gzip.get_encoder(SharedBuffer::default(), CompressionLevel::Level(u32::MAX)) { - Ok(_) => panic!("overflow level should return an error"), - Err(err) => err, - }; - assert!(matches!(err, ZipError::InvalidCompressionLevel(u32::MAX))); - } + #[tokio::test] + async fn test_get_decoder_round_trips_gzip_stream() { + let mut encoder = GzipEncoder::new(Vec::new()); + encoder.write_all(b"payload").await.expect("gzip encode should succeed"); + encoder.shutdown().await.expect("gzip encoder shutdown should succeed"); - #[test] - fn test_validate_archive_entry_name_rejects_absolute_path() { - let err = validate_archive_entry_name("/absolute.txt", ArchiveLimits::default()) - .expect_err("absolute path should fail validation"); - assert!(matches!(err, ZipError::UnsafeEntryPath(path) if path == "/absolute.txt")); + let mut decoder = CompressionFormat::Gzip + .get_decoder(std::io::Cursor::new(encoder.into_inner())) + .expect("gzip decoder should be created"); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).await.expect("gzip decode should succeed"); + + assert_eq!(decoded, b"payload"); } #[tokio::test] - async fn test_compressor_round_trip_gzip() { - let input = b"hello rustfs zip ".repeat(64); - let compressor = Compressor::new(CompressionFormat::Gzip); - - let compressed = compressor.compress(&input).await.expect("gzip compress should succeed"); - assert!(!compressed.is_empty()); - assert_ne!(compressed, input); - - let decompressed = compressor - .decompress(compressed) - .await - .expect("gzip decompress should succeed"); - assert_eq!(decompressed, input); - } - - #[tokio::test] - async fn test_compressor_round_trip_zstd() { - let input = b"zstd payload ".repeat(128); - let compressor = Compressor::new(CompressionFormat::Zstd).with_level(CompressionLevel::Best); - - let compressed = compressor.compress(&input).await.expect("zstd compress should succeed"); - let decompressed = compressor - .decompress(compressed) - .await - .expect("zstd decompress should succeed"); - assert_eq!(decompressed, input); - } - - #[tokio::test] - async fn test_zip_stream_encoder_is_rejected() { - let err = CompressionFormat::Zip - .get_encoder(SharedBuffer::default(), CompressionLevel::Default) + async fn test_get_decoder_rejects_zip_and_unknown_formats() { + let zip_err = CompressionFormat::Zip + .get_decoder(std::io::Cursor::new(Vec::::new())) .err() - .expect("zip stream encoder should be rejected"); + .expect("zip stream decoding should be rejected"); assert!(matches!( - err, - ZipError::UnsupportedFormat { - format: CompressionFormat::Zip, - operation: "stream encoding", - } - )); - } - - #[tokio::test] - async fn test_read_archive_entries_iterates_tar_gzip_entries() { - let gzip_bytes = - build_compressed_tar_bytes(CompressionFormat::Gzip, &[("nested/hello.txt", b"hello"), ("world.txt", b"world")]) - .await - .expect("tar.gz build should succeed"); - - let seen = collect_archive_entries(gzip_bytes, CompressionFormat::Gzip) - .await - .expect("tar.gz archive iteration should succeed"); - assert_eq!(seen.len(), 2); - assert_eq!(seen[0].0, "nested/hello.txt"); - assert_eq!(seen[0].1, b"hello"); - assert_eq!(seen[1].0, "world.txt"); - assert_eq!(seen[1].1, b"world"); - } - - #[tokio::test] - async fn test_read_archive_entries_iterates_tar_bzip2_entries() { - let payload = - build_compressed_tar_bytes(CompressionFormat::Bzip2, &[("nested/hello.txt", b"hello"), ("world.txt", b"world")]) - .await - .expect("tar.bz2 build should succeed"); - - let seen = collect_archive_entries(payload, CompressionFormat::Bzip2) - .await - .expect("tar.bz2 archive iteration should succeed"); - assert_eq!(seen.len(), 2); - assert_eq!(seen[0].0, "nested/hello.txt"); - assert_eq!(seen[1].0, "world.txt"); - } - - #[tokio::test] - async fn test_read_archive_entries_iterates_tar_xz_entries() { - let payload = - build_compressed_tar_bytes(CompressionFormat::Xz, &[("nested/hello.txt", b"hello"), ("world.txt", b"world")]) - .await - .expect("tar.xz build should succeed"); - - let seen = collect_archive_entries(payload, CompressionFormat::Xz) - .await - .expect("tar.xz archive iteration should succeed"); - assert_eq!(seen.len(), 2); - assert_eq!(seen[0].0, "nested/hello.txt"); - assert_eq!(seen[1].0, "world.txt"); - } - - #[tokio::test] - async fn test_read_archive_entries_iterates_tar_zstd_entries() { - let payload = - build_compressed_tar_bytes(CompressionFormat::Zstd, &[("nested/hello.txt", b"hello"), ("world.txt", b"world")]) - .await - .expect("tar.zst build should succeed"); - - let seen = collect_archive_entries(payload, CompressionFormat::Zstd) - .await - .expect("tar.zst archive iteration should succeed"); - assert_eq!(seen.len(), 2); - assert_eq!(seen[0].0, "nested/hello.txt"); - assert_eq!(seen[1].0, "world.txt"); - } - - #[tokio::test] - async fn test_extract_tar_entries_alias_matches_stream_behavior() { - let payload = build_compressed_tar_bytes(CompressionFormat::Gzip, &[("hello.txt", b"hello")]) - .await - .expect("tar.gz build should succeed"); - let seen = Arc::new(Mutex::new(Vec::::new())); - let seen_ref = Arc::clone(&seen); - - extract_tar_entries(std::io::Cursor::new(payload), CompressionFormat::Gzip, move |entry| { - let seen_ref = Arc::clone(&seen_ref); - async move { - seen_ref - .lock() - .expect("seen collection lock poisoned") - .push(entry.path()?.to_string_lossy().into_owned()); - Ok(()) - } - }) - .await - .expect("extract_tar_entries alias should succeed"); - - assert_eq!(seen.lock().expect("seen collection lock poisoned").as_slice(), ["hello.txt"]); - } - - #[tokio::test] - async fn test_read_archive_entries_rejects_zip_streams() { - let err = read_archive_entries(std::io::Cursor::new(Vec::::new()), CompressionFormat::Zip, |_entry| async { Ok(()) }) - .await - .expect_err("zip stream should be rejected"); - - assert!(matches!( - err, + zip_err, ZipError::UnsupportedFormat { format: CompressionFormat::Zip, operation: "stream decoding", } )); - } - - #[tokio::test] - async fn test_read_archive_entries_rejects_corrupt_tar_gzip_stream() { - let err = read_archive_entries( - std::io::Cursor::new(b"not-a-valid-gzip-stream".to_vec()), - CompressionFormat::Gzip, - |_entry| async { Ok(()) }, - ) - .await - .expect_err("corrupt tar.gz stream should fail"); - - assert!(matches!(err, ZipError::Io(_))); - } - - #[tokio::test] - async fn test_read_archive_entries_rejects_truncated_tar_gzip_stream() { - let payload = build_compressed_tar_bytes(CompressionFormat::Gzip, &[("hello.txt", b"hello world")]) - .await - .expect("tar.gz build should succeed"); - let truncated = payload[..payload.len() / 2].to_vec(); - - let err = read_archive_entries(std::io::Cursor::new(truncated), CompressionFormat::Gzip, |_entry| async { Ok(()) }) - .await - .expect_err("truncated tar.gz stream should fail"); - - assert!(matches!(err, ZipError::Io(_))); - } - - #[tokio::test] - async fn test_read_archive_entries_rejects_too_many_entries() { - let payload = build_compressed_tar_bytes(CompressionFormat::Gzip, &[("one.txt", b"1"), ("two.txt", b"2")]) - .await - .expect("tar.gz build should succeed"); - - let err = read_archive_entries_with_limits( - std::io::Cursor::new(payload), - CompressionFormat::Gzip, - ArchiveLimits { - max_entries: 1, - ..ArchiveLimits::default() - }, - |_entry| async { Ok(()) }, - ) - .await - .expect_err("entry count limit should fail"); - - assert!(matches!(err, ZipError::EntryCountLimitExceeded { count: 2, limit: 1 })); - } - - #[tokio::test] - async fn test_read_archive_entries_rejects_oversized_entry() { - let payload = build_compressed_tar_bytes(CompressionFormat::Gzip, &[("big.txt", b"hello world")]) - .await - .expect("tar.gz build should succeed"); - - let err = read_archive_entries_with_limits( - std::io::Cursor::new(payload), - CompressionFormat::Gzip, - ArchiveLimits { - max_entry_size: 4, - ..ArchiveLimits::default() - }, - |_entry| async { Ok(()) }, - ) - .await - .expect_err("entry size limit should fail"); + let unknown_err = CompressionFormat::Unknown + .get_decoder(std::io::Cursor::new(Vec::::new())) + .err() + .expect("unknown format decoding should be rejected"); assert!(matches!( - err, - ZipError::EntrySizeLimitExceeded { - path, - size: 11, - limit: 4, - } if path == "big.txt" + unknown_err, + ZipError::UnsupportedFormat { + format: CompressionFormat::Unknown, + operation: "decoding", + } )); } - - #[tokio::test] - async fn test_read_archive_entries_rejects_total_unpacked_size_limit() { - let payload = build_compressed_tar_bytes(CompressionFormat::Gzip, &[("one.txt", b"12345"), ("two.txt", b"67890")]) - .await - .expect("tar.gz build should succeed"); - - let err = read_archive_entries_with_limits( - std::io::Cursor::new(payload), - CompressionFormat::Gzip, - ArchiveLimits { - max_total_unpacked_size: 9, - ..ArchiveLimits::default() - }, - |_entry| async { Ok(()) }, - ) - .await - .expect_err("total unpacked size limit should fail"); - - assert!(matches!(err, ZipError::TotalUnpackedSizeLimitExceeded { size: 10, limit: 9 })); - } - - #[tokio::test] - async fn test_read_archive_entries_rejects_entry_path_length_limit() { - let payload = build_compressed_tar_bytes(CompressionFormat::Gzip, &[("nested/hello.txt", b"hello")]) - .await - .expect("tar.gz build should succeed"); - - let err = read_archive_entries_with_limits( - std::io::Cursor::new(payload), - CompressionFormat::Gzip, - ArchiveLimits { - max_path_length: 5, - ..ArchiveLimits::default() - }, - |_entry| async { Ok(()) }, - ) - .await - .expect_err("path length limit should fail"); - - assert!(matches!( - err, - ZipError::EntryPathTooLong { path, limit: 5, .. } if path == "nested/hello.txt" - )); - } - - #[tokio::test] - async fn test_create_and_extract_zip_round_trip() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("archive.zip"); - let extract_path = temp.path().join("extract"); - - create_zip_simple( - &zip_path, - vec![ - ("nested/hello.txt".to_string(), b"hello".to_vec()), - ("world.txt".to_string(), b"world".to_vec()), - ], - CompressionLevel::Default, - ) - .await - .expect("zip creation should succeed"); - - let entries = extract_zip_simple(&zip_path, &extract_path) - .await - .expect("zip extraction should succeed"); - assert_eq!(entries.len(), 2); - assert_eq!( - fs::read(extract_path.join("nested/hello.txt")) - .await - .expect("nested zip entry should be extracted"), - b"hello" - ); - assert_eq!( - fs::read(extract_path.join("world.txt")) - .await - .expect("root zip entry should be extracted"), - b"world" - ); - assert!(entries.iter().all(|entry| entry.archive_kind == ArchiveKind::Zip)); - assert!(entries.iter().all(|entry| entry.format == ArchiveFormat::Zip)); - } - - #[tokio::test] - async fn test_create_zip_with_directory_entries_and_extract_directory_scenarios() { - let temp = tempdir().expect("tempdir should be created"); - let explicit_zip_path = temp.path().join("explicit-directories.zip"); - let explicit_extract_path = temp.path().join("explicit-extract"); - let auto_zip_path = temp.path().join("auto-directories.zip"); - let auto_extract_path = temp.path().join("auto-extract"); - - create_zip_with_options( - &explicit_zip_path, - vec![ - ("nested/".to_string(), Vec::new()), - ("nested/deeper/".to_string(), Vec::new()), - ], - ZipWriteOptions { - compression_level: CompressionLevel::Default, - create_directory_entries: false, - }, - ) - .await - .expect("zip creation with explicit directory entries should succeed"); - - let explicit_entries = extract_zip_with_limits(&explicit_zip_path, &explicit_extract_path, ArchiveLimits::default()) - .await - .expect("zip extraction with explicit directory entries should succeed"); - - assert!( - explicit_entries - .iter() - .any(|entry| entry.name.trim_end_matches('/') == "nested" && entry.is_dir) - ); - assert!( - explicit_entries - .iter() - .any(|entry| entry.name.trim_end_matches('/') == "nested/deeper" && entry.is_dir) - ); - assert!( - fs::metadata(explicit_extract_path.join("nested")) - .await - .expect("nested directory should exist") - .is_dir() - ); - assert!( - fs::metadata(explicit_extract_path.join("nested/deeper")) - .await - .expect("nested deeper directory should exist") - .is_dir() - ); - - create_zip_with_options( - &auto_zip_path, - vec![("nested/deeper/file.txt".to_string(), b"hello".to_vec())], - ZipWriteOptions { - compression_level: CompressionLevel::Default, - create_directory_entries: true, - }, - ) - .await - .expect("zip creation with automatic directory entries should succeed"); - - let entries = extract_zip_with_limits(&auto_zip_path, &auto_extract_path, ArchiveLimits::default()) - .await - .expect("zip extraction with automatic directory entries should succeed"); - - assert!( - entries - .iter() - .any(|entry| entry.name.trim_end_matches('/') == "nested" && entry.is_dir) - ); - assert!( - entries - .iter() - .any(|entry| entry.name.trim_end_matches('/') == "nested/deeper" && entry.is_dir) - ); - assert!( - fs::metadata(auto_extract_path.join("nested")) - .await - .expect("nested directory should exist") - .is_dir() - ); - assert!( - fs::metadata(auto_extract_path.join("nested/deeper")) - .await - .expect("nested deeper directory should exist") - .is_dir() - ); - assert_eq!( - fs::read(auto_extract_path.join("nested/deeper/file.txt")) - .await - .expect("nested file should be extracted"), - b"hello" - ); - } - - #[tokio::test] - async fn test_create_zip_with_lots_of_small_files_round_trip() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("many-small-files.zip"); - let extract_path = temp.path().join("extract"); - let files = (0..32) - .map(|index| (format!("batch/file-{index}.txt"), format!("payload-{index}").into_bytes())) - .collect::>(); - - create_zip_with_options( - &zip_path, - files.clone(), - ZipWriteOptions { - compression_level: CompressionLevel::Default, - create_directory_entries: true, - }, - ) - .await - .expect("zip creation for many small files should succeed"); - - let entries = extract_zip_with_limits(&zip_path, &extract_path, ArchiveLimits::default()) - .await - .expect("zip extraction for many small files should succeed"); - - assert!(entries.len() >= files.len()); - for (path, expected) in files { - assert_eq!( - fs::read(extract_path.join(path)) - .await - .expect("small file should be extracted"), - expected - ); - } - } - - #[tokio::test] - async fn test_extract_zip_to_path_with_limits_returns_summary() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("summary.zip"); - let extract_path = temp.path().join("extract"); - - create_zip_with_options( - &zip_path, - vec![ - ("nested/".to_string(), Vec::new()), - ("nested/hello.txt".to_string(), b"hello".to_vec()), - ("world.txt".to_string(), b"world".to_vec()), - ], - ZipWriteOptions { - compression_level: CompressionLevel::Default, - create_directory_entries: false, - }, - ) - .await - .expect("zip creation for summary should succeed"); - - let summary = extract_zip_to_path_with_limits(&zip_path, &extract_path, ArchiveLimits::default()) - .await - .expect("zip extract summary should succeed"); - - assert_eq!(summary.entry_count, 3); - assert_eq!(summary.directory_count, 1); - assert_eq!(summary.file_count, 2); - assert_eq!(summary.total_unpacked_size, 10); - assert_eq!( - fs::read(extract_path.join("nested/hello.txt")) - .await - .expect("nested file should be extracted"), - b"hello" - ); - assert_eq!( - fs::read(extract_path.join("world.txt")) - .await - .expect("world file should be extracted"), - b"world" - ); - } - - #[tokio::test] - async fn test_zip_helper_exposes_stored_vs_deflated_metadata() { - let temp = tempdir().expect("tempdir should be created"); - let stored_zip_path = temp.path().join("stored.zip"); - let deflated_zip_path = temp.path().join("deflated.zip"); - let stored_extract_path = temp.path().join("stored-extract"); - let deflated_extract_path = temp.path().join("deflated-extract"); - let payload = b"compressible-content-".repeat(64); - - create_zip_with_options( - &stored_zip_path, - vec![("payload.txt".to_string(), payload.clone())], - ZipWriteOptions { - compression_level: CompressionLevel::Fastest, - create_directory_entries: false, - }, - ) - .await - .expect("stored zip creation should succeed"); - - create_zip_with_options( - &deflated_zip_path, - vec![("payload.txt".to_string(), payload.clone())], - ZipWriteOptions { - compression_level: CompressionLevel::Best, - create_directory_entries: false, - }, - ) - .await - .expect("deflated zip creation should succeed"); - - let stored_entries = extract_zip_with_limits(&stored_zip_path, &stored_extract_path, ArchiveLimits::default()) - .await - .expect("stored zip extraction should succeed"); - let deflated_entries = extract_zip_with_limits(&deflated_zip_path, &deflated_extract_path, ArchiveLimits::default()) - .await - .expect("deflated zip extraction should succeed"); - - assert_eq!(stored_entries[0].compression_method, "Stored"); - assert_eq!(deflated_entries[0].compression_method, "Deflated"); - assert_eq!( - fs::read(stored_extract_path.join("payload.txt")) - .await - .expect("stored payload should be extracted"), - payload - ); - assert_eq!( - fs::read(deflated_extract_path.join("payload.txt")) - .await - .expect("deflated payload should be extracted"), - payload - ); - assert!(deflated_entries[0].compressed_size <= stored_entries[0].compressed_size); - } - - #[tokio::test] - async fn test_create_zip_rejects_unsafe_entry_name() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("archive.zip"); - - let err = create_zip_simple( - &zip_path, - vec![("../escape.txt".to_string(), b"escape".to_vec())], - CompressionLevel::Default, - ) - .await - .unwrap_err(); - - assert!(matches!(err, ZipError::UnsafeEntryPath(path) if path == "../escape.txt")); - } - - #[tokio::test] - async fn test_extract_zip_rejects_parent_traversal_entry_and_writes_nothing_outside_target() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("traversal.zip"); - let extract_path = temp.path().join("extract"); - - build_zip_file_with_entries(&zip_path, &[("safe.txt", b"safe"), ("../escape.txt", b"escape")]) - .await - .expect("zip fixture should be created"); - - let err = extract_zip_simple(&zip_path, &extract_path) - .await - .expect_err("zip entry with parent traversal should be rejected"); - - assert!(matches!(err, ZipError::UnsafeEntryPath(path) if path == "../escape.txt")); - assert!( - fs::metadata(temp.path().join("escape.txt")).await.is_err(), - "traversal entry must not be written outside the extraction target" - ); - assert!( - fs::metadata(extract_path.join("escape.txt")).await.is_err(), - "traversal entry must not be written inside the extraction target either" - ); - } - - #[tokio::test] - async fn test_extract_zip_rejects_small_entry_with_corrupted_crc() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("corrupt-crc.zip"); - let extract_path = temp.path().join("extract"); - - // A stored entry well under the small-entry fast-path limit so extraction takes - // the in-memory buffer path rather than the streaming `io::copy` path. - let content = b"small-entry-crc-payload"; - assert!((content.len() as u64) <= SMALL_ZIP_EXTRACT_FAST_PATH_LIMIT); - build_zip_file_with_entries(&zip_path, &[("small.txt", content)]) - .await - .expect("zip fixture should be created"); - - // Corrupt the stored entry data so its bytes no longer match the recorded CRC32. - let mut raw = fs::read(&zip_path).await.expect("zip fixture should be readable"); - let offset = raw - .windows(content.len()) - .position(|window| window == content) - .expect("stored entry data should be present in the zip"); - raw[offset] ^= 0xFF; - fs::write(&zip_path, &raw).await.expect("corrupted zip should be writable"); - - let err = extract_zip_simple(&zip_path, &extract_path) - .await - .expect_err("small entry with a corrupted CRC should be rejected"); - - assert!( - matches!(err, ZipError::Io(ref e) if e.kind() == std::io::ErrorKind::InvalidData), - "expected an InvalidData checksum error, got {err:?}" - ); - } - - #[tokio::test] - async fn test_extract_zip_allows_entry_count_exactly_at_limit() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("at-limit.zip"); - let extract_path = temp.path().join("extract"); - - build_zip_file_with_entries(&zip_path, &[("one.txt", b"1"), ("two.txt", b"2")]) - .await - .expect("zip fixture should be created"); - - let entries = extract_zip_with_limits( - &zip_path, - &extract_path, - ArchiveLimits { - max_entries: 2, - ..ArchiveLimits::default() - }, - ) - .await - .expect("entry count exactly at the limit should extract successfully"); - - assert_eq!(entries.len(), 2); - } - - #[tokio::test] - async fn test_extract_zip_allows_total_unpacked_size_exactly_at_limit() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("total-at-limit.zip"); - let extract_path = temp.path().join("extract"); - - build_zip_file_with_entries(&zip_path, &[("one.txt", b"12345"), ("two.txt", b"67890")]) - .await - .expect("zip fixture should be created"); - - let entries = extract_zip_with_limits( - &zip_path, - &extract_path, - ArchiveLimits { - max_total_unpacked_size: 10, - ..ArchiveLimits::default() - }, - ) - .await - .expect("total unpacked size exactly at the limit should extract successfully"); - - assert_eq!(entries.len(), 2); - assert_eq!( - fs::read(extract_path.join("two.txt")) - .await - .expect("second entry should be extracted"), - b"67890" - ); - } - - #[tokio::test] - async fn test_extract_zip_rejects_too_many_entries() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("too-many.zip"); - let extract_path = temp.path().join("extract"); - - build_zip_file_with_entries(&zip_path, &[("one.txt", b"1"), ("two.txt", b"2")]) - .await - .expect("zip fixture should be created"); - - let err = extract_zip_with_limits( - &zip_path, - &extract_path, - ArchiveLimits { - max_entries: 1, - ..ArchiveLimits::default() - }, - ) - .await - .expect_err("zip entry count limit should fail"); - - assert!(matches!(err, ZipError::EntryCountLimitExceeded { count: 2, limit: 1 })); - } - - #[tokio::test] - async fn test_extract_zip_rejects_oversized_entry() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("oversized.zip"); - let extract_path = temp.path().join("extract"); - - build_zip_file_with_entries(&zip_path, &[("big.txt", b"hello world")]) - .await - .expect("zip fixture should be created"); - - let err = extract_zip_with_limits( - &zip_path, - &extract_path, - ArchiveLimits { - max_entry_size: 4, - ..ArchiveLimits::default() - }, - ) - .await - .expect_err("zip entry size limit should fail"); - - assert!(matches!( - err, - ZipError::EntrySizeLimitExceeded { - path, - size: 11, - limit: 4, - } if path == "big.txt" - )); - } - - #[tokio::test] - async fn test_extract_zip_rejects_total_unpacked_size_limit() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("too-large-total.zip"); - let extract_path = temp.path().join("extract"); - - build_zip_file_with_entries(&zip_path, &[("one.txt", b"12345"), ("two.txt", b"67890")]) - .await - .expect("zip fixture should be created"); - - let err = extract_zip_with_limits( - &zip_path, - &extract_path, - ArchiveLimits { - max_total_unpacked_size: 9, - ..ArchiveLimits::default() - }, - ) - .await - .expect_err("zip total size limit should fail"); - - assert!(matches!(err, ZipError::TotalUnpackedSizeLimitExceeded { size: 10, limit: 9 })); - } - - #[tokio::test] - async fn test_extract_zip_rejects_entry_path_length_limit() { - let temp = tempdir().expect("tempdir should be created"); - let zip_path = temp.path().join("long-path.zip"); - let extract_path = temp.path().join("extract"); - - build_zip_file_with_entries(&zip_path, &[("nested/hello.txt", b"hello")]) - .await - .expect("zip fixture should be created"); - - let err = extract_zip_with_limits( - &zip_path, - &extract_path, - ArchiveLimits { - max_path_length: 5, - ..ArchiveLimits::default() - }, - ) - .await - .expect_err("zip path length limit should fail"); - - assert!(matches!( - err, - ZipError::EntryPathTooLong { path, limit: 5, .. } if path == "nested/hello.txt" - )); - } - - #[tokio::test] - async fn test_decompress_file_round_trip() { - let temp = tempdir().expect("tempdir should be created"); - let input_path = temp.path().join("payload.txt.gz"); - let output_path = temp.path().join("payload.txt"); - let compressed = Compressor::new(CompressionFormat::Gzip) - .compress(b"payload") - .await - .expect("gzip compress should succeed"); - fs::write(&input_path, compressed) - .await - .expect("compressed input file should be written"); - - Decompressor::auto_detect(&input_path) - .decompress_file(&input_path, &output_path) - .await - .expect("gzip file decompress should succeed"); - - assert_eq!( - fs::read(&output_path) - .await - .expect("decompressed output file should be readable"), - b"payload" - ); - } } From 9852e53b4ccaa9052ccfbfd29e1bef45ab009a82 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 18:28:13 +0800 Subject: [PATCH 37/51] test(lifecycle,scanner): drop 47 no-op #[serial] markers (backlog#1846 T1) (#6213) Second batch of the #[serial] sweep started in #6209. nextest is the repository's authoritative runner and executes every test in its own process, so serial_test's in-process mutex cannot serialize tests against each other -- docs/testing/README.md documents this, and the mechanism that actually serializes across the process boundary is a .config/nextest.toml [test-groups] entry with max-threads = 1. Unlike the first batch (e2e, process-isolated by construction), these are in-crate unit tests that could genuinely share process state under the `cargo test` fallback runner, where #[serial] IS still effective. Every marker was therefore reviewed individually and removed only where the test provably touches neither the process environment nor a process-global. Removed (47, pure deletions, no test bodies touched): crates/lifecycle/src/core.rs 35 crates/scanner/src/scanner.rs 12 The lifecycle removals are all validate_* / filter_rules_* / has_active_rules_* / noncurrent_versions_expiration_limit_* tests that build a local BucketLifecycleConfiguration and call a &self method walking only that value. The scanner removals are pure duration arithmetic (randomized_cycle_delay_for, initial_scanner_delay_for with an explicit Some(secs), the bitrot-disabled early return of scanner_clean_idle_max_interval) and background_heal_info_for_scan_complete / _for_scan_result field comparisons over locally built values. Retained deliberately -- see the PR body for the full list and reasons: crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs 101 (all) crates/lifecycle/src/core.rs 44 crates/scanner/src/scanner.rs 53 bucket_lifecycle_ops.rs keeps every marker: its test module caches a process-wide `static STALE_MULTIPART_TEST_ENV: OnceLock<(Vec, Arc)>`, and its own reregister_env_local_disks helper documents in-tree that sibling #[serial] tests reset and reshape the shared local-disk registry for each other. That sharing is real, so the markers stay. No test was renamed, added, or deleted; no reserved migration-gate name substring is affected; no .config/nextest.toml entry references any of the 47 removed tests. --- crates/lifecycle/src/core.rs | 35 ----------------------------------- crates/scanner/src/scanner.rs | 12 ------------ 2 files changed, 47 deletions(-) diff --git a/crates/lifecycle/src/core.rs b/crates/lifecycle/src/core.rs index ff7d15505..0fe52b9df 100644 --- a/crates/lifecycle/src/core.rs +++ b/crates/lifecycle/src/core.rs @@ -1199,7 +1199,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_zero_expiration_days() { // S3 compatibility: Expiration.Days must be a positive integer (>= 1). AWS and // the ceph s3-tests `test_lifecycle_expiration_days0` case reject Days == 0 with @@ -1233,7 +1232,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_negative_expiration_days() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1263,7 +1261,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_accepts_positive_expiration_days() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1290,7 +1287,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_accepts_one_day_boundary_values() { // Pin the exact >= 1 boundary: a value of 1 is the smallest legal positive // integer and must be accepted for every day-count field tightened for S3 @@ -1325,7 +1321,6 @@ mod tests { } #[tokio::test] - #[serial] async fn has_active_rules_accepts_zero_day_expiration() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1350,7 +1345,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_zero_noncurrent_expiration_days() { // S3 compatibility: NoncurrentVersionExpiration.NoncurrentDays must be a positive // integer (>= 1); AWS rejects 0 with InvalidArgument. @@ -1382,7 +1376,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_negative_noncurrent_expiration_days() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1412,7 +1405,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_accepts_abort_incomplete_multipart_upload_only_rule() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1438,7 +1430,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_zero_abort_incomplete_multipart_upload_days() { // S3 compatibility: AbortIncompleteMultipartUpload.DaysAfterInitiation must be a // positive integer (>= 1); AWS rejects 0 with InvalidArgument. @@ -1469,7 +1460,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_missing_abort_incomplete_multipart_upload_days() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1495,7 +1485,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_negative_abort_incomplete_multipart_upload_days() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1556,7 +1545,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_non_midnight_expiration_date() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1638,7 +1626,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_accepts_multiple_rules_without_ids() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1682,7 +1669,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_rule_id_too_long() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1709,7 +1695,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_duplicate_rule_ids() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1752,7 +1737,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_transition_without_storage_class() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1780,7 +1764,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_transition_without_date_or_days() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -1808,7 +1791,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_noncurrent_transition_without_days() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -2365,7 +2347,6 @@ mod tests { } #[tokio::test] - #[serial] async fn noncurrent_versions_expiration_limit_returns_configured_limits() { let lc = Arc::new(BucketLifecycleConfiguration { expiry_updated_at: None, @@ -2456,7 +2437,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_invalid_status_case_sensitive() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -2483,7 +2463,6 @@ mod tests { } #[tokio::test] - #[serial] async fn filter_rules_respects_filter_prefix() { let filter = LifecycleRuleFilter { prefix: Some("prefix".to_string()), @@ -2528,7 +2507,6 @@ mod tests { } #[tokio::test] - #[serial] async fn filter_rules_respects_filter_and_prefix() { let and = s3s::dto::LifecycleRuleAndOperator { prefix: Some("prefix".to_string()), @@ -2578,7 +2556,6 @@ mod tests { } #[tokio::test] - #[serial] async fn filter_rules_respects_filter_tag() { let filter = LifecycleRuleFilter { tag: Some(s3s::dto::Tag { @@ -2632,7 +2609,6 @@ mod tests { } #[tokio::test] - #[serial] async fn filter_rules_respects_filter_and_tags() { let filter = LifecycleRuleFilter { and: Some(s3s::dto::LifecycleRuleAndOperator { @@ -3086,7 +3062,6 @@ mod tests { // --- TASK-002 tests: Object Lock + ExpiredObjectDeleteMarker compatibility --- #[tokio::test] - #[serial] async fn validate_allows_expired_object_delete_marker_on_locked_bucket() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -3118,7 +3093,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_allows_expired_object_delete_marker_on_unlocked_bucket() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -3146,7 +3120,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_allows_non_delete_marker_expiration_on_locked_bucket() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -3179,7 +3152,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_del_marker_expiration_on_locked_bucket() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -3210,7 +3182,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_rejects_zero_day_del_marker_expiration_on_locked_bucket() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -3594,7 +3565,6 @@ mod tests { // --- TASK-007 tests: Legacy Prefix/Filter conflict --- #[tokio::test] - #[serial] async fn validate_rejects_prefix_and_filter_both_present() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -3623,7 +3593,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_allows_prefix_without_filter() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -3650,7 +3619,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_allows_filter_without_prefix() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -3680,7 +3648,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_allows_empty_prefix_with_filter() { // Empty prefix should be treated as "not set" let lc = BucketLifecycleConfiguration { @@ -3713,7 +3680,6 @@ mod tests { // --- TASK-004 tests: ExpiredObjectAllVersions --- #[tokio::test] - #[serial] async fn validate_rejects_expired_object_all_versions_on_locked_bucket() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, @@ -3745,7 +3711,6 @@ mod tests { } #[tokio::test] - #[serial] async fn validate_allows_expired_object_all_versions_on_unlocked_bucket() { let lc = BucketLifecycleConfiguration { expiry_updated_at: None, diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 11023eaac..fe743a1c6 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -4574,7 +4574,6 @@ mod tests { } #[test] - #[serial] fn test_randomized_cycle_delay_keeps_configured_start_delay() { // 120s with ±10% jitter should stay clearly above the historic 30s cap. let delay = randomized_cycle_delay_for(Duration::from_secs(120)); @@ -4593,7 +4592,6 @@ mod tests { } #[test] - #[serial] fn test_initial_scanner_delay_uses_configured_start_delay() { let delay = initial_scanner_delay_for(Some(120)); assert!(delay >= Duration::from_secs(108)); @@ -4613,14 +4611,12 @@ mod tests { } #[test] - #[serial] fn test_initial_scanner_delay_skips_for_cold_usage_cache_with_buckets() { let delay = initial_scanner_delay_for_startup(Some(120), true, true, false); assert_eq!(delay, Duration::ZERO); } #[test] - #[serial] fn test_initial_scanner_delay_keeps_configured_delay_for_warm_usage_cache_no_replication() { let delay = initial_scanner_delay_for_startup(Some(120), false, true, false); assert!(delay >= Duration::from_secs(108)); @@ -4628,14 +4624,12 @@ mod tests { } #[test] - #[serial] fn test_initial_scanner_delay_skips_for_cold_usage_cache_without_buckets() { let delay = initial_scanner_delay_for_startup(Some(120), true, false, false); assert_eq!(delay, Duration::ZERO); } #[test] - #[serial] fn test_initial_scanner_delay_skips_for_active_replication_warm_cache() { // Warm cache + active replication rules → skip startup delay so that FAILED-status objects // from a crash are healed on the first cycle, not after a 27-33 min sleep. @@ -4644,7 +4638,6 @@ mod tests { } #[test] - #[serial] fn test_initial_scanner_delay_keeps_delay_for_replication_without_buckets() { // Active replication but no buckets → no objects to scan, keep normal delay. let delay = initial_scanner_delay_for_startup(Some(120), false, false, true); @@ -7399,7 +7392,6 @@ mod tests { } #[test] - #[serial] fn clean_idle_cap_allows_policy_max_when_bitrot_is_disabled() { let config = ScannerRuntimeConfig { bitrot_cycle: None, @@ -7515,7 +7507,6 @@ mod tests { } #[test] - #[serial] fn test_randomized_cycle_delay_handles_small_start_delay() { // 0 is treated as minimum 1 second before jitter, with lower bound preserved. let delay = randomized_cycle_delay_for(Duration::from_secs(0)); @@ -8174,7 +8165,6 @@ mod tests { } #[test] - #[serial] fn test_background_heal_info_for_scan_complete_marks_deep_idle() { let started_at = Utc::now(); let info = BackgroundHealInfo { @@ -8192,7 +8182,6 @@ mod tests { } #[test] - #[serial] fn test_background_heal_info_for_scan_complete_leaves_normal_scan_unchanged() { let info = BackgroundHealInfo { bitrot_start_time: Some(Utc::now()), @@ -8204,7 +8193,6 @@ mod tests { } #[test] - #[serial] fn test_background_heal_info_for_failed_scan_preserves_deep_mode() { let info = BackgroundHealInfo { bitrot_start_time: Some(Utc::now()), From 8a3c66e655c2baabcd1a27171947761093f4a406 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 18:28:46 +0800 Subject: [PATCH 38/51] test(admin): replace a source-text guard with a behavior test (#6212) --- crates/test-utils/src/lib.rs | 10 ++- rustfs/src/admin/handlers/system.rs | 78 +++++++++++++++++++++ rustfs/src/admin/route_registration_test.rs | 17 ----- rustfs/src/admin/storage_api.rs | 4 ++ 4 files changed, 91 insertions(+), 18 deletions(-) diff --git a/crates/test-utils/src/lib.rs b/crates/test-utils/src/lib.rs index 46f215a9d..59b4f3e0c 100644 --- a/crates/test-utils/src/lib.rs +++ b/crates/test-utils/src/lib.rs @@ -92,6 +92,13 @@ pub struct TestECStoreEnv { /// `init_local_disks` + `ECStore::new` on `127.0.0.1:0` (random port keeps /// nextest's process-per-test parallelism safe). pub ecstore: Arc, + /// The single-pool, single-set topology the store was built from. + /// + /// The bootstrap does **not** publish it on the instance context (server + /// startup is what calls `set_endpoints`, and that write is once-only), so + /// a test that needs `get_global_endpoints` to resolve — admin server-info + /// and other topology readers — publishes this value itself. + pub endpoint_pools: EndpointServerPools, } impl TestECStoreEnv { @@ -234,7 +241,7 @@ impl TestECStoreEnvBuilder { // Port 0 keeps ECStore-backed integration binaries parallel-safe under // nextest: no fixed peer port is ever shared between test processes. let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().expect("parse test addr"); - let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) + let ecstore = ECStore::new(server_addr, endpoint_pools.clone(), CancellationToken::new()) .await .expect("build test ECStore"); @@ -254,6 +261,7 @@ impl TestECStoreEnvBuilder { temp_root, disk_paths, ecstore, + endpoint_pools, } } } diff --git a/rustfs/src/admin/handlers/system.rs b/rustfs/src/admin/handlers/system.rs index 0909568e5..9df481c11 100644 --- a/rustfs/src/admin/handlers/system.rs +++ b/rustfs/src/admin/handlers/system.rs @@ -1541,6 +1541,84 @@ mod tests { assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); } + /// `ServerInfoHandler` must answer an authorized admin request with the + /// per-pool erasure-set topology (rustfs/backlog#1839). That map is only + /// filled when the server-info query is issued with pools included, so a + /// handler that stopped asking for them would still return 200 with an + /// empty `pools` object instead of failing. + #[tokio::test] + #[serial_test::serial] + async fn server_info_response_carries_pool_topology() { + use crate::admin::runtime_sources::{AppContext, publish_test_app_context}; + use crate::admin::storage_api::runtime::bootstrap_ctx; + use http_body_util::BodyExt as _; + use rustfs_iam::store::{Store as _, object::IAM_CONFIG_PREFIX}; + use std::sync::Arc; + + const ROOT_ACCESS_KEY: &str = "SERVERINFOROOTACCESSKEY"; + const ROOT_SECRET_KEY: &str = "serverInfoRootSecret123"; + + let _ = rustfs_credentials::init_global_action_credentials( + Some(ROOT_ACCESS_KEY.to_string()), + Some(ROOT_SECRET_KEY.to_string()), + ); + + let env = rustfs_test_utils::TestECStoreEnv::builder() + .prefix("admin_server_info_pools") + .disk_count(1) + .init_bucket_metadata(false) + .build() + .await; + // Server startup owns this write in production; the test bootstrap + // stops short of it, and without a topology the server-info query + // returns before it ever looks at drives. + bootstrap_ctx().set_endpoints(env.endpoint_pools.clone()); + rustfs_iam::store::object::ObjectStore::new(Arc::clone(&env.ecstore)) + .save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX)) + .await + .expect("seed IAM format"); + let iam = rustfs_iam::build_iam_sys(Arc::clone(&env.ecstore)) + .await + .expect("build test IAM"); + publish_test_app_context(Arc::new(AppContext::with_default_interfaces( + Arc::clone(&env.ecstore), + iam, + Arc::new(rustfs_kms::KmsServiceManager::new()), + ))); + + let request = S3Request { + input: Body::empty(), + method: Method::GET, + uri: Uri::from_static("/rustfs/admin/v3/info"), + headers: HeaderMap::new(), + extensions: Extensions::new(), + credentials: Some(s3s::auth::Credentials { + access_key: ROOT_ACCESS_KEY.to_string(), + secret_key: s3s::auth::SecretKey::from(ROOT_SECRET_KEY.to_string()), + }), + region: None, + service: None, + trailing_headers: None, + }; + + let (status, body) = super::ServerInfoHandler {} + .call(request, Params::new()) + .await + .expect("root admin credentials must be served server info") + .output; + assert_eq!(status, hyper::StatusCode::OK); + + let bytes = body.collect().await.expect("server info body should read").to_bytes(); + let payload: serde_json::Value = serde_json::from_slice(&bytes).expect("server info must be json"); + let pools = payload["info"]["pools"] + .as_object() + .expect("server info must carry a pools object"); + assert!( + pools.contains_key("0"), + "server info must report the erasure-set topology of pool 0, got {pools:?}" + ); + } + /// Authorization denial for this exact action is pinned to AccessDenied by /// `crate::admin::auth::tests::non_admin_credential_is_denied`. #[test] diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index e81f13d17..e48e09c94 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -1412,23 +1412,6 @@ fn test_health_routes_not_registered_when_disabled_by_env() { }); } -#[test] -fn test_phase5_admin_info_contract() { - let system_src = include_str!("handlers/system.rs"); - - let server_info_impl_marker = "impl Operation for ServerInfoHandler"; - let server_info_impl_start = system_src - .find(server_info_impl_marker) - .expect("Expected impl Operation for ServerInfoHandler in handlers/system.rs"); - let server_info_impl_block = &system_src[server_info_impl_start..]; - - assert!( - server_info_impl_block.contains("default_admin_usecase()") - && server_info_impl_block.contains("execute_query_server_info(QueryServerInfoRequest { include_pools: true })"), - "admin server info path must be served through admin runtime-source DefaultAdminUsecase::execute_query_server_info" - ); -} - fn extract_block_between_markers<'a>(src: &'a str, start_marker: &str, end_marker: &str) -> &'a str { let start = src .find(start_marker) diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index ec89f5e4e..dbf3a81df 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -948,6 +948,10 @@ pub(crate) mod runtime { #[cfg(test)] pub(crate) use super::{Endpoint, Endpoints, PoolEndpoints}; + /// Test-only: the process instance context, so a handler test can publish + /// the endpoint topology that server startup normally installs. + #[cfg(test)] + pub(crate) use crate::storage::storage_api::ecstore_runtime::bootstrap_ctx; } pub(crate) mod s3 { From 127b662f3f00f6c985a3e1203faecf90fc7fcf1a Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 19:53:06 +0800 Subject: [PATCH 39/51] feat(app): add opt-in small GET body once path (#6216) Use the merged s3s single-chunk StreamingBlob support for exact-length materialized GET bodies when RUSTFS_GET_SMALL_BODY_ONCE_ENABLE is enabled. Keep the default path unchanged and fall back to the guarded MemoryTrackedBytesStream on length mismatch. Co-authored-by: heihutu --- Cargo.lock | 5 +- Cargo.toml | 2 +- rustfs/src/app/object_usecase.rs | 84 +++++++++++++++++++++++++++++++- 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82f66641e..db75a7992 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10678,7 +10678,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3s" version = "0.14.1" -source = "git+https://github.com/rustfs/s3s.git?rev=d7028511a53f69d41ed3c69f36899f9b1aede647#d7028511a53f69d41ed3c69f36899f9b1aede647" +source = "git+https://github.com/rustfs/s3s.git?rev=ff8106f46098b60fce8ee88518f34974fbfbec4b#ff8106f46098b60fce8ee88518f34974fbfbec4b" dependencies = [ "arc-swap", "arrayvec", @@ -10706,6 +10706,7 @@ dependencies = [ "numeric_cast", "pin-project-lite", "quick-xml", + "regex", "serde", "serde_json", "serde_urlencoded", @@ -11809,7 +11810,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", diff --git a/Cargo.toml b/Cargo.toml index ae9b497e0..f2080a15f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -290,7 +290,7 @@ rustify = { version = "0.7", default-features = false } rustix = { version = "1.1.4" } rust-embed = { version = "8.12.0" } rustc-hash = { version = "2.1.3" } -s3s = { git = "https://github.com/rustfs/s3s.git", rev = "d7028511a53f69d41ed3c69f36899f9b1aede647" } +s3s = { git = "https://github.com/rustfs/s3s.git", rev = "ff8106f46098b60fce8ee88518f34974fbfbec4b" } serial_test = "4.0.1" shadow-rs = { default-features = false, version = "2.0.0" } siphasher = "1.0.3" diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 1bb337c69..bcce2b705 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -782,6 +782,7 @@ const MID_BODY_READER_STREAM_BUFFER_THRESHOLD_BYTES: i64 = MI_B as i64; const ENV_RUSTFS_GET_SEEK_BUFFER_ENABLE: &str = "RUSTFS_GET_SEEK_BUFFER_ENABLE"; const ENV_RUSTFS_GET_READER_STREAM_BUFFER_SIZE: &str = "RUSTFS_GET_READER_STREAM_BUFFER_SIZE"; const ENV_RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE: &str = "RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE"; +const ENV_RUSTFS_GET_SMALL_BODY_ONCE_ENABLE: &str = "RUSTFS_GET_SMALL_BODY_ONCE_ENABLE"; const GET_READER_STREAM_BUFFER_SOURCE_SELECTED: &str = "selected"; const GET_READER_STREAM_BUFFER_SOURCE_ENV_OVERRIDE: &str = "env_override"; const GET_READER_STREAM_POLL_PENDING: &str = "pending"; @@ -813,6 +814,18 @@ fn is_get_output_handoff_attribution_enabled() -> bool { *ENABLED.get_or_init(|| rustfs_utils::get_env_bool(ENV_RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE, false)) } +fn is_get_small_body_once_enabled() -> bool { + #[cfg(test)] + { + rustfs_utils::get_env_bool(ENV_RUSTFS_GET_SMALL_BODY_ONCE_ENABLE, false) + } + #[cfg(not(test))] + { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| rustfs_utils::get_env_bool(ENV_RUSTFS_GET_SMALL_BODY_ONCE_ENABLE, false)) + } +} + fn is_get_seek_buffer_enabled() -> bool { static ENABLED: OnceLock = OnceLock::new(); *ENABLED.get_or_init(|| rustfs_utils::get_env_bool(ENV_RUSTFS_GET_SEEK_BUFFER_ENABLE, false)) @@ -932,6 +945,30 @@ struct MemoryTrackedBytesStream { lifecycle: GetObjectBodyLifecycle, } +struct MemoryOnceBodyOwner { + bytes: Bytes, + _guard: Option, + // Body::Once has no poll hook, so this opt-in path only holds the request + // guard until the bytes are dropped; the result status remains unknown. + _lifecycle: GetObjectBodyLifecycle, +} + +impl MemoryOnceBodyOwner { + fn new(bytes: Bytes, guard: Option, lifecycle: GetObjectBodyLifecycle) -> Self { + Self { + bytes, + _guard: guard, + _lifecycle: lifecycle, + } + } +} + +impl AsRef<[u8]> for MemoryOnceBodyOwner { + fn as_ref(&self) -> &[u8] { + self.bytes.as_ref() + } +} + #[derive(Default)] struct GetObjectBodyLifecycle { request_guard: Option, @@ -4160,7 +4197,12 @@ impl DefaultObjectUsecase { let bytes_len = bytes.len(); let guard = rustfs_io_metrics::track_get_object_buffered_bytes(bytes_len); let remaining = usize::try_from(response_content_length.max(0)).unwrap_or(usize::MAX); - let blob = StreamingBlob::new(MemoryTrackedBytesStream::new(bytes, remaining, source, guard, lifecycle)); + let blob = if is_get_small_body_once_enabled() && bytes_len == remaining { + let owner = MemoryOnceBodyOwner::new(bytes, guard, lifecycle); + StreamingBlob::from_bytes(Bytes::from_owner(owner)) + } else { + StreamingBlob::new(MemoryTrackedBytesStream::new(bytes, remaining, source, guard, lifecycle)) + }; if let Some(handoff_start) = handoff_start { rustfs_io_metrics::record_get_object_response_handoff( "single_chunk", @@ -12955,6 +12997,46 @@ mod tests { assert_eq!(blob.remaining_length().exact(), Some(5)); } + #[test] + #[serial_test::serial] + fn memory_blob_once_fast_path_holds_guard_until_bytes_drop() { + temp_env::with_var(ENV_RUSTFS_GET_SMALL_BODY_ONCE_ENABLE, Some("true"), || { + let initial = GetObjectGuard::concurrent_count(); + let guard = GetObjectGuard::new(); + assert_eq!(GetObjectGuard::concurrent_count(), initial + 1); + + let blob = DefaultObjectUsecase::build_memory_bytes_blob( + Bytes::from_static(b"hello"), + 5, + GET_MEMORY_BODY_SOURCE_BUFFERED_BODY, + GetObjectBodyLifecycle::tracked(guard), + ); + let mut body = s3s::Body::from(blob); + let bytes = body.take_bytes().expect("opt-in exact memory body should stay on Body::Once"); + + assert_eq!(bytes, Bytes::from_static(b"hello")); + assert_eq!(GetObjectGuard::concurrent_count(), initial + 1); + drop(bytes); + assert_eq!(GetObjectGuard::concurrent_count(), initial); + }); + } + + #[test] + #[serial_test::serial] + fn memory_blob_once_fast_path_rejects_length_mismatch() { + temp_env::with_var(ENV_RUSTFS_GET_SMALL_BODY_ONCE_ENABLE, Some("true"), || { + let blob = DefaultObjectUsecase::build_memory_bytes_blob( + Bytes::from_static(b"test"), + 5, + GET_MEMORY_BODY_SOURCE_BUFFERED_BODY, + GetObjectBodyLifecycle::disabled(), + ); + let mut body = s3s::Body::from(blob); + + assert!(body.take_bytes().is_none(), "mismatched memory body must keep the guarded stream path"); + }); + } + #[tokio::test] async fn get_object_streaming_reader_times_out_when_body_stalls() { let reader = GetObjectStreamingReader::new( From 27d23b61351a81b1aec1125d7afb49f1fc1f1ee1 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 21:21:34 +0800 Subject: [PATCH 40/51] test(io-metrics): assert record helper emissions; fix census heuristic (#6217) * test(io-metrics): assert lib.rs record helper emissions The 29 assertion-less record_* smoke tests in io-metrics/src/lib.rs called their helpers and checked nothing; because METRICS_ENABLED defaults to false they did not even reach the emission bodies. Replace them with six DebuggingRecorder tests that enable the gate, pin every metric name the helpers own, and pin the derived values, branch selection and label mapping (rustfs/backlog#1836). * test(tooling): anchor assertless-census delegation tokens to name segments --- crates/io-metrics/src/lib.rs | 724 ++++++++++++++++++++++++------- scripts/find_assertless_tests.py | 19 +- 2 files changed, 578 insertions(+), 165 deletions(-) diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index 86b38ed5a..785cea246 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -2633,36 +2633,164 @@ mod tests { use std::sync::{Arc, Barrier, Mutex}; // Serialize tests that mutate the process-global PUT_STAGE_METRICS_ENABLED flag. - static METRICS_FLAG_LOCK: Mutex<()> = Mutex::new(()); + pub(crate) static METRICS_FLAG_LOCK: Mutex<()> = Mutex::new(()); - #[test] - fn test_record_zero_copy_read() { - record_zero_copy_read(1024, 10.5); - record_memory_copy_saved(1024); - record_zero_copy_fallback("test"); + /// One row of a `DebuggingRecorder` snapshot. + pub(crate) type MetricRow = ( + metrics_util::CompositeKey, + Option, + Option, + DebugValue, + ); + + /// Every metric name present in a snapshot. + pub(crate) fn emitted_names(rows: &[MetricRow]) -> std::collections::HashSet<&str> { + rows.iter().map(|(composite, _, _, _)| composite.key().name()).collect() } - #[test] - fn test_record_bytes_pool_metrics() { - record_bytes_pool_acquire("small", 4096, true); - record_bytes_pool_return("small"); - record_bytes_pool_allocated("small", 4096); - record_bytes_pool_hit_rate("small", 0.85); + /// Counter value for `name`, summed over every label set it was emitted with. + /// `None` means the counter never reached the recorder. + pub(crate) fn counter_total(rows: &[MetricRow], name: &str) -> Option { + let mut total = None; + for (composite, _, _, value) in rows { + if composite.kind() == MetricKind::Counter && composite.key().name() == name { + match value { + DebugValue::Counter(count) => *total.get_or_insert(0) += count, + other => panic!("{name} is registered as a counter but holds {other:?}"), + } + } + } + total } - #[test] - fn test_record_bytespool_acquisition_and_return() { - // Acquisition outcomes - record_bytespool_acquisition("small", "hit"); - record_bytespool_acquisition("medium", "miss"); - record_bytespool_acquisition("large", "hit"); - record_bytespool_acquisition("xlarge", "miss"); + /// Gauge value for `name`. Panics when several label sets carry it, so a caller + /// cannot silently assert on an arbitrary one. + pub(crate) fn gauge_value(rows: &[MetricRow], name: &str) -> Option { + let mut matching = rows + .iter() + .filter(|(composite, _, _, _)| composite.kind() == MetricKind::Gauge && composite.key().name() == name); + let value = matching.next().map(|(_, _, _, value)| match value { + DebugValue::Gauge(value) => value.0, + other => panic!("{name} is registered as a gauge but holds {other:?}"), + }); + assert!( + matching.next().is_none(), + "{name} carries several label sets; assert on the labelled rows instead" + ); + value + } - // Return outcomes - record_bytespool_return("small", "recycled"); - record_bytespool_return("medium", "dropped"); - record_bytespool_return("large", "recycled"); - record_bytespool_return("xlarge", "dropped"); + /// Histogram samples for `name` across every label set, sorted so the assertion + /// does not depend on registry iteration order. + pub(crate) fn histogram_samples(rows: &[MetricRow], name: &str) -> Vec { + let mut samples: Vec = rows + .iter() + .filter(|(composite, _, _, _)| composite.kind() == MetricKind::Histogram && composite.key().name() == name) + .flat_map(|(_, _, _, value)| match value { + DebugValue::Histogram(samples) => samples.iter().map(|sample| sample.0), + other => panic!("{name} is registered as a histogram but holds {other:?}"), + }) + .collect(); + samples.sort_by(f64::total_cmp); + samples + } + + /// Replaces four smoke tests that called the zero-copy and bytes-pool recorders + /// and asserted nothing (rustfs/backlog#1836). The same calls now run against a + /// local recorder: every metric name these helpers own must be emitted, the + /// `from_pool` branch must pick the hit/miss counter, and the derived values + /// (byte totals, the hit rate's percent conversion) must match the inputs. + #[test] + fn zero_copy_and_bytes_pool_helpers_emit_their_metrics() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + metrics::with_local_recorder(&recorder, || { + set_metrics_enabled(true); + record_zero_copy_read(1024, 10.5); + record_memory_copy_saved(1024); + record_zero_copy_fallback("test"); + record_zero_copy_write(2048, 20.5); + record_zero_copy_write_fallback("test"); + record_bytes_saved(4096); + record_bytes_pool_acquire("small", 4096, true); + record_bytes_pool_acquire("small", 4096, false); + record_bytes_pool_return("small"); + record_bytes_pool_allocated("small", 4096); + record_bytes_pool_hit_rate("small", 0.85); + record_bytespool_acquisition("small", "hit"); + record_bytespool_acquisition("medium", "miss"); + record_bytespool_return("small", "recycled"); + record_bytespool_return("medium", "dropped"); + set_metrics_enabled(false); + }); + + let rows = snapshotter.snapshot().into_vec(); + let names = emitted_names(&rows); + for expected in [ + "rustfs_zero_copy_reads_total", + "rustfs_zero_copy_read_size_bytes", + "rustfs_zero_copy_read_duration_ms", + mmap_copy::READS_TOTAL, + mmap_copy::READ_SIZE_BYTES, + mmap_copy::READ_DURATION_MS, + mmap_copy::BYTES_COPIED_TOTAL, + mmap_copy::FALLBACK_TOTAL, + "rustfs_zero_copy_memory_saved_bytes_total", + "rustfs_zero_copy_fallback_total", + "rustfs_zero_copy_write_total", + "rustfs_zero_copy_write_size_bytes", + "rustfs_zero_copy_write_duration_ms", + buffered_write::WRITES_TOTAL, + buffered_write::WRITE_SIZE_BYTES, + buffered_write::WRITE_DURATION_MS, + buffered_write::BYTES_COPIED_TOTAL, + buffered_write::FALLBACK_TOTAL, + "rustfs_zero_copy_write_fallback_total", + "rustfs_zero_copy_bytes_saved_total", + "rustfs_bytes_pool_acquisitions_total", + "rustfs_bytes_pool_size_bytes", + "rustfs_bytes_pool_hits_total", + "rustfs_bytes_pool_misses_total", + "rustfs_bytes_pool_returns_total", + "rustfs_bytes_pool_allocated_bytes", + "rustfs_bytes_pool_hit_rate", + "rustfs_io_bytespool_acquisition_total", + "rustfs_io_bytespool_return_total", + ] { + assert!(names.contains(expected), "{expected} must be emitted by its record helper"); + } + + assert_eq!( + counter_total(&rows, mmap_copy::BYTES_COPIED_TOTAL), + Some(1024), + "the read helper must count the read size, not the call" + ); + assert_eq!( + counter_total(&rows, buffered_write::BYTES_COPIED_TOTAL), + Some(2048), + "the write helper must count the write size, not the call" + ); + assert_eq!(counter_total(&rows, "rustfs_zero_copy_memory_saved_bytes_total"), Some(1024)); + assert_eq!(counter_total(&rows, "rustfs_zero_copy_bytes_saved_total"), Some(4096)); + assert_eq!(histogram_samples(&rows, "rustfs_zero_copy_read_duration_ms"), vec![10.5]); + assert_eq!(histogram_samples(&rows, "rustfs_zero_copy_write_duration_ms"), vec![20.5]); + assert_eq!( + counter_total(&rows, "rustfs_bytes_pool_hits_total"), + Some(1), + "only the from_pool acquisition counts as a hit" + ); + assert_eq!( + counter_total(&rows, "rustfs_bytes_pool_misses_total"), + Some(1), + "only the non-pool acquisition counts as a miss" + ); + assert_eq!( + gauge_value(&rows, "rustfs_bytes_pool_hit_rate"), + Some(85.0), + "the hit rate is exported as a percentage" + ); } #[test] @@ -2685,20 +2813,6 @@ mod tests { set_get_stage_metrics_enabled(false); } - #[test] - fn test_record_zero_copy_write() { - record_zero_copy_write(1024, 10.5); - record_zero_copy_write_fallback("test"); - record_bytes_saved(1024); - } - - // S3 Operation Metrics Tests - #[test] - fn test_record_get_object() { - record_get_object(100.0, 1024 * 1024); - record_get_object(50.0, 2048); - } - #[test] fn test_record_get_object_stage_metrics() { record_get_object_stage_duration("s3_handler", "request_context", 0.001); @@ -2847,17 +2961,95 @@ mod tests { assert!(0.0003_f64.is_sign_positive()); } + /// Replaces five smoke tests (`test_record_get_object`, `test_record_put_object`, + /// `test_record_put_object_request_metrics`, `test_record_list_objects`, + /// `test_record_delete_object`) that called the S3 operation recorders and + /// asserted nothing (rustfs/backlog#1836). Besides pinning the metric names, + /// this pins the conditional emissions each helper owns: the zero-copy alias + /// counters fire only for an eligible PUT, the truncated/version counters only + /// for the truncated listing and the versioned delete. #[test] - fn test_record_put_object() { - record_put_object(200.0, 1024 * 1024, true); - record_put_object(100.0, 512, false); - } + fn s3_operation_helpers_emit_their_metrics() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); - #[test] - fn test_record_put_object_request_metrics() { - record_put_object_request_start(3); - record_put_object_request_result("ok", 0.25); - record_put_object_request_result("error", 0.5); + metrics::with_local_recorder(&recorder, || { + set_metrics_enabled(true); + set_get_stage_metrics_enabled(true); + record_get_object(100.0, 1024 * 1024); + record_get_object(50.0, 2048); + record_put_object(200.0, 1024 * 1024, true); + record_put_object(100.0, 512, false); + record_put_object_request_start(3); + record_put_object_request_result("ok", 0.25); + record_put_object_request_result("error", 0.5); + record_list_objects(50.0, 100, false); + record_list_objects(75.0, 1000, true); + record_delete_object(25.0, false); + record_delete_object(30.0, true); + set_get_stage_metrics_enabled(false); + set_metrics_enabled(false); + }); + + let rows = snapshotter.snapshot().into_vec(); + let names = emitted_names(&rows); + for expected in [ + "rustfs_s3_get_object_total", + "rustfs_s3_get_object_duration_ms", + "rustfs_s3_get_object_size_bytes", + "rustfs_s3_put_object_total", + "rustfs_s3_put_object_duration_ms", + "rustfs_s3_put_object_size_bytes", + "rustfs_s3_put_object_zero_copy_enabled_total", + "rustfs_s3_put_object_zero_copy_eligible_total", + "rustfs_io_put_object_requests_total", + "rustfs_io_put_object_concurrent_requests", + "rustfs_io_put_object_request_results_total", + "rustfs_io_put_object_request_duration_seconds", + "rustfs_s3_list_objects_total", + "rustfs_s3_list_objects_duration_ms", + "rustfs_s3_list_objects_count", + "rustfs_s3_list_objects_truncated_total", + "rustfs_s3_delete_object_total", + "rustfs_s3_delete_object_duration_ms", + "rustfs_s3_delete_object_version_total", + ] { + assert!(names.contains(expected), "{expected} must be emitted by its record helper"); + } + + assert_eq!(counter_total(&rows, "rustfs_s3_get_object_total"), Some(2)); + assert_eq!(counter_total(&rows, "rustfs_s3_put_object_total"), Some(2)); + assert_eq!( + counter_total(&rows, "rustfs_s3_put_object_zero_copy_eligible_total"), + Some(1), + "only the zero-copy eligible PUT increments the eligibility counter" + ); + assert_eq!( + counter_total(&rows, "rustfs_s3_put_object_zero_copy_enabled_total"), + Some(1), + "the historical alias must stay in step with the eligibility counter" + ); + assert_eq!( + counter_total(&rows, "rustfs_s3_list_objects_truncated_total"), + Some(1), + "only the truncated listing increments the truncation counter" + ); + assert_eq!( + counter_total(&rows, "rustfs_s3_delete_object_version_total"), + Some(1), + "only the versioned delete increments the version counter" + ); + assert_eq!( + histogram_samples(&rows, "rustfs_s3_list_objects_count"), + vec![100.0, 1000.0], + "the object count, not the duration, belongs in the count histogram" + ); + assert_eq!( + gauge_value(&rows, "rustfs_io_put_object_concurrent_requests"), + Some(3.0), + "the concurrency gauge must carry the reported in-flight request count" + ); } #[test] @@ -3175,74 +3367,171 @@ mod tests { set_metrics_enabled(false); } + /// Replaces six smoke tests (`test_record_io_strategy`, `test_record_permit_wait`, + /// `test_record_io_load_level`, `test_record_cache_size`, `test_record_bandwidth`, + /// `test_record_data_transfer`) that called the scheduler, cache and bandwidth + /// recorders and asserted nothing (rustfs/backlog#1836). The derived bandwidth + /// value and the `all` tier fan-out are now pinned, not just the names. #[test] - fn test_record_list_objects() { - record_list_objects(50.0, 100, false); - record_list_objects(75.0, 1000, true); + fn io_scheduler_and_bandwidth_helpers_emit_their_metrics() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + metrics::with_local_recorder(&recorder, || { + set_metrics_enabled(true); + record_io_strategy("nvme", "sequential", 256 * 1024, 5); + record_io_strategy("ssd", "random", 64 * 1024, 10); + record_permit_wait(5.0); + record_permit_wait(10.5); + record_io_load_level("low", 2); + record_io_load_level("high", 15); + record_cache_size("l1", 50 * 1024 * 1024, 1000); + record_bandwidth(100 * 1024 * 1024, "high"); + record_data_transfer(1024 * 1024, 100.0); + record_data_transfer(2048, 50.0); + set_metrics_enabled(false); + }); + + let rows = snapshotter.snapshot().into_vec(); + let names = emitted_names(&rows); + for expected in [ + "rustfs_io_strategy_total", + "rustfs_io_buffer_size_bytes", + "rustfs_io_concurrent_requests", + "rustfs_io_permit_wait_duration_ms", + "rustfs_io_load_level", + "rustfs_cache_size_bytes", + "rustfs_cache_entries", + "rustfs_bandwidth_current_bps", + "rustfs_bandwidth_observed_bps", + "rustfs_io_transfer_bytes_total", + "rustfs_io_transfer_duration_ms", + "rustfs_io_transfer_bandwidth_bps", + ] { + assert!(names.contains(expected), "{expected} must be emitted by its record helper"); + } + + assert_eq!(counter_total(&rows, "rustfs_io_strategy_total"), Some(2)); + assert_eq!(counter_total(&rows, "rustfs_io_load_level"), Some(2)); + assert_eq!( + gauge_value(&rows, "rustfs_io_concurrent_requests"), + Some(15.0), + "the shared concurrency gauge must hold the last reported value" + ); + assert_eq!(histogram_samples(&rows, "rustfs_io_permit_wait_duration_ms"), vec![5.0, 10.5]); + assert_eq!(gauge_value(&rows, "rustfs_cache_entries"), Some(1000.0)); + assert_eq!(gauge_value(&rows, "rustfs_cache_size_bytes"), Some((50 * 1024 * 1024) as f64)); + assert_eq!( + counter_total(&rows, "rustfs_io_transfer_bytes_total"), + Some(1024 * 1024 + 2048), + "transferred bytes must accumulate across calls" + ); + assert_eq!( + histogram_samples(&rows, "rustfs_io_transfer_bandwidth_bps"), + vec![40960.0, 10_485_760.0], + "bandwidth must be derived as bytes * 1000 / duration_ms" + ); + + let mut bandwidth_by_tier: Vec<(&str, f64)> = rows + .iter() + .filter(|(composite, _, _, _)| composite.key().name() == "rustfs_bandwidth_current_bps") + .map(|(composite, _, _, value)| { + let tier = composite + .key() + .labels() + .find(|label| label.key() == "tier") + .map(|label| label.value()) + .expect("bandwidth gauges carry a tier label"); + match value { + DebugValue::Gauge(value) => (tier, value.0), + other => panic!("rustfs_bandwidth_current_bps holds {other:?}"), + } + }) + .collect(); + bandwidth_by_tier.sort_by(|left, right| left.0.cmp(right.0)); + assert_eq!( + bandwidth_by_tier, + vec![("all", 104_857_600.0), ("high", 104_857_600.0)], + "record_bandwidth must publish both the aggregate `all` series and the caller tier" + ); } + /// Replaces five smoke tests (`test_record_memory_usage`, + /// `test_record_process_memory_split`, `test_record_cgroup_memory_split`, + /// `test_record_cpu_usage`, `test_record_disk_io`) that called the system + /// resource recorders and asserted nothing (rustfs/backlog#1836). The gauge + /// values pin the argument order and the usage-percent derivation, which name + /// checks alone cannot catch. #[test] - fn test_record_delete_object() { - record_delete_object(25.0, false); - record_delete_object(30.0, true); + fn system_resource_helpers_emit_their_metrics() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + metrics::with_local_recorder(&recorder, || { + set_metrics_enabled(true); + record_memory_usage(1024 * 1024 * 1024, 4 * 1024 * 1024 * 1024); + record_process_memory_split(1024, 2048); + record_cgroup_memory_split(Some(1), Some(2), Some(3), Some(4), Some(5), Some(6)); + record_cpu_usage(25.5); + record_disk_io(1024 * 1024, 2048, 100, 50); + record_disk_io(2048, 4096, 200, 100); + set_metrics_enabled(false); + }); + + let rows = snapshotter.snapshot().into_vec(); + assert_eq!(gauge_value(&rows, "rustfs_memory_used_bytes"), Some((1024 * 1024 * 1024) as f64)); + assert_eq!(gauge_value(&rows, "rustfs_memory_total_bytes"), Some((4u64 * 1024 * 1024 * 1024) as f64)); + assert_eq!( + gauge_value(&rows, "rustfs_memory_usage_percent"), + Some(25.0), + "usage percent must be used/total * 100" + ); + assert_eq!(gauge_value(&rows, "rustfs_memory_process_resident_bytes"), Some(1024.0)); + assert_eq!( + gauge_value(&rows, "rustfs_memory_process_virtual_bytes"), + Some(2048.0), + "resident and virtual bytes must not be swapped" + ); + for (name, expected) in [ + ("rustfs_memory_cgroup_current_bytes", 1.0), + ("rustfs_memory_cgroup_limit_bytes", 2.0), + ("rustfs_memory_cgroup_anon_bytes", 3.0), + ("rustfs_memory_cgroup_file_bytes", 4.0), + ("rustfs_memory_cgroup_active_file_bytes", 5.0), + ("rustfs_memory_cgroup_inactive_file_bytes", 6.0), + ] { + assert_eq!(gauge_value(&rows, name), Some(expected), "{name} must receive its own argument"); + } + assert_eq!(gauge_value(&rows, "rustfs_cpu_usage_percent"), Some(25.5)); + assert_eq!(counter_total(&rows, "rustfs_disk_read_bytes_total"), Some(1024 * 1024 + 2048)); + assert_eq!(counter_total(&rows, "rustfs_disk_write_bytes_total"), Some(2048 + 4096)); + assert_eq!(counter_total(&rows, "rustfs_disk_read_ops_total"), Some(300)); + assert_eq!( + counter_total(&rows, "rustfs_disk_write_ops_total"), + Some(150), + "byte and op counters must not be crossed" + ); } - // I/O Scheduler Metrics Tests + /// Boundary companion of the `Some(..)` case above: an absent cgroup field must + /// emit no gauge at all. Publishing `0` for a field the kernel does not expose + /// would read as a real measurement (rustfs/backlog#1836). #[test] - fn test_record_io_strategy() { - record_io_strategy("nvme", "sequential", 256 * 1024, 5); - record_io_strategy("ssd", "random", 64 * 1024, 10); - } + fn cgroup_memory_split_skips_absent_fields() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); - #[test] - fn test_record_permit_wait() { - record_permit_wait(5.0); - record_permit_wait(10.5); - } + metrics::with_local_recorder(&recorder, || { + set_metrics_enabled(true); + record_cgroup_memory_split(None, None, None, None, None, None); + set_metrics_enabled(false); + }); - #[test] - fn test_record_io_load_level() { - record_io_load_level("low", 2); - record_io_load_level("medium", 5); - record_io_load_level("high", 15); - } - - #[test] - fn test_record_cache_size() { - record_cache_size("l1", 50 * 1024 * 1024, 1000); - record_cache_size("l2", 200 * 1024 * 1024, 5000); - } - - // Bandwidth Metrics Tests - #[test] - fn test_record_bandwidth() { - record_bandwidth(100 * 1024 * 1024, "high"); - record_bandwidth(50 * 1024 * 1024, "medium"); - } - - #[test] - fn test_record_data_transfer() { - record_data_transfer(1024 * 1024, 100.0); - record_data_transfer(2048, 50.0); - } - - // System Resource Metrics Tests - #[test] - fn test_record_memory_usage() { - record_memory_usage(1024 * 1024 * 1024, 4 * 1024 * 1024 * 1024); - record_memory_usage(2 * 1024 * 1024 * 1024, 8 * 1024 * 1024 * 1024); - } - - #[test] - fn test_record_process_memory_split() { - record_process_memory_split(1024, 2048); - record_process_memory_split(4096, 8192); - } - - #[test] - fn test_record_cgroup_memory_split() { - record_cgroup_memory_split(Some(1), Some(2), Some(3), Some(4), Some(5), Some(6)); - record_cgroup_memory_split(None, None, None, None, None, None); + let rows = snapshotter.snapshot().into_vec(); + assert!(rows.is_empty(), "absent cgroup fields must emit nothing, got {:?}", emitted_names(&rows)); } #[test] @@ -3460,36 +3749,75 @@ mod tests { assert_eq!(current_get_object_buffered_bytes(), 0); } + /// Replaces three smoke tests (`test_record_error`, `test_record_timeout`, + /// `test_record_retry`) that called the failure recorders and asserted nothing + /// (rustfs/backlog#1836). The histogram samples pin that the timeout duration + /// and the retry attempt number reach their histogram rather than being folded + /// into the counters. #[test] - fn test_record_cpu_usage() { - record_cpu_usage(25.5); - record_cpu_usage(50.0); - record_cpu_usage(75.5); - } + fn failure_helpers_emit_their_metrics() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); - #[test] - fn test_record_disk_io() { - record_disk_io(1024 * 1024, 2048, 100, 50); - record_disk_io(2048, 4096, 200, 100); - } + metrics::with_local_recorder(&recorder, || { + set_metrics_enabled(true); + record_error("get_object", "timeout"); + record_error("put_object", "disk_error"); + record_timeout("get_object", 5000.0); + record_timeout("list_objects", 10000.0); + record_retry("get_object", 1); + record_retry("put_object", 2); + set_metrics_enabled(false); + }); - // Error and Timeout Metrics Tests - #[test] - fn test_record_error() { - record_error("get_object", "timeout"); - record_error("put_object", "disk_error"); - } + let rows = snapshotter.snapshot().into_vec(); + let names = emitted_names(&rows); + for expected in [ + "rustfs_errors_total", + "rustfs_timeouts_total", + "rustfs_timeouts_duration_ms", + "rustfs_retries_total", + "rustfs_retries_attempt", + ] { + assert!(names.contains(expected), "{expected} must be emitted by its record helper"); + } - #[test] - fn test_record_timeout() { - record_timeout("get_object", 5000.0); - record_timeout("list_objects", 10000.0); - } + assert_eq!( + counter_total(&rows, "rustfs_errors_total"), + Some(2), + "each error is counted once under its own operation/type labels" + ); + assert_eq!(counter_total(&rows, "rustfs_timeouts_total"), Some(2)); + assert_eq!(counter_total(&rows, "rustfs_retries_total"), Some(2)); + assert_eq!(histogram_samples(&rows, "rustfs_timeouts_duration_ms"), vec![5000.0, 10000.0]); + assert_eq!( + histogram_samples(&rows, "rustfs_retries_attempt"), + vec![1.0, 2.0], + "the attempt number belongs in the histogram, not the retry counter" + ); - #[test] - fn test_record_retry() { - record_retry("get_object", 1); - record_retry("put_object", 2); + let mut error_labels: Vec<(&str, &str)> = rows + .iter() + .filter(|(composite, _, _, _)| composite.key().name() == "rustfs_errors_total") + .map(|(composite, _, _, _)| { + let label = |key: &str| { + composite + .key() + .labels() + .find(|label| label.key() == key) + .map(|label| label.value()) + .expect("error counters carry operation and type labels") + }; + (label("operation"), label("type")) + }) + .collect(); + error_labels.sort(); + assert_eq!( + error_labels, + vec![("get_object", "timeout"), ("put_object", "disk_error")], + "operation and error type must not be swapped" + ); } } @@ -3633,42 +3961,116 @@ pub fn update_zero_copy_performance_metrics(copy_count: u32, throughput_mbps: f6 #[cfg(test)] mod zero_copy_tests { use super::*; + use crate::tests::{METRICS_FLAG_LOCK, counter_total, emitted_names, gauge_value, histogram_samples}; + use metrics_util::debugging::DebuggingRecorder; + /// Replaces six smoke tests (`test_record_zero_copy_buffer_operation`, + /// `test_record_memory_copy`, `test_record_shared_ref_operation`, + /// `test_record_bufreader_optimization`, `test_record_direct_io_operation`, + /// `test_update_zero_copy_performance_metrics`) whose own comment admitted they + /// only checked that the helpers compile and run (rustfs/backlog#1836). The same + /// calls now run against a local recorder, and the assertions pin the counter + /// split (operations vs bytes, copies vs copied bytes), the success/fallback + /// label mapping, and the three same-typed performance gauges. #[test] - fn test_record_zero_copy_buffer_operation() { - // This test verifies the function compiles and runs - // Actual metric verification requires a metrics recorder - record_zero_copy_buffer_operation("read", 1024); - record_zero_copy_buffer_operation("write", 2048); - } + fn zero_copy_helpers_emit_their_metrics() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); - #[test] - fn test_record_memory_copy() { - record_memory_copy(1, 1024); - record_memory_copy(2, 2048); - } + metrics::with_local_recorder(&recorder, || { + set_metrics_enabled(true); + record_zero_copy_buffer_operation("read", 1024); + record_zero_copy_buffer_operation("write", 2048); + record_memory_copy(1, 1024); + record_memory_copy(2, 2048); + record_shared_ref_operation("create"); + record_shared_ref_operation("share"); + record_bufreader_optimization(1, 8192); + record_bufreader_optimization(2, 65536); + record_direct_io_operation("read", 4096, true); + record_direct_io_operation("write", 8192, false); + update_zero_copy_performance_metrics(2, 150.5, 1024 * 1024); + set_metrics_enabled(false); + }); - #[test] - fn test_record_shared_ref_operation() { - record_shared_ref_operation("create"); - record_shared_ref_operation("share"); - } + let rows = snapshotter.snapshot().into_vec(); + let names = emitted_names(&rows); + for expected in [ + zero_copy::BUFFER_OPERATIONS_TOTAL, + zero_copy::BUFFER_BYTES_TOTAL, + zero_copy::MEMORY_COPY_TOTAL, + zero_copy::MEMORY_COPY_BYTES_TOTAL, + "rustfs_memory_copy_size_bytes", + zero_copy::SHARED_REF_OPERATIONS_TOTAL, + zero_copy::BUFREADER_LAYERS_ELIMINATED_TOTAL, + zero_copy::BUFREADER_BUFFER_SIZE_BYTES, + zero_copy::DIRECT_IO_OPERATIONS_TOTAL, + zero_copy::DIRECT_IO_BYTES_TOTAL, + aligned_pread::OPERATIONS_TOTAL, + aligned_pread::BYTES_TOTAL, + zero_copy::AVG_COPY_COUNT, + zero_copy::THROUGHPUT_MBPS, + zero_copy::MEMORY_SAVED_BYTES, + ] { + assert!(names.contains(expected), "{expected} must be emitted by its record helper"); + } - #[test] - fn test_record_bufreader_optimization() { - record_bufreader_optimization(1, 8192); - record_bufreader_optimization(2, 65536); - } + assert_eq!(counter_total(&rows, zero_copy::BUFFER_OPERATIONS_TOTAL), Some(2)); + assert_eq!( + counter_total(&rows, zero_copy::BUFFER_BYTES_TOTAL), + Some(3072), + "buffer bytes must accumulate the sizes, not the call count" + ); + assert_eq!( + counter_total(&rows, zero_copy::MEMORY_COPY_TOTAL), + Some(3), + "the copy counter takes the copy count argument" + ); + assert_eq!( + counter_total(&rows, zero_copy::MEMORY_COPY_BYTES_TOTAL), + Some(3072), + "the copied-bytes counter takes the size argument" + ); + assert_eq!(histogram_samples(&rows, "rustfs_memory_copy_size_bytes"), vec![1024.0, 2048.0]); + assert_eq!(counter_total(&rows, zero_copy::SHARED_REF_OPERATIONS_TOTAL), Some(2)); + assert_eq!(counter_total(&rows, zero_copy::BUFREADER_LAYERS_ELIMINATED_TOTAL), Some(3)); + assert_eq!(histogram_samples(&rows, zero_copy::BUFREADER_BUFFER_SIZE_BYTES), vec![8192.0, 65536.0]); + assert_eq!(counter_total(&rows, zero_copy::DIRECT_IO_BYTES_TOTAL), Some(12288)); + assert_eq!( + counter_total(&rows, aligned_pread::BYTES_TOTAL), + Some(12288), + "the aligned-pread series must mirror the direct-IO series" + ); + assert_eq!(gauge_value(&rows, zero_copy::AVG_COPY_COUNT), Some(2.0)); + assert_eq!(gauge_value(&rows, zero_copy::THROUGHPUT_MBPS), Some(150.5)); + assert_eq!( + gauge_value(&rows, zero_copy::MEMORY_SAVED_BYTES), + Some((1024 * 1024) as f64), + "the three performance gauges must not be filled from each other's argument" + ); - #[test] - fn test_record_direct_io_operation() { - record_direct_io_operation("read", 4096, true); - record_direct_io_operation("write", 8192, false); - } - - #[test] - fn test_update_zero_copy_performance_metrics() { - update_zero_copy_performance_metrics(2, 150.5, 1024 * 1024); + let mut direct_io_labels: Vec<(&str, &str)> = rows + .iter() + .filter(|(composite, _, _, _)| composite.key().name() == zero_copy::DIRECT_IO_OPERATIONS_TOTAL) + .map(|(composite, _, _, _)| { + let label = |key: &str| { + composite + .key() + .labels() + .find(|label| label.key() == key) + .map(|label| label.value()) + .expect("direct-IO counters carry operation and status labels") + }; + (label("operation"), label("status")) + }) + .collect(); + direct_io_labels.sort(); + assert_eq!( + direct_io_labels, + vec![("read", "success"), ("write", "fallback")], + "the success flag must map to the success/fallback status label" + ); } #[test] diff --git a/scripts/find_assertless_tests.py b/scripts/find_assertless_tests.py index 90e52a21f..dc73447de 100755 --- a/scripts/find_assertless_tests.py +++ b/scripts/find_assertless_tests.py @@ -28,8 +28,15 @@ called helper. Known false-positive classes are excluded up front: parameters; the assert lives in the shared body — still scanned, but a body that asserts is not flagged anyway; the exclusion covers wrappers that only delegate to a suite runner). -- Functions whose body calls a helper with `assert`, `verify`, `check`, - `expect`, `run_` or `_case` in its name (suite-delegation pattern). +- Functions whose body calls a helper *named* like a shared check or suite + runner: an `assert_`/`verify_`/`check_`/`expect_`/`ensure_`/`run_` prefix, + or a `_case`/`_cases`/`_harness`/`_roundtrip` suffix. The name must carry + the token as its own leading or trailing segment — matching it anywhere + inside the identifier hid whole test bodies behind an unrelated domain + call such as `record_get_object_bitrot_verify_duration(..)`. +- Functions whose body only defines an unused inner `fn _name(..)`: that is + the compile-time shape check (exhaustive match, signature pin), where the + type system is the assertion. Usage: scripts/find_assertless_tests.py [path ...] # default: crates rustfs/src @@ -45,7 +52,11 @@ VERIFY_SIGNALS = re.compile( r"assert!|assert_eq!|assert_ne!|debug_assert|panic!\(|\.expect\(|\.unwrap\(|" r"unreachable!|matches!\(|insta::|proptest!|\.await\?|\)\?|\?;|should_panic" ) -DELEGATION = re.compile(r"\b[a-z0-9_]*(?:assert|verify|check|expect|run_case|_case|harness|round_trip|roundtrip)[a-z0-9_]*\s*\(") +DELEGATION = re.compile( + r"\b(?:assert|verify|check|expect|ensure|run)_[a-z0-9_]*\s*\(|" + r"\b[a-z0-9_]+_(?:case|cases|harness|roundtrip|round_trip)\s*\(" +) +COMPILE_TIME_CHECK = re.compile(r"\bfn\s+_[a-zA-Z0-9_]*\s*(?:<[^>]*>)?\s*\(") TEST_ATTR = re.compile(r"#\[(?:tokio::)?test[\](]") TEST_CASE_ATTR = re.compile(r"#\[test_case") FN_LINE = re.compile(r"^\s*(?:pub\s+)?(?:async\s+)?fn\s+([a-zA-Z0-9_]+)") @@ -94,7 +105,7 @@ def scan_file(path: Path): break k += 1 text = "\n".join(body) - if not VERIFY_SIGNALS.search(text) and not DELEGATION.search(text): + if not VERIFY_SIGNALS.search(text) and not DELEGATION.search(text) and not COMPILE_TIME_CHECK.search(text): print(f"{path}:{j + 1}: {name}") i = k + 1 From 1cf0f7af1583caeafea278c082688814017ba6af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Tue, 18 Aug 2026 21:45:38 +0800 Subject: [PATCH 41/51] feat(replication): split oversized hot-path functions, proxy unreplicated reads, and fail SSE-C passthrough closed (#6170) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(replication): split four oversized hot-path functions into focused helpers Pure-move decomposition of the four oversized functions flagged by the replication compatibility review (P1-18), unblocking migration milestone M2 which requires resyncer moves to stay mechanical: - resync_bucket (522 lines -> 61-line step sequence): leader lock, target resolution, walk/collector/worker spawning, and dispatch loop extracted into focused helpers; pure decision helpers (DTO builders, HEAD-result classification) separated from IO orchestration. - replicate_all (411 lines -> 113-line main body): initial target-info seeding, read/stat option builders, skip-path notes, target HEAD action resolution, and the multipart/single-put payload transport extracted as private free functions. - start_mrf_processor (306 lines -> 46-line spawn body): recovery guard, ledger load, per-entry replay (delete/object/metadata), and retained entry resolution extracted; retry bookkeeping semantics preserved exactly (inner continue-paths push inside helpers, outer Missed push stays in the loop). - apply_iam_item (255 lines -> match dispatch skeleton): one helper per IAM item type. No behavior change: log texts, error paths, event emissions, and metric counts are byte-identical; existing tests unchanged and green (238 ecstore replication/mrf/resync + 232 rustfs site-replication). * feat(replication): proxy GET/HEAD/Tagging for unreplicated objects to replication targets (#6172) * feat(replication): proxy GET/HEAD/Tagging for unreplicated objects to replication targets Implements the MinIO active-active read-proxy protocol (P1-5 of the replication compatibility review): when a GET/HEAD/GetObjectTagging/ PutObjectTagging/DeleteObjectTagging request fails locally with not-found and the bucket has replication targets, the request is proxied to the targets in rule order, mirroring bucket-replication.go proxyGetToReplicationTarget/proxyHeadToRepTarget/proxyTaggingToRepTarget. Protocol surface: - Anti-loop: inbound {x-rustfs-,x-minio-}source-proxy-request is parsed into ObjectOptions (proxy_request + proxy_header_set, matching MinIO ProxyRequest/ProxyHeaderSet); a request carrying the marker with ANY value is never re-proxied. Outbound client proxy calls send the marker as "true"; replication worker convergence HEADs send it as "false" so a peer's proxy layer cannot answer a convergence check by proxying back to the source (which would fake Completed without a PUT). - Target selection: new replication_proxy.rs get_proxy_targets — empty when the marker is set, versioning is suspended, or no replication config; otherwise filter_target_arns -> TargetClient lookup, skipping targets with proxying disabled. - TargetClient gains head_object_for_proxy/get_object (streaming) and the three tagging calls. Proxy calls never send the replication-check SSE-C exemption header; customer SSE-C keys are forwarded verbatim so the target performs real decryption. Conditional (If-*) headers are not forwarded (MinIO parity); Range and part_number are, with parts_count/tag_count/storage_class/expiration passed through. - Metrics: proxy counters now count only real client proxy traffic, MinIO-aligned (one total per proxied request, one failed when no target served it). The previous misattributed counters — replication worker HEAD/PUT (#2672) and local tagging operations (#2682) — are removed; ReplProxyMetric now maps the tagging counters instead of dropping them. e2e (fake_s3_target extended with tagging + header journaling): proxied GET body + outbound header contract (marker present, no replication-check, SSE-C passthrough), HEAD, anti-loop 404 with zero outbound requests, GetObjectTagging, and metric mapping unit tests. Rolling note: proxying only activates for buckets with replication targets; requests carrying the marker keep pre-upgrade behavior. Refs rustfs/backlog#1675 (P1-5) * fix(replication): fail SSE-C passthrough closed on targets that drop transport headers (#6178) SSE-C ciphertext passthrough replicates via X-Rustfs-Replication-* transport headers. A MinIO/generic-S3 target silently discards them, storing bare ciphertext with no decryption material — yet the PUT succeeded, so the object reported COMPLETED with a silently unreadable replica (backlog#1675 N2). Fail-closed design: - SsecPassthroughCapability {Unknown, Supported, Unsupported} cached in BucketTargetSys per target ARN with a recording timestamp. Entries reset whenever the target is rebuilt, edited, or removed (arn_remotes_map lifecycle) and expire after SSEC_PASSTHROUGH_CAPABILITY_TTL (10 minutes): an expired verdict in either direction is re-earned through the audit, so an Unsupported target recovers automatically after an upgrade (at most one wasted PUT+HEAD audit per bad target per TTL window) and a Supported verdict cannot outlive a backend swapped behind the same endpoint. - Replication worker (replicate_object and replicate_all): fresh Unsupported targets never receive the PUT — the attempt fails immediately into the normal MRF retry channel with a "run ?replication-check to re-probe" hint. Unknown or expired verdicts are audited: after the PUT the worker HEADs the replica back through the replication-check channel (source version id mapped through resolve_read_api_version_id, so null-version objects audit correctly) and requires SSE-C evidence (the echoed customer-algorithm header); missing evidence records Unsupported and fails the attempt. Convergence HEADs are audited the same way, so a broken ciphertext replica from an earlier attempt can never launder itself into COMPLETED via an ETag match. The gate/evidence policy is pure (replication_target_boundary, staleness folded in as an input) for the M2 worker migration. - replication-check grows an SsecPassthrough probe phase: a probe PUT carrying the live transport-header shape, HEAD-back for evidence, and a machine-readable Code BucketRemoteSsecPassthroughUnsupported on failure. The probe verdict is synced into the runtime capability cache. Unlike VersionFidelity, a failed SsecPassthrough phase does NOT fail the target overall — it is a capability limit, not a broken replication contract, and a plaintext-only deployment against such a target must not turn red. - fake_s3_target: default mode now models a RustFS target (stores the transport headers, echoes SSE-C evidence); the new drop_unlisted_replication_headers mode models MinIO. The journal records whether a request carried transport headers. Receiver-echo verification: the replication-check HEAD exemption only skips SSE-C key validation; the response has always built sse-customer-algorithm from stored metadata (rustfs/src/app/object_usecase.rs), so no receiver change was needed — pinned end to end by the replication-check e2e against a real RustFS target. Rolling-upgrade constraint: RustFS targets older than the replication-check HEAD exemption (#5898) answer the audit HEAD without SSE-C evidence (or fail it outright), so SSE-C replication to such targets reports FAILED. This is deliberate — FAILED-and-retryable beats a silently undecryptable replica — and self-heals: once the target is upgraded, the next TTL expiry (or a manual ?replication-check re-probe) re-audits and records Supported. Plaintext and managed-SSE replication are unaffected. The capability cache is per-node; each node audits independently. Known limitations: - The audit judges evidence from the echoed customer-algorithm header only. A hypothetical target that preserves that one header while dropping other transport headers (partial-drop) would pass the audit; no known target behaves this way — observed targets drop the whole unknown-header family. - A mixed-version target cluster can flap the verdict between audits routed to different target nodes until the rollout completes; the TTL bounds how long each stale verdict persists. New e2e (backlog#1675 C1 + N2, red-first): fail-closed against a header-dropping fake (FAILED + no second PUT via the capability cache, journal-asserted; red run showed the old COMPLETED), replication-check reports the SsecPassthrough phase Code while the target stays OK overall, SSE-C heal convergence after a real target outage, and SSE-C existing-object resync landing a REPLICA readable with the customer key. TTL expiry in both directions is pinned at the cache and gate seams. * refactor(replication): move resyncer pure decision logic into rustfs-replication (M2) (#6180) * refactor(replication): move resyncer pure decision logic into rustfs-replication (M2) Pure-move milestone M2 of the ECStore replication split (backlog#1675 P1-17): relocate the resyncer's IO-free decision helpers, with their unit tests, into the crates they already belong to by type ownership. No behavior change. Moved into crates/replication: - resync.rs: resync_status_duration - delete.rs: resync_existing_delete_replication_info, replicate_delete_outcome, target_delete_version_id, delete_marker_purge_version_id, delete_marker_purge_mrf_entry - object.rs: version_identity_drifted, is_replication_target_offline_error, SsecPassthroughCapability, SsecPassthroughGate, ssec_passthrough_gate, ssec_passthrough_evidence_present (param-demoted to the echoed customer-algorithm string; ECStore keeps the HeadObjectOutput adapter) - filemeta.rs: NULL_VERSION_ID wire literal (crate-owned copy per the filemeta-independence contract) ECStore rewiring (Rule #14: imports stay in *_boundary.rs): - resync/object-decision/target boundaries re-export the moved symbols; resyncer call sites are unchanged - bucket_target_sys keeps only the verdict cache + TTL and re-exports the capability enum so existing consumer paths keep compiling Not moved (signatures carry ECStore or aws-sdk types): verify_resync_head_result, resync_target_error_detail, the SdkError classifiers, the replicate_all_* option/info builders, and the env-coupled bounded_resync_max_jobs admission clamp. README milestone table updated. * chore(replication): retire the datatypes.rs relay early README sanctions retiring datatypes.rs ahead of M4. The module was a pure relay (resync boundary -> datatypes -> mod.rs facade) with no external consumer importing it directly, so the facade now re-exports ResyncStatusType from replication_resync_boundary and the relay file is deleted. Consumers stay behind the ECStore facade, keeping Migration Rule #15 intact — the original retirement wording ("consumers import through rustfs-replication directly") conflicted with that rule and is corrected in the README. * chore(arch): extend migration guards to the M2-moved decision contracts The adversarial review of the M2 move found the per-symbol ratchet in check_architecture_migration_rules.sh was not extended for the moved symbols, leaving them free to be redefined in ECStore or imported past their boundary without CI noticing: - resync definition pin + boundary fences gain resync_status_duration; - the object-decision boundary fences gain the five delete-family helpers (delete_marker_purge_mrf_entry, delete_marker_purge_version_id, replicate_delete_outcome, resync_existing_delete_replication_info, target_delete_version_id); - the target-boundary fence gains the SSE-C gate family, the offline classifier, and version_identity_drifted; - a new definition pin rejects ECStore redefinitions of the M2-moved fns/enums (ssec_passthrough_evidence_present deliberately excluded: ECStore keeps a thin HeadObjectOutput adapter under that name). Mutation-verified: a probe fn ssec_passthrough_gate under crates/ecstore/src/bucket/replication trips the new pin. Also anchors the intentionally-duplicated NULL_VERSION_ID wire literal from the filemeta side and tightens the M2 README note on bounded_resync_max_jobs. --- crates/e2e_test/src/fake_s3_target/README.md | 4 +- crates/e2e_test/src/fake_s3_target/mod.rs | 289 ++- .../src/replication_extension_test.rs | 722 +++++- crates/ecstore/src/api/mod.rs | 15 +- .../ecstore/src/bucket/bucket_target_sys.rs | 310 ++- .../ecstore/src/bucket/replication/README.md | 15 +- .../src/bucket/replication/datatypes.rs | 15 - crates/ecstore/src/bucket/replication/mod.rs | 6 +- .../replication_filemeta_boundary.rs | 7 +- .../replication_object_decision_boundary.rs | 11 +- .../bucket/replication/replication_pool.rs | 636 ++--- .../bucket/replication/replication_proxy.rs | 150 ++ .../replication_resync_boundary.rs | 9 +- .../replication/replication_resyncer.rs | 2044 +++++++++-------- .../bucket/replication/replication_state.rs | 25 + .../replication_target_boundary.rs | 47 +- crates/ecstore/src/object_api/types.rs | 14 + crates/filemeta/src/fileinfo.rs | 3 + crates/replication/src/delete.rs | 206 +- crates/replication/src/filemeta.rs | 5 + crates/replication/src/lib.rs | 14 +- crates/replication/src/object.rs | 213 +- crates/replication/src/resync.rs | 41 + rustfs/src/admin/handlers/site_replication.rs | 504 ++-- rustfs/src/admin/router.rs | 270 ++- rustfs/src/admin/storage_api.rs | 1 + rustfs/src/app/object_usecase.rs | 251 +- rustfs/src/app/storage_api.rs | 18 + rustfs/src/storage/ecfs.rs | 267 ++- rustfs/src/storage/mod.rs | 36 +- rustfs/src/storage/options.rs | 96 +- rustfs/src/storage/storage_api.rs | 26 +- scripts/check_architecture_migration_rules.sh | 36 +- 33 files changed, 4694 insertions(+), 1612 deletions(-) delete mode 100644 crates/ecstore/src/bucket/replication/datatypes.rs create mode 100644 crates/ecstore/src/bucket/replication/replication_proxy.rs diff --git a/crates/e2e_test/src/fake_s3_target/README.md b/crates/e2e_test/src/fake_s3_target/README.md index 78597195c..7f1feae0b 100644 --- a/crates/e2e_test/src/fake_s3_target/README.md +++ b/crates/e2e_test/src/fake_s3_target/README.md @@ -4,8 +4,8 @@ This module is the shared failure-injection boundary for replication end-to-end `FakeS3Target::start()` creates the listener. Add target buckets with `create_bucket`, point a RustFS remote target at `address()`, use `FAKE_ACCESS_KEY` / `FAKE_SECRET_KEY`, then enqueue per-operation faults with `inject`. Faults for one operation are consumed in FIFO order and do not consume faults queued for another operation. A fault is consumed only after `s3s` verifies the full request signature, so anonymous, other-access-key, and bad-signature traffic cannot disturb a script. -Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB. +Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB. -Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions. +Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions. Each record also journals a `ProxyHeaderSnapshot` — the read-proxy anti-loop marker (`x-{rustfs,minio}-source-proxy-request`), the replication-check exemption header, and the client SSE-C header family (algorithm and key-MD5 values; for the key itself only its presence) — so proxy tests can pin the exact wire contract. The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type at 1 KiB. A PUT or uploaded part is capped at 64 MiB; a completed multipart object and all stored object/part data are capped at 128 MiB. Body drain, body-permit waits, delay, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound. diff --git a/crates/e2e_test/src/fake_s3_target/mod.rs b/crates/e2e_test/src/fake_s3_target/mod.rs index c8ddcecf3..aad643043 100644 --- a/crates/e2e_test/src/fake_s3_target/mod.rs +++ b/crates/e2e_test/src/fake_s3_target/mod.rs @@ -30,10 +30,12 @@ use s3s::access::{S3Access, S3AccessContext}; use s3s::auth::SimpleAuth; use s3s::dto::{ AbortMultipartUploadInput, AbortMultipartUploadOutput, CompleteMultipartUploadInput, CompleteMultipartUploadOutput, - CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput, ETag, - GetBucketVersioningInput, GetBucketVersioningOutput, GetObjectInput, GetObjectOutput, HeadBucketInput, HeadBucketOutput, + CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput, + DeleteObjectTaggingInput, DeleteObjectTaggingOutput, ETag, GetBucketVersioningInput, GetBucketVersioningOutput, + GetObjectInput, GetObjectOutput, GetObjectTaggingInput, GetObjectTaggingOutput, HeadBucketInput, HeadBucketOutput, HeadObjectInput, HeadObjectOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ObjectVersionId, PutObjectInput, - PutObjectOutput, StreamingBlob, Timestamp, TimestampFormat, UploadPartInput, UploadPartOutput, + PutObjectOutput, PutObjectTaggingInput, PutObjectTaggingOutput, StreamingBlob, Tag, TagSet, Timestamp, TimestampFormat, + UploadPartInput, UploadPartOutput, }; use s3s::service::{S3Service, S3ServiceBuilder}; use s3s::validation::{AwsNameValidation, NameValidation}; @@ -88,6 +90,13 @@ const SOURCE_LEGALHOLD_TIMESTAMP_HEADERS: [&str; 2] = [ "x-rustfs-source-replication-legalhold-timestamp", "x-minio-source-replication-legalhold-timestamp", ]; +/// Wire prefix of the SSE-C passthrough replication transport headers +/// (`X-Rustfs-Replication-*`). In the default mode the fake stores them like a +/// RustFS target and echoes SSE-C evidence back on HEAD/GET; with +/// [`FakeS3Target::drop_unlisted_replication_headers`] it models MinIO / +/// generic S3, which silently discard unknown x-* headers. +const REPLICATION_SSE_TRANSPORT_PREFIX: &str = "x-rustfs-replication-"; +const REPLICATION_SSEC_ALGORITHM_TRANSPORT_HEADER: &str = "x-rustfs-replication-ssec-algorithm"; const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"]; const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"]; @@ -103,6 +112,9 @@ pub enum Operation { GetObject, HeadObject, DeleteObject, + GetObjectTagging, + PutObjectTagging, + DeleteObjectTagging, ListObjectVersions, CreateMultipartUpload, UploadPart, @@ -149,6 +161,42 @@ impl ReplicationTimestampHeaders { } } +/// Read-proxy related headers observed on a request, journaled so proxy +/// tests can assert the exact wire contract: the anti-loop marker present, +/// the replication-check exemption absent, and the client SSE-C key family +/// forwarded verbatim. The SSE-C key value itself is never retained — only +/// its presence. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ProxyHeaderSnapshot { + pub source_proxy_request: Option, + pub replication_check: Option, + pub ssec_algorithm: Option, + pub ssec_key_present: bool, + pub ssec_key_md5: Option, + /// Whether the request carried any `X-Rustfs-Replication-*` SSE-C + /// passthrough transport header, so fail-closed tests can assert the + /// sender really shipped the material a dropping target discarded. + pub ssec_transport_present: bool, +} + +impl ProxyHeaderSnapshot { + fn from_headers(headers: &HeaderMap) -> Self { + Self { + source_proxy_request: header_value(headers, &["x-rustfs-source-proxy-request", "x-minio-source-proxy-request"]) + .map(bounded_journal_value), + replication_check: header_value(headers, &["x-rustfs-source-replication-check", "x-minio-source-replication-check"]) + .map(bounded_journal_value), + ssec_algorithm: header_value(headers, &["x-amz-server-side-encryption-customer-algorithm"]) + .map(bounded_journal_value), + ssec_key_present: headers.contains_key("x-amz-server-side-encryption-customer-key"), + ssec_key_md5: header_value(headers, &["x-amz-server-side-encryption-customer-key-md5"]).map(bounded_journal_value), + ssec_transport_present: headers + .keys() + .any(|name| name.as_str().starts_with(REPLICATION_SSE_TRANSPORT_PREFIX)), + } + } +} + /// Credential-free request metadata retained for deterministic assertions. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RequestRecord { @@ -163,6 +211,7 @@ pub struct RequestRecord { pub content_length: Option, pub consumed_bytes: Option, pub replication_timestamps: ReplicationTimestampHeaders, + pub proxy_headers: ProxyHeaderSnapshot, pub fault: Option, } @@ -178,6 +227,10 @@ struct ControlState { struct StoreState { assign_own_version_ids: bool, assign_own_multipart_version_ids: bool, + /// MinIO-like mode: silently discard non-whitelisted replication + /// transport headers instead of storing them (see + /// [`REPLICATION_SSE_TRANSPORT_PREFIX`]). + drop_unlisted_replication_headers: bool, buckets: HashMap, uploads: HashMap, total_bytes: usize, @@ -199,6 +252,12 @@ struct ObjectVersion { delete_marker: bool, content_type: Option, metadata: Option>, + /// Object tags as ordered key/value pairs (PutObjectTagging replaces the + /// whole set, DeleteObjectTagging clears it). + tags: Vec<(String, String)>, + /// SSE-C passthrough transport headers stored with the version (RustFS + /// target behavior); empty when the drop mode discarded them. + replication_sse_headers: Vec<(String, String)>, } #[derive(Clone)] @@ -208,6 +267,7 @@ struct MultipartState { version_id: String, content_type: Option, metadata: Option>, + replication_sse_headers: Vec<(String, String)>, parts: BTreeMap, } @@ -428,6 +488,15 @@ impl FakeS3Target { /// Mint own version ids for the multipart path only — models a target /// that adopts PutObject version ids but not CreateMultipartUpload ones. + /// MinIO-like mode: silently drop every `X-Rustfs-Replication-*` SSE-C + /// passthrough transport header instead of storing it. The default (off) + /// models a RustFS target, which preserves the headers and echoes SSE-C + /// evidence (`x-amz-server-side-encryption-customer-algorithm`) on + /// HEAD/GET of the replica. + pub fn drop_unlisted_replication_headers(&self, enabled: bool) { + lock(&self.backend.store).drop_unlisted_replication_headers = enabled; + } + pub fn assign_own_multipart_version_ids(&self, enabled: bool) { lock(&self.backend.store).assign_own_multipart_version_ids = enabled; } @@ -569,6 +638,7 @@ impl S3Access for FaultAccess { .and_then(|value| value.to_str().ok()) .and_then(|value| value.parse().ok()); let replication_timestamps = ReplicationTimestampHeaders::from_headers(context.headers()); + let proxy_headers = ProxyHeaderSnapshot::from_headers(context.headers()); let fault = record_request( &self.control, operation, @@ -576,6 +646,7 @@ impl S3Access for FaultAccess { parsed, content_length, replication_timestamps, + proxy_headers, ); if let Some(RequestFault { action: FaultAction::Status(status), @@ -615,6 +686,9 @@ fn operation_from_s3_name(name: &str) -> Operation { "GetObject" => Operation::GetObject, "HeadObject" => Operation::HeadObject, "DeleteObject" => Operation::DeleteObject, + "GetObjectTagging" => Operation::GetObjectTagging, + "PutObjectTagging" => Operation::PutObjectTagging, + "DeleteObjectTagging" => Operation::DeleteObjectTagging, "CreateMultipartUpload" => Operation::CreateMultipartUpload, "UploadPart" => Operation::UploadPart, "CompleteMultipartUpload" => Operation::CompleteMultipartUpload, @@ -630,6 +704,7 @@ fn record_request( parsed: ParsedRequest, content_length: Option, replication_timestamps: ReplicationTimestampHeaders, + proxy_headers: ProxyHeaderSnapshot, ) -> Option { let mut state = lock(control); let action = parsed @@ -655,6 +730,7 @@ fn record_request( content_length, consumed_bytes: None, replication_timestamps, + proxy_headers, fault: action.clone(), }); action.map(|action| RequestFault { sequence, action }) @@ -721,6 +797,15 @@ fn parse_request(method: &Method, uri: &Uri) -> ParsedRequest { (&Method::POST, true) if query.contains_key("uploads") => Operation::CreateMultipartUpload, (&Method::POST, true) if upload_id.is_some() => Operation::CompleteMultipartUpload, (&Method::DELETE, true) if upload_id.is_some() => Operation::AbortMultipartUpload, + (&Method::GET, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => { + Operation::GetObjectTagging + } + (&Method::PUT, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => { + Operation::PutObjectTagging + } + (&Method::DELETE, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => { + Operation::DeleteObjectTagging + } // A replication PUT addresses the source version via `?versionId=`. (&Method::PUT, true) if only_query_keys(&["versionId"]) => Operation::PutObject, (&Method::GET, true) if only_query_keys(&["versionId"]) => Operation::GetObject, @@ -788,6 +873,29 @@ fn new_version_id(headers: &HeaderMap, assign_own: bool) -> S3Result { Ok(version_id.to_string()) } +/// Capture the SSE-C passthrough transport headers a replication PUT carried. +/// Returns an empty set in the MinIO-like drop mode. +fn captured_replication_sse_headers(headers: &HeaderMap, drop_unlisted: bool) -> Vec<(String, String)> { + if drop_unlisted { + return Vec::new(); + } + headers + .iter() + .filter(|(name, _)| name.as_str().starts_with(REPLICATION_SSE_TRANSPORT_PREFIX)) + .filter_map(|(name, value)| Some((name.as_str().to_string(), value.to_str().ok()?.to_string()))) + .collect() +} + +/// SSE-C evidence a RustFS-like target echoes for a stored passthrough +/// replica: the customer algorithm restored from the transport headers. +fn stored_sse_customer_algorithm(version: &ObjectVersion) -> Option { + version + .replication_sse_headers + .iter() + .find(|(name, _)| name == REPLICATION_SSEC_ALGORITHM_TRANSPORT_HEADER) + .map(|(_, value)| value.clone()) +} + fn source_etag(headers: &HeaderMap) -> S3Result> { header_value(headers, &SOURCE_ETAG_HEADERS) .map(|value| validate_retained_identifier(value, "source ETag").map(|value| normalize_etag(&value))) @@ -1135,6 +1243,33 @@ fn find_version(state: &StoreState, bucket: &str, key: &str, version_id: Option< Ok(version.clone()) } +/// Replace (or clear, with an empty vec) the tag set of the addressed +/// version, returning its version id. Mirrors `find_version` addressing: +/// explicit version id or the latest version, delete markers rejected. +fn set_version_tags( + state: &mut StoreState, + bucket: &str, + key: &str, + version_id: Option<&str>, + tags: Vec<(String, String)>, +) -> S3Result { + // Resolve first (immutable) so the error paths match find_version. + let resolved = find_version(state, bucket, key, version_id)?.version_id; + let versions = state + .buckets + .get_mut(bucket) + .expect("bucket existence checked by find_version") + .objects + .get_mut(key) + .expect("key existence checked by find_version"); + let version = versions + .iter_mut() + .find(|version| version.version_id == resolved) + .expect("version existence checked by find_version"); + version.tags = tags; + Ok(resolved) +} + #[async_trait] impl S3 for FakeBackend { async fn head_bucket(&self, req: S3Request) -> S3Result> { @@ -1231,7 +1366,10 @@ impl S3 for FakeBackend { let input = req.input; let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?; validate_stored_metadata(&input.content_type, &input.metadata)?; - let assign_own = lock(&self.store).assign_own_version_ids; + let (assign_own, drop_unlisted) = { + let state = lock(&self.store); + (state.assign_own_version_ids, state.drop_unlisted_replication_headers) + }; let version_id = new_version_id(&headers, assign_own)?; let e_tag = match source_etag(&headers)? { Some(value) => value, @@ -1248,6 +1386,8 @@ impl S3 for FakeBackend { delete_marker: false, content_type: input.content_type, metadata: input.metadata, + tags: Vec::new(), + replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted), }; upsert_version(&mut lock(&self.store), &input.bucket, input.key, version)?; Ok(apply_response_fault( @@ -1268,6 +1408,7 @@ impl S3 for FakeBackend { let state = lock(&self.store); find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())? }; + let sse_customer_algorithm = stored_sse_customer_algorithm(&version); Ok(apply_response_fault( S3Response::new(GetObjectOutput { body: Some(StreamingBlob::new(Body::from(version.body.clone()))), @@ -1277,6 +1418,7 @@ impl S3 for FakeBackend { e_tag: Some(ETag::Strong(version.e_tag)), last_modified: Some(version.last_modified.clone()), version_id: Some(version.version_id), + sse_customer_algorithm, ..Default::default() }), fault.as_ref(), @@ -1291,6 +1433,7 @@ impl S3 for FakeBackend { let state = lock(&self.store); find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())? }; + let sse_customer_algorithm = stored_sse_customer_algorithm(&version); Ok(apply_response_fault( S3Response::new(HeadObjectOutput { content_length: Some(version.body.len() as i64), @@ -1299,12 +1442,79 @@ impl S3 for FakeBackend { e_tag: Some(ETag::Strong(version.e_tag)), last_modified: Some(version.last_modified.clone()), version_id: Some(version.version_id), + sse_customer_algorithm, ..Default::default() }), fault.as_ref(), )) } + async fn get_object_tagging(&self, req: S3Request) -> S3Result> { + let fault = request_fault(&req); + apply_non_body_fault(fault.as_ref(), &self.control).await?; + let input = req.input; + let version = { + let state = lock(&self.store); + find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())? + }; + let tag_set: TagSet = version + .tags + .into_iter() + .map(|(key, value)| Tag { + key: Some(key), + value: Some(value), + }) + .collect(); + Ok(apply_response_fault( + S3Response::new(GetObjectTaggingOutput { + tag_set, + version_id: Some(ObjectVersionId::from(version.version_id)), + }), + fault.as_ref(), + )) + } + + async fn put_object_tagging(&self, req: S3Request) -> S3Result> { + let fault = request_fault(&req); + apply_non_body_fault(fault.as_ref(), &self.control).await?; + let input = req.input; + let tags = input + .tagging + .tag_set + .into_iter() + .map(|tag| (tag.key.unwrap_or_default(), tag.value.unwrap_or_default())) + .collect(); + let version_id = { + let mut state = lock(&self.store); + set_version_tags(&mut state, &input.bucket, &input.key, input.version_id.as_deref(), tags)? + }; + Ok(apply_response_fault( + S3Response::new(PutObjectTaggingOutput { + version_id: Some(ObjectVersionId::from(version_id)), + }), + fault.as_ref(), + )) + } + + async fn delete_object_tagging( + &self, + req: S3Request, + ) -> S3Result> { + let fault = request_fault(&req); + apply_non_body_fault(fault.as_ref(), &self.control).await?; + let input = req.input; + let version_id = { + let mut state = lock(&self.store); + set_version_tags(&mut state, &input.bucket, &input.key, input.version_id.as_deref(), Vec::new())? + }; + Ok(apply_response_fault( + S3Response::new(DeleteObjectTaggingOutput { + version_id: Some(ObjectVersionId::from(version_id)), + }), + fault.as_ref(), + )) + } + async fn delete_object(&self, req: S3Request) -> S3Result> { let fault = request_fault(&req); apply_non_body_fault(fault.as_ref(), &self.control).await?; @@ -1381,6 +1591,8 @@ impl S3 for FakeBackend { delete_marker: true, content_type: None, metadata: None, + tags: Vec::new(), + replication_sse_headers: Vec::new(), }, )?; Ok(apply_response_fault( @@ -1408,9 +1620,10 @@ impl S3 for FakeBackend { ensure_upload_budget(&state)?; validate_stored_metadata(&input.content_type, &input.metadata)?; let upload_id = Uuid::new_v4().to_string(); - // Read the flag before the mutable borrow of `state.uploads` below + // Read the flags before the mutable borrow of `state.uploads` below // (and never re-lock the store: the mutex is not reentrant). let mint_own = state.assign_own_version_ids || state.assign_own_multipart_version_ids; + let drop_unlisted = state.drop_unlisted_replication_headers; let version_id = new_version_id(&headers, mint_own)?; state.uploads.insert( upload_id.clone(), @@ -1420,6 +1633,7 @@ impl S3 for FakeBackend { version_id, content_type: input.content_type, metadata: input.metadata, + replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted), parts: BTreeMap::new(), }, ); @@ -1557,6 +1771,7 @@ impl S3 for FakeBackend { version_id: upload.version_id.clone(), content_type: upload.content_type.clone(), metadata: upload.metadata.clone(), + replication_sse_headers: upload.replication_sse_headers.clone(), parts: BTreeMap::new(), }, selected, @@ -1583,6 +1798,8 @@ impl S3 for FakeBackend { delete_marker: false, content_type: upload.content_type, metadata: upload.metadata, + tags: Vec::new(), + replication_sse_headers: upload.replication_sse_headers, }; let mut state = lock(&self.store); let current = state @@ -1787,6 +2004,65 @@ mod tests { Ok(()) } + /// Default mode is RustFS-like: SSE-C passthrough transport headers are + /// stored and the customer algorithm is echoed on HEAD/GET. Drop mode is + /// MinIO-like: the headers are silently discarded, so no evidence comes + /// back — the exact difference the N2 fail-closed audit keys on. Both + /// modes journal that the sender shipped the transport headers. + #[tokio::test] + async fn ssec_passthrough_headers_echo_and_drop_modes() -> Result<(), BoxError> { + let target = FakeS3Target::start().await?; + target.create_bucket("target-bucket"); + let client = client(&target); + + let put_with_transport_headers = |key: &'static str| { + client + .put_object() + .bucket("target-bucket") + .key(key) + .body(ByteStream::from_static(b"ciphertext")) + .customize() + .map_request(move |mut request| { + let headers = request.headers_mut(); + headers.insert("x-rustfs-replication-ssec-algorithm", "AES256"); + headers.insert("x-rustfs-replication-ssec-key-md5", "AAAAAAAAAAAAAAAAAAAAAA=="); + Ok::<_, std::convert::Infallible>(request) + }) + .send() + }; + + put_with_transport_headers("kept").await?; + let head = client.head_object().bucket("target-bucket").key("kept").send().await?; + assert_eq!(head.sse_customer_algorithm(), Some("AES256")); + let get = client.get_object().bucket("target-bucket").key("kept").send().await?; + assert_eq!(get.sse_customer_algorithm(), Some("AES256")); + + target.drop_unlisted_replication_headers(true); + put_with_transport_headers("dropped").await?; + let head = client.head_object().bucket("target-bucket").key("dropped").send().await?; + assert_eq!(head.sse_customer_algorithm(), None, "drop mode must discard SSE-C evidence"); + + let requests = target.requests(); + for key in ["kept", "dropped"] { + let record = requests + .iter() + .find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some(key)) + .expect("PUT must be journaled"); + assert!( + record.proxy_headers.ssec_transport_present, + "the journal must prove the sender shipped the transport headers for {key}" + ); + } + let plain_head = requests + .iter() + .find(|record| record.operation == Operation::HeadObject) + .expect("HEAD must be journaled"); + assert!(!plain_head.proxy_headers.ssec_transport_present); + + target.shutdown().await; + Ok(()) + } + macro_rules! assert_sdk_error { ($error:expr, $status:expr, $code:expr) => {{ let error = &$error; @@ -3052,6 +3328,7 @@ mod tests { version_id: index.to_string(), content_type: None, metadata: None, + replication_sse_headers: Vec::new(), parts: BTreeMap::new(), }, ); @@ -3074,6 +3351,7 @@ mod tests { }, Some(0), ReplicationTimestampHeaders::default(), + ProxyHeaderSnapshot::default(), ); } let records = lock(&control).requests.clone(); @@ -3096,6 +3374,7 @@ mod tests { }, None, ReplicationTimestampHeaders::default(), + ProxyHeaderSnapshot::default(), ); { let bounded_records = lock(&bounded_control); diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index ec847bfb7..8279479f1 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -2606,17 +2606,20 @@ async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box< assert_eq!(response.status(), StatusCode::OK); let payload: serde_json::Value = response.json().await?; - assert_eq!(payload["Status"], "OK"); + assert_eq!(payload["Status"], "OK", "{payload}"); assert_eq!(payload["ActiveMutation"], true); assert_eq!(payload["Targets"].as_array().map(Vec::len), Some(1)); - assert_eq!(payload["Targets"][0]["Status"], "OK"); - assert_eq!(payload["Targets"][0]["Phases"]["Put"]["Status"], "OK"); + assert_eq!(payload["Targets"][0]["Status"], "OK", "{payload}"); + assert_eq!(payload["Targets"][0]["Phases"]["Put"]["Status"], "OK", "{payload}"); // A RustFS target adopts the source version id, so the P1-19 // version-identity probe passes. - assert_eq!(payload["Targets"][0]["Phases"]["VersionFidelity"]["Status"], "OK"); - assert_eq!(payload["Targets"][0]["Phases"]["DeleteMarker"]["Status"], "OK"); - assert_eq!(payload["Targets"][0]["Phases"]["VersionDelete"]["Status"], "OK"); - assert_eq!(payload["Targets"][0]["Phases"]["Cleanup"]["Status"], "OK"); + assert_eq!(payload["Targets"][0]["Phases"]["VersionFidelity"]["Status"], "OK", "{payload}"); + // A RustFS target preserves the SSE-C passthrough transport headers and + // echoes the customer algorithm on the replication-check HEAD (N2). + assert_eq!(payload["Targets"][0]["Phases"]["SsecPassthrough"]["Status"], "OK", "{payload}"); + assert_eq!(payload["Targets"][0]["Phases"]["DeleteMarker"]["Status"], "OK", "{payload}"); + assert_eq!(payload["Targets"][0]["Phases"]["VersionDelete"]["Status"], "OK", "{payload}"); + assert_eq!(payload["Targets"][0]["Phases"]["Cleanup"]["Status"], "OK", "{payload}"); let target_client = target_env.create_s3_client(); let versions = target_client @@ -4616,6 +4619,410 @@ async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult { Ok(()) } +/// N2 (backlog#1675 P1-22): SSE-C passthrough replication to a target that +/// silently drops the `X-Rustfs-Replication-*` transport headers (MinIO-like +/// behavior, modeled by the fake target's drop mode) used to report COMPLETED +/// while the replica had irrecoverably lost its decryption material — the red +/// light this test was born failing on. Fail-closed contract now under test: +/// the first attempt PUTs, HEAD-backs the replica, finds no SSE-C evidence, +/// records the target Unsupported and reports FAILED; a second SSE-C object +/// fails without any PUT reaching the target (capability cache, proven from +/// the target journal); plaintext objects still replicate COMPLETED. +#[tokio::test] +#[serial] +async fn test_ssec_replication_fails_closed_when_target_drops_passthrough_headers() -> TestResult { + init_logging(); + + let target = FakeS3Target::start().await?; + let target_bucket = "ssec-drop-dst"; + target.create_bucket(target_bucket); + target.drop_unlisted_replication_headers(true); + + let mut source_env = RustFSTestEnvironment::new().await?; + let mut env_vars = replication_fast_env(); + env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV); + env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]); + source_env.start_rustfs_server_with_env(vec![], &env_vars).await?; + + let source_bucket = "ssec-drop-src"; + let source_client = source_env.create_s3_client(); + source_client.create_bucket().bucket(source_bucket).send().await?; + enable_bucket_versioning(&source_env, source_bucket).await?; + let target_arn = set_replication_target_with_options( + &source_env, + source_bucket, + ReplicationTargetOptions { + endpoint: &target.address(), + access_key: FAKE_ACCESS_KEY, + secret_key: FAKE_SECRET_KEY, + target_bucket, + secure: false, + skip_tls_verify: false, + ca_cert_pem: None, + }, + ) + .await?; + put_bucket_replication(&source_env, source_bucket, &target_arn).await?; + + let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY); + let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY); + let put_ssec = |key: &'static str| { + source_client + .put_object() + .bucket(source_bucket) + .key(key) + .body(ByteStream::from_static(b"ssec fail-closed payload")) + .sse_customer_algorithm("AES256") + .sse_customer_key(&customer_key) + .sse_customer_key_md5(&customer_key_md5) + .send() + }; + + // First SSE-C object: the audit must catch the dropped material. + put_ssec("ssec-first.txt").await?; + wait_for_source_replication_status(&source_client, source_bucket, "ssec-first.txt", "FAILED", true).await?; + + let requests = target.take_requests(); + let first_put = requests + .iter() + .find(|record| record.operation == FakeTargetOperation::PutObject && record.key.as_deref() == Some("ssec-first.txt")) + .ok_or("the first SSE-C object must have been PUT (capability was Unknown)")?; + assert!( + first_put.proxy_headers.ssec_transport_present, + "the replication PUT must have shipped the SSE-C transport headers the target then dropped" + ); + assert!( + requests.iter().any(|record| { + record.operation == FakeTargetOperation::HeadObject + && record.key.as_deref() == Some("ssec-first.txt") + && record.sequence > first_put.sequence + && record.proxy_headers.replication_check.as_deref() == Some("true") + }), + "the post-PUT HEAD-back audit must have run through the replication-check channel; journal: {requests:?}" + ); + + // Second SSE-C object: the cached Unsupported verdict fails it closed + // before any PUT — including MRF retries of the first object. + put_ssec("ssec-second.txt").await?; + wait_for_source_replication_status(&source_client, source_bucket, "ssec-second.txt", "FAILED", true).await?; + assert!( + !target.requests().iter().any(|record| { + record.operation == FakeTargetOperation::PutObject + && record.key.as_deref() != Some("plain-control.txt") + && record.proxy_headers.ssec_transport_present + }), + "no further SSE-C ciphertext may reach a target recorded Unsupported; journal: {:?}", + target.requests() + ); + + // The gate is scoped to SSE-C: plaintext replication keeps working. + source_client + .put_object() + .bucket(source_bucket) + .key("plain-control.txt") + .body(ByteStream::from_static(b"plaintext control payload")) + .send() + .await?; + wait_for_source_replication_status(&source_client, source_bucket, "plain-control.txt", "COMPLETED", false).await?; + assert!(target.has_object(target_bucket, "plain-control.txt")); + + target.shutdown().await; + Ok(()) +} + +/// N2 (backlog#1675 P1-22): the admin replication-check must expose the same +/// verdict operators would otherwise only learn from failing SSE-C objects — +/// an SsecPassthrough probe phase that fails with the machine-readable +/// `BucketRemoteSsecPassthroughUnsupported` code against a header-dropping +/// target, with no probe residue left behind. The target's overall status +/// stays OK: unlike version-identity drift, dropped passthrough headers are +/// a capability limit, and a plaintext-only deployment against a MinIO-like +/// target must not turn red. +#[tokio::test] +#[serial] +async fn test_replication_check_flags_ssec_passthrough_dropping_target() -> TestResult { + init_logging(); + + let target = FakeS3Target::start().await?; + let target_bucket = "ssec-check-dst"; + target.create_bucket(target_bucket); + target.drop_unlisted_replication_headers(true); + + let mut source_env = RustFSTestEnvironment::new().await?; + let mut env_vars = replication_fast_env(); + env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV); + env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]); + source_env.start_rustfs_server_with_env(vec![], &env_vars).await?; + + let source_bucket = "ssec-check-src"; + let source_client = source_env.create_s3_client(); + source_client.create_bucket().bucket(source_bucket).send().await?; + enable_bucket_versioning(&source_env, source_bucket).await?; + let target_arn = set_replication_target_with_options( + &source_env, + source_bucket, + ReplicationTargetOptions { + endpoint: &target.address(), + access_key: FAKE_ACCESS_KEY, + secret_key: FAKE_SECRET_KEY, + target_bucket, + secure: false, + skip_tls_verify: false, + ca_cert_pem: None, + }, + ) + .await?; + put_bucket_replication(&source_env, source_bucket, &target_arn).await?; + + let response = run_replication_check(&source_env, source_bucket).await?; + assert_eq!(response.status(), StatusCode::OK); + let payload: serde_json::Value = response.json().await?; + + assert_eq!( + payload["Status"], "OK", + "a capability-only SSE-C failure must not fail the check overall: {payload}" + ); + let target_report = &payload["Targets"][0]; + assert_eq!(target_report["Status"], "OK", "{payload}"); + let ssec = &target_report["Phases"]["SsecPassthrough"]; + assert_eq!(ssec["Status"], "FAILED", "SsecPassthrough phase must fail: {payload}"); + assert_eq!( + ssec["Code"], "BucketRemoteSsecPassthroughUnsupported", + "the failure must carry the machine-readable code: {payload}" + ); + // Basic replication of plaintext objects works on this target: every other + // phase passes, so the code is the discriminator operators branch on. + assert_eq!(target_report["Phases"]["Put"]["Status"], "OK", "{payload}"); + assert_eq!(target_report["Phases"]["VersionFidelity"]["Status"], "OK", "{payload}"); + assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "OK", "{payload}"); + assert_eq!(target_report["Phases"]["VersionDelete"]["Status"], "OK", "{payload}"); + assert_eq!(target_report["Phases"]["Cleanup"]["Status"], "OK", "{payload}"); + + // The SSE-C probe PUT must have shipped the real transport header names — + // a mangled or missing header set would fail the phase for the wrong + // reason and mask a working target. + let requests = target.requests(); + assert!( + requests + .iter() + .any(|record| record.operation == FakeTargetOperation::PutObject && record.proxy_headers.ssec_transport_present), + "the SSE-C probe PUT must carry the X-Rustfs-Replication-* transport headers; journal: {requests:?}" + ); + + // No probe residue, including the SSE-C probe version. + let probe_put = requests + .into_iter() + .find(|record| record.operation == FakeTargetOperation::PutObject) + .ok_or("the probe PUT never reached the fake target")?; + let probe_key = probe_put.key.ok_or("probe PUT journal record has no key")?; + assert!( + target.stored_versions(target_bucket, &probe_key).is_empty(), + "all probe versions must be cleaned up" + ); + + target.shutdown().await; + Ok(()) +} + +/// C1 (backlog#1675 P1-22): heal-path convergence for SSE-C. An SSE-C object +/// whose live replication failed during a target outage must converge through +/// the scanner/heal compensation once the target returns — passing the N2 +/// HEAD-back audit against the recovered RustFS target — and the replica must +/// be readable with the customer key. +#[tokio::test] +#[serial] +async fn test_bucket_replication_sse_c_heals_after_target_outage() -> TestResult { + init_logging(); + + let (source_env, mut target_env, source_bucket, target_bucket) = + build_sse_replication_pair("ssec-heal", false, false).await?; + let source_client = source_env.create_s3_client(); + let key = "ssec-heal-contract.txt"; + let body = b"repl-22 ssec heal payload".to_vec(); + let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY); + let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY); + + // Target outage: the SSE-C write cannot replicate. + target_env.stop_server(); + + source_client + .put_object() + .bucket(&source_bucket) + .key(key) + .body(ByteStream::from(body.clone())) + .sse_customer_algorithm("AES256") + .sse_customer_key(&customer_key) + .sse_customer_key_md5(&customer_key_md5) + .send() + .await?; + + // The failure is observable on the source (SSE-C HEAD needs the key). + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let head = source_client + .head_object() + .bucket(&source_bucket) + .key(key) + .sse_customer_algorithm("AES256") + .sse_customer_key(&customer_key) + .sse_customer_key_md5(&customer_key_md5) + .send() + .await?; + match head.replication_status().map(|status| status.as_str()) { + Some("PENDING") | Some("FAILED") => break, + other => { + if tokio::time::Instant::now() >= deadline { + return Err(format!("source SSE-C object never reported PENDING/FAILED; last status={other:?}").into()); + } + sleep(Duration::from_millis(200)).await; + } + } + } + + // Recover the target in place; the source scanner re-drives the failure. + target_env + .restart_server_preserving_data(vec![], &[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]) + .await?; + + wait_for_source_replication_status(&source_client, &source_bucket, key, "COMPLETED", true).await?; + + // The healed replica is a REPLICA (status surfaces on HEAD) readable with + // the customer key. + let target_client = target_env.create_s3_client(); + let replica_head = target_client + .head_object() + .bucket(&target_bucket) + .key(key) + .sse_customer_algorithm("AES256") + .sse_customer_key(&customer_key) + .sse_customer_key_md5(&customer_key_md5) + .send() + .await?; + assert_eq!( + replica_head.replication_status().map(|status| status.as_str()), + Some("REPLICA"), + "the healed copy must carry REPLICA status" + ); + let replica = target_client + .get_object() + .bucket(&target_bucket) + .key(key) + .sse_customer_algorithm("AES256") + .sse_customer_key(&customer_key) + .sse_customer_key_md5(&customer_key_md5) + .send() + .await?; + assert_eq!(replica.sse_customer_algorithm(), Some("AES256")); + assert_eq!(replica.body.collect().await?.into_bytes().as_ref(), body.as_slice()); + + Ok(()) +} + +/// C1 (backlog#1675 P1-22): existing-object resync for SSE-C. An SSE-C object +/// written BEFORE any replication config must reach the RustFS target through +/// the existing-object resync (`replicate_all` transport, N2-audited), land as +/// a REPLICA, and read back with the customer key. +#[tokio::test] +#[serial] +async fn test_bucket_replication_sse_c_existing_object_resync() -> TestResult { + init_logging(); + + let mut source_env = RustFSTestEnvironment::new().await?; + let mut source_process_env = replication_fast_env(); + source_process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV); + source_process_env.extend_from_slice(FAST_SCANNER_ENV); + source_process_env.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]); + source_env.start_rustfs_server_with_env(vec![], &source_process_env).await?; + + let mut target_env = RustFSTestEnvironment::new().await?; + target_env + .start_rustfs_server_without_cleanup_with_env(&[ + ("NO_PROXY", "127.0.0.1,localhost"), + ("HTTP_PROXY", ""), + ("HTTPS_PROXY", ""), + ]) + .await?; + + let source_bucket = "ssec-existing-src"; + let target_bucket = "ssec-existing-dst"; + let source_client = source_env.create_s3_client(); + let target_client = target_env.create_s3_client(); + source_client.create_bucket().bucket(source_bucket).send().await?; + target_client.create_bucket().bucket(target_bucket).send().await?; + enable_bucket_versioning(&source_env, source_bucket).await?; + enable_bucket_versioning(&target_env, target_bucket).await?; + + // The SSE-C object exists before any replication wiring. + let key = "ssec-existing-contract.txt"; + let body = b"repl-22 ssec existing-object payload".to_vec(); + let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY); + let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY); + source_client + .put_object() + .bucket(source_bucket) + .key(key) + .body(ByteStream::from(body.clone())) + .sse_customer_algorithm("AES256") + .sse_customer_key(&customer_key) + .sse_customer_key_md5(&customer_key_md5) + .send() + .await?; + + // Wire replication (existing-object enabled) and drive a resync. + let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?; + put_bucket_replication(&source_env, source_bucket, &target_arn).await?; + let (reset_arn, reset_id) = start_bucket_replication_reset(&source_env, source_bucket).await?; + assert_eq!(reset_arn, target_arn); + let terminal = wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |status| { + status.reset_id == reset_id && matches!(status.status.as_str(), "Completed" | "Failed") + }) + .await?; + assert_eq!(terminal.status, "Completed", "SSE-C existing-object resync must complete"); + assert!(terminal.replicated_count >= 1, "the existing SSE-C object must have been resynced"); + + // The replica is a REPLICA (status surfaces on HEAD) readable with the + // customer key. + let replica_head = target_client + .head_object() + .bucket(target_bucket) + .key(key) + .sse_customer_algorithm("AES256") + .sse_customer_key(&customer_key) + .sse_customer_key_md5(&customer_key_md5) + .send() + .await?; + assert_eq!( + replica_head.replication_status().map(|status| status.as_str()), + Some("REPLICA"), + "the resynced copy must carry REPLICA status" + ); + let replica = target_client + .get_object() + .bucket(target_bucket) + .key(key) + .sse_customer_algorithm("AES256") + .sse_customer_key(&customer_key) + .sse_customer_key_md5(&customer_key_md5) + .send() + .await?; + assert_eq!(replica.sse_customer_algorithm(), Some("AES256")); + assert_eq!(replica.body.collect().await?.into_bytes().as_ref(), body.as_slice()); + + // No plaintext leak: the replica stays unreadable without the key. + assert!( + target_client + .get_object() + .bucket(target_bucket) + .key(key) + .send() + .await + .is_err(), + "SSE-C replica must not be readable without the customer key" + ); + + Ok(()) +} + /// backlog#1147 repl-17 / backlog#1783: SSE-S3 objects replicate by decrypting /// at the source and re-encrypting on the target with the target's own KMS. /// The property backlog#1291 pinned — never a silent plaintext replica — still @@ -8348,3 +8755,304 @@ async fn test_scanner_never_compensates_when_existing_object_replication_disable Ok(()) } + +/// Shared setup for the P1-5 read-proxy scenarios (backlog#1675): a RustFS +/// source with an enabled replication rule pointing at the fake target, and +/// an object seeded DIRECTLY on the target — it exists remotely but not +/// locally, exactly the active-active replication-lag window the read proxy +/// serves. +async fn start_read_proxy_lab( + source_bucket: &str, + target_bucket: &str, +) -> Result<(FakeS3Target, RustFSTestEnvironment, Client, Client), Box> { + let target = FakeS3Target::start().await?; + target.create_bucket(target_bucket); + target.assign_own_version_ids(true); + + let mut source_env = RustFSTestEnvironment::new().await?; + let mut process_env = replication_fast_env(); + process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV); + process_env.extend_from_slice(&[ + ("NO_PROXY", "127.0.0.1,localhost"), + ("HTTP_PROXY", ""), + ("HTTPS_PROXY", ""), + ("RUST_LOG", "error"), + ]); + source_env.start_rustfs_server_with_env(vec![], &process_env).await?; + + let source_client = source_env.create_s3_client(); + source_client.create_bucket().bucket(source_bucket).send().await?; + enable_bucket_versioning(&source_env, source_bucket).await?; + let target_arn = set_replication_target_with_options( + &source_env, + source_bucket, + ReplicationTargetOptions { + endpoint: &target.address(), + access_key: FAKE_ACCESS_KEY, + secret_key: FAKE_SECRET_KEY, + target_bucket, + secure: false, + skip_tls_verify: false, + ca_cert_pem: None, + }, + ) + .await?; + put_bucket_replication(&source_env, source_bucket, &target_arn).await?; + + let target_client = Client::from_conf(crate::common::build_test_s3_config( + target.endpoint(), + FAKE_ACCESS_KEY, + FAKE_SECRET_KEY, + None, + "read-proxy-e2e", + )); + + Ok((target, source_env, source_client, target_client)) +} + +/// P1-5 (backlog#1675): during the active-active replication lag window a +/// GET/HEAD for an object the local site does not have yet is proxied to the +/// replication target. Pins the wire contract: the anti-loop +/// `source-proxy-request` marker is sent, the replication worker's +/// `source-replication-check` SSE-C exemption is NEVER sent, client SSE-C +/// headers are forwarded verbatim, and an inbound request that was itself +/// proxied is answered locally (404) without touching the target. +#[tokio::test] +#[serial] +async fn test_get_and_head_proxy_unreplicated_object_to_replication_target() -> TestResult { + init_logging(); + + let source_bucket = "proxy-read-src"; + let target_bucket = "proxy-read-dst"; + let (target, source_env, source_client, target_client) = start_read_proxy_lab(source_bucket, target_bucket).await?; + + let payload = b"proxy payload".to_vec(); + target_client + .put_object() + .bucket(target_bucket) + .key("proxy-only") + .body(ByteStream::from(payload.clone())) + .send() + .await?; + target.take_requests(); + + // a. GET of the locally-missing object is served through the proxy. + let got = source_client + .get_object() + .bucket(source_bucket) + .key("proxy-only") + .send() + .await + .map_err(|err| format!("proxied GET failed: {}", err.into_service_error()))?; + assert_eq!(got.content_length, Some(payload.len() as i64)); + let body = got.body.collect().await?.into_bytes(); + assert_eq!(body.as_ref(), payload.as_slice(), "proxied GET must stream the target's body"); + + let get_record = target + .requests() + .into_iter() + .find(|record| record.operation == FakeTargetOperation::GetObject && record.key.as_deref() == Some("proxy-only")) + .ok_or("fake target never received the proxied GET")?; + assert_eq!( + get_record.proxy_headers.source_proxy_request.as_deref(), + Some("true"), + "proxied GET must carry the anti-loop source-proxy-request marker" + ); + assert!( + get_record.proxy_headers.replication_check.is_none(), + "proxied GET must never carry the replication worker's source-replication-check exemption" + ); + assert!( + get_record.proxy_headers.ssec_algorithm.is_none() && !get_record.proxy_headers.ssec_key_present, + "no client SSE-C headers were sent, so none may be forwarded" + ); + + // a2. Client SSE-C headers travel verbatim to the target (the target owns + // the real SSE-C decryption; the plaintext fake simply ignores them). + target.take_requests(); + let ssec_key = "01234567890123456789012345678901"; + let ssec_key_b64 = BASE64_STANDARD.encode(ssec_key); + let ssec_key_md5 = sse_customer_key_md5_base64(ssec_key); + let _ = source_client + .get_object() + .bucket(source_bucket) + .key("proxy-only") + .sse_customer_algorithm("AES256") + .sse_customer_key(&ssec_key_b64) + .sse_customer_key_md5(&ssec_key_md5) + .send() + .await + .map_err(|err| format!("proxied SSE-C GET failed: {}", err.into_service_error()))?; + let ssec_record = target + .requests() + .into_iter() + .find(|record| record.operation == FakeTargetOperation::GetObject && record.key.as_deref() == Some("proxy-only")) + .ok_or("fake target never received the proxied SSE-C GET")?; + assert_eq!(ssec_record.proxy_headers.ssec_algorithm.as_deref(), Some("AES256")); + assert!(ssec_record.proxy_headers.ssec_key_present, "SSE-C key header must be forwarded verbatim"); + assert_eq!(ssec_record.proxy_headers.ssec_key_md5.as_deref(), Some(ssec_key_md5.as_str())); + assert!(ssec_record.proxy_headers.replication_check.is_none()); + + // b. HEAD of the locally-missing object is served through the proxy. + target.take_requests(); + let head = source_client + .head_object() + .bucket(source_bucket) + .key("proxy-only") + .send() + .await + .map_err(|err| format!("proxied HEAD failed: {}", err.into_service_error()))?; + assert_eq!(head.content_length, Some(payload.len() as i64)); + let head_record = target + .requests() + .into_iter() + .find(|record| record.operation == FakeTargetOperation::HeadObject && record.key.as_deref() == Some("proxy-only")) + .ok_or("fake target never received the proxied HEAD")?; + assert_eq!(head_record.proxy_headers.source_proxy_request.as_deref(), Some("true")); + assert!(head_record.proxy_headers.replication_check.is_none()); + + // c. Anti-loop: an inbound request that already carries the proxy marker + // is answered locally with 404 and never forwarded to the target. + target.take_requests(); + let err = source_client + .get_object() + .bucket(source_bucket) + .key("proxy-only") + .customize() + .mutate_request(|req| { + req.headers_mut().insert("x-minio-source-proxy-request", "true"); + }) + .send() + .await + .expect_err("anti-loop GET must fail locally instead of proxying"); + let service_err = err.into_service_error(); + assert!(service_err.is_no_such_key(), "anti-loop GET must 404, got: {service_err}"); + assert!( + !target + .requests() + .iter() + .any(|record| record.operation == FakeTargetOperation::GetObject), + "anti-loop GET must not reach the replication target; journal: {:?}", + target.requests() + ); + + // c2. MinIO ProxyHeaderSet parity: the header's mere PRESENCE disables + // proxying — "false" is exactly what a peer's replication worker sends on + // its convergence HEADs, and proxying that miss back would fake + // convergence. + target.take_requests(); + let err = source_client + .get_object() + .bucket(source_bucket) + .key("proxy-only") + .customize() + .mutate_request(|req| { + req.headers_mut().insert("x-minio-source-proxy-request", "false"); + }) + .send() + .await + .expect_err("proxy-header-set GET must fail locally instead of proxying"); + let service_err = err.into_service_error(); + assert!(service_err.is_no_such_key(), "proxy-header-set GET must 404, got: {service_err}"); + assert!( + !target + .requests() + .iter() + .any(|record| record.operation == FakeTargetOperation::GetObject), + "proxy-header-set GET must not reach the replication target; journal: {:?}", + target.requests() + ); + + // d. The replication worker's own convergence HEAD against the target + // must carry `source-proxy-request: false` (never proxied back) and the + // replication-check exemption. Trigger real replication and inspect the + // fake journal. + target.take_requests(); + source_client + .put_object() + .bucket(source_bucket) + .key("worker-replicated") + .body(ByteStream::from_static(b"worker payload")) + .send() + .await?; + wait_for_target_request_version_id(&target, FakeTargetOperation::PutObject, "worker-replicated").await?; + let worker_head = target + .requests() + .into_iter() + .find(|record| record.operation == FakeTargetOperation::HeadObject && record.key.as_deref() == Some("worker-replicated")) + .ok_or_else(|| format!("replication worker never HEAD-ed the target; journal: {:?}", target.requests()))?; + assert_eq!( + worker_head.proxy_headers.source_proxy_request.as_deref(), + Some("false"), + "worker convergence HEAD must send source-proxy-request: false so the target answers locally" + ); + assert_eq!( + worker_head.proxy_headers.replication_check.as_deref(), + Some("true"), + "worker convergence HEAD keeps the replication-check exemption" + ); + + drop(source_env); + target.shutdown().await; + Ok(()) +} + +/// P1-5 (backlog#1675): GetObjectTagging for an object missing locally is +/// proxied to the replication target with the anti-loop marker, mirroring +/// MinIO `proxyGetTaggingToRepTarget`. +#[tokio::test] +#[serial] +async fn test_get_object_tagging_proxies_unreplicated_object_to_replication_target() -> TestResult { + init_logging(); + + let source_bucket = "proxy-tag-src"; + let target_bucket = "proxy-tag-dst"; + let (target, source_env, source_client, target_client) = start_read_proxy_lab(source_bucket, target_bucket).await?; + + target_client + .put_object() + .bucket(target_bucket) + .key("proxy-tagged") + .body(ByteStream::from_static(b"tagged payload")) + .send() + .await?; + target_client + .put_object_tagging() + .bucket(target_bucket) + .key("proxy-tagged") + .tagging( + aws_sdk_s3::types::Tagging::builder() + .tag_set(aws_sdk_s3::types::Tag::builder().key("team").value("storage").build()?) + .build()?, + ) + .send() + .await?; + target.take_requests(); + + let tags = source_client + .get_object_tagging() + .bucket(source_bucket) + .key("proxy-tagged") + .send() + .await + .map_err(|err| format!("proxied GetObjectTagging failed: {}", err.into_service_error()))?; + assert_eq!(tags.tag_set.len(), 1, "proxied tagging read must return the target's tags"); + assert_eq!(tags.tag_set[0].key.as_str(), "team"); + assert_eq!(tags.tag_set[0].value.as_str(), "storage"); + + let record = target + .requests() + .into_iter() + .find(|record| record.operation == FakeTargetOperation::GetObjectTagging && record.key.as_deref() == Some("proxy-tagged")) + .ok_or("fake target never received the proxied GetObjectTagging")?; + assert_eq!( + record.proxy_headers.source_proxy_request.as_deref(), + Some("true"), + "proxied tagging read must carry the anti-loop marker" + ); + assert!(record.proxy_headers.replication_check.is_none()); + + drop(source_env); + target.shutdown().await; + Ok(()) +} diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 3a031d60a..17fffac3d 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -32,7 +32,7 @@ pub mod bucket { pub mod bucket_target_sys { pub use crate::bucket::bucket_target_sys::{ AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, - TargetClient, append_version_id_query, + SsecPassthroughCapability, TargetClient, append_version_id_query, }; } @@ -198,12 +198,13 @@ pub mod bucket { ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus, VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool, - get_global_replication_stats, init_background_replication, invalid_replication_config_status_field, - persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, - replication_statuses_map, replication_target_arns, resync_start_conflict_id, should_remove_replication_target, - should_schedule_delete_replication, should_use_existing_delete_replication_info, - should_use_existing_delete_replication_source, unsupported_replication_config_field, - validate_replication_config_structure, validate_replication_config_target_arns, version_purge_status_to_filemeta, + get_global_replication_stats, get_proxy_targets, init_background_replication, + invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog, + replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns, + resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication, + should_use_existing_delete_replication_info, should_use_existing_delete_replication_source, + unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns, + version_purge_status_to_filemeta, }; } diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index 00cf124f1..ef1918648 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -27,10 +27,15 @@ use aws_sdk_s3::config::SharedHttpClient; use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::error::SdkError; use aws_sdk_s3::operation::complete_multipart_upload::CompleteMultipartUploadOutput; +use aws_sdk_s3::operation::delete_object_tagging::{DeleteObjectTaggingError, DeleteObjectTaggingOutput}; +use aws_sdk_s3::operation::get_object::{GetObjectError, GetObjectOutput}; +use aws_sdk_s3::operation::get_object_tagging::{GetObjectTaggingError, GetObjectTaggingOutput}; use aws_sdk_s3::operation::head_bucket::HeadBucketError; use aws_sdk_s3::operation::head_object::HeadObjectError; +use aws_sdk_s3::operation::put_object_tagging::{PutObjectTaggingError, PutObjectTaggingOutput}; use aws_sdk_s3::operation::upload_part::UploadPartOutput; use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::Tagging as SdkTagging; use aws_sdk_s3::types::{ ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode, }; @@ -57,8 +62,8 @@ use rustfs_utils::http::{ is_rustfs_header, is_standard_header, is_storageclass_header, }; use rustfs_utils::http::{ - SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK, - SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST, + SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST, + SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID, insert_header, }; @@ -294,9 +299,41 @@ struct TargetClientBuildProbe { release: Arc, } +/// SSE-C passthrough capability verdicts (see the enum's own docs in +/// `rustfs-replication`) are cached here per target ARN: entries follow the +/// `arn_remotes_map` lifecycle (rebuilding or removing a target resets its +/// capability to `Unknown`) and additionally expire after +/// [`SSEC_PASSTHROUGH_CAPABILITY_TTL`], after which the next attempt +/// re-audits. Re-exported so existing `bucket_target_sys` consumers keep +/// their import path while the verdict vocabulary lives with the +/// replication decision logic. +pub use crate::bucket::replication::SsecPassthroughCapability; + +/// How long an audited SSE-C passthrough verdict stays authoritative. +/// +/// Trade-off: without a TTL a verdict is sticky for the process lifetime — +/// an `Unsupported` target that gets upgraded (or re-probed only via +/// replication-check) would keep failing SSE-C replication forever, and the +/// fail-open twin: a `Supported` verdict would outlive a backend swapped +/// behind the same endpoint/ARN. With the TTL, a bad target costs at most +/// one wasted PUT+HEAD audit per TTL window, and a changed backend is +/// re-discovered within the same window. +pub const SSEC_PASSTHROUGH_CAPABILITY_TTL: Duration = Duration::from_secs(10 * 60); + +/// A recorded SSE-C passthrough verdict plus when it was recorded, so reads +/// can report staleness against [`SSEC_PASSTHROUGH_CAPABILITY_TTL`]. +#[derive(Debug, Clone, Copy)] +struct SsecPassthroughRecord { + capability: SsecPassthroughCapability, + recorded_at: Instant, +} + #[derive(Debug, Default)] pub struct BucketTargetSys { pub arn_remotes_map: Arc>>, + /// SSE-C passthrough capability verdicts keyed by target ARN. See + /// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`. + ssec_passthrough_map: Arc>>, pub targets_map: Arc>>>, pub h_mutex: Arc>>, target_h_mutex: Arc>>, @@ -317,6 +354,7 @@ impl BucketTargetSys { fn new() -> Self { Self { arn_remotes_map: Arc::new(RwLock::new(HashMap::new())), + ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())), targets_map: Arc::new(RwLock::new(HashMap::new())), h_mutex: Arc::new(RwLock::new(HashMap::new())), target_h_mutex: Arc::new(RwLock::new(HashMap::new())), @@ -580,19 +618,59 @@ impl BucketTargetSys { let update_mutex = self.target_update_mutex(bucket).await; let _update_guard = update_mutex.lock().await; - // Lock order: targets_map, then arn_remotes_map, then target_h_mutex. + // Lock order: targets_map, then arn_remotes_map, then target_h_mutex, + // then ssec_passthrough_map (always last; also taken standalone by the + // capability accessors). let mut targets_map = self.targets_map.write().await; let mut arn_remotes_map = self.arn_remotes_map.write().await; let mut health_map = self.target_h_mutex.write().await; if let Some(targets) = targets_map.remove(bucket) { + let mut ssec_map = self.ssec_passthrough_map.write().await; for target in targets { arn_remotes_map.remove(&target.arn); health_map.remove(&target.arn); + ssec_map.remove(&target.arn); } } } + /// Cached SSE-C passthrough capability for a target ARN, plus whether the + /// verdict is older than [`SSEC_PASSTHROUGH_CAPABILITY_TTL`]. `(Unknown, + /// false)` when no verdict has been recorded since the target was built. + /// Staleness is computed here so the gate policy stays a pure function. + pub async fn ssec_passthrough_capability(&self, arn: &str) -> (SsecPassthroughCapability, bool) { + match self.ssec_passthrough_map.read().await.get(arn) { + Some(record) => (record.capability, record.recorded_at.elapsed() >= SSEC_PASSTHROUGH_CAPABILITY_TTL), + None => (SsecPassthroughCapability::Unknown, false), + } + } + + /// Record an audited SSE-C passthrough verdict for a target ARN. Written by + /// the replication worker's HEAD-back audit and by the replication-check + /// SsecPassthrough probe phase. + pub async fn record_ssec_passthrough_capability(&self, arn: &str, capability: SsecPassthroughCapability) { + self.ssec_passthrough_map.write().await.insert( + arn.to_string(), + SsecPassthroughRecord { + capability, + recorded_at: Instant::now(), + }, + ); + } + + /// Test hook: age an existing verdict so TTL expiry is observable without + /// waiting out the real window. + #[cfg(test)] + pub(crate) async fn backdate_ssec_passthrough_capability(&self, arn: &str, age: Duration) { + let backdated = Instant::now() + .checked_sub(age) + .expect("system uptime must exceed the backdate age"); + if let Some(record) = self.ssec_passthrough_map.write().await.get_mut(arn) { + record.recorded_at = backdated; + } + } + pub async fn set_target( &self, bucket: &str, @@ -948,15 +1026,21 @@ impl BucketTargetSys { } } - // Lock order: targets_map, then arn_remotes_map, then target_h_mutex. + // Lock order: targets_map, then arn_remotes_map, then target_h_mutex, + // then ssec_passthrough_map (always last; also taken standalone by the + // capability accessors). let mut targets_map = self.targets_map.write().await; let mut arn_remotes_map = self.arn_remotes_map.write().await; let mut health_map = self.target_h_mutex.write().await; // Remove existing targets if let Some(existing_targets) = targets_map.remove(bucket) { + let mut ssec_map = self.ssec_passthrough_map.write().await; for target in existing_targets { arn_remotes_map.remove(&target.arn); health_map.remove(&target.arn); + // A rebuilt/edited target may point at a different service: + // the SSE-C passthrough verdict must be re-audited from Unknown. + ssec_map.remove(&target.arn); self.update_bandwidth_limit(bucket, &target.arn, 0); } } @@ -1446,6 +1530,43 @@ fn resolve_put_api_version_id(source_version_id: &str) -> Option<&str> { } } +/// Resolve the S3 `versionId` for a proxied read against a remote target. +/// RustFS represents the null version internally as the nil UUID while the S3 +/// API addresses it as the literal "null" (same mapping as +/// [`resolve_put_api_version_id`]); empty means "no version requested". +pub(crate) fn resolve_read_api_version_id(version_id: Option) -> Option { + let version_id = version_id?; + let trimmed = version_id.trim(); + if trimmed.is_empty() { + None + } else if Uuid::parse_str(trimmed).is_ok_and(|uuid| uuid.is_nil()) { + Some(rustfs_filemeta::NULL_VERSION_ID.to_string()) + } else { + Some(trimmed.to_string()) + } +} + +/// Outbound header set for a proxied read: the caller-provided passthrough +/// headers (client SSE-C key family, conditional headers) plus the anti-loop +/// `source-proxy-request` marker in both the x-rustfs- and x-minio- prefixes +/// (a MinIO target only understands the latter). Never adds +/// `source-replication-check`: that exemption channel belongs exclusively to +/// the replication worker's HEAD. +fn proxy_outbound_headers(mut extra_headers: HeaderMap) -> HeaderMap { + insert_header(&mut extra_headers, SUFFIX_SOURCE_PROXY_REQUEST, "true"); + extra_headers +} + +/// Copy `headers` onto an SDK request inside `customize().map_request` (runs +/// before signing, so the headers join the SigV4 canonical request). +fn apply_extra_headers(mut req: HttpRequest, headers: &HeaderMap) -> Result { + for (k, v) in headers.iter() { + req.headers_mut() + .insert(k.as_str().to_string(), v.to_str().unwrap_or("").to_string()); + } + Ok(req) +} + /// Append `versionId=` to an already-built request URI. aws-sdk-s3's /// `PutObjectInput` / `CreateMultipartUploadInput` expose no version id /// member, so the query is spliced in via `map_request`, which runs at @@ -1853,6 +1974,13 @@ impl TargetClient { // worker cannot hold; otherwise SSE-C replicas never converge on HEAD. let mut headers = HeaderMap::new(); insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true"); + // `source-proxy-request: false` (MinIO `ProxyHeaderSet` semantics): + // the header's mere presence tells the receiver to answer LOCALLY + // instead of proxying the miss back to us. Without it, a not-found on + // the target gets read-proxied back to this source, echoes the source + // object with an identical ETag, and the worker concludes the object + // already converged — so it never actually replicates it. + insert_header(&mut headers, SUFFIX_SOURCE_PROXY_REQUEST, "false"); match self .client .head_object() @@ -1877,6 +2005,129 @@ impl TargetClient { } } + /// HEAD used by the read-proxy path (GET/HEAD of an object not yet + /// replicated locally, MinIO `proxyHeadToRepTarget`). + /// + /// Deliberately different from [`TargetClient::head_object`]: it must NOT + /// send `source-replication-check` — that header is the replication + /// worker's SSE-C metadata exemption channel. A proxied client request + /// instead forwards the client's own SSE-C headers (`extra_headers`) so + /// the target performs the real SSE-C validation/decryption. The + /// `source-proxy-request` marker is always added so the target does not + /// proxy the request onward (anti-loop). + pub async fn head_object_for_proxy( + &self, + bucket: &str, + object: &str, + version_id: Option, + range: Option, + part_number: Option, + extra_headers: HeaderMap, + ) -> Result> { + let headers = proxy_outbound_headers(extra_headers); + self.client + .head_object() + .bucket(bucket) + .key(object) + .set_version_id(resolve_read_api_version_id(version_id)) + .set_range(range) + .set_part_number(part_number) + .customize() + .map_request(move |req| apply_extra_headers(req, &headers)) + .send() + .await + } + + /// GET used by the read-proxy path (MinIO `proxyGetToReplicationTarget`). + /// Returns the streaming SDK output; callers must forward the body without + /// buffering it. Same header contract as [`Self::head_object_for_proxy`]: + /// anti-loop marker on, replication-check never sent, client SSE-C / + /// conditional headers forwarded verbatim via `extra_headers`. + pub async fn get_object( + &self, + bucket: &str, + object: &str, + version_id: Option, + range: Option, + part_number: Option, + extra_headers: HeaderMap, + ) -> Result> { + let headers = proxy_outbound_headers(extra_headers); + self.client + .get_object() + .bucket(bucket) + .key(object) + .set_version_id(resolve_read_api_version_id(version_id)) + .set_range(range) + .set_part_number(part_number) + .customize() + .map_request(move |req| apply_extra_headers(req, &headers)) + .send() + .await + } + + /// GetObjectTagging for the tagging read-proxy path + /// (MinIO `proxyGetTaggingToRepTarget`). Anti-loop marker always added. + pub async fn get_object_tagging( + &self, + bucket: &str, + object: &str, + version_id: Option, + ) -> Result> { + let headers = proxy_outbound_headers(HeaderMap::new()); + self.client + .get_object_tagging() + .bucket(bucket) + .key(object) + .set_version_id(resolve_read_api_version_id(version_id)) + .customize() + .map_request(move |req| apply_extra_headers(req, &headers)) + .send() + .await + } + + /// PutObjectTagging for the tagging proxy path + /// (MinIO `proxyTaggingToRepTarget`). Anti-loop marker always added. + pub async fn put_object_tagging( + &self, + bucket: &str, + object: &str, + version_id: Option, + tagging: SdkTagging, + ) -> Result> { + let headers = proxy_outbound_headers(HeaderMap::new()); + self.client + .put_object_tagging() + .bucket(bucket) + .key(object) + .set_version_id(resolve_read_api_version_id(version_id)) + .tagging(tagging) + .customize() + .map_request(move |req| apply_extra_headers(req, &headers)) + .send() + .await + } + + /// DeleteObjectTagging for the tagging proxy path + /// (MinIO `proxyTaggingToRepTarget`). Anti-loop marker always added. + pub async fn delete_object_tagging( + &self, + bucket: &str, + object: &str, + version_id: Option, + ) -> Result> { + let headers = proxy_outbound_headers(HeaderMap::new()); + self.client + .delete_object_tagging() + .bucket(bucket) + .key(object) + .set_version_id(resolve_read_api_version_id(version_id)) + .customize() + .map_request(move |req| apply_extra_headers(req, &headers)) + .send() + .await + } + /// On success returns the version id the target assigned (from /// `x-amz-version-id`), letting callers audit the version-identity /// contract — a target that adopts the source version echoes it back. @@ -2506,6 +2757,57 @@ mod tests { assert_eq!(health.last_online, Some(now)); } + /// N2 TTL contract, both flip directions: a recorded verdict is fresh + /// until [`SSEC_PASSTHROUGH_CAPABILITY_TTL`], then reads as expired; a + /// re-audit that records the OPPOSITE verdict replaces it as fresh. The + /// worker gate maps expired verdicts to ProceedWithAudit (pinned in + /// `replication_target_boundary`), so together this proves an Unsupported + /// target recovers to Supported through the audit once its verdict ages + /// out — and a stale Supported one is re-proven rather than trusted. + #[tokio::test] + async fn ssec_passthrough_capability_ttl_expires_and_reaudit_flips_verdict() { + let sys = BucketTargetSys::default(); + let arn = "arn:rustfs:replication:us-east-1:bucket:ssec-ttl"; + let expired_age = SSEC_PASSTHROUGH_CAPABILITY_TTL + Duration::from_secs(1); + + assert_eq!( + sys.ssec_passthrough_capability(arn).await, + (SsecPassthroughCapability::Unknown, false), + "an unrecorded target must read Unknown and never expired" + ); + + sys.record_ssec_passthrough_capability(arn, SsecPassthroughCapability::Unsupported) + .await; + assert_eq!( + sys.ssec_passthrough_capability(arn).await, + (SsecPassthroughCapability::Unsupported, false) + ); + + sys.backdate_ssec_passthrough_capability(arn, expired_age).await; + assert_eq!( + sys.ssec_passthrough_capability(arn).await, + (SsecPassthroughCapability::Unsupported, true), + "an aged-out Unsupported verdict must read expired so the gate re-audits" + ); + + // The re-audit against an upgraded target records Supported afresh. + sys.record_ssec_passthrough_capability(arn, SsecPassthroughCapability::Supported) + .await; + assert_eq!( + sys.ssec_passthrough_capability(arn).await, + (SsecPassthroughCapability::Supported, false), + "a fresh Supported verdict replaces the expired Unsupported one" + ); + + // And the fail-open twin: Supported also ages out. + sys.backdate_ssec_passthrough_capability(arn, expired_age).await; + assert_eq!( + sys.ssec_passthrough_capability(arn).await, + (SsecPassthroughCapability::Supported, true), + "an aged-out Supported verdict must read expired so the gate re-proves it" + ); + } + #[tokio::test] async fn list_targets_applies_health_stats_by_arn_and_preserves_endpoint_port() { let sys = BucketTargetSys::default(); diff --git a/crates/ecstore/src/bucket/replication/README.md b/crates/ecstore/src/bucket/replication/README.md index 68f43a32b..17e9ed7f6 100644 --- a/crates/ecstore/src/bucket/replication/README.md +++ b/crates/ecstore/src/bucket/replication/README.md @@ -11,9 +11,9 @@ paths. | Module | Current role | Split blocker | |---|---|---| | `config.rs` | Replication config helpers, rule matching, and tag filtering. | Uses replication-local filemeta/tagging boundaries and S3 DTOs directly. | -| `datatypes.rs` | ECStore compatibility re-export for resync status enums. | Re-exports `rustfs-replication` contracts while downstream facade consumers migrate. | | `replication_object_decision_boundary.rs` | Object replication option DTOs, resync target projection, delete replication decisions, and multipart planning helpers. | Keeps ECStore runtime modules from importing object decision contracts directly from `rustfs-replication`. | | `replication_pool.rs` | Replication queue, worker pool, MRF persistence, bucket stats, and delete/object scheduling. | Depends on bucket target sys, bucket metadata sys, metadata paths, queue contracts through the queue boundary, file metadata replication contracts through local boundaries, config storage, storage contracts through the replication storage boundary, runtime sources, and notification state. | +| `replication_proxy.rs` | Proxy-target selection for GET/HEAD/Tagging reads of objects not yet replicated locally (MinIO `getProxyTargets` parity: anti-loop, version-suspended, and no-config empty branches). | Uses replication config lookup, rule matching, and target clients through local boundaries. | | `replication_queue_boundary.rs` | Queue/admission DTOs, heal queue DTOs, worker sizing, and backpressure helpers. | Keeps ECStore runtime modules from importing queue/backpressure contracts directly from `rustfs-replication`. | | `replication_resync_boundary.rs` | Resync DTOs, status classifiers, persisted resync/MRF codec wrappers, and ECStore error mapping. | Keeps ECStore runtime modules from importing resync contract helpers directly from `rustfs-replication`. | | `replication_resyncer.rs` | Object replication, delete replication, resync execution, target calls, and multipart target upload paths. | Depends on target calls and target config types through the replication target boundary, metadata paths and metadata systems through the replication metadata boundary, file metadata replication contracts through the filemeta boundary, object decisions and multipart planning through the object decision boundary, resync contracts through the resync boundary, queue DTOs through the queue boundary, error contracts through the error boundary, versioning systems, storage contracts through the replication storage boundary, config-derived storage class labels through the config store, runtime sources, notification events and local event host selection through the event sink, bandwidth reader wrapping, and SetDisks lock timing. | @@ -117,9 +117,12 @@ Target end state: their file names — so batch-merging them beforehand is explicitly rejected: it forces synchronized guard-script/mod/import churn with zero functional gain; -- the only module that can retire early is `datatypes.rs`: delete it once its - facade consumers import the resync status enums through `rustfs-replication` - directly. +- `datatypes.rs` retired early (its sanctioned exception): it was a pure + relay (`boundary -> datatypes -> mod.rs`), so the facade now re-exports + `ResyncStatusType` from the resync boundary directly and the relay file is + deleted. Note the original retirement wording ("consumers import through + `rustfs-replication` directly") conflicted with Migration Rule #15 — + consumers stay behind the ECStore facade; only the relay hop dissolves. ## Milestones @@ -127,9 +130,9 @@ Target end state: |---|---|---| | M0 | Record the completion criteria and end state (this section). | Done | | M1 | Contract extraction: resync/queue/stats/object-decision/filemeta/storage wire contracts owned by `crates/replication`; ECStore imports concentrated in `*_boundary.rs`; event sink and runtime access behind local contracts. | Done — see Required Contracts | -| M2 | Move resyncer pure decision logic (no IO) into `crates/replication`. | Pending; sequence after splitting the oversized resyncer/pool functions (`resync_bucket`, `replicate_all`, `start_mrf_processor`) so moves stay mechanical | +| M2 | Move resyncer pure decision logic (no IO) into `crates/replication`. | Done — moved the pure decision helpers with their unit tests: `resync_status_duration` (resync), `resync_existing_delete_replication_info` / `replicate_delete_outcome` / `target_delete_version_id` / `delete_marker_purge_version_id` / `delete_marker_purge_mrf_entry` (delete), `version_identity_drifted` / `is_replication_target_offline_error` / the SSE-C passthrough gate family incl. `SsecPassthroughCapability` (object; `ssec_passthrough_evidence_present` was param-demoted to the echoed customer-algorithm string, ECStore keeps the `HeadObjectOutput` adapter). ECStore imports them through the resync/object-decision/target boundaries; `bucket_target_sys` keeps only the verdict cache + TTL and re-exports the capability enum. Not moved (signatures carry ECStore or aws-sdk types): `verify_resync_head_result`, `resync_target_error_detail`, the `SdkError` classifiers (`has_raw_status`, `is_version_id_format_mismatch`), the `replicate_all_*` option/info builders, and `bounded_resync_max_jobs` (itself a pure clamp, but it forms one local configuration unit with the env-reading `configured_resync_max_jobs` and its ECStore-local constants — moving the clamp alone has negative value). | | M3 | Move the worker runtime (`replication_pool.rs`, the IO paths of `replication_resyncer.rs`, `replication_state.rs`) once the contract traits are stable. Highest-risk step of the whole plan; do it last. | Pending | -| M4 | Retire the boundary modules together with their guard-script entries; delete `datatypes.rs`. | Pending | +| M4 | Retire the boundary modules together with their guard-script entries. | Pending (`datatypes.rs` already retired early alongside M2) | The original first code-bearing step (narrow `ReplicationEventSink` / `ReplicationRuntime` contracts) has landed — `replication_event_sink.rs` diff --git a/crates/ecstore/src/bucket/replication/datatypes.rs b/crates/ecstore/src/bucket/replication/datatypes.rs deleted file mode 100644 index 005a47966..000000000 --- a/crates/ecstore/src/bucket/replication/datatypes.rs +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub use super::replication_resync_boundary::ResyncStatusType; diff --git a/crates/ecstore/src/bucket/replication/mod.rs b/crates/ecstore/src/bucket/replication/mod.rs index d46f78eb3..be0de0121 100644 --- a/crates/ecstore/src/bucket/replication/mod.rs +++ b/crates/ecstore/src/bucket/replication/mod.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub mod datatypes; mod replication_bandwidth_boundary; mod replication_config_boundary; mod replication_config_store; @@ -29,6 +28,7 @@ mod replication_object_bridge; mod replication_object_config; mod replication_object_decision_boundary; pub(crate) mod replication_pool; +mod replication_proxy; mod replication_queue_boundary; mod replication_resync_boundary; mod replication_resyncer; @@ -43,7 +43,6 @@ pub(crate) mod replication_timing; mod replication_versioning_boundary; mod runtime_boundary; -pub use datatypes::ResyncStatusType; pub use replication_config_boundary::{ ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, @@ -74,13 +73,16 @@ pub use replication_pool::{ get_global_replication_pool, get_global_replication_stats, init_background_replication, persist_force_delete_intent, read_durable_mrf_backlog, resync_start_conflict_id, }; +pub use replication_proxy::get_proxy_targets; pub use replication_queue_boundary::{ DeletedObjectReplicationInfo, ReplicationBatchAdmission, ReplicationHealQueueResult, ReplicationOperation, ReplicationPriority, ReplicationQueueAdmission, }; +pub use replication_resync_boundary::ResyncStatusType; pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus}; pub use replication_scanner_bridge::ReplicationScannerBridge; pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog}; pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats}; pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage}; +pub use replication_target_boundary::SsecPassthroughCapability; pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge; diff --git a/crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs b/crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs index 5a89e0fb2..917ac65a8 100644 --- a/crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub(crate) use rustfs_filemeta::NULL_VERSION_ID; pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry}; pub(crate) use rustfs_replication::{ - REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos, - ReplicatedTargetInfo, ReplicationAction, ReplicationWorkerOperation, ResyncDecision, get_replication_state, - parse_replicate_decision, replicate_decision_for_admitted_targets, target_reset_header, version_purge_statuses_map, + REPLICATE_EXISTING, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, + ReplicationWorkerOperation, ResyncDecision, get_replication_state, parse_replicate_decision, + replicate_decision_for_admitted_targets, target_reset_header, version_purge_statuses_map, }; pub use rustfs_replication::{ REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationState, ReplicationStatusType, ReplicationType, diff --git a/crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs b/crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs index d6c4f1ceb..4360ed4ef 100644 --- a/crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs @@ -18,9 +18,10 @@ pub use rustfs_replication::{ should_use_existing_delete_replication_source, }; pub(crate) use rustfs_replication::{ - ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, - delete_replication_missing_source_decision, delete_replication_object_opts, heal_uses_delete_replication_path, - is_retryable_delete_replication_head_error, is_version_delete_replication, replication_etags_match, - replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_target_for_object, - should_retry_delete_marker_purge, + ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry, + delete_marker_purge_version_id, delete_replication_missing_source_decision, delete_replication_object_opts, + heal_uses_delete_replication_path, is_retryable_delete_replication_head_error, is_version_delete_replication, + replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size, + replication_multipart_part_plan, resync_existing_delete_replication_info, resync_target_for_object, + should_retry_delete_marker_purge, target_delete_version_id, }; diff --git a/crates/ecstore/src/bucket/replication/replication_pool.rs b/crates/ecstore/src/bucket/replication/replication_pool.rs index 58b3efdca..efe517b99 100644 --- a/crates/ecstore/src/bucket/replication/replication_pool.rs +++ b/crates/ecstore/src/bucket/replication/replication_pool.rs @@ -667,6 +667,368 @@ async fn acknowledge_mrf_recovery( Err(EcstoreError::PreconditionFailed) } +/// Acquires the MRF recovery leader lock for the startup replay. +/// Returns `None` (after logging) when the lock cannot be created or another +/// node is already processing the backlog. +async fn acquire_mrf_recovery_guard(storage: &Arc) -> Option { + let recovery_lock = match storage + .new_ns_lock( + ReplicationMetadataStore::rustfs_meta_bucket(), + ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK, + ) + .await + { + Ok(lock) => lock, + Err(error) => { + warn!( + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION, + error = %error, + "Failed to create the MRF recovery leader lock" + ); + return None; + } + }; + match recovery_lock + .get_write_lock_quiet(ReplicationLockTiming::acquire_timeout()) + .await + { + Ok(guard) => Some(guard), + Err(_) => { + debug!( + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION, + "Another node is already processing the MRF recovery backlog" + ); + None + } + } +} + +/// Reads and decodes the on-disk MRF recovery file. +/// Returns `None` when there is nothing to replay: missing file (publishes an +/// empty available summary), read failure, or corrupt data (quarantined). +async fn load_mrf_recovery_entries(storage: &Arc) -> Option> { + let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await { + Ok(d) => d, + Err(EcstoreError::ConfigNotFound) => { + set_durable_mrf_backlog_summary(DurableMrfBacklogSummary { + available: true, + buckets: Vec::new(), + }); + return None; + } + Err(e) => { + warn!( + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION, + error = %e, + "Failed to load MRF recovery file" + ); + return None; + } + }; + + match decode_mrf_file(&data) { + Ok(v) => Some(v), + Err(e) => { + warn!( + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION, + error = %e, + "Failed to decode MRF recovery file — preserving corrupt data" + ); + quarantine_mrf_file(storage, &data).await; + None + } + } +} + +/// Replays one MRF recovery entry by operation kind. +/// Returns `None` when the entry is skipped entirely (no admission outcome); +/// entries that must be retried later are pushed onto `retry_entries`. +async fn replay_mrf_entry( + entry: &MrfReplicateEntry, + storage: &Arc, + retry_entries: &mut Vec, +) -> Option { + match entry.op { + MrfOpKind::Delete => replay_mrf_delete_entry(entry, storage, retry_entries).await, + MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => { + replay_mrf_object_entry(entry, storage, retry_entries).await + } + MrfOpKind::Metadata => replay_mrf_metadata_entry(entry, storage, retry_entries).await, + } +} + +/// Replays a delete-kind MRF entry: force-delete intents replay directly, +/// stale force-delete generations are skipped, and plain deletes are +/// reconstructed as heal deletes. +async fn replay_mrf_delete_entry( + entry: &MrfReplicateEntry, + storage: &Arc, + retry_entries: &mut Vec, +) -> Option { + if should_replay_force_delete_intent(entry) { + let operation_id = entry.force_delete_id?; + let delete = force_delete_heal_replication_info(entry, operation_id); + if replicate_delete_with_outcome(delete, storage.clone()).await { + Some(ReplicationQueueAdmission::Queued) + } else { + Some(ReplicationQueueAdmission::Missed) + } + } else if entry.force_delete_id.is_some() { + Some(ReplicationQueueAdmission::Skipped) + } else { + replay_mrf_reconstructed_delete(entry, storage, retry_entries).await + } +} + +/// Pure DTO construction: heal replication info for a replayed force-delete intent. +fn force_delete_heal_replication_info(entry: &MrfReplicateEntry, operation_id: uuid::Uuid) -> DeletedObjectReplicationInfo { + DeletedObjectReplicationInfo { + delete_object: ReplicationDeletedObject { + object_name: entry.object.clone(), + force_delete: true, + force_delete_id: Some(operation_id), + force_delete_target_arns: entry.target_arns.clone(), + force_delete_generation: entry.force_delete_generation, + ..Default::default() + }, + bucket: entry.bucket.clone(), + op_type: ReplicationType::Heal, + event_type: REPLICATE_HEAL_DELETE.to_string(), + ..Default::default() + } +} + +/// Reconstruct a heal delete and re-queue it. We do NOT call +/// get_object_info here because the delete-marker or version may +/// already be absent from the local store — that is expected. +async fn replay_mrf_reconstructed_delete( + entry: &MrfReplicateEntry, + storage: &Arc, + retry_entries: &mut Vec, +) -> Option { + let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await; + let oi = ObjectInfo { + bucket: entry.bucket.clone(), + name: entry.object.clone(), + version_id: entry.version_id, + delete_marker: entry.delete_marker, + ..Default::default() + }; + let dsc = resolve_mrf_delete_replicate_decision(entry, &oi, versioned, retry_entries).await?; + let dv = reconstructed_heal_delete_info(entry, &oi, &dsc); + if replicate_delete_with_outcome(dv, storage.clone()).await { + Some(ReplicationQueueAdmission::Queued) + } else { + Some(ReplicationQueueAdmission::Missed) + } +} + +/// The MRF entry does not persist the replication decision and the +/// source object is gone, so re-derive the decision from the live +/// bucket config (mirroring get_heal_replicate_object_info) and set +/// it on the reconstructed delete. Without this the decision string +/// is empty and the delete replicates to zero targets — a silent +/// no-op that leaves replicas diverged (backlog#858 / #799 B9). +async fn resolve_mrf_delete_replicate_decision( + entry: &MrfReplicateEntry, + oi: &ObjectInfo, + versioned: bool, + retry_entries: &mut Vec, +) -> Option { + if entry.target_arns.is_empty() { + match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await { + Ok(None) => None, + Err(_) => { + retry_entries.push(entry.clone()); + None + } + Ok(Some(_)) => match check_replicate_delete_strict( + &entry.bucket, + &ObjectToDelete { + object_name: entry.object.clone(), + version_id: entry.version_id, + ..Default::default() + }, + oi, + &ObjectOptions { + versioned, + ..Default::default() + }, + None, + ) + .await + { + Ok(dsc) => Some(dsc), + Err(_) => { + retry_entries.push(entry.clone()); + None + } + }, + } + } else { + Some(replicate_decision_for_admitted_targets(&entry.target_arns)) + } +} + +/// Pure DTO construction: reconstructed heal delete carrying the re-derived +/// replication decision. +fn reconstructed_heal_delete_info( + entry: &MrfReplicateEntry, + oi: &ObjectInfo, + dsc: &ReplicateDecision, +) -> DeletedObjectReplicationInfo { + let mut rstate = oi.replication_state(); + rstate.replicate_decision_str = dsc.to_string(); + + let delete_marker_mtime = entry + .delete_marker_mtime + .and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok()); + + DeletedObjectReplicationInfo { + delete_object: ReplicationDeletedObject { + object_name: entry.object.clone(), + version_id: entry.version_id, + delete_marker_version_id: entry.delete_marker_version_id, + delete_marker: entry.delete_marker, + delete_marker_mtime, + force_delete: entry.force_delete, + replication_state: Some(rstate), + ..Default::default() + }, + bucket: entry.bucket.clone(), + op_type: ReplicationType::Heal, + event_type: REPLICATE_HEAL_DELETE.to_string(), + ..Default::default() + } +} + +/// Replays an Object/Heal/ExistingObject MRF entry against the live source object. +async fn replay_mrf_object_entry( + entry: &MrfReplicateEntry, + storage: &Arc, + retry_entries: &mut Vec, +) -> Option { + let opts = ObjectOptions { + version_id: entry.version_id.map(|u| u.to_string()), + ..Default::default() + }; + let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await { + Ok(oi) => oi, + Err(e) => { + debug!( + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION, + bucket = %entry.bucket, + object = %entry.object, + error = %e, + "MRF recovery: source object lookup failed" + ); + if should_retry_mrf_source_lookup(&e) { + retry_entries.push(entry.clone()); + } + return None; + } + }; + if entry.target_arns.is_empty() { + // Legacy entries predate target admission persistence. They cannot + // be safely attributed, so retain the old live-config fallback. + Some(queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await) + } else { + let roi = admitted_mrf_replicate_object(oi, entry, entry.op.replication_type()); + if replicate_object_with_outcome(roi, storage.clone()).await.1 { + Some(ReplicationQueueAdmission::Queued) + } else { + Some(ReplicationQueueAdmission::Missed) + } + } +} + +/// Replays a metadata-kind MRF entry against the live source object. +async fn replay_mrf_metadata_entry( + entry: &MrfReplicateEntry, + storage: &Arc, + retry_entries: &mut Vec, +) -> Option { + let opts = ObjectOptions { + version_id: entry.version_id.map(|u| u.to_string()), + ..Default::default() + }; + let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await { + Ok(oi) => oi, + Err(e) => { + debug!( + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION, + bucket = %entry.bucket, + object = %entry.object, + error = %e, + "MRF metadata recovery: source object lookup failed" + ); + if should_retry_mrf_source_lookup(&e) { + retry_entries.push(entry.clone()); + } + return None; + } + }; + if entry.target_arns.is_empty() { + Some(queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await) + } else { + let roi = admitted_mrf_replicate_object(oi, entry, ReplicationType::Metadata); + if replicate_object_with_outcome(roi, storage.clone()).await.1 { + Some(ReplicationQueueAdmission::Queued) + } else { + Some(ReplicationQueueAdmission::Missed) + } + } +} + +/// Pure DTO construction: replicate-object info for an entry with persisted +/// admitted targets, carrying over the entry's retry count. +fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_type: ReplicationType) -> ReplicateObjectInfo { + let dsc = replicate_decision_for_admitted_targets(&entry.target_arns); + let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type); + roi.retry_count = entry.retry_count.max(0) as u32; + roi +} + +/// Acknowledges the replayed MRF prefix and returns the retained backlog. +/// On acknowledgement failure the backlog is preserved for the next startup and +/// re-read (falling back to the replayed snapshot) so the published summary stays accurate. +async fn resolve_retained_mrf_entries( + storage: &Arc, + recovery_guard: &rustfs_lock::NamespaceLockGuard, + entries: &[MrfReplicateEntry], + retry_entries: &[MrfReplicateEntry], +) -> Vec { + match acknowledge_mrf_recovery(storage.clone(), recovery_guard, entries, retry_entries).await { + Ok(retained) => retained, + Err(error) => { + warn!( + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION, + error = %error, + "Failed to acknowledge the MRF recovery prefix; preserving it for the next startup" + ); + match read_mrf_entries(storage.clone()).await { + Ok(current) => current, + Err(read_error) => { + warn!( + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION, + error = %read_error, + "Failed to refresh the MRF backlog after acknowledgement failure" + ); + entries.to_vec() + } + } + } + } +} + #[derive(Debug, thiserror::Error)] #[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")] struct ResyncActiveConflictError { @@ -1221,71 +1583,12 @@ impl ReplicationPool { let storage = self.storage.clone(); let handle = tokio::spawn(async move { - let recovery_lock = match storage - .new_ns_lock( - ReplicationMetadataStore::rustfs_meta_bucket(), - ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK, - ) - .await - { - Ok(lock) => lock, - Err(error) => { - warn!( - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION, - error = %error, - "Failed to create the MRF recovery leader lock" - ); - return; - } - }; - let recovery_guard = match recovery_lock - .get_write_lock_quiet(ReplicationLockTiming::acquire_timeout()) - .await - { - Ok(guard) => guard, - Err(_) => { - debug!( - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION, - "Another node is already processing the MRF recovery backlog" - ); - return; - } + let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else { + return; }; - let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await { - Ok(d) => d, - Err(EcstoreError::ConfigNotFound) => { - set_durable_mrf_backlog_summary(DurableMrfBacklogSummary { - available: true, - buckets: Vec::new(), - }); - return; - } - Err(e) => { - warn!( - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION, - error = %e, - "Failed to load MRF recovery file" - ); - return; - } - }; - - let entries = match decode_mrf_file(&data) { - Ok(v) => v, - Err(e) => { - warn!( - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION, - error = %e, - "Failed to decode MRF recovery file — preserving corrupt data" - ); - quarantine_mrf_file(&storage, &data).await; - return; - } + let Some(entries) = load_mrf_recovery_entries(&storage).await else { + return; }; set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries)); @@ -1294,187 +1597,8 @@ impl ReplicationPool { let mut retry_entries = Vec::new(); for entry in entries.iter() { - let admission = match entry.op { - MrfOpKind::Delete => { - if should_replay_force_delete_intent(entry) { - let Some(operation_id) = entry.force_delete_id else { - continue; - }; - let delete = DeletedObjectReplicationInfo { - delete_object: ReplicationDeletedObject { - object_name: entry.object.clone(), - force_delete: true, - force_delete_id: Some(operation_id), - force_delete_target_arns: entry.target_arns.clone(), - force_delete_generation: entry.force_delete_generation, - ..Default::default() - }, - bucket: entry.bucket.clone(), - op_type: ReplicationType::Heal, - event_type: REPLICATE_HEAL_DELETE.to_string(), - ..Default::default() - }; - if replicate_delete_with_outcome(delete, storage.clone()).await { - ReplicationQueueAdmission::Queued - } else { - ReplicationQueueAdmission::Missed - } - } else if entry.force_delete_id.is_some() { - ReplicationQueueAdmission::Skipped - } else { - // Reconstruct a heal delete and re-queue it. We do NOT call - // get_object_info here because the delete-marker or version may - // already be absent from the local store — that is expected. - // - // The MRF entry does not persist the replication decision and the - // source object is gone, so re-derive the decision from the live - // bucket config (mirroring get_heal_replicate_object_info) and set - // it on the reconstructed delete. Without this the decision string - // is empty and the delete replicates to zero targets — a silent - // no-op that leaves replicas diverged (backlog#858 / #799 B9). - let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await; - let oi = ObjectInfo { - bucket: entry.bucket.clone(), - name: entry.object.clone(), - version_id: entry.version_id, - delete_marker: entry.delete_marker, - ..Default::default() - }; - let dsc = if entry.target_arns.is_empty() { - match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await { - Ok(None) => continue, - Err(_) => { - retry_entries.push(entry.clone()); - continue; - } - Ok(Some(_)) => match check_replicate_delete_strict( - &entry.bucket, - &ObjectToDelete { - object_name: entry.object.clone(), - version_id: entry.version_id, - ..Default::default() - }, - &oi, - &ObjectOptions { - versioned, - ..Default::default() - }, - None, - ) - .await - { - Ok(dsc) => dsc, - Err(_) => { - retry_entries.push(entry.clone()); - continue; - } - }, - } - } else { - replicate_decision_for_admitted_targets(&entry.target_arns) - }; - let mut rstate = oi.replication_state(); - rstate.replicate_decision_str = dsc.to_string(); - - let delete_marker_mtime = entry - .delete_marker_mtime - .and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok()); - - let dv = DeletedObjectReplicationInfo { - delete_object: ReplicationDeletedObject { - object_name: entry.object.clone(), - version_id: entry.version_id, - delete_marker_version_id: entry.delete_marker_version_id, - delete_marker: entry.delete_marker, - delete_marker_mtime, - force_delete: entry.force_delete, - replication_state: Some(rstate), - ..Default::default() - }, - bucket: entry.bucket.clone(), - op_type: ReplicationType::Heal, - event_type: REPLICATE_HEAL_DELETE.to_string(), - ..Default::default() - }; - if replicate_delete_with_outcome(dv, storage.clone()).await { - ReplicationQueueAdmission::Queued - } else { - ReplicationQueueAdmission::Missed - } - } - } - MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => { - let opts = ObjectOptions { - version_id: entry.version_id.map(|u| u.to_string()), - ..Default::default() - }; - let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await { - Ok(oi) => oi, - Err(e) => { - debug!( - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION, - bucket = %entry.bucket, - object = %entry.object, - error = %e, - "MRF recovery: source object lookup failed" - ); - if should_retry_mrf_source_lookup(&e) { - retry_entries.push(entry.clone()); - } - continue; - } - }; - if entry.target_arns.is_empty() { - // Legacy entries predate target admission persistence. They cannot - // be safely attributed, so retain the old live-config fallback. - queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await - } else { - let dsc = replicate_decision_for_admitted_targets(&entry.target_arns); - let mut roi = replicate_object_info_from_object_info(oi, dsc, entry.op.replication_type()); - roi.retry_count = entry.retry_count.max(0) as u32; - if replicate_object_with_outcome(roi, storage.clone()).await.1 { - ReplicationQueueAdmission::Queued - } else { - ReplicationQueueAdmission::Missed - } - } - } - MrfOpKind::Metadata => { - let opts = ObjectOptions { - version_id: entry.version_id.map(|u| u.to_string()), - ..Default::default() - }; - let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await { - Ok(oi) => oi, - Err(e) => { - debug!( - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION, - bucket = %entry.bucket, - object = %entry.object, - error = %e, - "MRF metadata recovery: source object lookup failed" - ); - if should_retry_mrf_source_lookup(&e) { - retry_entries.push(entry.clone()); - } - continue; - } - }; - if entry.target_arns.is_empty() { - queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await - } else { - let dsc = replicate_decision_for_admitted_targets(&entry.target_arns); - let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata); - roi.retry_count = entry.retry_count.max(0) as u32; - if replicate_object_with_outcome(roi, storage.clone()).await.1 { - ReplicationQueueAdmission::Queued - } else { - ReplicationQueueAdmission::Missed - } - } - } + let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else { + continue; }; if admission == ReplicationQueueAdmission::Missed { @@ -1484,29 +1608,7 @@ impl ReplicationPool { } } - let retained = match acknowledge_mrf_recovery(storage.clone(), &recovery_guard, &entries, &retry_entries).await { - Ok(retained) => retained, - Err(error) => { - warn!( - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION, - error = %error, - "Failed to acknowledge the MRF recovery prefix; preserving it for the next startup" - ); - match read_mrf_entries(storage.clone()).await { - Ok(current) => current, - Err(read_error) => { - warn!( - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION, - error = %read_error, - "Failed to refresh the MRF backlog after acknowledgement failure" - ); - entries.clone() - } - } - } - }; + let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await; let retained_count = retained.len(); set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained)); diff --git a/crates/ecstore/src/bucket/replication/replication_proxy.rs b/crates/ecstore/src/bucket/replication/replication_proxy.rs new file mode 100644 index 000000000..d71895f16 --- /dev/null +++ b/crates/ecstore/src/bucket/replication/replication_proxy.rs @@ -0,0 +1,150 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Proxy-target selection for reads of objects not yet replicated locally +//! (MinIO `getProxyTargets`, bucket-replication.go). +//! +//! During the active-active replication lag window a GET/HEAD/Tagging request +//! for an object the local site does not have yet may be served by proxying to +//! a replication target. This module only *selects* the candidate targets; the +//! request-path callers perform the remote calls and response translation. + +use std::sync::Arc; + +use tracing::debug; + +use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt as _}; +use super::replication_object_config::get_replication_config; +use super::replication_storage_boundary::ObjectOptions; +use super::replication_target_boundary::{ReplicationTargetStore, TargetClient}; + +/// Returns the replication-target clients eligible to serve a proxied read of +/// `bucket/object`, in rule order. Mirrors MinIO's `getProxyTargets`: +/// +/// - the `source-proxy-request` header family was present at all +/// (`opts.proxy_request` / `opts.proxy_header_set`, MinIO `ProxyRequest` / +/// `ProxyHeaderSet`) -> empty. "true" is the anti-loop marker of an +/// already-proxied client read; "false" is what a peer's replication +/// worker sends on convergence HEADs so the receiver answers locally — +/// proxying that miss back would echo the source object and fake +/// convergence, permanently skipping replication; +/// - the bucket's versioning is suspended for the object -> empty; +/// - no replication configuration / no matching rule -> empty; +/// - otherwise every distinct target ARN whose rules match the object, +/// resolved through the bucket target system, skipping targets that opted +/// out of proxying (`disable_proxy`). +pub async fn get_proxy_targets(bucket: &str, object: &str, opts: &ObjectOptions) -> Vec> { + if opts.proxy_request || opts.proxy_header_set { + return Vec::new(); + } + if opts.version_suspended { + return Vec::new(); + } + + let cfg = match get_replication_config(bucket).await { + Ok(Some(cfg)) => cfg, + Ok(None) => return Vec::new(), + Err(err) => { + debug!(bucket, object, error = %err, "read proxy: failed to load replication config; not proxying"); + return Vec::new(); + } + }; + + let arns = cfg.filter_target_arns(&ObjectOpts { + name: object.to_string(), + ..Default::default() + }); + + let mut targets = Vec::with_capacity(arns.len()); + for arn in arns { + let Some(client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else { + debug!(bucket, object, arn, "read proxy: no client for replication target ARN"); + continue; + }; + if client.disable_proxy { + continue; + } + targets.push(client); + } + + targets +} + +#[cfg(test)] +mod tests { + use super::*; + + fn opts() -> ObjectOptions { + ObjectOptions::default() + } + + /// Anti-loop: a request that was already proxied by a peer must never be + /// proxied onward, regardless of replication configuration. + #[tokio::test] + async fn proxy_request_yields_no_targets() { + let targets = get_proxy_targets( + "bucket", + "object", + &ObjectOptions { + proxy_request: true, + ..opts() + }, + ) + .await; + assert!(targets.is_empty()); + } + + /// MinIO `ProxyHeaderSet` parity: the header family being present at all + /// disables proxying, even with the value "false" — that is what a + /// peer's replication worker sends on convergence HEADs. + #[tokio::test] + async fn proxy_header_set_yields_no_targets() { + let targets = get_proxy_targets( + "bucket", + "object", + &ObjectOptions { + proxy_header_set: true, + proxy_request: false, + ..opts() + }, + ) + .await; + assert!(targets.is_empty()); + } + + /// Suspended versioning disables proxying (MinIO parity): the local null + /// version is authoritative and a remote read could resurrect data. + #[tokio::test] + async fn version_suspended_yields_no_targets() { + let targets = get_proxy_targets( + "bucket", + "object", + &ObjectOptions { + version_suspended: true, + ..opts() + }, + ) + .await; + assert!(targets.is_empty()); + } + + /// A bucket without replication configuration has nothing to proxy to. + /// (No metadata system is running in unit tests, so the config lookup + /// resolves to "no configuration" — the same empty-result contract.) + #[tokio::test] + async fn missing_replication_config_yields_no_targets() { + let targets = get_proxy_targets("bucket-without-replication", "object", &opts()).await; + assert!(targets.is_empty()); + } +} diff --git a/crates/ecstore/src/bucket/replication/replication_resync_boundary.rs b/crates/ecstore/src/bucket/replication/replication_resync_boundary.rs index 3b6dcdf5d..32e3f6c61 100644 --- a/crates/ecstore/src/bucket/replication/replication_resync_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_resync_boundary.rs @@ -15,10 +15,15 @@ use super::replication_error_boundary::{Error, Result}; use super::replication_filemeta_boundary::MrfReplicateEntry; +/// Kept test-only: the runtime consumer was the worker HEAD's fake proxy +/// counting (removed in backlog#1675 P1-5); the resyncer tests still pin the +/// classifier's semantics for the real client read-proxy failure accounting. +#[cfg(test)] +pub(crate) use rustfs_replication::should_count_head_proxy_failure; pub use rustfs_replication::{BucketReplicationResyncStatus, ResyncOpts, ResyncStatusType, TargetReplicationResyncStatus}; pub(crate) use rustfs_replication::{ - is_version_id_mismatch, resync_state_accepts_update, sanitize_resync_error_detail, should_auto_resume_resync, - should_count_head_proxy_failure, + is_version_id_mismatch, resync_state_accepts_update, resync_status_duration, sanitize_resync_error_detail, + should_auto_resume_resync, }; #[allow( diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index 30198a75b..882ece91c 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -15,13 +15,12 @@ use super::replication_bandwidth_boundary; use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt as _}; use super::replication_config_store::ReplicationConfigStore; -use super::replication_error_boundary::{Result, is_err_object_not_found, is_err_version_not_found}; +use super::replication_error_boundary::{Error, Result, is_err_object_not_found, is_err_version_not_found}; use super::replication_event_sink::{EventArgs, send_event, send_local_event}; use super::replication_filemeta_boundary::{ - MrfReplicateEntry, NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo, - ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType, ReplicationType, - ReplicationWorkerOperation, VersionPurgeStatusType, get_replication_state, parse_replicate_decision, - replication_statuses_map, target_reset_header, version_purge_statuses_map, + REPLICATE_EXISTING, ReplicateDecision, ReplicateObjectInfo, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, + ReplicationState, ReplicationStatusType, ReplicationType, VersionPurgeStatusType, get_replication_state, + parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map, }; use super::replication_lock_boundary::ReplicationLockTiming; use super::replication_logging::{EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION_RESYNC}; @@ -30,27 +29,35 @@ use super::replication_metadata_boundary::ReplicationMetadataStore; use super::replication_msgp_boundary::ReplicationMsgpCodec; use super::replication_object_config::{ReplicationConfig, get_replication_config, must_replicate}; use super::replication_object_decision_boundary::{ - MustReplicateOptions, ReplicationMultipartPartInput, heal_uses_delete_replication_path, - is_retryable_delete_replication_head_error, is_version_delete_replication, replication_etags_match, - replication_multipart_complete_actual_size, replication_multipart_part_plan, should_retry_delete_marker_purge, + MustReplicateOptions, ReplicationMultipartPartInput, delete_marker_purge_mrf_entry, delete_marker_purge_version_id, + heal_uses_delete_replication_path, is_retryable_delete_replication_head_error, is_version_delete_replication, + replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size, + replication_multipart_part_plan, resync_existing_delete_replication_info, should_retry_delete_marker_purge, + target_delete_version_id, }; use super::replication_queue_boundary::{DeletedObjectReplicationInfo, ReplicationQueueAdmission}; use super::replication_resync_boundary::ResyncStatusType; +#[cfg(test)] +use super::replication_resync_boundary::should_count_head_proxy_failure; use super::replication_resync_boundary::{ BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus, encode_resync_file, is_version_id_mismatch, - resync_state_accepts_update, sanitize_resync_error_detail, should_count_head_proxy_failure, + resync_state_accepts_update, resync_status_duration, sanitize_resync_error_detail, }; #[cfg(test)] use super::replication_resync_boundary::{RESYNC_META_FORMAT, RESYNC_META_VERSION, WIRE_ZERO_TIME_UNIX, decode_resync_file}; +#[cfg(test)] +use super::replication_storage_boundary::ReplicationDeletedObject; use super::replication_storage_boundary::{ - AdvancedGetOptions, EcstoreObjectOperations, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete, - ReplicationDeletedObject, ReplicationObjectIO, ReplicationStorage, StatObjectOptions, WalkOptions, + AdvancedGetOptions, EcstoreObjectOperations, GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete, + ReplicationObjectIO, ReplicationStorage, StatObjectOptions, StorageObjectInfoOrErr, WalkOptions, }; use super::replication_target_boundary::{ - PutObjectOptions, PutObjectPartOptions, ReplicationTargetStore, TargetClient, replication_action_for_target_head, - replication_complete_multipart_options, replication_delete_marker_purge_remove_options, replication_delete_remove_options, - replication_force_delete_remove_options, replication_object_is_ssec_encrypted, replication_put_object_header_size, - replication_put_object_options, replication_target_head_is_newer_null_version, + ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, PutObjectOptions, PutObjectPartOptions, ReplicationTargetStore, + SsecPassthroughCapability, SsecPassthroughGate, TargetClient, is_replication_target_offline_error, + replication_action_for_target_head, replication_complete_multipart_options, replication_delete_marker_purge_remove_options, + replication_delete_remove_options, replication_force_delete_remove_options, replication_object_is_ssec_encrypted, + replication_put_object_header_size, replication_put_object_options, replication_target_head_is_newer_null_version, + resolve_read_api_version_id, ssec_passthrough_evidence_present, ssec_passthrough_gate, version_identity_drifted, }; use super::replication_versioning_boundary::ReplicationVersioningStore; use super::runtime_boundary as runtime_sources; @@ -106,21 +113,6 @@ const EVENT_DELETE_MARKER_PURGE_FAILED: &str = "replication_delete_marker_purge_ const EVENT_DELETE_MARKER_PURGE_MRF: &str = "replication_delete_marker_purge_mrf"; const METRIC_DELETE_MARKER_PURGE_TOTAL: &str = "rustfs_replication_delete_marker_purge_total"; const EVENT_REPLICATION_VERSION_IDENTITY_DRIFT: &str = "replication_version_identity_drift"; -const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[ - "dispatch failure", - "timeouterror", - "timed out", - "connection refused", - "connection reset", - "connection closed", - "connection aborted", - "broken pipe", - "dns error", - "failed to lookup address", - "name or service not known", - "deadline has elapsed", - "tcp connect error", -]; #[allow( dead_code, @@ -182,15 +174,6 @@ fn has_raw_status(err: &SdkError, status: u16) -> bool { err.raw_response().is_some_and(|r| r.status().as_u16() == status) } -fn is_head_proxy_failure(err: &SdkError) -> bool { - let (is_not_found, code) = err - .as_service_error() - .map(|service_err| (service_err.is_not_found(), service_err.code())) - .unwrap_or((false, None)); - let raw_status = err.raw_response().map(|resp| resp.status().as_u16()); - should_count_head_proxy_failure(is_not_found, code, raw_status) -} - const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_identity_drift_total"; /// Targets that already produced a version-identity-drift warning this @@ -199,27 +182,6 @@ const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_id /// after a restart is acceptable. static VERSION_IDENTITY_WARNED_ARNS: LazyLock>> = LazyLock::new(|| StdMutex::new(HashSet::new())); -/// Runtime half of the P1-19 version-identity contract (the explicit probe -/// lives in replication-check's VersionFidelity phase): every replication PUT -/// response reveals whether the target adopted the source version id. A -/// target minting its own ids silently breaks version-addressed deletes and -/// heal, so surface it — once per target — instead of letting the divergence -/// accumulate unseen. -/// Pure drift judgment: the contract only applies when the source addressed a -/// real (non-nil) version uuid, and drift means the target answered with -/// anything else — including nothing at all. -fn version_identity_drifted(source_version_id: &str, assigned_version_id: Option<&str>) -> bool { - if source_version_id.is_empty() { - return false; - } - // A nil source uuid travels as the literal "null" (unversioned-source - // semantics); no identity contract applies to it. - if Uuid::parse_str(source_version_id).map(|uuid| uuid.is_nil()).unwrap_or(true) { - return false; - } - assigned_version_id != Some(source_version_id) -} - fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &str, assigned_version_id: Option<&str>) { if !version_identity_drifted(source_version_id, assigned_version_id) { return; @@ -242,23 +204,18 @@ fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: & } } -async fn record_proxy_request(bucket: &str, api: &str, is_err: bool) { - if let Some(stats) = runtime_sources::replication_stats() { - stats.inc_proxy(bucket, api, is_err).await; - } -} - -async fn head_object_with_proxy_stats( - source_bucket: &str, +/// HEAD against a replication target on behalf of the replication worker +/// (resync/heal/delete convergence checks). This is NOT a client read proxy: +/// it must not touch the proxy metrics — those count only real GET/HEAD/ +/// Tagging requests proxied for clients (see `replication_proxy.rs` / +/// `TargetClient::head_object_for_proxy`). +async fn head_object_for_worker( target_client: &TargetClient, target_bucket: &str, object: &str, version_id: Option, ) -> std::result::Result> { - let result = target_client.head_object(target_bucket, object, version_id).await; - let is_err = result.as_ref().err().is_some_and(is_head_proxy_failure); - record_proxy_request(source_bucket, "HeadObject", is_err).await; - result + target_client.head_object(target_bucket, object, version_id).await } fn is_version_id_format_mismatch(err: &SdkError) -> bool { @@ -267,13 +224,6 @@ fn is_version_id_format_mismatch(err: &SdkError) -> bool { is_version_id_mismatch(code, raw_status) } -fn is_replication_target_offline_error(err: &(impl Display + ?Sized)) -> bool { - let message = err.to_string().to_ascii_lowercase(); - REPLICATION_TARGET_OFFLINE_ERROR_MARKERS - .iter() - .any(|marker| message.contains(marker)) -} - async fn mark_replication_target_offline_if_needed(target_client: &Arc, err: &(impl Display + ?Sized)) { if is_replication_target_offline_error(err) { ReplicationTargetStore::mark_target_offline(target_client).await; @@ -281,44 +231,126 @@ async fn mark_replication_target_offline_if_needed(target_client: &Arc std::result::Result, SdkError> { - match head_object_with_proxy_stats(source_bucket, tgt_client, &tgt_client.bucket, object, None).await { + match head_object_for_worker(tgt_client, &tgt_client.bucket, object, None).await { Ok(oi) => Ok(Some(oi)), Err(e) if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) => Ok(None), Err(e) => Err(e), } } -static RESYNC_WORKER_COUNT: usize = 10; - -fn resync_status_duration( - status: ResyncStatusType, - start_time: Option, - now: OffsetDateTime, -) -> Option { - if !matches!( - status, - ResyncStatusType::ResyncCompleted | ResyncStatusType::ResyncFailed | ResyncStatusType::ResyncCanceled - ) { - return None; +/// Resolve the N2 fail-closed gate for an SSE-C passthrough attempt against +/// this target. Returns `Some(audit_required)` when replication may proceed; +/// on a freshly-flagged header-dropping target it settles `rinfo` as FAILED +/// (no PUT is ever sent — the object stays on the normal MRF retry channel +/// and re-audits once the verdict's TTL expires or replication-check +/// re-probes the target) and returns `None`. +async fn resolve_ssec_passthrough_gate( + ssec: bool, + tgt_client: &TargetClient, + bucket: &str, + object: &str, + rinfo: &mut ReplicatedTargetInfo, +) -> Option { + let (capability, expired) = ReplicationTargetStore::ssec_passthrough_capability(&tgt_client.arn).await; + match ssec_passthrough_gate(ssec, capability, expired) { + SsecPassthroughGate::Proceed => Some(false), + SsecPassthroughGate::ProceedWithAudit => Some(true), + SsecPassthroughGate::FailClosed => { + rinfo.replication_status = ReplicationStatusType::Failed; + rinfo.error = Some(ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED.to_string()); + warn!( + event = EVENT_RESYNC_TARGET_OPERATION_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + object = %object, + arn = %tgt_client.arn, + operation = "ssec_passthrough_gate", + error = ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, + "Replication target operation failed" + ); + None + } } - - let millis = (now - start_time?).whole_milliseconds(); - if millis < 0 { - return None; - } - - let millis = if millis > i128::from(u64::MAX) { - u64::MAX - } else { - u64::try_from(millis).ok()? - }; - Some(std::time::Duration::from_millis(millis)) } +/// Judge SSE-C passthrough evidence on a HEAD of the replica and record the +/// capability verdict for the target. Returns true when the SSE-C material +/// provably survived; otherwise records `Unsupported` and settles `rinfo` as +/// FAILED so the attempt never reports a silently unreadable COMPLETED. +async fn settle_ssec_passthrough_evidence( + head: &HeadObjectOutput, + tgt_client: &TargetClient, + bucket: &str, + object: &str, + rinfo: &mut ReplicatedTargetInfo, +) -> bool { + if ssec_passthrough_evidence_present(head) { + ReplicationTargetStore::record_ssec_passthrough_capability(&tgt_client.arn, SsecPassthroughCapability::Supported).await; + return true; + } + ReplicationTargetStore::record_ssec_passthrough_capability(&tgt_client.arn, SsecPassthroughCapability::Unsupported).await; + rinfo.replication_status = ReplicationStatusType::Failed; + rinfo.error = Some(ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED.to_string()); + warn!( + event = EVENT_RESYNC_TARGET_OPERATION_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + object = %object, + arn = %tgt_client.arn, + endpoint = %tgt_client.endpoint, + operation = "ssec_passthrough_audit", + error = ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, + "Replication target operation failed" + ); + false +} + +/// Post-PUT HEAD-back audit for an SSE-C passthrough replica, over the worker +/// HEAD channel (replication-check exemption plus the `source-proxy-request: +/// false` suppression header, so the target answers locally without a +/// customer key). A HEAD transport failure leaves the capability `Unknown` +/// but still fails this attempt: an unverifiable SSE-C replica must not +/// report COMPLETED. +async fn audit_ssec_passthrough_replica( + tgt_client: &Arc, + bucket: &str, + object: &str, + version_id: Option, + rinfo: &mut ReplicatedTargetInfo, +) -> bool { + // Address the replica the way the PUT named it: a nil source version id + // (versioning-suspended / null-version objects) maps to the "null" + // version, so the audit HEAD does not 4xx-loop on those objects. + let version_id = resolve_read_api_version_id(version_id); + match head_object_for_worker(tgt_client.as_ref(), &tgt_client.bucket, object, version_id).await { + Ok(head) => settle_ssec_passthrough_evidence(&head, tgt_client, bucket, object, rinfo).await, + Err(e) => { + rinfo.replication_status = ReplicationStatusType::Failed; + rinfo.error = Some(format!("SSE-C passthrough audit HEAD failed: {e}")); + warn!( + event = EVENT_RESYNC_TARGET_OPERATION_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + object = %object, + arn = %tgt_client.arn, + operation = "ssec_passthrough_audit_head", + error = %e, + "Replication target operation failed" + ); + mark_replication_target_offline_if_needed(tgt_client, &e).await; + false + } + } +} + +static RESYNC_WORKER_COUNT: usize = 10; + type ResyncCancelKey = (String, String, String); fn configured_resync_max_jobs() -> usize { @@ -632,31 +664,13 @@ impl ReplicationResyncer { } } - #[instrument(skip(cancellation_token, storage))] - pub async fn resync_bucket( - self: Arc, - cancellation_token: CancellationToken, - storage: Arc, - heal: bool, - opts: ResyncOpts, - ) { - // Check cancellation before starting the scan. - // NOTE: the previous design waited here on `worker_rx.resubscribe().recv()` to - // throttle concurrent resyncs, but `resubscribe()` positions the new receiver at - // the current write-head of the broadcast ring buffer, so all pre-sent bootstrap - // signals (written in `ReplicationResyncer::new`) are invisible to it. Every - // spawned task therefore blocked forever, which is why `resync start` reported - // "started" yet objects never moved. Throttling at this level is also incorrect - // for broadcast channels (one send unblocks ALL receivers). The inner - // per-object worker pool (mpsc channels, line ~877) already provides the right - // concurrency limit. - if cancellation_token.is_cancelled() { - return; - } - - // Acquire a cluster-wide leader lock for this (bucket, ARN) pair so that only - // one node runs the resync scan at a time. Without this, every cluster node would - // scan and replicate every object independently, causing N-fold duplicate traffic. + /// Acquire a cluster-wide leader lock for this (bucket, ARN) pair so that only + /// one node runs the resync scan at a time. Without this, every cluster node would + /// scan and replicate every object independently, causing N-fold duplicate traffic. + async fn acquire_resync_leader_lock( + storage: &Arc, + opts: &ResyncOpts, + ) -> Option { let resync_lock_key = ReplicationMetadataStore::resync_lock_key(&opts.bucket, &opts.arn); let resync_ns_lock = match storage .new_ns_lock(ReplicationMetadataStore::rustfs_meta_bucket(), &resync_lock_key) @@ -674,11 +688,11 @@ impl ReplicationResyncer { reason = "leader_lock_create_failed", "Failed to create resync leader lock — skipping resync" ); - return; + return None; } }; - let _resync_leader_guard = match resync_ns_lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await { - Ok(g) => g, + match resync_ns_lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await { + Ok(g) => Some(g), Err(_) => { debug!( event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED, @@ -689,14 +703,19 @@ impl ReplicationResyncer { reason = "leader_lock_held_by_another_node", "Another node is already running resync for this bucket/ARN — skipping" ); - return; + None } - }; - - let Some(_resync_admission_permit) = self.acquire_resync_admission(&cancellation_token).await else { - return; - }; + } + } + /// Resolve and validate the replication config plus the single remote target + /// client this resync run replicates to, marking the resync failed (and + /// returning `None`) when any lookup or validation step does not hold. + async fn resolve_resync_target( + &self, + opts: &ResyncOpts, + storage: &Arc, + ) -> Option<(ReplicationConfig, Arc)> { let cfg = match get_replication_config(&opts.bucket).await { Ok(cfg) => cfg, Err(err) => { @@ -712,7 +731,7 @@ impl ReplicationResyncer { ); self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone()) .await; - return; + return None; } }; @@ -730,7 +749,7 @@ impl ReplicationResyncer { ); self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone()) .await; - return; + return None; } }; @@ -748,7 +767,7 @@ impl ReplicationResyncer { ); self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone()) .await; - return; + return None; } let target_arns = if let Some(cfg) = cfg { @@ -773,7 +792,7 @@ impl ReplicationResyncer { ); self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone()) .await; - return; + return None; } let Some(target_client) = ReplicationTargetStore::remote_target_client(&opts.bucket, &target_arns[0]).await else { @@ -788,9 +807,15 @@ impl ReplicationResyncer { ); self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone()) .await; - return; + return None; }; + Some((rcfg, target_client)) + } + + /// Persist the `ResyncStarted` status for non-heal runs, logging (without + /// aborting the resync) when the status update fails. + async fn mark_resync_started(&self, heal: bool, opts: &ResyncOpts, storage: &Arc) { if !heal && let Err(e) = self .mark_status(ResyncStatusType::ResyncStarted, opts.clone(), storage.clone()) @@ -807,223 +832,152 @@ impl ReplicationResyncer { "Failed to update resync status" ); } + } - let (tx, mut rx) = tokio::sync::mpsc::channel(100); - let walk_failed = Arc::new(AtomicBool::new(false)); - let walk_failed_task = walk_failed.clone(); - let walk_storage = storage.clone(); - let walk_cancellation = cancellation_token.clone(); - let walk_bucket = opts.bucket.clone(); - let walk_arn = opts.arn.clone(); - let walk_task = tokio::spawn(async move { - if let Err(err) = walk_storage - .walk( - walk_cancellation, - &walk_bucket, - "", - tx, - WalkOptions::default().with_walkdir_timeouts(BACKGROUND_WALKDIR_TIMEOUT), - ) - .await - { - walk_failed_task.store(true, Ordering::Relaxed); - error!( - event = EVENT_RESYNC_RUNTIME_SKIPPED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %walk_bucket, - arn = %walk_arn, - reason = "walk_failed", - error = %err, - "Replication resync bucket walk failed" - ); - } - }); + /// Drain and join the resync worker tasks after a fatal dispatch error, + /// logging any observed task failure and persisting the failed status. + async fn finish_resync_failed( + &self, + worker_txs: Vec>, + results_tx: tokio::sync::mpsc::Sender, + futures: Vec>, + join_failure_reason: &str, + opts: &ResyncOpts, + storage: &Arc, + ) { + let worker_failed = finish_resync_workers(worker_txs, results_tx, futures, false).await; + if worker_failed { + error!( + event = EVENT_RESYNC_TASK_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %opts.bucket, + arn = %opts.arn, + reason = join_failure_reason, + "Replication resync worker cleanup observed task failure" + ); + } + self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone()) + .await; + } - let mut worker_txs = Vec::new(); + /// Abort the resync worker tasks after cancellation and persist the + /// canceled status. + async fn finish_resync_canceled( + &self, + worker_txs: Vec>, + results_tx: tokio::sync::mpsc::Sender, + futures: Vec>, + opts: &ResyncOpts, + storage: &Arc, + ) { + finish_resync_workers(worker_txs, results_tx, futures, true).await; + self.resync_bucket_mark_status(ResyncStatusType::ResyncCanceled, opts.clone(), storage.clone()) + .await; + } + + /// Spawn the collector task that folds per-object resync results into the + /// aggregated resync stats. + fn spawn_resync_results_collector( + resyncer: Arc, + opts: &ResyncOpts, + ) -> (tokio::sync::mpsc::Sender, JoinHandle<()>) { // mpsc, not broadcast: a lagging broadcast receiver returns Err(Lagged) which // would end the collector and silently drop every subsequent worker result. let (results_tx, mut results_rx) = tokio::sync::mpsc::channel::(RESYNC_WORKER_COUNT * 4); let opts_clone = opts.clone(); - let self_clone = self.clone(); - - let mut futures = vec![walk_task]; let results_fut = tokio::spawn(async move { while let Some(st) = results_rx.recv().await { - self_clone.inc_stats(&st, opts_clone.clone()).await; + resyncer.inc_stats(&st, opts_clone.clone()).await; } }); + (results_tx, results_fut) + } + + #[instrument(skip(cancellation_token, storage))] + pub async fn resync_bucket( + self: Arc, + cancellation_token: CancellationToken, + storage: Arc, + heal: bool, + opts: ResyncOpts, + ) { + // Check cancellation before starting the scan. + // NOTE: the previous design waited here on `worker_rx.resubscribe().recv()` to + // throttle concurrent resyncs, but `resubscribe()` positions the new receiver at + // the current write-head of the broadcast ring buffer, so all pre-sent bootstrap + // signals (written in `ReplicationResyncer::new`) are invisible to it. Every + // spawned task therefore blocked forever, which is why `resync start` reported + // "started" yet objects never moved. Throttling at this level is also incorrect + // for broadcast channels (one send unblocks ALL receivers). The inner + // per-object worker pool (mpsc channels, `spawn_resync_object_workers`) already + // provides the right concurrency limit. + if cancellation_token.is_cancelled() { + return; + } + + let Some(_resync_leader_guard) = Self::acquire_resync_leader_lock(&storage, &opts).await else { + return; + }; + + let Some(_resync_admission_permit) = self.acquire_resync_admission(&cancellation_token).await else { + return; + }; + + let Some((rcfg, target_client)) = self.resolve_resync_target(&opts, &storage).await else { + return; + }; + + self.mark_resync_started(heal, &opts, &storage).await; + + let (rx, walk_failed, walk_task) = spawn_resync_walk_task(&storage, &cancellation_token, &opts); + + let mut futures = vec![walk_task]; + + let (results_tx, results_fut) = Self::spawn_resync_results_collector(self.clone(), &opts); + futures.push(results_fut); - for _ in 0..RESYNC_WORKER_COUNT { - let (tx, mut rx) = tokio::sync::mpsc::channel::(100); - worker_txs.push(tx); + let worker_txs = + spawn_resync_object_workers(&cancellation_token, &target_client, &storage, &opts, &results_tx, &mut futures); - let cancel_token = cancellation_token.clone(); - let target_client = target_client.clone(); - let storage = storage.clone(); - let results_tx = results_tx.clone(); - let bucket_name = opts.bucket.clone(); - let target_arn = opts.arn.clone(); + self.drive_resync_dispatch( + &cancellation_token, + rx, + &rcfg, + ResyncRunState { + worker_txs, + results_tx, + futures, + walk_failed, + }, + &opts, + &storage, + ) + .await; + } - let f = tokio::spawn(async move { - while let Some(mut roi) = rx.recv().await { - if cancel_token.is_cancelled() { - return; - } - - if roi.delete_marker || !roi.version_purge_status.is_empty() { - let (version_id, dm_version_id) = if roi.version_purge_status.is_empty() { - (None, roi.version_id) - } else { - (roi.version_id, None) - }; - - let doi = DeletedObjectReplicationInfo { - delete_object: ReplicationDeletedObject { - object_name: roi.name.clone(), - delete_marker_version_id: dm_version_id, - version_id, - replication_state: roi.replication_state.clone(), - delete_marker: roi.delete_marker, - delete_marker_mtime: roi.mod_time, - ..Default::default() - }, - bucket: roi.bucket.clone(), - event_type: REPLICATE_EXISTING_DELETE.to_string(), - op_type: ReplicationType::ExistingObject, - target_arn: target_arn.clone(), - ..Default::default() - }; - replicate_delete(doi, storage.clone()).await; - } else { - roi.op_type = ReplicationType::ExistingObject; - roi.event_type = REPLICATE_EXISTING.to_string(); - replicate_object(roi.clone(), storage.clone()).await; - } - - let mut st = TargetReplicationResyncStatus { - object: roi.name.clone(), - bucket: roi.bucket.clone(), - ..Default::default() - }; - - let reset_id = target_client.reset_id.clone(); - - let head_result = head_object_with_proxy_stats( - &bucket_name, - target_client.as_ref(), - &target_client.bucket, - &roi.name, - roi.version_id.map(|v| v.to_string()), - ) - .await; - let (size, err) = match head_result { - Ok(_) => { - st.replicated_count += 1; - st.replicated_size += roi.size; - (roi.size, None) - } - Err(err) if roi.delete_marker => { - // Verifying a replicated delete marker: only a - // definitive 404/NoSuchKey or 405/MethodNotAllowed - // confirms the marker propagated. Any other - // (retryable/ambiguous) HEAD error leaves the outcome - // unverified, so it must count as failed — not as a - // blanket success (backlog#862 / #799 B13). - let retryable = { - let (is_not_found, code) = err - .as_service_error() - .map(|se| (se.is_not_found(), se.code())) - .unwrap_or((false, None)); - is_retryable_delete_replication_head_error(is_not_found, code) - }; - if retryable { - st.failed_count += 1; - (0, Some(err)) - } else { - st.replicated_count += 1; - (0, None) - } - } - Err(err) if is_version_id_format_mismatch(&err) => { - // AWS-style target rejects the RustFS UUID versionId - // (400). Re-verify without the versionId before - // concluding the object failed to replicate, instead - // of counting a well-replicated object as failed. - match head_object_fallback(&bucket_name, target_client.as_ref(), &roi.name).await { - Ok(Some(_)) => { - st.replicated_count += 1; - st.replicated_size += roi.size; - (roi.size, None) - } - Ok(None) => { - st.failed_count += 1; - (0, Some(err)) - } - Err(e2) => { - st.failed_count += 1; - (0, Some(e2)) - } - } - } - Err(err) => { - st.failed_count += 1; - (0, Some(err)) - } - }; - - if err.is_some() { - debug!( - event = EVENT_RESYNC_OBJECT_PROCESSED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - reset_id = %reset_id, - bucket = %bucket_name, - object = %roi.name, - version_id = %roi.version_id.unwrap_or_default(), - size, - error = ?err, - "Processed resync object with verification error" - ); - } else { - trace!( - event = EVENT_RESYNC_OBJECT_PROCESSED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - reset_id = %reset_id, - bucket = %bucket_name, - object = %roi.name, - version_id = %roi.version_id.unwrap_or_default(), - size, - "Processed resync object" - ); - } - st.error = err.as_ref().and_then(resync_target_error_detail); - - if cancel_token.is_cancelled() { - return; - } - - if let Err(err) = results_tx.send(st).await { - error!( - event = EVENT_RESYNC_RUNTIME_CHANNEL_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %bucket_name, - reason = "status_channel_send_failed", - error = %err, - "Failed to send resync status" - ); - } - } - }); - - futures.push(f); - } + /// Pump walked objects through classification into the hashed worker + /// queues, finalizing the resync status on dispatch error, cancellation, + /// or completion of the walk. + async fn drive_resync_dispatch( + &self, + cancellation_token: &CancellationToken, + mut rx: tokio::sync::mpsc::Receiver>, + rcfg: &ReplicationConfig, + state: ResyncRunState, + opts: &ResyncOpts, + storage: &Arc, + ) { + let ResyncRunState { + worker_txs, + results_tx, + futures, + walk_failed, + } = state; while let Some(res) = rx.recv().await { if let Some(err) = res.err { @@ -1039,27 +993,21 @@ impl ReplicationResyncer { ); cancellation_token.cancel(); drop(rx); - let worker_failed = finish_resync_workers(worker_txs, results_tx, futures, false).await; - if worker_failed { - error!( - event = EVENT_RESYNC_TASK_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %opts.bucket, - arn = %opts.arn, - reason = "worker_join_failed_after_object_info_error", - "Replication resync worker cleanup observed task failure" - ); - } - self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone()) - .await; + self.finish_resync_failed( + worker_txs, + results_tx, + futures, + "worker_join_failed_after_object_info_error", + opts, + storage, + ) + .await; return; } if cancellation_token.is_cancelled() { drop(rx); - finish_resync_workers(worker_txs, results_tx, futures, true).await; - self.resync_bucket_mark_status(ResyncStatusType::ResyncCanceled, opts.clone(), storage.clone()) + self.finish_resync_canceled(worker_txs, results_tx, futures, opts, storage) .await; return; } @@ -1068,7 +1016,7 @@ impl ReplicationResyncer { continue; }; - let roi = match get_heal_replicate_object_info(&object, &rcfg).await { + let roi = match get_heal_replicate_object_info(&object, rcfg).await { Ok(roi) => roi, Err(err) => { error!( @@ -1083,20 +1031,15 @@ impl ReplicationResyncer { ); cancellation_token.cancel(); drop(rx); - let worker_failed = finish_resync_workers(worker_txs, results_tx, futures, false).await; - if worker_failed { - error!( - event = EVENT_RESYNC_TASK_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %opts.bucket, - arn = %opts.arn, - reason = "worker_join_failed_after_classification_error", - "Replication resync worker cleanup observed task failure" - ); - } - self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone()) - .await; + self.finish_resync_failed( + worker_txs, + results_tx, + futures, + "worker_join_failed_after_classification_error", + opts, + storage, + ) + .await; return; } }; @@ -1106,8 +1049,7 @@ impl ReplicationResyncer { if cancellation_token.is_cancelled() { drop(rx); - finish_resync_workers(worker_txs, results_tx, futures, true).await; - self.resync_bucket_mark_status(ResyncStatusType::ResyncCanceled, opts.clone(), storage.clone()) + self.finish_resync_canceled(worker_txs, results_tx, futures, opts, storage) .await; return; } @@ -1127,26 +1069,21 @@ impl ReplicationResyncer { ); cancellation_token.cancel(); drop(rx); - let worker_failed = finish_resync_workers(worker_txs, results_tx, futures, false).await; - if worker_failed { - error!( - event = EVENT_RESYNC_TASK_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %opts.bucket, - arn = %opts.arn, - reason = "worker_join_failed_after_queue_send_error", - "Replication resync worker cleanup observed task failure" - ); - } - self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone()) - .await; + self.finish_resync_failed( + worker_txs, + results_tx, + futures, + "worker_join_failed_after_queue_send_error", + opts, + storage, + ) + .await; return; } } let worker_failed = finish_resync_workers(worker_txs, results_tx, futures, false).await; - let target_failed = self.target_has_resync_failures(&opts).await; + let target_failed = self.target_has_resync_failures(opts).await; let status = if walk_failed.load(Ordering::Relaxed) || worker_failed || target_failed { ResyncStatusType::ResyncFailed } else { @@ -1157,6 +1094,247 @@ impl ReplicationResyncer { } } +/// Worker-pool channel and task state for one resync run, handed from setup to +/// the dispatch loop. +struct ResyncRunState { + worker_txs: Vec>, + results_tx: tokio::sync::mpsc::Sender, + futures: Vec>, + walk_failed: Arc, +} + +/// Spawn the bucket walk task that feeds object listings into the resync +/// dispatch loop, surfacing walk failures through the returned flag. +fn spawn_resync_walk_task( + storage: &Arc, + cancellation_token: &CancellationToken, + opts: &ResyncOpts, +) -> ( + tokio::sync::mpsc::Receiver>, + Arc, + JoinHandle<()>, +) { + let (tx, rx) = tokio::sync::mpsc::channel(100); + let walk_failed = Arc::new(AtomicBool::new(false)); + let walk_failed_task = walk_failed.clone(); + let walk_storage = storage.clone(); + let walk_cancellation = cancellation_token.clone(); + let walk_bucket = opts.bucket.clone(); + let walk_arn = opts.arn.clone(); + let walk_task = tokio::spawn(async move { + if let Err(err) = walk_storage + .walk( + walk_cancellation, + &walk_bucket, + "", + tx, + WalkOptions::default().with_walkdir_timeouts(BACKGROUND_WALKDIR_TIMEOUT), + ) + .await + { + walk_failed_task.store(true, Ordering::Relaxed); + error!( + event = EVENT_RESYNC_RUNTIME_SKIPPED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %walk_bucket, + arn = %walk_arn, + reason = "walk_failed", + error = %err, + "Replication resync bucket walk failed" + ); + } + }); + (rx, walk_failed, walk_task) +} + +/// Classify the target HEAD verification result for one resynced object, +/// updating the per-object status counters and returning the accounted size +/// together with any verification error. +async fn verify_resync_head_result( + head_result: std::result::Result>, + roi: &ReplicateObjectInfo, + st: &mut TargetReplicationResyncStatus, + target_client: &Arc, +) -> (i64, Option>) { + match head_result { + Ok(_) => { + st.replicated_count += 1; + st.replicated_size += roi.size; + (roi.size, None) + } + Err(err) if roi.delete_marker => { + // Verifying a replicated delete marker: only a + // definitive 404/NoSuchKey or 405/MethodNotAllowed + // confirms the marker propagated. Any other + // (retryable/ambiguous) HEAD error leaves the outcome + // unverified, so it must count as failed — not as a + // blanket success (backlog#862 / #799 B13). + let retryable = { + let (is_not_found, code) = err + .as_service_error() + .map(|se| (se.is_not_found(), se.code())) + .unwrap_or((false, None)); + is_retryable_delete_replication_head_error(is_not_found, code) + }; + if retryable { + st.failed_count += 1; + (0, Some(err)) + } else { + st.replicated_count += 1; + (0, None) + } + } + Err(err) if is_version_id_format_mismatch(&err) => { + // AWS-style target rejects the RustFS UUID versionId + // (400). Re-verify without the versionId before + // concluding the object failed to replicate, instead + // of counting a well-replicated object as failed. + match head_object_fallback(target_client.as_ref(), &roi.name).await { + Ok(Some(_)) => { + st.replicated_count += 1; + st.replicated_size += roi.size; + (roi.size, None) + } + Ok(None) => { + st.failed_count += 1; + (0, Some(err)) + } + Err(e2) => { + st.failed_count += 1; + (0, Some(e2)) + } + } + } + Err(err) => { + st.failed_count += 1; + (0, Some(err)) + } + } +} + +/// Replicate one existing object (or delete marker / version purge) to the +/// resync target, verify the outcome via a target HEAD, and produce the +/// per-object resync status update. +async fn resync_worker_process_object( + mut roi: ReplicateObjectInfo, + storage: &Arc, + target_client: &Arc, + bucket_name: &str, + target_arn: &str, +) -> TargetReplicationResyncStatus { + if roi.delete_marker || !roi.version_purge_status.is_empty() { + let doi = resync_existing_delete_replication_info(&roi, target_arn); + replicate_delete(doi, storage.clone()).await; + } else { + roi.op_type = ReplicationType::ExistingObject; + roi.event_type = REPLICATE_EXISTING.to_string(); + replicate_object(roi.clone(), storage.clone()).await; + } + + let mut st = TargetReplicationResyncStatus { + object: roi.name.clone(), + bucket: roi.bucket.clone(), + ..Default::default() + }; + + let reset_id = target_client.reset_id.clone(); + + let head_result = head_object_for_worker( + target_client.as_ref(), + &target_client.bucket, + &roi.name, + roi.version_id.map(|v| v.to_string()), + ) + .await; + let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, target_client).await; + + if err.is_some() { + debug!( + event = EVENT_RESYNC_OBJECT_PROCESSED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + reset_id = %reset_id, + bucket = %bucket_name, + object = %roi.name, + version_id = %roi.version_id.unwrap_or_default(), + size, + error = ?err, + "Processed resync object with verification error" + ); + } else { + trace!( + event = EVENT_RESYNC_OBJECT_PROCESSED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + reset_id = %reset_id, + bucket = %bucket_name, + object = %roi.name, + version_id = %roi.version_id.unwrap_or_default(), + size, + "Processed resync object" + ); + } + st.error = err.as_ref().and_then(resync_target_error_detail); + + st +} + +/// Spawn the per-object resync worker pool, wiring every worker to the shared +/// results channel and registering its task handle for cleanup. +fn spawn_resync_object_workers( + cancellation_token: &CancellationToken, + target_client: &Arc, + storage: &Arc, + opts: &ResyncOpts, + results_tx: &tokio::sync::mpsc::Sender, + futures: &mut Vec>, +) -> Vec> { + let mut worker_txs = Vec::new(); + + for _ in 0..RESYNC_WORKER_COUNT { + let (tx, mut rx) = tokio::sync::mpsc::channel::(100); + worker_txs.push(tx); + + let cancel_token = cancellation_token.clone(); + let target_client = target_client.clone(); + let storage = storage.clone(); + let results_tx = results_tx.clone(); + let bucket_name = opts.bucket.clone(); + let target_arn = opts.arn.clone(); + + let f = tokio::spawn(async move { + while let Some(roi) = rx.recv().await { + if cancel_token.is_cancelled() { + return; + } + + let st = resync_worker_process_object(roi, &storage, &target_client, &bucket_name, &target_arn).await; + + if cancel_token.is_cancelled() { + return; + } + + if let Err(err) = results_tx.send(st).await { + error!( + event = EVENT_RESYNC_RUNTIME_CHANNEL_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket_name, + reason = "status_channel_send_failed", + error = %err, + "Failed to send resync status" + ); + } + } + }); + + futures.push(f); + } + + worker_txs +} + pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationConfig) -> Result { let mut oi = oi.clone(); let mut user_defined = (*oi.user_defined).clone(); @@ -1686,29 +1864,6 @@ pub(crate) async fn replicate_delete_with_outcome( ) } -/// Whether a delete replication fully succeeded — the MRF replay acknowledges -/// (drops) an entry exactly when this returns true. -/// -/// The delayed purge is deliberately NOT an input: holding the outcome hostage -/// to it (`&& !requires_delayed_purge`) forced `false` for every delete-marker -/// entry and retained them all in the durable MRF journal forever. Purge -/// failures persist their own purge-intent entry instead -/// (`watch_and_purge_source_delete_marker`), and replays of those entries -/// report purge success through `purge_stale_delete_marker_targets`. -fn replicate_delete_outcome( - expected_targets: usize, - replicated_targets: usize, - state_persisted: bool, - source_state_verified: bool, - replication_status: &ReplicationStatusType, -) -> bool { - expected_targets > 0 - && replicated_targets == expected_targets - && state_persisted - && source_state_verified - && *replication_status == ReplicationStatusType::Completed -} - async fn source_delete_marker_missing( storage: &S, bucket: &str, @@ -1733,29 +1888,6 @@ async fn source_delete_marker_missing( } } -/// Which version a delete-marker purge should address on one target. -/// -/// `None` means do not purge at all: the recorded mapping disagreed across the -/// dual internal prefixes, and guessing an id could destroy a live version on -/// the target. `Some(id)` is the exact version the target reported when it -/// accepted the marker; falling back to a source-derived id is only correct -/// when the target mirrors source version ids, which a generic S3 target does -/// not. -fn delete_marker_purge_version_id( - state: Option<&ReplicationState>, - arn: &str, - delete_marker_version_id: Uuid, -) -> Option> { - if state.is_some_and(|state| state.target_delete_marker_version_ids_corrupt) { - return None; - } - let recorded = state.and_then(|state| state.target_delete_marker_version_ids.get(arn).cloned()); - Some(match recorded { - Some(version_id) => Some(version_id), - None => target_delete_version_id(delete_marker_version_id, true), - }) -} - /// One purge pass over the eligible targets. Returns the ARNs that must be /// retried: the remote DELETE failed, or the target client was unavailable /// (e.g. a runtime cache miss). Inconsistent recorded version mappings are a @@ -1923,20 +2055,6 @@ async fn watch_and_purge_source_delete_marker( } } -/// Shape an exhausted purge intent as a marker-creation delete entry. Replay -/// reconstructs it with `delete_marker: true`, finds the source marker gone, -/// and funnels into the stale-marker branch of `replicate_delete_with_outcome` -/// — which re-runs the purge without touching source state and reports purge -/// success as the replay outcome. -fn delete_marker_purge_mrf_entry(dobj: &DeletedObjectReplicationInfo, failed_arns: Vec) -> MrfReplicateEntry { - let mut entry = dobj.to_mrf_entry(); - entry.delete_marker = true; - entry.version_id = None; - entry.retry_count = 0; - entry.target_arns = failed_arns; - entry -} - async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, failed_arns: Vec) { let arns = failed_arns.join(","); let miss_reason = match runtime_sources::replication_pool() { @@ -2296,14 +2414,6 @@ async fn replicate_force_delete_to_targets(dobj: &Deleted all_succeeded } -fn target_delete_version_id(version_id: Uuid, version_purge: bool) -> Option { - if version_id.is_nil() { - version_purge.then(|| NULL_VERSION_ID.to_string()) - } else { - Some(version_id.to_string()) - } -} - async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc) -> ReplicatedTargetInfo { let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id { version_id.to_owned() @@ -2346,8 +2456,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli let version_id = target_delete_version_id(version_id, is_version_purge); if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() { - match head_object_with_proxy_stats( - &dobj.bucket, + match head_object_for_worker( tgt_client.as_ref(), &tgt_client.bucket, &dobj.delete_object.object_name, @@ -2713,6 +2822,21 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { return rinfo; } + // N2 fail-closed: never PUT SSE-C ciphertext at a target known to drop + // the passthrough transport headers, and never trust a convergence HEAD + // against such a target — a previous broken replica matches by ETag. + let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo).await + else { + send_local_event(EventArgs { + event_name: EventName::ObjectReplicationNotTracked.to_string(), + bucket_name: bucket.clone(), + object: self.to_object_info(), + user_agent: "Internal: [Replication]".to_string(), + ..Default::default() + }); + return rinfo; + }; + let versioned = ReplicationVersioningStore::prefix_enabled(&bucket, &object).await; let version_suspended = ReplicationVersioningStore::prefix_suspended(&bucket, &object).await; @@ -2810,18 +2934,20 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { } let mut replication_action = replication_action; - match head_object_with_proxy_stats( - &bucket, - tgt_client.as_ref(), - &tgt_client.bucket, - &object, - self.version_id.map(|v| v.to_string()), - ) - .await + match head_object_for_worker(tgt_client.as_ref(), &tgt_client.bucket, &object, self.version_id.map(|v| v.to_string())) + .await { Ok(oi) => { replication_action = replication_action_for_target_head(&object_info, &oi, self.op_type); if replication_action == ReplicationAction::None { + // An SSE-C replica only counts as converged when the same + // HEAD proves its decryption material survived; a broken + // ciphertext copy from an earlier attempt matches by ETag. + if ssec_audit_required + && !settle_ssec_passthrough_evidence(&oi, &tgt_client, &bucket, &object, &mut rinfo).await + { + return rinfo; + } rinfo.replication_status = ReplicationStatusType::Completed; rinfo.replication_resynced = true; rinfo.replication_action = ReplicationAction::None; @@ -2834,8 +2960,13 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { // Object not on target yet → fall through to PUT. } else if is_version_id_format_mismatch(&e) { // Version-ID format mismatch: retry without versionId and compare ETags. - match head_object_fallback(&bucket, &tgt_client, &object).await { + match head_object_fallback(&tgt_client, &object).await { Ok(Some(oi)) if replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref()) => { + if ssec_audit_required + && !settle_ssec_passthrough_evidence(&oi, &tgt_client, &bucket, &object, &mut rinfo).await + { + return rinfo; + } rinfo.replication_status = ReplicationStatusType::Completed; rinfo.replication_resynced = true; rinfo.replication_action = ReplicationAction::None; @@ -2910,7 +3041,6 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { } }; - let has_tagging_replication = !put_opts.user_tags.is_empty(); if let Some(err) = if is_multipart { drop(gr); let result = replicate_object_with_multipart(MultipartReplicationContext { @@ -2925,10 +3055,6 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { put_opts, }) .await; - record_proxy_request(&bucket, "PutObject", result.is_err()).await; - if has_tagging_replication { - record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await; - } result.err() } else { gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn); @@ -2944,10 +3070,6 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { ) }) .map_err(|e| std::io::Error::other(e.to_string())); - record_proxy_request(&bucket, "PutObject", result.is_err()).await; - if has_tagging_replication { - record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await; - } result.err() } { rinfo.replication_status = ReplicationStatusType::Failed; @@ -2969,6 +3091,15 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { return rinfo; } + // First SSE-C passthrough PUT against this target: verify the replica + // kept its decryption material before reporting COMPLETED. + if ssec_audit_required + && !audit_ssec_passthrough_replica(&tgt_client, &bucket, &object, self.version_id.map(|v| v.to_string()), &mut rinfo) + .await + { + return rinfo; + } + rinfo.replication_status = ReplicationStatusType::Completed; rinfo @@ -2984,30 +3115,17 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { let bucket = self.bucket.clone(); let object = self.name.clone(); - let mut replication_action = ReplicationAction::Metadata; - let mut rinfo = ReplicatedTargetInfo { - arn: tgt_client.arn.clone(), - size: self.actual_size, - replication_action, - op_type: self.op_type, - replication_status: ReplicationStatusType::Failed, - prev_replication_status: self.target_replication_status(&tgt_client.arn), - endpoint: tgt_client.endpoint.clone(), - secure: tgt_client.secure, - ..Default::default() - }; + let mut rinfo = replicate_all_target_info(self, &tgt_client); if ReplicationTargetStore::target_is_offline(&tgt_client).await { - debug!( - event = EVENT_RESYNC_RUNTIME_SKIPPED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %bucket, - arn = %tgt_client.arn, - target = %tgt_client.to_url(), - reason = "target_offline", - "Skipped replication because target is offline" - ); + note_replicate_all_target_offline(self, &bucket, &tgt_client); + return rinfo; + } + + // N2 fail-closed: see the gate in `replicate_object` — the same policy + // applies to the metadata/existing-object transport. + let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo).await + else { send_local_event(EventArgs { event_name: EventName::ObjectReplicationNotTracked.to_string(), bucket_name: bucket.clone(), @@ -3015,49 +3133,22 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { user_agent: "Internal: [Replication]".to_string(), ..Default::default() }); + rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); return rinfo; - } + }; let versioned = ReplicationVersioningStore::prefix_enabled(&bucket, &object).await; let version_suspended = ReplicationVersioningStore::prefix_suspended(&bucket, &object).await; - let obj_opts = ObjectOptions { - version_id: self.version_id.map(|v| v.to_string()), - version_suspended, - versioned, - replication_request: true, - // SSE-C passthrough reads the stored ciphertext verbatim; the - // decrypting reader cannot serve it (no customer key server-side). - raw_data_movement_read: self.ssec, - ..Default::default() - }; + let obj_opts = replicate_all_read_options(self, versioned, version_suspended); - let mut gr = match storage + let gr = match storage .get_object_reader(&bucket, &object, None, HeaderMap::new(), &obj_opts) .await { Ok(gr) => gr, Err(e) => { - if !(is_err_object_not_found(&e) || is_err_version_not_found(&e)) { - debug!( - event = EVENT_RESYNC_RUNTIME_SKIPPED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %bucket, - arn = %tgt_client.arn, - error = %e, - reason = "object_reader_unavailable", - "Skipped replication because object reader is unavailable" - ); - send_local_event(EventArgs { - event_name: EventName::ObjectReplicationNotTracked.to_string(), - bucket_name: bucket.clone(), - object: self.to_object_info(), - user_agent: "Internal: [Replication]".to_string(), - ..Default::default() - }); - } - + note_replicate_all_reader_unavailable(self, &bucket, &tgt_client, &e); return rinfo; } }; @@ -3069,23 +3160,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { let size = match object_info.get_actual_size() { Ok(size) => size, Err(e) => { - debug!( - event = EVENT_RESYNC_RUNTIME_SKIPPED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %bucket, - arn = %tgt_client.arn, - error = %e, - reason = "actual_size_unavailable", - "Skipped replication because actual object size is unavailable" - ); - send_local_event(EventArgs { - event_name: EventName::ObjectReplicationNotTracked.to_string(), - bucket_name: bucket.clone(), - object: object_info, - user_agent: "Internal: [Replication]".to_string(), - ..Default::default() - }); + note_replicate_all_size_unavailable(&bucket, &tgt_client, object_info, &e); return rinfo; } }; @@ -3094,174 +3169,27 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { let transfer_size = if self.ssec { object_info.size } else { size }; if tgt_client.bucket.is_empty() { - debug!( - event = EVENT_RESYNC_RUNTIME_SKIPPED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %bucket, - arn = %tgt_client.arn, - reason = "target_bucket_empty", - "Skipped replication because target bucket is empty" - ); - send_local_event(EventArgs { - event_name: EventName::ObjectReplicationNotTracked.to_string(), - bucket_name: bucket.clone(), - object: object_info, - user_agent: "Internal: [Replication]".to_string(), - ..Default::default() - }); + note_replicate_all_target_bucket_empty(&bucket, &tgt_client, object_info); return rinfo; } - let mut sopts = StatObjectOptions { - version_id: object_info.version_id.map(|v| v.to_string()).unwrap_or_default(), - internal: AdvancedGetOptions { - replication_proxy_request: "false".to_string(), - ..Default::default() + let _sopts = replicate_all_stat_options(&object_info, &bucket, &tgt_client); + + let Some((replication_action, object_info)) = resolve_replicate_all_action( + ReplicateAllActionContext { + roi: self, + tgt_client: &tgt_client, + bucket: &bucket, + object: &object, + start_time, + ssec_audit_required, }, - ..Default::default() - }; - - if let Err(err) = sopts.set(AMZ_TAGGING_DIRECTIVE, "ACCESS") { - debug!( - event = EVENT_RESYNC_RUNTIME_SKIPPED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %bucket, - arn = %tgt_client.arn, - error = %err, - reason = "tagging_directive_header_invalid", - "Skipped replication tagging directive header detail" - ); - } - - match head_object_with_proxy_stats( - &bucket, - tgt_client.as_ref(), - &tgt_client.bucket, - &object, - self.version_id.map(|v| v.to_string()), + object_info, + &mut rinfo, ) .await - { - Ok(oi) => { - replication_action = replication_action_for_target_head(&object_info, &oi, self.op_type); - rinfo.replication_status = ReplicationStatusType::Completed; - if replication_action == ReplicationAction::None { - if self.op_type == ReplicationType::ExistingObject - && replication_target_head_is_newer_null_version(&object_info, &oi) - { - warn!( - event = EVENT_RESYNC_RUNTIME_SKIPPED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %bucket, - object = %object, - arn = %tgt_client.arn, - endpoint = %tgt_client.to_url(), - reason = "target_newer_than_source_null_version", - "Skipping replication because newer target version exists" - ); - send_local_event(EventArgs { - event_name: EventName::ObjectReplicationNotTracked.to_string(), - bucket_name: bucket.clone(), - object: object_info.clone(), - user_agent: "Internal: [Replication]".to_string(), - ..Default::default() - }); - } - - if object_info.target_replication_status(&tgt_client.arn) == ReplicationStatusType::Pending - || object_info.target_replication_status(&tgt_client.arn) == ReplicationStatusType::Failed - || self.op_type == ReplicationType::ExistingObject - { - rinfo.replication_action = replication_action; - rinfo.replication_status = ReplicationStatusType::Completed; - } - - if rinfo.replication_status == ReplicationStatusType::Completed - && self.op_type == ReplicationType::ExistingObject - && !tgt_client.reset_id.is_empty() - { - rinfo.resync_timestamp = format!( - "{};{}", - OffsetDateTime::now_utc() - .format(&Rfc3339) - .unwrap_or_else(|_| "invalid-time".to_string()), - tgt_client.reset_id - ); - rinfo.replication_resynced = true; - } - - rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); - - return rinfo; - } - } - Err(e) => { - if is_version_id_format_mismatch(&e) { - // Version-ID format mismatch: retry without versionId and compare ETags. - match head_object_fallback(&bucket, &tgt_client, &object).await { - Ok(Some(oi)) => { - replication_action = if replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref()) { - ReplicationAction::None - } else { - ReplicationAction::All - }; - } - Ok(None) => { - replication_action = ReplicationAction::All; - } - Err(e2) => { - rinfo.error = Some(e2.to_string()); - debug!( - event = EVENT_RESYNC_RUNTIME_SKIPPED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %bucket, - arn = %tgt_client.arn, - error = %e2, - reason = "head_object_fallback_failed", - "Failed replication head-object fallback" - ); - send_local_event(EventArgs { - event_name: EventName::ObjectReplicationNotTracked.to_string(), - bucket_name: bucket.clone(), - object: object_info, - user_agent: "Internal: [Replication]".to_string(), - ..Default::default() - }); - rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); - return rinfo; - } - } - } else if e.as_service_error().is_some_and(|se| se.is_not_found()) { - replication_action = ReplicationAction::All; - } else { - rinfo.error = Some(e.to_string()); - debug!( - event = EVENT_RESYNC_RUNTIME_SKIPPED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %bucket, - arn = %tgt_client.arn, - error = %e, - reason = "head_object_failed", - "Skipped replication because head-object failed" - ); - - send_local_event(EventArgs { - event_name: EventName::ObjectReplicationNotTracked.to_string(), - bucket_name: bucket.clone(), - object: object_info, - user_agent: "Internal: [Replication]".to_string(), - ..Default::default() - }); - - rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); - return rinfo; - } - } + else { + return rinfo; }; rinfo.replication_status = ReplicationStatusType::Completed; @@ -3276,14 +3204,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { // AWS-style targets permanently FAILED and never converging // (backlog#860 / #799 B11). if self.op_type == ReplicationType::ExistingObject && !tgt_client.reset_id.is_empty() { - rinfo.resync_timestamp = format!( - "{};{}", - OffsetDateTime::now_utc() - .format(&Rfc3339) - .unwrap_or_else(|_| "invalid-time".to_string()), - tgt_client.reset_id - ); - rinfo.replication_resynced = true; + apply_replication_resync_timestamp(&mut rinfo, &tgt_client.reset_id); } rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); return rinfo; @@ -3295,91 +3216,39 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { let (put_opts, is_multipart) = match replication_put_object_options(&tgt_client.storage_class, &object_info) { Ok((put_opts, is_mp)) => (put_opts, is_mp), Err(e) => { - // Unsupported source metadata (e.g. managed SSE) is a fail-closed - // condition: report FAILED so the composite status and the - // OperationFailedReplication event reflect that nothing reached - // the target, instead of leaking the optimistic Completed above. - rinfo.replication_status = ReplicationStatusType::Failed; - rinfo.error = Some(e.to_string()); - warn!( - event = EVENT_RESYNC_TARGET_OPERATION_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %bucket, - arn = %tgt_client.arn, - operation = "build_put_options", - error = %e, - "Replication target operation failed" - ); - send_local_event(EventArgs { - event_name: EventName::ObjectReplicationNotTracked.to_string(), - bucket_name: bucket.clone(), - object: object_info, - user_agent: "Internal: [Replication]".to_string(), - ..Default::default() - }); - - rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); + fail_replicate_all_put_options(&mut rinfo, &tgt_client, &bucket, object_info, &e, start_time); return rinfo; } }; - let has_tagging_replication = !put_opts.user_tags.is_empty(); - if let Some(err) = if is_multipart { - drop(gr); - let result = replicate_object_with_multipart(MultipartReplicationContext { - storage: storage.clone(), - cli: tgt_client.clone(), - src_bucket: &bucket, - dst_bucket: &tgt_client.bucket, + if let Some(err) = replicate_all_payload_to_target( + ReplicateAllPayloadContext { + storage: &storage, + tgt_client: &tgt_client, + bucket: &bucket, object: &object, object_info: &object_info, obj_opts: &obj_opts, arn: &rinfo.arn, + transfer_size, + is_multipart, put_opts, - }) - .await; - record_proxy_request(&bucket, "PutObject", result.is_err()).await; - if has_tagging_replication { - record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await; - } - result.err() - } else { - gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn); - let byte_stream = async_read_to_bytestream(gr.stream); - let result = tgt_client - .put_object(&tgt_client.bucket, &object, transfer_size, byte_stream, &put_opts) - .await - .map(|assigned_version_id| { - audit_target_version_identity( - &tgt_client, - &put_opts.internal.source_version_id, - assigned_version_id.as_deref(), - ) - }) - .map_err(|e| std::io::Error::other(e.to_string())); - record_proxy_request(&bucket, "PutObject", result.is_err()).await; - if has_tagging_replication { - record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await; - } - result.err() - } { - rinfo.replication_status = ReplicationStatusType::Failed; - rinfo.error = Some(err.to_string()); - warn!( - event = EVENT_RESYNC_TARGET_OPERATION_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = %bucket, - arn = %tgt_client.arn, - object = %object, - operation = "put_object", - error = ?err, - "Replication target operation failed" - ); - rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); + }, + gr, + ) + .await + { + fail_replicate_all_put_object(&mut rinfo, &tgt_client, &bucket, &object, &err, start_time).await; + return rinfo; + } - mark_replication_target_offline_if_needed(&tgt_client, &err).await; + // First SSE-C passthrough PUT against this target: verify the replica + // kept its decryption material before reporting COMPLETED. + if ssec_audit_required + && !audit_ssec_passthrough_replica(&tgt_client, &bucket, &object, self.version_id.map(|v| v.to_string()), &mut rinfo) + .await + { + rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); return rinfo; } @@ -3406,6 +3275,447 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { } } +/// Build the initial replication outcome DTO for `replicate_all`, seeded with +/// the metadata-only action and a failed status until the target confirms +/// otherwise. +fn replicate_all_target_info(roi: &ReplicateObjectInfo, tgt_client: &TargetClient) -> ReplicatedTargetInfo { + ReplicatedTargetInfo { + arn: tgt_client.arn.clone(), + size: roi.actual_size, + replication_action: ReplicationAction::Metadata, + op_type: roi.op_type, + replication_status: ReplicationStatusType::Failed, + prev_replication_status: roi.target_replication_status(&tgt_client.arn), + endpoint: tgt_client.endpoint.clone(), + secure: tgt_client.secure, + ..Default::default() + } +} + +/// Log and notify that replication was skipped because the target is offline. +fn note_replicate_all_target_offline(roi: &ReplicateObjectInfo, bucket: &str, tgt_client: &TargetClient) { + debug!( + event = EVENT_RESYNC_RUNTIME_SKIPPED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + arn = %tgt_client.arn, + target = %tgt_client.to_url(), + reason = "target_offline", + "Skipped replication because target is offline" + ); + send_local_event(EventArgs { + event_name: EventName::ObjectReplicationNotTracked.to_string(), + bucket_name: bucket.to_string(), + object: roi.to_object_info(), + user_agent: "Internal: [Replication]".to_string(), + ..Default::default() + }); +} + +/// Build the source-side read options for `replicate_all`. +fn replicate_all_read_options(roi: &ReplicateObjectInfo, versioned: bool, version_suspended: bool) -> ObjectOptions { + ObjectOptions { + version_id: roi.version_id.map(|v| v.to_string()), + version_suspended, + versioned, + replication_request: true, + // SSE-C passthrough reads the stored ciphertext verbatim; the + // decrypting reader cannot serve it (no customer key server-side). + raw_data_movement_read: roi.ssec, + ..Default::default() + } +} + +/// Log and notify that replication was skipped because the source object +/// reader is unavailable; missing objects/versions stay silent. +fn note_replicate_all_reader_unavailable(roi: &ReplicateObjectInfo, bucket: &str, tgt_client: &TargetClient, e: &Error) { + if !(is_err_object_not_found(e) || is_err_version_not_found(e)) { + debug!( + event = EVENT_RESYNC_RUNTIME_SKIPPED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + arn = %tgt_client.arn, + error = %e, + reason = "object_reader_unavailable", + "Skipped replication because object reader is unavailable" + ); + send_local_event(EventArgs { + event_name: EventName::ObjectReplicationNotTracked.to_string(), + bucket_name: bucket.to_string(), + object: roi.to_object_info(), + user_agent: "Internal: [Replication]".to_string(), + ..Default::default() + }); + } +} + +/// Log and notify that replication was skipped because the actual object size +/// is unavailable. +fn note_replicate_all_size_unavailable(bucket: &str, tgt_client: &TargetClient, object_info: ObjectInfo, e: &std::io::Error) { + debug!( + event = EVENT_RESYNC_RUNTIME_SKIPPED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + arn = %tgt_client.arn, + error = %e, + reason = "actual_size_unavailable", + "Skipped replication because actual object size is unavailable" + ); + send_local_event(EventArgs { + event_name: EventName::ObjectReplicationNotTracked.to_string(), + bucket_name: bucket.to_string(), + object: object_info, + user_agent: "Internal: [Replication]".to_string(), + ..Default::default() + }); +} + +/// Log and notify that replication was skipped because the target bucket is +/// empty. +fn note_replicate_all_target_bucket_empty(bucket: &str, tgt_client: &TargetClient, object_info: ObjectInfo) { + debug!( + event = EVENT_RESYNC_RUNTIME_SKIPPED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + arn = %tgt_client.arn, + reason = "target_bucket_empty", + "Skipped replication because target bucket is empty" + ); + send_local_event(EventArgs { + event_name: EventName::ObjectReplicationNotTracked.to_string(), + bucket_name: bucket.to_string(), + object: object_info, + user_agent: "Internal: [Replication]".to_string(), + ..Default::default() + }); +} + +/// Build the stat options for the target metadata comparison, logging (without +/// failing) when the tagging directive header cannot be set. +fn replicate_all_stat_options(object_info: &ObjectInfo, bucket: &str, tgt_client: &TargetClient) -> StatObjectOptions { + let mut sopts = StatObjectOptions { + version_id: object_info.version_id.map(|v| v.to_string()).unwrap_or_default(), + internal: AdvancedGetOptions { + replication_proxy_request: "false".to_string(), + ..Default::default() + }, + ..Default::default() + }; + + if let Err(err) = sopts.set(AMZ_TAGGING_DIRECTIVE, "ACCESS") { + debug!( + event = EVENT_RESYNC_RUNTIME_SKIPPED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + arn = %tgt_client.arn, + error = %err, + reason = "tagging_directive_header_invalid", + "Skipped replication tagging directive header detail" + ); + } + + sopts +} + +/// Record a failed payload transfer: mark the outcome FAILED, log the target +/// operation failure, and take the target offline when the error is a network +/// failure. +async fn fail_replicate_all_put_object( + rinfo: &mut ReplicatedTargetInfo, + tgt_client: &Arc, + bucket: &str, + object: &str, + err: &std::io::Error, + start_time: OffsetDateTime, +) { + rinfo.replication_status = ReplicationStatusType::Failed; + rinfo.error = Some(err.to_string()); + warn!( + event = EVENT_RESYNC_TARGET_OPERATION_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + arn = %tgt_client.arn, + object = %object, + operation = "put_object", + error = ?err, + "Replication target operation failed" + ); + rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); + + mark_replication_target_offline_if_needed(tgt_client, err).await; +} + +/// Stamp the replication outcome as resynced against the target's current +/// reset id. +fn apply_replication_resync_timestamp(rinfo: &mut ReplicatedTargetInfo, reset_id: &str) { + rinfo.resync_timestamp = format!( + "{};{}", + OffsetDateTime::now_utc() + .format(&Rfc3339) + .unwrap_or_else(|_| "invalid-time".to_string()), + reset_id + ); + rinfo.replication_resynced = true; +} + +/// Borrowed inputs for [`resolve_replicate_all_action`]. +struct ReplicateAllActionContext<'a> { + roi: &'a ReplicateObjectInfo, + tgt_client: &'a Arc, + bucket: &'a str, + object: &'a str, + start_time: OffsetDateTime, + /// N2: the target's SSE-C passthrough capability is still `Unknown`, so a + /// converged-looking replica must additionally prove its SSE-C material + /// survived before the comparison may settle COMPLETED. + ssec_audit_required: bool, +} + +/// Compare the source object against the target via HEAD and decide which +/// replication action is still required. Returns `None` after fully settling +/// `rinfo` when replication must stop here — either because the target already +/// matches or because the comparison failed. +async fn resolve_replicate_all_action( + ctx: ReplicateAllActionContext<'_>, + object_info: ObjectInfo, + rinfo: &mut ReplicatedTargetInfo, +) -> Option<(ReplicationAction, ObjectInfo)> { + let ReplicateAllActionContext { + roi, + tgt_client, + bucket, + object, + start_time, + ssec_audit_required, + } = ctx; + let replication_action; + match head_object_for_worker(tgt_client.as_ref(), &tgt_client.bucket, object, roi.version_id.map(|v| v.to_string())).await { + Ok(oi) => { + replication_action = replication_action_for_target_head(&object_info, &oi, roi.op_type); + rinfo.replication_status = ReplicationStatusType::Completed; + if replication_action == ReplicationAction::None { + // An SSE-C replica only counts as converged when the same HEAD + // proves its decryption material survived; a broken ciphertext + // copy from an earlier attempt matches by ETag. + if ssec_audit_required && !settle_ssec_passthrough_evidence(&oi, tgt_client, bucket, object, rinfo).await { + rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); + return None; + } + if roi.op_type == ReplicationType::ExistingObject + && replication_target_head_is_newer_null_version(&object_info, &oi) + { + warn!( + event = EVENT_RESYNC_RUNTIME_SKIPPED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + object = %object, + arn = %tgt_client.arn, + endpoint = %tgt_client.to_url(), + reason = "target_newer_than_source_null_version", + "Skipping replication because newer target version exists" + ); + send_local_event(EventArgs { + event_name: EventName::ObjectReplicationNotTracked.to_string(), + bucket_name: bucket.to_string(), + object: object_info.clone(), + user_agent: "Internal: [Replication]".to_string(), + ..Default::default() + }); + } + + if object_info.target_replication_status(&tgt_client.arn) == ReplicationStatusType::Pending + || object_info.target_replication_status(&tgt_client.arn) == ReplicationStatusType::Failed + || roi.op_type == ReplicationType::ExistingObject + { + rinfo.replication_action = replication_action; + rinfo.replication_status = ReplicationStatusType::Completed; + } + + if rinfo.replication_status == ReplicationStatusType::Completed + && roi.op_type == ReplicationType::ExistingObject + && !tgt_client.reset_id.is_empty() + { + apply_replication_resync_timestamp(rinfo, &tgt_client.reset_id); + } + + rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); + + return None; + } + } + Err(e) => { + if is_version_id_format_mismatch(&e) { + // Version-ID format mismatch: retry without versionId and compare ETags. + match head_object_fallback(tgt_client, object).await { + Ok(Some(oi)) => { + replication_action = if replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref()) { + if ssec_audit_required + && !settle_ssec_passthrough_evidence(&oi, tgt_client, bucket, object, rinfo).await + { + rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); + return None; + } + ReplicationAction::None + } else { + ReplicationAction::All + }; + } + Ok(None) => { + replication_action = ReplicationAction::All; + } + Err(e2) => { + rinfo.error = Some(e2.to_string()); + debug!( + event = EVENT_RESYNC_RUNTIME_SKIPPED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + arn = %tgt_client.arn, + error = %e2, + reason = "head_object_fallback_failed", + "Failed replication head-object fallback" + ); + send_local_event(EventArgs { + event_name: EventName::ObjectReplicationNotTracked.to_string(), + bucket_name: bucket.to_string(), + object: object_info, + user_agent: "Internal: [Replication]".to_string(), + ..Default::default() + }); + rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); + return None; + } + } + } else if e.as_service_error().is_some_and(|se| se.is_not_found()) { + replication_action = ReplicationAction::All; + } else { + rinfo.error = Some(e.to_string()); + debug!( + event = EVENT_RESYNC_RUNTIME_SKIPPED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + arn = %tgt_client.arn, + error = %e, + reason = "head_object_failed", + "Skipped replication because head-object failed" + ); + + send_local_event(EventArgs { + event_name: EventName::ObjectReplicationNotTracked.to_string(), + bucket_name: bucket.to_string(), + object: object_info, + user_agent: "Internal: [Replication]".to_string(), + ..Default::default() + }); + + rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); + return None; + } + } + }; + + Some((replication_action, object_info)) +} + +/// Record a fail-closed put-options failure. +/// Unsupported source metadata (e.g. managed SSE) is a fail-closed +/// condition: report FAILED so the composite status and the +/// OperationFailedReplication event reflect that nothing reached +/// the target, instead of leaking the optimistic Completed set earlier. +fn fail_replicate_all_put_options( + rinfo: &mut ReplicatedTargetInfo, + tgt_client: &TargetClient, + bucket: &str, + object_info: ObjectInfo, + e: &Error, + start_time: OffsetDateTime, +) { + rinfo.replication_status = ReplicationStatusType::Failed; + rinfo.error = Some(e.to_string()); + warn!( + event = EVENT_RESYNC_TARGET_OPERATION_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %bucket, + arn = %tgt_client.arn, + operation = "build_put_options", + error = %e, + "Replication target operation failed" + ); + send_local_event(EventArgs { + event_name: EventName::ObjectReplicationNotTracked.to_string(), + bucket_name: bucket.to_string(), + object: object_info, + user_agent: "Internal: [Replication]".to_string(), + ..Default::default() + }); + + rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); +} + +/// Borrowed inputs shared by both transports of the `replicate_all` payload +/// transfer step. +struct ReplicateAllPayloadContext<'a, S: ReplicationObjectIO> { + storage: &'a Arc, + tgt_client: &'a Arc, + bucket: &'a str, + object: &'a str, + object_info: &'a ObjectInfo, + obj_opts: &'a ObjectOptions, + arn: &'a str, + transfer_size: i64, + is_multipart: bool, + put_opts: PutObjectOptions, +} + +/// Ship the object payload to the replication target over the multipart or +/// single-put transport, returning the transport error when the upload fails. +async fn replicate_all_payload_to_target( + ctx: ReplicateAllPayloadContext<'_, S>, + mut gr: GetObjectReader, +) -> Option { + if ctx.is_multipart { + drop(gr); + let result = replicate_object_with_multipart(MultipartReplicationContext { + storage: ctx.storage.clone(), + cli: ctx.tgt_client.clone(), + src_bucket: ctx.bucket, + dst_bucket: &ctx.tgt_client.bucket, + object: ctx.object, + object_info: ctx.object_info, + obj_opts: ctx.obj_opts, + arn: ctx.arn, + put_opts: ctx.put_opts, + }) + .await; + result.err() + } else { + gr.stream = wrap_with_bandwidth_monitor(gr.stream, &ctx.put_opts, ctx.bucket, ctx.arn); + let byte_stream = async_read_to_bytestream(gr.stream); + let result = ctx + .tgt_client + .put_object(&ctx.tgt_client.bucket, ctx.object, ctx.transfer_size, byte_stream, &ctx.put_opts) + .await + .map(|assigned_version_id| { + audit_target_version_identity( + ctx.tgt_client, + &ctx.put_opts.internal.source_version_id, + assigned_version_id.as_deref(), + ) + }) + .map_err(|e| std::io::Error::other(e.to_string())); + result.err() + } +} + fn wrap_with_bandwidth_monitor_with_header( stream: Box, bucket: &str, @@ -3615,27 +3925,6 @@ mod tests { ReplicationTargetStore::register_test_target(target).await; } - /// P1-19 runtime spot-check exemption matrix: drift only applies when the - /// source addressed a real version uuid. - #[test] - fn test_version_identity_drift_judgment() { - let source = "6fa459ea-ee8a-3ca4-894e-db77e160355e"; - for (sent, got, expected) in [ - (source, Some(source), false), - (source, Some("0e304ce5-33e9-4b8a-9b12-9e40a53e6ded"), true), - (source, None, true), - ("", None, false), - ("null", Some("anything"), false), - ("00000000-0000-0000-0000-000000000000", Some("anything"), false), - ] { - assert_eq!( - version_identity_drifted(sent, got), - expected, - "sent {sent:?} got {got:?} must judge drift = {expected}" - ); - } - } - #[test] fn resync_admission_configuration_is_bounded() { assert_eq!(ENV_REPL_RESYNC_MAX_JOBS, "RUSTFS_REPL_RESYNC_MAX_JOBS"); @@ -3678,15 +3967,6 @@ mod tests { drop((first, second)); } - #[test] - fn replication_target_offline_error_classifier_is_network_scoped() { - assert!(is_replication_target_offline_error(&"put_object dispatch failure: connector error")); - assert!(is_replication_target_offline_error(&"request TimeoutError after retry")); - assert!(is_replication_target_offline_error(&"tcp connect error: connection refused")); - assert!(!is_replication_target_offline_error(&"put_object failed: AccessDenied: denied")); - assert!(!is_replication_target_offline_error(&"put_object failed: NoSuchBucket")); - } - #[tokio::test] async fn replication_target_network_failure_marks_target_offline() { let endpoint = format!("http://network-failure-{}.example:9000", Uuid::new_v4()); @@ -3960,27 +4240,6 @@ mod tests { ); } - /// P1-21 regression guard for the outcome formula. A fully successful - /// delete-marker replication must acknowledge its MRF entry: the formula - /// once carried `&& !requires_delayed_purge`, which pinned every - /// delete-marker entry to Missed and retained the whole backlog forever. - /// (Deterministically staging a marker-creation entry in the durable - /// journal from e2e would require saturating the worker queues, so the - /// formula is pinned here instead; the purge-intent replay half is pinned - /// by the delayed-purge e2e pair.) - #[test] - fn test_replicate_delete_outcome_is_not_held_hostage_by_the_delayed_purge() { - assert!( - replicate_delete_outcome(1, 1, true, true, &ReplicationStatusType::Completed), - "a completed delete-marker replication must be acknowledgeable even though a delayed purge watch is pending" - ); - assert!(!replicate_delete_outcome(0, 0, true, true, &ReplicationStatusType::Completed)); - assert!(!replicate_delete_outcome(2, 1, true, true, &ReplicationStatusType::Completed)); - assert!(!replicate_delete_outcome(1, 1, false, true, &ReplicationStatusType::Completed)); - assert!(!replicate_delete_outcome(1, 1, true, false, &ReplicationStatusType::Completed)); - assert!(!replicate_delete_outcome(1, 1, true, true, &ReplicationStatusType::Failed)); - } - /// P1-21 review follow-up: a target whose recorded marker version is /// inconsistent must be reported as a per-target FAILURE. Treating the /// refusal as success let the watcher and the MRF replay drop the purge @@ -4020,41 +4279,6 @@ mod tests { ); } - #[test] - fn test_delete_marker_purge_mrf_entry_replays_through_the_stale_marker_branch() { - let delete_marker_version_id = Uuid::new_v4(); - let dobj = DeletedObjectReplicationInfo { - delete_object: ReplicationDeletedObject { - object_name: "doc.txt".to_string(), - // A version-purge flavored source event: the entry must still - // be reshaped as a marker-creation delete so replay funnels - // into the stale-marker branch instead of re-running the full - // delete replication (whose source-state stamping would fail - // against the already-purged version). - delete_marker: false, - version_id: Some(Uuid::new_v4()), - delete_marker_version_id: Some(delete_marker_version_id), - ..Default::default() - }, - bucket: "bucket-a".to_string(), - ..Default::default() - }; - - let entry = delete_marker_purge_mrf_entry(&dobj, vec!["arn:a".to_string()]); - - assert!(entry.delete_marker, "purge intents must replay as marker-creation deletes"); - assert_eq!(entry.version_id, None, "the purged data version must not leak into the replay"); - assert_eq!(entry.delete_marker_version_id, Some(delete_marker_version_id)); - assert_eq!( - entry.target_arns, - vec!["arn:a".to_string()], - "only the targets whose purge failed may be retried" - ); - assert_eq!(entry.retry_count, 0); - assert_eq!(entry.bucket, "bucket-a"); - assert_eq!(entry.object, "doc.txt"); - } - #[test] fn test_is_retryable_delete_replication_head_error_allows_delete_marker_head_responses() { assert!( @@ -4562,60 +4786,4 @@ mod tests { assert!(resync_state_accepts_update(¤t, &matching)); assert!(!resync_state_accepts_update(¤t, &stale)); } - - #[test] - fn test_resync_status_duration_only_tracks_terminal_status() { - let start = match OffsetDateTime::from_unix_timestamp(1_700_000_000) { - Ok(start) => start, - Err(err) => panic!("valid test timestamp: {err}"), - }; - let end = start + time::Duration::seconds(2); - - assert_eq!( - resync_status_duration(ResyncStatusType::ResyncCompleted, Some(start), end), - Some(std::time::Duration::from_millis(2000)) - ); - assert_eq!(resync_status_duration(ResyncStatusType::ResyncStarted, Some(start), end), None); - assert_eq!(resync_status_duration(ResyncStatusType::ResyncFailed, None, end), None); - } - - #[test] - fn target_delete_version_id_preserves_explicit_null_purges() { - let version_id = Uuid::new_v4(); - - assert_eq!(target_delete_version_id(version_id, true), Some(version_id.to_string())); - assert_eq!(target_delete_version_id(Uuid::nil(), true).as_deref(), Some(NULL_VERSION_ID)); - assert_eq!(target_delete_version_id(Uuid::nil(), false), None); - } - - #[test] - fn delete_marker_purge_prefers_the_recorded_target_version() { - let source = Uuid::new_v4(); - let arn = "arn:rustfs:replication::target:bucket"; - - // No recorded mapping: fall back to deriving from the source uuid. - assert_eq!(delete_marker_purge_version_id(None, arn, source), Some(Some(source.to_string()))); - - // Recorded mapping wins — a generic S3 target assigns its own id, so the - // derived one would purge the wrong version or nothing at all. - let mut state = ReplicationState::default(); - state - .target_delete_marker_version_ids - .insert(arn.to_string(), "target-assigned-id".to_string()); - assert_eq!( - delete_marker_purge_version_id(Some(&state), arn, source), - Some(Some("target-assigned-id".to_string())) - ); - - // A mapping recorded for a different ARN must not be reused. - assert_eq!( - delete_marker_purge_version_id(Some(&state), "arn:rustfs:replication::other:bucket", source), - Some(Some(source.to_string())) - ); - - // Inconsistent persisted metadata: refuse to purge rather than guess. - let mut corrupt = state.clone(); - corrupt.target_delete_marker_version_ids_corrupt = true; - assert_eq!(delete_marker_purge_version_id(Some(&corrupt), arn, source), None); - } } diff --git a/crates/ecstore/src/bucket/replication/replication_state.rs b/crates/ecstore/src/bucket/replication/replication_state.rs index acd4b696a..ce2f1a36f 100644 --- a/crates/ecstore/src/bucket/replication/replication_state.rs +++ b/crates/ecstore/src/bucket/replication/replication_state.rs @@ -1161,6 +1161,31 @@ mod tests { assert!(all.contains_key("proxy-only-bucket")); } + /// Pins the read-proxy metric contract (backlog#1675 P1-5): the API + /// strings the GET/HEAD/Tagging proxy paths record map onto the + /// get/head/tagging totals, and only unexpected failures raise the + /// failed counters. + #[tokio::test] + async fn test_proxy_stats_map_read_proxy_apis_to_totals() { + let stats = ReplicationStats::new(); + stats.inc_proxy("proxy-bucket", "GetObject", false).await; + stats.inc_proxy("proxy-bucket", "GetObject", true).await; + stats.inc_proxy("proxy-bucket", "HeadObject", false).await; + stats.inc_proxy("proxy-bucket", "GetObjectTagging", false).await; + stats.inc_proxy("proxy-bucket", "PutObjectTagging", false).await; + stats.inc_proxy("proxy-bucket", "DeleteObjectTagging", true).await; + + let metric = stats.get_proxy_stats("proxy-bucket").await; + assert_eq!(metric.get_total, 2); + assert_eq!(metric.get_failed, 1); + assert_eq!(metric.head_total, 1); + assert_eq!(metric.head_failed, 0); + assert_eq!(metric.get_tag_total, 1); + assert_eq!(metric.put_tag_total, 1); + assert_eq!(metric.delete_tag_total, 1); + assert_eq!(metric.delete_tag_failed, 1); + } + #[tokio::test] async fn test_calculate_bucket_replication_stats_merges_resync_metrics() { let stats = ReplicationStats::new(); diff --git a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs index e82ab7e52..87d94779e 100644 --- a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs @@ -36,11 +36,15 @@ use time::OffsetDateTime; use time::format_description::well_known::Rfc3339; pub(crate) use crate::bucket::bucket_target_sys::{ - AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient, + AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient, resolve_read_api_version_id, }; #[cfg(test)] pub(crate) use crate::bucket::target::BucketTarget; pub(crate) use crate::bucket::target::BucketTargets; +pub use rustfs_replication::SsecPassthroughCapability; +pub(crate) use rustfs_replication::{ + SsecPassthroughGate, is_replication_target_offline_error, ssec_passthrough_gate, version_identity_drifted, +}; use super::replication_config_store::ReplicationConfigStore; use super::replication_error_boundary::{Error, Result}; @@ -65,6 +69,8 @@ static STANDARD_HEADERS: &[&str] = &[ ]; const ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED: &str = "replication source contains unsupported encryption metadata"; +pub(crate) const ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED: &str = "replication target does not support SSE-C passthrough: the replica would lose its decryption material \ + (run ?replication-check to re-probe)"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum ReplicationSourceEncryption { @@ -146,6 +152,13 @@ pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap bool { + rustfs_replication::ssec_passthrough_evidence_present(head.sse_customer_algorithm.as_deref()) +} + pub(crate) struct ReplicationTargetStore; impl ReplicationTargetStore { @@ -165,6 +178,17 @@ impl ReplicationTargetStore { BucketTargetSys::get().mark_target_offline(target_client).await } + /// Returns the cached verdict and whether it has outlived its TTL. + pub(crate) async fn ssec_passthrough_capability(arn: &str) -> (SsecPassthroughCapability, bool) { + BucketTargetSys::get().ssec_passthrough_capability(arn).await + } + + pub(crate) async fn record_ssec_passthrough_capability(arn: &str, capability: SsecPassthroughCapability) { + BucketTargetSys::get() + .record_ssec_passthrough_capability(arn, capability) + .await + } + #[cfg(test)] pub(crate) async fn register_test_target(target_client: &Arc) { BucketTargetSys::get().arn_remotes_map.write().await.insert( @@ -898,6 +922,27 @@ mod tests { } } + /// Pins the HeadObjectOutput field extraction feeding the crate-owned + /// evidence judgment (the gate/evidence policy matrix itself is pinned in + /// `rustfs-replication`'s object tests). + #[test] + fn ssec_passthrough_evidence_requires_customer_algorithm_echo() { + let with_evidence = HeadObjectOutput::builder().sse_customer_algorithm("AES256").build(); + assert!(ssec_passthrough_evidence_present(&with_evidence)); + + let empty_algorithm = HeadObjectOutput::builder().sse_customer_algorithm("").build(); + assert!( + !ssec_passthrough_evidence_present(&empty_algorithm), + "an empty echo is not evidence of preserved SSE-C material" + ); + + let without_evidence = HeadObjectOutput::builder().e_tag("\"abc\"").content_length(8).build(); + assert!( + !ssec_passthrough_evidence_present(&without_evidence), + "a plain HEAD response must classify the target as having dropped the material" + ); + } + #[test] fn replication_put_options_adds_ssec_checksum_metadata() { let metadata = HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]); diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 624cb7593..99ec6c83e 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -277,6 +277,20 @@ pub struct ObjectOptions { /// fence avoids recursively acquiring the read lock behind a queued writer. pub bucket_lifecycle_lock_fence: Option, pub replication_request: bool, + /// True when the inbound request carried the + /// `{x-rustfs-,x-minio-}source-proxy-request` header family with the + /// value "true": the request was already proxied by a replication peer, + /// so this server must not proxy a local miss onward (anti-loop, + /// MinIO-compatible). The header only disables proxying — it grants no + /// capability — so no authorization gate is required to honor it. + pub proxy_request: bool, + /// True when the `source-proxy-request` header family was present at + /// all, regardless of value (MinIO's `ProxyHeaderSet`). A replication + /// peer sends `source-proxy-request: false` on its worker convergence + /// HEADs precisely so the receiver answers locally instead of proxying + /// back — otherwise a proxied 404->200 echo makes the worker believe the + /// object already converged and it never replicates it. + pub proxy_header_set: bool, /// Source-cluster LWW timestamps carried by an authorized replication /// request; None when the source never modified the category. Only the /// replication-authorized options builders may set these. diff --git a/crates/filemeta/src/fileinfo.rs b/crates/filemeta/src/fileinfo.rs index 393975c18..42613185e 100644 --- a/crates/filemeta/src/fileinfo.rs +++ b/crates/filemeta/src/fileinfo.rs @@ -42,6 +42,9 @@ const FILEINFO_PART_BITMAP_WORD_BITS: usize = std::mem::size_of::() * 8; const FILEINFO_PART_BITMAP_WORDS: usize = MAX_FILEINFO_PARTS.div_ceil(FILEINFO_PART_BITMAP_WORD_BITS); // Additional constants from Go version +// Intentionally duplicated (S3 wire literal): rustfs-replication and +// rustfs-object-data-cache carry their own independent "null" constants so +// they stay free of a rustfs-filemeta dependency. Keep all three in sync. pub const NULL_VERSION_ID: &str = "null"; // pub const RUSTFS_ERASURE_UPGRADED: &str = "x-rustfs-internal-erasure-upgraded"; diff --git a/crates/replication/src/delete.rs b/crates/replication/src/delete.rs index 3a0c32bf8..3b6fbb016 100644 --- a/crates/replication/src/delete.rs +++ b/crates/replication/src/delete.rs @@ -14,8 +14,13 @@ use std::any::Any; +use uuid::Uuid; + use crate::storage_api::DeletedObject; -use crate::{MrfOpKind, MrfReplicateEntry, ReplicationState, ReplicationType, ReplicationWorkerOperation}; +use crate::{ + MrfOpKind, MrfReplicateEntry, NULL_VERSION_ID, REPLICATE_EXISTING_DELETE, ReplicateObjectInfo, ReplicationState, + ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, +}; #[derive(Debug, Clone, Default)] pub struct DeletedObjectReplicationInfo { @@ -117,17 +122,114 @@ pub fn is_retryable_delete_replication_head_error(is_not_found: bool, code: Opti !(is_not_found || matches!(code, Some("MethodNotAllowed" | "405"))) } +/// Build the delete-replication work item for an existing delete marker or +/// version purge discovered during a resync scan. +pub fn resync_existing_delete_replication_info(roi: &ReplicateObjectInfo, target_arn: &str) -> DeletedObjectReplicationInfo { + let (version_id, dm_version_id) = if roi.version_purge_status.is_empty() { + (None, roi.version_id) + } else { + (roi.version_id, None) + }; + + DeletedObjectReplicationInfo { + delete_object: DeletedObject { + object_name: roi.name.clone(), + delete_marker_version_id: dm_version_id, + version_id, + replication_state: roi.replication_state.clone(), + delete_marker: roi.delete_marker, + delete_marker_mtime: roi.mod_time, + ..Default::default() + }, + bucket: roi.bucket.clone(), + event_type: REPLICATE_EXISTING_DELETE.to_string(), + op_type: ReplicationType::ExistingObject, + target_arn: target_arn.to_string(), + ..Default::default() + } +} + +/// Whether a delete replication fully succeeded — the MRF replay acknowledges +/// (drops) an entry exactly when this returns true. +/// +/// The delayed purge is deliberately NOT an input: holding the outcome hostage +/// to it (`&& !requires_delayed_purge`) forced `false` for every delete-marker +/// entry and retained them all in the durable MRF journal forever. Purge +/// failures persist their own purge-intent entry instead +/// (`watch_and_purge_source_delete_marker`), and replays of those entries +/// report purge success through `purge_stale_delete_marker_targets`. +pub fn replicate_delete_outcome( + expected_targets: usize, + replicated_targets: usize, + state_persisted: bool, + source_state_verified: bool, + replication_status: &ReplicationStatusType, +) -> bool { + expected_targets > 0 + && replicated_targets == expected_targets + && state_persisted + && source_state_verified + && *replication_status == ReplicationStatusType::Completed +} + +pub fn target_delete_version_id(version_id: Uuid, version_purge: bool) -> Option { + if version_id.is_nil() { + version_purge.then(|| NULL_VERSION_ID.to_string()) + } else { + Some(version_id.to_string()) + } +} + +/// Which version a delete-marker purge should address on one target. +/// +/// `None` means do not purge at all: the recorded mapping disagreed across the +/// dual internal prefixes, and guessing an id could destroy a live version on +/// the target. `Some(id)` is the exact version the target reported when it +/// accepted the marker; falling back to a source-derived id is only correct +/// when the target mirrors source version ids, which a generic S3 target does +/// not. +pub fn delete_marker_purge_version_id( + state: Option<&ReplicationState>, + arn: &str, + delete_marker_version_id: Uuid, +) -> Option> { + if state.is_some_and(|state| state.target_delete_marker_version_ids_corrupt) { + return None; + } + let recorded = state.and_then(|state| state.target_delete_marker_version_ids.get(arn).cloned()); + Some(match recorded { + Some(version_id) => Some(version_id), + None => target_delete_version_id(delete_marker_version_id, true), + }) +} + +/// Shape an exhausted purge intent as a marker-creation delete entry. Replay +/// reconstructs it with `delete_marker: true`, finds the source marker gone, +/// and funnels into the stale-marker branch of `replicate_delete_with_outcome` +/// — which re-runs the purge without touching source state and reports purge +/// success as the replay outcome. +pub fn delete_marker_purge_mrf_entry(dobj: &DeletedObjectReplicationInfo, failed_arns: Vec) -> MrfReplicateEntry { + let mut entry = dobj.to_mrf_entry(); + entry.delete_marker = true; + entry.version_id = None; + entry.retry_count = 0; + entry.target_arns = failed_arns; + entry +} + #[cfg(test)] mod tests { use std::collections::HashMap; use super::{ - DeletedObjectReplicationInfo, is_retryable_delete_replication_head_error, is_version_delete_replication, - should_retry_delete_marker_purge, + DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id, + is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, + should_retry_delete_marker_purge, target_delete_version_id, }; use crate::storage_api::DeletedObject; use crate::{ - MrfOpKind, ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, VersionPurgeStatusType, + MrfOpKind, NULL_VERSION_ID, ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, + VersionPurgeStatusType, }; use uuid::Uuid; @@ -328,4 +430,100 @@ mod tests { assert!(!is_retryable_delete_replication_head_error(true, Some("NoSuchKey"))); assert!(is_retryable_delete_replication_head_error(false, Some("AccessDenied"))); } + + /// P1-21 regression guard for the outcome formula. A fully successful + /// delete-marker replication must acknowledge its MRF entry: the formula + /// once carried `&& !requires_delayed_purge`, which pinned every + /// delete-marker entry to Missed and retained the whole backlog forever. + /// (Deterministically staging a marker-creation entry in the durable + /// journal from e2e would require saturating the worker queues, so the + /// formula is pinned here instead; the purge-intent replay half is pinned + /// by the delayed-purge e2e pair.) + #[test] + fn test_replicate_delete_outcome_is_not_held_hostage_by_the_delayed_purge() { + assert!( + replicate_delete_outcome(1, 1, true, true, &ReplicationStatusType::Completed), + "a completed delete-marker replication must be acknowledgeable even though a delayed purge watch is pending" + ); + assert!(!replicate_delete_outcome(0, 0, true, true, &ReplicationStatusType::Completed)); + assert!(!replicate_delete_outcome(2, 1, true, true, &ReplicationStatusType::Completed)); + assert!(!replicate_delete_outcome(1, 1, false, true, &ReplicationStatusType::Completed)); + assert!(!replicate_delete_outcome(1, 1, true, false, &ReplicationStatusType::Completed)); + assert!(!replicate_delete_outcome(1, 1, true, true, &ReplicationStatusType::Failed)); + } + + #[test] + fn test_delete_marker_purge_mrf_entry_replays_through_the_stale_marker_branch() { + let delete_marker_version_id = Uuid::new_v4(); + let dobj = DeletedObjectReplicationInfo { + delete_object: DeletedObject { + object_name: "doc.txt".to_string(), + // A version-purge flavored source event: the entry must still + // be reshaped as a marker-creation delete so replay funnels + // into the stale-marker branch instead of re-running the full + // delete replication (whose source-state stamping would fail + // against the already-purged version). + delete_marker: false, + version_id: Some(Uuid::new_v4()), + delete_marker_version_id: Some(delete_marker_version_id), + ..Default::default() + }, + bucket: "bucket-a".to_string(), + ..Default::default() + }; + + let entry = delete_marker_purge_mrf_entry(&dobj, vec!["arn:a".to_string()]); + + assert!(entry.delete_marker, "purge intents must replay as marker-creation deletes"); + assert_eq!(entry.version_id, None, "the purged data version must not leak into the replay"); + assert_eq!(entry.delete_marker_version_id, Some(delete_marker_version_id)); + assert_eq!( + entry.target_arns, + vec!["arn:a".to_string()], + "only the targets whose purge failed may be retried" + ); + assert_eq!(entry.retry_count, 0); + assert_eq!(entry.bucket, "bucket-a"); + assert_eq!(entry.object, "doc.txt"); + } + + #[test] + fn target_delete_version_id_preserves_explicit_null_purges() { + let version_id = Uuid::new_v4(); + + assert_eq!(target_delete_version_id(version_id, true), Some(version_id.to_string())); + assert_eq!(target_delete_version_id(Uuid::nil(), true).as_deref(), Some(NULL_VERSION_ID)); + assert_eq!(target_delete_version_id(Uuid::nil(), false), None); + } + + #[test] + fn delete_marker_purge_prefers_the_recorded_target_version() { + let source = Uuid::new_v4(); + let arn = "arn:rustfs:replication::target:bucket"; + + // No recorded mapping: fall back to deriving from the source uuid. + assert_eq!(delete_marker_purge_version_id(None, arn, source), Some(Some(source.to_string()))); + + // Recorded mapping wins — a generic S3 target assigns its own id, so the + // derived one would purge the wrong version or nothing at all. + let mut state = ReplicationState::default(); + state + .target_delete_marker_version_ids + .insert(arn.to_string(), "target-assigned-id".to_string()); + assert_eq!( + delete_marker_purge_version_id(Some(&state), arn, source), + Some(Some("target-assigned-id".to_string())) + ); + + // A mapping recorded for a different ARN must not be reused. + assert_eq!( + delete_marker_purge_version_id(Some(&state), "arn:rustfs:replication::other:bucket", source), + Some(Some(source.to_string())) + ); + + // Inconsistent persisted metadata: refuse to purge rather than guess. + let mut corrupt = state.clone(); + corrupt.target_delete_marker_version_ids_corrupt = true; + assert_eq!(delete_marker_purge_version_id(Some(&corrupt), arn, source), None); + } } diff --git a/crates/replication/src/filemeta.rs b/crates/replication/src/filemeta.rs index 68064dc19..8dfdb26c5 100644 --- a/crates/replication/src/filemeta.rs +++ b/crates/replication/src/filemeta.rs @@ -27,6 +27,11 @@ use uuid::Uuid; pub const REPLICATION_RESET: &str = "replication-reset"; pub const REPLICATION_STATUS: &str = "replication-status"; +/// The S3 wire spelling of the unversioned ("null") version id. Owned here as +/// part of the replication wire contracts; `rustfs-filemeta` keeps its own +/// copy of the same literal (the crates are intentionally independent). +pub const NULL_VERSION_ID: &str = "null"; + // ReplicateQueued - replication being queued trail pub const REPLICATE_QUEUED: &str = "replicate:queue"; diff --git a/crates/replication/src/lib.rs b/crates/replication/src/lib.rs index bb0c08385..cb0761b5c 100644 --- a/crates/replication/src/lib.rs +++ b/crates/replication/src/lib.rs @@ -37,11 +37,12 @@ pub use config::{ validate_replication_config_target_arns, }; pub use delete::{ - DeletedObjectReplicationInfo, is_retryable_delete_replication_head_error, is_version_delete_replication, - should_retry_delete_marker_purge, + DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id, + is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, + resync_existing_delete_replication_info, should_retry_delete_marker_purge, target_delete_version_id, }; pub use filemeta::{ - REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING, + NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING, REPLICATE_INCOMING_DELETE, REPLICATE_MRF, REPLICATE_QUEUED, REPLICATION_RESET, REPLICATION_STATUS, ReplicateDecision, ReplicateObjectInfo, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, @@ -58,8 +59,9 @@ pub use multipart::{ replication_multipart_complete_actual_size, replication_multipart_part_plan, }; pub use object::{ - ReplicationSourceObject, ReplicationTargetObject, content_matches_by_etag, replication_action_for_target, - replication_etags_match, target_is_newer_than_source_null_version, + ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate, content_matches_by_etag, + is_replication_target_offline_error, replication_action_for_target, replication_etags_match, + ssec_passthrough_evidence_present, ssec_passthrough_gate, target_is_newer_than_source_null_version, version_identity_drifted, }; pub use operation::{ MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteSource, ReplicationDeleteStateSource, @@ -76,7 +78,7 @@ pub use queue::{ pub use resync::{ BucketReplicationResyncStatus, Error, RESYNC_FILE_MAX_BYTES, Result, ResyncOpts, ResyncStatusType, TargetReplicationResyncStatus, decode_resync_file, encode_resync_file, is_version_id_mismatch, resync_state_accepts_update, - sanitize_resync_error_detail, should_auto_resume_resync, should_count_head_proxy_failure, + resync_status_duration, sanitize_resync_error_detail, should_auto_resume_resync, should_count_head_proxy_failure, }; pub use rule::ReplicationRuleExt; pub use runtime::{ diff --git a/crates/replication/src/object.rs b/crates/replication/src/object.rs index afbb2ef46..7ce7761ca 100644 --- a/crates/replication/src/object.rs +++ b/crates/replication/src/object.rs @@ -157,11 +157,130 @@ fn comparable_metadata(metadata: Option<&HashMap>) -> HashMap) -> bool { + if source_version_id.is_empty() { + return false; + } + // A nil source uuid travels as the literal "null" (unversioned-source + // semantics); no identity contract applies to it. + if uuid::Uuid::parse_str(source_version_id) + .map(|uuid| uuid.is_nil()) + .unwrap_or(true) + { + return false; + } + assigned_version_id != Some(source_version_id) +} + +const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[ + "dispatch failure", + "timeouterror", + "timed out", + "connection refused", + "connection reset", + "connection closed", + "connection aborted", + "broken pipe", + "dns error", + "failed to lookup address", + "name or service not known", + "deadline has elapsed", + "tcp connect error", +]; + +/// True when a target operation error reads as a network/transport failure — +/// the only class of error that should mark a replication target offline. +pub fn is_replication_target_offline_error(err: &(impl std::fmt::Display + ?Sized)) -> bool { + let message = err.to_string().to_ascii_lowercase(); + REPLICATION_TARGET_OFFLINE_ERROR_MARKERS + .iter() + .any(|marker| message.contains(marker)) +} + +/// Whether a replication target preserves the SSE-C passthrough transport +/// headers (`X-Rustfs-Replication-*`) end to end. +/// +/// A target that silently drops those headers (MinIO, generic S3) stores the +/// forwarded ciphertext without its decryption material — an unreadable +/// replica that used to report COMPLETED. The replication worker audits the +/// first passthrough PUT per target (HEAD-back for SSE-C evidence) and caches +/// the verdict; a fresh `Unsupported` fails SSE-C replication closed before +/// any PUT is sent. The verdict cache (per-ARN map, lifecycle, and TTL) is +/// owned by the runtime's bucket target system; this crate owns only the +/// verdict vocabulary and the gate policy below. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum SsecPassthroughCapability { + #[default] + Unknown, + Supported, + Unsupported, +} + +/// Fail-closed decision for an SSE-C passthrough replication attempt, derived +/// from the target's cached [`SsecPassthroughCapability`]. Pure so the policy +/// can migrate with the worker (M2) without dragging the cache along; the +/// caller computes `expired` from the cache record's age (see the runtime's +/// `SSEC_PASSTHROUGH_CAPABILITY_TTL`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SsecPassthroughGate { + /// Not an SSE-C object, or the target has a fresh proof that it preserves + /// the passthrough transport headers: replicate without a HEAD-back audit. + Proceed, + /// No usable verdict — first SSE-C attempt since the target was (re)built, + /// or the recorded verdict (in either direction) aged out: PUT, then HEAD + /// the replica back and require SSE-C evidence before reporting COMPLETED. + ProceedWithAudit, + /// The target was recently proven to drop the passthrough headers: do not + /// send the PUT, report FAILED (the object stays on the normal MRF retry + /// channel and re-audits once the verdict expires). + FailClosed, +} + +pub fn ssec_passthrough_gate(ssec: bool, capability: SsecPassthroughCapability, expired: bool) -> SsecPassthroughGate { + if !ssec { + return SsecPassthroughGate::Proceed; + } + // An expired verdict — Supported or Unsupported — must be re-earned: a + // stale Unsupported would otherwise stick forever after a target upgrade, + // and a stale Supported would fail open after a backend swap behind the + // same endpoint. + if expired { + return SsecPassthroughGate::ProceedWithAudit; + } + match capability { + SsecPassthroughCapability::Supported => SsecPassthroughGate::Proceed, + SsecPassthroughCapability::Unknown => SsecPassthroughGate::ProceedWithAudit, + SsecPassthroughCapability::Unsupported => SsecPassthroughGate::FailClosed, + } +} + +/// True when a replication-check HEAD of the replica proves the SSE-C +/// material survived passthrough: a RustFS target restores the transport +/// headers into the stored SSE-C keys and its HEAD echoes +/// `x-amz-server-side-encryption-customer-algorithm` (the replication-check +/// exemption skips key validation but not the metadata echo). A target that +/// dropped the headers stored a plain object and echoes nothing. The caller +/// extracts the echoed customer-algorithm value from its HEAD response type. +pub fn ssec_passthrough_evidence_present(sse_customer_algorithm: Option<&str>) -> bool { + sse_customer_algorithm.is_some_and(|algo| !algo.is_empty()) +} + #[cfg(test)] mod tests { use super::{ - ReplicationSourceObject, ReplicationTargetObject, content_matches_by_etag, replication_action_for_target, - replication_etags_match, target_is_newer_than_source_null_version, + ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate, + content_matches_by_etag, is_replication_target_offline_error, replication_action_for_target, replication_etags_match, + ssec_passthrough_evidence_present, ssec_passthrough_gate, target_is_newer_than_source_null_version, + version_identity_drifted, }; use crate::filemeta::{ReplicationAction, ReplicationType}; use crate::http::AMZ_OBJECT_LOCK_MODE; @@ -269,6 +388,96 @@ mod tests { ); } + /// P1-19 runtime spot-check exemption matrix: drift only applies when the + /// source addressed a real version uuid. + #[test] + fn test_version_identity_drift_judgment() { + let source = "6fa459ea-ee8a-3ca4-894e-db77e160355e"; + for (sent, got, expected) in [ + (source, Some(source), false), + (source, Some("0e304ce5-33e9-4b8a-9b12-9e40a53e6ded"), true), + (source, None, true), + ("", None, false), + ("null", Some("anything"), false), + ("00000000-0000-0000-0000-000000000000", Some("anything"), false), + ] { + assert_eq!( + version_identity_drifted(sent, got), + expected, + "sent {sent:?} got {got:?} must judge drift = {expected}" + ); + } + } + + #[test] + fn replication_target_offline_error_classifier_is_network_scoped() { + assert!(is_replication_target_offline_error("put_object dispatch failure: connector error")); + assert!(is_replication_target_offline_error("request TimeoutError after retry")); + assert!(is_replication_target_offline_error("tcp connect error: connection refused")); + assert!(!is_replication_target_offline_error("put_object failed: AccessDenied: denied")); + assert!(!is_replication_target_offline_error("put_object failed: NoSuchBucket")); + } + + /// N2 fail-closed policy: SSE-C replication may only proceed silently + /// against a target with a FRESH proof that it preserves the passthrough + /// transport headers. Unknown targets must be audited; freshly-flagged + /// dropping targets must never receive the PUT; an expired verdict in + /// EITHER direction must be re-earned through the audit — a sticky + /// Unsupported would outlive a target upgrade, and a sticky Supported + /// would fail open after a backend swap behind the same endpoint. + #[test] + fn ssec_passthrough_gate_is_fail_closed_and_ttl_bounded() { + for capability in [ + SsecPassthroughCapability::Unknown, + SsecPassthroughCapability::Supported, + SsecPassthroughCapability::Unsupported, + ] { + for expired in [false, true] { + assert_eq!( + ssec_passthrough_gate(false, capability, expired), + SsecPassthroughGate::Proceed, + "non-SSE-C objects must never be gated on the passthrough capability" + ); + } + } + assert_eq!( + ssec_passthrough_gate(true, SsecPassthroughCapability::Supported, false), + SsecPassthroughGate::Proceed + ); + assert_eq!( + ssec_passthrough_gate(true, SsecPassthroughCapability::Unknown, false), + SsecPassthroughGate::ProceedWithAudit + ); + assert_eq!( + ssec_passthrough_gate(true, SsecPassthroughCapability::Unsupported, false), + SsecPassthroughGate::FailClosed + ); + // Expiry flips both directions back to the audit. + assert_eq!( + ssec_passthrough_gate(true, SsecPassthroughCapability::Unsupported, true), + SsecPassthroughGate::ProceedWithAudit, + "an expired Unsupported verdict must allow a re-audit (upgraded target recovers without operator action)" + ); + assert_eq!( + ssec_passthrough_gate(true, SsecPassthroughCapability::Supported, true), + SsecPassthroughGate::ProceedWithAudit, + "an expired Supported verdict must be re-proven (backend swap behind the same endpoint must not fail open)" + ); + } + + #[test] + fn ssec_passthrough_evidence_requires_customer_algorithm_echo() { + assert!(ssec_passthrough_evidence_present(Some("AES256"))); + assert!( + !ssec_passthrough_evidence_present(Some("")), + "an empty echo is not evidence of preserved SSE-C material" + ); + assert!( + !ssec_passthrough_evidence_present(None), + "a plain HEAD response must classify the target as having dropped the material" + ); + } + #[test] fn replication_action_detects_tags_and_object_lock_metadata_differences() { let mut source_metadata = HashMap::new(); diff --git a/crates/replication/src/resync.rs b/crates/replication/src/resync.rs index 2380d9e18..144181dd2 100644 --- a/crates/replication/src/resync.rs +++ b/crates/replication/src/resync.rs @@ -309,6 +309,31 @@ pub fn is_version_id_mismatch(code: Option<&str>, raw_status: Option) -> bo } } +pub fn resync_status_duration( + status: ResyncStatusType, + start_time: Option, + now: OffsetDateTime, +) -> Option { + if !matches!( + status, + ResyncStatusType::ResyncCompleted | ResyncStatusType::ResyncFailed | ResyncStatusType::ResyncCanceled + ) { + return None; + } + + let millis = (now - start_time?).whole_milliseconds(); + if millis < 0 { + return None; + } + + let millis = if millis > i128::from(u64::MAX) { + u64::MAX + } else { + u64::try_from(millis).ok()? + }; + Some(std::time::Duration::from_millis(millis)) +} + #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct BucketReplicationResyncStatus { pub version: u16, @@ -712,6 +737,22 @@ mod tests { assert!(!should_auto_resume_resync(ResyncStatusType::ResyncFailed)); } + #[test] + fn test_resync_status_duration_only_tracks_terminal_status() { + let start = match OffsetDateTime::from_unix_timestamp(1_700_000_000) { + Ok(start) => start, + Err(err) => panic!("valid test timestamp: {err}"), + }; + let end = start + time::Duration::seconds(2); + + assert_eq!( + resync_status_duration(ResyncStatusType::ResyncCompleted, Some(start), end), + Some(std::time::Duration::from_millis(2000)) + ); + assert_eq!(resync_status_duration(ResyncStatusType::ResyncStarted, Some(start), end), None); + assert_eq!(resync_status_duration(ResyncStatusType::ResyncFailed, None, end), None); + } + #[test] fn resync_state_accepts_update_only_for_matching_run() { let current = TargetReplicationResyncStatus { diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 18eef001b..219ebbffe 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -66,18 +66,20 @@ use rustfs_config::{ }; use rustfs_iam::error::is_err_no_such_service_account; use rustfs_iam::federation::OIDC_VIRTUAL_PARENT_CLAIM; +use rustfs_iam::store::object::ObjectStore; use rustfs_iam::store::{MappedPolicy, UserType, sr_wire_user_type, user_type_from_sr_wire}; use rustfs_iam::sys::{ - NewServiceAccountOpts, SITE_REPLICATOR_SERVICE_ACCOUNT, UpdateServiceAccountOpts, get_claims_from_token_with_secret, + IamSys, NewServiceAccountOpts, SITE_REPLICATOR_SERVICE_ACCOUNT, UpdateServiceAccountOpts, get_claims_from_token_with_secret, }; use rustfs_madmin::{ AddOrUpdateUserReq, BucketBandwidth, GroupAddRemove, GroupStatus, IDPSettings, InProgressMetric, InQueueMetric, LDAPConfigSettings, LDAPSettings, OpenIDProviderSettings, PeerInfo, PeerSite, QStat, ReplProxyMetric, ReplicateAddStatus, ReplicateEditStatus, ReplicateRemoveStatus, ResyncBucketStatus, SITE_REPL_API_VERSION, SR_IAM_ITEM_STS_ACC, SR_IAM_ITEM_STS_ACC_LEGACY, SRBucketInfo, SRBucketMeta, SRBucketStatsSummary, SRGroupInfo, SRGroupStatsSummary, SRIAMItem, - SRIAMPolicy, SRILMExpiryStatsSummary, SRInfo, SRMetric, SRMetricsSummary, SRPeerError, SRPeerJoinReq, SRPendingOperation, - SRPolicyMapping, SRPolicyStatsSummary, SRRemoveReq, SRResyncOpStatus, SRRetryStats, SRSessionPolicy, SRSiteSummary, - SRStateEditReq, SRStateInfo, SRStatusInfo, SRSvcAccCreate, SRUserStatsSummary, SiteReplicationInfo, SyncStatus, WorkerStat, + SRIAMPolicy, SRIAMUser, SRILMExpiryStatsSummary, SRInfo, SRMetric, SRMetricsSummary, SRPeerError, SRPeerJoinReq, + SRPendingOperation, SRPolicyMapping, SRPolicyStatsSummary, SRRemoveReq, SRResyncOpStatus, SRRetryStats, SRSTSCredential, + SRSessionPolicy, SRSiteSummary, SRStateEditReq, SRStateInfo, SRStatusInfo, SRSvcAccChange, SRSvcAccCreate, + SRUserStatsSummary, SiteReplicationInfo, SyncStatus, WorkerStat, }; use rustfs_policy::policy::{ Policy, @@ -4584,7 +4586,10 @@ async fn build_metrics_summary(local_peer: &PeerInfo) -> SRMetricsSummary { head_failed_total: non_negative_u64(node.proxy_head_failed), put_tag_total: non_negative_u64(node.proxy_put_tag_total), put_tag_failed_total: non_negative_u64(node.proxy_put_tag_failed), - ..Default::default() + get_tag_total: non_negative_u64(node.proxy_get_tag_total), + get_tag_failed_total: non_negative_u64(node.proxy_get_tag_failed), + remove_tag_total: non_negative_u64(node.proxy_delete_tag_total), + remove_tag_failed_total: non_negative_u64(node.proxy_delete_tag_failed), }, metrics, uptime: node.uptime, @@ -9358,247 +9363,16 @@ async fn apply_iam_item(item: SRIAMItem) -> S3Result<()> { let incoming_updated_at = item.updated_at; match item.r#type.as_str() { - "policy" => { - if let Some(policy) = item.policy { - let policy: Policy = - serde_json::from_value(policy).map_err(|e| s3_error!(InvalidRequest, "invalid policy body: {}", e))?; - iam_sys.set_policy(&item.name, policy).await.map_err(ApiError::from)?; - } else { - iam_sys.delete_policy(&item.name, true).await.map_err(ApiError::from)?; - } - Ok(()) - } - "policy-mapping" => { - let Some(mapping) = item.policy_mapping else { - return Err(s3_error!(InvalidRequest, "policyMapping is required")); - }; - let user_type = - user_type_from_sr_wire(mapping.user_type).ok_or_else(|| s3_error!(InvalidRequest, "invalid userType"))?; - iam_sys - .policy_db_set(&mapping.user_or_group, user_type, mapping.is_group, &mapping.policy) - .await - .map_err(ApiError::from)?; - Ok(()) - } - "group-info" => { - let Some(group_info) = item.group_info else { - return Err(s3_error!(InvalidRequest, "groupInfo is required")); - }; - let update = group_info.update_req; - if !group_info_requires_upsert(&update) { - iam_sys - .remove_users_from_group(&update.group, update.members) - .await - .map_err(ApiError::from)?; - return Ok(()); - } - - iam_sys - .add_users_to_group(&update.group, update.members) - .await - .map_err(ApiError::from)?; - iam_sys - .set_group_status(&update.group, matches!(update.status, GroupStatus::Enabled)) - .await - .map_err(ApiError::from)?; - Ok(()) - } + "policy" => apply_iam_policy_item(&iam_sys, &item.name, item.policy).await, + "policy-mapping" => apply_iam_policy_mapping_item(&iam_sys, item.policy_mapping).await, + "group-info" => apply_iam_group_info_item(&iam_sys, item.group_info).await, // MinIO madmin-go sends `SRIAMItemSTSAcc = "sts-account"`. The legacy alias // `sts-credential` (emitted by older RustFS releases) stays accepted permanently // so mixed-version RustFS sites keep replicating STS credentials during rolling // upgrades; it is a compatibility layer, not temporary code. - SR_IAM_ITEM_STS_ACC | SR_IAM_ITEM_STS_ACC_LEGACY => { - let Some(sts_credential) = item.sts_credential else { - return Err(s3_error!(InvalidRequest, "stsCredential is required")); - }; - let Some(secret) = current_token_signing_key() else { - return Err(s3_error!(InvalidRequest, "token signing key not initialized")); - }; - let claims = get_claims_from_token_with_secret(&sts_credential.session_token, &secret) - .map_err(|e| s3_error!(InvalidRequest, "invalid STS session token: {e}"))?; - let expiration = claims - .get("exp") - .and_then(claims_unix_timestamp) - .map(OffsetDateTime::from_unix_timestamp) - .transpose() - .map_err(|e| s3_error!(InvalidRequest, "invalid STS expiry: {e}"))?; - let groups = string_list_claim(&claims, "groups"); - let compatibility_policy = sts_replication_compatibility_policy(&claims, &sts_credential.parent_policy_mapping); - let cred = rustfs_credentials::Credentials { - access_key: sts_credential.access_key.clone(), - secret_key: sts_credential.secret_key.clone(), - session_token: sts_credential.session_token.clone(), - expiration, - status: "on".to_string(), - parent_user: sts_credential.parent_user.clone(), - groups, - claims: Some(claims), - ..Default::default() - }; - iam_sys - .set_temp_user(&sts_credential.access_key, &cred, compatibility_policy) - .await - .map_err(ApiError::from)?; - Ok(()) - } - "iam-user" => { - let Some(user) = item.iam_user else { - return Err(s3_error!(InvalidRequest, "iamUser is required")); - }; - if let Some(local) = iam_sys.get_user(&user.access_key).await - && is_stale_update(local.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH), incoming_updated_at) - { - return Ok(()); - } - if user.is_delete_req { - iam_sys.delete_user(&user.access_key, true).await.map_err(ApiError::from)?; - } else { - let Some(user_req) = user.user_req else { - return Err(s3_error!(InvalidRequest, "userReq is required")); - }; - let is_status_only_update = user_req.secret_key.is_empty() && user_req.policy.is_none(); - if is_status_only_update { - iam_sys - .set_user_status(&user.access_key, user_req.status) - .await - .map_err(ApiError::from)?; - } else { - iam_sys - .create_user(&user.access_key, &user_req) - .await - .map_err(ApiError::from)?; - } - } - Ok(()) - } - "service-account" => { - let Some(change) = item.svc_acc_change else { - return Err(s3_error!(InvalidRequest, "serviceAccountChange is required")); - }; - let envelope = change.oidc_service_account_envelope; - if let Some(create) = change.create { - let local_updated_at = iam_sys - .get_user(&create.access_key) - .await - .map(|local| local.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH)); - let replicated_policy = if create.access_key == SITE_REPLICATOR_SERVICE_ACCOUNT { - if local_updated_at.is_some_and(|local_updated_at| is_stale_update(local_updated_at, incoming_updated_at)) { - return Ok(()); - } - ReplicatedServiceAccountPolicy { - policy: Some(site_replicator_service_account_policy()?), - is_envelope: false, - } - } else { - let Some(replicated_policy) = decode_service_account_replication_policy( - &create, - envelope.as_ref(), - incoming_updated_at, - local_updated_at, - )? - else { - return Ok(()); - }; - replicated_policy - }; - match iam_sys.get_service_account(&create.access_key).await { - Ok((existing, _)) => { - if existing.parent_user != create.parent { - return Err(s3_error!( - InvalidRequest, - "service account {} already exists with a different parent user", - create.access_key - )); - } - iam_sys - .update_service_account( - &create.access_key, - UpdateServiceAccountOpts { - name: replicated_policy.metadata_for_existing_account(create.name), - description: replicated_policy.metadata_for_existing_account(create.description), - session_policy: replicated_policy.for_existing_account(), - secret_key: Some(create.secret_key), - expiration: create.expiration, - status: (!create.status.is_empty()).then_some(create.status), - parent_user: None, - allow_site_replicator_account: create.access_key == SITE_REPLICATOR_SERVICE_ACCOUNT, - }, - ) - .await - .map_err(ApiError::from)?; - } - Err(err) if is_err_no_such_service_account(&err) => { - iam_sys - .new_service_account( - &create.parent, - Some(create.groups), - NewServiceAccountOpts { - session_policy: replicated_policy.policy, - access_key: create.access_key, - secret_key: create.secret_key, - name: (!create.name.is_empty()).then_some(create.name), - description: (!create.description.is_empty()).then_some(create.description), - expiration: create.expiration, - allow_site_replicator_account: true, - claims: Some(create.claims), - }, - ) - .await - .map_err(ApiError::from)?; - } - Err(err) => return Err(ApiError::from(err).into()), - } - return Ok(()); - } - - if let Some(update) = change.update { - if let Some(local) = iam_sys.get_user(&update.access_key).await - && is_stale_update(local.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH), incoming_updated_at) - { - return Ok(()); - } - let allow_site_replicator_account = update.access_key == SITE_REPLICATOR_SERVICE_ACCOUNT; - let session_policy = if allow_site_replicator_account { - Some(site_replicator_service_account_policy()?) - } else { - update.session_policy.as_str().and_then(|raw| serde_json::from_str(raw).ok()) - }; - iam_sys - .update_service_account( - &update.access_key, - UpdateServiceAccountOpts { - session_policy, - secret_key: (!update.secret_key.is_empty()).then_some(update.secret_key), - name: (!update.name.is_empty()).then_some(update.name), - description: (!update.description.is_empty()).then_some(update.description), - expiration: update.expiration, - status: (!update.status.is_empty()).then_some(update.status), - // Peers replicate credentials, never the local parent binding: - // each site resolves its own parent from its own IAM. - parent_user: None, - allow_site_replicator_account, - }, - ) - .await - .map_err(ApiError::from)?; - return Ok(()); - } - - if let Some(delete) = change.delete { - if let Some(local) = iam_sys.get_user(&delete.access_key).await - && is_stale_update(local.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH), incoming_updated_at) - { - return Ok(()); - } - iam_sys - .delete_service_account(&delete.access_key, true) - .await - .map_err(ApiError::from)?; - return Ok(()); - } - - Err(s3_error!(InvalidRequest, "serviceAccountChange is empty")) - } + SR_IAM_ITEM_STS_ACC | SR_IAM_ITEM_STS_ACC_LEGACY => apply_iam_sts_account_item(&iam_sys, item.sts_credential).await, + "iam-user" => apply_iam_user_item(&iam_sys, item.iam_user, incoming_updated_at).await, + "service-account" => apply_iam_service_account_item(&iam_sys, item.svc_acc_change, incoming_updated_at).await, _ => Err(s3_error!( NotImplemented, "site replication IAM item type `{}` is not supported", @@ -9607,6 +9381,252 @@ async fn apply_iam_item(item: SRIAMItem) -> S3Result<()> { } } +async fn apply_iam_policy_item(iam_sys: &IamSys, name: &str, policy: Option) -> S3Result<()> { + if let Some(policy) = policy { + let policy: Policy = + serde_json::from_value(policy).map_err(|e| s3_error!(InvalidRequest, "invalid policy body: {}", e))?; + iam_sys.set_policy(name, policy).await.map_err(ApiError::from)?; + } else { + iam_sys.delete_policy(name, true).await.map_err(ApiError::from)?; + } + Ok(()) +} + +async fn apply_iam_policy_mapping_item(iam_sys: &IamSys, policy_mapping: Option) -> S3Result<()> { + let Some(mapping) = policy_mapping else { + return Err(s3_error!(InvalidRequest, "policyMapping is required")); + }; + let user_type = user_type_from_sr_wire(mapping.user_type).ok_or_else(|| s3_error!(InvalidRequest, "invalid userType"))?; + iam_sys + .policy_db_set(&mapping.user_or_group, user_type, mapping.is_group, &mapping.policy) + .await + .map_err(ApiError::from)?; + Ok(()) +} + +async fn apply_iam_group_info_item(iam_sys: &IamSys, group_info: Option) -> S3Result<()> { + let Some(group_info) = group_info else { + return Err(s3_error!(InvalidRequest, "groupInfo is required")); + }; + let update = group_info.update_req; + if !group_info_requires_upsert(&update) { + iam_sys + .remove_users_from_group(&update.group, update.members) + .await + .map_err(ApiError::from)?; + return Ok(()); + } + + iam_sys + .add_users_to_group(&update.group, update.members) + .await + .map_err(ApiError::from)?; + iam_sys + .set_group_status(&update.group, matches!(update.status, GroupStatus::Enabled)) + .await + .map_err(ApiError::from)?; + Ok(()) +} + +async fn apply_iam_sts_account_item(iam_sys: &IamSys, sts_credential: Option) -> S3Result<()> { + let Some(sts_credential) = sts_credential else { + return Err(s3_error!(InvalidRequest, "stsCredential is required")); + }; + let Some(secret) = current_token_signing_key() else { + return Err(s3_error!(InvalidRequest, "token signing key not initialized")); + }; + let claims = get_claims_from_token_with_secret(&sts_credential.session_token, &secret) + .map_err(|e| s3_error!(InvalidRequest, "invalid STS session token: {e}"))?; + let expiration = claims + .get("exp") + .and_then(claims_unix_timestamp) + .map(OffsetDateTime::from_unix_timestamp) + .transpose() + .map_err(|e| s3_error!(InvalidRequest, "invalid STS expiry: {e}"))?; + let groups = string_list_claim(&claims, "groups"); + let compatibility_policy = sts_replication_compatibility_policy(&claims, &sts_credential.parent_policy_mapping); + let cred = rustfs_credentials::Credentials { + access_key: sts_credential.access_key.clone(), + secret_key: sts_credential.secret_key.clone(), + session_token: sts_credential.session_token.clone(), + expiration, + status: "on".to_string(), + parent_user: sts_credential.parent_user.clone(), + groups, + claims: Some(claims), + ..Default::default() + }; + iam_sys + .set_temp_user(&sts_credential.access_key, &cred, compatibility_policy) + .await + .map_err(ApiError::from)?; + Ok(()) +} + +async fn apply_iam_user_item( + iam_sys: &IamSys, + iam_user: Option, + incoming_updated_at: Option, +) -> S3Result<()> { + let Some(user) = iam_user else { + return Err(s3_error!(InvalidRequest, "iamUser is required")); + }; + if let Some(local) = iam_sys.get_user(&user.access_key).await + && is_stale_update(local.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH), incoming_updated_at) + { + return Ok(()); + } + if user.is_delete_req { + iam_sys.delete_user(&user.access_key, true).await.map_err(ApiError::from)?; + } else { + let Some(user_req) = user.user_req else { + return Err(s3_error!(InvalidRequest, "userReq is required")); + }; + let is_status_only_update = user_req.secret_key.is_empty() && user_req.policy.is_none(); + if is_status_only_update { + iam_sys + .set_user_status(&user.access_key, user_req.status) + .await + .map_err(ApiError::from)?; + } else { + iam_sys + .create_user(&user.access_key, &user_req) + .await + .map_err(ApiError::from)?; + } + } + Ok(()) +} + +async fn apply_iam_service_account_item( + iam_sys: &IamSys, + svc_acc_change: Option, + incoming_updated_at: Option, +) -> S3Result<()> { + let Some(change) = svc_acc_change else { + return Err(s3_error!(InvalidRequest, "serviceAccountChange is required")); + }; + let envelope = change.oidc_service_account_envelope; + if let Some(create) = change.create { + let local_updated_at = iam_sys + .get_user(&create.access_key) + .await + .map(|local| local.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH)); + let replicated_policy = if create.access_key == SITE_REPLICATOR_SERVICE_ACCOUNT { + if local_updated_at.is_some_and(|local_updated_at| is_stale_update(local_updated_at, incoming_updated_at)) { + return Ok(()); + } + ReplicatedServiceAccountPolicy { + policy: Some(site_replicator_service_account_policy()?), + is_envelope: false, + } + } else { + let Some(replicated_policy) = + decode_service_account_replication_policy(&create, envelope.as_ref(), incoming_updated_at, local_updated_at)? + else { + return Ok(()); + }; + replicated_policy + }; + match iam_sys.get_service_account(&create.access_key).await { + Ok((existing, _)) => { + if existing.parent_user != create.parent { + return Err(s3_error!( + InvalidRequest, + "service account {} already exists with a different parent user", + create.access_key + )); + } + iam_sys + .update_service_account( + &create.access_key, + UpdateServiceAccountOpts { + name: replicated_policy.metadata_for_existing_account(create.name), + description: replicated_policy.metadata_for_existing_account(create.description), + session_policy: replicated_policy.for_existing_account(), + secret_key: Some(create.secret_key), + expiration: create.expiration, + status: (!create.status.is_empty()).then_some(create.status), + parent_user: None, + allow_site_replicator_account: create.access_key == SITE_REPLICATOR_SERVICE_ACCOUNT, + }, + ) + .await + .map_err(ApiError::from)?; + } + Err(err) if is_err_no_such_service_account(&err) => { + iam_sys + .new_service_account( + &create.parent, + Some(create.groups), + NewServiceAccountOpts { + session_policy: replicated_policy.policy, + access_key: create.access_key, + secret_key: create.secret_key, + name: (!create.name.is_empty()).then_some(create.name), + description: (!create.description.is_empty()).then_some(create.description), + expiration: create.expiration, + allow_site_replicator_account: true, + claims: Some(create.claims), + }, + ) + .await + .map_err(ApiError::from)?; + } + Err(err) => return Err(ApiError::from(err).into()), + } + return Ok(()); + } + + if let Some(update) = change.update { + if let Some(local) = iam_sys.get_user(&update.access_key).await + && is_stale_update(local.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH), incoming_updated_at) + { + return Ok(()); + } + let allow_site_replicator_account = update.access_key == SITE_REPLICATOR_SERVICE_ACCOUNT; + let session_policy = if allow_site_replicator_account { + Some(site_replicator_service_account_policy()?) + } else { + update.session_policy.as_str().and_then(|raw| serde_json::from_str(raw).ok()) + }; + iam_sys + .update_service_account( + &update.access_key, + UpdateServiceAccountOpts { + session_policy, + secret_key: (!update.secret_key.is_empty()).then_some(update.secret_key), + name: (!update.name.is_empty()).then_some(update.name), + description: (!update.description.is_empty()).then_some(update.description), + expiration: update.expiration, + status: (!update.status.is_empty()).then_some(update.status), + // Peers replicate credentials, never the local parent binding: + // each site resolves its own parent from its own IAM. + parent_user: None, + allow_site_replicator_account, + }, + ) + .await + .map_err(ApiError::from)?; + return Ok(()); + } + + if let Some(delete) = change.delete { + if let Some(local) = iam_sys.get_user(&delete.access_key).await + && is_stale_update(local.update_at.unwrap_or(OffsetDateTime::UNIX_EPOCH), incoming_updated_at) + { + return Ok(()); + } + iam_sys + .delete_service_account(&delete.access_key, true) + .await + .map_err(ApiError::from)?; + return Ok(()); + } + + Err(s3_error!(InvalidRequest, "serviceAccountChange is empty")) +} + fn claims_unix_timestamp(value: &Value) -> Option { match value { Value::Number(number) => number.as_i64(), diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 717de0bee..14ad1fb5e 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -17,7 +17,8 @@ use super::storage_api::bucket::metadata_sys; use super::storage_api::bucket::replication::{self, BucketReplicationResyncStatus, BucketStats, ReplicationStatusType}; use super::storage_api::bucket::target::{BucketTarget, BucketTargetType, BucketTargets}; use super::storage_api::bucket::target_sys::{ - BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, TargetClient, append_version_id_query, + BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, SsecPassthroughCapability, TargetClient, + append_version_id_query, }; use super::storage_api::bucket::versioning_sys::BucketVersioningSys; use super::storage_api::bucket::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; @@ -70,6 +71,9 @@ use rustfs_utils::egress::{OutboundDnsResolver, OutboundPolicy}; use rustfs_utils::http::headers::{ AMZ_CHECKSUM_CRC32, AMZ_CHECKSUM_CRC32C, AMZ_CHECKSUM_CRC64NVME, AMZ_CHECKSUM_SHA1, AMZ_CHECKSUM_SHA256, AMZ_CHECKSUM_TYPE, }; +use rustfs_utils::http::object_encryption_keys::{ + REPLICATION_SSEC_ALGORITHM_HEADER, REPLICATION_SSEC_KEY_MD5_HEADER, REPLICATION_SSEC_ORIGINAL_SIZE_HEADER, +}; use rustfs_utils::http::{ SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, get_source_scheme, insert_header, @@ -213,6 +217,13 @@ const REPLICATION_CHECK_ERROR_MAX_BYTES: usize = 512; /// RustFS extension code (no madmin analogue): the target does not adopt the /// source version id, breaking the version-identity replication contract. const REPLICATION_CHECK_CODE_VERSION_MISMATCH: &str = "BucketRemoteTargetVersionMismatch"; +/// RustFS extension code (no madmin analogue): the target drops the +/// `X-Rustfs-Replication-*` SSE-C passthrough headers, so an SSE-C replica +/// would lose its decryption material (N2 fail-closed). +const REPLICATION_CHECK_CODE_SSEC_PASSTHROUGH: &str = "BucketRemoteSsecPassthroughUnsupported"; +/// Syntactically valid stand-in SSE-C key MD5 for the passthrough probe (the +/// probe object is never decrypted; it only has to round-trip the metadata). +const REPLICATION_CHECK_SSEC_PROBE_KEY_MD5: &str = "AAAAAAAAAAAAAAAAAAAAAA=="; #[derive(Debug, Clone, serde::Serialize)] struct ReplicationCheckResponse { @@ -254,6 +265,8 @@ struct ReplicationCheckPhases { put: ReplicationCheckPhaseStatus, #[serde(rename = "VersionFidelity")] version_fidelity: ReplicationCheckPhaseStatus, + #[serde(rename = "SsecPassthrough")] + ssec_passthrough: ReplicationCheckPhaseStatus, #[serde(rename = "DeleteMarker")] delete_marker: ReplicationCheckPhaseStatus, #[serde(rename = "VersionDelete")] @@ -1852,7 +1865,7 @@ fn build_replication_check_response(mut targets: Vec { + BucketTargetSys::get() + .record_ssec_passthrough_capability(&target.arn, SsecPassthroughCapability::Supported) + .await; + } + ("FAILED", Some(REPLICATION_CHECK_CODE_SSEC_PASSTHROUGH)) => { + BucketTargetSys::get() + .record_ssec_passthrough_capability(&target.arn, SsecPassthroughCapability::Unsupported) + .await; + } + _ => {} + } + result } @@ -2087,6 +2119,15 @@ struct ReplicationProbePutOutcome { response_version_id: Option, } +/// Outcome of the SSE-C passthrough probe: whether the HEAD-back of the probe +/// replica echoed SSE-C evidence (the customer-algorithm header a RustFS +/// target restores from the passthrough transport headers), plus the version +/// the target assigned so cleanup can address it. +struct ReplicationSsecProbeOutcome { + evidence_present: bool, + version_id: Option, +} + struct ReplicationProbeMultipartError { primary: S3ClientError, cleanup_error: Option, @@ -2109,9 +2150,14 @@ trait ReplicationProbeOperations { /// there: a target can adopt PutObject version ids and still mint its own /// for CreateMultipartUpload. async fn multipart_put(&mut self) -> Result; + /// PUT a probe version carrying the SSE-C passthrough transport headers, + /// HEAD it back through the replication-check channel, and report whether + /// the SSE-C evidence survived. Cleanup of the created version is the + /// caller's job (the outcome carries its version id). + async fn ssec_passthrough_probe(&mut self) -> Result; async fn create_delete_marker(&mut self, version_id: Option<&str>) -> Result, S3ClientError>; async fn delete_version(&mut self, version_id: Option<&str>) -> Result<(), S3ClientError>; - async fn cleanup(&mut self, known_version_ids: [Option<&str>; 3]) -> Result<(), String>; + async fn cleanup(&mut self, known_version_ids: [Option<&str>; 4]) -> Result<(), String>; } struct RemoteReplicationProbeOperations<'a> { @@ -2131,6 +2177,10 @@ impl ReplicationProbeOperations for RemoteReplicationProbeOperations<'_> { multipart_put_replication_probe_object(self.client, self.bucket, self.key, self.time).await } + async fn ssec_passthrough_probe(&mut self) -> Result { + ssec_passthrough_probe_object(self.client, self.bucket, self.key, self.time).await + } + async fn create_delete_marker(&mut self, version_id: Option<&str>) -> Result, S3ClientError> { delete_replication_probe_object( self.client, @@ -2154,7 +2204,7 @@ impl ReplicationProbeOperations for RemoteReplicationProbeOperations<'_> { .map(|_| ()) } - async fn cleanup(&mut self, known_version_ids: [Option<&str>; 3]) -> Result<(), String> { + async fn cleanup(&mut self, known_version_ids: [Option<&str>; 4]) -> Result<(), String> { cleanup_replication_probe(self.client, self.bucket, self.key, known_version_ids).await } } @@ -2175,6 +2225,7 @@ fn version_fidelity_error(api: &str, outcome: &ReplicationProbePutOutcome) -> Op async fn execute_replication_probe(result: &mut ReplicationCheckTargetStatus, operations: &mut impl ReplicationProbeOperations) { let mut probe_version_id = None; let mut multipart_probe_version_id = None; + let mut ssec_probe_version_id = None; let mut delete_marker_version_id = None; let mut cleanup_required = true; let mut multipart_cleanup_error = None; @@ -2230,6 +2281,38 @@ async fn execute_replication_probe(result: &mut ReplicationCheckTargetStatus, op } } + // N2: probe SSE-C passthrough with the same transport headers live + // replication sends. A target that drops them (MinIO, generic S3) stores + // the probe as a plain object and echoes no SSE-C evidence on the + // HEAD-back; SSE-C replicas there would silently lose their decryption + // material, so the target must be flagged with a machine-readable code. + // Deliberately unlike VersionFidelity, a failed SsecPassthrough phase + // does NOT fail the target overall: version-identity drift breaks the + // replication contract for every object, while dropped SSE-C passthrough + // headers only limit a capability — a plaintext-only deployment against a + // MinIO target is perfectly healthy and must not turn red. The phase's + // own FAILED + machine-readable Code remains for madmin consumers (and + // the verdict still reaches the runtime capability cache). + if result.phases.put.status == "OK" && result.phases.version_fidelity.status == "OK" { + match operations.ssec_passthrough_probe().await { + Ok(outcome) => { + ssec_probe_version_id = outcome.version_id; + if outcome.evidence_present { + result.phases.ssec_passthrough = ReplicationCheckPhaseStatus::passed(); + } else { + let error = "target drops SSE-C passthrough replication headers; \ + SSE-C replicas would lose their decryption material on this target"; + result.phases.ssec_passthrough = + ReplicationCheckPhaseStatus::failed_with_code(error, REPLICATION_CHECK_CODE_SSEC_PASSTHROUGH); + } + } + Err(err) => { + let error = format_replication_check_client_error(&err, ReplicationCheckFailureContext::ReplicateObject); + result.phases.ssec_passthrough = ReplicationCheckPhaseStatus::failed(&error); + } + } + } + if result.phases.put.status == "OK" && result.phases.version_fidelity.status == "OK" { match operations.create_delete_marker(probe_version_id.as_deref()).await { Ok(version_id) => { @@ -2258,6 +2341,7 @@ async fn execute_replication_probe(result: &mut ReplicationCheckTargetStatus, op .cleanup([ probe_version_id.as_deref(), multipart_probe_version_id.as_deref(), + ssec_probe_version_id.as_deref(), delete_marker_version_id.as_deref(), ]) .await @@ -2552,6 +2636,72 @@ async fn put_replication_probe_object( }) } +/// PUT a fresh probe version carrying the SSE-C passthrough transport headers +/// (the wire shape live SSE-C replication uses), then HEAD it back through the +/// worker channel (replication-check exemption + proxy suppression). A RustFS +/// target restores the transport headers into stored SSE-C metadata and its +/// HEAD echoes `x-amz-server-side-encryption-customer-algorithm`; a target +/// that dropped the headers echoes nothing. The probe body is never SSE-C +/// encrypted — only the metadata round-trip matters — and the version is +/// deleted by the shared probe cleanup. +async fn ssec_passthrough_probe_object( + target_client: &TargetClient, + target_bucket: &str, + probe_key: &str, + now: OffsetDateTime, +) -> Result { + let options = build_replication_probe_put_options(now); + let sent_version_id = options.internal.source_version_id.clone(); + let mut headers = build_replication_probe_headers(&options); + // These are full wire names (not x-rustfs/x-minio suffixes), so they must + // be inserted verbatim — `insert_header` would mangle them. + for (name, value) in [ + (REPLICATION_SSEC_ALGORITHM_HEADER, "AES256"), + (REPLICATION_SSEC_KEY_MD5_HEADER, REPLICATION_CHECK_SSEC_PROBE_KEY_MD5), + (REPLICATION_SSEC_ORIGINAL_SIZE_HEADER, "8"), + ] { + let name = name + .parse::() + .map_err(|err| S3ClientError::new(format!("invalid ssec probe header name: {err}")))?; + let value = + HeaderValue::from_str(value).map_err(|err| S3ClientError::new(format!("invalid ssec probe header value: {err}")))?; + headers.insert(name, value); + } + + let query_version_id = sent_version_id.clone(); + let response = target_client + .client + .put_object() + .bucket(target_bucket) + .key(probe_key) + .content_length(8) + .body(AwsByteStream::from_static(b"aaaaaaaa")) + .customize() + .map_request(move |mut req| { + for (key, value) in headers.clone() { + req.headers_mut().insert(key.expect("operation should succeed"), value); + } + let uri = append_version_id_query(req.uri(), &query_version_id); + req.set_uri(uri).map_err(std::io::Error::other)?; + Result::<_, std::io::Error>::Ok(req) + }) + .send() + .await + .map_err(S3ClientError::from)?; + let version_id = response.version_id().map(ToOwned::to_owned); + + let head_version = version_id.clone().or_else(|| Some(sent_version_id.clone())); + let head = target_client + .head_object(target_bucket, probe_key, head_version) + .await + .map_err(S3ClientError::from)?; + + Ok(ReplicationSsecProbeOutcome { + evidence_present: head.sse_customer_algorithm().is_some_and(|algorithm| !algorithm.is_empty()), + version_id, + }) +} + async fn delete_replication_probe_object( target_client: &TargetClient, target_bucket: &str, @@ -3722,6 +3872,12 @@ mod tests { /// Same, for the multipart leg: a target may mirror PutObject ids and /// still mint its own at CreateMultipartUpload. minted_multipart_version_id: Option<&'static str>, + /// Transport failure of the SSE-C passthrough probe itself. + ssec_probe_error: Option<&'static str>, + /// Models a MinIO-like target that drops the SSE-C passthrough + /// headers: the probe HEAD-back echoes no SSE-C evidence. The default + /// (false) models a RustFS target that preserves them. + ssec_evidence_missing: bool, delete_marker_error: Option<&'static str>, version_delete_error: Option<&'static str>, cleanup_error: Option<&'static str>, @@ -3759,6 +3915,17 @@ mod tests { }) } + async fn ssec_passthrough_probe(&mut self) -> Result { + self.calls.push("ssec-probe"); + match self.ssec_probe_error { + Some(code) => Err(scripted_probe_error(code)), + None => Ok(ReplicationSsecProbeOutcome { + evidence_present: !self.ssec_evidence_missing, + version_id: Some("ssec-version".to_string()), + }), + } + } + async fn create_delete_marker(&mut self, _version_id: Option<&str>) -> Result, S3ClientError> { self.calls.push("delete-marker"); match self.delete_marker_error { @@ -3775,7 +3942,7 @@ mod tests { } } - async fn cleanup(&mut self, known_version_ids: [Option<&str>; 3]) -> Result<(), String> { + async fn cleanup(&mut self, known_version_ids: [Option<&str>; 4]) -> Result<(), String> { self.calls.push("cleanup"); self.cleanup_ids = known_version_ids .into_iter() @@ -3810,8 +3977,9 @@ mod tests { assert_eq!(result.phases.version_fidelity.code, Some(REPLICATION_CHECK_CODE_VERSION_MISMATCH)); assert_eq!(result.phases.delete_marker.status, "SKIPPED"); assert_eq!(result.phases.version_delete.status, "SKIPPED"); + assert_eq!(result.phases.ssec_passthrough.status, "SKIPPED"); assert_eq!(result.phases.cleanup.status, "OK"); - assert_eq!(operations.cleanup_ids, [Some("target-minted-version".to_string()), None, None]); + assert_eq!(operations.cleanup_ids, [Some("target-minted-version".to_string()), None, None, None]); } #[tokio::test] @@ -3824,6 +3992,70 @@ mod tests { assert_eq!(result.status, "OK"); assert_eq!(result.phases.version_fidelity.status, "OK"); assert_eq!(result.phases.version_fidelity.code, None); + assert_eq!(result.phases.ssec_passthrough.status, "OK"); + assert_eq!(result.phases.ssec_passthrough.code, None); + } + + /// N2: a target that drops the SSE-C passthrough transport headers must + /// fail the SsecPassthrough phase with the machine-readable code while the + /// target overall stays OK — deliberately unlike VersionFidelity: this is + /// a capability limit, not a broken replication contract, and a + /// plaintext-only deployment against such a target must not turn red. The + /// other mutation phases keep running and the probe version is cleaned up. + #[tokio::test] + async fn replication_probe_flags_ssec_passthrough_dropping_target_without_failing_target() { + let mut result = replication_check_target("arn:a", "OK", None); + let mut operations = ScriptedReplicationProbe { + ssec_evidence_missing: true, + ..Default::default() + }; + + execute_replication_probe(&mut result, &mut operations).await; + + assert_eq!( + operations.calls, + [ + "put", + "multipart-put", + "ssec-probe", + "delete-marker", + "version-delete", + "cleanup" + ] + ); + assert_eq!(result.status, "OK", "a capability-only failure must not fail the target overall"); + assert_eq!(result.error, None); + assert_eq!(result.phases.ssec_passthrough.status, "FAILED"); + assert_eq!(result.phases.ssec_passthrough.code, Some(REPLICATION_CHECK_CODE_SSEC_PASSTHROUGH)); + assert_eq!( + operations.cleanup_ids, + [ + Some("object-version".to_string()), + Some("multipart-version".to_string()), + Some("ssec-version".to_string()), + Some("marker-version".to_string()) + ] + ); + } + + /// A transport failure of the SSE-C probe is not evidence of a dropping + /// target: the phase fails without the capability code (the runtime cache + /// stays Unknown and the worker keeps auditing), and the target overall + /// stays OK. + #[tokio::test] + async fn replication_probe_ssec_transport_failure_carries_no_capability_code() { + let mut result = replication_check_target("arn:a", "OK", None); + let mut operations = ScriptedReplicationProbe { + ssec_probe_error: Some("InternalError"), + ..Default::default() + }; + + execute_replication_probe(&mut result, &mut operations).await; + + assert_eq!(result.status, "OK"); + assert_eq!(result.phases.ssec_passthrough.status, "FAILED"); + assert_eq!(result.phases.ssec_passthrough.code, None); + assert_eq!(operations.cleanup_ids[2], None, "a failed ssec probe leaves no version to clean"); } #[tokio::test] @@ -3854,12 +4086,23 @@ mod tests { execute_replication_probe(&mut result, &mut operations).await; - assert_eq!(operations.calls, ["put", "multipart-put", "delete-marker", "version-delete", "cleanup"]); + assert_eq!( + operations.calls, + [ + "put", + "multipart-put", + "ssec-probe", + "delete-marker", + "version-delete", + "cleanup" + ] + ); assert_eq!( operations.cleanup_ids, [ Some("object-version".to_string()), Some("multipart-version".to_string()), + Some("ssec-version".to_string()), None ] ); @@ -3879,12 +4122,23 @@ mod tests { execute_replication_probe(&mut result, &mut operations).await; - assert_eq!(operations.calls, ["put", "multipart-put", "delete-marker", "version-delete", "cleanup"]); + assert_eq!( + operations.calls, + [ + "put", + "multipart-put", + "ssec-probe", + "delete-marker", + "version-delete", + "cleanup" + ] + ); assert_eq!( operations.cleanup_ids, [ Some("object-version".to_string()), Some("multipart-version".to_string()), + Some("ssec-version".to_string()), Some("marker-version".to_string()) ] ); diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index dbf3a81df..eb38e9640 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -196,6 +196,7 @@ pub(crate) mod bucket_target_sys { pub(crate) type PutObjectOptions = super::ecstore_bucket::bucket_target_sys::PutObjectOptions; pub(crate) type RemoveObjectOptions = super::ecstore_bucket::bucket_target_sys::RemoveObjectOptions; pub(crate) type S3ClientError = super::ecstore_bucket::bucket_target_sys::S3ClientError; + pub(crate) type SsecPassthroughCapability = super::ecstore_bucket::bucket_target_sys::SsecPassthroughCapability; pub(crate) type TargetClient = super::ecstore_bucket::bucket_target_sys::TargetClient; } diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index bcce2b705..413a88464 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -46,9 +46,10 @@ use super::storage_api::object_usecase::bucket::{ replication::{ DeleteReplicationConfigSnapshot, REPLICATE_INCOMING_DELETE, ReplicationStatusType, commit_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id, deleted_object_has_pending_replication_delete, - force_delete_target_set, has_active_delete_rule, load_delete_config_snapshot, must_replicate_object, - persist_force_delete_intent, schedule_object_replication, schedule_replication_delete, schedule_replication_deletes, - set_deleted_object_replication_state, should_schedule_delete_replication, should_use_existing_delete_replication_info, + force_delete_target_set, get_read_proxy_targets, has_active_delete_rule, load_delete_config_snapshot, + must_replicate_object, persist_force_delete_intent, record_replication_proxy, schedule_object_replication, + schedule_replication_delete, schedule_replication_deletes, set_deleted_object_replication_state, + should_schedule_delete_replication, should_use_existing_delete_replication_info, }, tagging::decode_tags, validate_restore_request, @@ -6662,6 +6663,226 @@ impl DefaultObjectUsecase { }) } + /// Headers a proxied read forwards verbatim to the replication target: + /// only the client's SSE-C key family, so the target performs the real + /// SSE-C decryption (never the replication-check exemption). HTTP + /// conditional headers (If-Match & co.) are deliberately NOT forwarded — + /// MinIO does not forward them either, and a remote 304/412 would leak a + /// conditional evaluation against a replica the local site never saw. + /// Range and part-number travel as typed SDK parameters instead. + fn proxy_read_passthrough_headers(headers: &HeaderMap) -> HeaderMap { + const FORWARDED: &[&str] = &[ + "x-amz-server-side-encryption-customer-algorithm", + "x-amz-server-side-encryption-customer-key", + "x-amz-server-side-encryption-customer-key-md5", + ]; + let mut forwarded = HeaderMap::new(); + for name in FORWARDED { + if let Ok(header_name) = http::HeaderName::from_str(name) + && let Some(value) = headers.get(&header_name) + { + forwarded.insert(header_name, value.clone()); + } + } + forwarded + } + + /// True when a proxied SDK call failed because the target does not have + /// the object either (service-level not-found or a raw 404, which also + /// covers NoSuchVersion): the caller tries the next target silently. + fn proxy_sdk_error_is_not_found(err: &aws_sdk_s3::error::SdkError) -> bool { + err.raw_response().is_some_and(|resp| resp.status().as_u16() == 404) + } + + /// Serve a GET whose local read failed with not-found by proxying to the + /// bucket's replication targets (MinIO `proxyGetToReplicationTarget`, + /// backlog#1675 P1-5). Returns None when no target can serve the object; + /// the caller then returns the original local error. + async fn proxy_get_object_to_replication_targets( + req: &S3Request, + bucket: &str, + key: &str, + opts: &ObjectOptions, + ) -> Option { + let targets = get_read_proxy_targets(bucket, key, opts).await; + if targets.is_empty() { + return None; + } + let extra_headers = Self::proxy_read_passthrough_headers(&req.headers); + let range = req + .headers + .get(http::header::RANGE) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let part_number = req.input.part_number; + + for target in targets { + match target + .get_object( + &target.bucket, + key, + opts.version_id.clone(), + range.clone(), + part_number, + extra_headers.clone(), + ) + .await + { + Ok(remote) => { + // MinIO-aligned accounting: one total per proxy attempt + // (targets were available), one failed when no target + // served it — never per target. + record_replication_proxy(bucket, "GetObject", false).await; + return Some(Self::proxy_sdk_get_output_to_s3s(remote)); + } + Err(err) if Self::proxy_sdk_error_is_not_found(&err) => { + debug!(bucket, key, arn = %target.arn, "read proxy: target does not have the object"); + } + Err(err) => { + warn!(bucket, key, arn = %target.arn, error = %err, "read proxy: GET against replication target failed"); + } + } + } + record_replication_proxy(bucket, "GetObject", true).await; + None + } + + /// Serve a HEAD whose local lookup failed with not-found by proxying to + /// the bucket's replication targets (MinIO `proxyHeadToRepTarget`). + async fn proxy_head_object_to_replication_targets( + req: &S3Request, + bucket: &str, + key: &str, + opts: &ObjectOptions, + ) -> Option { + let targets = get_read_proxy_targets(bucket, key, opts).await; + if targets.is_empty() { + return None; + } + let extra_headers = Self::proxy_read_passthrough_headers(&req.headers); + let range = req + .headers + .get(http::header::RANGE) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let part_number = req.input.part_number; + + for target in targets { + match target + .head_object_for_proxy( + &target.bucket, + key, + opts.version_id.clone(), + range.clone(), + part_number, + extra_headers.clone(), + ) + .await + { + Ok(remote) => { + // MinIO-aligned accounting: one total per proxy attempt, + // one failed when no target served it. + record_replication_proxy(bucket, "HeadObject", false).await; + return Some(Self::proxy_sdk_head_output_to_s3s(remote)); + } + Err(err) if Self::proxy_sdk_error_is_not_found(&err) => { + debug!(bucket, key, arn = %target.arn, "read proxy: target does not have the object"); + } + Err(err) => { + warn!(bucket, key, arn = %target.arn, error = %err, "read proxy: HEAD against replication target failed"); + } + } + } + record_replication_proxy(bucket, "HeadObject", true).await; + None + } + + /// Translate a proxied SDK GET response into the s3s output, forwarding + /// the body as a stream (no buffering, no local persistence). + fn proxy_sdk_get_output_to_s3s(remote: aws_sdk_s3::operation::get_object::GetObjectOutput) -> GetObjectOutput { + let body = remote.body; + let body_stream = tokio_util::io::ReaderStream::with_capacity(body.into_async_read(), 64 * 1024); + GetObjectOutput { + body: Some(StreamingBlob::wrap(body_stream)), + content_length: remote.content_length, + content_range: remote.content_range, + content_type: remote.content_type.as_deref().and_then(|v| ContentType::from_str(v).ok()), + content_encoding: remote.content_encoding, + content_disposition: remote.content_disposition, + content_language: remote.content_language, + cache_control: remote.cache_control, + accept_ranges: Some(ACCEPT_RANGES_BYTES.to_string()), + e_tag: remote.e_tag.as_deref().and_then(|v| ETag::from_str(v).ok()), + last_modified: remote + .last_modified + .and_then(|dt| OffsetDateTime::from_unix_timestamp_nanos(dt.as_nanos()).ok()) + .map(Timestamp::from), + metadata: remote.metadata, + version_id: remote.version_id, + server_side_encryption: remote + .server_side_encryption + .map(|sse| ServerSideEncryption::from(sse.as_str().to_string())), + sse_customer_algorithm: remote.sse_customer_algorithm, + sse_customer_key_md5: remote.sse_customer_key_md5, + ssekms_key_id: remote.ssekms_key_id, + parts_count: remote.parts_count, + tag_count: remote.tag_count, + storage_class: remote.storage_class.map(|sc| StorageClass::from(sc.as_str().to_string())), + expiration: remote.expiration, + restore: remote.restore, + checksum_crc32: remote.checksum_crc32, + checksum_crc32c: remote.checksum_crc32_c, + checksum_crc64nvme: remote.checksum_crc64_nvme, + checksum_sha1: remote.checksum_sha1, + checksum_sha256: remote.checksum_sha256, + checksum_type: remote.checksum_type.map(|ct| ChecksumType::from(ct.as_str().to_string())), + ..Default::default() + } + } + + /// Translate a proxied SDK HEAD response into the s3s output. + /// + /// Known gaps: the SDK's HeadObjectOutput does not model 206/Content-Range + /// for a ranged HEAD (the SDK exposes no content_range member on HEAD), + /// and s3s' typed HeadObjectOutput has no tag_count field (the local path + /// injects x-amz-tagging-count as a raw header) — both are dropped for + /// proxied HEADs. + fn proxy_sdk_head_output_to_s3s(remote: aws_sdk_s3::operation::head_object::HeadObjectOutput) -> HeadObjectOutput { + HeadObjectOutput { + content_length: remote.content_length, + content_type: remote.content_type.as_deref().and_then(|v| ContentType::from_str(v).ok()), + content_encoding: remote.content_encoding, + content_disposition: remote.content_disposition, + content_language: remote.content_language, + cache_control: remote.cache_control, + accept_ranges: Some(ACCEPT_RANGES_BYTES.to_string()), + e_tag: remote.e_tag.as_deref().and_then(|v| ETag::from_str(v).ok()), + last_modified: remote + .last_modified + .and_then(|dt| OffsetDateTime::from_unix_timestamp_nanos(dt.as_nanos()).ok()) + .map(Timestamp::from), + metadata: remote.metadata, + version_id: remote.version_id, + server_side_encryption: remote + .server_side_encryption + .map(|sse| ServerSideEncryption::from(sse.as_str().to_string())), + sse_customer_algorithm: remote.sse_customer_algorithm, + sse_customer_key_md5: remote.sse_customer_key_md5, + ssekms_key_id: remote.ssekms_key_id, + parts_count: remote.parts_count, + storage_class: remote.storage_class.map(|sc| StorageClass::from(sc.as_str().to_string())), + expiration: remote.expiration, + restore: remote.restore, + checksum_crc32: remote.checksum_crc32, + checksum_crc32c: remote.checksum_crc32_c, + checksum_crc64nvme: remote.checksum_crc64_nvme, + checksum_sha1: remote.checksum_sha1, + checksum_sha256: remote.checksum_sha256, + checksum_type: remote.checksum_type.map(|ct| ChecksumType::from(ct.as_str().to_string())), + ..Default::default() + } + } + #[instrument(name = "execute_get_object", level = "trace", skip(self, req))] pub async fn execute_get_object(&self, req: S3Request) -> S3Result> { self.execute_get_object_boxed(req).await @@ -6787,6 +7008,19 @@ impl DefaultObjectUsecase { { Ok(prepared_read) => prepared_read, Err(err) => { + // Active-active replication lag window: an object missing + // locally (and only missing — other errors keep their + // semantics) may still be served by proxying the GET to a + // replication target (backlog#1675 P1-5). + if matches!(*err.code(), S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion) + && let Some(output) = Self::proxy_get_object_to_replication_targets(&req, &bucket, &key, &opts).await + { + lifecycle.finish_ok(); + let response = wrap_response_with_cors(&bucket, &req.method, &req.headers, output).await; + let result = Ok(response); + let _ = helper.version_id(version_id_for_event).complete(&result); + return result; + } lifecycle.finish_err(); return Err(err); } @@ -8696,6 +8930,17 @@ impl DefaultObjectUsecase { let msg = head_prefix_not_found_message(&bucket, &key, has_children); return Err(S3Error::with_message(S3ErrorCode::NoSuchKey, msg)); } + // Active-active replication lag window: an object missing + // locally may still be served by proxying the HEAD to a + // replication target (backlog#1675 P1-5). + if let Some(output) = Self::proxy_head_object_to_replication_targets(&req, &bucket, &key, &opts).await { + let response = wrap_response_with_cors(&bucket, &req.method, &req.headers, output).await; + let result = Ok(response); + let _ = helper + .version_id(req.input.version_id.clone().unwrap_or_default()) + .complete(&result); + return result; + } return Err(S3Error::new(S3ErrorCode::NoSuchKey)); } // Other errors, such as insufficient permissions, still return the original error diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index c6f03f240..0c2b3d4d4 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -627,6 +627,24 @@ pub(crate) mod bucket { #[cfg(test)] pub(crate) use replication_contracts::replication_statuses_map; + /// Remote replication-target client used by the read-proxy path. + pub(crate) type ProxyTargetClient = crate::storage::storage_api::ecstore_bucket::bucket_target_sys::TargetClient; + + /// Proxy-request metric recorder (get/head/tagging totals + failures). + pub(crate) use crate::storage::storage_api::record_replication_proxy; + + /// Replication targets eligible to serve a proxied GET/HEAD/Tagging of + /// an object not present locally (MinIO `getProxyTargets`; empty when + /// the request was itself proxied, versioning is suspended, or no + /// replication rule matches). backlog#1675 P1-5. + pub(crate) async fn get_read_proxy_targets( + bucket: &str, + object: &str, + opts: &crate::storage::storage_api::StorageObjectOptions, + ) -> Vec> { + replication_contracts::get_proxy_targets(bucket, object, opts).await + } + pub(crate) async fn persist_force_delete_intent( store: Arc, bucket: String, diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 9369d7224..da0c9b775 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +use super::StorageVersioningConfigExt as _; use super::{ BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG, BucketVersioningSys, OBJECT_LOCK_CONFIG, StorageError, check_retention_for_modification, decode_tags, decode_tags_to_map, delete_bucket_metadata_config_if_incarnation, encode_tags, get_bucket_accelerate_config, - get_bucket_logging_config, get_bucket_object_lock_config, get_bucket_replication_config, get_bucket_request_payment_config, - get_bucket_website_config, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, - record_replication_proxy, serialize, update_bucket_metadata_config_if_incarnation, + get_bucket_logging_config, get_bucket_object_lock_config, get_bucket_request_payment_config, get_bucket_website_config, + is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, record_replication_proxy, serialize, + update_bucket_metadata_config_if_incarnation, }; -use super::{StorageReplicationConfigExt as _, StorageVersioningConfigExt as _}; use crate::admin::handlers::site_replication::site_replication_bucket_meta_hook; use crate::error::ApiError; use crate::storage::access::{apply_bucket_generation_guard, bucket_config_mutation_incarnation, has_bypass_governance_header}; @@ -59,7 +59,7 @@ const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock"; const LOG_SUBSYSTEM_TAGGING: &str = "tagging"; use crate::app::storage_api::object_usecase::bucket::replication::{ - ReplicateDecision, must_replicate_metadata, schedule_metadata_replication, + ReplicateDecision, get_read_proxy_targets, must_replicate_metadata, schedule_metadata_replication, }; use crate::storage::storage_api::ecfs_consumer::StorageObjectOptions as ObjectOptions; @@ -105,18 +105,152 @@ impl FS { &self.server_ctx } - async fn replication_tagging_enabled(bucket: &str, object: &str) -> bool { - get_bucket_replication_config(bucket) - .await - .map(|(cfg, _)| cfg.has_active_rules(object, true)) - .unwrap_or(false) + /// Not-found classifier for proxied SDK tagging calls: a raw 404 covers + /// NoSuchKey and NoSuchVersion alike; the caller silently tries the next + /// replication target. + fn proxy_sdk_error_is_not_found(err: &aws_sdk_s3::error::SdkError) -> bool { + err.raw_response().is_some_and(|resp| resp.status().as_u16() == 404) } - async fn record_replication_tagging_metric(bucket: &str, object: &str, api: &str, is_err: bool) { - if !Self::replication_tagging_enabled(bucket, object).await { - return; + /// Selector options for a tagging proxy. Reuses `get_opts` so the + /// anti-loop `source-proxy-request` header family and the bucket's + /// version-suspension state gate proxying exactly like GET/HEAD. + async fn tagging_proxy_opts( + bucket: &str, + object: &str, + version_id: Option, + headers: &http::HeaderMap, + ) -> Option { + get_opts(bucket, object, version_id, None, headers).await.ok() + } + + /// Serve a GetObjectTagging for an object missing locally by proxying to + /// the bucket's replication targets (MinIO `proxyGetTaggingToRepTarget`, + /// backlog#1675 P1-5). None means no target had the object. + async fn proxy_get_object_tagging( + bucket: &str, + object: &str, + version_id: Option, + headers: &http::HeaderMap, + ) -> Option { + let opts = Self::tagging_proxy_opts(bucket, object, version_id, headers).await?; + let targets = get_read_proxy_targets(bucket, object, &opts).await; + if targets.is_empty() { + return None; } - record_replication_proxy(bucket, api, is_err).await; + for target in targets { + match target + .get_object_tagging(&target.bucket, object, opts.version_id.clone()) + .await + { + Ok(remote) => { + // MinIO-aligned accounting: one total per proxy attempt, + // one failed when no target served it. + record_replication_proxy(bucket, "GetObjectTagging", false).await; + return Some( + remote + .tag_set + .into_iter() + .map(|tag| Tag { + key: Some(tag.key), + value: Some(tag.value), + }) + .collect(), + ); + } + Err(err) if Self::proxy_sdk_error_is_not_found(&err) => { + debug!(bucket, object, arn = %target.arn, "tagging proxy: target does not have the object"); + } + Err(err) => { + warn!(bucket, object, arn = %target.arn, error = %err, "tagging proxy: GetObjectTagging against replication target failed"); + } + } + } + record_replication_proxy(bucket, "GetObjectTagging", true).await; + None + } + + /// Apply a PutObjectTagging for an object missing locally on a + /// replication target (MinIO `proxyTaggingToRepTarget`). + async fn proxy_put_object_tagging( + bucket: &str, + object: &str, + version_id: Option, + headers: &http::HeaderMap, + tag_set: &TagSet, + ) -> Option<()> { + let opts = Self::tagging_proxy_opts(bucket, object, version_id, headers).await?; + let mut tagging = aws_sdk_s3::types::Tagging::builder(); + for tag in tag_set { + let sdk_tag = aws_sdk_s3::types::Tag::builder() + .key(tag.key.clone().unwrap_or_default()) + .value(tag.value.clone().unwrap_or_default()) + .build() + .ok()?; + tagging = tagging.tag_set(sdk_tag); + } + let tagging = tagging.build().ok()?; + let targets = get_read_proxy_targets(bucket, object, &opts).await; + if targets.is_empty() { + return None; + } + for target in targets { + match target + .put_object_tagging(&target.bucket, object, opts.version_id.clone(), tagging.clone()) + .await + { + Ok(_) => { + // MinIO-aligned accounting: one total per proxy attempt, + // one failed when no target served it. + record_replication_proxy(bucket, "PutObjectTagging", false).await; + return Some(()); + } + Err(err) if Self::proxy_sdk_error_is_not_found(&err) => { + debug!(bucket, object, arn = %target.arn, "tagging proxy: target does not have the object"); + } + Err(err) => { + warn!(bucket, object, arn = %target.arn, error = %err, "tagging proxy: PutObjectTagging against replication target failed"); + } + } + } + record_replication_proxy(bucket, "PutObjectTagging", true).await; + None + } + + /// Apply a DeleteObjectTagging for an object missing locally on a + /// replication target (MinIO `proxyTaggingToRepTarget`). + async fn proxy_delete_object_tagging( + bucket: &str, + object: &str, + version_id: Option, + headers: &http::HeaderMap, + ) -> Option<()> { + let opts = Self::tagging_proxy_opts(bucket, object, version_id, headers).await?; + let targets = get_read_proxy_targets(bucket, object, &opts).await; + if targets.is_empty() { + return None; + } + for target in targets { + match target + .delete_object_tagging(&target.bucket, object, opts.version_id.clone()) + .await + { + Ok(_) => { + // MinIO-aligned accounting: one total per proxy attempt, + // one failed when no target served it. + record_replication_proxy(bucket, "DeleteObjectTagging", false).await; + return Some(()); + } + Err(err) if Self::proxy_sdk_error_is_not_found(&err) => { + debug!(bucket, object, arn = %target.arn, "tagging proxy: target does not have the object"); + } + Err(err) => { + warn!(bucket, object, arn = %target.arn, error = %err, "tagging proxy: DeleteObjectTagging against replication target failed"); + } + } + } + record_replication_proxy(bucket, "DeleteObjectTagging", true).await; + None } pub async fn get_object_tag_conditions_for_policy( @@ -447,7 +581,27 @@ impl S3 for FS { let mut opts = get_opts(&bucket, &object, version_id.clone(), None, &req.headers) .await .map_err(ApiError::from)?; - let existing_object_info = store.get_object_info(&bucket, &object, &opts).await.map_err(ApiError::from)?; + let existing_object_info = match store.get_object_info(&bucket, &object, &opts).await { + Ok(info) => info, + Err(e) => { + // Replication lag window: apply the tagging delete on a + // replication target that already has the object + // (backlog#1675 P1-5). No local object exists, so no bucket + // notification event is emitted for the proxied write. + if (is_err_object_not_found(&e) || is_err_version_not_found(&e)) + && Self::proxy_delete_object_tagging(&bucket, &object, version_id.clone(), &req.headers) + .await + .is_some() + { + counter!("rustfs_delete_object_tagging_success").increment(1); + let duration = start_time.elapsed(); + histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "delete") + .record(duration.as_secs_f64()); + return Ok(S3Response::new(DeleteObjectTaggingOutput { version_id })); + } + return Err(ApiError::from(e).into()); + } + }; let dsc = must_replicate_metadata( &bucket, &object, @@ -470,7 +624,6 @@ impl S3 for FS { } let delete_tags_result = store.delete_object_tags(&bucket, &object, &opts).await; - Self::record_replication_tagging_metric(&bucket, &object, "DeleteObjectTagging", delete_tags_result.is_err()).await; let object_info = delete_tags_result.map_err(|e| { error!( component = LOG_COMPONENT_STORAGE, @@ -928,32 +1081,49 @@ impl S3 for FS { ..Default::default() }; - let tags_result = store.get_object_tags(bucket, object, &opts).await; - Self::record_replication_tagging_metric(bucket, object, "GetObjectTagging", tags_result.is_err()).await; - let tags = tags_result.map_err(|e| { - if is_err_object_not_found(&e) { - debug!( + let tags = match store.get_object_tags(bucket, object, &opts).await { + Ok(tags) => tags, + Err(e) => { + // Replication lag window: the object may exist on a + // replication target even though it is missing locally — + // proxy the tagging read there (backlog#1675 P1-5). + if (is_err_object_not_found(&e) || is_err_version_not_found(&e)) + && let Some(tag_set) = + Self::proxy_get_object_tagging(bucket, object, req.input.version_id.clone(), &req.headers).await + { + counter!("rustfs_get_object_tagging_success").increment(1); + let duration = start_time.elapsed(); + histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "get") + .record(duration.as_secs_f64()); + return Ok(S3Response::new(GetObjectTaggingOutput { + tag_set, + version_id: req.input.version_id.clone(), + })); + } + if is_err_object_not_found(&e) { + debug!( + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_TAGGING, + event = "object_tagging_not_found", + bucket = %bucket, + object = %object, + error = %e, + "Object tags not found" + ); + return Err(s3_error!(NoSuchKey)); + } + error!( component = LOG_COMPONENT_STORAGE, subsystem = LOG_SUBSYSTEM_TAGGING, - event = "object_tagging_not_found", + event = "object_tagging_get_failed", bucket = %bucket, object = %object, error = %e, - "Object tags not found" + "Failed to load object tags" ); - return s3_error!(NoSuchKey); + return Err(ApiError::from(e).into()); } - error!( - component = LOG_COMPONENT_STORAGE, - subsystem = LOG_SUBSYSTEM_TAGGING, - event = "object_tagging_get_failed", - bucket = %bucket, - object = %object, - error = %e, - "Failed to load object tags" - ); - ApiError::from(e).into() - })?; + }; let tag_set = decode_tags(tags.as_str()); debug!( @@ -1629,14 +1799,36 @@ impl S3 for FS { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - let tags = encode_tags(tagging.tag_set); + let tags = encode_tags(tagging.tag_set.clone()); debug!("Encoded tags: {}", tags); let version_id = req.input.version_id.clone(); let mut opts = get_opts(&bucket, &object, version_id.clone(), None, &req.headers) .await .map_err(ApiError::from)?; - let existing_object_info = store.get_object_info(&bucket, &object, &opts).await.map_err(ApiError::from)?; + let existing_object_info = match store.get_object_info(&bucket, &object, &opts).await { + Ok(info) => info, + Err(e) => { + // Replication lag window: apply the tagging update on a + // replication target that already has the object + // (backlog#1675 P1-5). No local object exists, so no bucket + // notification event is emitted for the proxied write. + if (is_err_object_not_found(&e) || is_err_version_not_found(&e)) + && Self::proxy_put_object_tagging(&bucket, &object, version_id.clone(), &req.headers, &tagging.tag_set) + .await + .is_some() + { + counter!("rustfs_put_object_tagging_success").increment(1); + let duration = start_time.elapsed(); + histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "put") + .record(duration.as_secs_f64()); + return Ok(S3Response::new(PutObjectTaggingOutput { + version_id: req.input.version_id.clone(), + })); + } + return Err(ApiError::from(e).into()); + } + }; let dsc = must_replicate_metadata( &bucket, &object, @@ -1659,7 +1851,6 @@ impl S3 for FS { } let put_tags_result = store.put_object_tags(&bucket, &object, &tags, &opts).await; - Self::record_replication_tagging_metric(&bucket, &object, "PutObjectTagging", put_tags_result.is_err()).await; let object_info = put_tags_result.map_err(|e| { error!("Failed to put object tags: {}", e); counter!("rustfs_put_object_tagging_failure").increment(1); diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index e0817da90..f1fe52979 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -55,24 +55,24 @@ pub(crate) use storage_api::{ QuotaError, RUSTFS_META_BUCKET, RawFileInfo, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, ReplicationStats, ReplicationStatusType, Result, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDeletedObject, StorageDiskRpcExt, StorageError, StorageGetObjectReader, StorageObjectInfo, StorageObjectOptions, - StorageObjectToDelete, StoragePeerS3ClientExt, StoragePutObjReader, StorageReplicationConfigExt, StorageVersioningConfigExt, - TONIC_RPC_PREFIX, TierConfigMgr, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, WorkloadAdmissionSnapshotProviderRef, - WriteEncryption, WritePlan, access_consumer, add_object_lock_years, all_local_disk, all_local_disk_path, - check_retention_for_modification, collect_local_metrics, compression_metadata_value, contract, decode_tags, - decode_tags_to_map, delete_bucket_metadata_config, delete_bucket_metadata_config_if_incarnation, disk_drive_path, - disk_endpoint, ecfs_consumer, ecfs_extend_consumer, ecstore_admin, ecstore_bucket, ecstore_capacity, ecstore_client, - ecstore_cluster, ecstore_compression, ecstore_config, ecstore_data_usage, ecstore_disk, ecstore_error, ecstore_event, - ecstore_layout, ecstore_metrics, ecstore_notification, ecstore_rebalance, ecstore_rio, ecstore_rpc, ecstore_set_disk, - ecstore_storage, ecstore_tier, encode_tags, find_local_disk_by_ref, get_bucket_accelerate_config, get_bucket_cors_config, - get_bucket_logging_config, get_bucket_metadata, get_bucket_notification_config, get_bucket_object_lock_config, - get_bucket_replication_config, get_bucket_request_payment_config, get_bucket_sse_config, get_bucket_website_config, - get_local_server_property, get_lock_acquire_timeout, head_prefix_consumer, helper_consumer, init_background_replication, - init_bucket_metadata_sys, init_ecstore_config, init_local_disks_with_instance_ctx, init_lock_clients, - is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, is_valid_storage_class, options_consumer, - prewarm_local_disk_id_map_with_instance_ctx, read_config, record_replication_proxy, rpc_consumer, runtime_sources_consumer, - s3_api_consumer, serialize, table_catalog_path_hash, to_s3s_etag, topology_snapshot_from_endpoint_pools_with_capabilities, - try_migrate_bucket_metadata, try_migrate_iam_config, try_migrate_server_config, update_bucket_metadata_config, - update_bucket_metadata_config_if_incarnation, verify_rpc_signature, wrap_reader, + StorageObjectToDelete, StoragePeerS3ClientExt, StoragePutObjReader, StorageVersioningConfigExt, TONIC_RPC_PREFIX, + TierConfigMgr, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, WorkloadAdmissionSnapshotProviderRef, WriteEncryption, + WritePlan, access_consumer, add_object_lock_years, all_local_disk, all_local_disk_path, check_retention_for_modification, + collect_local_metrics, compression_metadata_value, contract, decode_tags, decode_tags_to_map, delete_bucket_metadata_config, + delete_bucket_metadata_config_if_incarnation, disk_drive_path, disk_endpoint, ecfs_consumer, ecfs_extend_consumer, + ecstore_admin, ecstore_bucket, ecstore_capacity, ecstore_client, ecstore_cluster, ecstore_compression, ecstore_config, + ecstore_data_usage, ecstore_disk, ecstore_error, ecstore_event, ecstore_layout, ecstore_metrics, ecstore_notification, + ecstore_rebalance, ecstore_rio, ecstore_rpc, ecstore_set_disk, ecstore_storage, ecstore_tier, encode_tags, + find_local_disk_by_ref, get_bucket_accelerate_config, get_bucket_cors_config, get_bucket_logging_config, get_bucket_metadata, + get_bucket_notification_config, get_bucket_object_lock_config, get_bucket_request_payment_config, get_bucket_sse_config, + get_bucket_website_config, get_local_server_property, get_lock_acquire_timeout, head_prefix_consumer, helper_consumer, + init_background_replication, init_bucket_metadata_sys, init_ecstore_config, init_local_disks_with_instance_ctx, + init_lock_clients, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, is_valid_storage_class, + options_consumer, prewarm_local_disk_id_map_with_instance_ctx, read_config, record_replication_proxy, rpc_consumer, + runtime_sources_consumer, s3_api_consumer, serialize, table_catalog_path_hash, to_s3s_etag, + topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata, try_migrate_iam_config, + try_migrate_server_config, update_bucket_metadata_config, update_bucket_metadata_config_if_incarnation, verify_rpc_signature, + wrap_reader, }; #[cfg(test)] diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index ac920b5c3..dc5a6ca19 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -19,9 +19,10 @@ use http::{HeaderMap, HeaderValue}; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_FORCE_DELETE, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, - SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, - SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, - SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID, SUFFIX_TAGGING_TIMESTAMP, get_header, + SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST, + SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST, + SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID, + SUFFIX_TAGGING_TIMESTAMP, get_header, header_compat::{MINIO_ENCRYPTION_PREFIX, RUSTFS_ENCRYPTION_PREFIX}, insert_header_map, insert_str, metadata_compat::{MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX}, @@ -276,6 +277,19 @@ pub async fn get_opts( // Background scanner still performs full integrity checks asynchronously. opts.skip_verify_bitrot = get_skip_verify_bitrot(); + // Anti-loop markers for the replication read proxy + // (`{x-rustfs-,x-minio-}source-proxy-request` header family). + // MinIO semantics: the header being PRESENT at all (`ProxyHeaderSet`) + // disables proxying, whatever its value — a peer's replication worker + // sends "false" on its convergence HEADs so the receiver answers locally + // instead of proxying the miss back (a proxied echo would fake + // convergence and the object would never replicate). Deliberately not + // gated on replication authorization: the header only disables proxying + // (it grants nothing). + let proxy_header = get_header(headers, SUFFIX_SOURCE_PROXY_REQUEST); + opts.proxy_header_set = proxy_header.is_some(); + opts.proxy_request = proxy_header.map(|v| v.as_ref() == "true").unwrap_or_default(); + fill_conditional_writes_opts_from_header(headers, &mut opts)?; Ok(opts) @@ -2544,4 +2558,80 @@ mod tests { } } } + + /// The replication read-proxy anti-loop markers must be honored under + /// both interop prefixes (a MinIO peer sends x-minio-, a RustFS peer + /// sends both). `proxy_request` is set only for the literal value + /// "true", while `proxy_header_set` (MinIO `ProxyHeaderSet`) is set by + /// the header's mere presence — "false" (the replication worker's + /// convergence-HEAD marker) and arbitrary values included — so the + /// selector refuses to proxy either way. + #[tokio::test] + async fn test_get_opts_parses_source_proxy_request_under_both_prefixes() { + for header_name in ["x-rustfs-source-proxy-request", "x-minio-source-proxy-request"] { + let mut headers = HeaderMap::new(); + headers.insert(header_name, HeaderValue::from_static("true")); + let opts = get_opts("test-bucket", "test-object", None, None, &headers) + .await + .expect("get_opts should succeed"); + assert!(opts.proxy_request, "{header_name} must set opts.proxy_request"); + assert!(opts.proxy_header_set, "{header_name} must set opts.proxy_header_set"); + } + + let opts = get_opts("test-bucket", "test-object", None, None, &HeaderMap::new()) + .await + .expect("get_opts should succeed"); + assert!(!opts.proxy_request, "absent header must leave proxy_request off"); + assert!(!opts.proxy_header_set, "absent header must leave proxy_header_set off"); + + for (header_name, value) in [ + ("x-minio-source-proxy-request", "false"), + ("x-rustfs-source-proxy-request", "false"), + ("x-minio-source-proxy-request", "anything-else"), + ] { + let mut headers = HeaderMap::new(); + headers.insert(header_name, HeaderValue::from_static(value)); + let opts = get_opts("test-bucket", "test-object", None, None, &headers) + .await + .expect("get_opts should succeed"); + assert!(!opts.proxy_request, "{header_name}: non-'true' value must leave proxy_request off"); + assert!( + opts.proxy_header_set, + "{header_name}: value {value:?} must still set proxy_header_set (presence disables proxying)" + ); + } + } + + /// Pin that the source-proxy-request transport family cannot be + /// materialized as bare stored metadata via an `x-*-meta-` disguise: the + /// reserved-key namespacing (`x-rustfs-source-` / `x-minio-source-` + /// prefixes in `is_reserved_user_metadata_key`) must keep covering it. + #[test] + fn test_source_proxy_request_family_is_reserved_user_metadata() { + let mut headers = HeaderMap::new(); + headers.insert("x-amz-meta-x-minio-source-proxy-request", HeaderValue::from_static("true")); + headers.insert("x-rustfs-meta-x-rustfs-source-proxy-request", HeaderValue::from_static("true")); + // The bare transport header itself is not a user-metadata prefix and + // must never land in stored metadata at all. + headers.insert("x-minio-source-proxy-request", HeaderValue::from_static("true")); + + let metadata = extract_metadata(&headers); + + assert!( + !metadata.contains_key("x-minio-source-proxy-request"), + "bare source-proxy-request key must not be storable: {metadata:?}" + ); + assert!( + !metadata.contains_key("x-rustfs-source-proxy-request"), + "bare source-proxy-request key must not be storable: {metadata:?}" + ); + assert!( + metadata.contains_key("x-amz-meta-x-minio-source-proxy-request"), + "disguised key must be namespaced back under x-amz-meta-: {metadata:?}" + ); + assert!( + metadata.contains_key("x-amz-meta-x-rustfs-source-proxy-request"), + "disguised key must be namespaced back under x-amz-meta-: {metadata:?}" + ); + } } diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index be9548619..534269546 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -809,6 +809,10 @@ impl StorageReplicationStatsHandle { proxy_head_failed: metrics.proxied.head_failed, proxy_put_tag_total: metrics.proxied.put_tag_total, proxy_put_tag_failed: metrics.proxied.put_tag_failed, + proxy_get_tag_total: metrics.proxied.get_tag_total, + proxy_get_tag_failed: metrics.proxied.get_tag_failed, + proxy_delete_tag_total: metrics.proxied.delete_tag_total, + proxy_delete_tag_failed: metrics.proxied.delete_tag_failed, replica_size: metrics.replica_size, replica_count: metrics.replica_count, } @@ -845,6 +849,10 @@ pub(crate) struct ReplicationSiteMetricsSnapshot { pub(crate) proxy_head_failed: i64, pub(crate) proxy_put_tag_total: i64, pub(crate) proxy_put_tag_failed: i64, + pub(crate) proxy_get_tag_total: i64, + pub(crate) proxy_get_tag_failed: i64, + pub(crate) proxy_delete_tag_total: i64, + pub(crate) proxy_delete_tag_failed: i64, pub(crate) replica_size: i64, pub(crate) replica_count: i64, } @@ -1491,12 +1499,6 @@ pub(crate) async fn get_bucket_object_lock_config( ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await } -pub(crate) async fn get_bucket_replication_config( - bucket: &str, -) -> Result<(s3s::dto::ReplicationConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_replication_config(bucket).await -} - pub(crate) async fn persist_force_delete_intent( api: Arc, entry: ecstore_bucket::replication::MrfReplicateEntry, @@ -1842,18 +1844,6 @@ pub(crate) async fn find_local_disk_by_ref(disk_ref: &str) -> Option ecstore_storage::find_local_disk_by_ref(disk_ref).await } -pub(crate) trait StorageReplicationConfigExt { - fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool; -} - -impl StorageReplicationConfigExt for s3s::dto::ReplicationConfiguration { - fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool { - ::has_active_rules( - self, prefix, recursive, - ) - } -} - pub(crate) trait StorageVersioningConfigExt { fn enabled(&self) -> bool; } diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 08c79d4ef..d510d49a6 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -242,6 +242,7 @@ REPLICATION_RESYNC_CONTRACT_BACKSLIDE_HITS_FILE="${TMP_DIR}/replication_resync_c REPLICATION_RESYNC_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_resync_boundary_bypass_hits.txt" REPLICATION_OBJECT_DECISION_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/replication_object_decision_boundary_bypass_hits.txt" REPLICATION_OBJECT_COMPARE_CONTRACT_BACKSLIDE_HITS_FILE="${TMP_DIR}/replication_object_compare_contract_backslide_hits.txt" +REPLICATION_M2_DECISION_CONTRACT_BACKSLIDE_HITS_FILE="${TMP_DIR}/replication_m2_decision_contract_backslide_hits.txt" REPLICATION_MRF_WIRE_FORMAT_BACKSLIDE_HITS_FILE="${TMP_DIR}/replication_mrf_wire_format_backslide_hits.txt" STORAGE_REPLICATION_HANDLE_BOUNDARY_BYPASS_HITS_FILE="${TMP_DIR}/storage_replication_handle_boundary_bypass_hits.txt" STORAGE_REPLICATION_CRATE_BYPASS_HITS_FILE="${TMP_DIR}/storage_replication_crate_bypass_hits.txt" @@ -2981,7 +2982,7 @@ fi ( cd "$ROOT_DIR" replication_resync_status=0 - rg -n --with-filename '^\s*(?:pub(?:\([^)]*\))?\s+)?(?:(?:struct|enum)\s+(?:ResyncOpts|TargetReplicationResyncStatus|BucketReplicationResyncStatus|ResyncStatusType)|fn\s+(?:resync_state_accepts_update|should_count_head_proxy_failure|should_auto_resume_resync|is_version_id_mismatch))\b' \ + rg -n --with-filename '^\s*(?:pub(?:\([^)]*\))?\s+)?(?:(?:struct|enum)\s+(?:ResyncOpts|TargetReplicationResyncStatus|BucketReplicationResyncStatus|ResyncStatusType)|fn\s+(?:resync_state_accepts_update|resync_status_duration|should_count_head_proxy_failure|should_auto_resume_resync|is_version_id_mismatch))\b' \ crates/ecstore/src/bucket/replication \ --glob '*.rs' >"$REPLICATION_RESYNC_CONTRACT_BACKSLIDE_HITS_FILE" || replication_resync_status=$? if [[ "$replication_resync_status" -ne 0 && "$replication_resync_status" -ne 1 ]]; then @@ -2996,7 +2997,7 @@ fi ( cd "$ROOT_DIR" replication_resync_boundary_status=0 - rg -n --with-filename 'rustfs_replication::(resync::(RESYNC_META_FORMAT|RESYNC_META_VERSION|WIRE_ZERO_TIME_UNIX)|mrf::(MRF_META_FORMAT|MRF_META_VERSION)|(encode_resync_file|decode_resync_file|encode_mrf_file|decode_mrf_file|BucketReplicationResyncStatus|ResyncOpts|TargetReplicationResyncStatus|resync_state_accepts_update|should_count_head_proxy_failure|should_auto_resume_resync|is_version_id_mismatch)\b)' \ + rg -n --with-filename 'rustfs_replication::(resync::(RESYNC_META_FORMAT|RESYNC_META_VERSION|WIRE_ZERO_TIME_UNIX)|mrf::(MRF_META_FORMAT|MRF_META_VERSION)|(encode_resync_file|decode_resync_file|encode_mrf_file|decode_mrf_file|BucketReplicationResyncStatus|ResyncOpts|TargetReplicationResyncStatus|resync_state_accepts_update|resync_status_duration|should_count_head_proxy_failure|should_auto_resume_resync|is_version_id_mismatch)\b)' \ crates/ecstore/src/bucket/replication \ --glob '*.rs' \ --glob '!replication_resync_boundary.rs' >"$REPLICATION_RESYNC_BOUNDARY_BYPASS_HITS_FILE" || replication_resync_boundary_status=$? @@ -3005,7 +3006,7 @@ fi fi replication_resync_boundary_grouped_status=0 - rg -n -U --with-filename 'use\s+rustfs_replication::\{[^}]*\b(encode_resync_file|decode_resync_file|encode_mrf_file|decode_mrf_file|BucketReplicationResyncStatus|ResyncOpts|TargetReplicationResyncStatus|resync_state_accepts_update|should_count_head_proxy_failure|should_auto_resume_resync|is_version_id_mismatch)\b' \ + rg -n -U --with-filename 'use\s+rustfs_replication::\{[^}]*\b(encode_resync_file|decode_resync_file|encode_mrf_file|decode_mrf_file|BucketReplicationResyncStatus|ResyncOpts|TargetReplicationResyncStatus|resync_state_accepts_update|resync_status_duration|should_count_head_proxy_failure|should_auto_resume_resync|is_version_id_mismatch)\b' \ crates/ecstore/src/bucket/replication \ --glob '*.rs' \ --glob '!replication_resync_boundary.rs' >>"$REPLICATION_RESYNC_BOUNDARY_BYPASS_HITS_FILE" || replication_resync_boundary_grouped_status=$? @@ -3021,7 +3022,7 @@ fi ( cd "$ROOT_DIR" replication_object_decision_boundary_status=0 - rg -n --with-filename 'rustfs_replication::(MustReplicateOptions|ReplicationDeleteSource|ReplicationMultipartPartInput|ReplicationResyncTargetObject|delete_replication_missing_source_decision|delete_replication_object_opts|heal_uses_delete_replication_path|is_retryable_delete_replication_head_error|is_version_delete_replication|replication_etags_match|replication_multipart_complete_actual_size|replication_multipart_part_plan|resync_target_for_object|should_retry_delete_marker_purge)\b' \ + rg -n --with-filename 'rustfs_replication::(MustReplicateOptions|ReplicationDeleteSource|ReplicationMultipartPartInput|ReplicationResyncTargetObject|delete_marker_purge_mrf_entry|delete_marker_purge_version_id|delete_replication_missing_source_decision|delete_replication_object_opts|heal_uses_delete_replication_path|is_retryable_delete_replication_head_error|is_version_delete_replication|replicate_delete_outcome|replication_etags_match|replication_multipart_complete_actual_size|replication_multipart_part_plan|resync_existing_delete_replication_info|resync_target_for_object|should_retry_delete_marker_purge|target_delete_version_id)\b' \ crates/ecstore/src/bucket/replication \ --glob '*.rs' \ --glob '!replication_object_decision_boundary.rs' >"$REPLICATION_OBJECT_DECISION_BOUNDARY_BYPASS_HITS_FILE" || replication_object_decision_boundary_status=$? @@ -3030,7 +3031,7 @@ fi fi replication_object_decision_boundary_grouped_status=0 - rg -n -U --with-filename 'use\s+rustfs_replication::\{[^}]*\b(MustReplicateOptions|ReplicationDeleteSource|ReplicationMultipartPartInput|ReplicationResyncTargetObject|delete_replication_missing_source_decision|delete_replication_object_opts|heal_uses_delete_replication_path|is_retryable_delete_replication_head_error|is_version_delete_replication|replication_etags_match|replication_multipart_complete_actual_size|replication_multipart_part_plan|resync_target_for_object|should_retry_delete_marker_purge)\b' \ + rg -n -U --with-filename 'use\s+rustfs_replication::\{[^}]*\b(MustReplicateOptions|ReplicationDeleteSource|ReplicationMultipartPartInput|ReplicationResyncTargetObject|delete_marker_purge_mrf_entry|delete_marker_purge_version_id|delete_replication_missing_source_decision|delete_replication_object_opts|heal_uses_delete_replication_path|is_retryable_delete_replication_head_error|is_version_delete_replication|replicate_delete_outcome|replication_etags_match|replication_multipart_complete_actual_size|replication_multipart_part_plan|resync_existing_delete_replication_info|resync_target_for_object|should_retry_delete_marker_purge|target_delete_version_id)\b' \ crates/ecstore/src/bucket/replication \ --glob '*.rs' \ --glob '!replication_object_decision_boundary.rs' >>"$REPLICATION_OBJECT_DECISION_BOUNDARY_BYPASS_HITS_FILE" || replication_object_decision_boundary_grouped_status=$? @@ -3058,6 +3059,25 @@ if [[ -s "$REPLICATION_OBJECT_COMPARE_CONTRACT_BACKSLIDE_HITS_FILE" ]]; then report_failure "replication object comparison contracts must stay in crates/replication: $(paste -sd '; ' "$REPLICATION_OBJECT_COMPARE_CONTRACT_BACKSLIDE_HITS_FILE")" fi +( + cd "$ROOT_DIR" + replication_m2_decision_status=0 + # M2-moved pure decision contracts must not be redefined in ECStore. The + # ssec_passthrough_evidence_present name is deliberately absent: ECStore + # keeps a thin HeadObjectOutput adapter under that name in + # replication_target_boundary.rs which delegates to the crate-owned pure fn. + rg -n --with-filename '^\s*(?:pub(?:\([^)]*\))?\s+)?(?:enum\s+(?:SsecPassthroughCapability|SsecPassthroughGate)|fn\s+(?:replicate_delete_outcome|target_delete_version_id|delete_marker_purge_version_id|delete_marker_purge_mrf_entry|resync_existing_delete_replication_info|version_identity_drifted|is_replication_target_offline_error|ssec_passthrough_gate))\b' \ + crates/ecstore/src/bucket/replication \ + --glob '*.rs' >"$REPLICATION_M2_DECISION_CONTRACT_BACKSLIDE_HITS_FILE" || replication_m2_decision_status=$? + if [[ "$replication_m2_decision_status" -ne 0 && "$replication_m2_decision_status" -ne 1 ]]; then + exit "$replication_m2_decision_status" + fi +) + +if [[ -s "$REPLICATION_M2_DECISION_CONTRACT_BACKSLIDE_HITS_FILE" ]]; then + report_failure "M2-moved replication decision contracts must stay in crates/replication: $(paste -sd '; ' "$REPLICATION_M2_DECISION_CONTRACT_BACKSLIDE_HITS_FILE")" +fi + ( cd "$ROOT_DIR" rg -n --with-filename 'rmp_serde::(to_vec_named|from_slice)|LittleEndian::(write_u16|read_u16)|const\s+MRF_META_(FORMAT|VERSION):\s+u16\s*=\s*1\b' \ @@ -3728,6 +3748,12 @@ fi crates/ecstore/src/bucket/replication \ --glob '*.rs' | rg -v '^crates/ecstore/src/bucket/replication/replication_target_boundary\.rs:' || true + # M2-moved target decision contracts (pure gate family, offline classifier, + # version identity drift) route through the target boundary only. + rg -n -U --with-filename 'rustfs_replication::\{[^;]*\b(?:SsecPassthroughCapability|SsecPassthroughGate|is_replication_target_offline_error|ssec_passthrough_gate|ssec_passthrough_evidence_present|version_identity_drifted)\b|rustfs_replication::(?:SsecPassthroughCapability|SsecPassthroughGate|is_replication_target_offline_error|ssec_passthrough_gate|ssec_passthrough_evidence_present|version_identity_drifted)\b' \ + crates/ecstore/src/bucket/replication \ + --glob '*.rs' | + rg -v '^crates/ecstore/src/bucket/replication/replication_target_boundary\.rs:' || true ) >"$REPLICATION_TARGET_BOUNDARY_BYPASS_HITS_FILE" if [[ -s "$REPLICATION_TARGET_BOUNDARY_BYPASS_HITS_FILE" ]]; then From 8315c23d49b4b0517d9db4643793d501dd6bc619 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 21:46:00 +0800 Subject: [PATCH 42/51] test(kms): move the Vault KV2 doc guard into check_fips_wording.sh (#6215) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test(kms): move the Vault KV2 Transit-wrapping doc guard into check_fips_wording.sh `test_vault_kv2_sources_do_not_claim_transit_wrapping` asserted that four `include_str!`-pinned files never describe the Vault KV2 backend as wrapping key material through Vault's Transit engine. The invariant is a documentation-claim invariant with no behavioral twin by construction, and the test form was weak in both directions: it saw only four files (the same prose in a fifth file passed silently) and it stopped compiling — rather than reporting a violation — as soon as one of them was renamed. Move the four literals verbatim into `scripts/check_fips_wording.sh`, which already guards the adjacent cryptographic over-claim class (unsupported FIPS validation wording) and is anchored to the same policy document. The guard now greps every file under `crates/kms` for the same four case-sensitive literals and separately reports a moved pinned source instead of failing to build. `check_fips_wording.sh` previously ran only in `make pre-commit` / `pre-pr`, so wire it into the Quick Checks job of both CI workflows to keep the invariant's failure visibility at least as strong as the deleted test's. --- .config/make/lint-fmt.mak | 4 +- .github/workflows/ci-docs-only.yml | 3 + .github/workflows/ci.yml | 3 + crates/kms/src/config.rs | 28 +----- .../kms-cryptographic-compliance.md | 2 + scripts/check_fips_wording.sh | 87 +++++++++++++++++-- 6 files changed, 92 insertions(+), 35 deletions(-) diff --git a/.config/make/lint-fmt.mak b/.config/make/lint-fmt.mak index f473845e2..46f659440 100644 --- a/.config/make/lint-fmt.mak +++ b/.config/make/lint-fmt.mak @@ -66,8 +66,8 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen ./scripts/check_s3s_footprint.sh .PHONY: fips-wording-check -fips-wording-check: ## Check outward docs do not make unsupported FIPS claims - @echo "📣 Checking FIPS wording guard..." +fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities + @echo "📣 Checking cryptographic capability wording guard..." ./scripts/check_fips_wording.sh .PHONY: log-analyzer-rules-check diff --git a/.github/workflows/ci-docs-only.yml b/.github/workflows/ci-docs-only.yml index 17eca6cba..a156a61b9 100644 --- a/.github/workflows/ci-docs-only.yml +++ b/.github/workflows/ci-docs-only.yml @@ -117,6 +117,9 @@ jobs: - name: Check s3s footprint ratchet run: ./scripts/check_s3s_footprint.sh + - name: Check cryptographic capability wording + run: ./scripts/check_fips_wording.sh + - name: Check no planning docs committed run: ./scripts/check_no_planning_docs.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1988cb54d..412145661 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,6 +152,9 @@ jobs: - name: Check s3s footprint ratchet run: ./scripts/check_s3s_footprint.sh + - name: Check cryptographic capability wording + run: ./scripts/check_fips_wording.sh + - name: Check no planning docs committed run: ./scripts/check_no_planning_docs.sh diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index bd6c4a0e1..6e8cc1d06 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -1729,30 +1729,10 @@ mod tests { assert!(config.validate().is_ok(), "deprecated mount_path must not be required"); } - #[test] - fn test_vault_kv2_sources_do_not_claim_transit_wrapping() { - let sources = [ - ("config.rs", include_str!("config.rs")), - ("api_types.rs", include_str!("api_types.rs")), - ("backends/vault.rs", include_str!("backends/vault.rs")), - ("lib.rs", include_str!("lib.rs")), - ]; - // Assemble the needles at runtime so this guard does not match its own source. - let needles = [ - format!("wrapping via {}", "Transit"), - format!("KV v2 + {}", "Transit"), - format!("KV2+{}", "Transit"), - format!("you would use Vault's {} engine", "transit"), - ]; - for (name, source) in sources { - for needle in &needles { - assert!( - !source.contains(needle.as_str()), - "{name} still describes the Vault KV2 backend with `{needle}`" - ); - } - } - } + // The "VaultKv2 must not claim Transit wrapping" documentation-claim + // invariant is enforced by scripts/check_fips_wording.sh, which scans every + // file in crates/kms rather than a fixed include_str! list + // (rustfs/backlog#1884). #[test] fn test_legacy_persisted_vault_transit_config_uses_metadata_defaults() { diff --git a/docs/operations/kms-cryptographic-compliance.md b/docs/operations/kms-cryptographic-compliance.md index c9ce1a4d4..08dd593e3 100644 --- a/docs/operations/kms-cryptographic-compliance.md +++ b/docs/operations/kms-cryptographic-compliance.md @@ -53,6 +53,8 @@ Suggested boilerplate when the topic cannot be avoided: `README.md` and `CHANGELOG.md` currently contain no FIPS-related wording; `scripts/check_fips_wording.sh` is the grep guard for that public baseline. Any future occurrence of the banned strings in either file should be treated as a defect and either removed or brought under the qualifier rule above. This document intentionally contains the terminology needed to define the policy and is not part of that narrow outward-material scan. +The same script carries a second block for the adjacent over-claim: no file under `crates/kms` may describe the Vault KV2 backend as wrapping key material through Vault's Transit engine. `KmsBackend::VaultKv2` stores RustFS-wrapped key material in Vault's KV v2 engine and never calls Transit, so that wording would tell an operator their key material is cryptographically isolated inside Vault when it is not. Use the `VaultTransit` backend when that isolation is the requirement. + ## The `rustfs-crypto` `fips` feature: what it actually does `crates/crypto/Cargo.toml` declares `default = ["crypto", "fips"]`, so the feature is on in every normal build. Its entire effect is **which algorithm the write path selects**; the implementation is RustCrypto either way. diff --git a/scripts/check_fips_wording.sh b/scripts/check_fips_wording.sh index 49027119c..6221f6d4f 100755 --- a/scripts/check_fips_wording.sh +++ b/scripts/check_fips_wording.sh @@ -1,10 +1,30 @@ #!/usr/bin/env bash set -euo pipefail -# Guard: outward README and CHANGELOG material must not make an unsupported -# FIPS validation or certification claim. The detailed policy and permitted -# qualifiers live in docs/operations/kms-cryptographic-compliance.md; this -# check intentionally scans only the two public project-facing documents. +# Guard: cryptographic capability wording must not over-claim what RustFS +# actually does. Two independent blocks, both anchored to the policy in +# docs/operations/kms-cryptographic-compliance.md: +# +# 1. Outward README and CHANGELOG material must not make an unsupported +# FIPS validation or certification claim. This block intentionally scans +# only the two public project-facing documents; the permitted qualifiers +# live in the policy document. +# +# 2. Nothing in crates/kms may describe the Vault KV2 backend as wrapping +# key material through Vault's Transit engine. `KmsBackend::VaultKv2` +# stores RustFS-wrapped key material in Vault's KV v2 engine and never +# calls Transit (see crates/kms/src/config.rs and +# docs/operations/kms-backend-security.md), so such prose tells operators +# their key material is cryptographically isolated inside Vault when it is +# not. +# +# Block 2 replaces the unit test `test_vault_kv2_sources_do_not_claim_transit_wrapping` +# that used to live in crates/kms/src/config.rs (rustfs/backlog#1884). The +# invariant is a documentation-claim invariant, so it has no behavioral twin by +# construction and belongs in a wording guard rather than in a test. The test +# could only see four `include_str!`-pinned files and stopped compiling — +# rather than reporting a violation — the moment one of them was renamed; this +# block scans every file in the crate and reports a rename explicitly. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="${CHECK_FIPS_WORDING_ROOT:-$(cd "${SCRIPT_DIR}/.." && pwd)}" @@ -21,12 +41,34 @@ FORBIDDEN_PATTERNS=( '(meets|satisfies)[[:space:]]+FIPS' ) -status=0 +KMS_CRATE_DIR="crates/kms" + +# The four files the retired unit test pinned with include_str!. They stay +# listed so that moving one out of crates/kms is reported here instead of +# silently shrinking the scan; the scan itself is not limited to them. +KMS_PINNED_SOURCES=( + "crates/kms/src/config.rs" + "crates/kms/src/api_types.rs" + "crates/kms/src/backends/vault.rs" + "crates/kms/src/lib.rs" +) + +# Literal, case-sensitive, and byte-for-byte the needles the retired test built +# at runtime via format!("wrapping via {}", "Transit") and friends. +KMS_VAULT_KV2_FORBIDDEN=( + 'wrapping via Transit' + 'KV v2 + Transit' + 'KV2+Transit' + "you would use Vault's transit engine" +) + +fips_status=0 +kms_status=0 for target in "${TARGETS[@]}"; do if [[ ! -f "$target" ]]; then printf 'FIPS wording guard failed: %s is missing\n' "$target" >&2 - status=1 + fips_status=1 continue fi @@ -35,14 +77,41 @@ for target in "${TARGETS[@]}"; do if [[ -n "$matches" ]]; then printf 'FIPS wording guard failed: forbidden pattern /%s/ in %s:\n%s\n' \ "$pattern" "$target" "$matches" >&2 - status=1 + fips_status=1 fi done done -if [[ "$status" -ne 0 ]]; then +for source in "${KMS_PINNED_SOURCES[@]}"; do + if [[ ! -f "$source" ]]; then + printf 'KMS wording guard failed: %s is missing; update KMS_PINNED_SOURCES in scripts/check_fips_wording.sh after moving it\n' \ + "$source" >&2 + kms_status=1 + fi +done + +if [[ -d "$KMS_CRATE_DIR" ]]; then + for pattern in "${KMS_VAULT_KV2_FORBIDDEN[@]}"; do + matches="$(grep -r -F -n -- "$pattern" "$KMS_CRATE_DIR" || true)" + if [[ -n "$matches" ]]; then + printf 'KMS wording guard failed: forbidden Vault KV2 claim "%s" in %s:\n%s\n' \ + "$pattern" "$KMS_CRATE_DIR" "$matches" >&2 + kms_status=1 + fi + done +fi + +if [[ "$fips_status" -ne 0 ]]; then printf 'Remove unsupported FIPS validation wording from README.md or CHANGELOG.md.\n' >&2 - exit "$status" +fi + +if [[ "$kms_status" -ne 0 ]]; then + printf 'The Vault KV2 backend does not wrap key material through Vault Transit; fix the wording in crates/kms.\n' >&2 +fi + +if [[ "$fips_status" -ne 0 || "$kms_status" -ne 0 ]]; then + exit 1 fi printf 'FIPS wording guard passed (README.md and CHANGELOG.md contain no forbidden claims).\n' +printf 'KMS wording guard passed (crates/kms claims no Vault KV2 Transit wrapping).\n' From 1d056d76055e0fb6fd72f9cb565f97b1acbe3dde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Tue, 18 Aug 2026 21:46:16 +0800 Subject: [PATCH 43/51] test(e2e): pin bounded physical reads for compressed multipart range GETs (#6167) Byte-exactness tests stay green if the compressed range seek regresses into decoding from byte zero: the returned bytes are still correct and only the read amplification explodes. Assert the cost side as well. The observation reuses rustfs_io_get_object_shard_read_observed_bytes_total, already emitted per shard read by the erasure layer, so no production code is instrumented. The OTLP collector learns to accumulate a second counter, keyed by its path/role/outcome labels rather than by data-point position, which is not stable across exports. Two failure modes the assertions guard against: - With RUSTFS_OBS_METER_INTERVAL=1, treating one unchanged sample as settled measures a delta of zero, because the range read's counter has not been exported yet. Settling now requires several consecutive equal samples. - An upper bound alone passes vacuously on a zero delta, so a lower bound turns "measured nothing" into a failure instead of a green run. Refs rustfs/rustfs#5957, backlog#1848. --- .../src/inline_fast_path_cluster_test.rs | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/crates/e2e_test/src/inline_fast_path_cluster_test.rs b/crates/e2e_test/src/inline_fast_path_cluster_test.rs index e2890c4a8..f1cdc1acf 100644 --- a/crates/e2e_test/src/inline_fast_path_cluster_test.rs +++ b/crates/e2e_test/src/inline_fast_path_cluster_test.rs @@ -67,6 +67,9 @@ type MetricValues = Arc>>; const KIB: usize = 1024; const READER_PATH_COUNTER: &str = "rustfs_io_get_object_reader_path_by_size_total"; +/// Physical bytes the erasure layer pulled from disk, emitted per shard read by +/// `crates/ecstore/src/erasure/coding/decode.rs`. +const SHARD_READ_BYTES_COUNTER: &str = "rustfs_io_get_object_shard_read_observed_bytes_total"; const MSGPACK_JSON_DECODE_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_decode_total"; const MSGPACK_JSON_FALLBACK_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_fallback_total"; const MSGPACK_JSON_DECODE_ERROR_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total"; @@ -146,6 +149,7 @@ struct OtlpMetricCollector { decode_values: MetricValues, fallback_values: MetricValues, decode_error_values: MetricValues, + shard_read_values: MetricValues, task: JoinHandle<()>, } @@ -157,10 +161,12 @@ impl OtlpMetricCollector { let decode_values = Arc::new(Mutex::new(BTreeMap::new())); let fallback_values = Arc::new(Mutex::new(BTreeMap::new())); let decode_error_values = Arc::new(Mutex::new(BTreeMap::new())); + let shard_read_values = Arc::new(Mutex::new(BTreeMap::new())); let task_values = values.clone(); let task_decode_values = decode_values.clone(); let task_fallback_values = fallback_values.clone(); let task_decode_error_values = decode_error_values.clone(); + let task_shard_read_values = shard_read_values.clone(); let task = tokio::spawn(async move { loop { let Ok((stream, _)) = listener.accept().await else { @@ -170,6 +176,7 @@ impl OtlpMetricCollector { let decode_values = task_decode_values.clone(); let fallback_values = task_fallback_values.clone(); let decode_error_values = task_decode_error_values.clone(); + let shard_read_values = task_shard_read_values.clone(); tokio::spawn(async move { let _ = hyper::server::conn::http1::Builder::new() .serve_connection( @@ -181,6 +188,7 @@ impl OtlpMetricCollector { decode_values.clone(), fallback_values.clone(), decode_error_values.clone(), + shard_read_values.clone(), ) }), ) @@ -194,10 +202,48 @@ impl OtlpMetricCollector { decode_values, fallback_values, decode_error_values, + shard_read_values, task, }) } + /// Total physical bytes read from disk across every shard-read label set. + async fn shard_read_bytes_total(&self) -> u64 { + self.shard_read_values + .lock() + .await + .values() + .map(|versions| versions.values().map(|(_, value)| *value).sum::()) + .sum() + } + + /// Waits until the shard-read counter stops advancing so a measurement window + /// is not polluted by exports still in flight. + /// + /// Requires several consecutive equal samples spanning more than one export + /// interval (`RUSTFS_OBS_METER_INTERVAL=1`): a single unchanged sample only + /// proves the latest export has not landed yet, which silently reads as "no + /// disk reads happened" and makes any upper-bound assertion vacuous. + async fn wait_for_shard_read_bytes_to_settle(&self) -> TestResult { + const REQUIRED_STABLE_SAMPLES: usize = 5; + let mut last = self.shard_read_bytes_total().await; + let mut stable = 0; + for _ in 0..60 { + sleep(Duration::from_millis(500)).await; + let current = self.shard_read_bytes_total().await; + if current == last { + stable += 1; + if stable >= REQUIRED_STABLE_SAMPLES { + return Ok(current); + } + } else { + stable = 0; + last = current; + } + } + Err("timed out waiting for shard-read byte counter to settle".into()) + } + async fn reader_path_total(&self, path: &str, object_class: &str, size_bucket: &str) -> u64 { self.reader_path_values(path, object_class, size_bucket).await.values().sum() } @@ -321,6 +367,7 @@ async fn handle_metric_export( decode_values: MetricValues, fallback_values: MetricValues, decode_error_values: MetricValues, + shard_read_values: MetricValues, ) -> Result>, Infallible> { if request.uri().path() != "/v1/metrics" { return Ok(response(StatusCode::NOT_FOUND)); @@ -354,7 +401,9 @@ async fn handle_metric_export( let mut decode_values = decode_values.lock().await; let mut fallback_values = fallback_values.lock().await; let mut decode_error_values = decode_error_values.lock().await; + let mut shard_read_values = shard_read_values.lock().await; record_reader_path_metrics(&export, &mut values); + record_shard_read_bytes_metrics(&export, &mut shard_read_values); record_msgpack_decode_metrics(&export, &mut decode_values); record_msgpack_fallback_metrics(&export, &mut fallback_values); record_msgpack_decode_error_metrics(&export, &mut decode_error_values); @@ -375,6 +424,50 @@ fn reader_path_metric_key(path: &str, object_class: &str, size_bucket: &str) -> format!("{path}\u{1f}{object_class}\u{1f}{size_bucket}") } +/// Accumulates `SHARD_READ_BYTES_COUNTER` across all label sets. Only the total +/// matters: it is the number of physical bytes the erasure layer actually pulled +/// from disk, which is what separates a bounded per-part read from a decode of +/// the whole object. +fn record_shard_read_bytes_metrics(export: &ExportMetricsServiceRequest, values: &mut BTreeMap) { + for resource_metrics in &export.resource_metrics { + for scope_metrics in &resource_metrics.scope_metrics { + for metric in &scope_metrics.metrics { + if metric.name != SHARD_READ_BYTES_COUNTER { + continue; + } + let Some(metric::Data::Sum(sum)) = &metric.data else { + continue; + }; + for point in &sum.data_points { + let Some(number_data_point::Value::AsInt(value)) = point.value.as_ref() else { + continue; + }; + let value = u64::try_from(*value).unwrap_or_default(); + // Keyed by labels, not by position: point order within an export + // is not guaranteed stable, so an index key would alias distinct + // series across batches. + let key = format!( + "{}\u{1f}{}\u{1f}{}", + attribute_string(&point.attributes, "path").unwrap_or_default(), + attribute_string(&point.attributes, "role").unwrap_or_default(), + attribute_string(&point.attributes, "outcome").unwrap_or_default(), + ); + values + .entry(key) + .or_default() + .entry(point.start_time_unix_nano) + .and_modify(|current| { + if point.time_unix_nano >= current.0 { + *current = (point.time_unix_nano, value); + } + }) + .or_insert((point.time_unix_nano, value)); + } + } + } + } +} + fn record_reader_path_metrics(export: &ExportMetricsServiceRequest, values: &mut BTreeMap) { for resource_metrics in &export.resource_metrics { for scope_metrics in &resource_metrics.scope_metrics { @@ -1864,6 +1957,86 @@ async fn four_node_multipart_disk_compression_roundtrip() -> TestResult { Ok(()) } +/// A tail range over a compressed multipart object must read only the physical +/// data it needs, not decode the object from byte zero. +/// +/// The byte-exactness tests around this one stay green even if the seek path +/// regresses into decoding from the start of the object: the bytes returned are +/// still correct, only the read amplification explodes. This asserts the cost +/// side, using `SHARD_READ_BYTES_COUNTER` — already emitted per shard read by the +/// erasure layer, so no production code is instrumented for the test. +/// +/// `get_compressed_offsets` skips whole preceding parts by their stored size and +/// then seeks inside the covering part via its compression index, so a bounded +/// read costs on the order of the covering part's block size against a ~5 MiB +/// object. +#[tokio::test] +#[serial] +async fn four_node_compressed_multipart_tail_range_reads_are_bounded() -> TestResult { + init_logging(); + + let collector = OtlpMetricCollector::start().await?; + let mut cluster = RustFSTestClusterEnvironment::new(4).await?; + configure_reader_metric_cluster(&mut cluster, &collector); + cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true"); + cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true"); + cluster.start().await?; + + let bucket = "inline-multipart-compression-tail-range"; + cluster.create_test_bucket(bucket).await?; + let client = cluster.create_s3_client(0)?; + let key = "multipart/tail-range.txt"; + let (body, _second_part, etag) = put_two_part_multipart(&client, bucket, key).await?; + + // Establish that the object really took the compressed read path; otherwise a + // small delta below would only prove compression never happened. + assert_reader_path( + &collector, + &client, + ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, COMPRESSED), + ) + .await?; + + let baseline = collector.wait_for_shard_read_bytes_to_settle().await?; + + let tail_len = 4 * KIB; + let start = body.len() - tail_len; + let end = body.len() - 1; + let range = client + .get_object() + .bucket(bucket) + .key(key) + .range(format!("bytes={start}-{end}")) + .send() + .await?; + let tail = range.body.collect().await?.into_bytes(); + assert_eq!(tail.as_ref(), &body[start..], "tail range returned wrong bytes"); + + let after = collector.wait_for_shard_read_bytes_to_settle().await?; + let read_bytes = after.saturating_sub(baseline); + + // A zero delta means the window caught nothing — an unexported counter, or a + // read served without touching the erasure layer — which would make the upper + // bound vacuously true. Fail instead of passing blind. + assert!( + read_bytes > 0, + "no shard reads observed for the tail range; the budget assertion below would be vacuous" + ); + + // Part 1 alone is MPU_PART_1_SIZE, so a whole-object decode cannot come in + // under it. Half the logical size leaves generous headroom for erasure padding + // and unrelated background reads while still failing loudly on a full decode. + let budget = (body.len() / 2) as u64; + assert!( + read_bytes < budget, + "tail range read {read_bytes} physical bytes for a {tail_len}-byte range (budget {budget}, object {} bytes): \ + the read is not bounded to the covering part", + body.len() + ); + + Ok(()) +} + #[tokio::test] #[serial] async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> TestResult { From 91c97f3416b143d1e993f58c3ec7148abd92a00a Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Tue, 18 Aug 2026 21:46:56 +0800 Subject: [PATCH 44/51] chore(deps): update s3s to upstream main (#6203) * deps: update s3s to upstream main * deps: refresh s3s upstream revision * deps: pin s3s to latest upstream main --------- Co-authored-by: houseme Co-authored-by: heihutu --- Cargo.lock | 2 +- Cargo.toml | 2 +- rustfs/src/app/metadata_route.rs | 24 ++++++++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db75a7992..c73ac8d64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10678,7 +10678,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3s" version = "0.14.1" -source = "git+https://github.com/rustfs/s3s.git?rev=ff8106f46098b60fce8ee88518f34974fbfbec4b#ff8106f46098b60fce8ee88518f34974fbfbec4b" +source = "git+https://github.com/rustfs/s3s.git?rev=d358a68783096df1db0c3e314127f2704603b29e#d358a68783096df1db0c3e314127f2704603b29e" dependencies = [ "arc-swap", "arrayvec", diff --git a/Cargo.toml b/Cargo.toml index f2080a15f..b6a37b028 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -290,7 +290,7 @@ rustify = { version = "0.7", default-features = false } rustix = { version = "1.1.4" } rust-embed = { version = "8.12.0" } rustc-hash = { version = "2.1.3" } -s3s = { git = "https://github.com/rustfs/s3s.git", rev = "ff8106f46098b60fce8ee88518f34974fbfbec4b" } +s3s = { git = "https://github.com/rustfs/s3s.git", rev = "d358a68783096df1db0c3e314127f2704603b29e" } serial_test = "4.0.1" shadow-rs = { default-features = false, version = "2.0.0" } siphasher = "1.0.3" diff --git a/rustfs/src/app/metadata_route.rs b/rustfs/src/app/metadata_route.rs index d6509f60e..3a49a41e7 100644 --- a/rustfs/src/app/metadata_route.rs +++ b/rustfs/src/app/metadata_route.rs @@ -349,6 +349,30 @@ mod tests { ); } + #[test] + fn metadata_operation_matches_unconfigured_host_fallbacks() { + let host = MultiDomain::new(["s3.example.com", "s3.example.com:9000"]).expect("valid test host domain"); + + let mut path_style_headers = HeaderMap::new(); + path_style_headers.insert(HOST, "localhost:9000".parse().expect("valid host header")); + let path_style = metadata_operation( + &Method::GET, + &uri("/path-bucket?list-type=2&metadata=true"), + &path_style_headers, + Some(&host), + ) + .expect("unmatched host with a port should use path-style routing"); + assert_eq!(path_style.operation, MetadataOperation::ListObjectsV2); + assert_eq!(path_style.bucket, "path-bucket"); + + let mut cname_headers = HeaderMap::new(); + cname_headers.insert(HOST, "cdn.example.org".parse().expect("valid host header")); + let cname = metadata_operation(&Method::GET, &uri("/?list-type=2&metadata=true"), &cname_headers, Some(&host)) + .expect("unmatched valid bucket host should use CNAME routing"); + assert_eq!(cname.operation, MetadataOperation::ListObjectsV2); + assert_eq!(cname.bucket, "cdn.example.org"); + } + #[test] fn list_objects_v2_input_parses_query_headers_and_decodes_bucket() { let mut headers = HeaderMap::new(); From eed0ca3612a530d750b7bd746ae71fef05b8fe4d Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 22:40:43 +0800 Subject: [PATCH 45/51] perf(ecstore): validate local IO paths with openat2 (#6221) Use Linux openat2 with RESOLVE_BENEATH and RESOLVE_NO_SYMLINKS for LocalDisk I/O path validation while keeping the existing lstat walk as the public-path and unsupported-kernel fallback. Add focused regression coverage for traversal, symlink swaps, missing leaves, recreated parents, high-cardinality prefixes, final symlink leaves, and concurrent validation. Co-authored-by: heihutu --- crates/ecstore/src/disk/local.rs | 364 ++++++++++++++++++++++++++++++- 1 file changed, 356 insertions(+), 8 deletions(-) diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index a71f2c55b..273ada7d7 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -5618,11 +5618,37 @@ impl LocalDisk { } fn io_get_object_path(&self, bucket: &str, key: &str) -> Result { - local_disk_object_path(self.io_root(), bucket, key) + self.local_disk_object_path(self.io_root(), bucket, key) } fn io_get_bucket_path(&self, bucket: &str) -> Result { - local_disk_bucket_path(self.io_root(), bucket) + self.local_disk_bucket_path(self.io_root(), bucket) + } + + fn local_disk_object_path(&self, root: &Path, bucket: &str, key: &str) -> Result { + let (bucket_path, path) = build_local_disk_object_path(root, bucket, key); + #[cfg(target_os = "linux")] + { + check_local_disk_valid_object_path_at(root, &self.mount_lease, &bucket_path, &path)?; + } + #[cfg(not(target_os = "linux"))] + { + check_local_disk_valid_object_path(root, &bucket_path, &path)?; + } + Ok(path) + } + + fn local_disk_bucket_path(&self, root: &Path, bucket: &str) -> Result { + let bucket_path = build_local_disk_bucket_path(root, bucket); + #[cfg(target_os = "linux")] + { + check_local_disk_valid_path_at(root, &self.mount_lease, &bucket_path)?; + } + #[cfg(not(target_os = "linux"))] + { + check_local_disk_valid_path(root, &bucket_path)?; + } + Ok(bucket_path) } // Check if a path is valid @@ -5631,7 +5657,14 @@ impl LocalDisk { reason = "method wrapper over the live free function check_local_disk_valid_path; no caller in this port (backlog#1823)" )] fn check_valid_path>(&self, path: P) -> Result<()> { - check_local_disk_valid_path(self.io_root(), path) + #[cfg(target_os = "linux")] + { + check_local_disk_valid_path_at(self.io_root(), &self.mount_lease, path) + } + #[cfg(not(target_os = "linux"))] + { + check_local_disk_valid_path(self.io_root(), path) + } } #[allow( @@ -5639,7 +5672,14 @@ impl LocalDisk { reason = "method wrapper over the live free function reject_local_disk_symlink_components; no caller in this port (backlog#1823)" )] fn reject_symlink_components(&self, path: &Path) -> Result<()> { - reject_local_disk_symlink_components(self.io_root(), path) + #[cfg(target_os = "linux")] + { + reject_local_disk_symlink_components_at(self.io_root(), &self.mount_lease, path) + } + #[cfg(not(target_os = "linux"))] + { + reject_local_disk_symlink_components(self.io_root(), path) + } } // Batch path generation with single lock acquisition @@ -7323,29 +7363,54 @@ fn skip_access_checks(p: impl AsRef) -> bool { } fn local_disk_object_path(root: &Path, bucket: &str, key: &str) -> Result { + let (bucket_path, path) = build_local_disk_object_path(root, bucket, key); + check_local_disk_valid_object_path(root, &bucket_path, &path)?; + Ok(path) +} + +fn build_local_disk_object_path(root: &Path, bucket: &str, key: &str) -> (PathBuf, PathBuf) { let cache_key = if key.is_empty() { bucket.to_string() } else { path_join_buf(&[bucket, key]) }; + #[cfg(windows)] + let bucket_path = root.join(bucket.replace('/', "\\")); + #[cfg(not(windows))] + let bucket_path = root.join(bucket); + #[cfg(windows)] let path = root.join(cache_key.replace('/', "\\")); #[cfg(not(windows))] let path = root.join(cache_key); - check_local_disk_valid_path(root, &path)?; - Ok(path) + (bucket_path, path) } fn local_disk_bucket_path(root: &Path, bucket: &str) -> Result { + let bucket_path = build_local_disk_bucket_path(root, bucket); + check_local_disk_valid_path(root, &bucket_path)?; + Ok(bucket_path) +} + +fn build_local_disk_bucket_path(root: &Path, bucket: &str) -> PathBuf { #[cfg(windows)] let bucket_path = root.join(bucket.replace('/', "\\")); #[cfg(not(windows))] let bucket_path = root.join(bucket); - check_local_disk_valid_path(root, &bucket_path)?; - Ok(bucket_path) + bucket_path +} + +fn check_local_disk_valid_object_path(root: &Path, bucket_path: &Path, path: &Path) -> Result<()> { + let bucket_path = normalize_path_components(bucket_path); + let path = normalize_path_components(path); + if !bucket_path.starts_with(root) || !path.starts_with(&bucket_path) { + return Err(DiskError::InvalidPath); + } + + reject_local_disk_symlink_components(root, &path) } fn check_local_disk_valid_path(root: &Path, path: impl AsRef) -> Result<()> { @@ -7357,6 +7422,80 @@ fn check_local_disk_valid_path(root: &Path, path: impl AsRef) -> Result<() reject_local_disk_symlink_components(root, &path) } +#[cfg(target_os = "linux")] +fn check_local_disk_valid_object_path_at(root: &Path, root_fd: &std::fs::File, bucket_path: &Path, path: &Path) -> Result<()> { + let bucket_path = normalize_path_components(bucket_path); + let path = normalize_path_components(path); + if !bucket_path.starts_with(root) || !path.starts_with(&bucket_path) { + return Err(DiskError::InvalidPath); + } + + reject_local_disk_symlink_components_at(root, root_fd, &path) +} + +#[cfg(target_os = "linux")] +fn check_local_disk_valid_path_at(root: &Path, root_fd: &std::fs::File, path: impl AsRef) -> Result<()> { + let path = normalize_path_components(path); + if !path.starts_with(root) { + return Err(DiskError::InvalidPath); + } + + reject_local_disk_symlink_components_at(root, root_fd, &path) +} + +#[cfg(target_os = "linux")] +fn reject_local_disk_symlink_components_at(root: &Path, root_fd: &std::fs::File, path: &Path) -> Result<()> { + let relative = path.strip_prefix(root).map_err(|_| DiskError::InvalidPath)?; + match validate_existing_local_disk_prefix_at(root_fd, relative) { + Ok(()) => Ok(()), + Err(LocalDiskPathValidationAtError::Unsupported) => reject_local_disk_symlink_components(root, path), + Err(LocalDiskPathValidationAtError::InvalidPath) => Err(DiskError::InvalidPath), + Err(LocalDiskPathValidationAtError::Io(err)) => Err(to_file_error(err).into()), + } +} + +#[cfg(target_os = "linux")] +enum LocalDiskPathValidationAtError { + Unsupported, + InvalidPath, + Io(std::io::Error), +} + +#[cfg(target_os = "linux")] +fn validate_existing_local_disk_prefix_at( + root_fd: &std::fs::File, + relative: &Path, +) -> core::result::Result<(), LocalDiskPathValidationAtError> { + use rustix::fs::{Mode, OFlags, ResolveFlags, openat2}; + use rustix::io::Errno; + + if relative.as_os_str().is_empty() { + return Ok(()); + } + + let mut candidate = relative.to_path_buf(); + loop { + match openat2( + root_fd, + &candidate, + OFlags::PATH | OFlags::CLOEXEC, + Mode::empty(), + ResolveFlags::BENEATH | ResolveFlags::NO_SYMLINKS, + ) { + Ok(_) => return Ok(()), + Err(Errno::NOSYS) => return Err(LocalDiskPathValidationAtError::Unsupported), + Err(Errno::LOOP | Errno::XDEV) => return Err(LocalDiskPathValidationAtError::InvalidPath), + Err(Errno::NOENT) => { + let Some(parent) = candidate.parent().filter(|parent| !parent.as_os_str().is_empty()) else { + return Ok(()); + }; + candidate = parent.to_path_buf(); + } + Err(err) => return Err(LocalDiskPathValidationAtError::Io(err.into())), + } + } +} + fn reject_local_disk_symlink_components(root: &Path, path: &Path) -> Result<()> { let relative = path.strip_prefix(root).map_err(|_| DiskError::InvalidPath)?; let mut current = root.to_path_buf(); @@ -18068,6 +18207,22 @@ mod test { assert!(matches!(disk.get_bucket_path("escape-bucket"), Err(DiskError::InvalidPath))); } + #[cfg(unix)] + #[tokio::test] + async fn get_bucket_path_for_io_rejects_symlink_escape() { + use std::os::unix::fs::symlink; + + let root_dir = tempfile::tempdir().expect("temp dir should be created"); + let outside_dir = tempfile::tempdir().expect("outside temp dir should be created"); + let link_path = root_dir.path().join("escape-bucket"); + symlink(outside_dir.path(), &link_path).expect("bucket symlink should be created"); + + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + + assert!(matches!(disk.get_bucket_path_for_io("escape-bucket"), Err(DiskError::InvalidPath))); + } + #[cfg(unix)] #[tokio::test] async fn test_get_object_path_rejects_symlink_component_escape() { @@ -18087,6 +18242,199 @@ mod test { assert!(matches!(disk.get_object_path("bucket", "escape/object.txt"), Err(DiskError::InvalidPath))); } + #[cfg(unix)] + #[tokio::test] + async fn get_object_path_for_io_rejects_symlink_leaf() { + use std::os::unix::fs::symlink; + + let root_dir = tempfile::tempdir().expect("temp dir should be created"); + let outside_file = root_dir.path().join("outside-file"); + fs::write(&outside_file, b"outside") + .await + .expect("outside file should be created"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + disk.make_volume("bucket").await.expect("bucket should be created"); + symlink(&outside_file, root_dir.path().join("bucket/object")).expect("object symlink should be created"); + + assert!(matches!(disk.get_object_path_for_io("bucket", "object"), Err(DiskError::InvalidPath))); + } + + #[tokio::test] + async fn get_object_path_rejects_key_traversal_out_of_bucket() { + let root_dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + + assert!(matches!(disk.get_object_path("bucket", "../outside"), Err(DiskError::InvalidPath))); + assert!(matches!( + disk.get_object_path("bucket", "prefix/../../outside"), + Err(DiskError::InvalidPath) + )); + } + + #[tokio::test] + async fn get_object_path_accepts_missing_leaf_under_existing_bucket() { + let root_dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + disk.make_volume("bucket").await.expect("bucket should be created"); + + let object_path = disk + .get_object_path("bucket", "missing-object") + .expect("missing leaf under a valid bucket should resolve"); + + assert_eq!(object_path, disk.root.join("bucket/missing-object")); + } + + #[tokio::test] + async fn get_object_path_for_io_rejects_key_traversal_out_of_bucket() { + let root_dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + + assert!(matches!(disk.get_object_path_for_io("bucket", "../outside"), Err(DiskError::InvalidPath))); + assert!(matches!( + disk.get_object_path_for_io("bucket", "prefix/../../outside"), + Err(DiskError::InvalidPath) + )); + } + + #[tokio::test] + async fn get_object_path_for_io_accepts_missing_leaf_under_existing_bucket() { + let root_dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + disk.make_volume("bucket").await.expect("bucket should be created"); + + let object_path = disk + .get_object_path_for_io("bucket", "missing-object") + .expect("missing leaf under a valid I/O bucket should resolve"); + + assert!(object_path.ends_with("bucket/missing-object")); + } + + #[cfg(unix)] + #[tokio::test] + async fn get_object_path_rejects_symlink_component_after_prior_valid_lookup() { + use std::os::unix::fs::symlink; + + let root_dir = tempfile::tempdir().expect("temp dir should be created"); + let outside_dir = tempfile::tempdir().expect("outside temp dir should be created"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + let prefix = root_dir.path().join("bucket/prefix"); + fs::create_dir_all(&prefix).await.expect("prefix should be created"); + + disk.get_object_path("bucket", "prefix/object") + .expect("initial lookup should validate the real prefix"); + fs::remove_dir(&prefix).await.expect("prefix should be removable"); + symlink(outside_dir.path(), &prefix).expect("prefix should be replaced by a symlink"); + + assert!(matches!(disk.get_object_path("bucket", "prefix/object"), Err(DiskError::InvalidPath))); + } + + #[cfg(unix)] + #[tokio::test] + async fn get_object_path_for_io_rejects_symlink_component_after_prior_valid_lookup() { + use std::os::unix::fs::symlink; + + let root_dir = tempfile::tempdir().expect("temp dir should be created"); + let outside_dir = tempfile::tempdir().expect("outside temp dir should be created"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + let prefix = root_dir.path().join("bucket/prefix"); + fs::create_dir_all(&prefix).await.expect("prefix should be created"); + + disk.get_object_path_for_io("bucket", "prefix/object") + .expect("initial I/O lookup should validate the real prefix"); + fs::remove_dir(&prefix).await.expect("prefix should be removable"); + symlink(outside_dir.path(), &prefix).expect("prefix should be replaced by a symlink"); + + assert!(matches!( + disk.get_object_path_for_io("bucket", "prefix/object"), + Err(DiskError::InvalidPath) + )); + } + + #[tokio::test] + async fn get_object_path_accepts_parent_recreated_after_prior_valid_lookup() { + let root_dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + let prefix = root_dir.path().join("bucket/prefix"); + fs::create_dir_all(&prefix).await.expect("prefix should be created"); + + disk.get_object_path("bucket", "prefix/object") + .expect("initial lookup should validate the real prefix"); + fs::remove_dir(&prefix).await.expect("prefix should be removable"); + fs::create_dir(&prefix).await.expect("prefix should be recreated"); + + let object_path = disk + .get_object_path("bucket", "prefix/object") + .expect("recreated non-symlink parent should validate"); + assert_eq!(object_path, disk.root.join("bucket/prefix/object")); + } + + #[tokio::test] + async fn get_object_path_for_io_accepts_parent_recreated_after_prior_valid_lookup() { + let root_dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + let prefix = root_dir.path().join("bucket/prefix"); + fs::create_dir_all(&prefix).await.expect("prefix should be created"); + + disk.get_object_path_for_io("bucket", "prefix/object") + .expect("initial I/O lookup should validate the real prefix"); + fs::remove_dir(&prefix).await.expect("prefix should be removable"); + fs::create_dir(&prefix).await.expect("prefix should be recreated"); + + let object_path = disk + .get_object_path_for_io("bucket", "prefix/object") + .expect("recreated non-symlink parent should validate for I/O"); + assert!(object_path.ends_with("bucket/prefix/object")); + } + + #[tokio::test] + async fn get_object_path_handles_many_unique_missing_prefixes_without_state_growth() { + let root_dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + disk.make_volume("bucket").await.expect("bucket should be created"); + + for index in 0..5000 { + let object_path = disk + .get_object_path("bucket", &format!("prefix-{index}/object")) + .expect("unique missing prefix should validate without shared state"); + assert!(object_path.ends_with(format!("bucket/prefix-{index}/object"))); + } + } + + #[tokio::test] + async fn get_object_path_concurrent_validation_keeps_paths_under_bucket() { + let root_dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + disk.make_volume("bucket").await.expect("bucket should be created"); + let barrier = Arc::new(tokio::sync::Barrier::new(32)); + let mut tasks = Vec::with_capacity(32); + + for index in 0..32 { + let disk = disk.clone(); + let barrier = barrier.clone(); + tasks.push(tokio::spawn(async move { + barrier.wait().await; + disk.get_object_path("bucket", &format!("object-{index}")) + .expect("concurrent validation should resolve object path") + })); + } + + for task in tasks { + let object_path = task.await.expect("validation task should complete"); + assert!(object_path.starts_with(disk.root.join("bucket"))); + } + } + #[tokio::test] async fn test_local_disk_file_operations() { let test_dir = "./test_local_disk_file_ops"; From 905082893fc0cc1a80148c5975a98d656f45398d Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 23:12:39 +0800 Subject: [PATCH 46/51] fix(e2e): import serial test attribute (#6222) --- crates/e2e_test/src/replication_extension_test.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index 8279479f1..b9645deba 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -60,6 +60,7 @@ use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; use s3s::header::X_AMZ_REPLICATION_STATUS; +use serial_test::serial; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; use std::convert::Infallible; From c86a94a2dc495e240f49aa81bd1e7513fd76128a Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:56:52 +0800 Subject: [PATCH 47/51] fix(package): declare /etc/default/rustfs as a deb conffile (#6220) Co-authored-by: houseme --- .github/workflows/package.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index b9536d263..95ec5a869 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -290,6 +290,12 @@ jobs: Homepage: https://rustfs.com EOF + # Declare /etc/default/rustfs as a conffile so dpkg preserves user + # modifications on upgrade instead of silently overwriting them. + cat > "${PKG_DIR}/DEBIAN/conffiles" << 'CONFFILES' + /etc/default/rustfs + CONFFILES + cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST' #!/bin/bash set -e From 50c39fec4518db61cbc5eddbbc7d531aea4f8cf6 Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 19 Aug 2026 01:20:16 +0800 Subject: [PATCH 48/51] perf(get): emit accept-ranges with static header (#6225) Avoid constructing the typed Accept-Ranges string on the GetObject output path. Inject the static header after CORS wrapping so the final S3 response remains unchanged while the hot path avoids one fixed per-GET allocation/conversion. Co-authored-by: heihutu --- rustfs/src/app/object_usecase.rs | 38 +++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 413a88464..b63110d39 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -2998,6 +2998,10 @@ pub(crate) fn inject_additional_checksum_headers(headers: &mut HeaderMap, pairs: } } +fn inject_accept_ranges_header(headers: &mut HeaderMap) { + headers.insert(http::header::ACCEPT_RANGES, HeaderValue::from_static(ACCEPT_RANGES_BYTES)); +} + /// Derive the response-header echo pairs for an additional-checksum algorithm /// (XXHash3/64/128, SHA-512) from the server-computed content checksum, for /// PutObject to echo back (#1256). Returns empty for the five s3s-typed algorithms @@ -6500,6 +6504,7 @@ impl DefaultObjectUsecase { }; let helper = helper.version_id(version_id_for_event); let mut response = wrap_response_with_cors(bucket, method, headers, output).await; + inject_accept_ranges_header(&mut response.headers); // Emit XXHash3/64/128 and SHA-512 checksums that s3s GetObjectOutput cannot // carry (#1257). This is the download-side integrity path AWS SDKs verify. inject_additional_checksum_headers(&mut response.headers, &extra_checksum_headers); @@ -6633,7 +6638,6 @@ impl DefaultObjectUsecase { content_encoding: info.content_encoding.clone(), cache_control, content_disposition, - accept_ranges: Some(ACCEPT_RANGES_BYTES.to_string()), content_range, e_tag: info.etag.map(|etag| to_s3s_etag(&etag)), metadata, @@ -6811,7 +6815,6 @@ impl DefaultObjectUsecase { content_disposition: remote.content_disposition, content_language: remote.content_language, cache_control: remote.cache_control, - accept_ranges: Some(ACCEPT_RANGES_BYTES.to_string()), e_tag: remote.e_tag.as_deref().and_then(|v| ETag::from_str(v).ok()), last_modified: remote .last_modified @@ -7016,7 +7019,8 @@ impl DefaultObjectUsecase { && let Some(output) = Self::proxy_get_object_to_replication_targets(&req, &bucket, &key, &opts).await { lifecycle.finish_ok(); - let response = wrap_response_with_cors(&bucket, &req.method, &req.headers, output).await; + let mut response = wrap_response_with_cors(&bucket, &req.method, &req.headers, output).await; + inject_accept_ranges_header(&mut response.headers); let result = Ok(response); let _ = helper.version_id(version_id_for_event).complete(&result); return result; @@ -10290,6 +10294,34 @@ mod tests { assert!(empty.is_empty()); } + #[test] + fn inject_accept_ranges_header_writes_static_bytes_value() { + let mut headers = HeaderMap::new(); + inject_accept_ranges_header(&mut headers); + + assert_eq!(headers.get(http::header::ACCEPT_RANGES).unwrap(), ACCEPT_RANGES_BYTES); + } + + #[tokio::test] + async fn finalize_get_object_response_injects_accept_ranges_header() { + let req = build_request(GetObjectInput::default(), Method::GET); + let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject).suppress_event(); + let response = DefaultObjectUsecase::finalize_get_object_response( + helper, + "bucket", + &req.method, + &req.headers, + None, + String::new(), + GetObjectOutput::default(), + Vec::new(), + ) + .await + .expect("finalize response"); + + assert_eq!(response.headers.get(http::header::ACCEPT_RANGES).unwrap(), ACCEPT_RANGES_BYTES); + } + fn build_request(input: T, method: Method) -> S3Request { S3Request { input, From 850445e95703581b36be9b1ab8e58836cdf7c8bb Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 06:24:15 +0800 Subject: [PATCH 49/51] fix(e2e): add missing serial_test import in replication_extension_test (#6224) From ff9ac1013ad2cf175b4b1c8ae95be9d6d78104bf Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 19 Aug 2026 06:24:45 +0800 Subject: [PATCH 50/51] feat(ecstore): add default-off dst dir fsync group commit (#6226) * feat(ecstore): add dst dir fsync group commit Co-Authored-By: heihutu * fix(ecstore): tidy dst dir fsync group open Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- .config/nextest.toml | 11 + crates/ecstore/src/disk/local.rs | 72 ++- crates/ecstore/src/disk/os.rs | 761 ++++++++++++++++++++++++++++++- 3 files changed, 840 insertions(+), 4 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index ade76a58c..4dd2d1d0e 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -87,6 +87,13 @@ test-group = 'ecstore-serial-flaky' filter = 'package(rustfs-ecstore) & (test(bucket_delete_waits_for_config_mutation_fence) | test(stale_config_request_cannot_mutate_a_recreated_bucket) | test(disk_incarnation_read_detects_stale_cache_until_peer_reload) | test(lifecycle_expiry_fails_closed_on_corrupt_object_lock_metadata) | test(expiry_configs_are_resolved_from_the_owning_store))' test-group = 'ecstore-serial-flaky' +# Serialize the default-off dst-dir fsync group-commit tests. They use +# process-global test hooks/registry to deterministically freeze fsync batches; +# no retries, just one at a time under nextest too. +[[profile.default.overrides]] +filter = 'package(rustfs-ecstore) & test(dst_dir_fsync_group_commit)' +test-group = 'ecstore-serial-flaky' + # Serialize the 4-disk reliability / degraded-read e2e tests (see the # e2e-reliability test-group note above). The matching ci-profile override is at # the end of the file, after [profile.ci] is declared. @@ -188,6 +195,10 @@ test-group = 'ecstore-serial-flaky' filter = 'package(rustfs-ecstore) & (test(bucket_delete_waits_for_config_mutation_fence) | test(stale_config_request_cannot_mutate_a_recreated_bucket) | test(disk_incarnation_read_detects_stale_cache_until_peer_reload) | test(lifecycle_expiry_fails_closed_on_corrupt_object_lock_metadata) | test(expiry_configs_are_resolved_from_the_owning_store))' test-group = 'ecstore-serial-flaky' +[[profile.ci.overrides]] +filter = 'package(rustfs-ecstore) & test(dst_dir_fsync_group_commit)' +test-group = 'ecstore-serial-flaky' + # --------------------------------------------------------------------------- # e2e-smoke profile — PR smoke subset of the e2e_test crate (backlog#1149 ci-4) # --------------------------------------------------------------------------- diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 273ada7d7..285a12ba1 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -9405,7 +9405,7 @@ impl DiskAPI for LocalDisk { && let Some(parent) = dst_file_path.parent() { let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dir(parent).await { + if let Err(err) = os::fsync_dst_dir_group_commit(parent).await { rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, fsync_started, @@ -13022,6 +13022,76 @@ mod test { ); } + #[tokio::test] + #[serial_test::serial(dst_dir_fsync_group_commit)] + async fn rename_data_non_inline_uses_dst_dir_fsync_group_commit_when_enabled() { + let _group_commit = os::set_dst_dir_fsync_group_commit_for_test(true); + let bucket = "grouped-dst-fsync-bucket"; + let object = "dir/object"; + let (disk, _dir) = commit_new_object(DurabilityMode::Strict, bucket, object).await; + let dst_meta_parent = disk + .get_object_path(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}")) + .expect("dst meta path should resolve") + .parent() + .expect("dst meta should have a parent") + .to_path_buf(); + + assert_eq!( + os::fsync_dir_recorder::grouped_batch_sizes(&dst_meta_parent), + vec![1], + "enabled non-inline rename_data must route the dst parent fsync through the group commit coordinator" + ); + } + + #[tokio::test] + #[serial_test::serial(dst_dir_fsync_group_commit)] + async fn rename_data_non_inline_dst_dir_fsync_group_commit_failure_rolls_back_fresh_put() { + use tempfile::tempdir; + + let _group_commit = os::set_dst_dir_fsync_group_commit_for_test(true); + let _mode = durability_mode_override::set(DurabilityMode::Strict); + let dir = tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + let bucket = "grouped-dst-fsync-failure-bucket"; + let object = "dir/object"; + let tmp_object = "tmp-grouped-dst-fsync-failure"; + let version_id = Uuid::parse_str("99999999-9999-9999-9999-999999999999").expect("version id should parse"); + let new_data_dir = Uuid::parse_str("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa").expect("data dir should parse"); + ensure_test_volume(&disk, bucket).await; + ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await; + + let tmp_data_dir = dir + .path() + .join(RUSTFS_META_TMP_BUCKET) + .join(tmp_object) + .join(new_data_dir.to_string()); + fs::create_dir_all(&tmp_data_dir) + .await + .expect("new tmp data dir should be created"); + fs::write(tmp_data_dir.join("part.1"), b"new-data") + .await + .expect("new tmp data should be written"); + let dst_meta_parent = dir.path().join(bucket).join(object); + os::fsync_dir_recorder::set_grouped_failure(&dst_meta_parent, io::ErrorKind::PermissionDenied); + + let new_fi = test_file_info(object, version_id, Some(new_data_dir), None); + let err = disk + .rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, new_fi, bucket, object) + .await + .expect_err("grouped dst dir fsync failure must fail the fresh PUT"); + + assert_eq!(err, DiskError::FileAccessDenied); + assert!( + !dst_meta_parent.join(STORAGE_FORMAT_FILE).exists(), + "fresh PUT rollback must remove the committed xl.meta after grouped dst dir fsync failure" + ); + assert!( + !dst_meta_parent.join(new_data_dir.to_string()).exists(), + "fresh PUT rollback must remove the committed data dir after grouped dst dir fsync failure" + ); + } + // Seed a first PUT of `object` (no prior version) through the non-inline // rename_data path and return (disk, tempdir). The object dir and any prefix // dirs are created during the commit. diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index decb36964..bc0ed5def 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -19,14 +19,14 @@ use futures::TryStreamExt; use parking_lot::Mutex; use rustfs_utils::path::SLASH_SEPARATOR; use std::{ - collections::HashMap, + collections::{HashMap, VecDeque}, io, path::{Component, Path, PathBuf}, sync::{Arc, LazyLock, Weak}, }; use tokio::fs; use tokio::sync::{ - Mutex as AsyncMutex, OwnedMutexGuard, OwnedRwLockReadGuard, OwnedSemaphorePermit, RwLock, Semaphore, SemaphorePermit, + Mutex as AsyncMutex, OwnedMutexGuard, OwnedRwLockReadGuard, OwnedSemaphorePermit, RwLock, Semaphore, SemaphorePermit, oneshot, }; use tracing::warn; @@ -79,6 +79,7 @@ pub fn check_path_length(path_name: &str) -> Result<()> { #[cfg(test)] pub(crate) mod fsync_dir_recorder { use std::collections::HashMap; + use std::io; use std::path::{Path, PathBuf}; use std::sync::Mutex; @@ -86,8 +87,17 @@ pub(crate) mod fsync_dir_recorder { static RECORDED: Mutex> = Mutex::new(Vec::new()); static LIMITED: Mutex> = Mutex::new(Vec::new()); + static GROUPED: Mutex> = Mutex::new(Vec::new()); static BEFORE_LIMITED: std::sync::LazyLock>> = std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); + static BEFORE_GROUP_BATCH: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); + static AFTER_GROUP_ENQUEUE: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); + static BEFORE_GROUPED: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); + static GROUPED_FAILURES: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); fn record_path(paths: &Mutex>, path: &Path, description: &str) { let mut paths = paths.lock().expect(description); @@ -106,6 +116,29 @@ pub(crate) mod fsync_dir_recorder { .any(|recorded| recorded == path || canonical.as_ref().is_some_and(|canonical| recorded == canonical)) } + fn remove_path_keyed(entries: &Mutex>, dir: &Path, description: &str) -> Option { + let mut entries = entries.lock().expect(description); + if let Some(value) = entries.remove(dir) { + return Some(value); + } + let canonical = dir.canonicalize().ok(); + let matching_key = entries + .keys() + .find(|registered| { + registered.as_path() == dir + || canonical.as_ref().is_some_and(|canonical| *registered == canonical) + || registered.canonicalize().ok().is_some_and(|registered_canonical| { + registered_canonical == dir || canonical.as_ref() == Some(®istered_canonical) + }) + }) + .cloned(); + matching_key.and_then(|key| entries.remove(&key)) + } + + fn remove_hook(hooks: &Mutex>, dir: &Path, description: &str) -> Option { + remove_path_keyed(hooks, dir, description) + } + pub(crate) fn record(dir: &Path) { record_path(&RECORDED, dir, "fsync dir recorder"); } @@ -116,7 +149,7 @@ pub(crate) mod fsync_dir_recorder { pub(crate) fn record_limited(dir: &Path) { record_path(&LIMITED, dir, "limited fsync dir recorder"); - let hook = BEFORE_LIMITED.lock().expect("limited fsync hook poisoned").remove(dir); + let hook = remove_hook(&BEFORE_LIMITED, dir, "limited fsync hook poisoned"); if let Some(hook) = hook { hook(); } @@ -132,6 +165,78 @@ pub(crate) mod fsync_dir_recorder { .expect("limited fsync hook poisoned") .insert(dir.to_path_buf(), Box::new(hook)); } + + pub(crate) fn record_grouped(dir: &Path, batch_len: usize) { + let mut grouped = GROUPED.lock().expect("grouped fsync dir recorder poisoned"); + grouped.push((dir.to_path_buf(), batch_len)); + if let Ok(canonical) = dir.canonicalize() + && canonical != dir + { + grouped.push((canonical, batch_len)); + } + drop(grouped); + let hook = remove_hook(&BEFORE_GROUPED, dir, "grouped fsync hook poisoned"); + if let Some(hook) = hook { + hook(); + } + } + + pub(crate) fn run_before_group_batch(dir: &Path) { + let hook = remove_hook(&BEFORE_GROUP_BATCH, dir, "grouped fsync batch hook poisoned"); + if let Some(hook) = hook { + hook(); + } + } + + pub(crate) fn set_before_group_batch(dir: &Path, hook: impl FnOnce() + Send + 'static) { + BEFORE_GROUP_BATCH + .lock() + .expect("grouped fsync batch hook poisoned") + .insert(dir.to_path_buf(), Box::new(hook)); + } + + pub(crate) fn run_after_group_enqueue(dir: &Path) { + let hook = remove_hook(&AFTER_GROUP_ENQUEUE, dir, "grouped fsync enqueue hook poisoned"); + if let Some(hook) = hook { + hook(); + } + } + + pub(crate) fn set_after_group_enqueue(dir: &Path, hook: impl FnOnce() + Send + 'static) { + AFTER_GROUP_ENQUEUE + .lock() + .expect("grouped fsync enqueue hook poisoned") + .insert(dir.to_path_buf(), Box::new(hook)); + } + + pub(crate) fn grouped_batch_sizes(dir: &Path) -> Vec { + let grouped = GROUPED.lock().expect("grouped fsync dir recorder poisoned"); + let canonical = dir.canonicalize().ok(); + grouped + .iter() + .filter_map(|(recorded, batch_len)| { + (recorded == dir || canonical.as_ref().is_some_and(|canonical| recorded == canonical)).then_some(*batch_len) + }) + .collect() + } + + pub(crate) fn set_before_grouped(dir: &Path, hook: impl FnOnce() + Send + 'static) { + BEFORE_GROUPED + .lock() + .expect("grouped fsync hook poisoned") + .insert(dir.to_path_buf(), Box::new(hook)); + } + + pub(crate) fn set_grouped_failure(dir: &Path, kind: io::ErrorKind) { + GROUPED_FAILURES + .lock() + .expect("grouped fsync failure hook poisoned") + .insert(dir.to_path_buf(), kind); + } + + pub(crate) fn take_grouped_failure(dir: &Path) -> Option { + remove_path_keyed(&GROUPED_FAILURES, dir, "grouped fsync failure hook poisoned") + } } #[cfg(all(test, windows))] @@ -218,6 +323,374 @@ pub async fn fsync_dir(dir: impl AsRef) -> io::Result<()> { } } +const ENV_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: &str = "RUSTFS_EXPERIMENTAL_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE"; +const DEFAULT_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: bool = false; +#[cfg(not(test))] +const MAX_DST_DIR_FSYNC_GROUPS: usize = 1024; +#[cfg(test)] +const MAX_DST_DIR_FSYNC_GROUPS: usize = 4; +#[cfg(not(test))] +const MAX_DST_DIR_FSYNC_WAITERS: usize = 8192; +#[cfg(test)] +const MAX_DST_DIR_FSYNC_WAITERS: usize = 8; +static DST_DIR_FSYNC_GROUP_COMMIT_ENABLED: LazyLock = LazyLock::new(|| { + rustfs_utils::get_env_bool(ENV_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE, DEFAULT_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE) +}); + +#[cfg(test)] +mod dst_dir_fsync_group_commit_override { + use std::sync::{Mutex, MutexGuard, PoisonError, RwLock}; + + static OVERRIDE: RwLock> = RwLock::new(None); + static SERIAL: Mutex<()> = Mutex::new(()); + + pub(crate) fn get() -> Option { + *OVERRIDE.read().unwrap_or_else(PoisonError::into_inner) + } + + pub(crate) struct OverrideGuard { + _serial: MutexGuard<'static, ()>, + } + + impl Drop for OverrideGuard { + fn drop(&mut self) { + *OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = None; + } + } + + pub(crate) fn set(enabled: bool) -> OverrideGuard { + let serial = SERIAL.lock().unwrap_or_else(PoisonError::into_inner); + *OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = Some(enabled); + OverrideGuard { _serial: serial } + } +} + +#[cfg(test)] +pub(crate) fn set_dst_dir_fsync_group_commit_for_test(enabled: bool) -> dst_dir_fsync_group_commit_override::OverrideGuard { + dst_dir_fsync_group_commit_override::set(enabled) +} + +fn dst_dir_fsync_group_commit_enabled() -> bool { + #[cfg(test)] + if let Some(enabled) = dst_dir_fsync_group_commit_override::get() { + return enabled; + } + + *DST_DIR_FSYNC_GROUP_COMMIT_ENABLED +} + +#[derive(Clone, Eq, Hash, PartialEq)] +struct DstDirFsyncGroupKey { + canonical_path: PathBuf, + #[cfg(unix)] + dev: u64, + #[cfg(unix)] + ino: u64, +} + +impl DstDirFsyncGroupKey { + fn from_metadata(canonical_path: PathBuf, metadata: std::fs::Metadata) -> io::Result { + if !metadata.is_dir() { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "dst dir fsync group key must be a directory")); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + Ok(Self { + canonical_path, + dev: metadata.dev(), + ino: metadata.ino(), + }) + } + #[cfg(not(unix))] + { + Ok(Self { canonical_path }) + } + } +} + +struct OpenedDstDirFsyncGroup { + key: DstDirFsyncGroupKey, + #[cfg(any(test, not(unix)))] + dir: PathBuf, + #[cfg(unix)] + dir_file: Arc, +} + +impl OpenedDstDirFsyncGroup { + fn open(dir: &Path) -> io::Result { + let canonical_path = dir.canonicalize()?; + #[cfg(unix)] + { + let file = std::fs::File::open(&canonical_path)?; + let key = DstDirFsyncGroupKey::from_metadata(canonical_path, file.metadata()?)?; + #[cfg(test)] + let dir = key.canonical_path.clone(); + Ok(Self { + key, + #[cfg(test)] + dir, + dir_file: Arc::new(file), + }) + } + #[cfg(not(unix))] + { + let metadata = std::fs::metadata(&canonical_path)?; + let key = DstDirFsyncGroupKey::from_metadata(canonical_path, metadata)?; + let dir = key.canonical_path.clone(); + Ok(Self { key, dir }) + } + } +} + +struct DstDirFsyncWaiter { + result_tx: oneshot::Sender, +} + +#[derive(Clone)] +struct SharedDstDirFsyncError { + kind: io::ErrorKind, + message: Arc, +} + +impl SharedDstDirFsyncError { + fn from_error(err: io::Error) -> Self { + Self { + kind: err.kind(), + message: Arc::from(err.to_string()), + } + } + + fn into_error(self) -> io::Error { + io::Error::new(self.kind, self.message.to_string()) + } +} + +type SharedDstDirFsyncResult = std::result::Result<(), SharedDstDirFsyncError>; + +struct DstDirFsyncGroup { + key: DstDirFsyncGroupKey, + #[cfg(any(test, not(unix)))] + dir: PathBuf, + #[cfg(unix)] + dir_file: Arc, + inner: Mutex, +} + +#[derive(Default)] +struct DstDirFsyncGroupInner { + worker_running: bool, + pending: VecDeque, +} + +#[derive(Default)] +struct DstDirFsyncGroupCommit { + inner: Mutex, +} + +#[derive(Default)] +struct DstDirFsyncGroupCommitInner { + groups: HashMap>, + total_waiters: usize, +} + +static DST_DIR_FSYNC_GROUP_COMMIT: LazyLock = LazyLock::new(DstDirFsyncGroupCommit::default); + +impl DstDirFsyncGroupCommit { + // Lock order: registry first, then per-group state. No path may hold a + // group lock while acquiring the registry lock. + fn enqueue_opened( + &self, + opened: OpenedDstDirFsyncGroup, + ) -> io::Result<(oneshot::Receiver, Option>)> { + let (result_tx, result_rx) = oneshot::channel(); + let mut registry = self.inner.lock(); + if registry.total_waiters >= MAX_DST_DIR_FSYNC_WAITERS { + return Err(io::Error::new( + io::ErrorKind::WouldBlock, + "dst dir fsync group commit waiter limit reached", + )); + } + let group = if let Some(group) = registry.groups.get(&opened.key) { + group.clone() + } else { + if registry.groups.len() >= MAX_DST_DIR_FSYNC_GROUPS { + return Err(io::Error::new( + io::ErrorKind::WouldBlock, + "dst dir fsync group commit active group limit reached", + )); + } + let group = Arc::new(DstDirFsyncGroup { + key: opened.key.clone(), + #[cfg(any(test, not(unix)))] + dir: opened.dir, + #[cfg(unix)] + dir_file: opened.dir_file, + inner: Mutex::new(DstDirFsyncGroupInner::default()), + }); + registry.groups.insert(opened.key, group.clone()); + group + }; + let mut group_state = group.inner.lock(); + group_state.pending.push_back(DstDirFsyncWaiter { result_tx }); + let start_worker = !group_state.worker_running; + if start_worker { + group_state.worker_running = true; + } + registry.total_waiters += 1; + drop(group_state); + drop(registry); + #[cfg(test)] + fsync_dir_recorder::run_after_group_enqueue(&group.dir); + + Ok((result_rx, start_worker.then_some(group))) + } + + fn complete_batch(&self, count: usize) { + let mut registry = self.inner.lock(); + registry.total_waiters = registry.total_waiters.saturating_sub(count); + } + + fn remove_idle_group(&self, group: &Arc) { + let mut registry = self.inner.lock(); + let group_state = group.inner.lock(); + if !group_state.worker_running && group_state.pending.is_empty() { + registry.groups.remove(&group.key); + } + } + + #[cfg(test)] + fn counts_for_test(&self) -> (usize, usize) { + let registry = self.inner.lock(); + (registry.groups.len(), registry.total_waiters) + } + + #[cfg(test)] + fn clear_for_test(&self) { + let mut registry = self.inner.lock(); + registry.groups.clear(); + registry.total_waiters = 0; + } + + #[cfg(test)] + fn enqueue_for_test( + &self, + dir: &Path, + ) -> io::Result<(oneshot::Receiver, Option>)> { + self.enqueue_opened(OpenedDstDirFsyncGroup::open(dir)?) + } +} + +#[cfg(unix)] +async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> { + #[cfg(test)] + let dir = group.dir.clone(); + let dir_file = group.dir_file.clone(); + tokio::task::spawn_blocking(move || { + #[cfg(test)] + { + if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) { + return Err(io::Error::new(kind, "injected grouped dst dir fsync failure")); + } + fsync_dir_recorder::record(&dir); + } + dir_file.sync_all() + }) + .await + .map_err(|err| io::Error::other(format!("blocking dst dir group fsync failed: {err}")))? +} + +#[cfg(not(unix))] +async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> { + fsync_dir(&group.dir).await +} + +async fn run_dst_dir_fsync_group_worker(group: Arc) { + loop { + #[cfg(test)] + fsync_dir_recorder::run_before_group_batch(&group.dir); + tokio::task::yield_now().await; + let batch: Vec = { + let mut group_state = group.inner.lock(); + group_state.pending.drain(..).collect() + }; + if batch.is_empty() { + let mut group_state = group.inner.lock(); + group_state.worker_running = false; + drop(group_state); + DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group); + return; + } + + #[cfg(test)] + fsync_dir_recorder::record_grouped(&group.dir, batch.len()); + let result = fsync_open_dst_dir_group(&group) + .await + .map_err(SharedDstDirFsyncError::from_error); + let batch_len = batch.len(); + DST_DIR_FSYNC_GROUP_COMMIT.complete_batch(batch_len); + + let should_stop = { + let mut group_state = group.inner.lock(); + if group_state.pending.is_empty() { + group_state.worker_running = false; + true + } else { + false + } + }; + if should_stop { + DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group); + } + for waiter in batch { + let _ = waiter.result_tx.send(result.clone()); + } + if should_stop { + return; + } + } +} + +async fn fsync_dst_dir_group_commit_with_enabled(dir: impl AsRef, enabled: bool) -> io::Result<()> { + if !enabled { + return fsync_dir(dir).await; + } + + let dir = dir.as_ref().to_path_buf(); + let opened = tokio::task::spawn_blocking(move || OpenedDstDirFsyncGroup::open(&dir)) + .await + .map_err(|err| io::Error::other(format!("blocking dst dir group open failed: {err}")))??; + let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened)?; + if let Some(group) = worker { + tokio::spawn(run_dst_dir_fsync_group_worker(group)); + } + + match result_rx.await { + Ok(Ok(())) => Ok(()), + Ok(Err(err)) => Err(err.into_error()), + Err(_) => Err(io::Error::other("dst dir fsync group worker dropped the waiter")), + } +} + +pub(crate) async fn fsync_dst_dir_group_commit(dir: impl AsRef) -> io::Result<()> { + fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled()).await +} + +#[cfg(test)] +pub(crate) async fn fsync_dst_dir_group_commit_for_test(dir: impl AsRef, enabled: bool) -> io::Result<()> { + fsync_dst_dir_group_commit_with_enabled(dir, enabled).await +} + +#[cfg(test)] +pub(crate) fn dst_dir_fsync_group_commit_counts_for_test() -> (usize, usize) { + DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test() +} + +#[cfg(test)] +fn clear_dst_dir_fsync_group_commit_for_test() { + DST_DIR_FSYNC_GROUP_COMMIT.clear_for_test(); +} + // Small object directories are cheaper to flush in one blocking task. Multipart // directories fan out only once enough files can amortize per-task scheduling. const PARALLEL_FILE_SYNC_THRESHOLD: usize = 16; @@ -4750,6 +5223,288 @@ mod tests { fsync_dir(temp_dir.path()).await.expect("fsync dir must succeed"); } + async fn wait_for_dst_dir_fsync_group_commit_idle() { + for _ in 0..100 { + if dst_dir_fsync_group_commit_counts_for_test() == (0, 0) { + return; + } + tokio::task::yield_now().await; + } + assert_eq!( + dst_dir_fsync_group_commit_counts_for_test(), + (0, 0), + "dst dir fsync group registry must release idle groups and waiters" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(dst_dir_fsync_group_commit)] + async fn dst_dir_fsync_group_commit_default_off_uses_direct_fsync() { + let temp_dir = tempdir().expect("create temp dir"); + let dir = temp_dir.path().join("object"); + std::fs::create_dir(&dir).expect("create object dir"); + + fsync_dst_dir_group_commit_for_test(&dir, false) + .await + .expect("direct dst dir fsync should succeed"); + + assert!(fsync_dir_recorder::was_fsynced(&dir), "default-off path must still fsync the dst dir"); + assert!( + fsync_dir_recorder::grouped_batch_sizes(&dir).is_empty(), + "default-off path must not enter the group commit coordinator" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(dst_dir_fsync_group_commit)] + async fn dst_dir_fsync_group_commit_batches_same_directory_waiters() { + use std::sync::mpsc; + let temp_dir = tempdir().expect("create temp dir"); + let dir = temp_dir.path().join("object"); + std::fs::create_dir(&dir).expect("create object dir"); + let (batch_entered_tx, batch_entered_rx) = mpsc::channel(); + let (release_batch_tx, release_batch_rx) = mpsc::channel(); + fsync_dir_recorder::set_before_group_batch(&dir, move || { + batch_entered_tx.send(()).expect("signal first worker before freezing batch"); + release_batch_rx.recv().expect("wait until second waiter is queued"); + }); + + let first_dir = dir.clone(); + let first = tokio::spawn(async move { fsync_dst_dir_group_commit_for_test(first_dir, true).await }); + tokio::task::spawn_blocking(move || batch_entered_rx.recv_timeout(Duration::from_secs(30))) + .await + .expect("batch hook waiter should run") + .expect("first worker should reach the batch hook"); + + let (second_enqueued_tx, second_enqueued_rx) = mpsc::channel(); + fsync_dir_recorder::set_after_group_enqueue(&dir, move || { + second_enqueued_tx.send(()).expect("signal second waiter enqueue"); + }); + let second_dir = dir.clone(); + let second = tokio::spawn(async move { fsync_dst_dir_group_commit_for_test(second_dir, true).await }); + tokio::task::spawn_blocking(move || second_enqueued_rx.recv_timeout(Duration::from_secs(30))) + .await + .expect("enqueue hook waiter should run") + .expect("second waiter should be enqueued"); + assert_eq!( + dst_dir_fsync_group_commit_counts_for_test(), + (1, 2), + "second waiter must be queued before the first batch is released" + ); + release_batch_tx.send(()).expect("release first batch"); + + let (first_result, second_result) = tokio::time::timeout(Duration::from_secs(30), async { tokio::join!(first, second) }) + .await + .expect("same-directory fsync waiters should complete"); + first_result + .expect("first waiter task should not panic") + .expect("first waiter should observe successful fsync"); + second_result + .expect("second waiter task should not panic") + .expect("second waiter should observe successful fsync"); + + assert_eq!( + fsync_dir_recorder::grouped_batch_sizes(&dir), + vec![2], + "two waiters queued before the batch freezes must share exactly one dst dir fsync" + ); + wait_for_dst_dir_fsync_group_commit_idle().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(dst_dir_fsync_group_commit)] + async fn dst_dir_fsync_group_commit_late_join_waits_for_next_fsync() { + use std::sync::mpsc; + let temp_dir = tempdir().expect("create temp dir"); + let dir = temp_dir.path().join("object"); + std::fs::create_dir(&dir).expect("create object dir"); + let (fsync_entered_tx, fsync_entered_rx) = mpsc::channel(); + let (release_fsync_tx, release_fsync_rx) = mpsc::channel(); + fsync_dir_recorder::set_before_grouped(&dir, move || { + fsync_entered_tx.send(()).expect("signal first frozen batch"); + release_fsync_rx.recv().expect("wait until late waiter is queued"); + }); + + let first_dir = dir.clone(); + let first = tokio::spawn(async move { fsync_dst_dir_group_commit_for_test(first_dir, true).await }); + tokio::task::spawn_blocking(move || fsync_entered_rx.recv_timeout(Duration::from_secs(30))) + .await + .expect("grouped fsync hook waiter should run") + .expect("first batch should reach fsync"); + + let second_dir = dir.clone(); + let second = tokio::spawn(async move { fsync_dst_dir_group_commit_for_test(second_dir, true).await }); + release_fsync_tx.send(()).expect("release first fsync"); + + let (first_result, second_result) = tokio::time::timeout(Duration::from_secs(30), async { tokio::join!(first, second) }) + .await + .expect("late waiter should complete after a second fsync"); + first_result + .expect("first waiter task should not panic") + .expect("first waiter should observe successful fsync"); + second_result + .expect("second waiter task should not panic") + .expect("late waiter should observe successful fsync"); + + assert_eq!( + fsync_dir_recorder::grouped_batch_sizes(&dir), + vec![1, 1], + "a waiter queued after the first batch is frozen must not be covered by the earlier fsync" + ); + wait_for_dst_dir_fsync_group_commit_idle().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(dst_dir_fsync_group_commit)] + async fn dst_dir_fsync_group_commit_propagates_shared_fsync_failure() { + let temp_dir = tempdir().expect("create temp dir"); + let dir = temp_dir.path().join("object"); + std::fs::create_dir(&dir).expect("create object dir"); + fsync_dir_recorder::set_grouped_failure(&dir, io::ErrorKind::Other); + + let err = fsync_dst_dir_group_commit_for_test(&dir, true) + .await + .expect_err("shared dst dir fsync failure must be returned to the waiter"); + + assert_eq!(err.kind(), io::ErrorKind::Other); + wait_for_dst_dir_fsync_group_commit_idle().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(dst_dir_fsync_group_commit)] + async fn dst_dir_fsync_group_commit_cancellation_releases_waiter_state() { + use std::sync::mpsc; + let temp_dir = tempdir().expect("create temp dir"); + let dir = temp_dir.path().join("object"); + std::fs::create_dir(&dir).expect("create object dir"); + let (fsync_entered_tx, fsync_entered_rx) = mpsc::channel(); + let (release_fsync_tx, release_fsync_rx) = mpsc::channel(); + fsync_dir_recorder::set_before_grouped(&dir, move || { + fsync_entered_tx.send(()).expect("signal grouped fsync"); + release_fsync_rx.recv().expect("wait for cancellation"); + }); + + let cancelled_dir = dir.clone(); + let cancelled = tokio::spawn(async move { fsync_dst_dir_group_commit_for_test(cancelled_dir, true).await }); + tokio::task::spawn_blocking(move || fsync_entered_rx.recv_timeout(Duration::from_secs(30))) + .await + .expect("grouped fsync hook waiter should run") + .expect("first grouped fsync should start"); + cancelled.abort(); + assert!( + cancelled + .await + .expect_err("cancelled waiter task should abort") + .is_cancelled(), + "waiter cancellation must be observable" + ); + release_fsync_tx.send(()).expect("release grouped fsync"); + + fsync_dst_dir_group_commit_for_test(&dir, true) + .await + .expect("a later waiter should not be blocked by cancelled waiter state"); + wait_for_dst_dir_fsync_group_commit_idle().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(dst_dir_fsync_group_commit)] + async fn dst_dir_fsync_group_commit_recreated_directory_gets_new_group() { + use std::sync::mpsc; + let temp_dir = tempdir().expect("create temp dir"); + let dir = temp_dir.path().join("object"); + std::fs::create_dir(&dir).expect("create object dir"); + let (fsync_entered_tx, fsync_entered_rx) = mpsc::channel(); + let (release_fsync_tx, release_fsync_rx) = mpsc::channel(); + let dir_for_hook = dir.clone(); + fsync_dir_recorder::set_before_grouped(&dir, move || { + std::fs::remove_dir(&dir_for_hook).expect("remove old object dir"); + std::fs::create_dir(&dir_for_hook).expect("recreate object dir at the same path"); + fsync_entered_tx.send(()).expect("signal grouped fsync"); + release_fsync_rx.recv().expect("wait until recreated dir is enqueued"); + }); + + let first_dir = dir.clone(); + let first = tokio::spawn(async move { fsync_dst_dir_group_commit_for_test(first_dir, true).await }); + tokio::task::spawn_blocking(move || fsync_entered_rx.recv_timeout(Duration::from_secs(30))) + .await + .expect("grouped fsync hook waiter should run") + .expect("first grouped fsync should start"); + + let (_result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT + .enqueue_for_test(&dir) + .expect("recreated dir should enqueue separately"); + assert!(worker.is_some(), "same path with a new inode must not join the stale in-flight group"); + assert_eq!( + dst_dir_fsync_group_commit_counts_for_test().0, + 2, + "old and recreated directory identities must be tracked as separate active groups" + ); + + release_fsync_tx.send(()).expect("release grouped fsync"); + first + .await + .expect("first waiter task should not panic") + .expect("first stale directory fd should still fsync successfully"); + clear_dst_dir_fsync_group_commit_for_test(); + assert_eq!( + dst_dir_fsync_group_commit_counts_for_test(), + (0, 0), + "test registry cleanup must release the unstarted recreated-directory waiter" + ); + } + + #[test] + #[serial_test::serial(dst_dir_fsync_group_commit)] + fn dst_dir_fsync_group_commit_rejects_active_group_overflow() { + let temp_dir = tempdir().expect("create temp dir"); + let mut receivers = Vec::new(); + for index in 0..MAX_DST_DIR_FSYNC_GROUPS { + let dir = temp_dir.path().join(format!("object-{index}")); + std::fs::create_dir(&dir).expect("create object dir"); + let (result_rx, _worker) = DST_DIR_FSYNC_GROUP_COMMIT + .enqueue_for_test(&dir) + .expect("group below cap should enqueue"); + receivers.push(result_rx); + } + let overflow_dir = temp_dir.path().join("overflow"); + std::fs::create_dir(&overflow_dir).expect("create overflow dir"); + + let err = match DST_DIR_FSYNC_GROUP_COMMIT.enqueue_for_test(&overflow_dir) { + Ok(_) => panic!("active group max+1 must fail closed"), + Err(err) => err, + }; + + assert_eq!(err.kind(), io::ErrorKind::WouldBlock); + clear_dst_dir_fsync_group_commit_for_test(); + assert_eq!(dst_dir_fsync_group_commit_counts_for_test(), (0, 0)); + drop(receivers); + } + + #[test] + #[serial_test::serial(dst_dir_fsync_group_commit)] + fn dst_dir_fsync_group_commit_rejects_waiter_overflow() { + let temp_dir = tempdir().expect("create temp dir"); + let dir = temp_dir.path().join("object"); + std::fs::create_dir(&dir).expect("create object dir"); + let mut receivers = Vec::new(); + for _ in 0..MAX_DST_DIR_FSYNC_WAITERS { + let (result_rx, _worker) = DST_DIR_FSYNC_GROUP_COMMIT + .enqueue_for_test(&dir) + .expect("waiter below cap should enqueue"); + receivers.push(result_rx); + } + + let err = match DST_DIR_FSYNC_GROUP_COMMIT.enqueue_for_test(&dir) { + Ok(_) => panic!("waiter max+1 must fail closed"), + Err(err) => err, + }; + + assert_eq!(err.kind(), io::ErrorKind::WouldBlock); + clear_dst_dir_fsync_group_commit_for_test(); + assert_eq!(dst_dir_fsync_group_commit_counts_for_test(), (0, 0)); + drop(receivers); + } + #[tokio::test] async fn file_sync_admission_is_reused_across_commit_barriers() { let temp_dir = tempdir().expect("create temp dir"); From 40c15c769bc02265faecfc52b5fb47bd8a6a2885 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 07:09:00 +0800 Subject: [PATCH 51/51] docs(architecture): describe io-core by its surviving surface (#6227) The Code Map entry still summarised io-core as "buffer pool, storage profiling, admission control", which predates #6201 removing eight zero-consumer modules. The crate now exposes pool, io_profile, config, backpressure, deadlock_detector, lock_optimizer, and progress, so the summary names the policy, lock, and progress helpers as well. Refs backlog#1824 --- ARCHITECTURE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 4475c2997..64ae563f1 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -55,7 +55,7 @@ rustfs/ # Workspace root (virtual manifest) ├── crates/ # library crates (authoritative list: Cargo.toml [workspace].members) │ ├── ecstore/ # Erasure-coded storage engine │ ├── rio/ # Reader I/O pipeline (encrypt, compress, hash) -│ ├── io-core/ # Buffer pool, storage profiling, admission control +│ ├── io-core/ # Buffer pool, storage profiling, backpressure/deadlock policy, lock optimizer, operation progress │ ├── io-metrics/ # I/O metrics collection │ ├── common/ # Shared runtime state, globals, data usage types │ ├── config/ # Configuration types and parsing