From 072b57b37559fbe626aeda8abf61617fdcbdd926 Mon Sep 17 00:00:00 2001 From: Chris Date: Fri, 9 Oct 2026 17:01:48 -0700 Subject: [PATCH 001/143] fix: auto-repair breaking changes from 446479791 (#8451) * fix: repair s3s footprint ratchet violation, clippy warnings, and stack overflow - Replace s3_error! macro with S3Error::with_message in ecfs.rs to keep s3s footprint ratchet at baseline (1586) after retention validation refactor - Remove redundant clone on Copy type in list_objects.rs (clippy::redundant_clone) - Remove clone_on_copy calls in access.rs test (clippy::clone_on_copy) - Use manual thread with 8MB stack for heal_concurrent_delete_test to fix stack overflow on macOS (consistent with sibling tests in same file) * test: align CI assertions with version authorization rules --- crates/e2e_test/src/sts_query_compat_test.rs | 2 +- crates/ecstore/src/store/list_objects.rs | 2 +- .../heal/tests/heal_concurrent_delete_test.rs | 26 +++++++++++++++++-- .../src/app/lifecycle_transition_api_test.rs | 6 ++++- rustfs/src/storage/access.rs | 11 ++------ rustfs/src/storage/ecfs.rs | 7 ++++- 6 files changed, 39 insertions(+), 15 deletions(-) diff --git a/crates/e2e_test/src/sts_query_compat_test.rs b/crates/e2e_test/src/sts_query_compat_test.rs index 134b3bfc6..3a703be6d 100644 --- a/crates/e2e_test/src/sts_query_compat_test.rs +++ b/crates/e2e_test/src/sts_query_compat_test.rs @@ -740,7 +740,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult { .version_id(version) .send() .await?; - opa.expect_bucket_tags("s3:GetObject", source, Some("finance")).await?; + opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?; client.list_objects_v2().bucket(source).send().await?; opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?; client.list_object_versions().bucket(source).send().await?; diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 13721bd4e..ce7eb0006 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -9221,7 +9221,7 @@ mod test { }; let snapshot = PersistentListMetadataObject::from_object_info(&object); assert!(snapshot.to_object_info("bucket").etag.is_none()); - let mut old_snapshot = snapshot.clone(); + let mut old_snapshot = snapshot; old_snapshot.etag = object.etag.clone(); let old_contents = format!( "# rustfs-listobjects-key-only-v2\n{}\n", diff --git a/crates/heal/tests/heal_concurrent_delete_test.rs b/crates/heal/tests/heal_concurrent_delete_test.rs index 61657c4b7..2770295dc 100644 --- a/crates/heal/tests/heal_concurrent_delete_test.rs +++ b/crates/heal/tests/heal_concurrent_delete_test.rs @@ -563,9 +563,31 @@ fn admin_root_completes_after_unversioned_delete() { run_delete_case(DeleteCase::Unversioned); } -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[test] #[serial] -async fn admin_root_rejects_recreated_bucket_before_successor_repair() { +fn admin_root_rejects_recreated_bucket_before_successor_repair() { + // Incarnation-bound repairs use a spawned owner; retain the server's stack + // budget for the whole scenario. block_on alone would poll that future on + // libtest's smaller calling-thread stack. + let stack_size = 8 * 1024 * 1024; + std::thread::Builder::new() + .name("heal-concurrent-delete".to_owned()) + .stack_size(stack_size) + .spawn(move || { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .thread_stack_size(stack_size) + .enable_all() + .build() + .expect("concurrent delete runtime"); + runtime.block_on(recreated_bucket_repair()); + }) + .expect("concurrent delete scenario thread starts") + .join() + .expect("concurrent delete scenario must not panic"); +} + +async fn recreated_bucket_repair() { let directory = tempfile::tempdir().expect("recreated bucket fixture"); let env = TestECStoreEnv::builder().base_dir(directory.path()).build().await; env.make_bucket(BUCKET, false).await; diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index a35a12446..c546cee72 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -964,7 +964,11 @@ async fn copy_object_allows_new_version_for_locked_destination_but_blocks_explic .await .expect_err("explicit CopyObject overwrite of a locked version should be blocked"); - assert_eq!(err.code(), &s3s::S3ErrorCode::AccessDenied); + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument); + assert_eq!( + err.message(), + Some("Invalid argument: A destination versionId requires replication authorization") + ); assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), dst_object).await, source_payload); assert_eq!(live_object_version_count(&ecstore, bucket.as_str(), dst_object).await, 2); } diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index 38109dc38..316eee5fc 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -4343,15 +4343,8 @@ mod tests { (S3Action::DeleteObjectTaggingAction, S3Action::DeleteObjectVersionTaggingAction), ] { for version in [None, Some("null"), Some("0194e0f1-0000-7000-8000-000000000000")] { - let action = versioned_tagging_action(current.clone(), version); - assert_eq!( - action, - Action::S3Action(if version.is_some() { - versioned.clone() - } else { - current.clone() - }) - ); + let action = versioned_tagging_action(current, version); + assert_eq!(action, Action::S3Action(if version.is_some() { versioned } else { current })); let args = Args { account: "writer", groups: &groups, diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index f1c045f84..3ac66cacf 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -415,7 +415,12 @@ pub(crate) fn object_lock_retention_from_metadata(metadata: &HashMap Date: Sat, 10 Oct 2026 19:01:40 +0800 Subject: [PATCH 002/143] refactor(iam): construct OIDC runtime directly (#8436) * refactor(iam): remove legacy OIDC federation paths * refactor(iam): construct OIDC runtime directly --- Cargo.lock | 1 + crates/iam/Cargo.toml | 1 + crates/iam/src/lib.rs | 167 ++++++++++++++++++++++++++----------- rustfs/src/startup_auth.rs | 12 ++- 4 files changed, 126 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4fe3d692c..44433ab1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10045,6 +10045,7 @@ dependencies = [ "tokio", "tokio-util", "tracing", + "tracing-subscriber", "url", ] diff --git a/crates/iam/Cargo.toml b/crates/iam/Cargo.toml index 7fb4f6b9f..dcfe70ce1 100644 --- a/crates/iam/Cargo.toml +++ b/crates/iam/Cargo.toml @@ -125,6 +125,7 @@ rustls-pki-types.workspace = true serial_test = { workspace = true } temp-env = { workspace = true, features = ["async_closure"] } tempfile = { workspace = true } +tracing-subscriber = { workspace = true, features = ["fmt"] } [lib] doctest = false diff --git a/crates/iam/src/lib.rs b/crates/iam/src/lib.rs index baffd0a62..ab6f28bfd 100644 --- a/crates/iam/src/lib.rs +++ b/crates/iam/src/lib.rs @@ -194,7 +194,6 @@ pub(crate) async fn notify_iam_load_policy_mapping( } static IAM_SYS: OnceLock>> = OnceLock::new(); -static OIDC_SYS: OnceLock> = OnceLock::new(); /// Build an IAM system bound to the given store without touching the process /// singleton (backlog#1052 S3): a per-server context can own the returned @@ -284,36 +283,11 @@ pub fn get_global_iam_sys() -> Option>> { IAM_SYS.get().cloned() } -/// Initialize the global OIDC system. Non-fatal if no OIDC providers are configured. -pub async fn init_oidc_sys() -> Result<()> { - init_oidc_sys_with_extra_root_ca(None).await -} - -/// Initialize the global OIDC system with an additional outbound root CA bundle. -pub async fn init_oidc_sys_with_extra_root_ca(root_ca_pem: Option<&[u8]>) -> Result<()> { - init_oidc_sys_with_extra_root_ca_provider_inner(None, root_ca_pem).await -} - -/// Initialize the global OIDC system with a reload-aware outbound root CA provider. -pub async fn init_oidc_sys_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result<()> { - init_oidc_sys_with_extra_root_ca_provider_inner(Some(extra_root_ca_provider), None).await -} - -async fn init_oidc_sys_with_extra_root_ca_provider_inner( - extra_root_ca_provider: Option, - root_ca_pem: Option<&[u8]>, -) -> Result<()> { - if OIDC_SYS.get().is_some() { - debug!( - event = EVENT_OIDC_STATE, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - state = "already_initialized", - "OIDC runtime already initialized" - ); - return Ok(()); - } - +/// Build the OIDC system with reload-aware outbound root CA material. +/// +/// Invalid OIDC setup remains non-fatal: the returned system has no providers, +/// matching the server startup behavior when OIDC cannot be initialized. +pub async fn build_oidc_sys_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result> { debug!( event = EVENT_OIDC_STATE, component = LOG_COMPONENT_IAM, @@ -322,10 +296,10 @@ async fn init_oidc_sys_with_extra_root_ca_provider_inner( "OIDC runtime starting" ); - let oidc_sys_result = match extra_root_ca_provider { - Some(provider) => OidcSys::new_with_extra_root_ca_provider(provider).await, - None => OidcSys::new_with_extra_root_ca(root_ca_pem).await, - }; + finish_oidc_sys_build(OidcSys::new_with_extra_root_ca_provider(extra_root_ca_provider).await) +} + +fn finish_oidc_sys_build(oidc_sys_result: std::result::Result) -> Result> { let oidc_sys = match oidc_sys_result { Ok(sys) => { if sys.has_providers() { @@ -361,20 +335,117 @@ async fn init_oidc_sys_with_extra_root_ca_provider_inner( } }; - if OIDC_SYS.set(Arc::new(oidc_sys)).is_err() { - warn!( - event = EVENT_OIDC_STATE, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - state = "singleton_set_race", - "OIDC runtime singleton set raced" - ); + Ok(Arc::new(oidc_sys)) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + io::{self, Write}, + sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, + }, + }; + use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt}; + + #[derive(Clone, Default)] + struct CapturedLogs(Arc>>); + + struct CapturedLogWriter(Arc>>); + + impl Write for CapturedLogWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0 + .lock() + .expect("captured log lock should be available") + .extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } } - Ok(()) -} + impl<'writer> MakeWriter<'writer> for CapturedLogs { + type Writer = CapturedLogWriter; -/// Get the global OIDC system. -pub fn get_oidc() -> Option> { - OIDC_SYS.get().cloned() + fn make_writer(&'writer self) -> Self::Writer { + CapturedLogWriter(Arc::clone(&self.0)) + } + } + + fn capture_oidc_build(result: std::result::Result) -> (Arc, String) { + let logs = CapturedLogs::default(); + let captured = Arc::clone(&logs.0); + let subscriber = Registry::default().with( + tracing_subscriber::fmt::layer() + .without_time() + .with_target(false) + .with_level(true) + .with_ansi(false) + .with_writer(logs), + ); + + let oidc = tracing::subscriber::with_default(subscriber, || { + finish_oidc_sys_build(result).expect("OIDC build completion should succeed") + }); + let output = String::from_utf8(captured.lock().expect("captured log lock should be available").clone()) + .expect("captured logs should be UTF-8"); + (oidc, output) + } + + #[test] + fn oidc_build_reports_empty_configuration() { + let (oidc, output) = capture_oidc_build(OidcSys::empty()); + + assert!(!oidc.has_providers()); + assert!(output.contains(EVENT_OIDC_STATE), "{output}"); + assert!(output.contains("state=\"empty\""), "{output}"); + } + + #[test] + fn oidc_build_reports_ready_configuration() { + let oidc = crate::oidc::make_test_sys(vec![crate::oidc::test_config("default")]); + let (oidc, output) = capture_oidc_build(Ok(oidc)); + + assert!(oidc.has_providers()); + assert_eq!(oidc.config_snapshot().providers().len(), 1); + assert!(output.contains(EVENT_OIDC_STATE), "{output}"); + assert!(output.contains("state=\"ready\""), "{output}"); + assert!(output.contains("provider_count=1"), "{output}"); + } + + #[test] + fn oidc_build_reports_error_and_returns_empty_runtime() { + let (oidc, output) = capture_oidc_build(Err("configured OIDC root CA is invalid".to_string())); + + assert!(!oidc.has_providers()); + assert!(output.contains("WARN"), "{output}"); + assert!(output.contains(EVENT_OIDC_STATE), "{output}"); + assert!(output.contains("state=\"init_failed_non_fatal\""), "{output}"); + assert!(output.contains("configured OIDC root CA is invalid"), "{output}"); + } + + #[tokio::test] + async fn oidc_builder_loads_root_ca_provider_and_keeps_failure_non_fatal() { + let load_count = Arc::new(AtomicUsize::new(0)); + let observed_load_count = Arc::clone(&load_count); + let provider = OidcExtraRootCaProvider::new(move || { + let observed_load_count = Arc::clone(&observed_load_count); + async move { + observed_load_count.fetch_add(1, Ordering::SeqCst); + Err("test root CA load failure".to_string()) + } + }); + + let oidc = build_oidc_sys_with_extra_root_ca_provider(provider) + .await + .expect("root CA load failure should produce an empty OIDC runtime"); + + assert_eq!(load_count.load(Ordering::SeqCst), 1); + assert!(!oidc.has_providers()); + } } diff --git a/rustfs/src/startup_auth.rs b/rustfs/src/startup_auth.rs index b3b10e4f8..c47afb3b1 100644 --- a/rustfs/src/startup_auth.rs +++ b/rustfs/src/startup_auth.rs @@ -13,11 +13,11 @@ // limitations under the License. use rustfs_iam::{ + build_oidc_sys_with_extra_root_ca_provider, federation::{ CoreFederatedAuthorizationMapper, FederatedIdentityService, oidc::{OidcConfigQuery, StandardOidcAdapter}, }, - get_oidc, init_oidc_sys_with_extra_root_ca_provider, oidc::{OidcExtraRootCaMaterial, OidcExtraRootCaProvider, OidcSys}, }; use std::{ @@ -56,12 +56,10 @@ pub(crate) async fn init_auth_integrations() -> Result<()> { } } - match init_oidc_sys_with_extra_root_ca_provider(oidc_extra_root_ca_provider()).await { - Ok(()) => { - if let Some(oidc) = get_oidc() { - let (service, oidc_config_query) = standard_oidc_runtime(oidc); - crate::runtime_sources::publish_federated_identity_runtime(service, oidc_config_query); - } + match build_oidc_sys_with_extra_root_ca_provider(oidc_extra_root_ca_provider()).await { + Ok(oidc) => { + let (service, oidc_config_query) = standard_oidc_runtime(oidc); + crate::runtime_sources::publish_federated_identity_runtime(service, oidc_config_query); } Err(e) => { warn!( From b8889fccc9ef16d734fab58229d47867ab60b1ce Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 11:50:24 +0800 Subject: [PATCH 003/143] fix(ecstore): allow cold-start recovery at EC read quorum (#8458) * fix(ecstore): load existing formats at EC read quorum * fix(ecstore): complete startup at EC read quorum --- .config/e2e-nightly-selection.txt | 4 +- crates/e2e_test/src/common.rs | 9 + .../src/namespace_lock_quorum_test.rs | 284 +++++++++++++++++- crates/ecstore/src/api/mod.rs | 3 +- crates/ecstore/src/bucket/migration.rs | 24 ++ crates/ecstore/src/set_disk/ops/bucket.rs | 18 +- crates/ecstore/src/store/bucket.rs | 117 +++++++- crates/ecstore/src/store/init.rs | 33 +- crates/ecstore/src/store/init_format.rs | 233 ++++++++++++++ rustfs/src/server/readiness.rs | 27 +- rustfs/src/startup_bucket_metadata.rs | 21 +- rustfs/src/storage/storage_api.rs | 4 +- 12 files changed, 708 insertions(+), 69 deletions(-) diff --git a/.config/e2e-nightly-selection.txt b/.config/e2e-nightly-selection.txt index 07dfe2811..bf672a947 100644 --- a/.config/e2e-nightly-selection.txt +++ b/.config/e2e-nightly-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=efe68675f4676a0d2901242c5815b1f7af5cdec8231f7cccd408f6dacf5c75db -sha256-linux=003ed6480935a0a95680cc061be143722fbd37bd477b8ff98607b1a064f3aef1 +sha256-darwin=5dbb9154b8feb94cedc8fb4aa6895d4b3259cc31b7df1f5c3b9928ef07f04535 +sha256-linux=d3a210be9eccf593d3723ae669b66680e9751dd01a6949f03df20614f913b707 diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index 622fe9015..e573fb00f 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -1542,6 +1542,15 @@ impl RustFSTestClusterEnvironment { self.start_node_from_binary(node_idx, &binary_path).await } + /// Start a process and wait only for TCP, allowing tests to bring up a + /// quorum one node at a time before the S3 service can become ready. + pub async fn start_node_process(&mut self, node_idx: usize) -> Result<(), Box> { + let binary_path = rustfs_binary_path(); + let volumes_arg = self.build_volumes_arg(); + self.spawn_node(node_idx, &binary_path, &volumes_arg)?; + self.wait_for_node_ready(&self.nodes[node_idx].address, node_idx).await + } + /// Start one stopped cluster node with a specific RustFS binary while /// preserving the cluster's volume layout and that node's data directory. pub async fn start_node_from_binary( diff --git a/crates/e2e_test/src/namespace_lock_quorum_test.rs b/crates/e2e_test/src/namespace_lock_quorum_test.rs index 46a59a943..1edf47c2b 100644 --- a/crates/e2e_test/src/namespace_lock_quorum_test.rs +++ b/crates/e2e_test/src/namespace_lock_quorum_test.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::common::RustFSTestClusterEnvironment; +use crate::common::{ClusterTopology, RustFSTestClusterEnvironment}; use aws_sdk_s3::Client; use aws_sdk_s3::error::SdkError; use bytes::Bytes; @@ -185,7 +185,11 @@ async fn wait_for_quorum_read_admission(clients: &[Client], bucket: &str) -> Tes return Ok(()); } if tokio::time::Instant::now() >= deadline { - return Err(format!("read quorum did not become available after lease convergence for {bucket}").into()); + let diagnostics = clients[0].get_object().bucket(bucket).key("warm-small").send().await; + return Err(format!( + "read quorum did not become available after lease convergence for {bucket}: GET={diagnostics:?}" + ) + .into()); } tokio::time::sleep(std::time::Duration::from_millis(100)).await; } @@ -195,11 +199,14 @@ async fn wait_for_quorum_read_admission(clients: &[Client], bucket: &str) -> Tes async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult { crate::common::init_logging(); - for (node_count, parity) in [(4, 2), (6, 3), (6, 2)] { + for (node_count, parity) in [(3, 1), (4, 2), (6, 3), (6, 2), (4, 0)] { let read_quorum = node_count - parity; let write_quorum = read_quorum + usize::from(read_quorum == parity); let mut cluster = RustFSTestClusterEnvironment::new(node_count).await?; cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{parity}")); + if parity == 0 { + cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0"); + } // Wait for every seed fanout before removing any physical shard. cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false"); cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false"); @@ -338,6 +345,277 @@ async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResul Ok(()) } +#[tokio::test] +async fn test_cold_start_read_quorum_follows_erasure_layout() -> TestResult { + crate::common::init_logging(); + for (node_count, parity) in [(3, 1), (4, 2), (6, 3), (6, 2), (4, 0)] { + let read_quorum = node_count - parity; + let write_quorum = read_quorum + usize::from(read_quorum == parity); + let mut cluster = RustFSTestClusterEnvironment::new(node_count).await?; + cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{parity}")); + if parity == 0 { + cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0"); + } + cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false"); + cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false"); + cluster.set_env("RUST_LOG", "warn"); + cluster.start().await?; + let clients = cluster + .create_all_clients()? + .into_iter() + .map(|client| { + Client::from_conf( + client + .config() + .to_builder() + .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1)) + .build(), + ) + }) + .collect::>(); + let bucket = format!("cold-read-quorum-{node_count}-{parity}"); + clients[0].create_bucket().bucket(&bucket).send().await?; + let small = b"cold startup follows the stored EC layout".to_vec(); + let large = vec![73; 256 * 1024]; + for (key, body) in [ + ("warm-small", &small), + ("warm-large", &large), + ("cold-large", &large), + ("below-quorum", &large), + ] { + clients[0] + .put_object() + .bucket(&bucket) + .key(key) + .body(Bytes::copy_from_slice(body).into()) + .send() + .await?; + for node in &cluster.nodes { + let census = + crate::chaos::census_object_version_on_disk(std::path::Path::new(&node.data_dir), &bucket, key, None)?; + assert!(census.is_complete(), "seed shard must exist before cold startup: {census:?}"); + assert_eq!((census.data_blocks, census.parity_blocks), (Some(read_quorum), Some(parity))); + } + } + for node in 0..node_count { + cluster.stop_node(node)?; + } + // Rebuild every process's in-memory state below read quorum, then + // cross the EC boundary. A format bootstrap must not admit S3 reads + // merely because half the peers are online (e.g. six disks, EC:2). + for node in 0..read_quorum - 1 { + cluster.start_node_process(node).await?; + } + let error = clients[0] + .get_object() + .bucket(&bucket) + .key("below-quorum") + .send() + .await + .expect_err("cold startup below the configured EC read quorum must stay unavailable"); + assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503)); + cluster.start_node_process(read_quorum - 1).await?; + wait_for_quorum_read_admission(&clients[..read_quorum], &bucket).await?; + for client in clients.iter().take(read_quorum) { + assert_quorum_object_body(client, &bucket, "warm-small", &small).await?; + assert_quorum_object_body(client, &bucket, "cold-large", &large).await?; + let write = client + .put_object() + .bucket(&bucket) + .key("cold-quorum-write") + .body(Bytes::copy_from_slice(&small).into()) + .send() + .await; + if read_quorum >= write_quorum { + write?; + assert_quorum_object_body(client, &bucket, "cold-quorum-write", &small).await?; + } else { + let error = write.expect_err("cold startup at read quorum must preserve the EC write quorum"); + assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503)); + } + } + for node in read_quorum..node_count { + cluster.start_node(node).await?; + } + for client in &clients { + assert_quorum_object_body(client, &bucket, "cold-large", &large).await?; + } + } + Ok(()) +} + +#[tokio::test] +async fn test_four_by_six_cold_start_at_read_quorum() -> TestResult { + crate::common::init_logging(); + let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, 6)).await?; + // Explicit URLs are consumed in order. Place one drive from each node in + // every set, matching a four-host, six-drive-per-host deployment. + let volumes = (0..6) + .flat_map(|drive| { + cluster + .nodes + .iter() + .map(move |node| format!("http://{}{}", node.address, node.data_dirs[drive])) + }) + .collect::>() + .join(" "); + cluster.set_env("RUSTFS_VOLUMES", volumes); + cluster.set_env("RUSTFS_ERASURE_SET_DRIVE_COUNT", "4"); + cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2"); + cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false"); + cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false"); + cluster.set_env("RUST_LOG", "error"); + cluster.start().await?; + let clients = cluster + .create_all_clients()? + .into_iter() + .map(|client| { + Client::from_conf( + client + .config() + .to_builder() + .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1)) + .build(), + ) + }) + .collect::>(); + let bucket = "cold-start-read-quorum"; + clients[0].create_bucket().bucket(bucket).send().await?; + let mut fixtures = Vec::new(); + let mut covered = [[false; 2]; 6]; + for index in 0..128 { + let key = match index { + 0 => "warm-small".to_string(), + 1 => "warm-large".to_string(), + _ => format!("cold-{index}"), + }; + let size_class = index % 2; + let size = if size_class == 0 { 71 } else { 256 * 1024 }; + let body = (0..size) + .map(|offset| u8::try_from(offset % 251).expect("bounded fixture byte")) + .collect::>(); + clients[0] + .put_object() + .bucket(bucket) + .key(&key) + .body(Bytes::copy_from_slice(&body).into()) + .send() + .await?; + for node in &cluster.nodes { + let mut complete_shards = 0; + for (drive, path) in node.data_dirs.iter().enumerate() { + let census = crate::chaos::census_object_version_on_disk(std::path::Path::new(path), bucket, &key, None)?; + if census.has_xl_meta { + assert!(census.is_complete(), "seed shard must be complete before shutdown: {census:?}"); + assert_eq!((census.data_blocks, census.parity_blocks), (Some(2), Some(2))); + covered[drive][size_class] = true; + complete_shards += 1; + } + } + assert_eq!(complete_shards, 1, "each node must own one shard of every seed object"); + } + fixtures.push((key, body)); + if covered.iter().flatten().all(|covered| *covered) { + break; + } + } + assert!( + covered.iter().flatten().all(|covered| *covered), + "fixtures must cover all six sets with inline and large objects" + ); + + for first in 0..4 { + for second in first + 1..4 { + for node in 0..4 { + cluster.stop_node(node)?; + } + cluster.start_node_process(first).await?; + let error = clients[first] + .get_object() + .bucket(bucket) + .key("warm-large") + .send() + .await + .expect_err("one node cannot satisfy read quorum after a cold start"); + assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503)); + cluster.start_node_process(second).await?; + wait_for_quorum_read_admission(&[clients[first].clone(), clients[second].clone()], bucket).await?; + for node in [first, second] { + for (key, body) in &fixtures { + assert_quorum_object_body(&clients[node], bucket, key, body).await?; + } + let range = clients[node] + .get_object() + .bucket(bucket) + .key("warm-large") + .range("bytes=17-1023") + .send() + .await?; + assert_eq!(range.body.collect().await?.into_bytes().as_ref(), &fixtures[1].1[17..1024]); + let listing = clients[node].list_objects_v2().bucket(bucket).send().await?; + for (key, _) in &fixtures { + assert!( + listing.contents().iter().any(|object| object.key() == Some(key.as_str())), + "listing omitted {key}" + ); + } + let error = clients[node] + .put_object() + .bucket(bucket) + .key("recovery-write") + .body(Bytes::from_static(b"after recovery").into()) + .send() + .await + .expect_err("two nodes must not authorize a write"); + assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503)); + } + let error = clients[first] + .delete_object() + .bucket(bucket) + .key("warm-large") + .send() + .await + .expect_err("two nodes must not authorize a delete"); + assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503)); + assert_quorum_object_body(&clients[first], bucket, "warm-large", &fixtures[1].1).await?; + + let remaining = (0..4).filter(|node| *node != first && *node != second).collect::>(); + cluster.start_node(remaining[0]).await?; + let deadline = tokio::time::Instant::now() + Duration::from_secs(90); + loop { + match clients[first] + .put_object() + .bucket(bucket) + .key("recovery-write") + .body(Bytes::from_static(b"after recovery").into()) + .send() + .await + { + Ok(_) => break, + Err(error) + if error.raw_response().is_some_and(|response| response.status().as_u16() == 503) + && tokio::time::Instant::now() < deadline => + { + tokio::time::sleep(Duration::from_millis(200)).await + } + Err(error) => return Err(error.into()), + } + } + for node in [first, second, remaining[0]] { + assert_quorum_object_body(&clients[node], bucket, "recovery-write", b"after recovery").await?; + } + cluster.start_node(remaining[1]).await?; + for client in &clients { + for (key, body) in &fixtures { + assert_quorum_object_body(client, bucket, key, body).await?; + } + assert_quorum_object_body(client, bucket, "recovery-write", b"after recovery").await?; + } + } + } + Ok(()) +} + async fn put_object(client: Client, payload: Vec, writer_id: usize) -> Result<(), String> { client .put_object() diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 3a28b840f..e5e22995f 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -210,7 +210,8 @@ pub mod bucket { pub mod migration { pub use crate::bucket::migration::{ - LegacyBlobDecryptFn, migration_startup_error, try_migrate_bucket_metadata, try_migrate_iam_config, + LegacyBlobDecryptFn, migration_startup_error, try_migrate_bucket_metadata, try_migrate_bucket_metadata_for_buckets, + try_migrate_iam_config, }; } diff --git a/crates/ecstore/src/bucket/migration.rs b/crates/ecstore/src/bucket/migration.rs index 723b75c69..6785795c2 100644 --- a/crates/ecstore/src/bucket/migration.rs +++ b/crates/ecstore/src/bucket/migration.rs @@ -273,6 +273,30 @@ where let buckets: Vec = buckets_list.into_iter().map(|b| b.name).collect(); + try_migrate_bucket_metadata_for_buckets(store, &buckets).await +} + +/// Migrate the already discovered startup candidates without repeating a +/// majority-only namespace listing. Every migration write retains its quorum. +pub async fn try_migrate_bucket_metadata_for_buckets(store: Arc, buckets: &[String]) -> Result<()> +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + > + ObjectOperations< + Error = Error, + ObjectInfo = ObjectInfo, + ObjectOptions = ObjectOptions, + FileInfo = FileInfo, + ObjectToDelete = ObjectToDelete, + DeletedObject = DeletedObject, + >, +{ if buckets.is_empty() { debug!("No migrating bucket metadata found"); return Ok(()); diff --git a/crates/ecstore/src/set_disk/ops/bucket.rs b/crates/ecstore/src/set_disk/ops/bucket.rs index d1dc1770a..738648df7 100644 --- a/crates/ecstore/src/set_disk/ops/bucket.rs +++ b/crates/ecstore/src/set_disk/ops/bucket.rs @@ -123,8 +123,18 @@ impl SetDisks { } pub(crate) async fn list_bucket_for_scanner(&self, _opts: &BucketOptions) -> Result<(Vec, bool)> { + self.list_bucket_with_quorum(BucketInfoQuorum::Write).await + } + + pub(crate) async fn list_bucket_with_quorum(&self, quorum: BucketInfoQuorum) -> Result<(Vec, bool)> { let disks = self.disk_inventory().await; let write_quorum = (disks.len() / 2) + 1; + // Discovery uses the same namespace read boundary as stat_bucket. + // It grants no authority to mutate buckets or accept object reads. + let required = match quorum { + BucketInfoQuorum::Read => disks.len().div_ceil(2).max(1), + BucketInfoQuorum::Write => write_quorum, + }; let mut futures = Vec::with_capacity(disks.len()); for disk in disks { @@ -153,7 +163,11 @@ impl SetDisks { } } - if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) { + let error = match quorum { + BucketInfoQuorum::Read => reduce_read_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, required), + BucketInfoQuorum::Write => reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, required), + }; + if let Some(err) = error { return Err(err.into()); } @@ -179,7 +193,7 @@ impl SetDisks { topology_complete &= counts.values().all(|(count, _)| *count >= write_quorum); let mut buckets = counts .into_values() - .filter_map(|(count, bucket)| (count >= write_quorum).then_some(bucket)) + .filter_map(|(count, bucket)| (count >= required).then_some(bucket)) .collect::>(); buckets.sort_by(|left, right| left.name.cmp(&right.name)); Ok((buckets, topology_complete)) diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 327769638..57ccd3581 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -1231,34 +1231,46 @@ impl ECStore { #[instrument(skip(self))] pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result> { // TODO(backlog): support cached bucket listing via opts.cached - Ok(self.list_bucket_from_sets(opts, usize::MAX).await?.buckets) + Ok(self + .list_bucket_from_sets(opts, usize::MAX, BucketInfoQuorum::Write) + .await? + .buckets) } pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result { - self.list_bucket_from_sets(opts, SCANNER_BUCKET_LIST_SET_CONCURRENCY).await + self.list_bucket_from_sets(opts, SCANNER_BUCKET_LIST_SET_CONCURRENCY, BucketInfoQuorum::Write) + .await + } + + /// Discover candidates for metadata initialization at each set's namespace + /// read quorum. Metadata loading still validates persisted configuration; + /// this listing must not be used as a complete scanner or mutation view. + pub async fn list_buckets_for_metadata_startup(&self) -> Result> { + Ok(self + .list_bucket_from_sets( + &BucketOptions { + no_metadata: true, + ..Default::default() + }, + SCANNER_BUCKET_LIST_SET_CONCURRENCY, + BucketInfoQuorum::Read, + ) + .await? + .buckets) } async fn list_bucket_from_sets( &self, opts: &BucketOptions, max_concurrency: usize, + quorum: BucketInfoQuorum, ) -> Result { let set_count = self.pools.iter().map(|pool| pool.disk_set.len()).sum(); let concurrency = bucket_list_set_concurrency(set_count, max_concurrency); - let deleted = opts.deleted; - let cached = opts.cached; - let no_metadata = opts.no_metadata; - let mut set_listings = stream::iter(self.bucket_sets().map(move |(pool_index, set_index, set)| { - let opts = BucketOptions { - deleted, - cached, - no_metadata, - }; - async move { - set.list_bucket_for_scanner(&opts) - .await - .map(|(buckets, complete)| (pool_index, set_index, buckets, complete)) - } + let mut set_listings = stream::iter(self.bucket_sets().map(move |(pool_index, set_index, set)| async move { + set.list_bucket_with_quorum(quorum) + .await + .map(|(buckets, complete)| (pool_index, set_index, buckets, complete)) })) .buffer_unordered(concurrency); let mut topology_complete = set_count != 0; @@ -3543,6 +3555,79 @@ mod tests { assert_eq!(info_err, StorageError::ErasureReadQuorum); } + #[tokio::test] + #[serial] + async fn metadata_startup_listing_uses_each_sets_read_quorum() { + for drives_per_pool in [vec![2], vec![3], vec![4], vec![6], vec![8], vec![4, 6]] { + let (_temp_dir, store) = setup_bucket_quorum_test_env(&drives_per_pool, None).await; + let bucket = "metadata-startup-read-quorum"; + for pool in &store.pools { + pool.disk_set[0] + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("healthy set should create the startup candidate"); + } + let mut offline = Vec::new(); + for (pool_index, &width) in drives_per_pool.iter().enumerate() { + let set = &store.pools[pool_index].disk_set[0]; + let indexes = (width.div_ceil(2)..width).collect::>(); + offline.push(take_set_disks_offline(&store, set, &indexes).await); + } + let listing = store + .list_buckets_for_metadata_startup() + .await + .expect("startup must discover buckets at every set's read quorum"); + assert_eq!(listing.len(), 1); + assert_eq!(listing[0].name, bucket); + if drives_per_pool.iter().any(|width| width % 2 == 0) { + let error = store + .list_bucket_for_scanner(&BucketOptions::default()) + .await + .expect_err("startup discovery must not weaken scanner quorum"); + assert_eq!(error, StorageError::ErasureWriteQuorum); + let error = store + .list_bucket(&BucketOptions { + no_metadata: true, + ..Default::default() + }) + .await + .expect_err("ordinary listing must retain its existing quorum"); + assert_eq!(error, StorageError::ErasureWriteQuorum); + } + // A healthy earlier pool must not hide read-quorum loss in a later pool. + let last_pool = store.pools.len() - 1; + let set = &store.pools[last_pool].disk_set[0]; + let extra = take_set_disks_offline(&store, set, &[0]).await; + let error = store + .list_buckets_for_metadata_startup() + .await + .expect_err("every set must retain read quorum for startup discovery"); + assert_eq!(error, StorageError::ErasureReadQuorum); + restore_set_disks(&store, set, extra).await; + for (pool_index, disks) in offline.into_iter().enumerate() { + restore_set_disks(&store, &store.pools[pool_index].disk_set[0], disks).await; + } + } + } + + #[tokio::test] + #[serial] + async fn metadata_startup_listing_does_not_accept_single_disk_bucket_remnants() { + let (temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await; + let bucket = "incomplete-startup-bucket"; + tokio::fs::create_dir(temp_dir.path().join("pool0-disk0").join(bucket)) + .await + .expect("create one uncommitted directory remnant"); + let set = &store.pools[0].disk_set[0]; + let offline = take_set_disks_offline(&store, set, &[2, 3]).await; + let listing = store + .list_buckets_for_metadata_startup() + .await + .expect("two responsive disks satisfy discovery quorum"); + assert!(listing.iter().all(|info| info.name != bucket)); + restore_set_disks(&store, set, offline).await; + } + #[tokio::test] #[serial] async fn scanner_bucket_listing_unions_every_erasure_set() { diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 0a2a855bb..4b5580e71 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -108,6 +108,10 @@ fn should_retry_format_load(err: &Error) -> bool { ) } +fn format_load_retry_exhausted(err: &Error, retries: u32) -> bool { + retries >= 10 && !matches!(err, Error::ErasureReadQuorum | Error::FirstDiskWait | Error::NotFirstDisk) +} + fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_resume_required: bool) -> bool { rebalance_resume_required && !decommission_running } @@ -484,7 +488,7 @@ impl ECStore { check_disk_fatal_errs(&errs)?; let loaded_format = { - let mut times = 0; + let mut times = 0_u32; let mut interval = 1; loop { match init_format::connect_load_init_formats_with_instance_ctx( @@ -500,13 +504,13 @@ impl ECStore { Ok(fm) => break Ok(fm), Err(e) if !should_retry_format_load(&e) => break Err(e), // Wrap the final error if we are giving up - Err(e) if times >= 10 => { + Err(e) if format_load_retry_exhausted(&e, times) => { break Err(Error::other(format!("store init failed to load formats after {times} retries: {e}"))); } // Retrying so just drop the error Err(_) => {} } - times += 1; + times = times.saturating_add(1); if interval < 16 { interval *= 2; } @@ -519,6 +523,12 @@ impl ECStore { "Retrying storage format load" ); select! { + _ = ctx.cancelled() => { + return Err(Error::other(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "storage format initialization cancelled", + ))); + } _ = tokio::signal::ctrl_c() => { info!( event = EVENT_STORE_FORMAT_RETRY, @@ -2201,6 +2211,23 @@ mod tests { assert_eq!(attempts.load(Ordering::SeqCst), 2); } + #[test] + fn format_peer_wait_does_not_exhaust_the_retry_budget() { + for retries in [0, 9, 10, u32::MAX] { + for error in [ + StorageError::ErasureReadQuorum, + StorageError::FirstDiskWait, + StorageError::NotFirstDisk, + ] { + assert!( + !super::format_load_retry_exhausted(&error, retries), + "peer availability must remain retryable" + ); + } + assert_eq!(super::format_load_retry_exhausted(&StorageError::Timeout, retries), retries >= 10); + } + } + #[test] fn test_should_retry_format_load_rejects_permanent_corruption() { assert!(!should_retry_format_load(&StorageError::CorruptedFormat)); diff --git a/crates/ecstore/src/store/init_format.rs b/crates/ecstore/src/store/init_format.rs index 9bfb62c60..c5f786848 100644 --- a/crates/ecstore/src/store/init_format.rs +++ b/crates/ecstore/src/store/init_format.rs @@ -134,6 +134,18 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx( { return Err(Error::CorruptedFormat); } + if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum))) + && let Some((format, members)) = select_existing_format_read_quorum(&formats, &errs, set_drive_count) + { + retain_format_quorum_members(instance_ctx, disks, &format, &members, set_drive_count).await?; + // Loading an existing format grants no authority to initialize pool + // metadata or migrate formats. The persisted storage classes and each + // object's EC metadata still determine service and operation quorums. + return Ok(LoadedFormat { + format, + pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None, + }); + } let resumable_partial_migration = formats_present > 0 && errs.iter().any(|error| matches!(error, Some(DiskError::UnformattedDisk))) && format_quorum.as_ref().is_some_and(Result::is_err) @@ -578,6 +590,43 @@ pub fn get_format_erasure_in_quorum(formats: &[Option], slot_offset: u select_format_erasure_in_quorum(formats, slot_offset).map(|(format, _)| format) } +/// Load a consistent existing layout with enough members per set to read the +/// max-parity system metadata containing the persisted storage classes. This +/// is not object-read admission: STANDARD, RRS and historical objects can have +/// different EC layouts, which are validated after this bootstrap read. +/// Format migration and healing still use strict-majority election. +fn select_existing_format_read_quorum( + formats: &[Option], + errors: &[Option], + set_drive_count: usize, +) -> Option<(FormatV3, Vec)> { + if set_drive_count == 0 || formats.is_empty() || formats.len() != errors.len() { + return None; + } + let metadata_parity = set_drive_count / 2; + let metadata_read_quorum = set_drive_count - metadata_parity; + let reference = formats.iter().flatten().next()?; + if !formats_match_reference_slots(formats, reference, 0) + || check_format_erasure_value_for_topology(reference, formats.len(), set_drive_count).is_err() + || formats.iter().zip(errors).any(|(format, error)| match (format, error) { + (Some(_), None) => false, + (None, Some(DiskError::DiskNotFound)) => false, + (None, Some(error)) => !is_network_like_disk_error(error), + _ => true, + }) + || formats + .chunks(set_drive_count) + .any(|set| set.iter().flatten().count() < metadata_read_quorum) + { + return None; + } + + let mut format = reference.clone(); + format.erasure.this = Uuid::nil(); + format.disk_info = None; + Some((format, formats.iter().map(Option::is_some).collect())) +} + pub(crate) fn select_format_erasure_in_quorum(formats: &[Option], slot_offset: usize) -> Result<(FormatV3, Vec)> { let mut candidates = HashMap::new(); let formats_present = formats.iter().flatten().count(); @@ -1120,6 +1169,190 @@ mod tests { assert!(!formats_match_reference_slots(&formats, &reference, 0)); } + fn half_pool_formats() -> (Vec>, Vec>) { + let reference = FormatV3::new(6, 4); + let formats = (0..24) + .map(|slot| { + (slot % 4 < 2).then(|| { + let mut format = reference.clone(); + format.erasure.this = reference.erasure.sets[slot / 4][slot % 4]; + format + }) + }) + .collect::>(); + let errors = formats + .iter() + .map(|format| format.is_none().then_some(DiskError::DiskNotFound)) + .collect(); + (formats, errors) + } + + #[test] + fn existing_format_read_quorum_follows_each_sets_metadata_layout() { + // These are system-metadata EC boundaries, independent of user-object + // storage classes and of how many drives are hosted by each node. + for (set_width, read_quorum) in [(1, 1), (2, 1), (3, 2), (4, 2), (6, 3), (8, 4), (12, 6), (16, 8)] { + for set_count in [1, 2, 5] { + let reference = FormatV3::new(set_count, set_width); + let mut formats = (0..set_count * set_width) + .map(|slot| { + let set = slot / set_width; + let drive = slot % set_width; + // Rotate surviving slots between sets so admission + // cannot depend on the first drive or node being up. + ((drive + set) % set_width < read_quorum).then(|| { + let mut format = reference.clone(); + format.erasure.this = reference.erasure.sets[set][drive]; + format + }) + }) + .collect::>(); + let mut errors = formats + .iter() + .map(|format| format.is_none().then_some(DiskError::DiskNotFound)) + .collect::>(); + let (loaded, members) = select_existing_format_read_quorum(&formats, &errors, set_width) + .unwrap_or_else(|| panic!("metadata read quorum must load {set_count} sets of {set_width} drives")); + assert_eq!(loaded.shared_identity(), reference.shared_identity()); + assert!(loaded.erasure.this.is_nil()); + assert_eq!(members.iter().filter(|member| **member).count(), set_count * read_quorum); + + // More members in other sets cannot compensate for one set + // falling below its own metadata reconstruction boundary. + for slot in 0..(set_count - 1) * set_width { + let mut format = reference.clone(); + format.erasure.this = reference.erasure.sets[slot / set_width][slot % set_width]; + formats[slot] = Some(format); + errors[slot] = None; + } + let lost_slot = formats + .iter() + .rposition(Option::is_some) + .expect("at least one surviving drive"); + formats[lost_slot] = None; + errors[lost_slot] = Some(DiskError::DiskNotFound); + assert!( + select_existing_format_read_quorum(&formats, &errors, set_width).is_none(), + "one set below read quorum must block {set_count} sets of {set_width} drives" + ); + } + } + } + + #[test] + fn existing_format_read_quorum_preserves_election_and_set_boundaries() { + let (mut formats, mut errors) = half_pool_formats(); + assert!(select_existing_format_read_quorum(&[], &[], 0).is_none()); + assert!(select_existing_format_read_quorum(&formats, &errors[..23], 4).is_none()); + assert!(select_existing_format_read_quorum(&formats, &errors, 0).is_none()); + assert!(select_existing_format_read_quorum(&formats, &errors, 6).is_none()); + assert!(select_existing_format_read_quorum(&formats, &errors, 4).is_some()); + assert!(matches!(get_format_erasure_in_quorum(&formats, 0), Err(Error::ErasureReadQuorum))); + + let mut extra = formats[0].as_ref().expect("seed format").clone(); + extra.erasure.this = extra.erasure.sets[0][2]; + formats[2] = Some(extra); + errors[2] = None; + assert!(get_format_erasure_in_quorum(&formats, 0).is_ok(), "13 votes must establish a majority"); + formats[5] = None; + errors[5] = Some(DiskError::DiskNotFound); + assert!( + select_existing_format_read_quorum(&formats, &errors, 4).is_none(), + "12 votes cannot compensate for a set with only one readable drive" + ); + formats[2] = None; + errors[2] = Some(DiskError::DiskNotFound); + assert!(select_existing_format_read_quorum(&formats, &errors, 4).is_none(), "11 votes must wait"); + } + + #[test] + fn existing_format_read_quorum_rejects_conflicts_invalid_slots_and_blank_disks() { + for failure in [ + "conflict", + "wrong_slot", + "nil", + "duplicate", + "blank", + "corrupt", + "access_denied", + "tie", + ] { + let (mut formats, mut errors) = half_pool_formats(); + match failure { + "conflict" => formats[1].as_mut().expect("second format").id = Uuid::new_v4(), + "wrong_slot" => { + let first_id = formats[0].as_ref().expect("first format").erasure.this; + formats[1].as_mut().expect("second format").erasure.this = first_id; + } + "nil" => { + for format in formats.iter_mut().flatten() { + format.id = Uuid::nil(); + } + } + "duplicate" => { + for format in formats.iter_mut().flatten() { + format.erasure.sets[5][3] = format.erasure.sets[0][0]; + } + } + "blank" => errors[2] = Some(DiskError::UnformattedDisk), + "corrupt" => errors[2] = Some(DiskError::CorruptedFormat), + "access_denied" => errors[2] = Some(DiskError::FileAccessDenied), + "tie" => { + let other = FormatV3::new(6, 4); + for slot in 0..24 { + if formats[slot].is_none() { + let mut format = other.clone(); + format.erasure.this = other.erasure.sets[slot / 4][slot % 4]; + formats[slot] = Some(format); + errors[slot] = None; + } + } + } + _ => unreachable!(), + } + assert!( + select_existing_format_read_quorum(&formats, &errors, 4).is_none(), + "{failure} must not authorize degraded startup" + ); + } + } + + #[tokio::test] + async fn existing_format_load_accepts_each_half_of_four_by_six_without_format_writes() { + let (_root, all_disks) = local_disks_with_set_width(24, 4).await; + let reference = FormatV3::new(6, 4); + for (slot, disk) in all_disks.iter().enumerate() { + let mut format = reference.clone(); + format.erasure.this = reference.erasure.sets[slot / 4][slot % 4]; + save_format_file(disk, &Some(format)).await.expect("persist existing format"); + } + let before = format_bytes(&all_disks).await; + for first in 0..4 { + for second in first + 1..4 { + let mut disks = all_disks.clone(); + for (slot, disk) in disks.iter_mut().enumerate() { + if slot % 4 != first && slot % 4 != second { + *disk = None; + } + } + let loaded = connect_load_init_formats_with_instance_ctx( + &Arc::new(InstanceContext::new()), + first == 0, + &mut disks, + 6, + 4, + None, + ) + .await + .expect("two nodes must load the existing format"); + assert_eq!(loaded.format.shared_identity(), reference.shared_identity()); + assert!(matches!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None)); + assert_eq!(disks.iter().flatten().count(), 12); + assert_eq!(format_bytes(&all_disks).await, before, "read startup must never rewrite formats"); + } + } + } + #[tokio::test] async fn existing_format_load_succeeds_with_a_strict_majority() { let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await; diff --git a/rustfs/src/server/readiness.rs b/rustfs/src/server/readiness.rs index 712522e1a..5de55bb7e 100644 --- a/rustfs/src/server/readiness.rs +++ b/rustfs/src/server/readiness.rs @@ -237,7 +237,10 @@ pub async fn publish_ready_when_runtime_ready( wait_for_runtime_readiness_with( startup_runtime_readiness_max_wait(), STARTUP_RUNTIME_READINESS_POLL_INTERVAL, - collect_node_readiness, + // Publish service readiness at the same per-set read and shared-lock + // boundary reported by the node probe. Mutation paths independently + // require write quorum and a writable pool-metadata state. + || async { collect_node_readiness_report().await.readiness }, |dependency_readiness| { readiness.mark_stage(rustfs_common::SystemStage::FullReady); mark_get_metadata_read_version_coalescing_service_ready(); @@ -1060,20 +1063,6 @@ fn cluster_health_timeout_report() -> DependencyReadinessReport { } } -async fn collect_node_readiness() -> DependencyReadiness { - // Startup publication retains its local dependency gate. Runtime probes - // additionally report storage quorum without changing S3 admission. - let storage = node_pool_meta_write_readiness().await; - let readiness = DependencyReadiness { - storage_ready: storage.ready, - iam_ready: runtime_sources::current_iam_ready(), - lock_quorum_ready: collect_lock_quorum_status().await.ready, - peer_health_ready: collect_peer_health_readiness(), - }; - record_readiness_report(&dependency_readiness_report_from_write_status(readiness, storage)); - readiness -} - pub async fn collect_cluster_read_dependency_readiness_report() -> DependencyReadinessReport { let iam_ready_raw = runtime_sources::current_iam_ready(); let storage_ready = collect_storage_read_readiness_uncached().await; @@ -1116,14 +1105,6 @@ async fn collect_lock_quorum_observation() -> LockQuorumObservation { } } -async fn node_pool_meta_write_readiness() -> StorageWriteReadinessStatus { - if let Some(store) = runtime_sources::current_object_store_handle() { - return pool_metadata_write_readiness(store.pool_meta_write_status().await); - } - - StorageWriteReadinessStatus::default() -} - async fn collect_storage_write_readiness_uncached() -> StorageWriteReadinessStatus { if let Some(store) = runtime_sources::current_object_store_handle() { let metadata = pool_metadata_write_readiness(store.pool_meta_write_status().await); diff --git a/rustfs/src/startup_bucket_metadata.rs b/rustfs/src/startup_bucket_metadata.rs index ad77a0a79..eed955cff 100644 --- a/rustfs/src/startup_bucket_metadata.rs +++ b/rustfs/src/startup_bucket_metadata.rs @@ -15,7 +15,6 @@ use crate::app::object::OnDemandMigrationWriteBack; use crate::module_switches::{on_demand_migration_enabled_from_env, set_on_demand_migration_module_enabled}; use crate::on_demand_migration::OnDemandMigrationSys; -use crate::storage_api::startup::bucket_metadata::contract::bucket::{BucketOperations, BucketOptions}; use crate::storage_api::startup::bucket_metadata::{ ECStore, Error as StorageError, Result as StorageResult, get_global_replication_pool, init_bucket_metadata_sys, reconcile_bucket_resync_target_intents, try_migrate_bucket_metadata, try_migrate_iam_config, @@ -58,19 +57,13 @@ const STARTUP_BACKGROUND_STATUS_CANCELED: f64 = 3.0; pub(crate) async fn init_embedded_bucket_metadata_runtime(store: Arc, ctx: &CancellationToken) -> IoResult> { let buckets_list = retry_metadata_startup(ctx, "list_buckets", || async { - store - .list_bucket(&BucketOptions { - no_metadata: true, - ..Default::default() - }) - .await - .map_err(IoError::other) + store.list_buckets_for_metadata_startup().await.map_err(IoError::other) }) .await?; let buckets: Vec = buckets_list.into_iter().map(|v| v.name).collect(); - retry_metadata_startup(ctx, "bucket_migration", || try_migrate_bucket_metadata(store.clone())).await?; + retry_metadata_startup(ctx, "bucket_migration", || try_migrate_bucket_metadata(store.clone(), &buckets)).await?; init_on_demand_migration_runtime(); init_bucket_metadata_sys(store.clone(), buckets.clone()).await; retry_metadata_startup(ctx, "iam_migration", || try_migrate_iam_config(store.clone())).await?; @@ -81,19 +74,13 @@ pub(crate) async fn init_embedded_bucket_metadata_runtime(store: Arc, c pub(crate) async fn init_bucket_metadata_runtime(store: Arc, ctx: CancellationToken) -> IoResult> { let buckets_list = retry_metadata_startup(&ctx, "list_buckets", || async { - store - .list_bucket(&BucketOptions { - no_metadata: true, - ..Default::default() - }) - .await - .map_err(IoError::other) + store.list_buckets_for_metadata_startup().await.map_err(IoError::other) }) .await?; let buckets: Vec = buckets_list.into_iter().map(|v| v.name).collect(); - retry_metadata_startup(&ctx, "bucket_migration", || try_migrate_bucket_metadata(store.clone())).await?; + retry_metadata_startup(&ctx, "bucket_migration", || try_migrate_bucket_metadata(store.clone(), &buckets)).await?; retry_metadata_startup(&ctx, "iam_migration", || try_migrate_iam_config(store.clone())).await?; init_on_demand_migration_runtime(); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 142948502..adc83492a 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -1196,8 +1196,8 @@ pub(crate) fn get_global_transition_state() -> Arc { ecstore_bucket::lifecycle::bucket_lifecycle_ops::get_global_transition_state() } -pub(crate) async fn try_migrate_bucket_metadata(store: Arc) -> std::io::Result<()> { - ecstore_bucket::migration::try_migrate_bucket_metadata(store) +pub(crate) async fn try_migrate_bucket_metadata(store: Arc, buckets: &[String]) -> std::io::Result<()> { + ecstore_bucket::migration::try_migrate_bucket_metadata_for_buckets(store, buckets) .await .map_err(ecstore_bucket::migration::migration_startup_error) } From 9ecb00d38815681496ecf20fc6fe939800c36a9c Mon Sep 17 00:00:00 2001 From: Hauser Date: Sun, 11 Oct 2026 11:54:12 +0800 Subject: [PATCH 004/143] fix(scanner): avoid placeholder retention and resume checkpoints (#8460) * fix(scanner): bound old-cache lookups and recover checkpoint saves Keep missing child lookups read-only and preserve the checkpoint receiver after ordinary persistence errors. Reload on stale CAS or indeterminate revision refresh, and report confirmed periodic saves separately. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(scanner): remove redundant checkpoint error conversion Return the existing StorageError directly so strict Clippy passes. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * docs(scanner): clarify checkpoint persistence guarantees Distinguish a confirmed periodic save from validated restart coverage. Keep adoption and subtree-skip requirements explicit. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(scanner): preserve resumable periodic coverage Materialize active ancestor accounting in bounded periodic snapshots and seal completed coverage independently of raw enumeration state. Preserve durable coverage during early resumed enumeration and exclude unfinished out-of-order heal work from periodic and cancellation receipts. Reuse ancestor frame slots to avoid growing child-set copies on descent, retain compacted accumulator safety, and report checkpoint adoption resets. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(scanner): reject covered active ancestors in checkpoints Check every suspended ancestor alongside the current folder before certifying completed coverage. An unfinished covered ancestor must not become a resumable subtree merely because its descendant sorts later. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(scanner): retain an incomplete root for missing namespaces Create the empty bucket root explicitly at the missing-namespace boundary. Preserve nonempty durable caches and keep namespace results unpublished. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/scanner-metrics/src/metrics.rs | 86 +- crates/scanner/src/data_usage_define.rs | 42 +- crates/scanner/src/data_usage_define/tests.rs | 28 +- crates/scanner/src/remote_scanner/stream.rs | 10 +- crates/scanner/src/scanner_folder.rs | 149 +++- crates/scanner/src/scanner_folder/tests.rs | 176 +++- .../tests/checkpoint_fixture.rs | 1 + .../checkpoint_fixture/periodic_snapshot.rs | 794 ++++++++++++++++++ crates/scanner/src/scanner_io/cache.rs | 48 +- crates/scanner/src/scanner_io/io_cache.rs | 19 +- crates/scanner/src/scanner_io/tests.rs | 171 +++- docs/operations/scanner-runtime-controls.md | 14 + 12 files changed, 1475 insertions(+), 63 deletions(-) create mode 100644 crates/scanner/src/scanner_folder/tests/checkpoint_fixture/periodic_snapshot.rs diff --git a/crates/scanner-metrics/src/metrics.rs b/crates/scanner-metrics/src/metrics.rs index d91d08497..1f018d51e 100644 --- a/crates/scanner-metrics/src/metrics.rs +++ b/crates/scanner-metrics/src/metrics.rs @@ -18,7 +18,7 @@ use jiff::Timestamp; use rustfs_heal_contracts::heal_channel::HealScanMode; use serde::{Deserialize, Serialize}; use std::{ - collections::{BTreeSet, HashMap}, + collections::{BTreeMap, BTreeSet, HashMap}, fmt::Display, future::Future, pin::Pin, @@ -994,6 +994,8 @@ pub struct Metrics { scanner_bitrot_cycle_enabled: AtomicBool, scanner_bitrot_cycle_millis: AtomicU64, scanner_checkpoint: Mutex>, + scanner_checkpoints_persisted: Mutex>, + scanner_checkpoint_save_failures: AtomicU64, scanner_checkpoint_used: AtomicU64, scanner_checkpoint_cleared: AtomicU64, scanner_checkpoint_ignored: AtomicU64, @@ -1137,6 +1139,15 @@ pub struct ScannerCheckpointReport { pub last_event: String, } +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct ScannerPersistedCheckpointReport { + pub pool_index: usize, + pub set_index: usize, + pub bucket: String, + pub saved_unix_secs: u64, + pub checkpoint: Option, +} + #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct ScannerSourceWorkSnapshot { pub source: String, @@ -1499,6 +1510,10 @@ pub struct ScannerMetricsReport { #[serde(skip_serializing_if = "Option::is_none")] pub scan_checkpoint: Option, #[serde(default)] + pub scan_checkpoints_persisted: Vec, + #[serde(default)] + pub scan_checkpoint_save_failures: u64, + #[serde(default)] pub scan_checkpoint_used: u64, #[serde(default)] pub scan_checkpoint_cleared: u64, @@ -2046,6 +2061,8 @@ impl Metrics { scanner_bitrot_cycle_enabled: AtomicBool::new(false), scanner_bitrot_cycle_millis: AtomicU64::new(0), scanner_checkpoint: Mutex::new(None), + scanner_checkpoints_persisted: Mutex::new(BTreeMap::new()), + scanner_checkpoint_save_failures: AtomicU64::new(0), scanner_checkpoint_used: AtomicU64::new(0), scanner_checkpoint_cleared: AtomicU64::new(0), scanner_checkpoint_ignored: AtomicU64::new(0), @@ -2353,6 +2370,34 @@ impl Metrics { } } + /// Keep only the latest confirmed periodic save per set on this process. + pub fn record_scanner_checkpoint_saved( + &self, + pool_index: usize, + set_index: usize, + bucket: &str, + checkpoint: Option, + ) { + let mut saved = self + .scanner_checkpoints_persisted + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + saved.insert( + (pool_index, set_index), + ScannerPersistedCheckpointReport { + pool_index, + set_index, + bucket: bucket.to_string(), + saved_unix_secs: unix_now_secs(), + checkpoint, + }, + ); + } + + pub fn record_scanner_checkpoint_save_failed(&self) { + self.scanner_checkpoint_save_failures.fetch_add(1, Ordering::Relaxed); + } + pub fn record_scanner_checkpoint_used(&self) { self.scanner_checkpoint_used.fetch_add(1, Ordering::Relaxed); self.update_scanner_checkpoint_event(SCANNER_CHECKPOINT_EVENT_USED); @@ -3512,6 +3557,14 @@ impl Metrics { Ok(checkpoint) => checkpoint.clone(), Err(poisoned) => poisoned.into_inner().clone(), }; + m.scan_checkpoints_persisted = self + .scanner_checkpoints_persisted + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .values() + .cloned() + .collect(); + m.scan_checkpoint_save_failures = self.scanner_checkpoint_save_failures.load(Ordering::Relaxed); m.scan_checkpoint_used = self.scanner_checkpoint_used.load(Ordering::Relaxed); m.scan_checkpoint_cleared = self.scanner_checkpoint_cleared.load(Ordering::Relaxed); m.scan_checkpoint_ignored = self.scanner_checkpoint_ignored.load(Ordering::Relaxed); @@ -3975,6 +4028,37 @@ mod tests { assert_eq!(report.scan_checkpoint_cleared, 1); } + #[tokio::test] + async fn report_distinguishes_persisted_checkpoints_from_memory_and_failures() { + let metrics = Metrics::new(); + let saved = ScannerCheckpointReport { + version: 1, + resume_after: "bucket/first".to_string(), + reason: "objects".to_string(), + last_event: "saved".to_string(), + }; + metrics.record_scanner_checkpoint_saved(0, 1, "bucket", Some(saved.clone())); + metrics.record_scanner_checkpoint_set(1, "bucket/second", "objects"); + metrics.record_scanner_checkpoint_save_failed(); + let report = metrics.report().await; + assert_eq!(report.scan_checkpoint_save_failures, 1); + assert_eq!(report.scan_checkpoint.expect("in-memory cursor").resume_after, "bucket/second"); + assert_eq!(report.scan_checkpoints_persisted.len(), 1); + assert_eq!(report.scan_checkpoints_persisted[0].checkpoint, Some(saved)); + assert_eq!(report.scan_checkpoints_persisted[0].set_index, 1); + assert!(report.scan_checkpoints_persisted[0].saved_unix_secs > 0); + metrics.record_scanner_checkpoint_saved(0, 1, "other-bucket", None); + metrics.record_scanner_checkpoint_saved(0, 2, "bucket", None); + let report = metrics.report().await; + assert_eq!( + report.scan_checkpoints_persisted.len(), + 2, + "retain at most one save per set, not per bucket" + ); + assert_eq!(report.scan_checkpoints_persisted[0].bucket, "other-bucket"); + assert!(report.scan_checkpoints_persisted[0].checkpoint.is_none()); + } + #[tokio::test] async fn report_includes_scanner_source_work() { let metrics = Metrics::new(); diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index f711f11fc..90ac72464 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -234,7 +234,7 @@ pub static DATA_USAGE_BLOOM_RECOVERY_PATH: LazyLock = pub static BACKGROUND_HEAL_INFO_PATH: LazyLock = LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}.background-heal.json")); -const MAX_DATA_USAGE_CACHE_DEPTH: usize = 1024; +pub(crate) const MAX_DATA_USAGE_CACHE_DEPTH: usize = 1024; /// Scanner-side accounting on the shared [`SizeSummary`]. /// @@ -948,6 +948,42 @@ impl DataUsageCache { return self.prepare_for_scan(name, next_cycle, leader_epoch, source, scan_plan_digest, true); } if !reusable { + let had_resume_state = self.info.scan_checkpoint.is_some() + || self.info.scan_resume_after.is_some() + || self.info.scan_coverage_receipt.is_some() + || self.info.scan_raw_enumeration_cursor.is_some() + || self.info.scan_raw_enumeration_page_index.is_some(); + if had_resume_state { + let reason = if name == DATA_USAGE_ROOT { + "invalid_bucket_scope" + } else if !identity.is_valid() { + "invalid_identity" + } else if self.info.name != name { + "bucket_mismatch" + } else if self.info.source != Some(source) { + "source_mismatch" + } else if self.info.cache_key_format != DATA_USAGE_CACHE_KEY_FORMAT { + "key_format_mismatch" + } else if self.info.scan_identity != Some(identity) + || self.info.tier_registry_generation != Some(identity.tier_registry_generation) + { + "identity_mismatch" + } else if self.info.leader_epoch != leader_epoch && !cross_epoch_checkpoint { + "unvalidated_handoff" + } else { + "incomplete_cache_scope" + }; + warn!( + target: "rustfs::scanner::data_usage", + event = EVENT_SCANNER_CACHE_LOAD_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_CACHE, + state = "checkpoint_reset", + bucket = %name, + reason, + "Scanner bucket checkpoint cannot be adopted" + ); + } let keep_debts = self.info.name == name && self .info @@ -1221,8 +1257,8 @@ impl DataUsageCache { self.cache.get(&hash_path(path).key()) } - pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap { - self.cache.entry(h.string()).or_default().children.clone() + pub fn find_children_copy(&self, h: &DataUsageHash) -> DataUsageHashMap { + self.cache.get(&h.0).map(|entry| entry.children.clone()).unwrap_or_default() } pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry { diff --git a/crates/scanner/src/data_usage_define/tests.rs b/crates/scanner/src/data_usage_define/tests.rs index 047d92105..5060911d2 100644 --- a/crates/scanner/src/data_usage_define/tests.rs +++ b/crates/scanner/src/data_usage_define/tests.rs @@ -2491,12 +2491,32 @@ fn checked_flatten_accepts_depth_limit_and_rejects_deeper_tree() { } #[test] -fn test_find_children_copy_preserves_missing_entry_behavior() { - let mut cache = DataUsageCache::default(); +fn test_find_children_copy_does_not_insert_missing_entries() { + let cache = DataUsageCache::default(); let missing_hash = hash_path("missing"); - assert!(cache.find_children_copy(missing_hash.clone()).is_empty()); - assert!(cache.cache.contains_key(&missing_hash.key())); + assert!(cache.find_children_copy(&missing_hash).is_empty()); + assert!(cache.cache.is_empty()); + assert_eq!(cache.cache.capacity(), 0); +} + +#[test] +fn test_find_children_copy_preserves_existing_children() { + let hash = hash_path("bucket"); + let children = HashSet::from(["bucket/child".to_string()]); + let mut cache = DataUsageCache::default(); + cache.cache.insert( + hash.key(), + DataUsageEntry { + children: children.clone(), + ..Default::default() + }, + ); + let mut copied = cache.find_children_copy(&hash); + assert_eq!(copied, children); + copied.clear(); + assert_eq!(cache.cache[&hash.0].children, children); + assert_eq!(cache.cache.len(), 1); } #[test] diff --git a/crates/scanner/src/remote_scanner/stream.rs b/crates/scanner/src/remote_scanner/stream.rs index 9ffc3ea2b..3749dc5e4 100644 --- a/crates/scanner/src/remote_scanner/stream.rs +++ b/crates/scanner/src/remote_scanner/stream.rs @@ -853,7 +853,7 @@ async fn scan_and_persist_local_bucket( ) .await { - ScannerCheckpointPersistResult::Saved => { + ScannerCheckpointPersistResult::Saved | ScannerCheckpointPersistResult::Skipped => { if guard.is_lock_lost() { scan_ctx.cancel(); let _ = tokio::time::timeout(NS_SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan.as_mut()).await; @@ -869,8 +869,12 @@ async fn scan_and_persist_local_bucket( "remote namespace scanner cache fence changed during checkpoint save", )); } - ScannerCheckpointPersistResult::Failed(_error) => { - checkpoint_channel_closed = true; + ScannerCheckpointPersistResult::RetryBucket(error) => { + scan_ctx.cancel(); + let _ = tokio::time::timeout(NS_SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan.as_mut()).await; + return Err(RemoteScannerServerError::retry_bucket(format!( + "remote namespace scanner checkpoint needs reload: {error}", + ))); } } } diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index fdc7a7f41..7bee115a8 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -21,8 +21,8 @@ use std::time::{Duration, Instant, SystemTime}; use crate::ReplTargetSizeSummary; use crate::data_usage_define::{ DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageHash, DataUsageHashMap, - DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason, PendingScannerHeal, - PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry, SizeSummary, hash_path, + DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason, MAX_DATA_USAGE_CACHE_DEPTH, + PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry, SizeSummary, hash_path, }; use crate::error::ScannerError; use crate::raw_page_index::{RawEnumerationPageIndex, RawEnumerationPageWriter}; @@ -426,6 +426,10 @@ fn folder_resume_match(folder_name: &str, resume_after: &str) -> Option bool { + folder <= frontier && !matches!(folder_resume_match(folder, frontier), Some(FolderResumeMatch::Descendant)) +} + fn order_items_for_resume(items: &mut [T], resume_after: Option<&str>, name: F) -> FolderResumeOrder where F: Fn(&T) -> &str, @@ -704,6 +708,12 @@ impl ScannerBucketPrefixScanScope { } /// Folder scanner for scanning directory structures +#[derive(Default)] +struct ScannerCheckpointAncestor { + hash: DataUsageHash, + entry: DataUsageEntry, +} + pub struct FolderScanner { root: String, old_cache: DataUsageCache, @@ -728,6 +738,8 @@ pub struct FolderScanner { updates: Option>, checkpoint_tx: Option>, + checkpoint_ancestors: Vec, + checkpoint_depth: usize, last_update: SystemTime, checkpoint_objects: u64, last_checkpoint_objects: u64, @@ -1210,10 +1222,42 @@ impl FolderScanner { .map_or((None, None), |progress| (progress.cursor(), progress.page_index())) } - fn maybe_send_checkpoint(&mut self) { + async fn scan_child_folder( + &mut self, + ctx: CancellationToken, + folder: CachedFolder, + parent_hash: &mut DataUsageHash, + parent_entry: &mut DataUsageEntry, + child_entry: &mut DataUsageEntry, + ) -> Result<(), ScannerError> { + if self.checkpoint_tx.is_none() || self.old_cache.info.scan_progress.is_none() { + return Box::pin(self.scan_folder(ctx, folder, child_entry)).await; + } + let depth = self.checkpoint_depth; + if depth >= MAX_DATA_USAGE_CACHE_DEPTH { + return Err(ScannerError::Other("scanner checkpoint ancestor depth limit exceeded".to_string())); + } + if self.checkpoint_ancestors.len() == depth { + self.checkpoint_ancestors.push(ScannerCheckpointAncestor::default()); + } + // Reuse the empty frame slots: moving suspended accumulators avoids + // cloning a growing child set or allocating histogram vectors per child. + std::mem::swap(parent_hash, &mut self.checkpoint_ancestors[depth].hash); + std::mem::swap(parent_entry, &mut self.checkpoint_ancestors[depth].entry); + self.checkpoint_depth += 1; + let result = Box::pin(self.scan_folder(ctx, folder, child_entry)).await; + self.checkpoint_depth -= 1; + // Restore the parent before cancellation/error handling stitches partial work. + std::mem::swap(parent_hash, &mut self.checkpoint_ancestors[depth].hash); + std::mem::swap(parent_entry, &mut self.checkpoint_ancestors[depth].entry); + result + } + + fn maybe_send_checkpoint(&mut self, folder: &CachedFolder, hash: &DataUsageHash, into: &DataUsageEntry) { let Some(checkpoint_tx) = self.checkpoint_tx.clone() else { return }; let elapsed = self.last_checkpoint_at.elapsed(); - if self.new_cache.info.scan_progress.is_none() + if into.compacted + || self.new_cache.info.scan_progress.is_none() || elapsed < SCANNER_CHECKPOINT_MIN_INTERVAL || (self.checkpoint_objects.saturating_sub(self.last_checkpoint_objects) < SCANNER_CHECKPOINT_OBJECT_INTERVAL && elapsed < SCANNER_CHECKPOINT_INTERVAL) @@ -1233,34 +1277,51 @@ impl FolderScanner { } let Ok(permit) = checkpoint_tx.try_reserve() else { return }; + // Invalid partial graphs must not trigger a fresh O(N) clone on every object. + self.last_checkpoint_objects = self.checkpoint_objects; + self.last_checkpoint_at = Instant::now(); + // Heal revisits are not sorted with the forward sweep. An unfinished + // earlier subtree, including a suspended ancestor, must not be signed + // as completed prefix coverage. + if self.coverage_frontier.as_deref().is_some_and(|frontier| { + folder_is_covered_by_frontier(&folder.name, frontier) + || self.checkpoint_ancestors[..self.checkpoint_depth] + .iter() + .any(|ancestor| folder_is_covered_by_frontier(&ancestor.hash.0, frontier)) + }) { + return; + } let mut snapshot = self.new_cache.clone(); + let mut parent = None; + for ancestor in &self.checkpoint_ancestors[..self.checkpoint_depth] { + snapshot.replace_hashed(&ancestor.hash, &parent, &ancestor.entry); + parent = Some(ancestor.hash.clone()); + } + snapshot.replace_hashed(hash, &folder.parent, into); + if snapshot.checked_flatten_complete_scope(&snapshot.info.name).is_none() { + return; + } snapshot.info.last_update = Some(SystemTime::now()); snapshot.info.snapshot_complete = false; let (cursor, page_index) = self.peek_raw_enumeration_resume_state(); - if cursor.is_some() || page_index.is_some() { - snapshot.info.scan_raw_enumeration_cursor = cursor; - snapshot.info.scan_raw_enumeration_page_index = page_index; - snapshot.info.scan_resume_after = None; - snapshot.info.scan_checkpoint = None; - snapshot.info.scan_coverage_receipt = None; - } else if snapshot.seal_scan_frontier(self.coverage_frontier.as_deref()).is_err() { + snapshot.info.scan_raw_enumeration_cursor = cursor; + snapshot.info.scan_raw_enumeration_page_index = page_index; + if snapshot.seal_scan_frontier(self.coverage_frontier.as_deref()).is_err() { return; } - if snapshot.root().is_none() - && snapshot.info.scan_raw_enumeration_cursor.is_none() - && snapshot.info.scan_raw_enumeration_page_index.is_none() - { + // A resumed readdir can run before covered subtrees are copied. Keep + // its durable receipt until this snapshot can represent that coverage. + if self.coverage_frontier.is_some() && snapshot.info.scan_coverage_receipt.is_none() { return; } + // The receipt was just sealed from this snapshot; do not hash it twice. if snapshot.info.scan_raw_enumeration_cursor.is_none() && snapshot.info.scan_raw_enumeration_page_index.is_none() - && snapshot.validated_scan_frontier().is_none() + && snapshot.info.scan_coverage_receipt.is_none() { return; } permit.send(snapshot); - self.last_checkpoint_objects = self.checkpoint_objects; - self.last_checkpoint_at = Instant::now(); } fn carry_forward_old_children(&mut self, parent_hash: &DataUsageHash, entry: &mut DataUsageEntry) { @@ -1297,6 +1358,15 @@ impl FolderScanner { child_entry: &DataUsageEntry, ) { if data_usage_entry_has_progress(child_entry) { + // A cancelled out-of-order heal revisit is not completed prefix work. + if self + .coverage_frontier + .as_deref() + .is_some_and(|frontier| folder_is_covered_by_frontier(&child_hash.0, frontier)) + { + self.coverage_frontier = None; + self.coverage_gap = true; + } let mut child_entry = child_entry.clone(); self.carry_forward_old_children(child_hash, &mut child_entry); self.record_scan_resume_hint_if_not_ancestor(&child_hash.key()); @@ -1513,7 +1583,7 @@ impl FolderScanner { return Err(ScannerError::Other("Operation cancelled".to_string())); } - let this_hash = hash_path(&folder.name); + let mut this_hash = hash_path(&folder.name); // Store initial compaction state. let was_compacted = into.compacted; @@ -1526,7 +1596,7 @@ impl FolderScanner { let mut abandoned_children: DataUsageHashMap = HashSet::new(); if !into.compacted { - abandoned_children = self.old_cache.find_children_copy(this_hash.clone()); + abandoned_children = self.old_cache.find_children_copy(&this_hash); } debug!( @@ -1651,7 +1721,7 @@ impl FolderScanner { } } self.record_raw_enumeration_entry(&folder.name, &file_name); - self.maybe_send_checkpoint(); + self.maybe_send_checkpoint(&folder, &this_hash, into); let is_storage_format_entry = file_name == STORAGE_FORMAT_FILE; let file_path = entry.path().to_string_lossy().to_string(); @@ -1958,7 +2028,7 @@ impl FolderScanner { object_count += 1; self.budget.record_object_scanned(); self.checkpoint_objects = self.checkpoint_objects.saturating_add(1); - self.maybe_send_checkpoint(); + self.maybe_send_checkpoint(&folder, &this_hash, into); timer.sleep().await; @@ -2159,10 +2229,9 @@ impl FolderScanner { let h = hash_path(&folder_item.name); if forward_sweep && !into.compacted - && forward_resume_after.as_deref().is_some_and(|resume| { - folder_item.name.as_str() <= resume - && !matches!(folder_resume_match(&folder_item.name, resume), Some(FolderResumeMatch::Descendant)) - }) + && forward_resume_after + .as_deref() + .is_some_and(|resume| folder_is_covered_by_frontier(&folder_item.name, resume)) && self.old_cache.find(&folder_item.name).is_some() { self.new_cache.copy_with_children(&self.old_cache, &h, &folder_item.parent); @@ -2233,8 +2302,10 @@ impl FolderScanner { let mut dst = DataUsageEntry::default(); // Use Box::pin for recursive async call - let fut = Box::pin(self.scan_folder(ctx.clone(), folder_item.clone(), &mut dst)); - if let Err(e) = fut.await { + if let Err(e) = self + .scan_child_folder(ctx.clone(), folder_item.clone(), &mut this_hash, into, &mut dst) + .await + { if ctx.is_cancelled() { self.preserve_partial_child_progress(&folder_item.parent, &h, into, &dst) .await; @@ -2258,7 +2329,7 @@ impl FolderScanner { into.add_child(&h); self.record_completed_child(&folder_item.name, dst.failed_objects == 0); - self.maybe_send_checkpoint(); + self.maybe_send_checkpoint(&folder, &this_hash, into); // We scanned a folder, optionally send update. self.update_cache.delete_recursive(&h); self.update_cache.copy_with_children(&self.new_cache, &h, &folder_item.parent); @@ -2635,8 +2706,10 @@ impl FolderScanner { let h = hash_path(&folder_item.name); // Use Box::pin for recursive async call - let fut = Box::pin(self.scan_folder(ctx.clone(), folder_item.clone(), &mut dst)); - if let Err(e) = fut.await { + if let Err(e) = self + .scan_child_folder(ctx.clone(), folder_item.clone(), &mut this_hash, into, &mut dst) + .await + { if ctx.is_cancelled() { self.preserve_partial_child_progress(&folder_item.parent, &h, into, &dst) .await; @@ -2662,7 +2735,7 @@ impl FolderScanner { self.update_cache.delete_recursive(&h); self.update_cache.copy_with_children(&self.new_cache, &h, &folder_item.parent); self.send_update().await; - self.maybe_send_checkpoint(); + self.maybe_send_checkpoint(&folder, &this_hash, into); } } } @@ -2803,6 +2876,9 @@ pub(crate) async fn scan_data_folder_scoped( cache.info.tier_registry_generation = Some(tier_registry.generation); let resume_frontier = cache.validated_scan_frontier().map(str::to_owned); + if resume_frontier.is_some() { + global_metrics().record_scanner_checkpoint_used(); + } let bucket_name = cache.info.name.clone(); let next_cycle = cache.info.next_cycle; let had_scan_checkpoint = cache.info.scan_checkpoint.is_some(); @@ -2834,6 +2910,8 @@ pub(crate) async fn scan_data_folder_scoped( disks_quorum, updates, checkpoint_tx, + checkpoint_ancestors: Vec::new(), + checkpoint_depth: 0, last_update: SystemTime::UNIX_EPOCH, checkpoint_objects: 0, last_checkpoint_objects: 0, @@ -2988,6 +3066,13 @@ pub(crate) async fn scan_data_folder_scoped( } if matches!(&e, ScannerError::Io(io) if io.kind() == ErrorKind::NotFound) { let mut partial_cache = scanner.old_cache; + // An empty input still needs a root for the non-publishable + // partial result; retain nonempty durable caches unchanged. + if partial_cache.cache.is_empty() { + partial_cache + .cache + .insert(hash_path(&bucket_name).0, DataUsageEntry::default()); + } partial_cache.info.last_update = Some(SystemTime::now()); partial_cache.info.next_cycle = next_cycle; partial_cache.info.snapshot_complete = false; diff --git a/crates/scanner/src/scanner_folder/tests.rs b/crates/scanner/src/scanner_folder/tests.rs index bde002039..d6f6c9d4f 100644 --- a/crates/scanner/src/scanner_folder/tests.rs +++ b/crates/scanner/src/scanner_folder/tests.rs @@ -343,6 +343,8 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) { disks_quorum: 0, updates: None, checkpoint_tx: None, + checkpoint_ancestors: Vec::new(), + checkpoint_depth: 0, last_update: SystemTime::UNIX_EPOCH, checkpoint_objects: 0, last_checkpoint_objects: 0, @@ -409,14 +411,21 @@ async fn periodic_checkpoint_emits_at_object_threshold_without_wall_clock_wait() .seal_scan_frontier(Some("bucket/a")) .expect("fixture frontier"); assert_eq!(scanner.new_cache.validated_scan_frontier(), Some("bucket/a")); + let checkpoint_folder = CachedFolder { + name: "bucket".to_string(), + parent: None, + object_heal_prob_div: 1, + }; + let checkpoint_hash = hash_path("bucket"); + let checkpoint_entry = scanner.new_cache.root().expect("fixture root"); - scanner.maybe_send_checkpoint(); + scanner.maybe_send_checkpoint(&checkpoint_folder, &checkpoint_hash, &checkpoint_entry); scanner.checkpoint_objects = SCANNER_CHECKPOINT_OBJECT_INTERVAL * 2; scanner.last_checkpoint_at = Instant::now() .checked_sub(SCANNER_CHECKPOINT_MIN_INTERVAL) .expect("test instant subtraction"); - scanner.maybe_send_checkpoint(); + scanner.maybe_send_checkpoint(&checkpoint_folder, &checkpoint_hash, &checkpoint_entry); assert_eq!( scanner.last_checkpoint_objects, SCANNER_CHECKPOINT_OBJECT_INTERVAL, "a full checkpoint queue must reject before cloning or advancing progress" @@ -427,11 +436,73 @@ async fn periodic_checkpoint_emits_at_object_threshold_without_wall_clock_wait() assert!(!checkpoint.info.snapshot_complete); assert_eq!(scanner.last_checkpoint_objects, SCANNER_CHECKPOINT_OBJECT_INTERVAL); assert!(checkpoint_rx.try_recv().is_err(), "checkpoint queue remains bounded"); + scanner + .new_cache + .cache + .insert("bucket/orphan".to_string(), DataUsageEntry::default()); + scanner.maybe_send_checkpoint(&checkpoint_folder, &checkpoint_hash, &checkpoint_entry); + assert!( + checkpoint_rx.try_recv().is_err(), + "unreachable observations must not become resumable snapshots" + ); + assert_eq!( + scanner.last_checkpoint_objects, + SCANNER_CHECKPOINT_OBJECT_INTERVAL * 2, + "an invalid attempt must still consume the producer interval" + ); + scanner.new_cache.cache.remove("bucket/orphan"); + scanner.checkpoint_objects = SCANNER_CHECKPOINT_OBJECT_INTERVAL * 3; + scanner.maybe_send_checkpoint(&checkpoint_folder, &checkpoint_hash, &checkpoint_entry); + assert!(checkpoint_rx.try_recv().is_err(), "a repaired graph still waits for the minimum interval"); + tokio::fs::remove_dir_all(temp_dir) .await .expect("remove test scanner directory"); } +#[tokio::test] +async fn checkpoint_ancestor_depth_limit_preserves_suspended_accounting() { + let (mut scanner, temp_dir) = build_test_scanner().await; + let _guard = TestGuard { + temp_dir: Some(temp_dir), + }; + let (tx, _rx) = mpsc::channel(1); + scanner.checkpoint_tx = Some(tx); + scanner.old_cache.info.scan_progress = Some(crate::DataUsageScanProgress { + started_plan: crate::DataUsageScanPlanDigest([1; 32]), + requested_plan: crate::DataUsageScanPlanDigest([1; 32]), + }); + scanner.checkpoint_depth = MAX_DATA_USAGE_CACHE_DEPTH; + scanner.checkpoint_ancestors = (0..MAX_DATA_USAGE_CACHE_DEPTH) + .map(|_| ScannerCheckpointAncestor::default()) + .collect(); + let mut hash = hash_path("bucket/parent"); + let mut parent = DataUsageEntry { + objects: 3, + size: 7, + ..Default::default() + }; + let mut child = DataUsageEntry::default(); + let error = scanner + .scan_child_folder( + CancellationToken::new(), + CachedFolder { + name: "bucket/parent/child".to_string(), + parent: Some(hash.clone()), + object_heal_prob_div: 1, + }, + &mut hash, + &mut parent, + &mut child, + ) + .await + .expect_err("unsupported depth must stop before recursion"); + assert!(matches!(error, ScannerError::Other(message) if message.contains("depth limit"))); + assert_eq!(hash, hash_path("bucket/parent")); + assert_eq!((parent.objects, parent.size), (3, 7)); + assert_eq!(scanner.checkpoint_depth, MAX_DATA_USAGE_CACHE_DEPTH); +} + struct TestGuard { temp_dir: Option, } @@ -1658,6 +1729,51 @@ fn metadata_for_object_version(bucket: &str, object: &str, version_id: Option partial, + other => panic!("missing namespace must remain incomplete, got {other:?}"), + }; + assert!(!partial.info.snapshot_complete); + assert_eq!(partial.info.next_cycle, 9); + assert_eq!(partial.cache.len(), 1); + assert_eq!(partial.cache.keys().next().expect("retained orphan key").as_ptr(), retained_key_ptr); + assert!( + partial.root().is_none(), + "a disconnected durable cache must not gain an invented zero root" + ); + assert!(partial.checked_flatten_complete_scope("missing-bucket").is_none()); + assert_eq!((partial.cache[&orphan.0].objects, partial.cache[&orphan.0].size), (7, 70)); +} + #[tokio::test] #[serial] async fn test_scan_data_folder_resume_hint_orders_across_new_and_existing_folders() { diff --git a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs index 14fc3c425..bdda4458b 100644 --- a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs +++ b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs @@ -21,6 +21,7 @@ use std::io::Cursor; use tokio::io::AsyncReadExt; mod deep_compacted; +mod periodic_snapshot; mod segment_observation; const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin"; diff --git a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/periodic_snapshot.rs b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/periodic_snapshot.rs new file mode 100644 index 000000000..c838d9fc4 --- /dev/null +++ b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/periodic_snapshot.rs @@ -0,0 +1,794 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Periodic snapshots captured while nested traversal frames are still active. + +use super::*; +use tokio::sync::Notify; + +const OBJECTS: u64 = 8; + +fn prepare_scanner(scanner: &mut FolderScanner, cache: DataUsageCache) { + scanner.resume_frontier = cache.validated_scan_frontier().map(str::to_owned); + scanner.coverage_frontier = scanner.resume_frontier.clone(); + scanner.new_cache = DataUsageCache { + info: cache.info.clone(), + ..Default::default() + }; + scanner.update_cache = DataUsageCache { + info: cache.info.clone(), + ..Default::default() + }; + scanner.old_cache = cache; + scanner.is_erasure_mode = true; + scanner.skip_heal.store(true, Ordering::SeqCst); + scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest); +} + +async fn nested_periodic_snapshot( + corrupt_first: bool, + root_scalars: (usize, usize), +) -> (DataUsageCache, crate::DataUsageScanIdentity, FolderScanner, TestGuard) { + let (mut scanner, root) = build_test_scanner().await; + let guard = TestGuard { + temp_dir: Some(root.clone()), + }; + for branch in ["a", "b"] { + for object in 0..4 { + write_checkpoint_object(&root, &format!("prefix/{branch}/{object:04}"), &[(None, 1)]).await; + } + } + if corrupt_first { + write_test_object_metadata_bytes(&root, "bucket", "prefix/a/0000", b"not-valid-filemeta").await; + } + let (_, identity) = bound_checkpoint(); + let mut cache = DataUsageCache::default(); + cache.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity); + prepare_scanner(&mut scanner, cache); + let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel(1); + scanner.checkpoint_tx = Some(checkpoint_tx); + scanner.checkpoint_objects = SCANNER_CHECKPOINT_OBJECT_INTERVAL - 2; + scanner.last_checkpoint_at = Instant::now() + .checked_sub(SCANNER_CHECKPOINT_MIN_INTERVAL) + .expect("seed elapsed checkpoint interval"); + let paused = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let pause_path = if corrupt_first { + "bucket/prefix/a/0003" + } else { + "bucket/prefix/a/0002" + }; + scanner.update_current_path = Arc::new({ + let paused = paused.clone(); + let release = release.clone(); + move |path: &str| { + let pause_here = path == pause_path; + let paused = paused.clone(); + let release = release.clone(); + Box::pin(async move { + if pause_here { + paused.notify_one(); + release.notified().await; + } + }) + } + }); + let parent = CancellationToken::new(); + scanner.budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default()); + let budget = scanner.budget.clone(); + let mut entry = DataUsageEntry { + objects: root_scalars.0, + size: root_scalars.1, + ..Default::default() + }; + let snapshot = { + let scan = scanner.scan_folder( + parent.clone(), + CachedFolder { + name: "bucket".to_string(), + parent: None, + object_heal_prob_div: 1, + }, + &mut entry, + ); + tokio::pin!(scan); + let snapshot = tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + snapshot = checkpoint_rx.recv() => snapshot.expect("real walk emits periodic snapshot"), + result = &mut scan => panic!("walk finished before periodic snapshot: {result:?}"), + } + }) + .await + .expect("bounded nested checkpoint emission"); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + () = paused.notified() => {}, + result = &mut scan => panic!("nested ancestors unwound before capture: {result:?}"), + } + }) + .await + .expect("walk pauses before third healthy object"); + assert_eq!(budget.progress().0, 2, "capture must precede ancestor unwind and further object reads"); + assert!(!parent.is_cancelled(), "periodic capture must not use cancellation linking"); + snapshot + }; + assert!(!snapshot.info.snapshot_complete); + assert!( + snapshot.validated_raw_enumeration_cursor().is_some() || snapshot.validated_raw_enumeration_page_index().is_some(), + "enumeration progress must coexist with completed subtree coverage" + ); + let totals = snapshot + .checked_flatten_complete_scope("bucket") + .expect("periodic snapshot must connect every active ancestor to the bucket root"); + assert_eq!( + totals.objects, + 2 + root_scalars.0, + "snapshot must retain suspended ancestor and current frame object accounting" + ); + assert_eq!( + totals.size, + 2 + root_scalars.1, + "suspended ancestor and current frame scalar bytes must survive snapshot construction" + ); + let store = FixtureStore::new(); + let revisions = DataUsageCache::default() + .load_with_revisions(store.clone(), CACHE_NAME) + .await + .expect("initial periodic fixture revisions"); + snapshot + .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0) + .await + .expect("persist real mid-walk periodic snapshot"); + let mut loaded = DataUsageCache::default(); + loaded + .load(store.clone(), CACHE_NAME) + .await + .expect("reload real periodic snapshot"); + assert_eq!(loaded.info.scan_coverage_receipt, snapshot.info.scan_coverage_receipt); + assert_eq!(loaded.info.scan_raw_enumeration_cursor, snapshot.info.scan_raw_enumeration_cursor); + assert_eq!( + loaded + .checked_flatten_complete_scope("bucket") + .expect("reloaded complete scope") + .objects, + 2 + root_scalars.0 + ); + assert_eq!( + loaded.prepare_bucket_checkpoint("bucket", 11, 8, SOURCE, PLAN, identity), + crate::DataUsageCachePrepareOutcome::Reused, + "a higher leader must adopt the persisted periodic snapshot" + ); + (loaded, identity, scanner, guard) +} + +async fn finish_resumed_walk(cache: DataUsageCache, original: &FolderScanner) -> (DataUsageCache, u64, TestGuard) { + let (mut scanner, unused_root) = build_test_scanner().await; + let guard = TestGuard { + temp_dir: Some(unused_root), + }; + scanner.root = original.root.clone(); + scanner.local_disk = original.local_disk.clone(); + prepare_scanner(&mut scanner, cache); + let parent = CancellationToken::new(); + scanner.budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default()); + let mut entry = DataUsageEntry::default(); + scanner + .scan_folder( + parent, + CachedFolder { + name: "bucket".to_string(), + parent: None, + object_heal_prob_div: 1, + }, + &mut entry, + ) + .await + .expect("finish resumed nested periodic walk"); + let visits = scanner.budget.progress().0; + (scanner.new_cache, visits, guard) +} + +#[tokio::test] +#[serial] +async fn periodic_nested_snapshot_resumes_after_leader_handoff_with_exact_totals() { + let (loaded, _, original, _guard) = nested_periodic_snapshot(false, (0, 0)).await; + assert_eq!(loaded.validated_scan_frontier(), Some("bucket/prefix/a/0000")); + let (finished, visits, _unused_guard) = finish_resumed_walk(loaded, &original).await; + assert_eq!(visits, OBJECTS - 1, "the completed nested sibling must skip its metadata read"); + let totals = finished + .checked_flatten_complete_scope("bucket") + .expect("finished connected root"); + assert_eq!(u64::try_from(totals.objects).expect("object count"), OBJECTS); + assert_eq!(u64::try_from(totals.size).expect("byte count"), OBJECTS); + assert_eq!(totals.failed_objects, 0); +} + +#[tokio::test] +#[serial] +async fn periodic_nested_snapshot_keeps_failed_prefix_outside_resume_frontier() { + let (loaded, _, original, _guard) = nested_periodic_snapshot(true, (0, 0)).await; + assert!( + loaded.validated_scan_frontier().is_none(), + "an earlier corrupt object must block subtree skipping" + ); + assert!(!loaded.info.failed_objects.is_empty(), "persisted failure debt must survive handoff"); + let (finished, visits, _unused_guard) = finish_resumed_walk(loaded, &original).await; + assert_eq!(visits, OBJECTS - 1, "every healthy object must be re-read after the coverage gap"); + let totals = finished + .checked_flatten_complete_scope("bucket") + .expect("failed sweep still has connected accounting"); + assert_eq!(u64::try_from(totals.objects).expect("healthy count"), OBJECTS - 1); + assert_eq!(u64::try_from(totals.size).expect("healthy bytes"), OBJECTS - 1); + assert!(!finished.info.failed_objects.is_empty(), "unrepaired metadata must remain pending"); +} + +#[tokio::test] +#[serial] +async fn periodic_nested_snapshot_preserves_suspended_ancestor_scalars_through_reload() { + let (loaded, _, _, _guard) = nested_periodic_snapshot(false, (3, 21)).await; + let root = loaded.root().expect("reloaded root accumulator"); + assert_eq!(root.objects, 3, "suspended root object accounting must not become an empty topology node"); + assert_eq!(root.size, 21, "suspended root bytes must remain independent of descendant totals"); + let totals = loaded + .checked_flatten_complete_scope("bucket") + .expect("reloaded ancestor accounting"); + assert_eq!(totals.objects, 5); + assert_eq!(totals.size, 23); + assert_eq!(loaded.validated_scan_frontier(), Some("bucket/prefix/a/0000")); +} + +#[tokio::test] +#[serial] +async fn periodic_nested_snapshot_keeps_durable_frontier_across_early_repeated_handoffs() { + let (cache, identity, original, _guard) = nested_periodic_snapshot(false, (0, 0)).await; + let store = FixtureStore::new(); + let revisions = DataUsageCache::default() + .load_with_revisions(store.clone(), CACHE_NAME) + .await + .expect("initial repeated-handoff revisions"); + cache + .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0) + .await + .expect("save the inherited valid subtree frontier"); + let (mut scanner, unused_root) = build_test_scanner().await; + let _unused_guard = TestGuard { + temp_dir: Some(unused_root), + }; + scanner.root = original.root.clone(); + scanner.local_disk = original.local_disk.clone(); + prepare_scanner(&mut scanner, cache); + let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel(1); + scanner.checkpoint_tx = Some(checkpoint_tx); + scanner.checkpoint_objects = SCANNER_CHECKPOINT_OBJECT_INTERVAL; + scanner.last_checkpoint_at = Instant::now() + .checked_sub(SCANNER_CHECKPOINT_MIN_INTERVAL) + .expect("early resumed emitter cadence"); + let folder = CachedFolder { + name: "bucket".to_string(), + parent: None, + object_heal_prob_div: 1, + }; + scanner.record_raw_enumeration_entry("bucket", "prefix"); + scanner.maybe_send_checkpoint(&folder, &hash_path("bucket"), &DataUsageEntry::default()); + assert!( + matches!(checkpoint_rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)), + "an early raw-only snapshot must not overwrite inherited completed coverage" + ); + assert_eq!(scanner.last_checkpoint_objects, SCANNER_CHECKPOINT_OBJECT_INTERVAL); + let mut inherited = DataUsageCache::default(); + let revisions = inherited + .load_with_revisions(store.clone(), CACHE_NAME) + .await + .expect("reload unchanged durable frontier after deferred early emission"); + assert_eq!(inherited.validated_scan_frontier(), Some("bucket/prefix/a/0000")); + assert_eq!( + inherited.prepare_bucket_checkpoint("bucket", 11, 9, SOURCE, PLAN, identity), + crate::DataUsageCachePrepareOutcome::Reused + ); + assert_eq!(inherited.validated_scan_frontier(), Some("bucket/prefix/a/0000")); + prepare_scanner(&mut scanner, inherited); + scanner.checkpoint_objects = SCANNER_CHECKPOINT_OBJECT_INTERVAL - 2; + scanner.last_checkpoint_objects = 0; + scanner.last_checkpoint_at = Instant::now() + .checked_sub(SCANNER_CHECKPOINT_MIN_INTERVAL) + .expect("later useful resumed emitter cadence"); + let parent = CancellationToken::new(); + scanner.budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default()); + let mut entry = DataUsageEntry::default(); + let candidate = { + let scan = scanner.scan_folder(parent, folder, &mut entry); + tokio::pin!(scan); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + checkpoint = checkpoint_rx.recv() => checkpoint.expect("resumed walk can later emit useful coverage"), + result = &mut scan => { + result.expect("later resumed walk completes"); + checkpoint_rx.try_recv().expect("completed walk emitted a useful periodic checkpoint") + } + } + }) + .await + .expect("bounded later useful emission") + }; + assert_eq!(candidate.validated_scan_frontier(), Some("bucket/prefix/a/0001")); + assert_eq!( + candidate + .checked_flatten_complete_scope("bucket") + .expect("later snapshot connected accounting") + .objects, + 3, + "inherited completed object and both newly read objects must survive" + ); + candidate + .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0) + .await + .expect("persist later coverage without losing the inherited prefix"); + let mut latest = DataUsageCache::default(); + latest + .load(store, CACHE_NAME) + .await + .expect("reload later periodic checkpoint"); + assert_eq!( + latest.prepare_bucket_checkpoint("bucket", 11, 10, SOURCE, PLAN, identity), + crate::DataUsageCachePrepareOutcome::Reused + ); + assert_eq!(latest.validated_scan_frontier(), Some("bucket/prefix/a/0001")); +} + +#[tokio::test] +#[serial] +async fn periodic_nested_snapshot_defers_unfinished_paths_at_or_before_completed_frontier() { + let (mut scanner, root) = build_test_scanner().await; + let _guard = TestGuard { temp_dir: Some(root) }; + let (_, identity) = bound_checkpoint(); + let mut cache = DataUsageCache::default(); + cache.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity); + cache.replace("bucket", "", DataUsageEntry::default()); + cache.replace( + "bucket/z", + "bucket", + DataUsageEntry { + objects: 3, + size: 9, + ..Default::default() + }, + ); + cache + .seal_scan_frontier(Some("bucket/z")) + .expect("completed later sibling coverage"); + prepare_scanner(&mut scanner, cache.clone()); + scanner.new_cache = cache; + let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel(1); + scanner.checkpoint_tx = Some(checkpoint_tx); + let partial = DataUsageEntry { + objects: 2, + size: 5, + ..Default::default() + }; + for active in ["bucket/a", "bucket/z"] { + scanner.record_raw_enumeration_entry(active, STORAGE_FORMAT_FILE); + scanner.checkpoint_objects += SCANNER_CHECKPOINT_OBJECT_INTERVAL; + scanner.last_checkpoint_at = Instant::now() + .checked_sub(SCANNER_CHECKPOINT_MIN_INTERVAL) + .expect("unordered active path cadence"); + scanner.maybe_send_checkpoint( + &CachedFolder { + name: active.to_string(), + parent: Some(hash_path("bucket")), + object_heal_prob_div: 1, + }, + &hash_path(active), + &partial, + ); + assert!( + matches!(checkpoint_rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)), + "unfinished path {active} must not become certified completed coverage" + ); + assert_eq!( + scanner.last_checkpoint_objects, scanner.checkpoint_objects, + "deferred paths still consume cadence" + ); + } + scanner.new_cache.replace("bucket/a", "bucket", partial); + let root_entry = scanner.new_cache.root().expect("completed subtree owner"); + scanner.checkpoint_objects += SCANNER_CHECKPOINT_OBJECT_INTERVAL; + scanner.last_checkpoint_at = Instant::now() + .checked_sub(SCANNER_CHECKPOINT_MIN_INTERVAL) + .expect("completed owner cadence"); + scanner.maybe_send_checkpoint( + &CachedFolder { + name: "bucket".to_string(), + parent: None, + object_heal_prob_div: 1, + }, + &hash_path("bucket"), + &root_entry, + ); + let checkpoint = checkpoint_rx.try_recv().expect("completed subtree owner may safely emit"); + assert_eq!(checkpoint.validated_scan_frontier(), Some("bucket/z")); + let totals = checkpoint + .checked_flatten_complete_scope("bucket") + .expect("both completed sibling links remain reachable"); + assert_eq!(totals.objects, 5); + assert_eq!(totals.size, 14); + assert_eq!(checkpoint.cache.len(), 3); +} + +#[tokio::test] +#[serial] +async fn periodic_nested_snapshot_checks_suspended_ancestors_before_certifying_frontier() { + for (ancestor, frontier, may_emit) in [ + ("bucket/z", "bucket/z", false), + ("bucket/a", "bucket/a-", false), + ("bucket/a", "bucket/a/done", true), + ] { + let (mut scanner, root) = build_test_scanner().await; + let _guard = TestGuard { temp_dir: Some(root) }; + let (_, identity) = bound_checkpoint(); + let mut cache = DataUsageCache::default(); + cache.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity); + cache.replace("bucket", "", DataUsageEntry::default()); + cache.replace(ancestor, "bucket", DataUsageEntry::default()); + let completed = format!("{ancestor}/done"); + cache.replace( + &completed, + ancestor, + DataUsageEntry { + objects: 3, + size: 9, + ..Default::default() + }, + ); + let sibling_frontier = frontier != ancestor && frontier != completed; + if sibling_frontier { + cache.replace( + frontier, + "bucket", + DataUsageEntry { + objects: 3, + size: 9, + ..Default::default() + }, + ); + } + cache.seal_scan_frontier(Some(frontier)).expect("initial completed coverage"); + prepare_scanner(&mut scanner, cache); + scanner.new_cache.replace("bucket", "", DataUsageEntry::default()); + scanner + .new_cache + .copy_with_children(&scanner.old_cache, &hash_path(&completed), &Some(hash_path(ancestor))); + if sibling_frontier { + scanner + .new_cache + .copy_with_children(&scanner.old_cache, &hash_path(frontier), &Some(hash_path("bucket"))); + } + let root_entry = scanner.new_cache.root().expect("suspended root owner").clone(); + let mut ancestor_entry = scanner.new_cache.find(ancestor).expect("suspended subtree owner").clone(); + ancestor_entry.objects = 1; + ancestor_entry.size = 2; + scanner.checkpoint_ancestors = vec![ + ScannerCheckpointAncestor { + hash: hash_path("bucket"), + entry: root_entry, + }, + ScannerCheckpointAncestor { + hash: hash_path(ancestor), + entry: ancestor_entry, + }, + ]; + scanner.checkpoint_depth = 2; + let current = format!("{ancestor}/next"); + assert!( + current.as_str() > frontier, + "the current-folder-only guard must not already reject this fixture" + ); + scanner.record_raw_enumeration_entry(¤t, STORAGE_FORMAT_FILE); + let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel(1); + scanner.checkpoint_tx = Some(checkpoint_tx); + scanner.checkpoint_objects = SCANNER_CHECKPOINT_OBJECT_INTERVAL; + scanner.last_checkpoint_at = Instant::now() + .checked_sub(SCANNER_CHECKPOINT_MIN_INTERVAL) + .expect("suspended ancestor emitter cadence"); + scanner.maybe_send_checkpoint( + &CachedFolder { + name: current.clone(), + parent: Some(hash_path(ancestor)), + object_heal_prob_div: 1, + }, + &hash_path(¤t), + &DataUsageEntry { + objects: 2, + size: 5, + ..Default::default() + }, + ); + assert_eq!(scanner.last_checkpoint_objects, SCANNER_CHECKPOINT_OBJECT_INTERVAL); + if may_emit { + let snapshot = checkpoint_rx + .try_recv() + .expect("an ancestor containing the frontier may safely emit"); + assert_eq!(snapshot.validated_scan_frontier(), Some(frontier)); + let totals = snapshot + .checked_flatten_complete_scope("bucket") + .expect("all suspended and current accounting connected"); + assert_eq!(totals.objects, 6); + assert_eq!(totals.size, 16); + assert_eq!(snapshot.cache.len(), 4); + } else { + assert!( + matches!(checkpoint_rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)), + "unfinished ancestor {ancestor} must not be certified through {frontier}" + ); + } + } +} + +#[tokio::test] +#[serial] +async fn periodic_nested_snapshot_partial_preservation_invalidates_covered_child_but_keeps_inner_frontier() { + for (child, frontier, frontier_inside_child, expected_objects) in [ + ("bucket/a", "bucket/z", false, 5), + ("bucket/z", "bucket/z", false, 2), + ("bucket/parent", "bucket/parent/done", true, 5), + ] { + let (mut scanner, root) = build_test_scanner().await; + let _guard = TestGuard { temp_dir: Some(root) }; + let (_, identity) = bound_checkpoint(); + let mut cache = DataUsageCache::default(); + cache.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity); + cache.replace("bucket", "", DataUsageEntry::default()); + if frontier_inside_child { + cache.replace(child, "bucket", DataUsageEntry::default()); + } + cache.replace( + frontier, + if frontier_inside_child { child } else { "bucket" }, + DataUsageEntry { + objects: 3, + size: 9, + ..Default::default() + }, + ); + cache + .seal_scan_frontier(Some(frontier)) + .expect("initial completed child coverage"); + prepare_scanner(&mut scanner, cache.clone()); + scanner.new_cache = cache; + let mut parent_entry = scanner.new_cache.root().expect("partial child owner"); + let child_entry = DataUsageEntry { + objects: 2, + size: 5, + ..Default::default() + }; + scanner + .preserve_partial_child_progress(&Some(hash_path("bucket")), &hash_path(child), &mut parent_entry, &child_entry) + .await; + let retained_frontier = scanner.coverage_frontier.clone(); + scanner.new_cache.replace_hashed(&hash_path("bucket"), &None, &parent_entry); + scanner + .new_cache + .seal_scan_frontier(retained_frontier.as_deref()) + .expect("seal the final partial cache after child preservation"); + let totals = scanner + .new_cache + .checked_flatten_complete_scope("bucket") + .expect("preserved partial child and completed siblings remain connected"); + assert_eq!(totals.objects, expected_objects, "partial child {child} must be counted once"); + assert_eq!(totals.size, if expected_objects == 2 { 5 } else { 14 }); + if frontier_inside_child { + assert!(!scanner.coverage_gap); + assert_eq!(scanner.new_cache.validated_scan_frontier(), Some(frontier)); + } else { + assert!( + scanner.coverage_gap, + "unfinished covered child {child} must block further frontier advancement" + ); + assert!(retained_frontier.is_none()); + assert!(scanner.new_cache.info.scan_coverage_receipt.is_none()); + assert!(scanner.new_cache.validated_scan_frontier().is_none()); + } + } +} + +async fn assert_nested_frame_restoration(max_objects: Option) { + let (mut scanner, root) = build_test_scanner().await; + let _guard = TestGuard { + temp_dir: Some(root.clone()), + }; + for branch in ["a", "b"] { + for object in 0..4 { + write_checkpoint_object(&root, &format!("prefix/{branch}/{object:04}"), &[(None, 1)]).await; + } + } + let (_, identity) = bound_checkpoint(); + let mut cache = DataUsageCache::default(); + cache.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity); + prepare_scanner(&mut scanner, cache); + let (checkpoint_tx, _checkpoint_rx) = mpsc::channel(1); + scanner.checkpoint_tx = Some(checkpoint_tx); + let parent = CancellationToken::new(); + scanner.budget = ScannerCycleBudget::new_with_progress_tracking( + &parent, + ScannerCycleBudgetConfig { + max_objects, + ..Default::default() + }, + ); + let mut entry = DataUsageEntry { + objects: usize::from(max_objects.is_some()) * 3, + size: usize::from(max_objects.is_some()) * 21, + ..Default::default() + }; + let result = scanner + .scan_folder( + scanner.budget.token(), + CachedFolder { + name: "bucket".to_string(), + parent: None, + object_heal_prob_div: 1, + }, + &mut entry, + ) + .await; + match max_objects { + Some(_) => { + assert!(result.is_err(), "object budget must interrupt the actual nested walk"); + assert_eq!(scanner.budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects)); + assert!( + scanner.budget.token().is_cancelled(), + "partial preservation must follow actual cancellation" + ); + assert_eq!( + entry.objects, 3, + "suspended root object accounting must be restored before returning the error" + ); + assert_eq!( + entry.size, 21, + "suspended root byte accounting must be restored before cancellation linking" + ); + } + None => result.expect("complete real nested walk with periodic sender enabled"), + } + let totals = scanner + .new_cache + .checked_flatten_complete_scope("bucket") + .expect("complete nested accounting"); + let expected = max_objects.unwrap_or(OBJECTS); + assert_eq!(u64::try_from(totals.objects).expect("object count"), expected); + assert_eq!(u64::try_from(totals.size).expect("byte count"), expected); + assert_eq!(scanner.checkpoint_depth, 0, "every completed child must restore its parent frame"); + assert_eq!( + scanner.checkpoint_ancestors.len(), + 3, + "sibling count must not increase depth-bounded spare storage" + ); + for spare in &scanner.checkpoint_ancestors { + assert!(spare.hash.0.is_empty(), "reusable frame must release its suspended path"); + assert!( + !data_usage_root_has_progress(&spare.entry), + "completed frame must not retain ancestor accounting" + ); + assert!(!spare.entry.compacted); + assert!(spare.entry.obj_sizes.is_empty()); + assert!(spare.entry.obj_versions.is_empty()); + assert!(spare.entry.tier_accounting_proof.is_none()); + } +} + +#[tokio::test] +#[serial] +async fn periodic_nested_snapshot_reuses_empty_ancestor_frames_after_complete_walk() { + assert_nested_frame_restoration(None).await; +} + +#[tokio::test] +#[serial] +async fn periodic_nested_snapshot_restores_ancestor_frames_before_cancellation_linking() { + assert_nested_frame_restoration(Some(2)).await; +} + +#[tokio::test] +#[serial] +async fn periodic_nested_snapshot_adoption_records_checkpoint_used_on_real_scan_startup() { + let (scanner, root) = build_test_scanner().await; + let _guard = TestGuard { + temp_dir: Some(root.clone()), + }; + tokio::fs::create_dir_all(root.join("bucket/static")) + .await + .expect("create the real cached subtree directory"); + let (mut cache, _) = bound_checkpoint(); + cache.info.skip_healing = true; + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_progress_tracking( + &parent, + ScannerCycleBudgetConfig { + max_directories: Some(1), + ..Default::default() + }, + ); + let before = global_metrics().report().await.scan_checkpoint_used; + let completed = scan_data_folder( + budget.token(), + budget.clone(), + vec![scanner.local_disk.clone()], + scanner.local_disk, + cache, + None, + HealScanMode::Normal, + DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest), + ) + .await + .expect("validated frontier must skip the child within a root-only directory budget"); + let after = global_metrics().report().await.scan_checkpoint_used; + assert_eq!(after, before + 1, "adopting a verified forward-sweep frontier must be observable"); + assert!(completed.info.snapshot_complete); + assert_eq!( + completed + .checked_flatten_complete_scope("bucket") + .expect("completed adopted scope") + .objects, + 3 + ); + assert_eq!(budget.progress().1, 1, "adoption must avoid entering the completed subtree"); + assert_eq!(budget.reason(), None); +} + +#[tokio::test] +#[serial] +async fn periodic_nested_snapshot_does_not_publish_shared_compacted_accumulator_under_child_path() { + let (mut scanner, root) = build_test_scanner().await; + let _guard = TestGuard { + temp_dir: Some(root.clone()), + }; + for object in 0..4 { + write_checkpoint_object(&root, &format!("prefix/nested/{object:04}"), &[(None, 1)]).await; + } + let (_, identity) = bound_checkpoint(); + let mut cache = DataUsageCache::default(); + cache.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity); + prepare_scanner(&mut scanner, cache); + let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel(1); + scanner.checkpoint_tx = Some(checkpoint_tx); + scanner.checkpoint_objects = SCANNER_CHECKPOINT_OBJECT_INTERVAL - 2; + scanner.last_checkpoint_at = Instant::now() + .checked_sub(SCANNER_CHECKPOINT_MIN_INTERVAL) + .expect("seed compacted checkpoint interval"); + let mut entry = DataUsageEntry { + compacted: true, + ..Default::default() + }; + scanner + .scan_folder( + CancellationToken::new(), + CachedFolder { + name: "bucket".to_string(), + parent: None, + object_heal_prob_div: 1, + }, + &mut entry, + ) + .await + .expect("walk with shared compacted owner accumulator"); + assert_eq!(entry.objects, 4); + assert_eq!(entry.size, 4); + assert!( + entry.children.is_empty(), + "compacted totals must not acquire independently accounted child links" + ); + assert!( + matches!(checkpoint_rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)), + "a shared compacted parent accumulator must not be snapshotted as the current child" + ); +} diff --git a/crates/scanner/src/scanner_io/cache.rs b/crates/scanner/src/scanner_io/cache.rs index 4ae95d6a0..ff2da937c 100644 --- a/crates/scanner/src/scanner_io/cache.rs +++ b/crates/scanner/src/scanner_io/cache.rs @@ -28,7 +28,8 @@ pub(crate) fn scanner_cache_lock_timeout() -> Duration { pub(crate) enum ScannerCheckpointPersistResult { Saved, FenceChanged, - Failed(StorageError), + Skipped, + RetryBucket(StorageError), } pub(crate) struct ScannerCheckpointPersistContext<'a> { @@ -80,7 +81,12 @@ where ) .await; if let Err(error) = checkpoint_save { - return ScannerCheckpointPersistResult::Failed(error); + record_checkpoint_save_failure(cache_name, checkpoint, &error); + return if matches!(error, StorageError::PreconditionFailed) { + ScannerCheckpointPersistResult::RetryBucket(error) + } else { + ScannerCheckpointPersistResult::Skipped + }; } if crate::remote_scanner::validate_remote_scanner_request_fence_with_store(context.cycle, context.leader_epoch, fence_store) @@ -101,12 +107,48 @@ where { Ok(next_revisions) => { *revisions = next_revisions; + if let Some(source) = checkpoint.info.source { + let saved = checkpoint.info.scan_checkpoint.as_ref().map(|cursor| { + rustfs_scanner_metrics::metrics::ScannerCheckpointReport { + version: cursor.version, + resume_after: cursor.resume_after.clone(), + reason: cursor.reason.as_str().to_string(), + last_event: "saved".to_string(), + } + }); + global_metrics().record_scanner_checkpoint_saved( + source.pool_index, + source.set_index, + &checkpoint.info.name, + saved, + ); + } ScannerCheckpointPersistResult::Saved } - Err(error) => ScannerCheckpointPersistResult::Failed(error), + Err(error) => { + // A successful PUT consumed the old CAS revision. Reload the bucket + // before another write instead of rebasing this scan over a newer writer. + record_checkpoint_save_failure(cache_name, checkpoint, &error); + ScannerCheckpointPersistResult::RetryBucket(error) + } } } +fn record_checkpoint_save_failure(cache_name: &str, checkpoint: &DataUsageCache, error: &StorageError) { + global_metrics().record_scanner_checkpoint_save_failed(); + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + state = "periodic_checkpoint_save_failed", + bucket = %checkpoint.info.name, + cache_name = %cache_name, + error = %error, + "Scanner periodic checkpoint save failed" + ); +} + #[derive(Debug)] pub(crate) struct ScannerCacheLockGuards { scoped: NamespaceLockGuard, diff --git a/crates/scanner/src/scanner_io/io_cache.rs b/crates/scanner/src/scanner_io/io_cache.rs index e8920915e..aa51abae0 100644 --- a/crates/scanner/src/scanner_io/io_cache.rs +++ b/crates/scanner/src/scanner_io/io_cache.rs @@ -1209,7 +1209,7 @@ impl ScannerIOCache for SetDisks { ) .await { - ScannerCheckpointPersistResult::Saved => { + ScannerCheckpointPersistResult::Saved | ScannerCheckpointPersistResult::Skipped => { if cache_guard.is_lock_lost() || ctx_clone.is_cancelled() { scan_ctx.cancel(); await_scanner_disk_shutdown(scan.as_mut()).await; @@ -1221,19 +1221,10 @@ impl ScannerIOCache for SetDisks { await_scanner_disk_shutdown(scan.as_mut()).await; break Err(Error::other("scanner bucket cache fence changed during checkpoint save")); } - ScannerCheckpointPersistResult::Failed(error) => { - error!( - target: "rustfs::scanner::io", - event = EVENT_SCANNER_CACHE_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_IO, - bucket = %bucket.name, - cache_name = %cache_name, - state = "periodic_checkpoint_save_failed", - error = %error, - "Scanner periodic checkpoint save failed" - ); - checkpoint_channel_closed = true; + ScannerCheckpointPersistResult::RetryBucket(error) => { + scan_ctx.cancel(); + await_scanner_disk_shutdown(scan.as_mut()).await; + break Err(error); } } } diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index 05f51fbe3..08d6799e6 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -717,7 +717,7 @@ async fn checkpoint_uses_global_cycle_fence_with_set_scoped_cache_revisions() { &mut revisions, ) .await, - ScannerCheckpointPersistResult::Failed(StorageError::PreconditionFailed) + ScannerCheckpointPersistResult::RetryBucket(StorageError::PreconditionFailed) ), "matching global fence must not authorize stale cache CAS revisions" ); @@ -734,6 +734,175 @@ async fn checkpoint_uses_global_cycle_fence_with_set_scoped_cache_revisions() { assert!(mismatched_sets > 0, "multi-pool fixture must expose a set-scoped cycle fence mismatch"); } +#[derive(Debug)] +struct CheckpointFaultStore { + set: Arc, + fail_put: AtomicBool, + fail_revision_read: AtomicBool, +} + +#[async_trait::async_trait] +impl crate::storage_api::owner::ObjectIO for CheckpointFaultStore { + type Error = crate::EcstoreError; + type RangeSpec = crate::storage_api::owner::HTTPRangeSpec; + type HeaderMap = http::HeaderMap; + type ObjectOptions = crate::ScannerObjectOptions; + type ObjectInfo = crate::ScannerObjectInfo; + type GetObjectReader = crate::ScannerGetObjectReader; + type PutObjectReader = crate::ScannerPutObjReader; + + async fn get_object_reader( + &self, + bucket: &str, + object: &str, + range: Option, + headers: Self::HeaderMap, + opts: &Self::ObjectOptions, + ) -> crate::EcstoreResult { + if self.fail_revision_read.swap(false, Ordering::SeqCst) { + return Err(StorageError::Io(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "injected revision read failure", + ))); + } + self.set.get_object_reader(bucket, object, range, headers, opts).await + } + + async fn put_object( + &self, + bucket: &str, + object: &str, + data: &mut Self::PutObjectReader, + opts: &Self::ObjectOptions, + ) -> crate::EcstoreResult { + if self.fail_put.swap(false, Ordering::SeqCst) { + return Err(StorageError::Lock(LockError::Timeout { + resource: object.to_string(), + timeout: Duration::from_secs(5), + })); + } + self.set.put_object(bucket, object, data, opts).await + } +} + +#[async_trait::async_trait] +impl crate::ScannerConfigObjectDelete for CheckpointFaultStore { + async fn delete_config_object( + &self, + bucket: &str, + object: &str, + opts: ScannerObjectOptions, + ) -> crate::EcstoreResult { + crate::ScannerConfigObjectDelete::delete_config_object(self.set.as_ref(), bucket, object, opts).await + } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + crate::ScannerConfigObjectDelete::scanner_data_usage_publication_admission(self.set.as_ref()).await + } +} + +#[tokio::test] +#[serial] +async fn checkpoint_retries_next_snapshot_after_lock_failure_and_reloads_after_revision_failure() { + let (_temp_dir, fence_store) = setup_local_scanner_store(1, 4).await; + let cycle = 8457; + let leader_epoch = 19; + save_config( + fence_store.clone(), + DATA_USAGE_BLOOM_NAME_PATH.as_str(), + crate::scanner::encode_scanner_cycle_fence_for_test(cycle, leader_epoch), + ) + .await + .expect("seed global cycle fence"); + let set = fence_store.pools[0].disk_set[0].clone(); + let publication_epoch = scanner_publication_epoch(set.clone()).await.expect("read publication epoch"); + let cache_name = "checkpoint-recovery/.usage-cache.bin"; + DataUsageCache::default() + .save(set.clone(), cache_name) + .await + .expect("seed cache"); + let mut revisions = DataUsageCache::read_revisions(set.clone(), cache_name) + .await + .expect("read initial revisions"); + let initial_revisions = revisions.clone(); + let store = Arc::new(CheckpointFaultStore { + set: set.clone(), + fail_put: AtomicBool::new(true), + fail_revision_read: AtomicBool::new(false), + }); + let ctx = CancellationToken::new(); + let context = || ScannerCheckpointPersistContext { + ctx: &ctx, + expected_publication_epoch: publication_epoch, + cycle, + leader_epoch, + }; + let mut checkpoint = DataUsageCache::default(); + checkpoint.info.name = "checkpoint-recovery".to_string(); + checkpoint.info.source = Some(DataUsageCacheSource::new(0, 0)); + checkpoint.info.scan_checkpoint = Some(crate::DataUsageScanCheckpoint::new( + "checkpoint-recovery/first".to_string(), + crate::DataUsageScanCheckpointReason::Objects, + )); + assert!(matches!( + persist_scanner_checkpoint(store.clone(), fence_store.clone(), context(), cache_name, &checkpoint, &mut revisions).await, + ScannerCheckpointPersistResult::Skipped + )); + assert_eq!(revisions, initial_revisions); + assert_eq!( + DataUsageCache::read_revisions(set.clone(), cache_name) + .await + .expect("read after failed save"), + initial_revisions + ); + + checkpoint + .info + .scan_checkpoint + .as_mut() + .expect("checkpoint cursor") + .resume_after = "checkpoint-recovery/second".to_string(); + assert!(matches!( + persist_scanner_checkpoint(store.clone(), fence_store.clone(), context(), cache_name, &checkpoint, &mut revisions).await, + ScannerCheckpointPersistResult::Saved + )); + assert_ne!(revisions, initial_revisions); + let mut loaded = DataUsageCache::default(); + loaded.load(set.clone(), cache_name).await.expect("load recovered checkpoint"); + assert_eq!( + loaded.info.scan_checkpoint, checkpoint.info.scan_checkpoint, + "next snapshot must become durable" + ); + + let saved_revisions = revisions.clone(); + checkpoint + .info + .scan_checkpoint + .as_mut() + .expect("checkpoint cursor") + .resume_after = "checkpoint-recovery/third".to_string(); + store.fail_revision_read.store(true, Ordering::SeqCst); + assert!(matches!( + persist_scanner_checkpoint(store, fence_store, context(), cache_name, &checkpoint, &mut revisions).await, + ScannerCheckpointPersistResult::RetryBucket(StorageError::Io(_)) + )); + assert_eq!(revisions, saved_revisions, "indeterminate revision refresh must not rebase the writer"); + loaded + .load(set.clone(), cache_name) + .await + .expect("load committed checkpoint after refresh failure"); + assert_eq!( + loaded.info.scan_checkpoint, checkpoint.info.scan_checkpoint, + "PUT completed before revision refresh failed" + ); + assert_ne!( + DataUsageCache::read_revisions(set, cache_name) + .await + .expect("read committed revisions"), + saved_revisions + ); +} + #[derive(Debug)] struct CheckpointFenceStore { before: Vec, diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index 2f497cffe..33a182860 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -172,6 +172,20 @@ catch_up_estimate.discovered_expiry_items catch_up_estimate.discovered_transition_items ``` +### Periodic checkpoint persistence + +`metrics.scan_checkpoint` is the in-memory resume hint; it does not confirm a durable save. `metrics.scan_checkpoints_persisted` lists the latest confirmed periodic save for each `(pool_index, set_index)` handled by this process, with its bucket, `saved_unix_secs`, and optional checkpoint cursor. It retains one observation per set, is reset on process restart, and does not include the final bucket save. Query the peer executing the walk for remote-worker observations. `metrics.scan_checkpoint_save_failures` counts failed periodic saves, including failures to refresh CAS revisions after a write; failed attempts do not advance the confirmed save observation. + +An ordinary failed save drops that snapshot and leaves the bounded checkpoint receiver active, so the next producer interval tries again. A CAS conflict or an unreadable revision after a successful write cancels the bucket attempt and reloads persisted state through the existing retry path. Leader, cycle, and publication fencing remain mandatory. This preserves future periodic save attempts for a continuing bucket walk. + +Periodic snapshots include the suspended ancestor accounting needed to connect a nested active branch to the bucket root. Completed coverage is sealed independently of raw enumeration bookkeeping. A valid coverage frontier can therefore skip completed subtrees after a fenced leader handoff without counting unfinished work as completed. Emission remains bounded by the existing interval/object thresholds and capacity-one channel; invalid snapshot attempts also consume the producer interval so they cannot repeatedly clone the cache on every object. + +A resumed walk retains its previous durable coverage while initial enumeration has not recopied the covered subtrees. Unfinished out-of-order heal revisits cannot be certified as completed prefix work, including in cancellation snapshots. + +Shared compacted accumulators defer periodic emission until the owning subtree completes; replay inside that compacted subtree remains possible after an interruption. When bucket checkpoint preparation returns `Reset` for existing resume state, it logs `checkpoint_reset` with the validation reason. The `scan_checkpoint_used` counter includes validated coverage adoption by the forward sweep. + +A confirmed periodic save does not prove restart resumability. Adoption still validates bucket/set identity and, for non-empty caches, connected cache scope. Skipping completed subtrees additionally requires a validated coverage frontier; raw enumeration state alone does not prove completed subtree coverage. + ## Usage State Reset The supported break-glass route for rebuilding scanner usage state is `POST /v3/scanner/usage-state/reset` with body `{"mode":"full-rebuild"}`, authenticated as an admin identity holding `ConfigUpdateAdminAction` (route registered in `rustfs/src/admin/route_registration_test.rs`). Use it only after the scanner status shows a usage-floor load failure, a conflicting persisted usage floor, or an operator decision to discard the durable usage baseline and rebuild it from a full scanner pass. From 93a64fdca3e64947f406957c047324b191e22b54 Mon Sep 17 00:00:00 2001 From: Chris Date: Sat, 10 Oct 2026 21:34:41 -0700 Subject: [PATCH 005/143] fix(dev): restore VS Code debug configurations (#8450) --- .vscode/launch.json | 48 ++++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 394c1e979..d842e5118 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,7 +7,9 @@ "name": "Debug(only) executable 'rustfs'", "env": { "RUST_LOG": "rustfs=info,ecstore=info,s3s=info,iam=info", - "RUSTFS_SKIP_BACKGROUND_TASK": "on" + "RUSTFS_SKIP_BACKGROUND_TASK": "on", + // Local test volumes share one physical disk. + "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true" //"RUSTFS_OBS_LOG_DIRECTORY": "./deploy/logs", // "RUSTFS_POLICY_PLUGIN_URL":"http://localhost:8181/v1/data/rustfs/authz/allow", // "RUSTFS_POLICY_PLUGIN_AUTH_TOKEN":"your-opa-token" @@ -44,6 +46,7 @@ "env": { "RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug,iam=debug", "RUSTFS_SKIP_BACKGROUND_TASK": "on", + "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true", //"RUSTFS_OBS_LOG_DIRECTORY": "./deploy/logs", // "RUSTFS_POLICY_PLUGIN_URL":"http://localhost:8181/v1/data/rustfs/authz/allow", // "RUSTFS_POLICY_PLUGIN_AUTH_TOKEN":"your-opa-token" @@ -117,7 +120,8 @@ "env": { "RUSTFS_ACCESS_KEY": "rustfsadmin", "RUSTFS_SECRET_KEY": "rustfsadmin", - //"RUSTFS_VOLUMES": "./target/volume/test{1...4}", + "RUSTFS_VOLUMES": "./target/volume/test{1...4}", + "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true", "RUSTFS_ADDRESS": ":9000", "RUSTFS_CONSOLE_ENABLE": "true", // "RUSTFS_OBS_TRACE_ENDPOINT": "http://127.0.0.1:4318/v1/traces", // jeager otlp http endpoint @@ -135,41 +139,30 @@ { "type": "lldb", "request": "launch", - "name": "Debug test_lifecycle_transition_basic", + "name": "Debug test_transition_and_restore_flows", "cargo": { "args": [ "test", + "--no-run", "-p", "rustfs-scanner", "--test", "lifecycle_integration_test", - "serial_tests::test_lifecycle_transition_basic", "-j", "1" - ] + ], + "filter": { + "name": "lifecycle_integration_test", + "kind": "test" + } }, - "args": [], - "cwd": "${workspaceFolder}" - }, - { - "name": "Debug executable target/debug/test", - "type": "lldb", - "request": "launch", - "program": "${workspaceFolder}/target/debug/deps/lifecycle_integration_test-5915cbfcab491b3b", "args": [ - "--skip", - "test_lifecycle_expiry_basic", - "--skip", - "test_lifecycle_expiry_deletemarker", - //"--skip", - //"test_lifecycle_transition_basic", - ], - "cwd": "${workspaceFolder}", - //"stopAtEntry": false, - //"preLaunchTask": "cargo build", - "sourceLanguages": [ - "rust" + "serial_tests::test_transition_and_restore_flows", + "--exact", + "--ignored", + "--test-threads=1" ], + "cwd": "${workspaceFolder}" }, { "name": "Debug executable target/debug/rustfs with sse kms", @@ -285,7 +278,7 @@ "RUSTFS_COMPRESSION_ENABLED": "true", "RUSTFS_KMS_ENABLE": "true", "RUSTFS_KMS_BACKEND": "local", - "RUSTFS_KMS_KEY_DIR": "./target/rio-v2/kms-key-dir", + "RUSTFS_KMS_KEY_DIR": "${workspaceFolder}/target/rio-v2/kms-key-dir", "RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key", }, @@ -307,6 +300,7 @@ "RUSTFS_ACCESS_KEY": "rustfsadmin", "RUSTFS_SECRET_KEY": "rustfsadmin", "RUSTFS_VOLUMES": "./target/e2e-test/test{1...4}", + "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true", "RUSTFS_REGION": "us-east-1", "RUSTFS_ADDRESS": ":9000", "RUSTFS_CONSOLE_ENABLE": "true", @@ -315,7 +309,7 @@ "RUSTFS_KMS_ENABLE": "true", "RUSTFS_KMS_BACKEND": "local", - "RUSTFS_KMS_KEY_DIR": "./target/e2e-key-dir", + "RUSTFS_KMS_KEY_DIR": "${workspaceFolder}/target/e2e-key-dir", "RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key", }, From 0263d5fcb9e4641693380af385579bfbe226c9d9 Mon Sep 17 00:00:00 2001 From: Hauser Date: Sun, 11 Oct 2026 12:41:13 +0800 Subject: [PATCH 006/143] fix(storage): preserve heal failures and timeout responses (#8462) * fix(storage): preserve heal failures and timeout responses Stop repairs with no writable target before opening source readers. Consolidate target failure aggregation, preserve typed causes, and distinguish all-ignored diagnostics from actual successful nil entries. Map typed disk timeouts to retryable S3 responses without changing quorum decisions. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * refactor(heal): unify staging failure cleanup Route data-stage failures through one local result boundary after temporary readers and writers are released. Preserve typed report failures, outer reconstruction errors, partial repair, and the existing commit fences. Exercise actual source loss after second-part writer admission using the existing test pause registry. Assert all staged parts are removed, committed metadata stays unchanged, and restoring the source allows a full-byte retry. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: zhi22915 Co-authored-by: heihutu Co-authored-by: Chris --- crates/ecstore/src/disk/error_reduce.rs | 31 +- crates/ecstore/src/set_disk/ops/heal.rs | 793 +++++++++++++++--------- rustfs/src/error.rs | 32 + rustfs/src/storage_api.rs | 6 + scripts/error-other-format-baseline.txt | 2 +- 5 files changed, 555 insertions(+), 309 deletions(-) diff --git a/crates/ecstore/src/disk/error_reduce.rs b/crates/ecstore/src/disk/error_reduce.rs index 298905459..34d13fade 100644 --- a/crates/ecstore/src/disk/error_reduce.rs +++ b/crates/ecstore/src/disk/error_reduce.rs @@ -131,6 +131,15 @@ pub fn build_write_quorum_failure_summary( fn dominant_error_label(errors: &[Option], ignored_errs: &[Error], dominant_error: Option<&Error>) -> &'static str { let Some(dominant_error) = dominant_error else { + // The reducer also returns None when every failure is ignored for + // quorum voting. That case contains no successful nil entry. + if !errors.is_empty() && errors.iter().all(Option::is_some) { + return if errors.iter().all(|error| error.as_ref() == Some(&Error::DiskNotFound)) { + "disk_not_found" + } else { + "all_ignored" + }; + } return "nil_dominated"; }; @@ -387,9 +396,9 @@ mod tests { } #[test] - fn test_build_write_quorum_failure_summary_all_ignored_is_nil_dominated() { - // All disks offline (all DiskNotFound). The dominant error must be None and - // the label must be "nil_dominated" rather than falling back to "other_error". + fn test_build_write_quorum_failure_summary_all_missing_is_disk_not_found() { + // Ignoring unavailable disks for quorum reduction must not imply a + // successful nil entry in the diagnostic label. let errors = vec![ Some(Error::DiskNotFound), Some(Error::DiskNotFound), @@ -397,7 +406,21 @@ mod tests { ]; let summary = build_write_quorum_failure_summary(&errors, OBJECT_OP_IGNORED_ERRS, 2); assert_eq!(summary.dominant_error, None); - assert_eq!(summary.dominant_error_label, "nil_dominated"); + assert_eq!(summary.achieved, 0); + assert_eq!(summary.dominant_error_label, "disk_not_found"); + } + + #[test] + fn test_build_write_quorum_failure_summary_all_ignored_has_no_success() { + let errors = [Some(Error::FaultyDisk), Some(Error::DiskNotFound)]; + let summary = build_write_quorum_failure_summary(&errors, OBJECT_OP_IGNORED_ERRS, 2); + assert_eq!(summary.dominant_error, None); + assert_eq!(summary.achieved, 0); + assert_eq!(summary.dominant_error_label, "all_ignored"); + assert_eq!( + reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, 2), + Some(Error::ErasureWriteQuorum) + ); } #[test] diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 39924f1a4..07df7f69c 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -99,6 +99,7 @@ static HEAL_RENAME_FAILURES: std::sync::Mutex> = st #[derive(Clone, Copy, PartialEq, Eq)] enum ReadRepairPausePhase { Data, + PartStaging(usize), Commit, } @@ -250,7 +251,16 @@ fn should_fail_heal_rename(_bucket: &str, _object: &str, _disk_index: usize) -> } #[cfg(test)] -static HEAL_WRITER_FAILURES: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); +struct HealWriterFailure { + bucket: String, + object: String, + part_number: Option, + disk_index: usize, + error: DiskError, +} + +#[cfg(test)] +static HEAL_WRITER_FAILURES: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); #[cfg(test)] struct HealWriterFailureScope { @@ -260,21 +270,23 @@ struct HealWriterFailureScope { #[cfg(test)] impl HealWriterFailureScope { - fn install(bucket: &str, object: &str, disk_indexes: &[usize], error: DiskError) -> Self { + fn install(bucket: &str, object: &str, part_number: Option, disk_indexes: &[usize], error: DiskError) -> Self { let mut failures = HEAL_WRITER_FAILURES .lock() .expect("heal writer failure registry should not poison"); assert!( - !failures.iter().any(|(registered_bucket, registered_object, _, _)| { - registered_bucket == bucket && registered_object == object - }), + !failures + .iter() + .any(|failure| failure.bucket == bucket && failure.object == object), "heal writer failures must be installed once per object" ); - failures.extend( - disk_indexes - .iter() - .map(|index| (bucket.to_string(), object.to_string(), *index, error.clone())), - ); + failures.extend(disk_indexes.iter().map(|index| HealWriterFailure { + bucket: bucket.to_string(), + object: object.to_string(), + part_number, + disk_index: *index, + error: error.clone(), + })); Self { bucket: bucket.to_string(), object: object.to_string(), @@ -288,25 +300,28 @@ impl Drop for HealWriterFailureScope { HEAL_WRITER_FAILURES .lock() .expect("heal writer failure registry should not poison") - .retain(|(bucket, object, _, _)| bucket != &self.bucket || object != &self.object); + .retain(|failure| failure.bucket != self.bucket || failure.object != self.object); } } #[cfg(test)] -fn injected_heal_writer_error(bucket: &str, object: &str, disk_index: usize) -> Option { +fn injected_heal_writer_error(bucket: &str, object: &str, part_number: usize, disk_index: usize) -> Option { let mut failures = HEAL_WRITER_FAILURES .lock() .expect("heal writer failure registry should not poison"); failures .iter() - .position(|(registered_bucket, registered_object, registered_index, _)| { - registered_bucket == bucket && registered_object == object && *registered_index == disk_index + .position(|failure| { + failure.bucket == bucket + && failure.object == object + && failure.disk_index == disk_index + && failure.part_number.is_none_or(|part| part == part_number) }) - .map(|position| failures.swap_remove(position).3) + .map(|position| failures.swap_remove(position).error) } #[cfg(not(test))] -fn injected_heal_writer_error(_bucket: &str, _object: &str, _disk_index: usize) -> Option { +fn injected_heal_writer_error(_bucket: &str, _object: &str, _part_number: usize, _disk_index: usize) -> Option { None } @@ -518,30 +533,64 @@ fn is_retryable_heal_writer_error(error: &DiskError) -> bool { ) || error.is_retryable_internode_write_failure() } -fn warn_heal_writer_failures( - bucket: &str, - object: &str, - version_id: &str, - writer_failure_count: usize, - result: &'static str, - first_failure: &(usize, usize, String), -) { - let (first_part_number, first_disk_index, first_error) = first_failure; - warn!( - event = EVENT_SET_DISK_HEAL, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_SET_DISK, - bucket, - object, - version_id, - writer_failure_count, - first_part_number, - first_disk_index, - error = %first_error, - result, - state = "writer_unavailable", - "Set disk object heal writer failures" - ); +struct HealWriterFailures { + target_count: usize, + count: usize, + first: Option<(usize, usize, String)>, + error: Option, +} + +impl HealWriterFailures { + fn new(target_count: usize) -> Self { + Self { + target_count, + count: 0, + first: None, + error: None, + } + } + + fn record(&mut self, part_number: usize, disk_index: usize, error: DiskError) { + self.count += 1; + if self.first.is_none() { + self.first = Some((part_number, disk_index, heal_writer_error_summary(&error))); + } + // A permanent failure must not be hidden behind an earlier transient + // target failure when the remaining targets also become unavailable. + if self + .error + .as_ref() + .is_none_or(|first| is_retryable_heal_writer_error(first) && !is_retryable_heal_writer_error(&error)) + { + self.error = Some(error); + } + } + + fn into_error(self) -> DiskError { + self.error.unwrap_or(DiskError::ErasureWriteQuorum) + } + + fn warn(&self, bucket: &str, object: &str, version_id: &str, result: &'static str) { + let Some((first_part_number, first_disk_index, first_error)) = &self.first else { + return; + }; + warn!( + event = EVENT_SET_DISK_HEAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + state = "writer_unavailable", + result, + bucket, + object, + version_id, + target_count = self.target_count, + writer_failure_count = self.count, + first_part_number, + first_disk_index, + error = %first_error, + "Set disk object heal writer failures" + ); + } } impl SetDisks { @@ -1548,196 +1597,169 @@ impl SetDisks { let dst_data_dir = data_dir; if !latest_meta.deleted && !latest_meta.is_remote() { - let erasure_info = latest_meta.erasure.clone(); - let mut writer_failure_count = 0usize; - let mut first_writer_failure = None; - let mut first_writer_error = None; - let mut all_writer_failures_retryable = true; - let mut writer_failure_warned = false; + let stage_result: disk::error::Result> = async { + let erasure_info = latest_meta.erasure.clone(); + let mut writer_failures = HealWriterFailures::new(disks_to_heal_count); - for (part_index, part) in latest_meta.parts.iter().enumerate() { - use crate::io_support::shard_integrity::{PartProofReader, PreparedIntegrity, ShardVerifier}; - let proof = part - .integrity - .as_ref() - .map(|part| { - PartProofReader::new(part.clone(), &parts_metadata, &latest_disks, bucket, object) - }) - .transpose() - .map_err(DiskError::from)?; - // This reader covers the whole encoded shard. A range - // offset can exclude the final encoded byte when the - // logical part ends inside a data block, allowing a - // one-byte tail truncation to evade deep Heal. - let till_offset = usize::try_from(erasure.shard_file_size(part.size as i64)) - .map_err(|_| DiskError::FileCorrupt)?; - let use_mmap_read = object_mmap_read_enabled(); + for (part_index, part) in latest_meta.parts.iter().enumerate() { + use crate::io_support::shard_integrity::{PartProofReader, PreparedIntegrity, ShardVerifier}; + let proof = part + .integrity + .as_ref() + .map(|part| { + PartProofReader::new(part.clone(), &parts_metadata, &latest_disks, bucket, object) + }) + .transpose()?; + // Cover the complete encoded shard, including a logical + // part's partial final data block, when checking truncation. + let till_offset = usize::try_from(erasure.shard_file_size(part.size as i64)) + .map_err(|_| DiskError::FileCorrupt)?; + let use_mmap_read = object_mmap_read_enabled(); - let mut readers = Vec::with_capacity(latest_disks.len()); - let mut writers = Vec::with_capacity(out_dated_disks.len()); - // let mut errors = Vec::with_capacity(out_dated_disks.len()); + let mut writers = Vec::with_capacity(out_dated_disks.len()); - let mut prefer = vec![false; latest_disks.len()]; - for (index, disk) in latest_disks.iter().enumerate() { + // Preserve the committed layout: recomputing inline-ness here + // (with a hardcoded unversioned threshold) makes healed replicas + // diverge from healthy ones in quorum identity, so heal would + // flag them forever. + let is_inline_buffer = latest_meta.inline_data(); + // create writers for all disk positions, but only for outdated disks + for (index, disk_op) in out_dated_disks.iter().enumerate() { + if let Some(outdated_disk) = disk_op { + let writer_result = if let Some(error) = + injected_heal_writer_error(bucket, object, part.number, index) + { + Err(error) + } else { + create_bitrot_writer( + is_inline_buffer, + Some(outdated_disk), + RUSTFS_META_TMP_BUCKET, + &path_join_buf(&[ + &tmp_id.to_string(), + &dst_data_dir.to_string(), + &format!("part.{}", part.number), + ]), + erasure.shard_file_size(part.size as i64), + erasure.shard_size(), + HashAlgorithm::HighwayHash256S, + ) + .await + }; + let writer = match writer_result { + Ok(writer) => writer, + Err(err) => { + writer_failures.record(part.number, index, err); + writers.push(None); + continue; + } + }; + let mut writer = writer; + if let Some(proof) = &proof { + let verifier = ShardVerifier::new(std::sync::Arc::clone(proof), index, 0, None)?; + writer.set_integrity(verifier); + } + writers.push(Some(writer)); + } else { + writers.push(None); + } + } + + // Heal each part. erasure.Heal() will write the healed + // part to .rustfs/tmp/uuid/ which needs to be renamed + // later to the final location. + if writers.iter().all(Option::is_none) { + writer_failures.warn(bucket, object, version_id, "all_targets_unavailable"); + // No target can accept this part. Preserve the setup + // cause before opening source readers or reconstructing. + return Ok(Some(writer_failures.into_error())); + } + #[cfg(test)] + pause_read_repair(bucket, object, ReadRepairPausePhase::PartStaging(part.number)).await; + + let mut readers = Vec::with_capacity(latest_disks.len()); + + let mut prefer = vec![false; latest_disks.len()]; let this_part_errs = Self::shuffle_check_parts(&data_errs_by_part[&part_index], &erasure_info.distribution); - if this_part_errs[index] != CHECK_PART_SUCCESS { - trace!( - event = EVENT_SET_DISK_HEAL, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_SET_DISK, - part_number = part.number, - disk_index = index, - part_status = this_part_errs[index], - state = "source_shard_skipped", - "Set disk source shard skipped" - ); - readers.push(None); - continue; - } - - if let (Some(disk), Some(metadata)) = (disk, ©_parts_metadata[index]) { - let checksum_info = metadata.erasure.get_checksum_info(part.number); - let checksum_algo = if metadata.uses_legacy_checksum - && checksum_info.algorithm == HashAlgorithm::HighwayHash256S - { - HashAlgorithm::HighwayHash256SLegacy - } else { - checksum_info.algorithm - }; - - match create_bitrot_reader( - metadata.data.as_deref(), - Some(disk), - bucket, - &path_join_buf(&[object, &src_data_dir, &format!("part.{}", part.number)]), - 0, - till_offset, - erasure.shard_size(), - checksum_algo.clone(), - false, - use_mmap_read, - ) - .await - { - Ok(Some(mut reader)) => { - if let Some(proof) = &proof { - reader - .set_integrity( - ShardVerifier::new(std::sync::Arc::clone(proof), index, 0, None) - .map_err(DiskError::from)?, - ) - .map_err(DiskError::from)?; - } - readers.push(Some(reader)); - } - Ok(None) => { - readers.push(None); - continue; - } - Err(_e) => { - readers.push(None); - continue; - } + for (index, disk) in latest_disks.iter().enumerate() { + if this_part_errs[index] != CHECK_PART_SUCCESS { + trace!( + event = EVENT_SET_DISK_HEAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + part_number = part.number, + disk_index = index, + part_status = this_part_errs[index], + state = "source_shard_skipped", + "Set disk source shard skipped" + ); + readers.push(None); + continue; } - prefer[index] = disk.host_name().is_empty(); - } else { - readers.push(None); - // errors.push(Some(DiskError::DiskNotFound)); - } - } + if let (Some(disk), Some(metadata)) = (disk, ©_parts_metadata[index]) { + let checksum_info = metadata.erasure.get_checksum_info(part.number); + let checksum_algo = if metadata.uses_legacy_checksum + && checksum_info.algorithm == HashAlgorithm::HighwayHash256S + { + HashAlgorithm::HighwayHash256SLegacy + } else { + checksum_info.algorithm + }; - // Preserve the committed layout: recomputing inline-ness here - // (with a hardcoded unversioned threshold) makes healed replicas - // diverge from healthy ones in quorum identity, so heal would - // flag them forever. - let is_inline_buffer = latest_meta.inline_data(); - // create writers for all disk positions, but only for outdated disks - for (index, disk_op) in out_dated_disks.iter().enumerate() { - if let Some(outdated_disk) = disk_op { - let writer_result = if let Some(error) = injected_heal_writer_error(bucket, object, index) - { - Err(error) - } else { - create_bitrot_writer( - is_inline_buffer, - Some(outdated_disk), - RUSTFS_META_TMP_BUCKET, - &path_join_buf(&[ - &tmp_id.to_string(), - &dst_data_dir.to_string(), - &format!("part.{}", part.number), - ]), - erasure.shard_file_size(part.size as i64), + match create_bitrot_reader( + metadata.data.as_deref(), + Some(disk), + bucket, + &path_join_buf(&[object, &src_data_dir, &format!("part.{}", part.number)]), + 0, + till_offset, erasure.shard_size(), - HashAlgorithm::HighwayHash256S, + checksum_algo.clone(), + false, + use_mmap_read, ) .await - }; - let writer = match writer_result { - Ok(writer) => writer, - Err(err) => { - writer_failure_count += 1; - all_writer_failures_retryable &= is_retryable_heal_writer_error(&err); - if first_writer_failure.is_none() { - first_writer_failure = - Some((part.number, index, heal_writer_error_summary(&err))); - first_writer_error = Some(err); + { + Ok(Some(mut reader)) => { + if let Some(proof) = &proof { + let verifier = + ShardVerifier::new(std::sync::Arc::clone(proof), index, 0, None)?; + reader.set_integrity(verifier)?; + } + readers.push(Some(reader)); + } + Ok(None) => { + readers.push(None); + continue; + } + Err(_e) => { + readers.push(None); + continue; } - writers.push(None); - continue; } - }; - let mut writer = writer; - if let Some(proof) = &proof { - writer.set_integrity( - ShardVerifier::new(std::sync::Arc::clone(proof), index, 0, None) - .map_err(DiskError::from)?, - ); - } - writers.push(Some(writer)); - } else { - writers.push(None); - } - } - // Heal each part. erasure.Heal() will write the healed - // part to .rustfs/tmp/uuid/ which needs to be renamed - // later to the final location. - if writer_failure_count > 0 - && writers.iter().all(Option::is_none) - && let Some(first_failure) = first_writer_failure.as_ref() - { - warn_heal_writer_failures( - bucket, - object, - version_id, - writer_failure_count, - "all_targets_unavailable", - first_failure, - ); - writer_failure_warned = true; - } - let heal_part = erasure.heal(&mut writers, readers, part.size, &prefer); - let heal_part_result = if read_repair_uses_shared_lock { - match tokio::time::timeout(READ_REPAIR_DATA_PHASE_TIMEOUT, heal_part).await { - Ok(result) => result, - Err(_) => Err(DiskError::Timeout), + prefer[index] = disk.host_name().is_empty(); + } else { + readers.push(None); + // errors.push(Some(DiskError::DiskNotFound)); + } } - } else { - heal_part.await - }; - if let Err(e) = heal_part_result { - // Don't leak the partially-written healed shards in - // .rustfs/tmp when heal fails midway (backlog#799 B20). - let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await; - return Err(e); - } - // close_bitrot_writers(&mut writers).await?; - if !is_inline_buffer && let Some(proof) = &proof { - let before = out_dated_disks.iter().filter(|disk| disk.is_some()).count(); - let proof_result = async { + + let heal_part = erasure.heal(&mut writers, readers, part.size, &prefer); + let heal_part_result = if read_repair_uses_shared_lock { + match tokio::time::timeout(READ_REPAIR_DATA_PHASE_TIMEOUT, heal_part).await { + Ok(result) => result, + Err(_) => Err(DiskError::Timeout), + } + } else { + heal_part.await + }; + heal_part_result?; + // close_bitrot_writers(&mut writers).await?; + if !is_inline_buffer && let Some(proof) = &proof { + let before = out_dated_disks.iter().filter(|disk| disk.is_some()).count(); PreparedIntegrity::copy_from(proof) .await? .write( @@ -1746,89 +1768,71 @@ impl SetDisks { RUSTFS_META_TMP_BUCKET, &format!("{tmp_id}/{dst_data_dir}"), ) - .await - } - .await; - if let Err(error) = proof_result { - let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await; - return Err(error.into()); - } - disks_to_heal_count -= before - out_dated_disks.iter().filter(|disk| disk.is_some()).count(); - } - - for (index, disk_op) in out_dated_disks.iter_mut().enumerate() { - if disk_op.is_none() { - continue; + .await?; + disks_to_heal_count -= + before - out_dated_disks.iter().filter(|disk| disk.is_some()).count(); } - if writers[index].is_none() { - *disk_op = None; - disks_to_heal_count -= 1; - continue; - } - - parts_metadata[index].data_dir = Some(dst_data_dir); - parts_metadata[index].add_object_part( - part.number, - part.etag.clone(), - part.size, - part.mod_time, - part.actual_size, - part.index.clone(), - part.checksums.clone(), - ); - if let Some(updated) = parts_metadata[index] - .parts - .iter_mut() - .find(|updated| updated.number == part.number) - { - updated.integrity.clone_from(&part.integrity); - } - if is_inline_buffer { - if let Some(writer) = writers[index].take() { - // if let Some(w) = writer.as_any().downcast_ref::() { - // parts_metadata[index].data = Some(w.inline_data().to_vec()); - // } - parts_metadata[index].data = - Some(writer.into_inline_data().map(Bytes::from).unwrap_or_default()); + for (index, disk_op) in out_dated_disks.iter_mut().enumerate() { + if disk_op.is_none() { + continue; } - parts_metadata[index].set_inline_data(); - } else { - parts_metadata[index].data = None; - } - } - if disks_to_heal_count == 0 { - if !writer_failure_warned && let Some(first_failure) = first_writer_failure.as_ref() { - warn_heal_writer_failures( - bucket, - object, - version_id, - writer_failure_count, - "all_targets_unavailable", - first_failure, + if writers[index].is_none() { + *disk_op = None; + disks_to_heal_count -= 1; + continue; + } + + parts_metadata[index].data_dir = Some(dst_data_dir); + parts_metadata[index].add_object_part( + part.number, + part.etag.clone(), + part.size, + part.mod_time, + part.actual_size, + part.index.clone(), + part.checksums.clone(), ); + if let Some(updated) = parts_metadata[index] + .parts + .iter_mut() + .find(|updated| updated.number == part.number) + { + updated.integrity.clone_from(&part.integrity); + } + if is_inline_buffer { + if let Some(writer) = writers[index].take() { + // if let Some(w) = writer.as_any().downcast_ref::() { + // parts_metadata[index].data = Some(w.inline_data().to_vec()); + // } + parts_metadata[index].data = + Some(writer.into_inline_data().map(Bytes::from).unwrap_or_default()); + } + parts_metadata[index].set_inline_data(); + } else { + parts_metadata[index].data = None; + } } - // Clean up healed shards written to .rustfs/tmp before bailing (B20). - let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await; - let error = if all_writer_failures_retryable { - first_writer_error.take().unwrap_or(DiskError::FaultyDisk) - } else { - DiskError::other(format!("all drives had write errors, unable to heal {bucket}/{object}")) - }; - return Ok((result, Some(error))); - } - } - if !writer_failure_warned && let Some(first_failure) = first_writer_failure.as_ref() { - warn_heal_writer_failures( - bucket, - object, - version_id, - writer_failure_count, - "partial_targets_unavailable", - first_failure, - ); + if disks_to_heal_count == 0 { + writer_failures.warn(bucket, object, version_id, "all_targets_unavailable"); + return Ok(Some(writer_failures.into_error())); + } + } + + writer_failures.warn(bucket, object, version_id, "partial_targets_unavailable"); + Ok(None) + } + .await; + match stage_result { + Ok(None) => {} + failure => { + // Release readers and writers before removing all parts + // staged under this heal's exclusively owned prefix. + let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await; + return failure.map(|error| (result, error)); + } } } let _read_repair_write_lock_guard = if let Some(expected_fingerprint) = read_repair_commit_fingerprint { @@ -3531,8 +3535,8 @@ impl SetDisks { #[cfg(test)] mod heal_result_report_tests { use super::{ - AbsenceProofRequest, DanglingCheckPartsFailure, DanglingDeleteFailure, DanglingDeleteSafety, ReadRepairCommitFingerprint, - ReadRepairPauseScope, SetDisks, heal_writer_error_summary, is_retryable_heal_writer_error, + AbsenceProofRequest, DanglingCheckPartsFailure, DanglingDeleteFailure, DanglingDeleteSafety, HealWriterFailures, + ReadRepairCommitFingerprint, ReadRepairPauseScope, SetDisks, heal_writer_error_summary, is_retryable_heal_writer_error, }; use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope, HealWriterFailureScope}; use crate::disk::endpoint::Endpoint; @@ -3786,10 +3790,17 @@ mod heal_result_report_tests { #[tokio::test(flavor = "current_thread")] #[serial_test::serial] async fn heal_writer_failures_emit_one_aggregate_warning_per_object() { - for (case, failed_target_count, expected_result, expect_error) in [ - ("partial", 1usize, "partial_targets_unavailable", false), - ("all", 2usize, "all_targets_unavailable", true), + for (case, target_count, failed_target_count, injected_error) in [ + ("partial", 2usize, 1usize, DiskError::DiskFull), + ("all-full", 2, 2, DiskError::DiskFull), + ("all-timeout", 2, 2, DiskError::Timeout), + ("single-faulty", 1, 1, DiskError::FaultyDisk), ] { + let expected_result = if target_count == failed_target_count { + "all_targets_unavailable" + } else { + "partial_targets_unavailable" + }; let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; let bucket = format!("heal-writer-{case}"); let object = "object.bin"; @@ -3816,11 +3827,13 @@ mod heal_result_report_tests { .read_version("", &bucket, object, "", &ReadOptions::default()) .await .expect("source metadata should be readable"); - let mut target_slots = [source.erasure.distribution[0] - 1, source.erasure.distribution[1] - 1]; + let mut target_slots = (0..target_count) + .map(|index| source.erasure.distribution[index] - 1) + .collect::>(); target_slots.sort_unstable(); - for index in [0, 1] { - remove_current_object_part(&temp_dirs[index], &bucket, object) + for dir in temp_dirs.iter().take(target_count) { + remove_current_object_part(dir, &bucket, object) .await .expect("target shard should be removed before heal"); } @@ -3834,7 +3847,9 @@ mod heal_result_report_tests { .without_time() .finish(); let subscriber_guard = tracing::subscriber::set_default(subscriber); - let failure_scope = HealWriterFailureScope::install(&bucket, object, failed_slots, DiskError::DiskFull); + let tmp_before = non_trash_tmp_entries(&temp_dirs).await; + let injected_summary = heal_writer_error_summary(&injected_error); + let failure_scope = HealWriterFailureScope::install(&bucket, object, None, failed_slots, injected_error.clone()); let heal_outcome = set .heal_object( @@ -3851,12 +3866,22 @@ mod heal_result_report_tests { drop(failure_scope); drop(subscriber_guard); + let (_, error) = heal_outcome.expect("writer setup failures must retain a heal report and their typed error"); assert_eq!( - heal_outcome.is_err(), - expect_error, - "{case}: aggregate heal result should match writer outcomes" + error, + (target_count == failed_target_count).then_some(injected_error), + "{case}: partial repair succeeds and all-target failure retains its cause" + ); + assert_eq!( + non_trash_tmp_entries(&temp_dirs).await, + tmp_before, + "{case}: heal must clean temporary shards" ); let output = logs.contents(); + assert!( + !output.contains("Erasure encode write quorum unavailable"), + "{case}: unavailable heal targets must not fabricate an all-disk encode failure: {output}" + ); assert_eq!( output.matches("Set disk object heal writer failures").count(), 1, @@ -3879,12 +3904,172 @@ mod heal_result_report_tests { "{case}: warning must distinguish partial from all-target failure: {output}" ); assert!( - output.contains("error=drive path full"), + output.contains(&format!("error={injected_summary}")), "{case}: warning must preserve a redacted failure reason: {output}" ); } } + #[test] + fn heal_writer_failure_aggregation_retains_permanent_cause() { + for errors in [ + [DiskError::Timeout, DiskError::DiskFull], + [DiskError::DiskFull, DiskError::Timeout], + ] { + let mut failures = HealWriterFailures::new(2); + for (index, error) in errors.into_iter().enumerate() { + failures.record(1, index, error); + } + assert_eq!(failures.count, 2); + assert_eq!(failures.into_error(), DiskError::DiskFull); + } + } + + #[tokio::test] + #[serial_test::serial] + async fn heal_second_part_failures_clean_staged_parts_and_retry() { + use crate::set_disk::CompletePart; + use crate::storage_api_contracts::multipart::MultipartOperations as _; + use tokio::io::AsyncReadExt as _; + + let (dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "heal-writer-second-part"; + let object = "multipart.bin"; + for disk in &disks { + disk.make_volume(bucket).await.expect("fixture bucket"); + } + let options = ObjectOptions { + no_lock: true, + ..Default::default() + }; + let upload = set + .new_multipart_upload(bucket, object, &options) + .await + .expect("multipart upload"); + let payloads = [vec![0x35; 5 * 1024 * 1024], vec![0x69; 1024 * 1024]]; + let mut completed = Vec::new(); + for (index, payload) in payloads.iter().enumerate() { + let part = set + .put_object_part( + bucket, + object, + &upload.upload_id, + index + 1, + &mut PutObjReader::from_vec(payload.clone()), + &options, + ) + .await + .expect("multipart part"); + completed.push(CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }); + } + set.clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, completed, &options) + .await + .expect("completed multipart object"); + let source = disks[2] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("healthy source metadata"); + let data_dir = source.data_dir.expect("multipart data directory").to_string(); + let target_slots = [source.erasure.distribution[0] - 1, source.erasure.distribution[1] - 1]; + for dir in &dirs[..2] { + tokio::fs::remove_dir_all(dir.path().join(bucket).join(object).join(&data_dir)) + .await + .expect("remove target shards"); + } + let tmp_before = non_trash_tmp_entries(&dirs).await; + let scope = HealWriterFailureScope::install(bucket, object, Some(2), &target_slots, DiskError::Timeout); + let heal_options = HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }; + let (_, error) = set + .heal_object(bucket, object, "", &heal_options) + .await + .expect("failed heal report"); + drop(scope); + assert_eq!(error, Some(DiskError::Timeout)); + assert_eq!( + non_trash_tmp_entries(&dirs).await, + tmp_before, + "first-part staging must be cleaned after the second-part setup failure" + ); + + let source_path = dirs[2].path().join(bucket).join(object).join(&data_dir).join("part.2"); + let source_bytes = tokio::fs::read(&source_path).await.expect("committed source shard"); + let mut metadata_before = Vec::new(); + for dir in &dirs { + metadata_before.push( + tokio::fs::read(dir.path().join(bucket).join(object).join("xl.meta")) + .await + .expect("committed metadata before reconstruction failure"), + ); + } + let (pause_scope, arrived, release) = + ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(2)); + let lose_source = async { + arrived.notified().await; + assert_ne!( + non_trash_tmp_entries(&dirs).await, + tmp_before, + "the first part must already be staged before the source loss" + ); + tokio::fs::remove_file(&source_path) + .await + .expect("lose one remaining source shard"); + release.notify_one(); + }; + let (outcome, ()) = tokio::time::timeout(std::time::Duration::from_secs(10), async { + tokio::join!(set.heal_object(bucket, object, "", &heal_options), lose_source) + }) + .await + .expect("heal must report source loss without waiting indefinitely"); + drop(pause_scope); + assert_eq!( + outcome.expect_err("source loss after writer admission must preserve the outer failure"), + DiskError::ErasureReadQuorum + ); + assert_eq!( + non_trash_tmp_entries(&dirs).await, + tmp_before, + "reconstruction failure must clean every staged part" + ); + for (dir, expected) in dirs.iter().zip(metadata_before) { + assert_eq!( + tokio::fs::read(dir.path().join(bucket).join(object).join("xl.meta")) + .await + .expect("metadata after reconstruction failure"), + expected, + "a failed data stage must not publish replacement metadata" + ); + } + tokio::fs::write(&source_path, source_bytes) + .await + .expect("restore the committed source shard"); + + let (_, error) = set + .heal_object(bucket, object, "", &heal_options) + .await + .expect("retry after target recovery"); + assert!(error.is_none(), "recovered targets should heal both parts: {error:?}"); + let mut reader = set + .get_object_reader(bucket, object, None, Default::default(), &options) + .await + .expect("read repaired multipart object"); + let mut actual = Vec::new(); + reader + .stream + .read_to_end(&mut actual) + .await + .expect("complete repaired object body"); + assert_eq!(actual, payloads.concat()); + } + async fn real_disk() -> (TempDir, Endpoint, DiskStore) { let dir = tempfile::tempdir().expect("tempdir should be created"); let endpoint = diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index 6f7242b91..e8cc2ae0b 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -695,6 +695,7 @@ impl From for ApiError { StorageError::FaultyDisk | StorageError::FaultyRemoteDisk | StorageError::DiskNotFound + | StorageError::Timeout | StorageError::TooManyOpenFiles => S3ErrorCode::ServiceUnavailable, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _) => { S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into()) @@ -1719,6 +1720,37 @@ mod tests { } } + #[test] + fn disk_quorum_timeouts_map_to_retryable_service_unavailable() { + use crate::storage_api::error::{DiskError, OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs}; + + let errors = [ + Some(DiskError::Timeout), + Some(DiskError::Timeout), + Some(DiskError::Timeout), + None, + ]; + for reduced in [ + reduce_read_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, 2), + reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, 3), + ] { + let reduced = reduced.expect("quorum of timeouts must retain its typed cause"); + assert_eq!(reduced, DiskError::Timeout); + let error = ApiError::from(StorageError::from(reduced)); + assert_eq!(error.code, S3ErrorCode::ServiceUnavailable); + let response = S3Error::from(error); + assert_eq!(response.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE)); + } + let degraded = [None, None, None, Some(DiskError::Timeout)]; + assert!(reduce_write_quorum_errs(°raded, OBJECT_OP_IGNORED_ERRS, 3).is_none()); + for failure in [DiskError::FileCorrupt, DiskError::Io(IoError::other("unknown disk failure"))] { + let errors = [Some(failure.clone()), Some(failure.clone()), Some(failure), None]; + let reduced = + reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, 3).expect("permanent failures must remain errors"); + assert_eq!(ApiError::from(StorageError::from(reduced)).code, S3ErrorCode::InternalError); + } + } + #[test] fn test_api_error_from_unexpected_eof_maps_to_incomplete_body() { let io_error = IoError::new(ErrorKind::UnexpectedEof, rustfs_rio::IncompleteBody { remaining: 7 }); diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index b7528a860..2e51ba37e 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -79,6 +79,12 @@ pub(crate) mod config_test { } pub(crate) mod error { + #[cfg(test)] + pub(crate) use crate::storage::storage_api::ecstore_disk::error::Error as DiskError; + #[cfg(test)] + pub(crate) use crate::storage::storage_api::ecstore_disk::error_reduce::{ + OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs, + }; pub(crate) mod contract { pub(crate) use super::super::storage_contracts::error::StorageErrorCode; diff --git a/scripts/error-other-format-baseline.txt b/scripts/error-other-format-baseline.txt index 967a87143..b0fc0f007 100644 --- a/scripts/error-other-format-baseline.txt +++ b/scripts/error-other-format-baseline.txt @@ -59,7 +59,7 @@ 7|crates/ecstore/src/set_disk/core/io_primitives.rs 1|crates/ecstore/src/set_disk/mod.rs 3|crates/ecstore/src/set_disk/ops/bitrot_self_verify.rs -2|crates/ecstore/src/set_disk/ops/heal.rs +1|crates/ecstore/src/set_disk/ops/heal.rs 8|crates/ecstore/src/set_disk/ops/multipart.rs 2|crates/ecstore/src/set_disk/ops/object.rs 3|crates/ecstore/src/set_disk/read.rs From 367f13f3f9a87b09b9ce3c22fef89525129733d3 Mon Sep 17 00:00:00 2001 From: Chris Date: Sat, 10 Oct 2026 22:19:25 -0700 Subject: [PATCH 007/143] ci: reduce repeated PR runner work (#8463) --- .config/make/tests.mak | 1 + .github/workflows/ci.yml | 6 +- .github/workflows/e2e-upgrade.yml | 78 ++++++++++++------------ docs/testing/ci-gates.md | 12 ++-- scripts/ci_gate.py | 20 ++++++- scripts/test_upgrade_workflow.py | 99 +++++++++++++++++++++++++++++++ 6 files changed, 166 insertions(+), 50 deletions(-) create mode 100644 scripts/test_upgrade_workflow.py diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 72f39e663..78ffecfb9 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -52,6 +52,7 @@ script-tests: ## Run shell script tests ./scripts/check_embedded_secrets.sh --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py + $(RUSTFS_PYTHON_BIN) ./scripts/test_upgrade_workflow.py $(RUSTFS_PYTHON_BIN) ./scripts/test_migration_gate_evidence.py $(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe1907458..3d02f9966 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -390,7 +390,9 @@ jobs: test-and-lint-rio-v2: name: Test and Lint (rio-v2) - if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed') + # Alternate feature configurations run on main, merge groups, and scheduled + # or manual validation; the default workspace remains the PR gate. + if: needs.classify-changes.outputs.mode == 'full' && github.event_name != 'pull_request' needs: [ quick-checks, classify-changes ] runs-on: sm-standard-4 timeout-minutes: 90 @@ -506,7 +508,7 @@ jobs: test-and-lint-protocols: name: "Test and Lint (${{ matrix.features.name }})" - if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed') + if: needs.classify-changes.outputs.mode == 'full' && github.event_name != 'pull_request' needs: [ quick-checks, classify-changes ] runs-on: sm-standard-4 timeout-minutes: 90 diff --git a/.github/workflows/e2e-upgrade.yml b/.github/workflows/e2e-upgrade.yml index 21125fe13..b926b613d 100644 --- a/.github/workflows/e2e-upgrade.yml +++ b/.github/workflows/e2e-upgrade.yml @@ -96,38 +96,8 @@ jobs: overwrite: true upgrade: - name: ${{ matrix.name }} + name: Upgrade compatibility needs: build - strategy: - fail-fast: false - matrix: - include: - # The two `_from_rc2_` tests keep their names: they assert - # release-independent object contracts and pass unchanged against the - # newer pinned source, so renaming them would only churn history and - # the CI required-check names. UPGRADE_SOURCE_VERSION above is the - # single source of truth for which release they actually run against. - - name: Direct upgrade from the previous release - test: direct_upgrade_from_rc2_preserves_object_contracts - artifact: direct-upgrade - - name: Mixed-version rolling upgrade from the previous release - test: rolling_upgrade_from_rc2_preserves_mixed_version_contracts - artifact: mixed-version-upgrade - - name: Bucket configuration survives the upgrade - test: direct_upgrade_from_previous_release_preserves_bucket_configuration - artifact: bucket-config-upgrade - - name: Rollback reads current bucket metadata - test: rollback_to_previous_release_reads_current_bucket_metadata - artifact: bucket-config-rollback - - name: ODM configuration recovery after rc.5 rollback - test: rc5_rollback_requires_restoring_odm_configuration - artifact: odm-config-rollback - - name: Multipart layouts survive the rc.5 upgrade - test: direct_upgrade_from_rc5_preserves_multipart_layouts - artifact: multipart-layout-upgrade - - name: rc.5 multipart replication baseline - test: rc5_baseline_replicates_multipart_layouts - artifact: multipart-layout-baseline runs-on: sm-standard-2 timeout-minutes: 60 env: @@ -142,8 +112,7 @@ jobs: uses: ./.github/actions/setup with: cache-shared-key: e2e-upgrade-tests - # One main-branch consumer warms the shared test-harness dependencies. - cache-save-if: ${{ github.ref == 'refs/heads/main' && matrix.artifact == 'direct-upgrade' }} + cache-save-if: ${{ github.ref == 'refs/heads/main' }} install-build-packaging-tools: "false" install-test-tools: "false" @@ -170,22 +139,49 @@ jobs: echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV" echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV" - - name: Run upgrade compatibility test + # Reuse one compiled harness, while keeping each case in its own process. + # Continue after a failure to retain the former matrix's complete verdict. + - name: Run upgrade compatibility tests + shell: bash env: - RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR: ${{ runner.temp }}/rustfs-upgrade-g09-evidence/${{ matrix.artifact }} NO_PROXY: 127.0.0.1,localhost,::1 run: | export no_proxy="$NO_PROXY" - python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test \ - "upgrade_compatibility_test::${{ matrix.test }}" \ - -- --ignored --exact --nocapture + cases=( + direct-upgrade:direct_upgrade_from_rc2_preserves_object_contracts + mixed-version-upgrade:rolling_upgrade_from_rc2_preserves_mixed_version_contracts + bucket-config-upgrade:direct_upgrade_from_previous_release_preserves_bucket_configuration + bucket-config-rollback:rollback_to_previous_release_reads_current_bucket_metadata + odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration + multipart-layout-upgrade:direct_upgrade_from_rc5_preserves_multipart_layouts + multipart-layout-baseline:rc5_baseline_replicates_multipart_layouts + ) + status=0 + for entry in "${cases[@]}"; do + artifact="${entry%%:*}" + test="${entry#*:}" + export RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR="$RUNNER_TEMP/rustfs-upgrade-g09-evidence/$artifact" + export RUSTFS_E2E_LOG_DIR="$RUNNER_TEMP/rustfs-upgrade-logs/$artifact" + mkdir -p "$RUSTFS_E2E_LOG_DIR" + echo "::group::$test" + if python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test \ + "upgrade_compatibility_test::$test" -- --ignored --exact --nocapture \ + 2>&1 | tee "$RUSTFS_E2E_LOG_DIR/test.log"; then + echo "PASS: $test" + else + echo "::error::Upgrade compatibility failed: $test" + status=1 + fi + echo "::endgroup::" + done + exit "$status" - name: Upload scanner/heal G09 evidence if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: ${{ matrix.artifact }}-g09-evidence-${{ github.run_number }} - path: ${{ runner.temp }}/rustfs-upgrade-g09-evidence/${{ matrix.artifact }} + name: upgrade-g09-evidence-${{ github.run_number }} + path: ${{ runner.temp }}/rustfs-upgrade-g09-evidence if-no-files-found: ignore retention-days: 14 @@ -193,7 +189,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: ${{ matrix.artifact }}-server-logs-${{ github.run_number }} + name: upgrade-server-logs-${{ github.run_number }} path: ${{ runner.temp }}/rustfs-upgrade-logs if-no-files-found: warn retention-days: 14 diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 29cc67db4..389193326 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -15,10 +15,12 @@ The `main` ruleset (`6436880`) requires exactly these contexts, with `strict_req | `Quick Checks` | `ci.yml` job `quick-checks` | Formatting and repository guard scripts | | `Test and Lint` | `ci.yml` job `required-checks` | Exact expected results for every CI validation job, including workspace checks, critical E2E, feature lanes, and event-specific full suites | -Every PR enters `ci.yml`. The `classify-changes` job uses the base revision of `scripts/ci_gate.py` to select a conservative documentation-only path: root Markdown/licenses, `AGENTS.md`, Markdown under `docs/` or `.agents/skills/`, and documentation images. Unknown paths, unavailable Git history, an empty diff, or a missing base policy select the full matrix. Renames include their deleted source path. Documentation-only PRs still run Quick Checks and Typos; the aggregate requires the expensive jobs to be skipped exactly as selected. +Every PR enters `ci.yml`. The `classify-changes` job uses the base revision of `scripts/ci_gate.py` to select a conservative documentation-only path: root Markdown/licenses, `AGENTS.md`, Markdown under `docs/` or `.agents/skills/`, and documentation images. Unknown paths, unavailable Git history, an empty diff, or a missing base policy select the full scope for that event. Renames include their deleted source path. Documentation-only PRs still run Quick Checks and Typos; the aggregate requires the expensive jobs to be skipped exactly as selected. PRs changing only Rust sources under `crates/e2e_test/src/` and known E2E selection digests (optionally with those documentation paths) use the `e2e` scope. They keep workspace validation, the server build, E2E smoke, and both S3 lanes; independent workflows retain their path selection. Connect boundary, serial ILM, optional protocol/rio-v2, and real io_uring jobs are skipped because their sources and test inputs are unchanged. Manifests, fixtures, shared nextest configuration, scripts, production sources, and unknown paths still select `full`. A dependency on `e2e_test` from another workspace member or reachable local dependency disables this shortcut. All non-PR events retain their full selection. +Alternate-feature unit tests and Clippy for rio-v2, Swift, SFTP, and HTTP/3 run on main pushes, merge groups, and scheduled/manual validation. PRs run the default workspace and retain ILM, S3, real io_uring, and Connect security boundaries in their applicable scopes. The aggregate requires the alternate-feature jobs to be skipped on PRs and successful on non-PR events. This reduces repeated builds across PR revisions; feature-specific regressions can be detected after merging when the merge queue is not used. For changes to these features, dispatch `ci.yml` against the candidate branch before merging when that signal is needed, including PRs targeting `gateway/integration`, which has no automatic push trigger in this workflow. Scheduled results supplement the main baseline and do not replace it. + `required-checks` runs even after failed or skipped dependencies. `scripts/ci_gate.py verify` rejects missing jobs, unexpected jobs, failure, cancellation, and unexpected skips; optional lanes are required only on their declared events. `Workspace Test and Lint` is the ordinary Rust job, while `Test and Lint` uniquely names the aggregate. New validation jobs must update both its direct dependencies and the script contract. Test this wiring and its failure cases with `python3 scripts/ci_gate.py --self-test`. The `gateway/integration` ruleset (`24663286`) requires the same three contexts with the same review rules for the s3s to gateway integration branch (rustfs/backlog#2734). It omits `required_linear_history` because `gateway-integration-sync.yml` lands `--no-ff` merge commits from `main` on that branch every weekday, and it names the `rustfs-cla-assistant` app as the bypass actor that pushes them. `ci.yml` runs on pull requests targeting either branch. @@ -48,7 +50,7 @@ The four site-replication state-writer concurrency proofs reserve the available | PR, non-doc change | `Workspace Test and Lint` | `ci.yml` `test-and-lint` | Via aggregate | `cargo clippy --all-targets -- -D warnings`; `cargo nextest run --profile ci --all --exclude e2e_test`; `cargo test --all --doc`; migration evidence check described below | | PR, non-doc change | `Typos` | `ci.yml` `typos` | Via aggregate | `typos` | | PR, full selection | `ILM Integration (serial)` | `ci.yml` `test-ilm-integration-serial` | Via aggregate | exact command in the job | -| PR, full selection | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate | `cargo nextest run` with the job's `--features` | +| Main push, merge group, schedule, manual dispatch | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)`, `Test and Lint (http3)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate on these events; skipped on PRs | `cargo nextest run` with the job's `--features` | | PR, full selection | `Connect Short Credential Boundary` | `ci.yml` `connect-short-credential-boundary` | Via aggregate | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `cargo check -p rustfs --release --features connect-e2e-short-credentials` must fail | | PR, full selection | `Offline Enrollment E2E Root Boundary` | `ci.yml` `offline-enrollment-e2e` | Via aggregate | `scripts/check_offline_enrollment_e2e.sh` | | PR, non-doc change | `Build RustFS Debug Binary` | `ci.yml` `build-rustfs-debug-binary` | Via aggregate; prerequisite for black-box jobs | `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks` (binary plus its `rustfs.e2e.json` sidecar) | @@ -61,7 +63,7 @@ The four site-replication state-writer concurrency proofs reserve the available | PR touching `paths` in `fuzz.yml` | `Build Fuzz Harness`, `Smoke / ` | `fuzz.yml` `fuzz-build`, `pr-fuzz-smoke` | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` | | PR touching `paths` in `windows-filesystem.yml` | `Rename Safety` | `windows-filesystem.yml` `rename-safety` | Report-only | the `cargo test -p rustfs-ecstore --lib ` commands in the job, on Windows | | PR touching `paths` in `coverage.yml` | `Workspace line coverage` | `coverage.yml` `coverage` | Report-only | `make coverage`; `python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json` | -| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from the previous release`, `Mixed-version rolling upgrade from the previous release`, `Bucket configuration survives the upgrade`, `Rollback reads current bucket metadata` | `e2e-upgrade.yml` `upgrade` matrix | Report-only | `python3 scripts/e2e_binary.py build`, then the job's `python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test` command with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) | +| PR touching `paths` in `e2e-upgrade.yml` | `Upgrade compatibility` | `e2e-upgrade.yml` `upgrade` | Report-only | `python3 scripts/e2e_binary.py build`, then the job's `python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test` commands with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) | | PR touching `paths` in `oidc-keycloak.yml` | `OIDC Keycloak live gate` | `oidc-keycloak.yml` `oidc-keycloak-live` | Report-only | `cargo build --locked -p rustfs --bin rustfs`, then `bash scripts/test/oidc_keycloak_live.sh ./target/debug/rustfs` | | PR touching `paths` in `targets-integration.yml` | `PostgreSQL, MySQL, AMQP, and NATS` | `targets-integration.yml` `targets-live` | Report-only | start the containers as in the job, export the `RUSTFS_TEST_*` DSNs, then the job's `cargo test --locked -p rustfs-targets --test -- --ignored --test-threads=1` commands | | PR, documentation-only selection | `Quick Checks`, `Typos`, `Test and Lint` | `ci.yml` `quick-checks`, `typos`, `required-checks` | Required directly or via aggregate | Quick Checks commands; `python3 scripts/ci_gate.py --self-test` | @@ -69,7 +71,7 @@ The four site-replication state-writer concurrency proofs reserve the available e2e filters live in `.config/nextest.toml`; extend a profile instead of adding a second selector. Before a profile runs, `scripts/check_test_wiring.py` compares its listing to the committed digest in `.config/e2e--selection.txt`, so a silent test drop fails closed. -Architecture migration rules run once in the required Quick Checks job. Migration proofs reuse the successful workspace run: `scripts/check_migration_gate_count.sh evidence ` checks the unchanged name selection and committed floor, and requires exactly one successful execution without retries for each proof. Missing, filtered, skipped, failed, or duplicate evidence fails the gate. Local `check` and `run` modes remain available. Upgrade compatibility builds one current server and shares its binary and provenance sidecar across the seven cases; each case independently verifies the pinned previous release checksum and runs its existing test. +Architecture migration rules run once in the required Quick Checks job. Migration proofs reuse the successful workspace run: `scripts/check_migration_gate_count.sh evidence ` checks the unchanged name selection and committed floor, and requires exactly one successful execution without retries for each proof. Missing, filtered, skipped, failed, or duplicate evidence fails the gate. Local `check` and `run` modes remain available. Upgrade compatibility builds one current server and shares its binary and provenance sidecar with one test runner. That runner verifies the pinned previous release checksum once, reuses the compiled harness, and executes every existing upgrade case in a separate process. Failed cases do not stop the remaining cases; any failure fails the job. Logs and G09 evidence retain per-case subdirectories in the aggregate artifacts. `scripts/test_upgrade_workflow.py` checks case membership and failure propagation. Scanner usage and heal rebuild coverage are intentionally split by risk and cost. `data_usage_test` runs in the PR `e2e-smoke` lane so changes that affect @@ -99,7 +101,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched | `mint.yml` (weekly) | `mint` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json` | yes | pinned Docker sequence in the workflow | | `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes | `make coverage` | | `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch | -| `e2e-upgrade.yml` (weekly) | `upgrade` (4-case matrix) | upgrade and rollback gate; server logs | no | see the PR row | +| `e2e-upgrade.yml` (weekly) | `upgrade` (serial cases sharing one harness build) | upgrade and rollback gate; server logs | no | see the PR row | | `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | no | see the PR row | | `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | no | see the PR row | | `gateway-integration-sync.yml` (weekdays) | `sync` | merges `main` into `gateway/integration` as a `--no-ff` merge commit; a conflicting merge is aborted and opens a `sync/-` pull request listing the conflicting files, which keeps the run green; a rejected push or missing pull request fails it | no | dispatch, optionally with `source_ref` | diff --git a/scripts/ci_gate.py b/scripts/ci_gate.py index 3979991f7..e8cf52e75 100644 --- a/scripts/ci_gate.py +++ b/scripts/ci_gate.py @@ -21,6 +21,9 @@ CODE_JOBS = ( "build-rustfs-debug-binary", "uring-integration", "e2e-tests", "s3-implemented-tests", "s3-lifecycle-behavior-tests", ) +# Full alternate-feature tests remain mandatory on main, merge groups, and +# scheduled/manual runs, without repeating four builds on every PR revision. +EXTENDED_FEATURE_JOBS = ("test-and-lint-rio-v2", "test-and-lint-protocols") # These jobs still exercise the server and its black-box harness when only E2E # Rust sources change. Production code, manifests and shared test configuration # always select the full matrix. No production package depends on e2e_test. @@ -127,6 +130,8 @@ def expected_results(mode: str, event: str, ref: str) -> dict[str, str]: raise ValueError(f"invalid CI selection: {mode!r} for {event!r}") expected = {job: "success" for job in ALWAYS_JOBS} expected.update({job: "success" if mode == "full" or (mode == "e2e" and job in E2E_JOBS) else "skipped" for job in CODE_JOBS}) + if event == "pull_request": + expected.update({job: "skipped" for job in EXTENDED_FEATURE_JOBS}) rio = mode == "full" and event in ("schedule", "workflow_dispatch") expected.update({job: "success" if rio else "skipped" for job in OPTIONAL_JOBS[:2]}) full = mode == "full" and (event in ("merge_group", "workflow_dispatch") or (event == "push" and ref == "refs/heads/main")) @@ -215,6 +220,10 @@ def check_workflow(root: Path) -> list[str]: errors.append("Test and Lint must uniquely name the aggregate gate") if scalar(gate, "if", 4) != "always() && (github.event_name != 'pull_request' || github.event.action != 'closed')": errors.append("required-checks must run after failed or skipped dependencies") + for job in EXTENDED_FEATURE_JOBS: + block = yaml_block(jobs, job, 2) or [] + if scalar(block, "if", 4) != "needs.classify-changes.outputs.mode == 'full' && github.event_name != 'pull_request'": + errors.append(f"{job} must retain full validation on every non-PR event") if scalar(gate, "shell", 8) != "bash" or scalar(gate, "run", 8) != "python3 scripts/ci_gate.py verify" or scalar(gate, "CI_NEEDS", 10) != "${{ toJSON(needs) }}": errors.append("required-checks must verify the actual needs results") if any(re.match(r'''\s+(?:- )?(?:["']?continue-on-error["']?\s*:|["']?if["']?\s*:)''', line) and not line.startswith(" if:") for line in gate): @@ -315,13 +324,14 @@ class SelfTests(unittest.TestCase): def test_event_contract_requires_complete_candidate_and_optional_lanes(self): ordinary = expected_results("full", "pull_request", "refs/pull/1/merge") - self.assertEqual({job for job, state in ordinary.items() if state == "skipped"}, set(OPTIONAL_JOBS)) + self.assertEqual({job for job, state in ordinary.items() if state == "skipped"}, set(OPTIONAL_JOBS + EXTENDED_FEATURE_JOBS)) docs = expected_results("docs", "pull_request", "refs/pull/1/merge") self.assertEqual({job for job, state in docs.items() if state == "success"}, set(ALWAYS_JOBS)) e2e = expected_results("e2e", "pull_request", "refs/pull/1/merge") self.assertEqual({job for job, state in e2e.items() if state == "success"}, set(ALWAYS_JOBS + E2E_JOBS)) for event in ("schedule", "workflow_dispatch", "merge_group", "push"): result = expected_results("full", event, "refs/heads/main") + self.assertTrue(all(result[job] == "success" for job in CODE_JOBS)) self.assertEqual(result["e2e-full"], "skipped" if event == "schedule" else "success") self.assertEqual(result["e2e-tests-rio-v2"], "success" if event in ("schedule", "workflow_dispatch") else "skipped") with self.assertRaises(ValueError): @@ -330,7 +340,7 @@ class SelfTests(unittest.TestCase): expected_results("e2e", event, "refs/heads/main") def test_every_wrong_result_missing_job_or_selection_fails_closed(self): - for mode, event in (("full", "pull_request"), ("docs", "pull_request"), ("e2e", "pull_request"), ("full", "schedule"), ("full", "workflow_dispatch"), ("full", "merge_group")): + for mode, event in (("full", "pull_request"), ("docs", "pull_request"), ("e2e", "pull_request"), ("full", "push"), ("full", "schedule"), ("full", "workflow_dispatch"), ("full", "merge_group")): good = {job: {"result": value} for job, value in expected_results(mode, event, "refs/heads/main").items()} good["classify-changes"]["outputs"] = {"mode": mode} self.assertEqual(verify_results(good, event, "refs/heads/main"), []) @@ -377,6 +387,12 @@ class SelfTests(unittest.TestCase): (root / ".github/workflows").mkdir(parents=True) source = (ROOT / ".github/workflows/ci.yml").read_text() path = root / ".github/workflows/ci.yml" + for job in EXTENDED_FEATURE_JOBS: + before, block = source.split(f" {job}:\n", 1) + for condition in ("false", "github.event_name == 'schedule'", "github.event_name != 'pull_request'"): + changed = re.sub(r"^ if: .*", f" if: {condition}", block, count=1, flags=re.MULTILINE) + path.write_text(before + f" {job}:\n" + changed) + self.assertIn(f"{job} must retain full validation on every non-PR event", check_workflow(root)) for job in ALWAYS_JOBS + CODE_JOBS + OPTIONAL_JOBS: before, gate = source.split(" required-checks:\n", 1) path.write_text(before + " required-checks:\n" + gate.replace(f" - {job}\n", "", 1)) diff --git a/scripts/test_upgrade_workflow.py b/scripts/test_upgrade_workflow.py new file mode 100644 index 000000000..46a89c121 --- /dev/null +++ b/scripts/test_upgrade_workflow.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Exercise the consolidated upgrade lane without building RustFS.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import unittest + +from check_test_wiring import yaml_block +from test_security_workflow import named_steps, shell_body + + +ROOT = Path(__file__).resolve().parents[1] + + +class UpgradeWorkflowTests(unittest.TestCase): + def setUp(self): + source = (ROOT / ".github/workflows/e2e-upgrade.yml").read_text().splitlines() + self.job = yaml_block(source, "upgrade", 2) + self.steps = named_steps(self.job) + self.step = self.steps["Run upgrade compatibility tests"] + # Explicit bash enables pipefail in Actions, so tee cannot hide a failure. + self.assertIn(" shell: bash", self.step) + self.assertNotIn(" strategy:", self.job) + rust = (ROOT / "crates/e2e_test/src/upgrade_compatibility_test.rs").read_text() + self.expected = set(re.findall(r'#\[ignore[^\n]*\]\s*async fn (\w+)\(', rust)) + self.assertTrue(self.expected) + + def run_cases(self, failed_test="", fail_tee=False): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + commands = root / "commands.jsonl" + fake_python = root / "python3" + fake_python.write_text(f"#!{sys.executable}\n" + '''import json, os, sys +from pathlib import Path +args = sys.argv[1:] +assert args[:8] == ["scripts/e2e_binary.py", "run", "--", "cargo", "test", "--locked", "-p", "e2e_test"] +assert args[9:] == ["--", "--ignored", "--exact", "--nocapture"] +test = args[8].removeprefix("upgrade_compatibility_test::") +assert args[8] != test +assert os.environ["no_proxy"] == "127.0.0.1,localhost,::1" +with open(os.environ["COMMANDS"], "a") as output: + output.write(json.dumps({"test": test, "logs": os.environ["RUSTFS_E2E_LOG_DIR"], + "evidence": os.environ["RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR"]}) + "\\n") +Path(os.environ["RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR"]).mkdir(parents=True) +print("executed " + test) +sys.exit(17 if test == os.environ["FAILED_TEST"] else 0) +''') + fake_python.chmod(0o755) + if fail_tee: + tee = root / "tee" + tee.write_text("#!/bin/sh\ncat >/dev/null\nexit 23\n") + tee.chmod(0o755) + env = dict(os.environ, PATH=f"{root}{os.pathsep}{os.environ['PATH']}", + RUNNER_TEMP=str(root), COMMANDS=str(commands), FAILED_TEST=failed_test, + NO_PROXY="127.0.0.1,localhost,::1") + result = subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", shell_body(self.step)], + cwd=root, env=env, capture_output=True, text=True) + calls = [json.loads(line) for line in commands.read_text().splitlines()] + self.assertEqual({call["test"] for call in calls}, self.expected) + self.assertEqual(len(calls), len(self.expected)) + for field in ("logs", "evidence"): + self.assertEqual(len({call[field] for call in calls}), len(calls)) + self.assertTrue(all(Path(call[field]).is_dir() for call in calls)) + if not fail_tee: + for call in calls: + self.assertIn(call["test"], (Path(call["logs"]) / "test.log").read_text()) + return result + + def test_all_ignored_upgrade_cases_run_once_with_separate_evidence(self): + result = self.run_cases() + self.assertEqual(result.returncode, 0, result.stderr) + + def test_early_or_late_failure_does_not_hide_remaining_cases(self): + for name in ("direct_upgrade_from_rc2_preserves_object_contracts", "rc5_baseline_replicates_multipart_layouts"): + with self.subTest(name=name): + result = self.run_cases(failed_test=name) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn(f"::error::Upgrade compatibility failed: {name}", result.stdout) + + def test_log_write_failure_fails_the_job(self): + self.assertNotEqual(self.run_cases(fail_tee=True).returncode, 0) + + def test_reports_are_uploaded_even_after_failure(self): + for name, directory in (("Upload scanner/heal G09 evidence", "rustfs-upgrade-g09-evidence"), + ("Upload server logs", "rustfs-upgrade-logs")): + step = self.steps[name] + self.assertIn(" if: always()", step) + self.assertIn(" path: ${{ runner.temp }}/" + directory, step) + + +if __name__ == "__main__": + unittest.main() From da42266bfb72d4ff040229893160a51aa4663244 Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Sun, 11 Oct 2026 13:27:46 +0800 Subject: [PATCH 008/143] refactor(iam): organize OIDC runtime modules (#8445) --- .github/workflows/oidc-keycloak.yml | 4 +- crates/iam/src/federation/mod.rs | 5 +- crates/iam/src/federation/oidc/claims.rs | 68 - crates/iam/src/federation/oidc/config.rs | 68 - crates/iam/src/federation/oidc/discovery.rs | 63 - crates/iam/src/federation/oidc/flow.rs | 109 - crates/iam/src/federation/oidc/http.rs | 57 - crates/iam/src/federation/oidc/mod.rs | 22 - crates/iam/src/federation/transaction.rs | 2 +- crates/iam/src/lib.rs | 1 - crates/iam/src/oidc.rs | 4710 ----------------- crates/iam/src/oidc/config.rs | 434 ++ crates/iam/src/oidc/config_tests.rs | 307 ++ crates/iam/src/oidc/mod.rs | 37 + crates/iam/src/oidc/provider.rs | 632 +++ crates/iam/src/oidc/provider_tests.rs | 712 +++ crates/iam/src/oidc/runtime.rs | 897 ++++ crates/iam/src/oidc/runtime_tests.rs | 754 +++ .../iam/src/{oidc_state.rs => oidc/state.rs} | 34 +- crates/iam/src/oidc/test_support.rs | 526 ++ crates/iam/src/oidc/transport.rs | 660 +++ crates/iam/src/oidc/transport_tests.rs | 167 + docs/operations/oidc-console-integration.md | 4 +- docs/operations/oidc-provider-requirements.md | 20 +- docs/operations/outbound-connection-policy.md | 2 +- rustfs/src/app/context.rs | 11 +- rustfs/src/app/context/global.rs | 6 +- rustfs/src/app/context/handles.rs | 13 +- rustfs/src/app/context/interfaces.rs | 6 +- rustfs/src/runtime_sources.rs | 9 +- rustfs/src/startup_auth.rs | 7 +- 31 files changed, 5176 insertions(+), 5171 deletions(-) delete mode 100644 crates/iam/src/federation/oidc/claims.rs delete mode 100644 crates/iam/src/federation/oidc/config.rs delete mode 100644 crates/iam/src/federation/oidc/discovery.rs delete mode 100644 crates/iam/src/federation/oidc/flow.rs delete mode 100644 crates/iam/src/federation/oidc/http.rs delete mode 100644 crates/iam/src/federation/oidc/mod.rs delete mode 100644 crates/iam/src/oidc.rs create mode 100644 crates/iam/src/oidc/config.rs create mode 100644 crates/iam/src/oidc/config_tests.rs create mode 100644 crates/iam/src/oidc/mod.rs create mode 100644 crates/iam/src/oidc/provider.rs create mode 100644 crates/iam/src/oidc/provider_tests.rs create mode 100644 crates/iam/src/oidc/runtime.rs create mode 100644 crates/iam/src/oidc/runtime_tests.rs rename crates/iam/src/{oidc_state.rs => oidc/state.rs} (92%) create mode 100644 crates/iam/src/oidc/test_support.rs create mode 100644 crates/iam/src/oidc/transport.rs create mode 100644 crates/iam/src/oidc/transport_tests.rs diff --git a/.github/workflows/oidc-keycloak.yml b/.github/workflows/oidc-keycloak.yml index 16b736521..f32287fb2 100644 --- a/.github/workflows/oidc-keycloak.yml +++ b/.github/workflows/oidc-keycloak.yml @@ -20,7 +20,7 @@ on: - ".github/workflows/oidc-keycloak.yml" - "crates/config/src/constants/oidc.rs" - "crates/iam/src/federation/**" - - "crates/iam/src/oidc.rs" + - "crates/iam/src/oidc/**" - "rustfs/src/admin/handlers/oidc.rs" - "rustfs/src/admin/handlers/sts.rs" - "scripts/test/oidc_keycloak_live.sh" @@ -31,7 +31,7 @@ on: - ".github/workflows/oidc-keycloak.yml" - "crates/config/src/constants/oidc.rs" - "crates/iam/src/federation/**" - - "crates/iam/src/oidc.rs" + - "crates/iam/src/oidc/**" - "rustfs/src/admin/handlers/oidc.rs" - "rustfs/src/admin/handlers/sts.rs" - "scripts/test/oidc_keycloak_live.sh" diff --git a/crates/iam/src/federation/mod.rs b/crates/iam/src/federation/mod.rs index dc1d930f5..b7498cf0a 100644 --- a/crates/iam/src/federation/mod.rs +++ b/crates/iam/src/federation/mod.rs @@ -16,14 +16,15 @@ mod binding; mod error; mod mapper; mod model; -pub mod oidc; mod provider; mod transaction; pub use binding::FederatedSessionBinding; pub use error::{FederatedSessionBindingError, FederationError, Result}; +pub(crate) use mapper::FederatedAuthorizationRule; +#[cfg(test)] +pub(crate) use mapper::FederatedAuthorizationRuleRef; pub use mapper::{CoreFederatedAuthorizationMapper, FederatedAuthorizationRules}; -pub(crate) use mapper::{FederatedAuthorizationRule, FederatedAuthorizationRuleRef}; pub(crate) use model::OpaqueLogoutContinuation; pub use model::{ FederatedAuthorization, FederatedClaims, FederatedLoginSession, FederatedProviderRef, FederatedSession, diff --git a/crates/iam/src/federation/oidc/claims.rs b/crates/iam/src/federation/oidc/claims.rs deleted file mode 100644 index 210bef001..000000000 --- a/crates/iam/src/federation/oidc/claims.rs +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::{ - federation::{FederatedClaims, FederatedProviderRef, VerifiedFederatedIdentity}, - oidc::OidcClaims, -}; - -pub(super) fn verified_identity(provider: FederatedProviderRef, claims: OidcClaims) -> VerifiedFederatedIdentity { - VerifiedFederatedIdentity::from_claims( - provider, - FederatedClaims { - sub: claims.sub, - email: claims.email, - username: claims.username, - groups: claims.groups, - raw: claims.raw, - }, - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - use std::collections::HashMap; - - #[test] - fn verified_identity_preserves_provider_and_claim_values() { - let raw = HashMap::from([ - ("iss".to_string(), json!("https://corp.example.test")), - ("department".to_string(), json!("engineering")), - ("roles".to_string(), json!(["reader", "admin"])), - ]); - let identity = verified_identity( - FederatedProviderRef::new("corp".to_string()), - OidcClaims { - sub: " subject-123 ".to_string(), - email: " user@example.test ".to_string(), - username: " user ".to_string(), - groups: vec![ - "source-ops".to_string(), - "source-developers".to_string(), - "source-ops".to_string(), - ], - raw: raw.clone(), - }, - ); - - assert_eq!(identity.provider().as_str(), "corp"); - assert_eq!(identity.subject(), " subject-123 "); - assert_eq!(identity.email(), " user@example.test "); - assert_eq!(identity.username(), " user "); - assert_eq!(identity.source_groups(), ["source-ops", "source-developers", "source-ops"]); - assert_eq!(identity.attributes(), &raw); - } -} diff --git a/crates/iam/src/federation/oidc/config.rs b/crates/iam/src/federation/oidc/config.rs deleted file mode 100644 index f2374c251..000000000 --- a/crates/iam/src/federation/oidc/config.rs +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::{ - federation::{FederatedAuthorizationRule, FederatedAuthorizationRules, FederatedRedirectPolicy}, - oidc::{OidcConfigSnapshot, OidcSys}, -}; - -pub(crate) fn authorization_rules(oidc: &OidcSys) -> FederatedAuthorizationRules { - FederatedAuthorizationRules::new(oidc.provider_configs().map(|config| { - FederatedAuthorizationRule::new( - config.id.clone(), - config.claim_name.clone(), - config.claim_prefix.clone(), - config.role_policy.clone(), - config.groups_claim.clone(), - config.roles_claim.clone(), - ) - })) -} - -/// Read-only access to the active standard OIDC configuration. -/// -/// This interface stays separate from authentication and generic provider -/// views because its snapshots include OIDC-specific fields and secrets. -pub trait OidcConfigQuery: Send + Sync { - fn config_snapshot(&self) -> OidcConfigSnapshot; -} - -pub(super) fn redirect_policy(oidc: &OidcSys, provider_id: &str) -> Option { - oidc.get_provider_config(provider_id).map(|config| FederatedRedirectPolicy { - redirect_uri: config.redirect_uri.clone(), - allow_request_origin: config.redirect_uri_dynamic, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn redirect_policy_keeps_configured_uri_and_dynamic_setting() { - let mut config = crate::oidc::test_config("default"); - config.redirect_uri = Some("https://console.example.test/callback".to_string()); - config.redirect_uri_dynamic = false; - let oidc = crate::oidc::make_test_sys(vec![config]); - - assert_eq!( - redirect_policy(&oidc, "default"), - Some(FederatedRedirectPolicy { - redirect_uri: Some("https://console.example.test/callback".to_string()), - allow_request_origin: false, - }) - ); - assert_eq!(redirect_policy(&oidc, "missing"), None); - } -} diff --git a/crates/iam/src/federation/oidc/discovery.rs b/crates/iam/src/federation/oidc/discovery.rs deleted file mode 100644 index dd50fe3ac..000000000 --- a/crates/iam/src/federation/oidc/discovery.rs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::{federation::FederatedProviderView, oidc::OidcSys}; - -fn provider_view(provider: &crate::oidc::OidcProviderConfig) -> FederatedProviderView { - FederatedProviderView { - provider_id: provider.id.clone(), - display_name: provider.display_name.clone(), - } -} - -pub(super) fn list_providers(oidc: &OidcSys) -> Vec { - oidc.provider_configs().map(provider_view).collect() -} - -pub(super) fn list_visible_providers(oidc: &OidcSys) -> Vec { - oidc.provider_configs() - .filter(|provider| !provider.hide_from_ui) - .map(provider_view) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn provider_views_preserve_names_and_visibility() { - let mut visible = crate::oidc::test_config("default"); - visible.display_name = "Company SSO".to_string(); - let mut hidden = crate::oidc::test_config("workload"); - hidden.hide_from_ui = true; - let oidc = crate::oidc::make_test_sys(vec![visible, hidden]); - - let all = list_providers(&oidc); - assert_eq!(all.len(), 2); - assert!( - all.iter() - .any(|provider| { provider.provider_id == "default" && provider.display_name == "Company SSO" }) - ); - assert!(all.iter().any(|provider| provider.provider_id == "workload")); - - assert_eq!( - list_visible_providers(&oidc), - vec![FederatedProviderView { - provider_id: "default".to_string(), - display_name: "Company SSO".to_string(), - }] - ); - } -} diff --git a/crates/iam/src/federation/oidc/flow.rs b/crates/iam/src/federation/oidc/flow.rs deleted file mode 100644 index e338590e7..000000000 --- a/crates/iam/src/federation/oidc/flow.rs +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::config::OidcConfigQuery; -use super::{claims, config, discovery, http}; -use crate::{ - federation::{ - CoreFederatedAuthorizationMapper, FederatedAuthorizationRules, FederatedIdentityService, FederatedProviderQuery, - FederatedProviderRef, FederatedProviderView, FederatedRedirectPolicy, FederationError, OpaqueLogoutContinuation, Result, - StandardOidcAuthentication, VerifiedFederatedCodeExchange, - }, - oidc::{OidcConfigSnapshot, OidcSys}, -}; -use std::sync::Arc; - -pub struct StandardOidcAdapter { - oidc: Arc, - authorization_rules: FederatedAuthorizationRules, -} - -impl StandardOidcAdapter { - pub fn new(oidc: Arc) -> Self { - let authorization_rules = config::authorization_rules(&oidc); - Self { - oidc, - authorization_rules, - } - } - - pub fn authorization_rules(&self) -> FederatedAuthorizationRules { - self.authorization_rules.clone() - } - - pub fn into_service(self: Arc, mapper: CoreFederatedAuthorizationMapper) -> FederatedIdentityService { - let authentication: Arc = self.clone(); - let provider_query: Arc = self; - FederatedIdentityService::from_standard_oidc_parts(provider_query, authentication, mapper) - } -} - -impl OidcConfigQuery for StandardOidcAdapter { - fn config_snapshot(&self) -> OidcConfigSnapshot { - self.oidc.config_snapshot() - } -} - -#[async_trait::async_trait] -impl StandardOidcAuthentication for StandardOidcAdapter { - async fn authorize_url(&self, provider_id: &str, redirect_uri: &str, redirect_after: Option) -> Result { - http::authorize_url(&self.oidc, provider_id, redirect_uri, redirect_after).await - } - - async fn exchange_identity(&self, state: &str, code: &str, redirect_uri: &str) -> Result { - let (oidc_claims, provider_id, session, id_token) = http::exchange_code(&self.oidc, state, code, redirect_uri).await?; - let provider = FederatedProviderRef::new(provider_id); - Ok(VerifiedFederatedCodeExchange::new( - claims::verified_identity(provider.clone(), oidc_claims), - session.redirect_after, - OpaqueLogoutContinuation::new(provider, id_token), - )) - } - - async fn verify_identity(&self, jwt: &str) -> Result { - let (oidc_claims, provider_id) = self - .oidc - .verify_web_identity_token(jwt) - .await - .map_err(FederationError::TokenVerification)?; - Ok(claims::verified_identity(FederatedProviderRef::new(provider_id), oidc_claims)) - } - - async fn create_logout_token(&self, continuation: OpaqueLogoutContinuation) -> Result { - let (provider, id_token) = continuation.into_parts(); - http::create_logout_token(&self.oidc, provider.as_str(), &id_token).await - } - - async fn build_logout_url(&self, logout_token: &str, post_logout_redirect_uri: &str) -> Result> { - http::build_logout_url(&self.oidc, logout_token, post_logout_redirect_uri).await - } -} - -impl FederatedProviderQuery for StandardOidcAdapter { - fn has_providers(&self) -> bool { - self.oidc.has_providers() - } - - fn list_providers(&self) -> Vec { - discovery::list_providers(&self.oidc) - } - - fn list_visible_providers(&self) -> Vec { - discovery::list_visible_providers(&self.oidc) - } - - fn redirect_policy(&self, provider_id: &str) -> Option { - config::redirect_policy(&self.oidc, provider_id) - } -} diff --git a/crates/iam/src/federation/oidc/http.rs b/crates/iam/src/federation/oidc/http.rs deleted file mode 100644 index 3b4f0dcd0..000000000 --- a/crates/iam/src/federation/oidc/http.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::{ - federation::{FederationError, Result}, - oidc::{OidcClaims, OidcSys}, - oidc_state::OidcAuthSession, -}; - -pub(super) async fn authorize_url( - oidc: &OidcSys, - provider_id: &str, - redirect_uri: &str, - redirect_after: Option, -) -> Result { - oidc.authorize_url(provider_id, redirect_uri, redirect_after) - .await - .map_err(FederationError::Authorization) -} - -pub(super) async fn exchange_code( - oidc: &OidcSys, - state: &str, - code: &str, - redirect_uri: &str, -) -> Result<(OidcClaims, String, OidcAuthSession, String)> { - oidc.exchange_code(state, code, redirect_uri) - .await - .map_err(FederationError::CodeExchange) -} - -pub(super) async fn create_logout_token(oidc: &OidcSys, provider_id: &str, id_token: &str) -> Result { - oidc.create_logout_token(provider_id, id_token) - .await - .map_err(FederationError::Logout) -} - -pub(super) async fn build_logout_url( - oidc: &OidcSys, - logout_token: &str, - post_logout_redirect_uri: &str, -) -> Result> { - oidc.build_logout_url(logout_token, post_logout_redirect_uri) - .await - .map_err(FederationError::Logout) -} diff --git a/crates/iam/src/federation/oidc/mod.rs b/crates/iam/src/federation/oidc/mod.rs deleted file mode 100644 index 52e6b6432..000000000 --- a/crates/iam/src/federation/oidc/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod claims; -mod config; -mod discovery; -mod flow; -mod http; - -pub use config::OidcConfigQuery; -pub use flow::StandardOidcAdapter; diff --git a/crates/iam/src/federation/transaction.rs b/crates/iam/src/federation/transaction.rs index b9bda054d..3f7ec15dc 100644 --- a/crates/iam/src/federation/transaction.rs +++ b/crates/iam/src/federation/transaction.rs @@ -61,7 +61,7 @@ impl FederatedIdentityService { self.authentication.create_logout_token(continuation).await } - pub(super) fn from_standard_oidc_parts( + pub(crate) fn from_standard_oidc_parts( provider_query: Arc, authentication: Arc, mapper: CoreFederatedAuthorizationMapper, diff --git a/crates/iam/src/lib.rs b/crates/iam/src/lib.rs index ab6f28bfd..2f139630c 100644 --- a/crates/iam/src/lib.rs +++ b/crates/iam/src/lib.rs @@ -34,7 +34,6 @@ pub mod keyring; pub mod manager; pub mod mfa; pub mod oidc; -pub mod oidc_state; mod root_credentials; mod runtime_sources; mod server_config; diff --git a/crates/iam/src/oidc.rs b/crates/iam/src/oidc.rs deleted file mode 100644 index 7a4adaa2f..000000000 --- a/crates/iam/src/oidc.rs +++ /dev/null @@ -1,4710 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! OIDC Provider Manager -//! -//! Implements the OpenID Connect Authorization Code Flow with PKCE using the -//! `openidconnect` crate for standards-compliant discovery, token exchange, -//! and ID token verification. - -use crate::{ - EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC, - federation::{CoreFederatedAuthorizationMapper, FederatedAuthorizationRuleRef}, - oidc_state::{OidcAuthSession, OidcLogoutSession, OidcStateStore}, -}; -use openidconnect::core::{ - CoreAuthenticationFlow, CoreClient, CoreIdToken, CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm, -}; -use openidconnect::{ - AsyncHttpClient, Audience, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, DiscoveryError, IssuerUrl, - JsonWebKeySetUrl, LogoutRequest, Nonce, PkceCodeChallenge, PkceCodeVerifier, PostLogoutRedirectUrl, - ProviderMetadataWithLogout, RedirectUrl, RequestTokenError, Scope, TokenUrl, -}; -use reqwest::{Certificate, Client}; -use rustfs_config::oidc::*; -use rustfs_config::server_config::{Config as ServerConfig, KVS}; -use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_OIDC_RESPONSE_SIZE}; -use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive}; -use rustfs_utils::egress::{ENV_OUTBOUND_ALLOW_ORIGINS, OutboundPolicy, find_outbound_dns_policy_rejection}; -use serde::{Deserialize, Serialize}; -use std::borrow::Cow; -use std::collections::{HashMap, VecDeque}; -use std::fmt; -use std::future::Future; -use std::net::IpAddr; -use std::pin::Pin; -use std::sync::Arc; -use std::sync::{LazyLock, Mutex, MutexGuard, RwLock}; -use std::time::{Duration as StdDuration, Instant}; -use tokio::time::sleep; -use tracing::{debug, error, warn}; -use url::Url; - -const EVENT_OIDC_HTTP: &str = "oidc_http"; -const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60); -const OIDC_DISCOVERY_TRANSPORT_RETRIES: usize = 3; -const OIDC_DISCOVERY_TRANSPORT_RETRY_DELAY: StdDuration = StdDuration::from_millis(50); -const OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY: &str = "JWKS request blocked by outbound policy"; -const OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY: &str = "OIDC provider discovery blocked by outbound policy"; -const OIDC_HTTP_REQUEST_TIMEOUT: StdDuration = StdDuration::from_secs(10); -const OIDC_HTTP_CONNECT_TIMEOUT: StdDuration = StdDuration::from_secs(3); -const OIDC_PLUGIN_AUTHN_WINDOW: StdDuration = StdDuration::from_secs(60); - -#[derive(Debug, Clone, Copy, Default)] -pub struct OidcPluginAuthnMetricsSnapshot { - pub failed_requests_minute: u64, - pub last_fail_seconds: u64, - pub last_succ_seconds: u64, - pub succ_avg_rtt_ms_minute: u64, - pub succ_max_rtt_ms_minute: u64, - pub total_requests_minute: u64, -} - -#[derive(Debug, Clone)] -struct OidcPluginAuthnSample { - observed_at: Instant, - succeeded: bool, - rtt_ms: u64, -} - -#[derive(Debug, Default)] -struct OidcPluginAuthnMetrics { - samples: Mutex>, - last_fail_at: Mutex>, - last_succ_at: Mutex>, -} - -fn lock_oidc_plugin_authn_metrics<'a, T>(mutex: &'a Mutex, metric: &'static str) -> MutexGuard<'a, T> { - match mutex.lock() { - Ok(guard) => guard, - Err(err) => { - warn!(metric, "Recovering poisoned OIDC authn metrics lock"); - err.into_inner() - } - } -} - -fn seconds_since(now: Instant, observed_at: Option) -> u64 { - observed_at - .map(|instant| now.duration_since(instant).as_secs()) - .unwrap_or_default() -} - -impl OidcPluginAuthnMetrics { - fn record(&self, rtt_ms: u64, succeeded: bool) { - let now = Instant::now(); - let mut samples = lock_oidc_plugin_authn_metrics(&self.samples, "samples"); - samples.push_back(OidcPluginAuthnSample { - observed_at: now, - succeeded, - rtt_ms, - }); - while samples - .front() - .is_some_and(|sample| now.duration_since(sample.observed_at) > OIDC_PLUGIN_AUTHN_WINDOW) - { - samples.pop_front(); - } - drop(samples); - - if succeeded { - *lock_oidc_plugin_authn_metrics(&self.last_succ_at, "last_succ_at") = Some(now); - } else { - *lock_oidc_plugin_authn_metrics(&self.last_fail_at, "last_fail_at") = Some(now); - } - } - - fn snapshot(&self) -> OidcPluginAuthnMetricsSnapshot { - let now = Instant::now(); - let (total_requests_minute, failed_requests_minute, succ_avg_rtt_ms_minute, succ_max_rtt_ms_minute) = { - let mut samples = lock_oidc_plugin_authn_metrics(&self.samples, "samples"); - while samples - .front() - .is_some_and(|sample| now.duration_since(sample.observed_at) > OIDC_PLUGIN_AUTHN_WINDOW) - { - samples.pop_front(); - } - - let mut failed_requests_minute = 0u64; - let mut successful_requests = 0u64; - let mut successful_rtt_sum = 0u64; - let mut succ_max_rtt_ms_minute = 0u64; - - for sample in samples.iter() { - if sample.succeeded { - successful_requests += 1; - successful_rtt_sum += sample.rtt_ms; - succ_max_rtt_ms_minute = succ_max_rtt_ms_minute.max(sample.rtt_ms); - } else { - failed_requests_minute += 1; - } - } - - let succ_avg_rtt_ms_minute = successful_rtt_sum.checked_div(successful_requests).unwrap_or_default(); - - ( - samples.len() as u64, - failed_requests_minute, - succ_avg_rtt_ms_minute, - succ_max_rtt_ms_minute, - ) - }; - - let last_fail_seconds = seconds_since(now, *lock_oidc_plugin_authn_metrics(&self.last_fail_at, "last_fail_at")); - let last_succ_seconds = seconds_since(now, *lock_oidc_plugin_authn_metrics(&self.last_succ_at, "last_succ_at")); - - OidcPluginAuthnMetricsSnapshot { - failed_requests_minute, - last_fail_seconds, - last_succ_seconds, - succ_avg_rtt_ms_minute, - succ_max_rtt_ms_minute, - total_requests_minute, - } - } -} - -static OIDC_PLUGIN_AUTHN_METRICS: LazyLock = LazyLock::new(OidcPluginAuthnMetrics::default); - -pub fn oidc_plugin_authn_metrics_snapshot() -> OidcPluginAuthnMetricsSnapshot { - OIDC_PLUGIN_AUTHN_METRICS.snapshot() -} - -/// Header names whose values may carry OIDC secrets (client credentials, cookies, -/// bearer tokens). Their values are never emitted to logs, only their byte length. -const SENSITIVE_HEADER_NAMES: [&str; 4] = ["authorization", "proxy-authorization", "cookie", "set-cookie"]; - -fn is_sensitive_header(name: &str) -> bool { - SENSITIVE_HEADER_NAMES - .iter() - .any(|candidate| name.eq_ignore_ascii_case(candidate)) -} - -fn format_http_headers(headers: &http::HeaderMap) -> String { - headers - .iter() - .map(|(name, value)| { - if is_sensitive_header(name.as_str()) { - format!("{}=", name.as_str(), value.as_bytes().len()) - } else { - let value = value.to_str().unwrap_or(""); - format!("{}={}", name.as_str(), value) - } - }) - .collect::>() - .join("; ") -} - -#[derive(Debug, Default)] -struct TokenResponseBodyShape { - json_object: bool, - json_keys: String, - has_access_token: bool, - has_id_token: bool, - has_token_type: bool, - has_expires_in: bool, - has_error: bool, - has_error_description: bool, - looks_like_html: bool, -} - -fn inspect_token_response_body(body: &[u8]) -> TokenResponseBodyShape { - let mut shape = TokenResponseBodyShape { - looks_like_html: body - .iter() - .copied() - .find(|byte| !byte.is_ascii_whitespace()) - .is_some_and(|byte| byte == b'<'), - ..Default::default() - }; - - let Ok(value) = serde_json::from_slice::(body) else { - return shape; - }; - let Some(object) = value.as_object() else { - return shape; - }; - - shape.json_object = true; - shape.has_access_token = object.contains_key("access_token"); - shape.has_id_token = object.contains_key("id_token"); - shape.has_token_type = object.contains_key("token_type"); - shape.has_expires_in = object.contains_key("expires_in"); - shape.has_error = object.contains_key("error"); - shape.has_error_description = object.contains_key("error_description"); - - let mut keys: Vec<&str> = object.keys().map(String::as_str).collect(); - keys.sort_unstable(); - keys.truncate(16); - shape.json_keys = keys.join(","); - - shape -} - -fn oidc_http_error_diagnostics(error: &OidcHttpError) -> (&'static str, String) { - match error { - OidcHttpError::Reqwest(err) if err.is_timeout() => ("timeout", String::new()), - OidcHttpError::Reqwest(err) if err.is_connect() => ("connect", String::new()), - OidcHttpError::Reqwest(err) if err.status().is_some() => { - ("http_status", err.status().map(|status| status.as_u16().to_string()).unwrap_or_default()) - } - OidcHttpError::Reqwest(_) => ("request", String::new()), - OidcHttpError::Http(_) => ("http_build", String::new()), - OidcHttpError::ExtraRootCa(_) => ("extra_root_ca", String::new()), - OidcHttpError::ForbiddenOutbound(_) => ("forbidden_outbound", String::new()), - OidcHttpError::ResponseTooLarge(limit) => ("response_too_large", limit.to_string()), - } -} - -// ---- HTTP Client Adapter ---- - -/// Error type for the OIDC HTTP client adapter. -#[derive(Debug)] -pub enum OidcHttpError { - Reqwest(reqwest::Error), - Http(http::Error), - ExtraRootCa(String), - /// The outbound destination was rejected by the shared egress policy before any - /// connection was attempted (invalid URL, loopback/link-local/metadata/private IP, - /// or a malformed allow-origins configuration). - ForbiddenOutbound(String), - /// The provider response body exceeded [`MAX_OIDC_RESPONSE_SIZE`] and was abandoned - /// instead of being buffered in full. - ResponseTooLarge(usize), -} - -impl std::fmt::Display for OidcHttpError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Reqwest(e) => write!(f, "{e}"), - Self::Http(e) => write!(f, "{e}"), - Self::ExtraRootCa(reason) => write!(f, "failed to load OIDC extra root CA bundle: {reason}"), - Self::ForbiddenOutbound(reason) => write!(f, "outbound request rejected: {reason}"), - Self::ResponseTooLarge(limit) => write!(f, "oidc response body exceeds {limit} bytes"), - } - } -} - -impl std::error::Error for OidcHttpError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - Self::Reqwest(e) => Some(e), - Self::Http(e) => Some(e), - Self::ExtraRootCa(_) | Self::ForbiddenOutbound(_) | Self::ResponseTooLarge(_) => None, - } - } -} - -#[derive(Clone, Debug, Default)] -pub struct OidcExtraRootCaMaterial { - pub generation: u64, - pub root_ca_pem: Option>, -} - -type OidcExtraRootCaFuture = Pin> + Send>>; -type OidcExtraRootCaLoader = dyn Fn() -> OidcExtraRootCaFuture + Send + Sync; - -#[derive(Clone)] -pub struct OidcExtraRootCaProvider { - loader: Arc, -} - -impl OidcExtraRootCaProvider { - pub fn new(loader: F) -> Self - where - F: Fn() -> Fut + Send + Sync + 'static, - Fut: Future> + Send + 'static, - { - Self { - loader: Arc::new(move || Box::pin(loader())), - } - } - - async fn load(&self) -> Result { - (self.loader)().await - } -} - -#[derive(Clone, Default)] -struct CachedOidcExtraRootCerts { - generation: u64, - initialized: bool, - certs: Vec, -} - -/// HTTP client adapter bridging reqwest 0.13 to the `openidconnect` `AsyncHttpClient` trait. -/// -/// A fresh client is built for every request so the destination is re-validated and the -/// resolved IP is re-classified at connection time. This closes the SSRF / DNS-rebinding -/// gap where a one-shot URL string check is bypassed by a hostname that resolves to an -/// internal address only at connection time, and it also covers endpoints discovered from -/// the provider metadata (JWKS, token) rather than only the operator-configured `config_url`. -pub(crate) struct ReqwestHttpClient { - /// `None` in production: the process-cached outbound policy from the environment is used. - /// `Some(..)` only in tests, to explicitly allow a loopback mock endpoint. - policy_override: Option, - extra_root_certs: Arc>, - extra_root_ca_provider: Option, - #[cfg(test)] - dns_resolver_override: Option>, -} - -fn parse_oidc_extra_root_certs(source: &str, pem: &[u8]) -> Result, String> { - if pem.iter().all(|byte| byte.is_ascii_whitespace()) { - return Ok(Vec::new()); - } - Certificate::from_pem_bundle(pem).map_err(|err| format!("failed to parse OIDC extra root CA bundle from {source}: {err}")) -} - -fn oidc_extra_root_certs(root_ca_pem: Option<&[u8]>) -> Result, String> { - match root_ca_pem { - Some(pem) => parse_oidc_extra_root_certs("RustFS outbound TLS material", pem), - None => Ok(Vec::new()), - } -} - -/// Build a reqwest client pinned to the shared outbound egress policy for a single request. -/// -/// [`OutboundPolicy::resolver_for`] validates the URL shape and rejects loopback, -/// link-local, metadata, multicast and unauthorized private addresses up front, and the -/// returned `OutboundDnsResolver` re-resolves and re-classifies the host on every new -/// connection so DNS rebinding fails closed. Redirects are not followed: a redirect target -/// would otherwise skip URL-shape re-validation. The timeouts bound how long a slow or -/// stalled provider can pin the calling task. -fn build_oidc_http_client( - uri: &str, - policy_override: Option<&OutboundPolicy>, - extra_root_certs: &[Certificate], - #[cfg(test)] dns_resolver_override: Option>, -) -> Result<(Client, Url), OidcHttpError> { - let url = Url::parse(uri).map_err(|_| OidcHttpError::ForbiddenOutbound("invalid outbound OIDC URL".to_string()))?; - let resolver = match policy_override { - Some(policy) => policy.resolver_for(&url), - None => OutboundPolicy::from_env_cached() - .map_err(|err| OidcHttpError::ForbiddenOutbound(err.to_string()))? - .resolver_for(&url), - } - .map_err(|err| { - let base = err.to_string(); - let origin = url.origin().ascii_serialization(); - let can_allow_origin = - OutboundPolicy::from_allowed_origins(&origin).is_ok_and(|allowlisted| allowlisted.validate_url(&url).is_ok()); - oidc_forbidden_outbound_error(&url, base, can_allow_origin) - })?; - let bypass_proxy = should_bypass_proxy_for_oidc_uri(uri); - #[cfg(test)] - let bypass_proxy = bypass_proxy || dns_resolver_override.is_some(); - #[cfg(test)] - let resolver: Arc = dns_resolver_override.unwrap_or_else(|| Arc::new(resolver)); - - let mut builder = reqwest::Client::builder() - .dns_resolver(resolver) - .redirect(reqwest::redirect::Policy::none()) - .timeout(OIDC_HTTP_REQUEST_TIMEOUT) - .connect_timeout(OIDC_HTTP_CONNECT_TIMEOUT); - if bypass_proxy { - builder = builder.no_proxy(); - } - if !extra_root_certs.is_empty() { - builder = builder.tls_certs_merge(extra_root_certs.iter().cloned()); - } - builder.build().map(|client| (client, url)).map_err(OidcHttpError::Reqwest) -} - -fn oidc_forbidden_outbound_error(url: &Url, base: String, can_allow_origin: bool) -> OidcHttpError { - let reason = if can_allow_origin { - let origin = url.origin().ascii_serialization(); - format!( - "{base}; add {origin} to {ENV_OUTBOUND_ALLOW_ORIGINS} (comma-separated) and restart RustFS to allow this operator-owned OIDC provider (origin only, no path)" - ) - } else { - base - }; - OidcHttpError::ForbiddenOutbound(reason) -} - -fn oidc_http_error_from_reqwest(url: &Url, error: reqwest::Error) -> OidcHttpError { - if let Some(rejection) = find_outbound_dns_policy_rejection(&error) { - let base = rejection.to_string(); - return oidc_forbidden_outbound_error(url, base, rejection.allow_origin_can_recover()); - } - OidcHttpError::Reqwest(error) -} - -/// Buffer a provider response body, failing closed once `limit` bytes have been seen. -/// -/// `Response::bytes` would buffer the whole body unconditionally, so a hostile or compromised -/// provider endpoint could stream an arbitrarily large (or endless) body into memory. -async fn read_bounded_response_body(response: reqwest::Response, limit: usize) -> Result, OidcHttpError> { - if response.content_length().is_some_and(|len| len > limit as u64) { - return Err(OidcHttpError::ResponseTooLarge(limit)); - } - - let mut response = response; - let mut body = Vec::new(); - while let Some(chunk) = response.chunk().await.map_err(OidcHttpError::Reqwest)? { - if body.len() + chunk.len() > limit { - return Err(OidcHttpError::ResponseTooLarge(limit)); - } - body.extend_from_slice(&chunk); - } - Ok(body) -} - -fn should_bypass_proxy_for_oidc_uri(uri: &str) -> bool { - let Some(host) = Url::parse(uri).ok().and_then(|url| url.host_str().map(str::to_owned)) else { - return false; - }; - let host = host.trim_matches(['[', ']']); - - host.eq_ignore_ascii_case("localhost") || host.parse::().is_ok_and(|addr| addr.is_loopback()) -} - -impl ReqwestHttpClient { - fn new() -> Result { - Self::new_with_extra_root_certs(Vec::new()) - } - - fn extra_root_cert_cache(certs: Vec) -> Arc> { - Arc::new(RwLock::new(CachedOidcExtraRootCerts { - generation: 0, - initialized: true, - certs, - })) - } - - fn new_with_extra_root_certs(extra_root_certs: Vec) -> Result { - Ok(Self { - policy_override: None, - extra_root_certs: Self::extra_root_cert_cache(extra_root_certs), - extra_root_ca_provider: None, - #[cfg(test)] - dns_resolver_override: None, - }) - } - - fn new_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result { - Ok(Self { - policy_override: None, - extra_root_certs: Arc::new(RwLock::new(CachedOidcExtraRootCerts::default())), - extra_root_ca_provider: Some(extra_root_ca_provider), - #[cfg(test)] - dns_resolver_override: None, - }) - } - - async fn current_extra_root_certs(&self) -> Result, OidcHttpError> { - let Some(provider) = self.extra_root_ca_provider.as_ref() else { - return self - .extra_root_certs - .read() - .map(|cache| cache.certs.clone()) - .map_err(|e| OidcHttpError::ExtraRootCa(format!("extra root certificate cache lock poisoned: {e}"))); - }; - - let material = provider.load().await.map_err(OidcHttpError::ExtraRootCa)?; - if let Ok(cache) = self.extra_root_certs.read() - && cache.initialized - && cache.generation == material.generation - { - return Ok(cache.certs.clone()); - } - - let certs = oidc_extra_root_certs(material.root_ca_pem.as_deref()).map_err(OidcHttpError::ExtraRootCa)?; - let mut cache = self - .extra_root_certs - .write() - .map_err(|e| OidcHttpError::ExtraRootCa(format!("extra root certificate cache lock poisoned: {e}")))?; - cache.generation = material.generation; - cache.initialized = true; - cache.certs = certs.clone(); - Ok(certs) - } - - /// Test-only constructor that pins outbound requests to an explicit policy, so a - /// loopback mock server can be reached without depending on process-wide environment. - #[cfg(test)] - fn with_policy(policy: OutboundPolicy) -> Self { - Self { - policy_override: Some(policy), - extra_root_certs: Self::extra_root_cert_cache(Vec::new()), - extra_root_ca_provider: None, - dns_resolver_override: None, - } - } - - #[cfg(test)] - fn with_policy_and_extra_root_certs(policy: OutboundPolicy, extra_root_certs: Vec) -> Self { - Self { - policy_override: Some(policy), - extra_root_certs: Self::extra_root_cert_cache(extra_root_certs), - extra_root_ca_provider: None, - dns_resolver_override: None, - } - } - - #[cfg(test)] - fn with_policy_and_extra_root_ca_provider(policy: OutboundPolicy, extra_root_ca_provider: OidcExtraRootCaProvider) -> Self { - Self { - policy_override: Some(policy), - extra_root_certs: Arc::new(RwLock::new(CachedOidcExtraRootCerts::default())), - extra_root_ca_provider: Some(extra_root_ca_provider), - dns_resolver_override: None, - } - } - - #[cfg(test)] - fn with_policy_and_dns_resolver(policy: OutboundPolicy, resolver: Arc) -> Self { - Self { - policy_override: Some(policy), - extra_root_certs: Self::extra_root_cert_cache(Vec::new()), - extra_root_ca_provider: None, - dns_resolver_override: Some(resolver), - } - } -} - -impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient { - type Error = OidcHttpError; - type Future = Pin>, Self::Error>> + Send + 'c>>; - - fn call(&'c self, request: http::Request>) -> Self::Future { - Box::pin(async move { - let started_at = Instant::now(); - let (parts, body) = request.into_parts(); - let method = parts.method.clone(); - let uri = parts.uri.to_string(); - if tracing::enabled!(tracing::Level::DEBUG) { - let request_headers = format_http_headers(&parts.headers); - debug!( - event = EVENT_OIDC_HTTP, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "request", - method = %method, - uri = %uri, - request_headers = %request_headers, - request_body_len = body.len(), - "oidc outbound http" - ); - } - - let extra_root_certs = self.current_extra_root_certs().await?; - let (client, url) = build_oidc_http_client( - &uri, - self.policy_override.as_ref(), - &extra_root_certs, - #[cfg(test)] - self.dns_resolver_override.clone(), - )?; - let response = client - .request(parts.method, uri.clone()) - .headers(parts.headers) - .body(body) - .send() - .await; - - let elapsed_ms = started_at.elapsed().as_millis().min(u128::from(u64::MAX)) as u64; - let succeeded = response.as_ref().is_ok_and(|resp| resp.status().is_success()); - OIDC_PLUGIN_AUTHN_METRICS.record(elapsed_ms, succeeded); - - let response = response.map_err(|err| { - let error = oidc_http_error_from_reqwest(&url, err); - error!( - event = EVENT_OIDC_HTTP, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "request_failed", - method = %method, - uri = %uri, - elapsed_ms, - error = %error, - "oidc outbound http" - ); - error - })?; - - let status = response.status(); - let headers = response.headers().clone(); - let body_bytes = read_bounded_response_body(response, MAX_OIDC_RESPONSE_SIZE) - .await - .map_err(|err| { - error!( - event = EVENT_OIDC_HTTP, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "response_body_failed", - method = %method, - uri = %uri, - status = status.as_u16(), - elapsed_ms, - error = %err, - "oidc outbound http" - ); - err - })?; - if tracing::enabled!(tracing::Level::DEBUG) { - let response_headers = format_http_headers(&headers); - debug!( - event = EVENT_OIDC_HTTP, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "response", - method = %method, - uri = %uri, - status = status.as_u16(), - status_success = status.is_success(), - elapsed_ms, - response_headers = %response_headers, - response_body_len = body_bytes.len(), - "oidc outbound http" - ); - } - - let mut http_response = http::Response::builder() - .status(status) - .body(body_bytes) - .map_err(OidcHttpError::Http)?; - *http_response.headers_mut() = headers; - - Ok(http_response) - }) - } -} - -// ---- Public types ---- - -const REDACTED_SECRET: &str = "***redacted***"; - -fn redacted_optional_secret(value: Option<&str>) -> &'static str { - value.filter(|secret| !secret.is_empty()).map_or("", |_| REDACTED_SECRET) -} - -/// Parsed configuration for a single OIDC provider. -#[derive(Clone, PartialEq, Eq)] -pub struct OidcProviderConfig { - pub id: String, - pub enabled: bool, - pub config_url: String, - pub issuer: Option, - pub client_id: String, - pub client_secret: Option, - pub scopes: Vec, - pub other_audiences: Vec, - pub redirect_uri: Option, - pub redirect_uri_dynamic: bool, - pub claim_name: String, - pub claim_prefix: String, - pub role_policy: String, - pub display_name: String, - pub groups_claim: String, - pub roles_claim: String, - pub email_claim: String, - pub username_claim: String, - pub hide_from_ui: bool, -} - -impl fmt::Debug for OidcProviderConfig { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("OidcProviderConfig") - .field("id", &self.id) - .field("enabled", &self.enabled) - .field("config_url", &self.config_url) - .field("issuer", &self.issuer) - .field("client_id", &self.client_id) - .field("client_secret", &redacted_optional_secret(self.client_secret.as_deref())) - .field("scopes", &self.scopes) - .field("other_audiences", &self.other_audiences) - .field("redirect_uri", &self.redirect_uri) - .field("redirect_uri_dynamic", &self.redirect_uri_dynamic) - .field("claim_name", &self.claim_name) - .field("claim_prefix", &self.claim_prefix) - .field("role_policy", &self.role_policy) - .field("display_name", &self.display_name) - .field("groups_claim", &self.groups_claim) - .field("roles_claim", &self.roles_claim) - .field("email_claim", &self.email_claim) - .field("username_claim", &self.username_claim) - .field("hide_from_ui", &self.hide_from_ui) - .finish() - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum OidcProviderConfigSource { - Env, - Persisted, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SourcedOidcProviderConfig { - pub config: OidcProviderConfig, - pub source: OidcProviderConfigSource, -} - -/// Immutable OIDC configuration entries returned by configuration queries. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct OidcConfigSnapshot { - providers: Vec, -} - -impl OidcConfigSnapshot { - pub fn new(mut providers: Vec) -> Self { - providers.sort_by(|left, right| { - (left.config.id != "default") - .cmp(&(right.config.id != "default")) - .then_with(|| left.config.id.cmp(&right.config.id)) - }); - Self { providers } - } - - pub fn providers(&self) -> &[SourcedOidcProviderConfig] { - &self.providers - } - - pub fn into_providers(self) -> Vec { - self.providers - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct OidcProviderValidationResult { - pub issuer: String, - pub authorization_endpoint: Option, - pub token_endpoint: Option, -} - -/// Compatibility name for the provider summary returned by OIDC list APIs. -pub type OidcProviderSummary = crate::federation::FederatedProviderView; - -/// Claims extracted from an OIDC ID token. -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct OidcClaims { - pub sub: String, - pub email: String, - pub username: String, - pub groups: Vec, - pub raw: HashMap, -} - -// ---- Internal provider state ---- - -/// Discovered OIDC provider metadata. -/// We store metadata (which includes JWKS after discovery) separately rather than -/// a `CoreClient` because the crate uses type-state generics that make storing -/// the configured client in a HashMap impractical. The client is reconstructed -/// on-the-fly from metadata when needed. -#[derive(Clone)] -struct ProviderState { - metadata: DiscoveredProviderMetadata, - discovered_at: Instant, -} - -// Workload issuers do not implement the browser authorization flow. Keep their -// verification metadata separate rather than inventing an authorization URL. -#[serde_with::serde_as] -#[derive(Clone, Deserialize)] -struct WorkloadProviderMetadata { - issuer: IssuerUrl, - jwks_uri: JsonWebKeySetUrl, - token_endpoint: Option, - #[serde_as(as = "serde_with::VecSkipError<_>")] - id_token_signing_alg_values_supported: Vec, - #[serde(skip)] - jwks: CoreJsonWebKeySet, - // Discovery is extensible; report unsupported fields without logging values. - #[serde(flatten)] - additional_fields: HashMap, -} - -#[derive(Clone)] -enum DiscoveredProviderMetadata { - Console(Box), - Workload(Box), -} - -impl DiscoveredProviderMetadata { - fn parse(body: &[u8], hide_from_ui: bool) -> Result { - let document: serde_json::Value = serde_json::from_slice(body).map_err(|err| err.to_string())?; - if hide_from_ui - && document - .as_object() - .is_some_and(|fields| !fields.contains_key("authorization_endpoint")) - { - let mut metadata: WorkloadProviderMetadata = serde_json::from_slice(body).map_err(|err| err.to_string())?; - if !metadata.additional_fields.is_empty() { - warn!( - event = EVENT_OIDC_DIAGNOSTICS, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "workload_discovery_additional_fields", - field_count = metadata.additional_fields.len(), - "workload discovery contains additional fields" - ); - metadata.additional_fields.clear(); - } - Ok(Self::Workload(Box::new(metadata))) - } else { - serde_json::from_slice(body) - .map(|metadata| Self::Console(Box::new(metadata))) - .map_err(|err| err.to_string()) - } - } - - fn console(&self) -> Result<&ProviderMetadataWithLogout, String> { - match self { - Self::Console(metadata) => Ok(metadata), - Self::Workload(_) => Err("OIDC provider has no authorization endpoint; only web identity is supported".into()), - } - } - - fn issuer(&self) -> &IssuerUrl { - match self { - Self::Console(metadata) => metadata.issuer(), - Self::Workload(metadata) => &metadata.issuer, - } - } - - fn jwks_uri(&self) -> &JsonWebKeySetUrl { - match self { - Self::Console(metadata) => metadata.jwks_uri(), - Self::Workload(metadata) => &metadata.jwks_uri, - } - } - - fn set_jwks(self, jwks: CoreJsonWebKeySet) -> Self { - match self { - Self::Console(metadata) => Self::Console(Box::new(metadata.set_jwks(jwks))), - Self::Workload(mut metadata) => { - metadata.jwks = jwks; - Self::Workload(metadata) - } - } - } - - fn authorization_endpoint(&self) -> Option { - match self { - Self::Console(metadata) => Some(metadata.authorization_endpoint().to_string()), - Self::Workload(_) => None, - } - } - - fn token_endpoint(&self) -> Option<&TokenUrl> { - match self { - Self::Console(metadata) => metadata.token_endpoint(), - Self::Workload(metadata) => metadata.token_endpoint.as_ref(), - } - } - - fn verifier(&self, config: &OidcProviderConfig) -> CoreIdTokenVerifier<'static> { - let client_id = ClientId::new(config.client_id.clone()); - let secret = config.client_secret.as_ref().map(|secret| ClientSecret::new(secret.clone())); - let (issuer, jwks, algorithms) = match self { - Self::Console(metadata) => (metadata.issuer(), metadata.jwks(), metadata.id_token_signing_alg_values_supported()), - Self::Workload(metadata) => (&metadata.issuer, &metadata.jwks, &metadata.id_token_signing_alg_values_supported), - }; - let verifier = match secret { - Some(secret) => CoreIdTokenVerifier::new_confidential_client(client_id, secret, issuer.clone(), jwks.clone()), - None => CoreIdTokenVerifier::new_public_client(client_id, issuer.clone(), jwks.clone()), - }; - verifier.set_allowed_algs(algorithms.clone()) - } -} - -// This adapter is used only for discovery/JWKS fetches, never token exchange. -struct JwksAcceptClient<'a> { - inner: &'a ReqwestHttpClient, - discovery_url: Option, -} - -impl<'c> AsyncHttpClient<'c> for JwksAcceptClient<'_> { - type Error = OidcHttpError; - type Future = >::Future; - - fn call(&'c self, mut request: http::Request>) -> Self::Future { - if !self.discovery_url.as_ref().is_some_and(|url| request.uri() == url.as_str()) { - request.headers_mut().insert( - http::header::ACCEPT, - http::HeaderValue::from_static("application/json, application/jwk-set+json"), - ); - } - self.inner.call(request) - } -} - -impl ProviderState { - fn is_stale(&self) -> bool { - self.discovered_at.elapsed() >= OIDC_JWKS_REFRESH_INTERVAL - } -} - -// ---- Core OIDC system ---- - -/// Global OIDC manager for all configured providers. -pub struct OidcSys { - configs: HashMap, - provider_states: RwLock>, - state_store: OidcStateStore, - http_client: ReqwestHttpClient, -} - -fn trusted_aud(other_audiences: &[String], audience: &Audience) -> bool { - for aud in other_audiences { - if audience.as_str() == aud.as_str() { - return true; - } - } - false -} - -impl OidcSys { - /// Parse environment variables and discover all configured OIDC providers. - pub async fn new() -> Result { - Self::new_with_extra_root_ca(None).await - } - - /// Parse environment variables and discover providers with an additional outbound root CA bundle. - pub(crate) async fn new_with_extra_root_ca(root_ca_pem: Option<&[u8]>) -> Result { - let http_client = ReqwestHttpClient::new_with_extra_root_certs(oidc_extra_root_certs(root_ca_pem)?)?; - Self::new_with_http_client(http_client).await - } - - pub(crate) async fn new_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result { - let http_client = ReqwestHttpClient::new_with_extra_root_ca_provider(extra_root_ca_provider)?; - http_client.current_extra_root_certs().await.map_err(|err| err.to_string())?; - Self::new_with_http_client(http_client).await - } - - async fn new_with_http_client(http_client: ReqwestHttpClient) -> Result { - let server_config = crate::server_config::current_server_config(); - let parsed_configs = load_oidc_config_snapshot(server_config.as_ref()); - let mut configs = HashMap::new(); - let mut provider_states = HashMap::new(); - - for sourced_config in parsed_configs.into_providers() { - let config = &sourced_config.config; - if !config.enabled { - debug!(provider = %config.id, "OIDC provider disabled"); - continue; - } - - match Self::discover_provider(config, &http_client).await { - Ok(state) => { - debug!(provider = %config.id, "OIDC provider discovered"); - provider_states.insert(config.id.clone(), state); - configs.insert(config.id.clone(), sourced_config); - } - Err(e) => { - error!( - event = EVENT_OIDC_DIAGNOSTICS, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "provider_discovery_failed", - provider_id = %config.id, - config_url = %config.config_url, - client_id = %config.client_id, - scopes = ?config.scopes, - redirect_uri = %config.redirect_uri.as_deref().unwrap_or(""), - redirect_uri_dynamic = config.redirect_uri_dynamic, - error = %e, - "oidc provider discovery failed" - ); - } - } - } - - Ok(Self { - configs, - provider_states: RwLock::new(provider_states), - state_store: OidcStateStore::new(), - http_client, - }) - } - - /// Create an OidcSys with no providers (useful for when OIDC is not configured). - pub fn empty() -> Result { - Ok(Self { - configs: HashMap::new(), - provider_states: RwLock::new(HashMap::new()), - state_store: OidcStateStore::new(), - http_client: ReqwestHttpClient::new()?, - }) - } - - /// Return true if any OIDC providers are configured and enabled. - pub fn has_providers(&self) -> bool { - !self.configs.is_empty() - } - - pub(crate) fn provider_configs(&self) -> impl Iterator { - self.configs.values().map(|provider| &provider.config) - } - - /// List all providers, including providers hidden from the login UI. - pub fn list_providers(&self) -> Vec { - self.provider_configs() - .map(|config| OidcProviderSummary { - provider_id: config.id.clone(), - display_name: config.display_name.clone(), - }) - .collect() - } - - /// List providers visible in the login UI. - pub fn list_visible_providers(&self) -> Vec { - self.provider_configs() - .filter(|config| !config.hide_from_ui) - .map(|config| OidcProviderSummary { - provider_id: config.id.clone(), - display_name: config.display_name.clone(), - }) - .collect() - } - - pub fn config_snapshot(&self) -> OidcConfigSnapshot { - OidcConfigSnapshot::new(self.configs.values().cloned().collect()) - } - - /// Build the PKCE authorization URL for a provider, store state in the state store. - pub async fn authorize_url( - &self, - provider_id: &str, - redirect_uri: &str, - redirect_after: Option, - ) -> Result { - let config = self - .get_provider_config(provider_id) - .ok_or_else(|| format!("unknown OIDC provider: {provider_id}"))?; - let state = self.ensure_provider_state(provider_id, config).await?; - - let (pkce_challenge, pkce_verifier) = PkceCodeChallenge::new_random_sha256(); - - let redirect = RedirectUrl::new(redirect_uri.to_string()).map_err(|e| format!("invalid redirect URI: {e}"))?; - - let client = CoreClient::from_provider_metadata( - state.metadata.console()?.clone(), - ClientId::new(config.client_id.clone()), - config.client_secret.as_ref().map(|s| ClientSecret::new(s.clone())), - ) - .set_auth_type(AuthType::RequestBody); - - let mut auth_req = - client.authorize_url(CoreAuthenticationFlow::AuthorizationCode, CsrfToken::new_random, Nonce::new_random); - auth_req = auth_req.set_redirect_uri(Cow::Owned(redirect)); - - for scope in &config.scopes { - auth_req = auth_req.add_scope(Scope::new(scope.clone())); - } - - auth_req = auth_req.set_pkce_challenge(pkce_challenge); - - let (auth_url, csrf_token, nonce) = auth_req.url(); - - // Store the state for callback validation - self.state_store - .insert( - csrf_token.secret().clone(), - OidcAuthSession { - provider_id: provider_id.to_string(), - pkce_verifier: pkce_verifier.secret().clone(), - nonce: nonce.secret().clone(), - redirect_after, - }, - ) - .await; - - Ok(auth_url.to_string()) - } - - /// Exchange an authorization code for tokens and extract claims. - pub async fn exchange_code( - &self, - state: &str, - code: &str, - redirect_uri: &str, - ) -> Result<(OidcClaims, String, OidcAuthSession, String), String> { - // Retrieve and consume the state (single-use) - let session = self - .state_store - .take(state) - .await - .ok_or_else(|| "invalid or expired OIDC state".to_string())?; - - let config = self - .get_provider_config(&session.provider_id) - .ok_or_else(|| format!("unknown provider: {}", session.provider_id))?; - let provider_state = self.get_provider_state(&session.provider_id)?; - let issuer = provider_state.metadata.issuer().to_string(); - let token_endpoint = provider_state - .metadata - .token_endpoint() - .map(ToString::to_string) - .unwrap_or_default(); - - // Construct CoreClient on-the-fly with JWKS from discovery - let client = CoreClient::from_provider_metadata( - provider_state.metadata.console()?.clone(), - ClientId::new(config.client_id.clone()), - config.client_secret.as_ref().map(|s| ClientSecret::new(s.clone())), - ) - .set_auth_type(AuthType::RequestBody); - - let redirect = RedirectUrl::new(redirect_uri.to_string()).map_err(|e| format!("invalid redirect URI: {e}"))?; - - // Exchange code for tokens - let token_response = client - .exchange_code(AuthorizationCode::new(code.to_string())) - .map_err(|e| { - error!( - event = EVENT_OIDC_DIAGNOSTICS, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "token_endpoint_missing", - provider_id = %session.provider_id, - config_url = %config.config_url, - issuer = %issuer, - client_id = %config.client_id, - redirect_uri = %redirect_uri, - scopes = ?config.scopes, - error = %e, - "oidc token exchange failed" - ); - format!( - "token endpoint not configured: {e}: provider_id={}, config_url={}, issuer={}, redirect_uri={}, client_id={}", - session.provider_id, config.config_url, issuer, redirect_uri, config.client_id - ) - })? - .set_pkce_verifier(PkceCodeVerifier::new(session.pkce_verifier.clone())) - .set_redirect_uri(Cow::Owned(redirect)) - .request_async(&self.http_client) - .await - .map_err(|e| match &e { - RequestTokenError::ServerResponse(response) => { - error!( - event = EVENT_OIDC_DIAGNOSTICS, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "token_server_response", - provider_id = %session.provider_id, - config_url = %config.config_url, - issuer = %issuer, - token_endpoint = %token_endpoint, - client_id = %config.client_id, - client_secret_configured = config.client_secret.as_deref().is_some_and(|secret| !secret.is_empty()), - redirect_uri = %redirect_uri, - scopes = ?config.scopes, - oauth_error = %response.error(), - oauth_error_description = %response.error_description().map(String::as_str).unwrap_or(""), - oauth_error_uri = %response.error_uri().map(String::as_str).unwrap_or(""), - error = %e, - "oidc token exchange failed" - ); - format!( - "token exchange failed: {e}: stage=token_server_response, provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}, oauth_error={}, oauth_error_description={}", - session.provider_id, - config.config_url, - issuer, - token_endpoint, - redirect_uri, - config.client_id, - response.error(), - response.error_description().map(String::as_str).unwrap_or("") - ) - } - RequestTokenError::Request(err) => { - let (request_error_kind, request_error_status) = oidc_http_error_diagnostics(err); - error!( - event = EVENT_OIDC_DIAGNOSTICS, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "token_request_failed", - provider_id = %session.provider_id, - config_url = %config.config_url, - issuer = %issuer, - token_endpoint = %token_endpoint, - client_id = %config.client_id, - redirect_uri = %redirect_uri, - request_error_kind = %request_error_kind, - request_error_status = %request_error_status, - error = %err, - "oidc token exchange failed" - ); - format!( - "token exchange failed: stage=token_request_failed, provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}, request_error_kind={}, request_error_status={}, request_error={}", - session.provider_id, - config.config_url, - issuer, - token_endpoint, - redirect_uri, - config.client_id, - request_error_kind, - request_error_status, - err - ) - } - RequestTokenError::Parse(parse_err, body) => { - let shape = inspect_token_response_body(body); - error!( - event = EVENT_OIDC_DIAGNOSTICS, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "token_response_parse_failed", - provider_id = %session.provider_id, - config_url = %config.config_url, - issuer = %issuer, - token_endpoint = %token_endpoint, - client_id = %config.client_id, - redirect_uri = %redirect_uri, - parse_error_path = %parse_err.path(), - response_body_len = body.len(), - response_json_object = shape.json_object, - response_json_keys = %shape.json_keys, - response_has_access_token = shape.has_access_token, - response_has_id_token = shape.has_id_token, - response_has_token_type = shape.has_token_type, - response_has_expires_in = shape.has_expires_in, - response_has_error = shape.has_error, - response_has_error_description = shape.has_error_description, - response_looks_like_html = shape.looks_like_html, - error = %e, - "oidc token exchange failed" - ); - format!( - "token exchange failed: {e}: stage=token_response_parse_failed, provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}, parse_error_path={}, response_body_len={}, response_json_keys={}, response_has_id_token={}, response_has_error={}, response_looks_like_html={}", - session.provider_id, - config.config_url, - issuer, - token_endpoint, - redirect_uri, - config.client_id, - parse_err.path(), - body.len(), - shape.json_keys, - shape.has_id_token, - shape.has_error, - shape.looks_like_html - ) - } - RequestTokenError::Other(message) => { - error!( - event = EVENT_OIDC_DIAGNOSTICS, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "token_exchange_other_error", - provider_id = %session.provider_id, - config_url = %config.config_url, - issuer = %issuer, - token_endpoint = %token_endpoint, - client_id = %config.client_id, - redirect_uri = %redirect_uri, - error = %message, - "oidc token exchange failed" - ); - format!( - "token exchange failed: {e}: stage=token_exchange_other_error, provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}", - session.provider_id, config.config_url, issuer, token_endpoint, redirect_uri, config.client_id - ) - } - })?; - - // Verify the ID token (signature, issuer, audience, expiry, nonce) - let id_token = token_response - .extra_fields() - .id_token() - .ok_or_else(|| { - error!( - event = EVENT_OIDC_DIAGNOSTICS, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "token_response_missing_id_token", - provider_id = %session.provider_id, - config_url = %config.config_url, - issuer = %issuer, - token_endpoint = %token_endpoint, - client_id = %config.client_id, - redirect_uri = %redirect_uri, - scopes = ?config.scopes, - "oidc token exchange failed" - ); - format!( - "no id_token in token response: provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}, scopes={}", - session.provider_id, - config.config_url, - issuer, - token_endpoint, - redirect_uri, - config.client_id, - config.scopes.join(",") - ) - })?; - - let verifier = client - .id_token_verifier() - .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); - let verified = id_token.claims(&verifier, &Nonce::new(session.nonce.clone())); - if let Err(e) = verified { - let verification_error = e.to_string(); - warn!( - event = EVENT_OIDC_DIAGNOSTICS, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "id_token_verification_retry", - provider_id = %session.provider_id, - config_url = %config.config_url, - issuer = %issuer, - token_endpoint = %token_endpoint, - client_id = %config.client_id, - other_audiences = ?config.other_audiences, - error = %verification_error, - "oidc id token verification failed" - ); - let refreshed_state = self - .refresh_provider_state(&session.provider_id, config) - .await - .map_err(|refresh_err| { - format!( - "ID token verification failed: {verification_error}; failed to refresh provider metadata: {refresh_err}" - ) - })?; - - warn!( - event = EVENT_OIDC_DIAGNOSTICS, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "jwks_metadata_refreshed", - provider_id = %session.provider_id, - config_url = %config.config_url, - issuer = %issuer, - "oidc provider metadata refreshed" - ); - - let client = CoreClient::from_provider_metadata( - refreshed_state.metadata.console()?.clone(), - ClientId::new(config.client_id.clone()), - config.client_secret.as_ref().map(|s| ClientSecret::new(s.clone())), - ) - .set_auth_type(AuthType::RequestBody); - - let verifier = client - .id_token_verifier() - .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); - id_token - .claims(&verifier, &Nonce::new(session.nonce.clone())) - .map_err(|retry_err| { - error!( - event = EVENT_OIDC_DIAGNOSTICS, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "id_token_verification_failed", - provider_id = %session.provider_id, - config_url = %config.config_url, - issuer = %issuer, - token_endpoint = %token_endpoint, - client_id = %config.client_id, - other_audiences = ?config.other_audiences, - original_error = %verification_error, - retry_error = %retry_err, - "oidc id token verification failed" - ); - format!( - "ID token verification failed after JWKS refresh: {retry_err}; original_error={verification_error}; provider_id={}, config_url={}, issuer={}, token_endpoint={}, client_id={}, other_audiences={}", - session.provider_id, - config.config_url, - issuer, - token_endpoint, - config.client_id, - config.other_audiences.join(",") - ) - })?; - } - - // Extract raw claims from the verified JWT for custom claim support - // (the crate verifies signature/expiry/nonce; we decode payload for non-standard claims) - let raw_jwt = id_token.to_string(); - let raw = decode_jwt_payload(&raw_jwt); - - let claims = OidcClaims { - sub: extract_string_claim(&raw, "sub"), - email: extract_string_claim(&raw, &config.email_claim), - username: extract_string_claim(&raw, &config.username_claim), - groups: extract_canonical_group_values(&raw, &config.groups_claim, &config.roles_claim), - raw, - }; - - Ok((claims, session.provider_id.clone(), session, raw_jwt)) - } - - /// Store a one-time logout session keyed by an opaque token so the console can - /// trigger browser logout without persisting the raw ID token. - pub async fn create_logout_token(&self, provider_id: &str, id_token: &str) -> Result { - if !self.configs.contains_key(provider_id) { - return Err(format!("unknown OIDC provider: {provider_id}")); - } - - let token = CsrfToken::new_random().secret().clone(); - self.state_store - .insert_logout( - token.clone(), - OidcLogoutSession { - provider_id: provider_id.to_string(), - id_token: id_token.to_string(), - }, - ) - .await; - - Ok(token) - } - - /// Build the RP-initiated logout URL for a previously issued logout token. - /// Returns `Ok(None)` when the provider does not advertise an end-session endpoint. - pub async fn build_logout_url(&self, logout_token: &str, post_logout_redirect_uri: &str) -> Result, String> { - let session = self - .state_store - .take_logout(logout_token) - .await - .ok_or_else(|| "invalid or expired OIDC logout token".to_string())?; - - let config = self - .get_provider_config(&session.provider_id) - .ok_or_else(|| format!("unknown OIDC provider: {}", session.provider_id))?; - let state = self.ensure_provider_state(&session.provider_id, config).await?; - let Some(end_session_endpoint) = state.metadata.console()?.additional_metadata().end_session_endpoint.clone() else { - return Ok(None); - }; - - let id_token: CoreIdToken = session - .id_token - .parse() - .map_err(|e: serde_json::Error| format!("failed to parse ID token for logout: {e}"))?; - let post_logout_redirect_uri = PostLogoutRedirectUrl::new(post_logout_redirect_uri.to_string()) - .map_err(|e| format!("invalid post logout redirect URI: {e}"))?; - - let logout_url = LogoutRequest::from(end_session_endpoint) - .set_id_token_hint(&id_token) - .set_client_id(ClientId::new(config.client_id.clone())) - .set_post_logout_redirect_uri(post_logout_redirect_uri) - .http_get_url() - .to_string(); - - Ok(Some(logout_url)) - } - - /// Map OIDC claims to rustfs policy names. - pub fn map_claims_to_policies(&self, provider_id: &str, claims: &OidcClaims) -> (Vec, Vec) { - let Some(rule) = self.authorization_rule(provider_id) else { - return (Vec::new(), Vec::new()); - }; - CoreFederatedAuthorizationMapper::map_policies_and_groups(provider_id, rule, &claims.groups, &claims.raw, true) - } - - /// Policy names produced only by the canonical groups in claim-based mode: the groups claim - /// plus the roles claim values that `extract_canonical_group_values` merges into it. - /// - /// Identity providers emit built-in groups and roles that can never have a matching policy - /// (for example `DOMAIN\Domain Users` or Keycloak's `offline_access`), so the session binding - /// may ignore these names when no such policy exists. Names from a fixed role policy or a - /// dedicated policy claim are never included, so those configurations keep requiring every - /// policy to resolve. - pub fn group_claim_policy_names(&self, provider_id: &str, claims: &OidcClaims) -> Vec { - let Some(rule) = self.authorization_rule(provider_id) else { - return Vec::new(); - }; - CoreFederatedAuthorizationMapper::group_claim_policy_names(rule, &claims.groups, &claims.raw) - } - - fn authorization_rule(&self, provider_id: &str) -> Option> { - self.get_provider_config(provider_id).map(|config| { - FederatedAuthorizationRuleRef::new( - &config.claim_name, - &config.claim_prefix, - &config.role_policy, - &config.groups_claim, - &config.roles_claim, - ) - }) - } - - /// Verify a raw JWT (id_token) for the AssumeRoleWithWebIdentity flow. - /// - /// Unlike the authorization code flow, ARWWI receives a raw JWT directly - /// (not via code exchange). This method: - /// 1. Decodes the JWT payload to extract the `iss` claim - /// 2. Finds the OIDC provider whose issuer matches - /// 3. Verifies signature, issuer, audience, and expiry (nonce is skipped) - /// 4. Extracts claims using the provider's claim configuration - pub async fn verify_web_identity_token(&self, jwt: &str) -> Result<(OidcClaims, String /* provider_id */), String> { - // Decode JWT payload without verification to get the issuer claim - let raw_claims = decode_jwt_payload(jwt); - let issuer = raw_claims - .get("iss") - .and_then(|v| v.as_str()) - .ok_or_else(|| "JWT missing 'iss' claim".to_string())?; - - // Find matching provider by issuer - let (provider_id, config, mut state) = self - .find_provider_by_issuer(issuer) - .ok_or_else(|| format!("no OIDC provider configured for issuer: {issuer}"))?; - - state = self.ensure_provider_state_if_stale(&provider_id, &config, &state).await?; - - // Parse raw JWT string into CoreIdToken - let id_token: CoreIdToken = jwt - .parse() - .map_err(|e: serde_json::Error| format!("failed to parse JWT as ID token: {e}"))?; - - // Verify the token (signature, issuer, audience, expiry) — skip nonce - // (nonce is only required for the authorization code flow) - let verifier = state - .metadata - .verifier(&config) - .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); - if let Err(e) = id_token.claims(&verifier, |_: Option<&Nonce>| Ok(())) { - state = self - .refresh_provider_state(&provider_id, &config) - .await - .map_err(|refresh_err| { - format!("ID token verification failed: {e}; failed to refresh provider metadata: {refresh_err}") - })?; - - let verifier = state - .metadata - .verifier(&config) - .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); - id_token - .claims(&verifier, |_: Option<&Nonce>| Ok(())) - .map_err(|retry_err| format!("ID token verification failed after JWKS refresh: {retry_err}"))?; - } - - // Extract claims using the provider's claim configuration - let claims = OidcClaims { - sub: extract_string_claim(&raw_claims, "sub"), - email: extract_string_claim(&raw_claims, &config.email_claim), - username: extract_string_claim(&raw_claims, &config.username_claim), - groups: extract_canonical_group_values(&raw_claims, &config.groups_claim, &config.roles_claim), - raw: raw_claims, - }; - - Ok((claims, provider_id.to_string())) - } - - /// Find a provider whose discovered issuer matches the given JWT issuer string. - fn find_provider_by_issuer(&self, issuer: &str) -> Option<(String, OidcProviderConfig, ProviderState)> { - let (issuer_scheme, issuer_host, issuer_port, issuer_path) = normalize_issuer(issuer)?; - let map = self - .provider_states - .read() - .map_err(|e| format!("provider state lock poisoned: {e}")) - .ok()?; - for (id, state) in map.iter() { - let provider_issuer = state.metadata.issuer().as_str(); - let Some((provider_scheme, provider_host, provider_port, provider_path)) = normalize_issuer(provider_issuer) else { - continue; - }; - - if issuer_scheme == provider_scheme - && issuer_host == provider_host - && issuer_port == provider_port - && issuer_path == provider_path - && let Some(config) = self.get_provider_config(id) - { - return Some((id.clone(), config.clone(), state.clone())); - } - } - None - } - - fn get_provider_state(&self, provider_id: &str) -> Result { - self.provider_states - .read() - .map_err(|e| format!("provider state lock poisoned: {e}"))? - .get(provider_id) - .cloned() - .ok_or_else(|| format!("provider not discovered: {provider_id}")) - } - - async fn refresh_provider_state(&self, provider_id: &str, config: &OidcProviderConfig) -> Result { - let state = Self::discover_provider(config, &self.http_client).await?; - let mut map = self.provider_states.write().map_err(|e| { - let msg = e.to_string(); - format!("provider state lock poisoned: {msg}") - })?; - map.insert(provider_id.to_string(), state.clone()); - - Ok(state) - } - - async fn ensure_provider_state(&self, provider_id: &str, config: &OidcProviderConfig) -> Result { - let state = self.get_provider_state(provider_id)?; - if state.is_stale() { - self.refresh_provider_state(provider_id, config).await.or_else(|refresh_err| { - warn!( - "OIDC provider '{}' JWKS metadata refresh skipped due to transient network issue: {}", - provider_id, refresh_err - ); - Ok(state) - }) - } else { - Ok(state) - } - } - - async fn ensure_provider_state_if_stale( - &self, - provider_id: &str, - config: &OidcProviderConfig, - state: &ProviderState, - ) -> Result { - if state.is_stale() { - self.refresh_provider_state(provider_id, config).await.or_else(|refresh_err| { - warn!( - "OIDC provider '{}' JWKS metadata refresh skipped due to transient network issue: {}", - provider_id, refresh_err - ); - Ok(state.clone()) - }) - } else { - Ok(state.clone()) - } - } - - /// Get the state store (used by HTTP handlers). - pub fn state_store(&self) -> &OidcStateStore { - &self.state_store - } - - /// Get a provider config by ID. - pub fn get_provider_config(&self, id: &str) -> Option<&OidcProviderConfig> { - self.configs.get(id).map(|provider| &provider.config) - } - - /// Parse all OIDC provider configs from environment variables. - fn parse_env_configs() -> Vec { - let mut configs = Vec::new(); - - // Check for the default provider (no suffix) - if let Some(config) = Self::parse_single_provider("", "default") { - configs.push(config); - } - - // Scan for suffixed providers by checking all OIDC env var prefixes. - // This allows providers to be discovered without requiring a separate ENABLE_ key. - let mut provider_ids: Vec = Vec::new(); - let scan_prefixes: Vec = ENV_IDENTITY_OPENID_KEYS.iter().map(|k| format!("{k}_")).collect(); - for (key, _) in std::env::vars() { - for prefix in &scan_prefixes { - if let Some(suffix) = key.strip_prefix(prefix.as_str()) - && !suffix.is_empty() - && suffix != "default" - { - provider_ids.push(suffix.to_string()); - break; - } - } - } - - provider_ids.sort(); - provider_ids.dedup(); - - for id in provider_ids { - let suffix = format!("_{id}"); - if let Some(config) = Self::parse_single_provider(&suffix, &id) { - configs.push(config); - } - } - - configs - } - - fn parse_persisted_configs(cfg: &ServerConfig) -> Vec { - let Some(subsystem) = cfg.0.get(IDENTITY_OPENID_SUB_SYS) else { - return Vec::new(); - }; - - let mut configs = Vec::new(); - let mut provider_ids: Vec = subsystem.keys().cloned().collect(); - provider_ids.sort(); - - for raw_id in provider_ids { - let Some(kvs) = subsystem.get(&raw_id) else { - continue; - }; - - let id = if raw_id == DEFAULT_DELIMITER { - "default" - } else { - raw_id.as_str() - }; - if let Some(config) = Self::parse_single_persisted_provider(kvs, id) { - configs.push(config); - } - } - - configs - } - - /// Parse a string as an `EnableState` boolean. - /// Returns `default_if_empty` when the input is empty, and `default_on_error` - /// when parsing fails. - fn parse_enable_state(value: &str, default_if_empty: bool, default_on_error: bool) -> bool { - if value.is_empty() { - return default_if_empty; - } - value - .parse::() - .map(|s| s.is_enabled()) - .unwrap_or(default_on_error) - } - - /// Parse a single provider's config from env vars with the given suffix. - fn parse_single_provider(env_suffix: &str, id: &str) -> Option { - let get_env = |base: &str| -> String { std::env::var(format!("{base}{env_suffix}")).unwrap_or_default() }; - - let enable_val = get_env(ENV_IDENTITY_OPENID_ENABLE); - let config_url = get_env(ENV_IDENTITY_OPENID_CONFIG_URL); - let issuer = get_env(ENV_IDENTITY_OPENID_ISSUER); - - // Skip if no config URL - if config_url.is_empty() { - return None; - } - - let enabled = Self::parse_enable_state(&enable_val, true, false); - - let scopes_str = get_env(ENV_IDENTITY_OPENID_SCOPES); - let scopes = if scopes_str.is_empty() { - OIDC_DEFAULT_SCOPES.split(',').map(String::from).collect() - } else { - scopes_str.split(',').map(|s| s.trim().to_string()).collect() - }; - - let other_audiences_str = get_env(ENV_IDENTITY_OPENID_OTHER_AUDIENCES); - let other_audiences = other_audiences_str - .split(',') - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()) - .collect(); - - let redirect_uri_dynamic = Self::parse_enable_state(&get_env(ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC), true, true); - - let claim_name = { - let v = get_env(ENV_IDENTITY_OPENID_CLAIM_NAME); - if v.is_empty() { - OIDC_DEFAULT_CLAIM_NAME.to_string() - } else { - v - } - }; - let groups_claim = { - let v = get_env(ENV_IDENTITY_OPENID_GROUPS_CLAIM); - if v.is_empty() { - OIDC_DEFAULT_GROUPS_CLAIM.to_string() - } else { - v - } - }; - let roles_claim = get_env(ENV_IDENTITY_OPENID_ROLES_CLAIM); - let email_claim = { - let v = get_env(ENV_IDENTITY_OPENID_EMAIL_CLAIM); - if v.is_empty() { - OIDC_DEFAULT_EMAIL_CLAIM.to_string() - } else { - v - } - }; - let username_claim = { - let v = get_env(ENV_IDENTITY_OPENID_USERNAME_CLAIM); - if v.is_empty() { - OIDC_DEFAULT_USERNAME_CLAIM.to_string() - } else { - v - } - }; - let display_name = { - let v = get_env(ENV_IDENTITY_OPENID_DISPLAY_NAME); - if v.is_empty() { id.to_string() } else { v } - }; - let redirect_uri = { - let v = get_env(ENV_IDENTITY_OPENID_REDIRECT_URI); - if v.is_empty() { None } else { Some(v) } - }; - let client_secret = { - let v = get_env(ENV_IDENTITY_OPENID_CLIENT_SECRET); - if v.is_empty() { None } else { Some(v) } - }; - let hide_from_ui = Self::parse_enable_state(&get_env(ENV_IDENTITY_OPENID_HIDE_FROM_UI), false, false); - - Some(OidcProviderConfig { - id: id.to_string(), - enabled, - config_url, - issuer: if issuer.is_empty() { None } else { Some(issuer) }, - client_id: get_env(ENV_IDENTITY_OPENID_CLIENT_ID), - client_secret, - scopes, - other_audiences, - redirect_uri, - redirect_uri_dynamic, - claim_name, - claim_prefix: get_env(ENV_IDENTITY_OPENID_CLAIM_PREFIX), - role_policy: get_env(ENV_IDENTITY_OPENID_ROLE_POLICY), - display_name, - groups_claim, - roles_claim, - email_claim, - username_claim, - hide_from_ui, - }) - } - - fn parse_single_persisted_provider(kvs: &KVS, id: &str) -> Option { - let config_url = kvs.get(OIDC_CONFIG_URL); - if config_url.is_empty() { - return None; - } - - let enabled = Self::parse_enable_state(&kvs.lookup(ENABLE_KEY).unwrap_or_default(), false, false); - - let scopes_str = kvs.get(OIDC_SCOPES); - let scopes = if scopes_str.is_empty() { - OIDC_DEFAULT_SCOPES.split(',').map(String::from).collect() - } else { - scopes_str.split(',').map(|s| s.trim().to_string()).collect() - }; - - let other_audiences_str = kvs.get(OIDC_OTHER_AUDIENCES); - let other_audiences = other_audiences_str - .split(',') - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()) - .collect(); - - let redirect_uri_dynamic = - Self::parse_enable_state(&kvs.lookup(OIDC_REDIRECT_URI_DYNAMIC).unwrap_or_default(), true, true); - - let claim_name = kvs - .lookup(OIDC_CLAIM_NAME) - .unwrap_or_else(|| OIDC_DEFAULT_CLAIM_NAME.to_string()); - let groups_claim = kvs - .lookup(OIDC_GROUPS_CLAIM) - .unwrap_or_else(|| OIDC_DEFAULT_GROUPS_CLAIM.to_string()); - let roles_claim = kvs - .lookup(OIDC_ROLES_CLAIM) - .unwrap_or_else(|| OIDC_DEFAULT_ROLES_CLAIM.to_string()); - let email_claim = kvs - .lookup(OIDC_EMAIL_CLAIM) - .unwrap_or_else(|| OIDC_DEFAULT_EMAIL_CLAIM.to_string()); - let username_claim = kvs - .lookup(OIDC_USERNAME_CLAIM) - .unwrap_or_else(|| OIDC_DEFAULT_USERNAME_CLAIM.to_string()); - let display_name = kvs.lookup(OIDC_DISPLAY_NAME).unwrap_or_else(|| id.to_string()); - let redirect_uri = kvs.lookup(OIDC_REDIRECT_URI).filter(|v| !v.is_empty()); - let client_secret = kvs.lookup(OIDC_CLIENT_SECRET).filter(|v| !v.is_empty()); - let hide_from_ui = Self::parse_enable_state(&kvs.lookup(OIDC_HIDE_FROM_UI).unwrap_or_default(), false, false); - - Some(OidcProviderConfig { - id: id.to_string(), - enabled, - config_url, - issuer: kvs.lookup(OIDC_ISSUER).filter(|v| !v.is_empty()), - client_id: kvs.get(OIDC_CLIENT_ID), - client_secret, - scopes, - other_audiences, - redirect_uri, - redirect_uri_dynamic, - claim_name, - claim_prefix: kvs.get(OIDC_CLAIM_PREFIX), - role_policy: kvs.get(OIDC_ROLE_POLICY), - display_name, - groups_claim, - roles_claim, - email_claim, - username_claim, - hide_from_ui, - }) - } - - /// Perform OIDC discovery for a provider. - /// `discover_async` fetches the discovery document and JWKS in one step. - async fn discover_provider(config: &OidcProviderConfig, http_client: &ReqwestHttpClient) -> Result { - if let Some(issuer) = config.issuer.as_deref().filter(|issuer| !issuer.trim().is_empty()) { - return Self::discover_provider_from_config_url(config, issuer, http_client).await; - } - - // The openidconnect crate expects the issuer URL (base), not the - // .well-known/openid-configuration URL. - let base_issuer = normalize_config_url(&config.config_url)?; - let candidates = issuer_candidates(&base_issuer); - let mut last_errors = Vec::new(); - - for candidate_issuer in candidates.iter() { - let issuer_url = IssuerUrl::new(candidate_issuer.clone()).map_err(|e| format!("invalid issuer URL: {e}"))?; - - for attempt in 0..OIDC_DISCOVERY_TRANSPORT_RETRIES { - let discovered = if config.hide_from_ui { - Self::discover_provider_from_config_url(config, candidate_issuer, http_client).await - } else { - let client = JwksAcceptClient { - inner: http_client, - discovery_url: Some( - issuer_url - .join(".well-known/openid-configuration") - .map_err(|err| err.to_string())?, - ), - }; - ProviderMetadataWithLogout::discover_async(issuer_url.clone(), &client) - .await - .map(|metadata| ProviderState { - metadata: DiscoveredProviderMetadata::Console(Box::new(metadata)), - discovered_at: Instant::now(), - }) - .map_err(|err| match err { - DiscoveryError::Request(OidcHttpError::ForbiddenOutbound(reason)) => { - format!("{OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY}: {reason}") - } - err => format!("discovery failed: {err}"), - }) - }; - match discovered { - Ok(state) => return Ok(state), - Err(error) - if error.starts_with(OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY) - || error.starts_with(OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY) => - { - return Err(error); - } - Err(error) => { - let is_transient_transport = error.contains("Request failed"); - let should_retry = is_transient_transport && attempt + 1 < OIDC_DISCOVERY_TRANSPORT_RETRIES; - if should_retry { - warn!( - event = EVENT_OIDC_DIAGNOSTICS, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "provider_discovery_transport_retry", - provider_id = %config.id, - config_url = %config.config_url, - issuer_candidate = %candidate_issuer, - attempt = attempt + 1, - max_attempts = OIDC_DISCOVERY_TRANSPORT_RETRIES, - error = %error, - "oidc provider discovery failed" - ); - sleep(OIDC_DISCOVERY_TRANSPORT_RETRY_DELAY).await; - continue; - } - - last_errors.push(format!("issuer '{candidate_issuer}': {error}")); - warn!( - event = EVENT_OIDC_DIAGNOSTICS, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "provider_discovery_candidate_failed", - provider_id = %config.id, - config_url = %config.config_url, - issuer_candidate = %candidate_issuer, - attempt = attempt + 1, - max_attempts = OIDC_DISCOVERY_TRANSPORT_RETRIES, - error = %error, - "oidc provider discovery failed" - ); - break; - } - } - } - } - - Err(format!( - "discovery failed for all issuer variants {:?}: {}", - candidates, - last_errors.join("; ") - )) - } - - async fn discover_provider_from_config_url( - config: &OidcProviderConfig, - issuer: &str, - http_client: &ReqwestHttpClient, - ) -> Result { - let issuer_url = IssuerUrl::new(issuer.trim().to_string()).map_err(|e| format!("invalid issuer URL: {e}"))?; - let explicit_issuer = config.issuer.as_deref().is_some_and(|issuer| !issuer.trim().is_empty()); - let discovery_url = if explicit_issuer { - discovery_url_from_config_url(&config.config_url)? - } else { - issuer_url - .join(".well-known/openid-configuration") - .map_err(|err| err.to_string())? - }; - let request = http::Request::builder() - .uri(discovery_url.to_string()) - .method(http::Method::GET) - .header(http::header::ACCEPT, "application/json") - .body(Vec::new()) - .map_err(|err| format!("failed to prepare discovery request: {err}"))?; - - let response = match http_client.call(request).await { - Ok(response) => response, - Err(OidcHttpError::ForbiddenOutbound(reason)) => { - return Err(format!("{OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY}: {reason}")); - } - Err(err) => return Err(format!("discovery request failed: Request failed: {err}")), - }; - if response.status() != http::StatusCode::OK { - return Err(format!("discovery failed: HTTP status code {} at {}", response.status(), discovery_url)); - } - - if !explicit_issuer - && let Some(content_type) = response.headers().get(http::header::CONTENT_TYPE) - && !content_type.to_str().ok().is_some_and(|value| { - value - .split(';') - .next() - .is_some_and(|essence| essence.eq_ignore_ascii_case("application/json")) - }) - { - return Err("Unexpected response Content-Type: expected application/json".into()); - } - - let provider_metadata = DiscoveredProviderMetadata::parse(response.body(), config.hide_from_ui) - .map_err(|err| format!("failed to parse discovery response: {err}"))?; - if provider_metadata.issuer() != &issuer_url { - return Err(format!( - "unexpected issuer URI `{}` (expected `{}`)", - provider_metadata.issuer().as_str(), - issuer_url.as_str() - )); - } - - let jwks_url = if explicit_issuer { - jwks_url_from_config_url(&config.config_url, &issuer_url, provider_metadata.jwks_uri())? - } else { - provider_metadata.jwks_uri().clone() - }; - let jwks = match CoreJsonWebKeySet::fetch_async( - &jwks_url, - &JwksAcceptClient { - inner: http_client, - discovery_url: None, - }, - ) - .await - { - Ok(jwks) => jwks, - Err(DiscoveryError::Request(OidcHttpError::ForbiddenOutbound(reason))) => { - return Err(format!("{OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY}: {reason}")); - } - Err(err) => return Err(format!("failed to fetch JWKS: {err}")), - }; - - Ok(ProviderState { - metadata: provider_metadata.set_jwks(jwks), - discovered_at: Instant::now(), - }) - } -} - -pub fn load_oidc_provider_configs_from_env() -> Vec { - OidcSys::parse_env_configs() -} - -pub fn load_oidc_provider_configs_from_server_config(cfg: &ServerConfig) -> Vec { - OidcSys::parse_persisted_configs(cfg) -} - -pub fn merge_oidc_provider_configs( - env_configs: Vec, - persisted_configs: Vec, -) -> Vec { - let mut effective = HashMap::new(); - - for config in persisted_configs { - effective.insert( - config.id.clone(), - SourcedOidcProviderConfig { - config, - source: OidcProviderConfigSource::Persisted, - }, - ); - } - - for config in env_configs { - effective.insert( - config.id.clone(), - SourcedOidcProviderConfig { - config, - source: OidcProviderConfigSource::Env, - }, - ); - } - - let mut configs: Vec = effective.into_values().collect(); - configs.sort_by(|lhs, rhs| lhs.config.id.cmp(&rhs.config.id)); - configs -} - -pub fn load_effective_oidc_provider_configs(server_config: Option<&ServerConfig>) -> Vec { - let env_configs = load_oidc_provider_configs_from_env(); - let persisted_configs = server_config - .map(load_oidc_provider_configs_from_server_config) - .unwrap_or_default(); - merge_oidc_provider_configs(env_configs, persisted_configs) -} - -pub fn load_oidc_config_snapshot(server_config: Option<&ServerConfig>) -> OidcConfigSnapshot { - OidcConfigSnapshot::new(load_effective_oidc_provider_configs(server_config)) -} - -pub async fn validate_oidc_provider_config(config: &OidcProviderConfig) -> Result { - validate_oidc_provider_config_with_extra_root_ca(config, None).await -} - -pub async fn validate_oidc_provider_config_with_extra_root_ca( - config: &OidcProviderConfig, - root_ca_pem: Option<&[u8]>, -) -> Result { - let http_client = ReqwestHttpClient::new_with_extra_root_certs(oidc_extra_root_certs(root_ca_pem)?)?; - let state = OidcSys::discover_provider(config, &http_client).await?; - - Ok(OidcProviderValidationResult { - issuer: state.metadata.issuer().to_string(), - authorization_endpoint: state.metadata.authorization_endpoint(), - token_endpoint: state.metadata.token_endpoint().map(ToString::to_string), - }) -} - -// --- Helper functions --- - -fn normalize_issuer(raw: &str) -> Option<(String, String, u16, String)> { - let parsed = Url::parse(raw).ok()?; - if parsed.scheme() != "http" && parsed.scheme() != "https" { - return None; - } - - let host = parsed.host_str()?.to_ascii_lowercase(); - let port = parsed.port_or_known_default()?; - let normalized_path = { - let path = parsed.path().trim_end_matches('/').to_string(); - if path.is_empty() { "/".to_string() } else { path } - }; - - Some((parsed.scheme().to_string(), host, port, normalized_path)) -} - -fn normalize_config_url(config_url: &str) -> Result { - let config_url = config_url.trim(); - let url = Url::parse(config_url).map_err(|e| format!("invalid config_url: {e}"))?; - if url.scheme() != "http" && url.scheme() != "https" { - return Err(format!("invalid config_url scheme: {}", url.scheme())); - } - let host = url.host_str().ok_or_else(|| "config_url missing host".to_string())?; - let path = url.path(); - - // Strip `/.well-known/openid-configuration` (with optional trailing slash) if present. - // Everything else is preserved exactly so the issuer URL matches the provider's discovery - // document (e.g. Authentik includes a trailing slash, Keycloak does not). - let normalized_path = path - .strip_suffix('/') - .unwrap_or(path) - .strip_suffix("/.well-known/openid-configuration") - .unwrap_or(if path == "/" { "" } else { path }); - - if normalized_path.contains("/.well-known/") { - return Err("config_url uses an unsupported .well-known discovery URL".into()); - } - - let mut issuer = format!("{}://{host}", url.scheme()); - if let Some(port) = url.port() { - issuer.push(':'); - issuer.push_str(&port.to_string()); - } - - if !normalized_path.is_empty() { - issuer.push_str(normalized_path); - } - - Ok(issuer) -} - -fn discovery_url_from_config_url(config_url: &str) -> Result { - let mut url = Url::parse(config_url.trim()).map_err(|e| format!("invalid config_url: {e}"))?; - if url.scheme() != "http" && url.scheme() != "https" { - return Err(format!("invalid config_url scheme: {}", url.scheme())); - } - if url.host_str().is_none() { - return Err("config_url missing host".to_string()); - } - - let path = url.path().to_string(); - let without_trailing_slash = path.strip_suffix('/').unwrap_or(&path); - if without_trailing_slash.ends_with("/.well-known/openid-configuration") { - url.set_path(without_trailing_slash); - return Ok(url); - } - if without_trailing_slash.contains("/.well-known/") { - return Err("config_url uses an unsupported .well-known discovery URL".into()); - } - - let discovery_path = if without_trailing_slash.is_empty() || without_trailing_slash == "/" { - "/.well-known/openid-configuration".to_string() - } else { - format!("{without_trailing_slash}/.well-known/openid-configuration") - }; - url.set_path(&discovery_path); - Ok(url) -} - -fn jwks_url_from_config_url( - config_url: &str, - issuer_url: &IssuerUrl, - jwks_url: &JsonWebKeySetUrl, -) -> Result { - let issuer = issuer_url.url(); - let jwks = jwks_url.url(); - if issuer.origin() != jwks.origin() { - return Ok(jwks_url.clone()); - } - - let issuer_path = issuer.path().trim_end_matches('/'); - let Some(suffix) = jwks.path().strip_prefix(issuer_path) else { - return Ok(jwks_url.clone()); - }; - if !suffix.is_empty() && !suffix.starts_with('/') { - return Ok(jwks_url.clone()); - } - - let mut internal_url = - Url::parse(&normalize_config_url(config_url)?).map_err(|err| format!("invalid config_url issuer base: {err}"))?; - let internal_path = internal_url.path().trim_end_matches('/'); - internal_url.set_path(&format!("{internal_path}{suffix}")); - internal_url.set_query(jwks.query()); - Ok(JsonWebKeySetUrl::from_url(internal_url)) -} - -fn issuer_candidates(base: &str) -> Vec { - let original = base.trim(); - let mut variants = Vec::with_capacity(2); - variants.push(original.to_string()); - - let toggled = if original.ends_with('/') { - original.trim_end_matches('/').to_string() - } else { - format!("{original}/") - }; - variants.push(toggled); - - variants -} - -/// Decode the payload section of a JWT without validation (token must already be verified). -pub(crate) fn decode_jwt_payload(token: &str) -> HashMap { - let parts: Vec<&str> = token.split('.').collect(); - if parts.len() < 2 { - return HashMap::new(); - } - let payload_bytes = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(parts[1]); - match payload_bytes { - Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_default(), - Err(_) => HashMap::new(), - } -} - -/// Extract a string claim from raw claims with case-insensitive fallback. -fn extract_string_claim(claims: &HashMap, key: &str) -> String { - match get_claim_case_insensitive(claims, key) { - ClaimLookup::Found(value) => value.as_str().unwrap_or_default().to_string(), - ClaimLookup::Missing | ClaimLookup::Ambiguous => String::new(), - } -} - -/// Extract a groups/array claim from raw claims with case-insensitive fallback. Handles both string arrays and single strings. -fn extract_groups_claim(claims: &HashMap, key: &str) -> Vec { - match get_claim_case_insensitive(claims, key) { - ClaimLookup::Found(serde_json::Value::Array(arr)) => arr.iter().filter_map(|v| v.as_str().map(String::from)).collect(), - ClaimLookup::Found(serde_json::Value::String(s)) => s.split(',').map(|s| s.trim().to_string()).collect(), - _ => vec![], - } -} - -fn extract_canonical_group_values( - claims: &HashMap, - groups_claim: &str, - roles_claim: &str, -) -> Vec { - let mut groups = extract_groups_claim(claims, groups_claim); - if !roles_claim.is_empty() && roles_claim != groups_claim { - groups.extend(extract_groups_claim(claims, roles_claim)); - } - groups.retain(|g| !g.is_empty()); - groups.sort(); - groups.dedup(); - groups -} - -#[cfg(any(test, feature = "test-util"))] -#[doc(hidden)] -pub fn make_test_sys(configs: Vec) -> OidcSys { - let configs = configs - .into_iter() - .map(test_sourced_config) - .map(|provider| (provider.config.id.clone(), provider)) - .collect(); - OidcSys { - configs, - provider_states: RwLock::new(HashMap::new()), - state_store: OidcStateStore::new(), - http_client: ReqwestHttpClient::new().expect("failed to initialize OIDC HTTP clients"), - } -} - -#[cfg(any(test, feature = "test-util"))] -fn test_sourced_config(config: OidcProviderConfig) -> SourcedOidcProviderConfig { - SourcedOidcProviderConfig { - config, - source: OidcProviderConfigSource::Persisted, - } -} - -#[cfg(any(test, feature = "test-util"))] -#[doc(hidden)] -pub fn test_config(id: &str) -> OidcProviderConfig { - OidcProviderConfig { - id: id.to_string(), - enabled: true, - config_url: format!("https://example.com/{id}/.well-known/openid-configuration"), - issuer: None, - client_id: "client-id".to_string(), - client_secret: None, - scopes: vec!["openid".to_string()], - other_audiences: vec![], - redirect_uri: None, - redirect_uri_dynamic: true, - claim_name: "groups".to_string(), - claim_prefix: String::new(), - role_policy: String::new(), - display_name: id.to_string(), - groups_claim: "groups".to_string(), - roles_claim: String::new(), - email_claim: "email".to_string(), - username_claim: "preferred_username".to_string(), - hide_from_ui: false, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use jsonwebtoken::{Algorithm, EncodingKey, Header}; - use rustfs_utils::egress::OutboundDnsPolicyRejection; - use std::sync::atomic::{AtomicUsize, Ordering}; - - #[derive(Clone)] - struct RejectingDnsResolver { - allow_origin_can_recover: bool, - calls: Option>, - } - - impl reqwest::dns::Resolve for RejectingDnsResolver { - fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { - if let Some(calls) = &self.calls { - calls.fetch_add(1, Ordering::Relaxed); - } - let host = name.as_str().to_string(); - let rejection = OutboundDnsPolicyRejection::new(host, self.allow_origin_can_recover); - Box::pin(async move { Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, rejection).into()) }) - } - } - - #[test] - fn test_extract_string_claim() { - let mut claims = HashMap::new(); - claims.insert("email".to_string(), serde_json::json!("user@example.com")); - claims.insert("sub".to_string(), serde_json::json!("12345")); - - assert_eq!(extract_string_claim(&claims, "email"), "user@example.com"); - assert_eq!(extract_string_claim(&claims, "sub"), "12345"); - assert_eq!(extract_string_claim(&claims, "missing"), ""); - } - - #[test] - fn format_http_headers_redacts_sensitive_values() { - let mut headers = http::HeaderMap::new(); - headers.insert(http::header::AUTHORIZATION, "Basic Y2xpZW50OnNlY3JldA==".parse().unwrap()); - headers.insert(http::header::CONTENT_TYPE, "application/json".parse().unwrap()); - headers.insert(http::header::COOKIE, "session=super-secret".parse().unwrap()); - - let rendered = format_http_headers(&headers); - - // Sensitive header values never appear; only their length is emitted. - assert!(!rendered.contains("Y2xpZW50OnNlY3JldA=="), "authorization value leaked: {rendered}"); - assert!(!rendered.contains("super-secret"), "cookie value leaked: {rendered}"); - assert!( - rendered.contains("authorization= OidcProviderConfig { - OidcProviderConfig { - id: id.to_string(), - enabled: true, - config_url: config_url.to_string(), - issuer: None, - client_id: "rustfs-oidc-test".to_string(), - client_secret: None, - scopes: vec!["openid".to_string()], - other_audiences: vec![], - redirect_uri: None, - redirect_uri_dynamic: false, - claim_name: "sub".to_string(), - claim_prefix: "oidc".to_string(), - role_policy: String::new(), - display_name: "mock-oidc".to_string(), - groups_claim: "groups".to_string(), - roles_claim: String::new(), - email_claim: "email".to_string(), - username_claim: "username".to_string(), - hide_from_ui: false, - } - } - - fn read_mock_oidc_request(stream: &mut impl std::io::Read) -> String { - let mut request_bytes = Vec::new(); - let mut buffer = [0u8; 4096]; - loop { - match stream.read(&mut buffer) { - Ok(0) => break, - Ok(n) => request_bytes.extend_from_slice(&buffer[..n]), - Err(e) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => break, - Err(_) => break, - } - if request_bytes.windows(4).any(|w| w == b"\r\n\r\n") { - break; - } - if request_bytes.len() >= 8192 { - break; - } - } - String::from_utf8_lossy(&request_bytes).into_owned() - } - - fn read_mock_oidc_request_path(stream: &mut impl std::io::Read) -> String { - read_mock_oidc_request(stream) - .lines() - .next() - .unwrap_or("") - .split_whitespace() - .nth(1) - .unwrap_or("") - .to_string() - } - - fn mock_oidc_response(path: &str, discovery_body: &str, expected_jwks_path: &str, jwks_body: &str) -> String { - let (status, body) = if path.ends_with("/.well-known/openid-configuration") { - (200, discovery_body) - } else if path == expected_jwks_path { - (200, jwks_body) - } else { - (404, r#"{"error":"not found"}"#) - }; - - format!( - "HTTP/1.1 {status} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - if status == 200 { "OK" } else { "Not Found" }, - body.len() - ) - } - - fn start_mock_oidc_discovery_server_with_jwks( - build_discovery_issuer: F, - max_requests: usize, - signing_alg: &'static str, - workload: bool, - jwks_response: J, - ) -> Option<(String, std::thread::JoinHandle<()>)> - where - F: Fn(&str) -> (String, String, String) + Send + 'static, - J: Fn(usize) -> String + Send + 'static, - { - use std::io::Write; - use std::net::{Shutdown, TcpListener}; - use std::sync::mpsc; - use std::time::{Duration, Instant}; - - // After the last completed response, exit if no new connection arrives within this window. - // Keep the mock server alive long enough for slower CI/macOS test environments to finish - // discovery + JWKS requests without racing the shutdown timer. - const IDLE_SHUTDOWN: Duration = Duration::from_secs(1); - const ABSOLUTE_CAP: Duration = Duration::from_secs(5); - - let listener = match TcpListener::bind("127.0.0.1:0") { - Ok(listener) => listener, - Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, - Err(err) => panic!("test listener should bind: {err}"), - }; - let base = format!("http://{}", listener.local_addr().expect("listener local address should be available")); - let (discovery_issuer, discovery_jwks_uri, expected_jwks_path) = build_discovery_issuer(&base); - let mut discovery_document = serde_json::json!({ - "issuer": discovery_issuer, - "authorization_endpoint": format!("{base}/authorize"), - "token_endpoint": format!("{base}/token"), - "jwks_uri": discovery_jwks_uri, - "response_types_supported": ["code"], - "response_modes_supported": ["query"], - "subject_types_supported": ["public"], - "id_token_signing_alg_values_supported": [signing_alg], - }); - if workload { - let fields = discovery_document.as_object_mut().expect("mock metadata is an object"); - fields.remove("authorization_endpoint"); - fields.remove("token_endpoint"); - fields.insert("response_types_supported".into(), serde_json::json!(["id_token"])); - } - let discovery_body = discovery_document.to_string(); - let (ready_tx, ready_rx) = mpsc::channel(); - - let handle = std::thread::spawn(move || { - listener - .set_nonblocking(true) - .expect("failed to set discovery mock listener non-blocking"); - let _ = ready_tx.send(()); - - let mut seen = 0usize; - let mut jwks_fetches = 0usize; - let start = Instant::now(); - let mut last_completed = Instant::now(); - - loop { - if seen > 0 && last_completed.elapsed() >= IDLE_SHUTDOWN { - break; - } - if start.elapsed() >= ABSOLUTE_CAP { - break; - } - - let mut stream = match listener.accept() { - Ok((stream, _)) => stream, - Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { - std::thread::sleep(Duration::from_millis(5)); - continue; - } - Err(_) => break, - }; - stream - .set_nonblocking(false) - .expect("failed to set discovery mock stream blocking"); - - seen += 1; - stream - .set_nonblocking(false) - .expect("failed to set discovery mock stream blocking"); - stream - .set_read_timeout(Some(Duration::from_secs(1))) - .expect("failed to set discovery mock read timeout"); - - let request = read_mock_oidc_request(&mut stream); - let path = request - .lines() - .next() - .and_then(|line| line.split_whitespace().nth(1)) - .unwrap_or(""); - let jwks_body = jwks_response(jwks_fetches); - if path == expected_jwks_path { - jwks_fetches += 1; - } - let mut response = mock_oidc_response(path, &discovery_body, &expected_jwks_path, &jwks_body); - if path.contains("/.well-known/openid-configuration") { - assert!( - request - .lines() - .filter_map(|line| line.split_once(':')) - .any(|(name, value)| { name.eq_ignore_ascii_case("accept") && value.trim() == "application/json" }), - "discovery Accept must remain unchanged" - ); - } - if path == expected_jwks_path { - let expected_type = if workload { - "application/jwk-set+json" - } else { - "application/json" - }; - let accepts_type = request.lines().filter_map(|line| line.split_once(':')).any(|(name, value)| { - name.eq_ignore_ascii_case("accept") && value.split(',').any(|item| item.trim() == expected_type) - }); - if !accepts_type { - response = "HTTP/1.1 406 Not Acceptable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".into(); - } else if workload { - response = response.replace("Content-Type: application/json", "Content-Type: application/jwk-set+json"); - } - } - let _ = stream.write_all(response.as_bytes()); - let _ = stream.flush(); - let _ = stream.shutdown(Shutdown::Both); - last_completed = Instant::now(); - - if seen >= max_requests { - break; - } - } - }); - ready_rx - .recv_timeout(Duration::from_millis(100)) - .expect("mock OIDC discovery server should become ready"); - - Some((base, handle)) - } - - fn start_mock_oidc_discovery_server( - build_discovery_issuer: F, - max_requests: usize, - ) -> Option<(String, std::thread::JoinHandle<()>)> - where - F: Fn(&str) -> (String, String, String) + Send + 'static, - { - start_mock_oidc_discovery_server_with_jwks(build_discovery_issuer, max_requests, "RS256", false, |_| { - r#"{"keys":[]}"#.to_string() - }) - } - - fn oidc_es256_key_and_jwk(kid: &str) -> (EncodingKey, serde_json::Value) { - let certified = - rcgen::generate_simple_self_signed(vec![format!("{kid}.invalid")]).expect("OIDC signing key should generate"); - let encoding_key = - EncodingKey::from_ec_pem(certified.signing_key.serialize_pem().as_bytes()).expect("OIDC signing key should encode"); - let mut jwk = jsonwebtoken::jwk::Jwk::from_encoding_key(&encoding_key, Algorithm::ES256) - .expect("OIDC public JWK should derive from signing key"); - jwk.common.key_id = Some(kid.to_string()); - jwk.common.public_key_use = Some(jsonwebtoken::jwk::PublicKeyUse::Signature); - (encoding_key, serde_json::to_value(jwk).expect("OIDC JWK should serialize")) - } - - #[tokio::test] - async fn web_identity_verification_refreshes_rotated_jwks() { - let (_, initial_jwk) = oidc_es256_key_and_jwk("initial"); - let (rotated_key, rotated_jwk) = oidc_es256_key_and_jwk("rotated"); - let initial_jwks = serde_json::json!({ "keys": [initial_jwk] }).to_string(); - let rotated_jwks = serde_json::json!({ "keys": [rotated_jwk] }).to_string(); - let Some((base, handle)) = start_mock_oidc_discovery_server_with_jwks( - |base| (base.to_string(), format!("{base}/jwks"), "/jwks".to_string()), - 4, - "ES256", - false, - move |fetch| { - if fetch == 0 { - initial_jwks.clone() - } else { - rotated_jwks.clone() - } - }, - ) else { - return; - }; - - let config = build_mocked_oidc_provider_config("rotating", &base); - let policy = OutboundPolicy::from_allowed_origins(&base).expect("loopback origin should be allowed"); - let http_client = ReqwestHttpClient::with_policy(policy); - let state = OidcSys::discover_provider(&config, &http_client) - .await - .expect("initial OIDC discovery should succeed"); - let sys = OidcSys { - configs: HashMap::from([(config.id.clone(), test_sourced_config(config.clone()))]), - provider_states: RwLock::new(HashMap::from([(config.id.clone(), state)])), - state_store: OidcStateStore::new(), - http_client, - }; - - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("system clock should be after Unix epoch") - .as_secs(); - let mut header = Header::new(Algorithm::ES256); - header.kid = Some("rotated".to_string()); - let token = jsonwebtoken::encode( - &header, - &serde_json::json!({ - "iss": base, - "sub": "rotated-user", - "aud": config.client_id, - "iat": now, - "exp": now + 300, - "groups": ["readwrite"], - }), - &rotated_key, - ) - .expect("rotated OIDC token should sign"); - - let (claims, provider_id) = sys - .verify_web_identity_token(&token) - .await - .expect("verification should refresh JWKS and accept the rotated key"); - assert_eq!(provider_id, "rotating"); - assert_eq!(claims.sub, "rotated-user"); - assert_eq!(claims.groups, vec!["readwrite"]); - handle.join().expect("rotating JWKS mock server should exit cleanly"); - } - - #[tokio::test] - async fn complete_provider_console_login_preserves_hidden_and_issuer_modes() { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - - for hidden in [false, true] { - for explicit in [false, true] { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base = format!("http://{}", listener.local_addr().unwrap()); - let issuer = if explicit { - "https://issuer.example.com".to_string() - } else { - base.clone() - }; - let mut config = - build_mocked_oidc_provider_config("console", &format!("{base}/.well-known/openid-configuration")); - config.hide_from_ui = hidden; - config.issuer = explicit.then(|| issuer.clone()); - config.client_secret = Some(Nonce::new_random().secret().clone()); - let server_config = config.clone(); - let server_base = base.clone(); - let redirect = "https://console.example.com/oauth_callback"; - let (key, jwk) = oidc_es256_key_and_jwk("console"); - let (auth_tx, auth_rx) = tokio::sync::oneshot::channel::>(); - let server = tokio::spawn(async move { - let mut auth_rx = Some(auth_rx); - for expected_path in ["/.well-known/openid-configuration", "/jwks", "/token"] { - let (mut stream, _) = tokio::time::timeout(StdDuration::from_secs(10), listener.accept()) - .await - .unwrap() - .unwrap(); - let mut bytes = Vec::new(); - let header_end = loop { - bytes.push(stream.read_u8().await.unwrap()); - assert!(bytes.len() < 8192); - if bytes.ends_with(b"\r\n\r\n") { - break bytes.len(); - } - }; - let headers = String::from_utf8(bytes).unwrap(); - let request_line = headers.lines().next().unwrap(); - assert_eq!(request_line.split_whitespace().nth(1), Some(expected_path)); - let headers_map: HashMap<_, _> = headers - .lines() - .skip(1) - .filter_map(|line| line.split_once(':')) - .map(|(name, value)| (name.to_ascii_lowercase(), value.trim().to_string())) - .collect(); - let body = match expected_path { - "/.well-known/openid-configuration" => { - assert!(request_line.starts_with("GET ")); - assert_eq!(headers_map["accept"], "application/json"); - serde_json::json!({ - "issuer": issuer, "authorization_endpoint": format!("{server_base}/authorize"), - "token_endpoint": format!("{server_base}/token"), "jwks_uri": format!("{server_base}/jwks"), - "response_types_supported": ["code"], "subject_types_supported": ["public"], - "id_token_signing_alg_values_supported": ["ES256"] - }) - } - "/jwks" => { - assert!(request_line.starts_with("GET ")); - assert!(headers_map["accept"].contains("application/json")); - serde_json::json!({"keys": [jwk]}) - } - "/token" => { - assert!(request_line.starts_with("POST ")); - assert_eq!(headers_map["accept"], "application/json"); - assert!(headers_map["content-type"].starts_with("application/x-www-form-urlencoded")); - let length: usize = headers_map["content-length"].parse().unwrap(); - assert!(header_end + length < 16384); - let mut body = vec![0; length]; - stream.read_exact(&mut body).await.unwrap(); - let form: HashMap = url::form_urlencoded::parse(&body).into_owned().collect(); - assert_eq!(form["grant_type"], "authorization_code"); - assert_eq!(form["code"], "test-authorization-code"); - assert_eq!(form["client_id"], server_config.client_id); - assert_eq!(Some(&form["client_secret"]), server_config.client_secret.as_ref()); - assert_eq!(form["redirect_uri"], redirect); - let auth = auth_rx.take().unwrap().await.unwrap(); - let challenge = PkceCodeChallenge::from_code_verifier_sha256(&PkceCodeVerifier::new(form["code_verifier"].clone())); - assert_eq!(challenge.as_str(), auth["code_challenge"]); - let now = time::OffsetDateTime::now_utc().unix_timestamp(); - let mut header = Header::new(Algorithm::ES256); - header.kid = Some("console".into()); - let token = jsonwebtoken::encode(&header, &serde_json::json!({ - "iss": issuer, "sub": "existing-user", "aud": server_config.client_id, - "iat": now, "exp": now + 300, "nonce": auth["nonce"], - "email": "user@example.com", "groups": ["readwrite"] - }), &key).unwrap(); - serde_json::json!({"access_token": "test-access-token", "token_type": "Bearer", "id_token": token}) - } - _ => unreachable!(), - }.to_string(); - stream.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).as_bytes()).await.unwrap(); - } - }); - let http_client = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).unwrap()); - let discovered = OidcSys::discover_provider(&config, &http_client).await.unwrap(); - let sys = OidcSys { - configs: HashMap::from([(config.id.clone(), test_sourced_config(config))]), - provider_states: RwLock::new(HashMap::from([("console".into(), discovered)])), - state_store: OidcStateStore::new(), - http_client, - }; - let auth_url = sys.authorize_url("console", redirect, Some("/buckets".into())).await.unwrap(); - let auth_url = Url::parse(&auth_url).unwrap(); - assert_eq!(auth_url.as_str().split('?').next(), Some(format!("{base}/authorize").as_str())); - let auth: HashMap = auth_url.query_pairs().into_owned().collect(); - assert_eq!(auth["response_type"], "code"); - assert_eq!(auth["client_id"], "rustfs-oidc-test"); - assert!(!auth["nonce"].is_empty()); - assert!(!auth["state"].is_empty()); - assert_eq!(auth["redirect_uri"], redirect); - assert_eq!(auth["code_challenge_method"], "S256"); - assert!(auth["scope"].split_whitespace().any(|scope| scope == "openid")); - let state = auth["state"].clone(); - auth_tx.send(auth).unwrap(); - let (claims, provider, session, _) = sys - .exchange_code(&state, "test-authorization-code", redirect) - .await - .unwrap_or_else(|err| panic!("hidden={hidden}, explicit={explicit}: {err}")); - assert_eq!(provider, "console"); - assert_eq!(claims.sub, "existing-user"); - assert_eq!(claims.email, "user@example.com"); - assert_eq!(claims.groups, vec!["readwrite"]); - assert_eq!(session.redirect_after.as_deref(), Some("/buckets")); - assert!(matches!(sys.exchange_code(&state, "test-authorization-code", redirect).await, - Err(error) if error == "invalid or expired OIDC state")); - server.await.unwrap(); - } - } - } - - #[test] - fn workload_metadata_requires_hidden_provider_and_valid_verification_fields() { - let document = serde_json::json!({ - "issuer": "https://issuer.example.com", - "jwks_uri": "https://issuer.example.com/jwks", - "response_types_supported": ["id_token"], - "subject_types_supported": ["public"], - "id_token_signing_alg_values_supported": ["ES256"], - }); - let parse = - |value: &serde_json::Value, hidden| DiscoveredProviderMetadata::parse(&serde_json::to_vec(value).unwrap(), hidden); - let metadata = parse(&document, true).expect("hidden workload metadata should parse"); - assert!(metadata.authorization_endpoint().is_none()); - assert!(metadata.console().err().unwrap().contains("only web identity")); - assert!(parse(&document, false).err().unwrap().contains("authorization_endpoint")); - for field in ["issuer", "jwks_uri", "id_token_signing_alg_values_supported"] { - let mut invalid = document.clone(); - invalid.as_object_mut().unwrap().remove(field); - assert!(parse(&invalid, true).err().unwrap().contains(field), "missing {field}"); - } - for endpoint in [serde_json::Value::Null, serde_json::json!(""), serde_json::json!("not a URL")] { - let mut invalid = document.clone(); - invalid["authorization_endpoint"] = endpoint; - assert!(parse(&invalid, true).is_err(), "invalid endpoint must not select workload metadata"); - } - let mut complete = document; - complete["authorization_endpoint"] = serde_json::json!("https://issuer.example.com/authorize"); - for hidden in [true, false] { - let metadata = parse(&complete, hidden).expect("full providers keep the existing parser"); - assert!(metadata.console().is_ok()); - assert!(metadata.token_endpoint().is_none(), "token endpoint remains optional"); - } - } - - #[test] - fn workload_metadata_rejects_duplicate_fields() { - for hidden in [false, true] { - let document = format!( - r#"{{"issuer":"https://wrong.example.com","issuer":"https://issuer.example.com",{}"jwks_uri":"https://issuer.example.com/jwks","response_types_supported":["id_token"],"subject_types_supported":["public"],"id_token_signing_alg_values_supported":["ES256"]}}"#, - if hidden { - "" - } else { - r#""authorization_endpoint":"https://issuer.example.com/authorize","# - }, - ); - let error = DiscoveredProviderMetadata::parse(document.as_bytes(), hidden) - .err() - .expect("duplicate issuer must fail"); - assert!(error.contains("duplicate field"), "{error}"); - } - } - - #[test] - fn workload_verifier_preserves_algorithm_secret_and_audience_policy() { - let secret = Nonce::new_random().secret().to_string(); - let mut config = build_mocked_oidc_provider_config("workload", "https://issuer.example.com"); - config.client_secret = Some(secret.clone()); - config.other_audiences = vec!["additional-audience".into()]; - let now = time::OffsetDateTime::now_utc().unix_timestamp(); - let payload = serde_json::json!({ - "iss": config.config_url, "sub": "repo:example/project:ref:refs/heads/main", - "aud": [config.client_id, "additional-audience"], "iat": now, "exp": now + 300, - }); - let signed = - jsonwebtoken::encode(&Header::new(Algorithm::HS256), &payload, &EncodingKey::from_secret(secret.as_bytes())).unwrap(); - let token: CoreIdToken = signed.parse().unwrap(); - for workload in [false, true] { - for (algorithms, accepted) in [ - (serde_json::json!(["HS256", "unsupported-future-algorithm"]), true), - (serde_json::json!(["ES256"]), false), - (serde_json::json!(["unsupported-future-algorithm"]), false), - ] { - let mut document = serde_json::json!({ - "issuer": config.config_url, "jwks_uri": "https://issuer.example.com/jwks", - "response_types_supported": ["id_token"], "subject_types_supported": ["public"], - "id_token_signing_alg_values_supported": algorithms, - }); - if !workload { - document["authorization_endpoint"] = serde_json::json!("https://issuer.example.com/authorize"); - } - let metadata = DiscoveredProviderMetadata::parse(&serde_json::to_vec(&document).unwrap(), true).unwrap(); - let verifier = metadata - .verifier(&config) - .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); - assert_eq!( - token.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(), - accepted, - "workload={workload}, algorithms={algorithms}" - ); - if accepted { - assert!( - token.claims(&metadata.verifier(&config), |_: Option<&Nonce>| Ok(())).is_err(), - "additional audiences require explicit trust" - ); - let mut wrong_secret = config.clone(); - wrong_secret.client_secret = Some(Nonce::new_random().secret().to_string()); - let verifier = metadata - .verifier(&wrong_secret) - .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); - assert!( - token.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(), - "incorrect client secret must fail" - ); - } - } - } - } - - #[tokio::test] - async fn workload_discovery_stops_after_forbidden_jwks() { - let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let seen = Arc::clone(&requests); - let (base, handle) = start_mock_oidc_discovery_server_with_jwks( - |base| (base.to_string(), "http://192.168.65.254:8080/jwks".into(), "/jwks".into()), - 2, - "ES256", - true, - move |_| { - seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - serde_json::json!({"keys": []}).to_string() - }, - ) - .expect("workload discovery mock must bind"); - let mut config = build_mocked_oidc_provider_config("workload", &base); - config.hide_from_ui = true; - let client = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).unwrap()); - let error = OidcSys::discover_provider(&config, &client) - .await - .err() - .expect("private JWKS must be blocked"); - assert!(error.starts_with(OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY), "{error}"); - handle.join().unwrap(); - assert_eq!( - requests.load(std::sync::atomic::Ordering::SeqCst), - 1, - "a policy denial must not retry discovery" - ); - } - - #[tokio::test] - async fn workload_config_validation_reports_absent_console_endpoints() { - let (base, handle) = start_mock_oidc_discovery_server_with_jwks( - |base| (base.to_string(), format!("{base}/jwks"), "/jwks".into()), - 2, - "ES256", - true, - |_| serde_json::json!({"keys": []}).to_string(), - ) - .expect("workload discovery mock must bind"); - let mut config = build_mocked_oidc_provider_config("workload", &base); - config.hide_from_ui = true; - // Inferred issuers keep the library's discovery URL construction. - config.config_url = format!("{base}/.well-known/openid-configuration?ignored=1#ignored"); - let result = validate_mocked_oidc_provider_config(&config) - .await - .expect("hidden workload configuration should validate"); - assert_eq!(result.issuer, base); - assert!(result.authorization_endpoint.is_none()); - assert!(result.token_endpoint.is_none()); - handle.join().unwrap(); - } - - #[tokio::test] - async fn workload_discovery_verification_and_rotation() { - for explicit_issuer in [false, true] { - let (_, initial_jwk) = oidc_es256_key_and_jwk("initial"); - let (key, rotated_jwk) = oidc_es256_key_and_jwk("rotated"); - let initial_jwks = serde_json::json!({"keys": [initial_jwk]}).to_string(); - let rotated_jwks = serde_json::json!({"keys": [rotated_jwk]}).to_string(); - let (base, handle) = start_mock_oidc_discovery_server_with_jwks( - |base| (base.to_string(), format!("{base}/jwks"), "/jwks".into()), - 4, - "ES256", - true, - move |fetch| { - if fetch == 0 { - initial_jwks.clone() - } else { - rotated_jwks.clone() - } - }, - ) - .expect("workload discovery mock must bind"); - let mut config = build_mocked_oidc_provider_config("workload", &base); - config.hide_from_ui = true; - if explicit_issuer { - config.issuer = Some(base.clone()); - } - let http_client = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).unwrap()); - let state = OidcSys::discover_provider(&config, &http_client) - .await - .expect("workload discovery must succeed"); - assert!(state.metadata.authorization_endpoint().is_none()); - let sys = OidcSys { - configs: HashMap::from([(config.id.clone(), test_sourced_config(config.clone()))]), - provider_states: RwLock::new(HashMap::from([(config.id.clone(), state)])), - state_store: OidcStateStore::new(), - http_client, - }; - assert!(sys.has_providers()); - assert!(sys.provider_configs().all(|provider| provider.hide_from_ui)); - let error = sys - .authorize_url(&config.id, "https://console.example.com/callback", None) - .await - .unwrap_err(); - assert!(error.contains("only web identity")); - let now = time::OffsetDateTime::now_utc().unix_timestamp(); - let payload = serde_json::json!({ - "iss": base, "sub": "system:serviceaccount:default:reader", "aud": [config.client_id], - "iat": now, "exp": now + 300, "groups": ["readonly"], - "kubernetes.io": {"namespace": "default", "serviceaccount": {"name": "reader"}}, - }); - let mut header = Header::new(Algorithm::ES256); - header.kid = Some("rotated".into()); - let token = jsonwebtoken::encode(&header, &payload, &key).unwrap(); - let (claims, provider) = sys - .verify_web_identity_token(&token) - .await - .expect("rotation must retain workload discovery support"); - assert_eq!(provider, config.id); - assert_eq!(claims.sub, "system:serviceaccount:default:reader"); - assert_eq!(claims.groups, ["readonly"]); - // Repeat after the mock exits: the verified snapshot must be cached. - handle.join().unwrap(); - assert!(sys.verify_web_identity_token(&token).await.is_ok()); - for (field, value, expected) in [ - ("iss", serde_json::json!("https://wrong.example.com"), "issuer"), - ("aud", serde_json::json!("wrong-audience"), "audience"), - ("exp", serde_json::json!(now - 60), "expired"), - ] { - let mut invalid = payload.clone(); - invalid[field] = value; - let token = jsonwebtoken::encode(&header, &invalid, &key).unwrap(); - let error = sys - .verify_web_identity_token(&token) - .await - .expect_err("invalid workload token must fail"); - assert!(error.to_lowercase().contains(expected), "{field}: {error}"); - } - let (wrong_key, _) = oidc_es256_key_and_jwk("wrong"); - let invalid = jsonwebtoken::encode(&header, &payload, &wrong_key).unwrap(); - let error = sys - .verify_web_identity_token(&invalid) - .await - .expect_err("wrong signature must fail"); - assert!(error.to_lowercase().contains("signature"), "{error}"); - assert!( - sys.verify_web_identity_token(&token).await.is_ok(), - "failed refresh must preserve the cached keys" - ); - } - } - - fn start_mock_oidc_tls_discovery_server( - build_discovery_issuer: F, - max_requests: usize, - ) -> Option<(String, String, std::thread::JoinHandle<()>)> - where - F: Fn(&str) -> (String, String, String) + Send + 'static, - { - use std::io::Write; - use std::net::{Shutdown, TcpListener}; - use std::sync::mpsc; - use std::time::{Duration, Instant}; - - const IDLE_SHUTDOWN: Duration = Duration::from_secs(1); - const ABSOLUTE_CAP: Duration = Duration::from_secs(5); - - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - let certified = - rcgen::generate_simple_self_signed(vec!["127.0.0.1".to_string()]).expect("generate OIDC TLS test certificate"); - let cert_pem = certified.cert.pem(); - let server_config = rustls::ServerConfig::builder() - .with_no_client_auth() - .with_single_cert( - vec![certified.cert.der().clone()], - rustls_pki_types::PrivateKeyDer::try_from(certified.signing_key.serialize_der()) - .expect("convert OIDC TLS test private key"), - ) - .expect("build OIDC TLS mock server config"); - - let listener = match TcpListener::bind("127.0.0.1:0") { - Ok(listener) => listener, - Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, - Err(err) => panic!("test TLS listener should bind: {err}"), - }; - let base = format!("https://{}", listener.local_addr().expect("listener local address should be available")); - let (discovery_issuer, discovery_jwks_uri, expected_jwks_path) = build_discovery_issuer(&base); - let discovery_body = serde_json::json!({ - "issuer": discovery_issuer, - "authorization_endpoint": format!("{base}/authorize"), - "token_endpoint": format!("{base}/token"), - "jwks_uri": discovery_jwks_uri, - "response_types_supported": ["code"], - "response_modes_supported": ["query"], - "subject_types_supported": ["public"], - "id_token_signing_alg_values_supported": ["RS256"], - }) - .to_string(); - let jwks_body = r#"{"keys":[]}"#; - let (ready_tx, ready_rx) = mpsc::channel(); - - let handle = std::thread::spawn(move || { - let server_config = Arc::new(server_config); - listener - .set_nonblocking(true) - .expect("failed to set TLS discovery mock listener non-blocking"); - let _ = ready_tx.send(()); - - let mut seen = 0usize; - let start = Instant::now(); - let mut last_completed = Instant::now(); - - loop { - if seen > 0 && last_completed.elapsed() >= IDLE_SHUTDOWN { - break; - } - if start.elapsed() >= ABSOLUTE_CAP { - break; - } - - let tcp_stream = match listener.accept() { - Ok((stream, _)) => stream, - Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { - std::thread::sleep(Duration::from_millis(5)); - continue; - } - Err(_) => break, - }; - tcp_stream - .set_nonblocking(false) - .expect("failed to set TLS discovery mock stream blocking"); - tcp_stream - .set_read_timeout(Some(Duration::from_secs(1))) - .expect("failed to set TLS discovery mock read timeout"); - - seen += 1; - let connection = match rustls::ServerConnection::new(server_config.clone()) { - Ok(connection) => connection, - Err(_) => break, - }; - let mut stream = rustls::StreamOwned::new(connection, tcp_stream); - let path = read_mock_oidc_request_path(&mut stream); - let response = mock_oidc_response(&path, &discovery_body, &expected_jwks_path, jwks_body); - let _ = stream.write_all(response.as_bytes()); - let _ = stream.flush(); - let _ = stream.sock.shutdown(Shutdown::Both); - last_completed = Instant::now(); - - if seen >= max_requests { - break; - } - } - }); - ready_rx - .recv_timeout(Duration::from_millis(100)) - .expect("mock TLS OIDC discovery server should become ready"); - - Some((base, cert_pem, handle)) - } - - fn discovery_error_contains_all_variants(err: &str, base: &str) -> bool { - err.contains(base) && err.contains(&format!("{base}/")) && err.contains("discovery failed for all issuer variants") - } - - async fn validate_mocked_oidc_provider_config(config: &OidcProviderConfig) -> Result { - // The mock discovery/JWKS/token endpoints share the loopback origin of `config_url`. - // Explicitly allow that origin so the egress policy does not reject the loopback mock. - let origin = Url::parse(&config.config_url) - .map_err(|_| "invalid mock config_url".to_string())? - .origin() - .ascii_serialization(); - let policy = OutboundPolicy::from_allowed_origins(&origin).map_err(|err| err.to_string())?; - let http_client = ReqwestHttpClient::with_policy(policy); - let state = OidcSys::discover_provider(config, &http_client).await?; - - Ok(OidcProviderValidationResult { - issuer: state.metadata.issuer().to_string(), - authorization_endpoint: state.metadata.authorization_endpoint(), - token_endpoint: state.metadata.token_endpoint().map(ToString::to_string), - }) - } - - #[tokio::test] - async fn oidc_discovery_accepts_extra_root_ca_for_https_provider() { - let Some((base, ca_pem, handle)) = start_mock_oidc_tls_discovery_server( - |base| (format!("{base}/application/o/rustfs"), format!("{base}/jwks"), "/jwks".to_string()), - 4, - ) else { - return; - }; - let config_url = format!("{base}/application/o/rustfs"); - let config = build_mocked_oidc_provider_config("default", &config_url); - let origin = Url::parse(&config.config_url) - .expect("mock config_url should parse") - .origin() - .ascii_serialization(); - let policy = OutboundPolicy::from_allowed_origins(&origin).expect("loopback TLS origin should be allowed"); - let extra_root_certs = - parse_oidc_extra_root_certs("test OIDC TLS CA", ca_pem.as_bytes()).expect("test CA bundle should parse"); - let http_client = ReqwestHttpClient::with_policy_and_extra_root_certs(policy, extra_root_certs); - - let state = OidcSys::discover_provider(&config, &http_client) - .await - .expect("OIDC discovery should trust the extra root CA"); - - assert_eq!(state.metadata.issuer().to_string(), format!("{base}/application/o/rustfs")); - assert!(handle.join().is_ok()); - } - - #[tokio::test] - async fn oidc_discovery_refreshes_extra_root_ca_when_generation_changes() { - let Some((base_a, ca_pem_a, handle_a)) = start_mock_oidc_tls_discovery_server( - |base| (format!("{base}/application/o/rustfs-a"), format!("{base}/jwks"), "/jwks".to_string()), - 4, - ) else { - return; - }; - let Some((base_b, ca_pem_b, handle_b)) = start_mock_oidc_tls_discovery_server( - |base| (format!("{base}/application/o/rustfs-b"), format!("{base}/jwks"), "/jwks".to_string()), - 4, - ) else { - assert!(handle_a.join().is_ok()); - return; - }; - - let origin_a = Url::parse(&base_a) - .expect("mock base A should parse") - .origin() - .ascii_serialization(); - let origin_b = Url::parse(&base_b) - .expect("mock base B should parse") - .origin() - .ascii_serialization(); - let allowed_origins = format!("{origin_a},{origin_b}"); - let policy = OutboundPolicy::from_allowed_origins(&allowed_origins).expect("loopback TLS origins should be allowed"); - let material = Arc::new(Mutex::new(OidcExtraRootCaMaterial { - generation: 1, - root_ca_pem: Some(ca_pem_a.into_bytes()), - })); - let provider = OidcExtraRootCaProvider::new({ - let material = material.clone(); - move || { - let material = material.clone(); - async move { - material - .lock() - .map(|material| material.clone()) - .map_err(|e| format!("test OIDC extra CA material lock poisoned: {e}")) - } - } - }); - let http_client = ReqwestHttpClient::with_policy_and_extra_root_ca_provider(policy, provider); - - let config_a = build_mocked_oidc_provider_config("a", &format!("{base_a}/application/o/rustfs-a")); - let state_a = OidcSys::discover_provider(&config_a, &http_client) - .await - .expect("OIDC discovery should trust initial extra root CA"); - assert_eq!(state_a.metadata.issuer().to_string(), format!("{base_a}/application/o/rustfs-a")); - - { - let mut material = material - .lock() - .expect("test OIDC extra CA material lock should not be poisoned"); - material.generation = 2; - material.root_ca_pem = Some(ca_pem_b.into_bytes()); - } - let config_b = build_mocked_oidc_provider_config("b", &format!("{base_b}/application/o/rustfs-b")); - let state_b = OidcSys::discover_provider(&config_b, &http_client) - .await - .expect("OIDC discovery should refresh extra root CA after generation change"); - - assert_eq!(state_b.metadata.issuer().to_string(), format!("{base_b}/application/o/rustfs-b")); - assert!(handle_a.join().is_ok()); - assert!(handle_b.join().is_ok()); - } - - #[tokio::test] - async fn test_validate_oidc_provider_config_retries_with_issuer_candidates() { - // Discovery document must advertise the canonical issuer path. The first candidate has no - // trailing slash; openidconnect rejects issuer mismatch, then the second variant succeeds. - let Some((base, handle)) = start_mock_oidc_discovery_server( - |base| (format!("{base}/application/o/rustfs/"), format!("{base}/jwks"), "/jwks".to_string()), - 8, - ) else { - return; - }; - let config_url = format!("{base}/application/o/rustfs"); - let config = build_mocked_oidc_provider_config("default", &config_url); - - let result = validate_mocked_oidc_provider_config(&config).await; - - let validation_result = result.expect("OIDC provider validation should succeed"); - assert_eq!(validation_result.issuer, format!("{base}/application/o/rustfs/")); - assert!(handle.join().is_ok()); - } - - #[tokio::test] - async fn test_validate_oidc_provider_config_fetches_issuer_relative_jwks_from_config_url() { - let Some((base, handle)) = start_mock_oidc_discovery_server( - |_| { - ( - "http://127.0.0.1:1/public/realms/app".to_string(), - "http://127.0.0.1:1/public/realms/app/jwks?version=1".to_string(), - "/internal/realms/app/jwks?version=1".to_string(), - ) - }, - 2, - ) else { - return; - }; - let mut config = - build_mocked_oidc_provider_config("default", &format!("{base}/internal/realms/app/.well-known/openid-configuration")); - config.issuer = Some("http://127.0.0.1:1/public/realms/app".to_string()); - - let validation_result = validate_mocked_oidc_provider_config(&config) - .await - .expect("OIDC provider validation should succeed"); - - assert_eq!(validation_result.issuer, "http://127.0.0.1:1/public/realms/app"); - assert!(handle.join().is_ok()); - } - - #[test] - fn test_jwks_url_from_config_url_preserves_unrelated_urls() { - let issuer = IssuerUrl::new("https://public.example.com/realms/app".to_string()).expect("issuer URL should parse"); - for raw_jwks_url in [ - "https://keys.example.com/jwks", - "http://public.example.com/realms/app/jwks", - "https://public.example.com:8443/realms/app/jwks", - "https://public.example.com/keys/jwks", - "https://public.example.com/realms/application/jwks", - ] { - let jwks_url = JsonWebKeySetUrl::new(raw_jwks_url.to_string()).expect("JWKS URL should parse"); - - let resolved = jwks_url_from_config_url( - "http://keycloak.internal/realms/app/.well-known/openid-configuration", - &issuer, - &jwks_url, - ) - .expect("JWKS URL should resolve"); - - assert_eq!(resolved.as_str(), raw_jwks_url); - } - - let issuer_root_jwks = JsonWebKeySetUrl::new(issuer.as_str().to_string()).expect("JWKS URL should parse"); - let resolved = jwks_url_from_config_url( - "http://keycloak.internal/realms/app/.well-known/openid-configuration", - &issuer, - &issuer_root_jwks, - ) - .expect("issuer-root JWKS URL should resolve"); - assert_eq!(resolved.as_str(), "http://keycloak.internal/realms/app"); - - let issuer_with_slash = - IssuerUrl::new("https://public.example.com/realms/app/".to_string()).expect("issuer URL should parse"); - let jwks_url = - JsonWebKeySetUrl::new("https://public.example.com/realms/app/jwks".to_string()).expect("JWKS URL should parse"); - let resolved = jwks_url_from_config_url( - "http://keycloak.internal/realms/app/.well-known/openid-configuration", - &issuer_with_slash, - &jwks_url, - ) - .expect("issuer-relative JWKS URL should resolve"); - assert_eq!(resolved.as_str(), "http://keycloak.internal/realms/app/jwks"); - } - - #[tokio::test] - async fn test_validate_oidc_provider_config_rejects_separate_issuer_mismatch() { - let Some((base, handle)) = start_mock_oidc_discovery_server( - |base| { - ( - "https://public.example.com/realms/other".to_string(), - format!("{base}/jwks"), - "/jwks".to_string(), - ) - }, - 1, - ) else { - return; - }; - let mut config = - build_mocked_oidc_provider_config("default", &format!("{base}/internal/realms/app/.well-known/openid-configuration")); - config.issuer = Some("https://public.example.com/realms/app".to_string()); - - let err = validate_mocked_oidc_provider_config(&config) - .await - .expect_err("OIDC provider validation should fail"); - - assert!(err.contains("unexpected issuer URI")); - assert!(err.contains("https://public.example.com/realms/app")); - assert!(handle.join().is_ok()); - } - - #[tokio::test] - async fn test_validate_oidc_provider_config_returns_detailed_errors() { - let Some((base, handle)) = start_mock_oidc_discovery_server( - |base| (format!("{base}/application/o/other"), format!("{base}/jwks"), "/jwks".to_string()), - 8, - ) else { - return; - }; - let config_url = format!("{base}/application/o/rustfs"); - let config = build_mocked_oidc_provider_config("default", &config_url); - - let err = validate_mocked_oidc_provider_config(&config) - .await - .expect_err("OIDC provider validation should fail"); - assert!(discovery_error_contains_all_variants(&err, &base)); - assert!(err.contains("issuer '")); - assert!(err.contains(&format!("issuer '{base}/application/o/rustfs'"))); - assert!(err.contains(&format!("issuer '{base}/application/o/rustfs/'"))); - assert!(handle.join().is_ok()); - } - - #[test] - fn test_decode_jwt_payload_invalid() { - assert!(decode_jwt_payload("not-a-jwt").is_empty()); - assert!(decode_jwt_payload("").is_empty()); - } - - #[test] - fn test_core_token_response_accepts_rfc3339_updated_at() { - // Signature verification happens later; this test covers token response deserialization. - let id_token = "eyJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJodHRwczovL2F1dGguZXhhbXBsZS5jb20vb2lkYyIsInN1YiI6InVzZXItMSIsImF1ZCI6InJ1c3RmcyIsImV4cCI6MTc4NDQzMjc2OSwiaWF0IjoxNzgzMjIzMTY5LCJ1cGRhdGVkX2F0IjoiMjAyNi0wNy0wM1QwNDo1MDo1MC44MTFaIn0.c2ln"; - let body = serde_json::json!({ - "scope": "openid roles profile email", - "token_type": "Bearer", - "access_token": "access-token", - "expires_in": 1209600, - "id_token": id_token, - }) - .to_string(); - - let response: openidconnect::core::CoreTokenResponse = - serde_json::from_str(&body).expect("RFC3339 updated_at should parse in token response"); - - assert!(response.extra_fields().id_token().is_some()); - } - - #[test] - fn test_map_claims_to_policies_no_provider() { - let sys = OidcSys::empty().expect("failed to initialize empty OIDC system"); - - let claims = OidcClaims { - sub: "user123".to_string(), - email: "user@example.com".to_string(), - username: "user".to_string(), - groups: vec!["admin".to_string(), "devs".to_string()], - raw: HashMap::new(), - }; - - let (policies, groups) = sys.map_claims_to_policies("nonexistent", &claims); - assert!(policies.is_empty()); - assert!(groups.is_empty()); - } - - #[test] - fn test_oidc_claims_default() { - let claims = OidcClaims::default(); - assert!(claims.sub.is_empty()); - assert!(claims.email.is_empty()); - assert!(claims.username.is_empty()); - assert!(claims.groups.is_empty()); - assert!(claims.raw.is_empty()); - } - - #[test] - fn test_oidc_claims_serde_roundtrip() { - let claims = OidcClaims { - sub: "user123".to_string(), - email: "user@example.com".to_string(), - username: "testuser".to_string(), - groups: vec!["admin".to_string(), "devs".to_string()], - raw: { - let mut m = HashMap::new(); - m.insert("custom".to_string(), serde_json::json!("value")); - m - }, - }; - - let json = serde_json::to_string(&claims).unwrap(); - let deserialized: OidcClaims = serde_json::from_str(&json).unwrap(); - assert_eq!(deserialized.sub, "user123"); - assert_eq!(deserialized.email, "user@example.com"); - assert_eq!(deserialized.groups.len(), 2); - } - - #[test] - fn test_oidc_provider_summary_serde() { - let summary = OidcProviderSummary { - provider_id: "okta".to_string(), - display_name: "Okta SSO".to_string(), - }; - - let json = serde_json::to_string(&summary).unwrap(); - let deserialized: OidcProviderSummary = serde_json::from_str(&json).unwrap(); - assert_eq!(deserialized.provider_id, "okta"); - assert_eq!(deserialized.display_name, "Okta SSO"); - } - - #[test] - fn test_parse_single_provider_no_config_url() { - let config = OidcSys::parse_single_provider("_TEST_EMPTY", "test_empty"); - assert!(config.is_none()); - } - - #[test] - fn test_parse_single_provider_reads_issuer() { - temp_env::with_vars( - [ - ( - ENV_IDENTITY_OPENID_CONFIG_URL, - Some("http://keycloak.ns.svc.cluster.local:8080/realms/app/.well-known/openid-configuration"), - ), - (ENV_IDENTITY_OPENID_ISSUER, Some("https://app.local/realms/app")), - (ENV_IDENTITY_OPENID_CLIENT_ID, Some("console")), - ], - || { - let config = OidcSys::parse_single_provider("", "default").expect("provider config should parse"); - - assert_eq!(config.issuer.as_deref(), Some("https://app.local/realms/app")); - }, - ); - } - - #[test] - fn test_parse_persisted_provider_config() { - let mut cfg = ServerConfig::new(); - let mut kvs = KVS(vec![ - rustfs_config::server_config::KV { - key: ENABLE_KEY.to_string(), - value: EnableState::Off.to_string(), - hidden_if_empty: false, - }, - rustfs_config::server_config::KV { - key: OIDC_CONFIG_URL.to_string(), - value: String::new(), - hidden_if_empty: false, - }, - rustfs_config::server_config::KV { - key: OIDC_CLIENT_ID.to_string(), - value: String::new(), - hidden_if_empty: false, - }, - ]); - kvs.insert( - OIDC_CONFIG_URL.to_string(), - "https://example.com/.well-known/openid-configuration".to_string(), - ); - kvs.insert(OIDC_CLIENT_ID.to_string(), "console".to_string()); - kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); - kvs.insert(OIDC_ISSUER.to_string(), "https://issuer.example".to_string()); - kvs.insert(OIDC_ROLES_CLAIM.to_string(), "app_roles".to_string()); - - cfg.0 - .entry(IDENTITY_OPENID_SUB_SYS.to_string()) - .or_default() - .insert(DEFAULT_DELIMITER.to_string(), kvs); - - let parsed = OidcSys::parse_persisted_configs(&cfg); - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].id, "default"); - assert_eq!(parsed[0].client_id, "console"); - assert_eq!(parsed[0].issuer.as_deref(), Some("https://issuer.example")); - assert!(parsed[0].enabled); - assert_eq!(parsed[0].roles_claim, "app_roles"); - } - - #[test] - fn test_parse_persisted_provider_config_omitted_roles_claim_is_empty() { - let mut cfg = ServerConfig::new(); - let mut kvs = KVS(vec![ - rustfs_config::server_config::KV { - key: ENABLE_KEY.to_string(), - value: EnableState::Off.to_string(), - hidden_if_empty: false, - }, - rustfs_config::server_config::KV { - key: OIDC_CONFIG_URL.to_string(), - value: String::new(), - hidden_if_empty: false, - }, - rustfs_config::server_config::KV { - key: OIDC_CLIENT_ID.to_string(), - value: String::new(), - hidden_if_empty: false, - }, - ]); - kvs.insert( - OIDC_CONFIG_URL.to_string(), - "https://example.com/.well-known/openid-configuration".to_string(), - ); - kvs.insert(OIDC_CLIENT_ID.to_string(), "console".to_string()); - kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); - - cfg.0 - .entry(IDENTITY_OPENID_SUB_SYS.to_string()) - .or_default() - .insert(DEFAULT_DELIMITER.to_string(), kvs); - - let parsed = OidcSys::parse_persisted_configs(&cfg); - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].roles_claim, ""); - } - - #[test] - fn test_merge_oidc_provider_configs_prefers_env() { - let mut persisted = test_config("default"); - persisted.display_name = "Persisted".to_string(); - - let mut env = test_config("default"); - env.display_name = "Environment".to_string(); - - let merged = merge_oidc_provider_configs(vec![env], vec![persisted]); - assert_eq!(merged.len(), 1); - assert_eq!(merged[0].config.display_name, "Environment"); - assert_eq!(merged[0].source, OidcProviderConfigSource::Env); - } - - #[test] - fn test_oidc_sys_empty() { - let sys = OidcSys::empty().expect("failed to initialize empty OIDC system"); - assert!(!sys.has_providers()); - assert!(sys.config_snapshot().providers().is_empty()); - } - - #[test] - fn build_oidc_http_client_rejects_forbidden_targets_without_allowlist() { - // Cloud metadata endpoint is never allowed. - assert!( - matches!( - build_oidc_http_client("http://169.254.169.254/latest/meta-data/", None, &[], None), - Err(OidcHttpError::ForbiddenOutbound(_)) - ), - "metadata endpoint must be rejected" - ); - // Loopback is rejected by default (no allow-origins configured). - assert!( - matches!( - build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", None, &[], None), - Err(OidcHttpError::ForbiddenOutbound(_)) - ), - "loopback must be rejected by default" - ); - // A public hostname passes the up-front shape/host check; the resolved IP is still - // re-classified at connection time by the pinned resolver. - assert!( - build_oidc_http_client("https://accounts.example.com/.well-known/openid-configuration", None, &[], None).is_ok(), - "public https endpoint should build" - ); - } - - #[test] - fn test_should_bypass_proxy_for_oidc_uri_loopback_only() { - assert!(should_bypass_proxy_for_oidc_uri("http://127.0.0.1:9000/.well-known/openid-configuration")); - assert!(should_bypass_proxy_for_oidc_uri("http://localhost:9000/.well-known/openid-configuration")); - assert!(should_bypass_proxy_for_oidc_uri("http://[::1]:9000/.well-known/openid-configuration")); - assert!(!should_bypass_proxy_for_oidc_uri( - "https://idp.example.com/.well-known/openid-configuration" - )); - assert!(!should_bypass_proxy_for_oidc_uri("not-a-url")); - } - - #[test] - fn build_oidc_http_client_honors_explicit_allowlist_for_loopback() { - let policy = OutboundPolicy::from_allowed_origins("http://127.0.0.1:8080").expect("origin should parse"); - assert!( - build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", Some(&policy), &[], None).is_ok(), - "explicitly allow-listed loopback origin should build" - ); - // A metadata endpoint stays forbidden even when a loopback origin is allow-listed. - assert!( - matches!( - build_oidc_http_client("http://169.254.169.254/", Some(&policy), &[], None), - Err(OidcHttpError::ForbiddenOutbound(_)) - ), - "metadata endpoint stays forbidden despite an unrelated allow-list entry" - ); - } - - #[tokio::test] - async fn oidc_discovery_reports_forbidden_outbound_without_retrying() { - let config_url = "http://192.168.65.254:8080/realms/rustfs/.well-known/openid-configuration"; - let config = build_mocked_oidc_provider_config("default", config_url); - let http_client = ReqwestHttpClient::with_policy(OutboundPolicy::default()); - - let error = match OidcSys::discover_provider(&config, &http_client).await { - Ok(_) => panic!("private OIDC provider should require an explicit allowlist origin"), - Err(error) => error, - }; - - assert!(error.contains("OIDC provider discovery blocked by outbound policy")); - assert!(error.contains(&format!("add http://192.168.65.254:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}"))); - assert!(!error.contains("discovery failed for all issuer variants")); - } - - #[tokio::test] - async fn oidc_explicit_issuer_reports_forbidden_discovery_endpoint() { - let config_url = "http://192.168.65.254:8080/realms/rustfs/.well-known/openid-configuration"; - let mut config = build_mocked_oidc_provider_config("default", config_url); - config.issuer = Some("https://idp.example.com/realms/rustfs".to_string()); - let http_client = ReqwestHttpClient::with_policy(OutboundPolicy::default()); - - let error = match OidcSys::discover_provider(&config, &http_client).await { - Ok(_) => panic!("private OIDC provider should require an explicit allowlist origin"), - Err(error) => error, - }; - - assert!(error.contains("OIDC provider discovery blocked by outbound policy")); - assert!(error.contains(&format!("add http://192.168.65.254:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}"))); - } - - #[tokio::test] - async fn oidc_explicit_issuer_reports_forbidden_jwks_endpoint() { - let Some((base, handle)) = start_mock_oidc_discovery_server( - |base| { - ( - format!("{base}/realms/rustfs"), - "http://192.168.65.254:8080/realms/rustfs/protocol/openid-connect/certs".to_string(), - "/unused".to_string(), - ) - }, - 1, - ) else { - return; - }; - let mut config = - build_mocked_oidc_provider_config("default", &format!("{base}/realms/rustfs/.well-known/openid-configuration")); - config.issuer = Some(format!("{base}/realms/rustfs")); - let policy = OutboundPolicy::from_allowed_origins(&base).expect("loopback discovery origin should be allowed"); - let http_client = ReqwestHttpClient::with_policy(policy); - - let error = match OidcSys::discover_provider(&config, &http_client).await { - Ok(_) => panic!("private JWKS endpoint should require an explicit allowlist origin"), - Err(error) => error, - }; - - assert!(error.contains("JWKS request blocked by outbound policy")); - assert!(error.contains(&format!("add http://192.168.65.254:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}"))); - assert!(handle.join().is_ok()); - } - - #[tokio::test] - async fn oidc_token_exchange_reports_forbidden_token_endpoint() { - let provider_id = "default"; - let token_endpoint = "http://192.168.65.254:8080/realms/rustfs/protocol/openid-connect/token"; - let metadata = serde_json::from_value::(serde_json::json!({ - "issuer": "https://idp.example.com/realms/rustfs", - "authorization_endpoint": "https://idp.example.com/realms/rustfs/protocol/openid-connect/auth", - "token_endpoint": token_endpoint, - "jwks_uri": "https://idp.example.com/realms/rustfs/protocol/openid-connect/certs", - "response_types_supported": ["code"], - "subject_types_supported": ["public"], - "id_token_signing_alg_values_supported": ["RS256"] - })) - .expect("provider metadata should parse"); - let config = build_mocked_oidc_provider_config( - provider_id, - "https://idp.example.com/realms/rustfs/.well-known/openid-configuration", - ); - let state_store = OidcStateStore::new(); - state_store - .insert( - "test-state".to_string(), - OidcAuthSession { - provider_id: provider_id.to_string(), - pkce_verifier: "test-pkce-verifier".to_string(), - nonce: "test-nonce".to_string(), - redirect_after: None, - }, - ) - .await; - let sys = OidcSys { - configs: HashMap::from([(provider_id.to_string(), test_sourced_config(config))]), - provider_states: RwLock::new(HashMap::from([( - provider_id.to_string(), - ProviderState { - metadata: DiscoveredProviderMetadata::Console(Box::new(metadata)), - discovered_at: Instant::now(), - }, - )])), - state_store, - http_client: ReqwestHttpClient::with_policy(OutboundPolicy::default()), - }; - - let error = match sys - .exchange_code("test-state", "test-code", "https://console.example.com/oauth_callback") - .await - { - Ok(_) => panic!("private token endpoint should require an explicit allowlist origin"), - Err(error) => error, - }; - - assert!(error.contains("request_error_kind=forbidden_outbound")); - assert!(error.contains(&format!("add http://192.168.65.254:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}"))); - } - - #[tokio::test] - async fn oidc_reqwest_dns_policy_rejection_stays_typed() { - let calls = Arc::new(AtomicUsize::new(0)); - let config = build_mocked_oidc_provider_config( - "default", - "http://keycloak.internal:8080/realms/rustfs/.well-known/openid-configuration", - ); - let http_client = ReqwestHttpClient::with_policy_and_dns_resolver( - OutboundPolicy::default(), - Arc::new(RejectingDnsResolver { - allow_origin_can_recover: true, - calls: Some(calls.clone()), - }), - ); - - let error = match OidcSys::discover_provider(&config, &http_client).await { - Ok(_) => panic!("private DNS answer should fail discovery"), - Err(error) => error, - }; - - assert!(error.contains("OIDC provider discovery blocked by outbound policy")); - assert!(error.contains(&format!("add http://keycloak.internal:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}"))); - assert_eq!(calls.load(Ordering::Relaxed), 1, "policy rejection must not be retried"); - } - - #[tokio::test] - async fn oidc_explicit_issuer_preserves_dns_policy_rejection() { - let calls = Arc::new(AtomicUsize::new(0)); - let mut config = build_mocked_oidc_provider_config( - "default", - "http://keycloak.internal:8080/realms/rustfs/.well-known/openid-configuration", - ); - config.issuer = Some("https://idp.example.com/realms/rustfs".to_string()); - let http_client = ReqwestHttpClient::with_policy_and_dns_resolver( - OutboundPolicy::default(), - Arc::new(RejectingDnsResolver { - allow_origin_can_recover: true, - calls: Some(calls.clone()), - }), - ); - - let error = match OidcSys::discover_provider(&config, &http_client).await { - Ok(_) => panic!("private DNS answer should fail discovery"), - Err(error) => error, - }; - - assert!(error.contains("OIDC provider discovery blocked by outbound policy")); - assert!(error.contains(&format!("add http://keycloak.internal:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}"))); - assert_eq!(calls.load(Ordering::Relaxed), 1, "policy rejection must not be retried"); - } - - #[tokio::test] - async fn oidc_nonrecoverable_dns_policy_rejection_omits_allowlist_hint() { - let uri = "http://metadata.internal/latest/meta-data"; - let http_client = ReqwestHttpClient::with_policy_and_dns_resolver( - OutboundPolicy::default(), - Arc::new(RejectingDnsResolver { - allow_origin_can_recover: false, - calls: None, - }), - ); - let request = http::Request::builder() - .uri(uri) - .body(Vec::new()) - .expect("request should build"); - - let error = http_client - .call(request) - .await - .expect_err("metadata DNS answer should be rejected"); - let message = error.to_string(); - - assert!(matches!(error, OidcHttpError::ForbiddenOutbound(_))); - assert!(message.contains("metadata.internal")); - assert!(!message.contains(&format!("add http://metadata.internal to {ENV_OUTBOUND_ALLOW_ORIGINS}"))); - } - - #[test] - fn oidc_metadata_endpoint_rejection_does_not_offer_allowlist_bypass() { - let error = - build_oidc_http_client("http://169.254.169.254/latest/meta-data/", Some(&OutboundPolicy::default()), &[], None) - .expect_err("metadata endpoint must remain forbidden"); - let message = error.to_string(); - - assert!(message.contains("metadata endpoint")); - assert!(!message.contains(&format!("add http://169.254.169.254 to {ENV_OUTBOUND_ALLOW_ORIGINS}"))); - } - - /// Serve exactly `body_len` bytes with no `Content-Length`, so the body ends only at EOF - /// and the size guard cannot rely on an advertised length. - fn start_unbounded_body_server(body_len: usize) -> Option<(String, std::thread::JoinHandle<()>)> { - use std::io::{Read, Write}; - use std::net::TcpListener; - use std::sync::mpsc; - use std::time::Duration; - - let listener = match TcpListener::bind("127.0.0.1:0") { - Ok(listener) => listener, - Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, - Err(err) => panic!("test listener should bind: {err}"), - }; - let base = format!("http://{}", listener.local_addr().expect("listener local address should be available")); - let (ready_tx, ready_rx) = mpsc::channel(); - - let handle = std::thread::spawn(move || { - let _ = ready_tx.send(()); - let Ok((mut stream, _)) = listener.accept() else { - return; - }; - let _ = stream.set_read_timeout(Some(Duration::from_secs(1))); - let mut buffer = [0u8; 4096]; - let _ = stream.read(&mut buffer); - let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n"); - - let chunk = vec![b'a'; 64 * 1024]; - let mut written = 0usize; - while written < body_len { - let take = chunk.len().min(body_len - written); - if stream.write_all(&chunk[..take]).is_err() { - break; - } - written += take; - } - let _ = stream.flush(); - }); - ready_rx - .recv_timeout(Duration::from_millis(100)) - .expect("mock body server should become ready"); - - Some((base, handle)) - } - - async fn fetch_oidc_mock_body(base: &str) -> Result, OidcHttpError> { - let policy = OutboundPolicy::from_allowed_origins(base).expect("origin should parse"); - let client = ReqwestHttpClient::with_policy(policy); - let request = http::Request::builder() - .method(http::Method::GET) - .uri(base) - .body(Vec::new()) - .expect("request should build"); - client.call(request).await.map(http::Response::into_body) - } - - #[tokio::test] - async fn oidc_response_body_at_the_limit_is_accepted() { - let Some((base, handle)) = start_unbounded_body_server(MAX_OIDC_RESPONSE_SIZE) else { - return; - }; - - let body = fetch_oidc_mock_body(&base) - .await - .expect("a body at the limit must be accepted"); - - assert_eq!(body.len(), MAX_OIDC_RESPONSE_SIZE); - handle.join().expect("mock body server thread should exit"); - } - - #[tokio::test] - async fn oidc_response_body_past_the_limit_is_rejected() { - let Some((base, handle)) = start_unbounded_body_server(MAX_OIDC_RESPONSE_SIZE + 1) else { - return; - }; - - let err = fetch_oidc_mock_body(&base) - .await - .map(|body| body.len()) - .expect_err("an oversized provider response must fail closed instead of being buffered"); - - assert!( - matches!(err, OidcHttpError::ResponseTooLarge(MAX_OIDC_RESPONSE_SIZE)), - "unexpected error: {err}" - ); - handle.join().expect("mock body server thread should exit"); - } - - #[test] - fn test_oidc_provider_config_debug_redacts_client_secret() { - let config = OidcProviderConfig { - client_secret: Some("oidc-client-secret".to_string()), - ..test_config("default") - }; - let sourced = SourcedOidcProviderConfig { - config, - source: OidcProviderConfigSource::Persisted, - }; - - let rendered = format!("{sourced:?}"); - - assert!(!rendered.contains("oidc-client-secret")); - assert!(rendered.contains(REDACTED_SECRET)); - assert!(rendered.contains("client-id")); - } - - #[test] - fn test_parse_enable_state_on() { - assert!(OidcSys::parse_enable_state("on", false, false)); - } - - #[test] - fn test_parse_enable_state_off() { - assert!(!OidcSys::parse_enable_state("off", true, true)); - } - - #[test] - fn test_parse_enable_state_empty_returns_default() { - assert!(OidcSys::parse_enable_state("", true, false)); - assert!(!OidcSys::parse_enable_state("", false, true)); - } - - #[test] - fn test_parse_enable_state_invalid_returns_error_default() { - assert!(!OidcSys::parse_enable_state("garbage", true, false)); - assert!(OidcSys::parse_enable_state("garbage", false, true)); - } - - #[test] - fn test_list_visible_providers_hides_hidden_provider() { - let visible = test_config("dex"); - let mut hidden = test_config("kubernetes"); - hidden.hide_from_ui = true; - - let sys = make_test_sys(vec![visible, hidden]); - let listed = sys.list_visible_providers(); - - assert_eq!(listed.len(), 1); - assert!(listed.iter().any(|provider| provider.provider_id == "dex")); - assert!(!listed.iter().any(|provider| provider.provider_id == "kubernetes")); - } - - #[test] - fn test_hidden_provider_still_resolvable_for_sts() { - let visible = test_config("dex"); - let mut hidden = test_config("kubernetes"); - hidden.hide_from_ui = true; - - let sys = make_test_sys(vec![visible, hidden]); - - assert!(sys.get_provider_config("kubernetes").is_some()); - assert!(sys.get_provider_config("dex").is_some()); - } - - #[test] - fn test_config_snapshot_includes_hidden_for_replication() { - let visible = test_config("dex"); - let mut hidden = test_config("kubernetes"); - hidden.hide_from_ui = true; - - let sys = make_test_sys(vec![visible, hidden]); - - assert_eq!(sys.config_snapshot().providers().len(), 2); - } - - #[test] - fn test_list_providers_includes_hidden_for_compatibility() { - let visible = test_config("dex"); - let mut hidden = test_config("kubernetes"); - hidden.hide_from_ui = true; - - let sys = make_test_sys(vec![visible, hidden]); - - assert_eq!(sys.list_providers().len(), 2); - assert_eq!(sys.list_visible_providers().len(), 1); - } - - #[test] - fn test_hide_from_ui_default_is_false() { - let config = test_config("default"); - assert!(!config.hide_from_ui); - } - - #[test] - fn test_parse_persisted_hide_from_ui_off_is_false() { - let mut cfg = ServerConfig::new(); - let mut kvs = KVS(vec![ - rustfs_config::server_config::KV { - key: ENABLE_KEY.to_string(), - value: EnableState::On.to_string(), - hidden_if_empty: false, - }, - rustfs_config::server_config::KV { - key: OIDC_CONFIG_URL.to_string(), - value: "https://example.com/.well-known/openid-configuration".to_string(), - hidden_if_empty: false, - }, - rustfs_config::server_config::KV { - key: OIDC_CLIENT_ID.to_string(), - value: "console".to_string(), - hidden_if_empty: false, - }, - ]); - kvs.insert(OIDC_HIDE_FROM_UI.to_string(), EnableState::Off.to_string()); - - cfg.0 - .entry(IDENTITY_OPENID_SUB_SYS.to_string()) - .or_default() - .insert(DEFAULT_DELIMITER.to_string(), kvs); - - let parsed = OidcSys::parse_persisted_configs(&cfg); - assert_eq!(parsed.len(), 1); - assert!(!parsed[0].hide_from_ui); - } - - #[test] - fn test_parse_persisted_hide_from_ui_missing_defaults_false() { - let mut cfg = ServerConfig::new(); - let kvs = KVS(vec![ - rustfs_config::server_config::KV { - key: ENABLE_KEY.to_string(), - value: EnableState::On.to_string(), - hidden_if_empty: false, - }, - rustfs_config::server_config::KV { - key: OIDC_CONFIG_URL.to_string(), - value: "https://example.com/.well-known/openid-configuration".to_string(), - hidden_if_empty: false, - }, - rustfs_config::server_config::KV { - key: OIDC_CLIENT_ID.to_string(), - value: "console".to_string(), - hidden_if_empty: false, - }, - ]); - - cfg.0 - .entry(IDENTITY_OPENID_SUB_SYS.to_string()) - .or_default() - .insert(DEFAULT_DELIMITER.to_string(), kvs); - - let parsed = OidcSys::parse_persisted_configs(&cfg); - assert_eq!(parsed.len(), 1); - assert!(!parsed[0].hide_from_ui); - } - - #[test] - fn test_parse_persisted_hide_from_ui() { - let mut cfg = ServerConfig::new(); - let mut kvs = KVS(vec![ - rustfs_config::server_config::KV { - key: ENABLE_KEY.to_string(), - value: EnableState::On.to_string(), - hidden_if_empty: false, - }, - rustfs_config::server_config::KV { - key: OIDC_CONFIG_URL.to_string(), - value: "https://example.com/.well-known/openid-configuration".to_string(), - hidden_if_empty: false, - }, - rustfs_config::server_config::KV { - key: OIDC_CLIENT_ID.to_string(), - value: "console".to_string(), - hidden_if_empty: false, - }, - ]); - kvs.insert(OIDC_HIDE_FROM_UI.to_string(), EnableState::On.to_string()); - - cfg.0 - .entry(IDENTITY_OPENID_SUB_SYS.to_string()) - .or_default() - .insert(DEFAULT_DELIMITER.to_string(), kvs); - - let parsed = OidcSys::parse_persisted_configs(&cfg); - assert_eq!(parsed.len(), 1); - assert!(parsed[0].hide_from_ui); - } - - #[test] - fn role_policy_does_not_map_groups_as_policies() { - let mut config = test_config("authentik"); - config.role_policy = "consoleAdmin".to_string(); - config.claim_name = "policy".to_string(); - - let sys = make_test_sys(vec![config]); - - let claims = OidcClaims { - groups: vec!["authentik Admins".to_string(), "users".to_string()], - raw: HashMap::from([("policy".to_string(), serde_json::json!(["readonly"]))]), - ..Default::default() - }; - - let (policies, groups) = sys.map_claims_to_policies("authentik", &claims); - assert_eq!(groups, vec!["authentik Admins", "users"]); - assert_eq!(policies, vec!["consoleAdmin"]); - } - - #[test] - fn group_claim_policy_names_cover_only_prefixed_group_values() { - let mut config = test_config("ad"); - config.claim_prefix = "oidc-".to_string(); - let sys = make_test_sys(vec![config]); - let claims = OidcClaims { - groups: vec![ - "EXAMPLE\\Domain Users".to_string(), - "admins".to_string(), - "admins".to_string(), - ], - ..Default::default() - }; - - assert_eq!( - sys.group_claim_policy_names("ad", &claims), - vec!["oidc-EXAMPLE\\Domain Users", "oidc-admins"] - ); - assert!(sys.group_claim_policy_names("missing-provider", &claims).is_empty()); - } - - #[test] - fn group_claim_policy_names_include_merged_roles_claim_values() { - let mut config = test_config("keycloak"); - config.roles_claim = "roles".to_string(); - let raw = HashMap::from([ - ("groups".to_string(), serde_json::json!(["readonly"])), - ("roles".to_string(), serde_json::json!(["offline_access", "default-roles-corp"])), - ]); - let claims = OidcClaims { - groups: extract_canonical_group_values(&raw, &config.groups_claim, &config.roles_claim), - raw, - ..Default::default() - }; - let sys = make_test_sys(vec![config]); - - assert_eq!( - sys.group_claim_policy_names("keycloak", &claims), - vec!["default-roles-corp", "offline_access", "readonly"] - ); - } - - #[test] - fn group_claim_policy_names_exclude_role_policy_and_dedicated_policy_claim() { - let mut role_config = test_config("role"); - role_config.role_policy = "consoleAdmin".to_string(); - let mut claim_config = test_config("claim"); - claim_config.claim_name = "policy".to_string(); - let sys = make_test_sys(vec![role_config, claim_config]); - let claims = OidcClaims { - groups: vec!["readonly".to_string(), "unmapped".to_string()], - raw: HashMap::from([("policy".to_string(), serde_json::json!(["readonly", "writeonly"]))]), - ..Default::default() - }; - - assert!(sys.group_claim_policy_names("role", &claims).is_empty()); - // `readonly` is also requested explicitly through the policy claim, so it must resolve. - assert_eq!(sys.group_claim_policy_names("claim", &claims), vec!["unmapped"]); - } - - #[test] - fn test_map_claims_to_policies_with_prefix() { - let mut config = test_config("azure"); - config.claim_prefix = "oidc-".to_string(); - config.display_name = "Azure AD".to_string(); - - let sys = make_test_sys(vec![config]); - - let claims = OidcClaims { - sub: "user456".to_string(), - email: "user@corp.com".to_string(), - username: "user".to_string(), - groups: vec!["engineers".to_string()], - raw: HashMap::new(), - }; - - let (policies, groups) = sys.map_claims_to_policies("azure", &claims); - assert_eq!(groups, vec!["engineers"]); - assert!(policies.contains(&"oidc-engineers".to_string())); - assert_eq!(policies.len(), 1); - } - - #[test] - fn blank_role_policy_uses_claim_mapping() { - let mut config = test_config("keycloak"); - config.role_policy = " ".to_string(); - - let sys = make_test_sys(vec![config]); - let claims = OidcClaims { - groups: vec!["readonly".to_string()], - ..Default::default() - }; - - let (policies, groups) = sys.map_claims_to_policies("keycloak", &claims); - assert_eq!(groups, vec!["readonly"]); - assert_eq!(policies, vec!["readonly"]); - } - - #[test] - fn claim_mapping_keeps_groups_with_distinct_primary_claim() { - let mut config = test_config("keycloak"); - config.claim_name = "policy".to_string(); - - let sys = make_test_sys(vec![config]); - let claims = OidcClaims { - groups: vec!["developers".to_string()], - raw: HashMap::from([("policy".to_string(), serde_json::json!(["readonly"]))]), - ..Default::default() - }; - - let (policies, groups) = sys.map_claims_to_policies("keycloak", &claims); - assert_eq!(groups, vec!["developers"]); - assert_eq!(policies, vec!["developers", "readonly"]); - } - - #[test] - fn test_config_snapshot_lists_providers() { - let mut config = test_config("keycloak"); - config.display_name = "Keycloak SSO".to_string(); - - let sys = make_test_sys(vec![config]); - - assert!(sys.has_providers()); - let snapshot = sys.config_snapshot(); - assert_eq!(snapshot.providers().len(), 1); - assert_eq!(snapshot.providers()[0].config.id, "keycloak"); - assert_eq!(snapshot.providers()[0].config.display_name, "Keycloak SSO"); - assert_eq!(snapshot.providers()[0].source, OidcProviderConfigSource::Persisted); - } - - #[test] - fn test_active_config_snapshot_puts_default_first() { - let sys = make_test_sys(vec![test_config("zeta"), test_config("default"), test_config("alpha")]); - - assert_eq!( - sys.config_snapshot() - .providers() - .iter() - .map(|provider| provider.config.id.as_str()) - .collect::>(), - ["default", "alpha", "zeta"] - ); - } - - #[test] - fn test_get_provider_config() { - let mut config = test_config("test"); - config.client_id = "my-client".to_string(); - config.client_secret = Some("secret".to_string()); - - let sys = make_test_sys(vec![config]); - - assert!(sys.get_provider_config("test").is_some()); - assert_eq!(sys.get_provider_config("test").unwrap().client_id, "my-client"); - assert!(sys.get_provider_config("nonexistent").is_none()); - } - - #[test] - fn test_oidc_provider_config_defaults() { - let config = OidcProviderConfig { - id: "test".to_string(), - enabled: true, - config_url: "https://example.com/.well-known/openid-configuration".to_string(), - issuer: None, - client_id: "my-client".to_string(), - client_secret: Some("secret".to_string()), - scopes: vec!["openid".to_string(), "profile".to_string(), "email".to_string()], - other_audiences: vec![], - redirect_uri: None, - redirect_uri_dynamic: true, - claim_name: "groups".to_string(), - claim_prefix: "".to_string(), - role_policy: "readwrite".to_string(), - display_name: "Test Provider".to_string(), - groups_claim: "groups".to_string(), - roles_claim: String::new(), - email_claim: "email".to_string(), - username_claim: "preferred_username".to_string(), - hide_from_ui: false, - }; - - assert_eq!(config.id, "test"); - assert!(config.enabled); - assert_eq!(config.scopes.len(), 3); - assert!(config.redirect_uri_dynamic); - } -} diff --git a/crates/iam/src/oidc/config.rs b/crates/iam/src/oidc/config.rs new file mode 100644 index 000000000..6169ea60b --- /dev/null +++ b/crates/iam/src/oidc/config.rs @@ -0,0 +1,434 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use rustfs_config::oidc::*; +use rustfs_config::server_config::{Config as ServerConfig, KVS}; +use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fmt; + +pub(super) const REDACTED_SECRET: &str = "***redacted***"; + +fn redacted_optional_secret(value: Option<&str>) -> &'static str { + value.filter(|secret| !secret.is_empty()).map_or("", |_| REDACTED_SECRET) +} + +/// Parsed configuration for a single OIDC provider. +#[derive(Clone, PartialEq, Eq)] +pub struct OidcProviderConfig { + pub id: String, + pub enabled: bool, + pub config_url: String, + pub issuer: Option, + pub client_id: String, + pub client_secret: Option, + pub scopes: Vec, + pub other_audiences: Vec, + pub redirect_uri: Option, + pub redirect_uri_dynamic: bool, + pub claim_name: String, + pub claim_prefix: String, + pub role_policy: String, + pub display_name: String, + pub groups_claim: String, + pub roles_claim: String, + pub email_claim: String, + pub username_claim: String, + pub hide_from_ui: bool, +} + +impl fmt::Debug for OidcProviderConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OidcProviderConfig") + .field("id", &self.id) + .field("enabled", &self.enabled) + .field("config_url", &self.config_url) + .field("issuer", &self.issuer) + .field("client_id", &self.client_id) + .field("client_secret", &redacted_optional_secret(self.client_secret.as_deref())) + .field("scopes", &self.scopes) + .field("other_audiences", &self.other_audiences) + .field("redirect_uri", &self.redirect_uri) + .field("redirect_uri_dynamic", &self.redirect_uri_dynamic) + .field("claim_name", &self.claim_name) + .field("claim_prefix", &self.claim_prefix) + .field("role_policy", &self.role_policy) + .field("display_name", &self.display_name) + .field("groups_claim", &self.groups_claim) + .field("roles_claim", &self.roles_claim) + .field("email_claim", &self.email_claim) + .field("username_claim", &self.username_claim) + .field("hide_from_ui", &self.hide_from_ui) + .finish() + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum OidcProviderConfigSource { + Env, + Persisted, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourcedOidcProviderConfig { + pub config: OidcProviderConfig, + pub source: OidcProviderConfigSource, +} + +/// Immutable OIDC configuration entries returned by configuration queries. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OidcConfigSnapshot { + providers: Vec, +} + +impl OidcConfigSnapshot { + pub fn new(mut providers: Vec) -> Self { + providers.sort_by(|left, right| { + (left.config.id != "default") + .cmp(&(right.config.id != "default")) + .then_with(|| left.config.id.cmp(&right.config.id)) + }); + Self { providers } + } + + pub fn providers(&self) -> &[SourcedOidcProviderConfig] { + &self.providers + } + + pub fn into_providers(self) -> Vec { + self.providers + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct OidcProviderValidationResult { + pub issuer: String, + pub authorization_endpoint: Option, + pub token_endpoint: Option, +} + +/// Compatibility name for the provider summary returned by OIDC list APIs. +pub(crate) type OidcProviderSummary = crate::federation::FederatedProviderView; + +/// Read-only access to the active standard OIDC configuration. +/// +/// This interface stays separate from authentication and generic provider +/// views because its snapshots include OIDC-specific fields and secrets. +pub trait OidcConfigQuery: Send + Sync { + fn config_snapshot(&self) -> OidcConfigSnapshot; +} + +/// Parse all OIDC provider configs from environment variables. +pub(super) fn parse_env_configs() -> Vec { + let mut configs = Vec::new(); + + // Check for the default provider (no suffix) + if let Some(config) = parse_single_provider("", "default") { + configs.push(config); + } + + // Scan for suffixed providers by checking all OIDC env var prefixes. + // This allows providers to be discovered without requiring a separate ENABLE_ key. + let mut provider_ids: Vec = Vec::new(); + let scan_prefixes: Vec = ENV_IDENTITY_OPENID_KEYS.iter().map(|k| format!("{k}_")).collect(); + for (key, _) in std::env::vars() { + for prefix in &scan_prefixes { + if let Some(suffix) = key.strip_prefix(prefix.as_str()) + && !suffix.is_empty() + && suffix != "default" + { + provider_ids.push(suffix.to_string()); + break; + } + } + } + + provider_ids.sort(); + provider_ids.dedup(); + + for id in provider_ids { + let suffix = format!("_{id}"); + if let Some(config) = parse_single_provider(&suffix, &id) { + configs.push(config); + } + } + + configs +} + +pub(super) fn parse_persisted_configs(cfg: &ServerConfig) -> Vec { + let Some(subsystem) = cfg.0.get(IDENTITY_OPENID_SUB_SYS) else { + return Vec::new(); + }; + + let mut configs = Vec::new(); + let mut provider_ids: Vec = subsystem.keys().cloned().collect(); + provider_ids.sort(); + + for raw_id in provider_ids { + let Some(kvs) = subsystem.get(&raw_id) else { + continue; + }; + + let id = if raw_id == DEFAULT_DELIMITER { + "default" + } else { + raw_id.as_str() + }; + if let Some(config) = parse_single_persisted_provider(kvs, id) { + configs.push(config); + } + } + + configs +} + +/// Parse a string as an `EnableState` boolean. +/// Returns `default_if_empty` when the input is empty, and `default_on_error` +/// when parsing fails. +pub(super) fn parse_enable_state(value: &str, default_if_empty: bool, default_on_error: bool) -> bool { + if value.is_empty() { + return default_if_empty; + } + value + .parse::() + .map(|s| s.is_enabled()) + .unwrap_or(default_on_error) +} + +/// Parse a single provider's config from env vars with the given suffix. +pub(super) fn parse_single_provider(env_suffix: &str, id: &str) -> Option { + let get_env = |base: &str| -> String { std::env::var(format!("{base}{env_suffix}")).unwrap_or_default() }; + + let enable_val = get_env(ENV_IDENTITY_OPENID_ENABLE); + let config_url = get_env(ENV_IDENTITY_OPENID_CONFIG_URL); + let issuer = get_env(ENV_IDENTITY_OPENID_ISSUER); + + // Skip if no config URL + if config_url.is_empty() { + return None; + } + + let enabled = parse_enable_state(&enable_val, true, false); + + let scopes_str = get_env(ENV_IDENTITY_OPENID_SCOPES); + let scopes = if scopes_str.is_empty() { + OIDC_DEFAULT_SCOPES.split(',').map(String::from).collect() + } else { + scopes_str.split(',').map(|s| s.trim().to_string()).collect() + }; + + let other_audiences_str = get_env(ENV_IDENTITY_OPENID_OTHER_AUDIENCES); + let other_audiences = other_audiences_str + .split(',') + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .collect(); + + let redirect_uri_dynamic = parse_enable_state(&get_env(ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC), true, true); + + let claim_name = { + let v = get_env(ENV_IDENTITY_OPENID_CLAIM_NAME); + if v.is_empty() { + OIDC_DEFAULT_CLAIM_NAME.to_string() + } else { + v + } + }; + let groups_claim = { + let v = get_env(ENV_IDENTITY_OPENID_GROUPS_CLAIM); + if v.is_empty() { + OIDC_DEFAULT_GROUPS_CLAIM.to_string() + } else { + v + } + }; + let roles_claim = get_env(ENV_IDENTITY_OPENID_ROLES_CLAIM); + let email_claim = { + let v = get_env(ENV_IDENTITY_OPENID_EMAIL_CLAIM); + if v.is_empty() { + OIDC_DEFAULT_EMAIL_CLAIM.to_string() + } else { + v + } + }; + let username_claim = { + let v = get_env(ENV_IDENTITY_OPENID_USERNAME_CLAIM); + if v.is_empty() { + OIDC_DEFAULT_USERNAME_CLAIM.to_string() + } else { + v + } + }; + let display_name = { + let v = get_env(ENV_IDENTITY_OPENID_DISPLAY_NAME); + if v.is_empty() { id.to_string() } else { v } + }; + let redirect_uri = { + let v = get_env(ENV_IDENTITY_OPENID_REDIRECT_URI); + if v.is_empty() { None } else { Some(v) } + }; + let client_secret = { + let v = get_env(ENV_IDENTITY_OPENID_CLIENT_SECRET); + if v.is_empty() { None } else { Some(v) } + }; + let hide_from_ui = parse_enable_state(&get_env(ENV_IDENTITY_OPENID_HIDE_FROM_UI), false, false); + + Some(OidcProviderConfig { + id: id.to_string(), + enabled, + config_url, + issuer: if issuer.is_empty() { None } else { Some(issuer) }, + client_id: get_env(ENV_IDENTITY_OPENID_CLIENT_ID), + client_secret, + scopes, + other_audiences, + redirect_uri, + redirect_uri_dynamic, + claim_name, + claim_prefix: get_env(ENV_IDENTITY_OPENID_CLAIM_PREFIX), + role_policy: get_env(ENV_IDENTITY_OPENID_ROLE_POLICY), + display_name, + groups_claim, + roles_claim, + email_claim, + username_claim, + hide_from_ui, + }) +} + +pub(super) fn parse_single_persisted_provider(kvs: &KVS, id: &str) -> Option { + let config_url = kvs.get(OIDC_CONFIG_URL); + if config_url.is_empty() { + return None; + } + + let enabled = parse_enable_state(&kvs.lookup(ENABLE_KEY).unwrap_or_default(), false, false); + + let scopes_str = kvs.get(OIDC_SCOPES); + let scopes = if scopes_str.is_empty() { + OIDC_DEFAULT_SCOPES.split(',').map(String::from).collect() + } else { + scopes_str.split(',').map(|s| s.trim().to_string()).collect() + }; + + let other_audiences_str = kvs.get(OIDC_OTHER_AUDIENCES); + let other_audiences = other_audiences_str + .split(',') + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .collect(); + + let redirect_uri_dynamic = parse_enable_state(&kvs.lookup(OIDC_REDIRECT_URI_DYNAMIC).unwrap_or_default(), true, true); + + let claim_name = kvs + .lookup(OIDC_CLAIM_NAME) + .unwrap_or_else(|| OIDC_DEFAULT_CLAIM_NAME.to_string()); + let groups_claim = kvs + .lookup(OIDC_GROUPS_CLAIM) + .unwrap_or_else(|| OIDC_DEFAULT_GROUPS_CLAIM.to_string()); + let roles_claim = kvs + .lookup(OIDC_ROLES_CLAIM) + .unwrap_or_else(|| OIDC_DEFAULT_ROLES_CLAIM.to_string()); + let email_claim = kvs + .lookup(OIDC_EMAIL_CLAIM) + .unwrap_or_else(|| OIDC_DEFAULT_EMAIL_CLAIM.to_string()); + let username_claim = kvs + .lookup(OIDC_USERNAME_CLAIM) + .unwrap_or_else(|| OIDC_DEFAULT_USERNAME_CLAIM.to_string()); + let display_name = kvs.lookup(OIDC_DISPLAY_NAME).unwrap_or_else(|| id.to_string()); + let redirect_uri = kvs.lookup(OIDC_REDIRECT_URI).filter(|v| !v.is_empty()); + let client_secret = kvs.lookup(OIDC_CLIENT_SECRET).filter(|v| !v.is_empty()); + let hide_from_ui = parse_enable_state(&kvs.lookup(OIDC_HIDE_FROM_UI).unwrap_or_default(), false, false); + + Some(OidcProviderConfig { + id: id.to_string(), + enabled, + config_url, + issuer: kvs.lookup(OIDC_ISSUER).filter(|v| !v.is_empty()), + client_id: kvs.get(OIDC_CLIENT_ID), + client_secret, + scopes, + other_audiences, + redirect_uri, + redirect_uri_dynamic, + claim_name, + claim_prefix: kvs.get(OIDC_CLAIM_PREFIX), + role_policy: kvs.get(OIDC_ROLE_POLICY), + display_name, + groups_claim, + roles_claim, + email_claim, + username_claim, + hide_from_ui, + }) +} + +pub fn load_oidc_provider_configs_from_env() -> Vec { + parse_env_configs() +} + +pub fn load_oidc_provider_configs_from_server_config(cfg: &ServerConfig) -> Vec { + parse_persisted_configs(cfg) +} + +pub(super) fn merge_oidc_provider_configs( + env_configs: Vec, + persisted_configs: Vec, +) -> Vec { + let mut effective = HashMap::new(); + + for config in persisted_configs { + effective.insert( + config.id.clone(), + SourcedOidcProviderConfig { + config, + source: OidcProviderConfigSource::Persisted, + }, + ); + } + + for config in env_configs { + effective.insert( + config.id.clone(), + SourcedOidcProviderConfig { + config, + source: OidcProviderConfigSource::Env, + }, + ); + } + + let mut configs: Vec = effective.into_values().collect(); + configs.sort_by(|lhs, rhs| lhs.config.id.cmp(&rhs.config.id)); + configs +} + +pub(super) fn load_effective_oidc_provider_configs(server_config: Option<&ServerConfig>) -> Vec { + let env_configs = load_oidc_provider_configs_from_env(); + let persisted_configs = server_config + .map(load_oidc_provider_configs_from_server_config) + .unwrap_or_default(); + merge_oidc_provider_configs(env_configs, persisted_configs) +} + +pub fn load_oidc_config_snapshot(server_config: Option<&ServerConfig>) -> OidcConfigSnapshot { + OidcConfigSnapshot::new(load_effective_oidc_provider_configs(server_config)) +} + +#[cfg(test)] +#[path = "config_tests.rs"] +mod tests; diff --git a/crates/iam/src/oidc/config_tests.rs b/crates/iam/src/oidc/config_tests.rs new file mode 100644 index 000000000..70936545a --- /dev/null +++ b/crates/iam/src/oidc/config_tests.rs @@ -0,0 +1,307 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::oidc::test_support::*; +use rustfs_config::server_config::{Config as ServerConfig, KVS}; +use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY}; + +#[test] +fn test_parse_single_provider_no_config_url() { + let config = parse_single_provider("_TEST_EMPTY", "test_empty"); + assert!(config.is_none()); +} +#[test] +fn test_parse_single_provider_reads_issuer() { + temp_env::with_vars( + [ + ( + ENV_IDENTITY_OPENID_CONFIG_URL, + Some("http://keycloak.ns.svc.cluster.local:8080/realms/app/.well-known/openid-configuration"), + ), + (ENV_IDENTITY_OPENID_ISSUER, Some("https://app.local/realms/app")), + (ENV_IDENTITY_OPENID_CLIENT_ID, Some("console")), + ], + || { + let config = parse_single_provider("", "default").expect("provider config should parse"); + + assert_eq!(config.issuer.as_deref(), Some("https://app.local/realms/app")); + }, + ); +} +#[test] +fn test_parse_persisted_provider_config() { + let mut cfg = ServerConfig::new(); + let mut kvs = KVS(vec![ + rustfs_config::server_config::KV { + key: ENABLE_KEY.to_string(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + rustfs_config::server_config::KV { + key: OIDC_CONFIG_URL.to_string(), + value: String::new(), + hidden_if_empty: false, + }, + rustfs_config::server_config::KV { + key: OIDC_CLIENT_ID.to_string(), + value: String::new(), + hidden_if_empty: false, + }, + ]); + kvs.insert( + OIDC_CONFIG_URL.to_string(), + "https://example.com/.well-known/openid-configuration".to_string(), + ); + kvs.insert(OIDC_CLIENT_ID.to_string(), "console".to_string()); + kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); + kvs.insert(OIDC_ISSUER.to_string(), "https://issuer.example".to_string()); + kvs.insert(OIDC_ROLES_CLAIM.to_string(), "app_roles".to_string()); + + cfg.0 + .entry(IDENTITY_OPENID_SUB_SYS.to_string()) + .or_default() + .insert(DEFAULT_DELIMITER.to_string(), kvs); + + let parsed = parse_persisted_configs(&cfg); + assert_eq!(parsed.len(), 1); + assert_eq!(parsed[0].id, "default"); + assert_eq!(parsed[0].client_id, "console"); + assert_eq!(parsed[0].issuer.as_deref(), Some("https://issuer.example")); + assert!(parsed[0].enabled); + assert_eq!(parsed[0].roles_claim, "app_roles"); +} +#[test] +fn test_parse_persisted_provider_config_omitted_roles_claim_is_empty() { + let mut cfg = ServerConfig::new(); + let mut kvs = KVS(vec![ + rustfs_config::server_config::KV { + key: ENABLE_KEY.to_string(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + rustfs_config::server_config::KV { + key: OIDC_CONFIG_URL.to_string(), + value: String::new(), + hidden_if_empty: false, + }, + rustfs_config::server_config::KV { + key: OIDC_CLIENT_ID.to_string(), + value: String::new(), + hidden_if_empty: false, + }, + ]); + kvs.insert( + OIDC_CONFIG_URL.to_string(), + "https://example.com/.well-known/openid-configuration".to_string(), + ); + kvs.insert(OIDC_CLIENT_ID.to_string(), "console".to_string()); + kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); + + cfg.0 + .entry(IDENTITY_OPENID_SUB_SYS.to_string()) + .or_default() + .insert(DEFAULT_DELIMITER.to_string(), kvs); + + let parsed = parse_persisted_configs(&cfg); + assert_eq!(parsed.len(), 1); + assert_eq!(parsed[0].roles_claim, ""); +} +#[test] +fn test_merge_oidc_provider_configs_prefers_env() { + let mut persisted = test_config("default"); + persisted.display_name = "Persisted".to_string(); + + let mut env = test_config("default"); + env.display_name = "Environment".to_string(); + + let merged = merge_oidc_provider_configs(vec![env], vec![persisted]); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].config.display_name, "Environment"); + assert_eq!(merged[0].source, OidcProviderConfigSource::Env); +} +#[test] +fn test_oidc_provider_config_debug_redacts_client_secret() { + let config = OidcProviderConfig { + client_secret: Some("oidc-client-secret".to_string()), + ..test_config("default") + }; + let sourced = SourcedOidcProviderConfig { + config, + source: OidcProviderConfigSource::Persisted, + }; + + let rendered = format!("{sourced:?}"); + + assert!(!rendered.contains("oidc-client-secret")); + assert!(rendered.contains(REDACTED_SECRET)); + assert!(rendered.contains("client-id")); +} +#[test] +fn test_parse_enable_state_on() { + assert!(parse_enable_state("on", false, false)); +} +#[test] +fn test_parse_enable_state_off() { + assert!(!parse_enable_state("off", true, true)); +} +#[test] +fn test_parse_enable_state_empty_returns_default() { + assert!(parse_enable_state("", true, false)); + assert!(!parse_enable_state("", false, true)); +} +#[test] +fn test_parse_enable_state_invalid_returns_error_default() { + assert!(!parse_enable_state("garbage", true, false)); + assert!(parse_enable_state("garbage", false, true)); +} +#[test] +fn test_hide_from_ui_default_is_false() { + let config = test_config("default"); + assert!(!config.hide_from_ui); +} +#[test] +fn test_parse_persisted_hide_from_ui_off_is_false() { + let mut cfg = ServerConfig::new(); + let mut kvs = KVS(vec![ + rustfs_config::server_config::KV { + key: ENABLE_KEY.to_string(), + value: EnableState::On.to_string(), + hidden_if_empty: false, + }, + rustfs_config::server_config::KV { + key: OIDC_CONFIG_URL.to_string(), + value: "https://example.com/.well-known/openid-configuration".to_string(), + hidden_if_empty: false, + }, + rustfs_config::server_config::KV { + key: OIDC_CLIENT_ID.to_string(), + value: "console".to_string(), + hidden_if_empty: false, + }, + ]); + kvs.insert(OIDC_HIDE_FROM_UI.to_string(), EnableState::Off.to_string()); + + cfg.0 + .entry(IDENTITY_OPENID_SUB_SYS.to_string()) + .or_default() + .insert(DEFAULT_DELIMITER.to_string(), kvs); + + let parsed = parse_persisted_configs(&cfg); + assert_eq!(parsed.len(), 1); + assert!(!parsed[0].hide_from_ui); +} +#[test] +fn test_parse_persisted_hide_from_ui_missing_defaults_false() { + let mut cfg = ServerConfig::new(); + let kvs = KVS(vec![ + rustfs_config::server_config::KV { + key: ENABLE_KEY.to_string(), + value: EnableState::On.to_string(), + hidden_if_empty: false, + }, + rustfs_config::server_config::KV { + key: OIDC_CONFIG_URL.to_string(), + value: "https://example.com/.well-known/openid-configuration".to_string(), + hidden_if_empty: false, + }, + rustfs_config::server_config::KV { + key: OIDC_CLIENT_ID.to_string(), + value: "console".to_string(), + hidden_if_empty: false, + }, + ]); + + cfg.0 + .entry(IDENTITY_OPENID_SUB_SYS.to_string()) + .or_default() + .insert(DEFAULT_DELIMITER.to_string(), kvs); + + let parsed = parse_persisted_configs(&cfg); + assert_eq!(parsed.len(), 1); + assert!(!parsed[0].hide_from_ui); +} +#[test] +fn test_parse_persisted_hide_from_ui() { + let mut cfg = ServerConfig::new(); + let mut kvs = KVS(vec![ + rustfs_config::server_config::KV { + key: ENABLE_KEY.to_string(), + value: EnableState::On.to_string(), + hidden_if_empty: false, + }, + rustfs_config::server_config::KV { + key: OIDC_CONFIG_URL.to_string(), + value: "https://example.com/.well-known/openid-configuration".to_string(), + hidden_if_empty: false, + }, + rustfs_config::server_config::KV { + key: OIDC_CLIENT_ID.to_string(), + value: "console".to_string(), + hidden_if_empty: false, + }, + ]); + kvs.insert(OIDC_HIDE_FROM_UI.to_string(), EnableState::On.to_string()); + + cfg.0 + .entry(IDENTITY_OPENID_SUB_SYS.to_string()) + .or_default() + .insert(DEFAULT_DELIMITER.to_string(), kvs); + + let parsed = parse_persisted_configs(&cfg); + assert_eq!(parsed.len(), 1); + assert!(parsed[0].hide_from_ui); +} +#[test] +fn test_active_config_snapshot_puts_default_first() { + let sys = make_test_sys(vec![test_config("zeta"), test_config("default"), test_config("alpha")]); + + assert_eq!( + sys.config_snapshot() + .providers() + .iter() + .map(|provider| provider.config.id.as_str()) + .collect::>(), + ["default", "alpha", "zeta"] + ); +} +#[test] +fn test_oidc_provider_config_defaults() { + let config = OidcProviderConfig { + id: "test".to_string(), + enabled: true, + config_url: "https://example.com/.well-known/openid-configuration".to_string(), + issuer: None, + client_id: "my-client".to_string(), + client_secret: Some("secret".to_string()), + scopes: vec!["openid".to_string(), "profile".to_string(), "email".to_string()], + other_audiences: vec![], + redirect_uri: None, + redirect_uri_dynamic: true, + claim_name: "groups".to_string(), + claim_prefix: "".to_string(), + role_policy: "readwrite".to_string(), + display_name: "Test Provider".to_string(), + groups_claim: "groups".to_string(), + roles_claim: String::new(), + email_claim: "email".to_string(), + username_claim: "preferred_username".to_string(), + hide_from_ui: false, + }; + + assert_eq!(config.id, "test"); + assert!(config.enabled); + assert_eq!(config.scopes.len(), 3); + assert!(config.redirect_uri_dynamic); +} diff --git a/crates/iam/src/oidc/mod.rs b/crates/iam/src/oidc/mod.rs new file mode 100644 index 000000000..4acfb6364 --- /dev/null +++ b/crates/iam/src/oidc/mod.rs @@ -0,0 +1,37 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//! OpenID Connect configuration, provider discovery, runtime flows, and state. + +mod config; +mod provider; +mod runtime; +mod state; +mod transport; + +#[cfg(any(test, feature = "test-util"))] +mod test_support; + +pub use config::{ + OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderConfigSource, OidcProviderValidationResult, + SourcedOidcProviderConfig, load_oidc_config_snapshot, load_oidc_provider_configs_from_env, + load_oidc_provider_configs_from_server_config, +}; +pub use provider::validate_oidc_provider_config_with_extra_root_ca; +pub use runtime::{OidcSys, StandardOidcAdapter}; +pub use transport::{ + OidcExtraRootCaMaterial, OidcExtraRootCaProvider, OidcPluginAuthnMetricsSnapshot, oidc_plugin_authn_metrics_snapshot, +}; + +#[cfg(any(test, feature = "test-util"))] +pub use test_support::{make_test_sys, test_config}; diff --git a/crates/iam/src/oidc/provider.rs b/crates/iam/src/oidc/provider.rs new file mode 100644 index 000000000..6f0c72a32 --- /dev/null +++ b/crates/iam/src/oidc/provider.rs @@ -0,0 +1,632 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use super::{ + config::{OidcProviderConfig, OidcProviderValidationResult, SourcedOidcProviderConfig}, + transport::{OidcHttpError, ReqwestHttpClient, oidc_extra_root_certs}, +}; +use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC}; +use openidconnect::core::{CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm}; +use openidconnect::{ + AsyncHttpClient, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, ProviderMetadataWithLogout, TokenUrl, +}; +use serde::Deserialize; +use std::collections::HashMap; +use std::sync::RwLock; +use std::time::{Duration as StdDuration, Instant}; +use tokio::time::sleep; +use tracing::warn; +use url::Url; + +pub(super) const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60); +pub(super) const OIDC_DISCOVERY_TRANSPORT_RETRIES: usize = 3; +pub(super) const OIDC_DISCOVERY_TRANSPORT_RETRY_DELAY: StdDuration = StdDuration::from_millis(50); +pub(super) const OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY: &str = "JWKS request blocked by outbound policy"; +pub(super) const OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY: &str = "OIDC provider discovery blocked by outbound policy"; + +// ---- Internal provider state ---- + +/// Discovered OIDC provider metadata. +/// We store metadata (which includes JWKS after discovery) separately rather than +/// a `CoreClient` because the crate uses type-state generics that make storing +/// the configured client in a HashMap impractical. The client is reconstructed +/// on-the-fly from metadata when needed. +#[derive(Clone)] +pub(super) struct ProviderState { + pub(super) metadata: DiscoveredProviderMetadata, + pub(super) discovered_at: Instant, +} + +pub(super) struct ProviderRuntime { + states: RwLock>, + http_client: ReqwestHttpClient, +} + +// Workload issuers do not implement the browser authorization flow. Keep their +// verification metadata separate rather than inventing an authorization URL. +#[serde_with::serde_as] +#[derive(Clone, Deserialize)] +pub(super) struct WorkloadProviderMetadata { + issuer: IssuerUrl, + jwks_uri: JsonWebKeySetUrl, + token_endpoint: Option, + #[serde_as(as = "serde_with::VecSkipError<_>")] + id_token_signing_alg_values_supported: Vec, + #[serde(skip)] + jwks: CoreJsonWebKeySet, + // Discovery is extensible; report unsupported fields without logging values. + #[serde(flatten)] + additional_fields: HashMap, +} + +#[derive(Clone)] +pub(super) enum DiscoveredProviderMetadata { + Console(Box), + Workload(Box), +} + +impl DiscoveredProviderMetadata { + pub(super) fn parse(body: &[u8], hide_from_ui: bool) -> Result { + let document: serde_json::Value = serde_json::from_slice(body).map_err(|err| err.to_string())?; + if hide_from_ui + && document + .as_object() + .is_some_and(|fields| !fields.contains_key("authorization_endpoint")) + { + let mut metadata: WorkloadProviderMetadata = serde_json::from_slice(body).map_err(|err| err.to_string())?; + if !metadata.additional_fields.is_empty() { + warn!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "workload_discovery_additional_fields", + field_count = metadata.additional_fields.len(), + "workload discovery contains additional fields" + ); + metadata.additional_fields.clear(); + } + Ok(Self::Workload(Box::new(metadata))) + } else { + serde_json::from_slice(body) + .map(|metadata| Self::Console(Box::new(metadata))) + .map_err(|err| err.to_string()) + } + } + + pub(super) fn console(&self) -> Result<&ProviderMetadataWithLogout, String> { + match self { + Self::Console(metadata) => Ok(metadata), + Self::Workload(_) => Err("OIDC provider has no authorization endpoint; only web identity is supported".into()), + } + } + + pub(super) fn issuer(&self) -> &IssuerUrl { + match self { + Self::Console(metadata) => metadata.issuer(), + Self::Workload(metadata) => &metadata.issuer, + } + } + + pub(super) fn jwks_uri(&self) -> &JsonWebKeySetUrl { + match self { + Self::Console(metadata) => metadata.jwks_uri(), + Self::Workload(metadata) => &metadata.jwks_uri, + } + } + + pub(super) fn set_jwks(self, jwks: CoreJsonWebKeySet) -> Self { + match self { + Self::Console(metadata) => Self::Console(Box::new(metadata.set_jwks(jwks))), + Self::Workload(mut metadata) => { + metadata.jwks = jwks; + Self::Workload(metadata) + } + } + } + + pub(super) fn authorization_endpoint(&self) -> Option { + match self { + Self::Console(metadata) => Some(metadata.authorization_endpoint().to_string()), + Self::Workload(_) => None, + } + } + + pub(super) fn token_endpoint(&self) -> Option<&TokenUrl> { + match self { + Self::Console(metadata) => metadata.token_endpoint(), + Self::Workload(metadata) => metadata.token_endpoint.as_ref(), + } + } + + pub(super) fn verifier(&self, config: &OidcProviderConfig) -> CoreIdTokenVerifier<'static> { + let client_id = ClientId::new(config.client_id.clone()); + let secret = config.client_secret.as_ref().map(|secret| ClientSecret::new(secret.clone())); + let (issuer, jwks, algorithms) = match self { + Self::Console(metadata) => (metadata.issuer(), metadata.jwks(), metadata.id_token_signing_alg_values_supported()), + Self::Workload(metadata) => (&metadata.issuer, &metadata.jwks, &metadata.id_token_signing_alg_values_supported), + }; + let verifier = match secret { + Some(secret) => CoreIdTokenVerifier::new_confidential_client(client_id, secret, issuer.clone(), jwks.clone()), + None => CoreIdTokenVerifier::new_public_client(client_id, issuer.clone(), jwks.clone()), + }; + verifier.set_allowed_algs(algorithms.clone()) + } +} + +// This adapter is used only for discovery/JWKS fetches, never token exchange. +struct JwksAcceptClient<'a> { + inner: &'a ReqwestHttpClient, + discovery_url: Option, +} + +impl<'c> AsyncHttpClient<'c> for JwksAcceptClient<'_> { + type Error = OidcHttpError; + type Future = >::Future; + + fn call(&'c self, mut request: http::Request>) -> Self::Future { + if !self.discovery_url.as_ref().is_some_and(|url| request.uri() == url.as_str()) { + request.headers_mut().insert( + http::header::ACCEPT, + http::HeaderValue::from_static("application/json, application/jwk-set+json"), + ); + } + self.inner.call(request) + } +} + +impl ProviderState { + fn is_stale(&self) -> bool { + self.discovered_at.elapsed() >= OIDC_JWKS_REFRESH_INTERVAL + } +} + +impl ProviderRuntime { + pub(super) fn new(http_client: ReqwestHttpClient, states: HashMap) -> Self { + Self { + states: RwLock::new(states), + http_client, + } + } + + pub(super) fn http_client(&self) -> &ReqwestHttpClient { + &self.http_client + } + + /// Find a provider whose discovered issuer matches the given JWT issuer string. + pub(super) fn find_provider_by_issuer( + &self, + issuer: &str, + configs: &HashMap, + ) -> Option<(String, OidcProviderConfig, ProviderState)> { + let (issuer_scheme, issuer_host, issuer_port, issuer_path) = normalize_issuer(issuer)?; + let map = self + .states + .read() + .map_err(|e| format!("provider state lock poisoned: {e}")) + .ok()?; + for (id, state) in map.iter() { + let provider_issuer = state.metadata.issuer().as_str(); + let Some((provider_scheme, provider_host, provider_port, provider_path)) = normalize_issuer(provider_issuer) else { + continue; + }; + + if issuer_scheme == provider_scheme + && issuer_host == provider_host + && issuer_port == provider_port + && issuer_path == provider_path + && let Some(config) = configs.get(id) + { + return Some((id.clone(), config.config.clone(), state.clone())); + } + } + None + } + + pub(super) fn get_provider_state(&self, provider_id: &str) -> Result { + self.states + .read() + .map_err(|e| format!("provider state lock poisoned: {e}"))? + .get(provider_id) + .cloned() + .ok_or_else(|| format!("provider not discovered: {provider_id}")) + } + + pub(super) async fn refresh_provider_state( + &self, + provider_id: &str, + config: &OidcProviderConfig, + ) -> Result { + let state = discover_provider(config, &self.http_client).await?; + let mut map = self.states.write().map_err(|e| { + let msg = e.to_string(); + format!("provider state lock poisoned: {msg}") + })?; + map.insert(provider_id.to_string(), state.clone()); + + Ok(state) + } + + pub(super) async fn ensure_provider_state( + &self, + provider_id: &str, + config: &OidcProviderConfig, + ) -> Result { + let state = self.get_provider_state(provider_id)?; + if state.is_stale() { + self.refresh_provider_state(provider_id, config).await.or_else(|refresh_err| { + warn!( + "OIDC provider '{}' JWKS metadata refresh skipped due to transient network issue: {}", + provider_id, refresh_err + ); + Ok(state) + }) + } else { + Ok(state) + } + } + + pub(super) async fn ensure_provider_state_if_stale( + &self, + provider_id: &str, + config: &OidcProviderConfig, + state: &ProviderState, + ) -> Result { + if state.is_stale() { + self.refresh_provider_state(provider_id, config).await.or_else(|refresh_err| { + warn!( + "OIDC provider '{}' JWKS metadata refresh skipped due to transient network issue: {}", + provider_id, refresh_err + ); + Ok(state.clone()) + }) + } else { + Ok(state.clone()) + } + } +} + +/// Perform OIDC discovery for a provider. +/// `discover_async` fetches the discovery document and JWKS in one step. +pub(super) async fn discover_provider( + config: &OidcProviderConfig, + http_client: &ReqwestHttpClient, +) -> Result { + if let Some(issuer) = config.issuer.as_deref().filter(|issuer| !issuer.trim().is_empty()) { + return discover_provider_from_config_url(config, issuer, http_client).await; + } + + // The openidconnect crate expects the issuer URL (base), not the + // .well-known/openid-configuration URL. + let base_issuer = normalize_config_url(&config.config_url)?; + let candidates = issuer_candidates(&base_issuer); + let mut last_errors = Vec::new(); + + for candidate_issuer in candidates.iter() { + let issuer_url = IssuerUrl::new(candidate_issuer.clone()).map_err(|e| format!("invalid issuer URL: {e}"))?; + + for attempt in 0..OIDC_DISCOVERY_TRANSPORT_RETRIES { + let discovered = if config.hide_from_ui { + discover_provider_from_config_url(config, candidate_issuer, http_client).await + } else { + let client = JwksAcceptClient { + inner: http_client, + discovery_url: Some( + issuer_url + .join(".well-known/openid-configuration") + .map_err(|err| err.to_string())?, + ), + }; + ProviderMetadataWithLogout::discover_async(issuer_url.clone(), &client) + .await + .map(|metadata| ProviderState { + metadata: DiscoveredProviderMetadata::Console(Box::new(metadata)), + discovered_at: Instant::now(), + }) + .map_err(|err| match err { + DiscoveryError::Request(OidcHttpError::ForbiddenOutbound(reason)) => { + format!("{OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY}: {reason}") + } + err => format!("discovery failed: {err}"), + }) + }; + match discovered { + Ok(state) => return Ok(state), + Err(error) + if error.starts_with(OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY) + || error.starts_with(OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY) => + { + return Err(error); + } + Err(error) => { + let is_transient_transport = error.contains("Request failed"); + let should_retry = is_transient_transport && attempt + 1 < OIDC_DISCOVERY_TRANSPORT_RETRIES; + if should_retry { + warn!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "provider_discovery_transport_retry", + provider_id = %config.id, + config_url = %config.config_url, + issuer_candidate = %candidate_issuer, + attempt = attempt + 1, + max_attempts = OIDC_DISCOVERY_TRANSPORT_RETRIES, + error = %error, + "oidc provider discovery failed" + ); + sleep(OIDC_DISCOVERY_TRANSPORT_RETRY_DELAY).await; + continue; + } + + last_errors.push(format!("issuer '{candidate_issuer}': {error}")); + warn!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "provider_discovery_candidate_failed", + provider_id = %config.id, + config_url = %config.config_url, + issuer_candidate = %candidate_issuer, + attempt = attempt + 1, + max_attempts = OIDC_DISCOVERY_TRANSPORT_RETRIES, + error = %error, + "oidc provider discovery failed" + ); + break; + } + } + } + } + + Err(format!( + "discovery failed for all issuer variants {:?}: {}", + candidates, + last_errors.join("; ") + )) +} + +pub(super) async fn validate_oidc_provider_config(config: &OidcProviderConfig) -> Result { + let http_client = ReqwestHttpClient::new()?; + validate_oidc_provider_config_with_http_client(config, &http_client).await +} + +pub async fn validate_oidc_provider_config_with_extra_root_ca( + config: &OidcProviderConfig, + root_ca_pem: Option<&[u8]>, +) -> Result { + if root_ca_pem.is_none() { + return validate_oidc_provider_config(config).await; + } + + let http_client = ReqwestHttpClient::new_with_extra_root_certs(oidc_extra_root_certs(root_ca_pem)?)?; + validate_oidc_provider_config_with_http_client(config, &http_client).await +} + +async fn validate_oidc_provider_config_with_http_client( + config: &OidcProviderConfig, + http_client: &ReqwestHttpClient, +) -> Result { + let state = discover_provider(config, http_client).await?; + + Ok(OidcProviderValidationResult { + issuer: state.metadata.issuer().to_string(), + authorization_endpoint: state.metadata.authorization_endpoint(), + token_endpoint: state.metadata.token_endpoint().map(ToString::to_string), + }) +} + +async fn discover_provider_from_config_url( + config: &OidcProviderConfig, + issuer: &str, + http_client: &ReqwestHttpClient, +) -> Result { + let issuer_url = IssuerUrl::new(issuer.trim().to_string()).map_err(|e| format!("invalid issuer URL: {e}"))?; + let explicit_issuer = config.issuer.as_deref().is_some_and(|issuer| !issuer.trim().is_empty()); + let discovery_url = if explicit_issuer { + discovery_url_from_config_url(&config.config_url)? + } else { + issuer_url + .join(".well-known/openid-configuration") + .map_err(|err| err.to_string())? + }; + let request = http::Request::builder() + .uri(discovery_url.to_string()) + .method(http::Method::GET) + .header(http::header::ACCEPT, "application/json") + .body(Vec::new()) + .map_err(|err| format!("failed to prepare discovery request: {err}"))?; + + let response = match http_client.call(request).await { + Ok(response) => response, + Err(OidcHttpError::ForbiddenOutbound(reason)) => { + return Err(format!("{OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY}: {reason}")); + } + Err(err) => return Err(format!("discovery request failed: Request failed: {err}")), + }; + if response.status() != http::StatusCode::OK { + return Err(format!("discovery failed: HTTP status code {} at {}", response.status(), discovery_url)); + } + + if !explicit_issuer + && let Some(content_type) = response.headers().get(http::header::CONTENT_TYPE) + && !content_type.to_str().ok().is_some_and(|value| { + value + .split(';') + .next() + .is_some_and(|essence| essence.eq_ignore_ascii_case("application/json")) + }) + { + return Err("Unexpected response Content-Type: expected application/json".into()); + } + + let provider_metadata = DiscoveredProviderMetadata::parse(response.body(), config.hide_from_ui) + .map_err(|err| format!("failed to parse discovery response: {err}"))?; + if provider_metadata.issuer() != &issuer_url { + return Err(format!( + "unexpected issuer URI `{}` (expected `{}`)", + provider_metadata.issuer().as_str(), + issuer_url.as_str() + )); + } + + let jwks_url = if explicit_issuer { + jwks_url_from_config_url(&config.config_url, &issuer_url, provider_metadata.jwks_uri())? + } else { + provider_metadata.jwks_uri().clone() + }; + let jwks = match CoreJsonWebKeySet::fetch_async( + &jwks_url, + &JwksAcceptClient { + inner: http_client, + discovery_url: None, + }, + ) + .await + { + Ok(jwks) => jwks, + Err(DiscoveryError::Request(OidcHttpError::ForbiddenOutbound(reason))) => { + return Err(format!("{OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY}: {reason}")); + } + Err(err) => return Err(format!("failed to fetch JWKS: {err}")), + }; + + Ok(ProviderState { + metadata: provider_metadata.set_jwks(jwks), + discovered_at: Instant::now(), + }) +} + +pub(super) fn normalize_issuer(raw: &str) -> Option<(String, String, u16, String)> { + let parsed = Url::parse(raw).ok()?; + if parsed.scheme() != "http" && parsed.scheme() != "https" { + return None; + } + + let host = parsed.host_str()?.to_ascii_lowercase(); + let port = parsed.port_or_known_default()?; + let normalized_path = { + let path = parsed.path().trim_end_matches('/').to_string(); + if path.is_empty() { "/".to_string() } else { path } + }; + + Some((parsed.scheme().to_string(), host, port, normalized_path)) +} + +pub(super) fn normalize_config_url(config_url: &str) -> Result { + let config_url = config_url.trim(); + let url = Url::parse(config_url).map_err(|e| format!("invalid config_url: {e}"))?; + if url.scheme() != "http" && url.scheme() != "https" { + return Err(format!("invalid config_url scheme: {}", url.scheme())); + } + let host = url.host_str().ok_or_else(|| "config_url missing host".to_string())?; + let path = url.path(); + + // Strip `/.well-known/openid-configuration` (with optional trailing slash) if present. + // Everything else is preserved exactly so the issuer URL matches the provider's discovery + // document (e.g. Authentik includes a trailing slash, Keycloak does not). + let normalized_path = path + .strip_suffix('/') + .unwrap_or(path) + .strip_suffix("/.well-known/openid-configuration") + .unwrap_or(if path == "/" { "" } else { path }); + + if normalized_path.contains("/.well-known/") { + return Err("config_url uses an unsupported .well-known discovery URL".into()); + } + + let mut issuer = format!("{}://{host}", url.scheme()); + if let Some(port) = url.port() { + issuer.push(':'); + issuer.push_str(&port.to_string()); + } + + if !normalized_path.is_empty() { + issuer.push_str(normalized_path); + } + + Ok(issuer) +} + +pub(super) fn discovery_url_from_config_url(config_url: &str) -> Result { + let mut url = Url::parse(config_url.trim()).map_err(|e| format!("invalid config_url: {e}"))?; + if url.scheme() != "http" && url.scheme() != "https" { + return Err(format!("invalid config_url scheme: {}", url.scheme())); + } + if url.host_str().is_none() { + return Err("config_url missing host".to_string()); + } + + let path = url.path().to_string(); + let without_trailing_slash = path.strip_suffix('/').unwrap_or(&path); + if without_trailing_slash.ends_with("/.well-known/openid-configuration") { + url.set_path(without_trailing_slash); + return Ok(url); + } + if without_trailing_slash.contains("/.well-known/") { + return Err("config_url uses an unsupported .well-known discovery URL".into()); + } + + let discovery_path = if without_trailing_slash.is_empty() || without_trailing_slash == "/" { + "/.well-known/openid-configuration".to_string() + } else { + format!("{without_trailing_slash}/.well-known/openid-configuration") + }; + url.set_path(&discovery_path); + Ok(url) +} + +pub(super) fn jwks_url_from_config_url( + config_url: &str, + issuer_url: &IssuerUrl, + jwks_url: &JsonWebKeySetUrl, +) -> Result { + let issuer = issuer_url.url(); + let jwks = jwks_url.url(); + if issuer.origin() != jwks.origin() { + return Ok(jwks_url.clone()); + } + + let issuer_path = issuer.path().trim_end_matches('/'); + let Some(suffix) = jwks.path().strip_prefix(issuer_path) else { + return Ok(jwks_url.clone()); + }; + if !suffix.is_empty() && !suffix.starts_with('/') { + return Ok(jwks_url.clone()); + } + + let mut internal_url = + Url::parse(&normalize_config_url(config_url)?).map_err(|err| format!("invalid config_url issuer base: {err}"))?; + let internal_path = internal_url.path().trim_end_matches('/'); + internal_url.set_path(&format!("{internal_path}{suffix}")); + internal_url.set_query(jwks.query()); + Ok(JsonWebKeySetUrl::from_url(internal_url)) +} + +pub(super) fn issuer_candidates(base: &str) -> Vec { + let original = base.trim(); + let mut variants = Vec::with_capacity(2); + variants.push(original.to_string()); + + let toggled = if original.ends_with('/') { + original.trim_end_matches('/').to_string() + } else { + format!("{original}/") + }; + variants.push(toggled); + + variants +} + +#[cfg(test)] +#[path = "provider_tests.rs"] +mod tests; diff --git a/crates/iam/src/oidc/provider_tests.rs b/crates/iam/src/oidc/provider_tests.rs new file mode 100644 index 000000000..efa6c543b --- /dev/null +++ b/crates/iam/src/oidc/provider_tests.rs @@ -0,0 +1,712 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::oidc::{runtime::*, state::*, test_support::*, transport::*}; +use jsonwebtoken::{Algorithm, EncodingKey, Header}; +use openidconnect::core::CoreIdToken; +use openidconnect::{IssuerUrl, JsonWebKeySetUrl, Nonce}; +use rustfs_utils::egress::{ENV_OUTBOUND_ALLOW_ORIGINS, OutboundPolicy}; +use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use url::Url; + +#[test] +fn test_normalize_issuer_matches() { + let lhs = normalize_issuer("https://idp.example.com/.well-known/openid-configuration/").unwrap(); + let rhs = normalize_issuer("https://idp.example.com/.well-known/openid-configuration").unwrap(); + assert_eq!(lhs, rhs); + assert_eq!( + lhs, + ( + "https".to_string(), + "idp.example.com".to_string(), + 443, + "/.well-known/openid-configuration".to_string() + ) + ); +} +#[test] +fn test_normalize_config_url() { + // --- Well-known suffix stripping --- + // Bare well-known URL → stripped to just the host + assert_eq!( + normalize_config_url("https://idp.example.com/.well-known/openid-configuration").unwrap(), + "https://idp.example.com" + ); + // Trailing slash after well-known suffix is also stripped + assert_eq!( + normalize_config_url("https://idp.example.com/.well-known/openid-configuration/").unwrap(), + "https://idp.example.com" + ); + // Well-known under a sub-path (Keycloak realms) + assert_eq!( + normalize_config_url("https://keycloak.example.com/realms/myrealm/.well-known/openid-configuration").unwrap(), + "https://keycloak.example.com/realms/myrealm" + ); + + // --- Providers WITHOUT trailing slash (Keycloak, Auth0, Okta, Google) --- + assert_eq!( + normalize_config_url("https://keycloak.example.com/realms/myrealm").unwrap(), + "https://keycloak.example.com/realms/myrealm" + ); + assert_eq!( + normalize_config_url("https://idp.example.com/custom/realm").unwrap(), + "https://idp.example.com/custom/realm" + ); + + // --- Providers WITH trailing slash (Authentik) --- + assert_eq!( + normalize_config_url("https://auth.example.com/application/o/myapp/").unwrap(), + "https://auth.example.com/application/o/myapp/" + ); + + // --- Root-level issuer (bare host) --- + assert_eq!(normalize_config_url("https://idp.example.com").unwrap(), "https://idp.example.com"); + assert_eq!(normalize_config_url("https://idp.example.com/").unwrap(), "https://idp.example.com"); + + // --- Custom port --- + assert_eq!( + normalize_config_url("https://idp.example.com:8443/auth/realms/test").unwrap(), + "https://idp.example.com:8443/auth/realms/test" + ); + assert_eq!( + normalize_config_url("http://localhost:8080/application/o/app/").unwrap(), + "http://localhost:8080/application/o/app/" + ); + + // --- Error cases --- + assert!(normalize_config_url("https://idp.example.com/.well-known/invalid").is_err()); + assert!(normalize_config_url("gopher://idp.example.com").is_err()); + assert!(normalize_config_url("not-a-url").is_err()); +} +#[test] +fn test_discovery_url_from_config_url() { + assert_eq!( + discovery_url_from_config_url("https://idp.example.com/.well-known/openid-configuration") + .expect("config URL should parse") + .as_str(), + "https://idp.example.com/.well-known/openid-configuration" + ); + assert_eq!( + discovery_url_from_config_url("https://idp.example.com/realms/app") + .expect("issuer URL should derive discovery URL") + .as_str(), + "https://idp.example.com/realms/app/.well-known/openid-configuration" + ); + assert!(discovery_url_from_config_url("https://idp.example.com/.well-known/not-openid").is_err()); +} +#[test] +fn test_issuer_candidates() { + assert_eq!( + issuer_candidates("https://idp.example.com/realm"), + vec![ + "https://idp.example.com/realm".to_string(), + "https://idp.example.com/realm/".to_string() + ] + ); + assert_eq!( + issuer_candidates("https://idp.example.com/realm/"), + vec![ + "https://idp.example.com/realm/".to_string(), + "https://idp.example.com/realm".to_string() + ] + ); + assert_eq!( + issuer_candidates("https://idp.example.com"), + vec!["https://idp.example.com".to_string(), "https://idp.example.com/".to_string()] + ); +} +#[test] +fn workload_metadata_requires_hidden_provider_and_valid_verification_fields() { + let document = serde_json::json!({ + "issuer": "https://issuer.example.com", + "jwks_uri": "https://issuer.example.com/jwks", + "response_types_supported": ["id_token"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["ES256"], + }); + let parse = + |value: &serde_json::Value, hidden| DiscoveredProviderMetadata::parse(&serde_json::to_vec(value).unwrap(), hidden); + let metadata = parse(&document, true).expect("hidden workload metadata should parse"); + assert!(metadata.authorization_endpoint().is_none()); + assert!(metadata.console().err().unwrap().contains("only web identity")); + assert!(parse(&document, false).err().unwrap().contains("authorization_endpoint")); + for field in ["issuer", "jwks_uri", "id_token_signing_alg_values_supported"] { + let mut invalid = document.clone(); + invalid.as_object_mut().unwrap().remove(field); + assert!(parse(&invalid, true).err().unwrap().contains(field), "missing {field}"); + } + for endpoint in [serde_json::Value::Null, serde_json::json!(""), serde_json::json!("not a URL")] { + let mut invalid = document.clone(); + invalid["authorization_endpoint"] = endpoint; + assert!(parse(&invalid, true).is_err(), "invalid endpoint must not select workload metadata"); + } + let mut complete = document; + complete["authorization_endpoint"] = serde_json::json!("https://issuer.example.com/authorize"); + for hidden in [true, false] { + let metadata = parse(&complete, hidden).expect("full providers keep the existing parser"); + assert!(metadata.console().is_ok()); + assert!(metadata.token_endpoint().is_none(), "token endpoint remains optional"); + } +} +#[test] +fn workload_metadata_rejects_duplicate_fields() { + for hidden in [false, true] { + let document = format!( + r#"{{"issuer":"https://wrong.example.com","issuer":"https://issuer.example.com",{}"jwks_uri":"https://issuer.example.com/jwks","response_types_supported":["id_token"],"subject_types_supported":["public"],"id_token_signing_alg_values_supported":["ES256"]}}"#, + if hidden { + "" + } else { + r#""authorization_endpoint":"https://issuer.example.com/authorize","# + }, + ); + let error = DiscoveredProviderMetadata::parse(document.as_bytes(), hidden) + .err() + .expect("duplicate issuer must fail"); + assert!(error.contains("duplicate field"), "{error}"); + } +} +#[test] +fn workload_verifier_preserves_algorithm_secret_and_audience_policy() { + let secret = Nonce::new_random().secret().to_string(); + let mut config = build_mocked_oidc_provider_config("workload", "https://issuer.example.com"); + config.client_secret = Some(secret.clone()); + config.other_audiences = vec!["additional-audience".into()]; + let now = time::OffsetDateTime::now_utc().unix_timestamp(); + let payload = serde_json::json!({ + "iss": config.config_url, "sub": "repo:example/project:ref:refs/heads/main", + "aud": [config.client_id, "additional-audience"], "iat": now, "exp": now + 300, + }); + let signed = + jsonwebtoken::encode(&Header::new(Algorithm::HS256), &payload, &EncodingKey::from_secret(secret.as_bytes())).unwrap(); + let token: CoreIdToken = signed.parse().unwrap(); + for workload in [false, true] { + for (algorithms, accepted) in [ + (serde_json::json!(["HS256", "unsupported-future-algorithm"]), true), + (serde_json::json!(["ES256"]), false), + (serde_json::json!(["unsupported-future-algorithm"]), false), + ] { + let mut document = serde_json::json!({ + "issuer": config.config_url, "jwks_uri": "https://issuer.example.com/jwks", + "response_types_supported": ["id_token"], "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": algorithms, + }); + if !workload { + document["authorization_endpoint"] = serde_json::json!("https://issuer.example.com/authorize"); + } + let metadata = DiscoveredProviderMetadata::parse(&serde_json::to_vec(&document).unwrap(), true).unwrap(); + let verifier = metadata + .verifier(&config) + .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); + assert_eq!( + token.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(), + accepted, + "workload={workload}, algorithms={algorithms}" + ); + if accepted { + assert!( + token.claims(&metadata.verifier(&config), |_: Option<&Nonce>| Ok(())).is_err(), + "additional audiences require explicit trust" + ); + let mut wrong_secret = config.clone(); + wrong_secret.client_secret = Some(Nonce::new_random().secret().to_string()); + let verifier = metadata + .verifier(&wrong_secret) + .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); + assert!( + token.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(), + "incorrect client secret must fail" + ); + } + } + } +} +#[tokio::test] +async fn workload_discovery_stops_after_forbidden_jwks() { + let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let seen = Arc::clone(&requests); + let (base, handle) = start_mock_oidc_discovery_server_with_jwks( + |base| (base.to_string(), "http://192.168.65.254:8080/jwks".into(), "/jwks".into()), + 2, + "ES256", + true, + move |_| { + seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + serde_json::json!({"keys": []}).to_string() + }, + ) + .expect("workload discovery mock must bind"); + let mut config = build_mocked_oidc_provider_config("workload", &base); + config.hide_from_ui = true; + let client = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).unwrap()); + let error = discover_provider(&config, &client) + .await + .err() + .expect("private JWKS must be blocked"); + assert!(error.starts_with(OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY), "{error}"); + handle.join().unwrap(); + assert_eq!( + requests.load(std::sync::atomic::Ordering::SeqCst), + 1, + "a policy denial must not retry discovery" + ); +} +#[tokio::test] +async fn workload_discovery_verification_and_rotation() { + for explicit_issuer in [false, true] { + let (_, initial_jwk) = oidc_es256_key_and_jwk("initial"); + let (key, rotated_jwk) = oidc_es256_key_and_jwk("rotated"); + let initial_jwks = serde_json::json!({"keys": [initial_jwk]}).to_string(); + let rotated_jwks = serde_json::json!({"keys": [rotated_jwk]}).to_string(); + let (base, handle) = start_mock_oidc_discovery_server_with_jwks( + |base| (base.to_string(), format!("{base}/jwks"), "/jwks".into()), + 4, + "ES256", + true, + move |fetch| { + if fetch == 0 { + initial_jwks.clone() + } else { + rotated_jwks.clone() + } + }, + ) + .expect("workload discovery mock must bind"); + let mut config = build_mocked_oidc_provider_config("workload", &base); + config.hide_from_ui = true; + if explicit_issuer { + config.issuer = Some(base.clone()); + } + let http_client = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).unwrap()); + let state = discover_provider(&config, &http_client) + .await + .expect("workload discovery must succeed"); + assert!(state.metadata.authorization_endpoint().is_none()); + let sys = OidcSys { + configs: HashMap::from([(config.id.clone(), test_sourced_config(config.clone()))]), + provider_runtime: ProviderRuntime::new(http_client, HashMap::from([(config.id.clone(), state)])), + state_store: OidcStateStore::new(), + }; + assert!(sys.has_providers()); + assert!(sys.provider_configs().all(|provider| provider.hide_from_ui)); + let error = sys + .authorize_url(&config.id, "https://console.example.com/callback", None) + .await + .unwrap_err(); + assert!(error.contains("only web identity")); + let now = time::OffsetDateTime::now_utc().unix_timestamp(); + let payload = serde_json::json!({ + "iss": base, "sub": "system:serviceaccount:default:reader", "aud": [config.client_id], + "iat": now, "exp": now + 300, "groups": ["readonly"], + "kubernetes.io": {"namespace": "default", "serviceaccount": {"name": "reader"}}, + }); + let mut header = Header::new(Algorithm::ES256); + header.kid = Some("rotated".into()); + let token = jsonwebtoken::encode(&header, &payload, &key).unwrap(); + let (claims, provider) = sys + .verify_web_identity_token(&token) + .await + .expect("rotation must retain workload discovery support"); + assert_eq!(provider, config.id); + assert_eq!(claims.sub, "system:serviceaccount:default:reader"); + assert_eq!(claims.groups, ["readonly"]); + // Repeat after the mock exits: the verified snapshot must be cached. + handle.join().unwrap(); + assert!(sys.verify_web_identity_token(&token).await.is_ok()); + for (field, value, expected) in [ + ("iss", serde_json::json!("https://wrong.example.com"), "issuer"), + ("aud", serde_json::json!("wrong-audience"), "audience"), + ("exp", serde_json::json!(now - 60), "expired"), + ] { + let mut invalid = payload.clone(); + invalid[field] = value; + let token = jsonwebtoken::encode(&header, &invalid, &key).unwrap(); + let error = sys + .verify_web_identity_token(&token) + .await + .expect_err("invalid workload token must fail"); + assert!(error.to_lowercase().contains(expected), "{field}: {error}"); + } + let (wrong_key, _) = oidc_es256_key_and_jwk("wrong"); + let invalid = jsonwebtoken::encode(&header, &payload, &wrong_key).unwrap(); + let error = sys + .verify_web_identity_token(&invalid) + .await + .expect_err("wrong signature must fail"); + assert!(error.to_lowercase().contains("signature"), "{error}"); + assert!( + sys.verify_web_identity_token(&token).await.is_ok(), + "failed refresh must preserve the cached keys" + ); + } +} +#[tokio::test] +async fn oidc_discovery_accepts_extra_root_ca_for_https_provider() { + let Some((base, ca_pem, handle)) = start_mock_oidc_tls_discovery_server( + |base| (format!("{base}/application/o/rustfs"), format!("{base}/jwks"), "/jwks".to_string()), + 4, + ) else { + return; + }; + let config_url = format!("{base}/application/o/rustfs"); + let config = build_mocked_oidc_provider_config("default", &config_url); + let origin = Url::parse(&config.config_url) + .expect("mock config_url should parse") + .origin() + .ascii_serialization(); + let policy = OutboundPolicy::from_allowed_origins(&origin).expect("loopback TLS origin should be allowed"); + let extra_root_certs = + parse_oidc_extra_root_certs("test OIDC TLS CA", ca_pem.as_bytes()).expect("test CA bundle should parse"); + let http_client = ReqwestHttpClient::with_policy_and_extra_root_certs(policy, extra_root_certs); + + let state = discover_provider(&config, &http_client) + .await + .expect("OIDC discovery should trust the extra root CA"); + + assert_eq!(state.metadata.issuer().to_string(), format!("{base}/application/o/rustfs")); + assert!(handle.join().is_ok()); +} +#[tokio::test] +async fn oidc_discovery_refreshes_extra_root_ca_when_generation_changes() { + let Some((base_a, ca_pem_a, handle_a)) = start_mock_oidc_tls_discovery_server( + |base| (format!("{base}/application/o/rustfs-a"), format!("{base}/jwks"), "/jwks".to_string()), + 4, + ) else { + return; + }; + let Some((base_b, ca_pem_b, handle_b)) = start_mock_oidc_tls_discovery_server( + |base| (format!("{base}/application/o/rustfs-b"), format!("{base}/jwks"), "/jwks".to_string()), + 4, + ) else { + assert!(handle_a.join().is_ok()); + return; + }; + + let origin_a = Url::parse(&base_a) + .expect("mock base A should parse") + .origin() + .ascii_serialization(); + let origin_b = Url::parse(&base_b) + .expect("mock base B should parse") + .origin() + .ascii_serialization(); + let allowed_origins = format!("{origin_a},{origin_b}"); + let policy = OutboundPolicy::from_allowed_origins(&allowed_origins).expect("loopback TLS origins should be allowed"); + let material = Arc::new(Mutex::new(OidcExtraRootCaMaterial { + generation: 1, + root_ca_pem: Some(ca_pem_a.into_bytes()), + })); + let provider = OidcExtraRootCaProvider::new({ + let material = material.clone(); + move || { + let material = material.clone(); + async move { + material + .lock() + .map(|material| material.clone()) + .map_err(|e| format!("test OIDC extra CA material lock poisoned: {e}")) + } + } + }); + let http_client = ReqwestHttpClient::with_policy_and_extra_root_ca_provider(policy, provider); + + let config_a = build_mocked_oidc_provider_config("a", &format!("{base_a}/application/o/rustfs-a")); + let state_a = discover_provider(&config_a, &http_client) + .await + .expect("OIDC discovery should trust initial extra root CA"); + assert_eq!(state_a.metadata.issuer().to_string(), format!("{base_a}/application/o/rustfs-a")); + + { + let mut material = material + .lock() + .expect("test OIDC extra CA material lock should not be poisoned"); + material.generation = 2; + material.root_ca_pem = Some(ca_pem_b.into_bytes()); + } + let config_b = build_mocked_oidc_provider_config("b", &format!("{base_b}/application/o/rustfs-b")); + let state_b = discover_provider(&config_b, &http_client) + .await + .expect("OIDC discovery should refresh extra root CA after generation change"); + + assert_eq!(state_b.metadata.issuer().to_string(), format!("{base_b}/application/o/rustfs-b")); + assert!(handle_a.join().is_ok()); + assert!(handle_b.join().is_ok()); +} +#[test] +fn test_jwks_url_from_config_url_preserves_unrelated_urls() { + let issuer = IssuerUrl::new("https://public.example.com/realms/app".to_string()).expect("issuer URL should parse"); + for raw_jwks_url in [ + "https://keys.example.com/jwks", + "http://public.example.com/realms/app/jwks", + "https://public.example.com:8443/realms/app/jwks", + "https://public.example.com/keys/jwks", + "https://public.example.com/realms/application/jwks", + ] { + let jwks_url = JsonWebKeySetUrl::new(raw_jwks_url.to_string()).expect("JWKS URL should parse"); + + let resolved = + jwks_url_from_config_url("http://keycloak.internal/realms/app/.well-known/openid-configuration", &issuer, &jwks_url) + .expect("JWKS URL should resolve"); + + assert_eq!(resolved.as_str(), raw_jwks_url); + } + + let issuer_root_jwks = JsonWebKeySetUrl::new(issuer.as_str().to_string()).expect("JWKS URL should parse"); + let resolved = jwks_url_from_config_url( + "http://keycloak.internal/realms/app/.well-known/openid-configuration", + &issuer, + &issuer_root_jwks, + ) + .expect("issuer-root JWKS URL should resolve"); + assert_eq!(resolved.as_str(), "http://keycloak.internal/realms/app"); + + let issuer_with_slash = + IssuerUrl::new("https://public.example.com/realms/app/".to_string()).expect("issuer URL should parse"); + let jwks_url = + JsonWebKeySetUrl::new("https://public.example.com/realms/app/jwks".to_string()).expect("JWKS URL should parse"); + let resolved = jwks_url_from_config_url( + "http://keycloak.internal/realms/app/.well-known/openid-configuration", + &issuer_with_slash, + &jwks_url, + ) + .expect("issuer-relative JWKS URL should resolve"); + assert_eq!(resolved.as_str(), "http://keycloak.internal/realms/app/jwks"); +} +#[tokio::test] +async fn oidc_discovery_reports_forbidden_outbound_without_retrying() { + let config_url = "http://192.168.65.254:8080/realms/rustfs/.well-known/openid-configuration"; + let config = build_mocked_oidc_provider_config("default", config_url); + let http_client = ReqwestHttpClient::with_policy(OutboundPolicy::default()); + + let error = match discover_provider(&config, &http_client).await { + Ok(_) => panic!("private OIDC provider should require an explicit allowlist origin"), + Err(error) => error, + }; + + assert!(error.contains("OIDC provider discovery blocked by outbound policy")); + assert!(error.contains(&format!("add http://192.168.65.254:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}"))); + assert!(!error.contains("discovery failed for all issuer variants")); +} +#[tokio::test] +async fn oidc_explicit_issuer_reports_forbidden_discovery_endpoint() { + let config_url = "http://192.168.65.254:8080/realms/rustfs/.well-known/openid-configuration"; + let mut config = build_mocked_oidc_provider_config("default", config_url); + config.issuer = Some("https://idp.example.com/realms/rustfs".to_string()); + let http_client = ReqwestHttpClient::with_policy(OutboundPolicy::default()); + + let error = match discover_provider(&config, &http_client).await { + Ok(_) => panic!("private OIDC provider should require an explicit allowlist origin"), + Err(error) => error, + }; + + assert!(error.contains("OIDC provider discovery blocked by outbound policy")); + assert!(error.contains(&format!("add http://192.168.65.254:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}"))); +} +#[tokio::test] +async fn oidc_explicit_issuer_reports_forbidden_jwks_endpoint() { + let Some((base, handle)) = start_mock_oidc_discovery_server( + |base| { + ( + format!("{base}/realms/rustfs"), + "http://192.168.65.254:8080/realms/rustfs/protocol/openid-connect/certs".to_string(), + "/unused".to_string(), + ) + }, + 1, + ) else { + return; + }; + let mut config = + build_mocked_oidc_provider_config("default", &format!("{base}/realms/rustfs/.well-known/openid-configuration")); + config.issuer = Some(format!("{base}/realms/rustfs")); + let policy = OutboundPolicy::from_allowed_origins(&base).expect("loopback discovery origin should be allowed"); + let http_client = ReqwestHttpClient::with_policy(policy); + + let error = match discover_provider(&config, &http_client).await { + Ok(_) => panic!("private JWKS endpoint should require an explicit allowlist origin"), + Err(error) => error, + }; + + assert!(error.contains("JWKS request blocked by outbound policy")); + assert!(error.contains(&format!("add http://192.168.65.254:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}"))); + assert!(handle.join().is_ok()); +} +#[tokio::test] +async fn oidc_reqwest_dns_policy_rejection_stays_typed() { + let calls = Arc::new(AtomicUsize::new(0)); + let config = build_mocked_oidc_provider_config( + "default", + "http://keycloak.internal:8080/realms/rustfs/.well-known/openid-configuration", + ); + let http_client = ReqwestHttpClient::with_policy_and_dns_resolver( + OutboundPolicy::default(), + Arc::new(RejectingDnsResolver { + allow_origin_can_recover: true, + calls: Some(calls.clone()), + }), + ); + + let error = match discover_provider(&config, &http_client).await { + Ok(_) => panic!("private DNS answer should fail discovery"), + Err(error) => error, + }; + + assert!(error.contains("OIDC provider discovery blocked by outbound policy")); + assert!(error.contains(&format!("add http://keycloak.internal:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}"))); + assert_eq!(calls.load(Ordering::Relaxed), 1, "policy rejection must not be retried"); +} +#[tokio::test] +async fn oidc_explicit_issuer_preserves_dns_policy_rejection() { + let calls = Arc::new(AtomicUsize::new(0)); + let mut config = build_mocked_oidc_provider_config( + "default", + "http://keycloak.internal:8080/realms/rustfs/.well-known/openid-configuration", + ); + config.issuer = Some("https://idp.example.com/realms/rustfs".to_string()); + let http_client = ReqwestHttpClient::with_policy_and_dns_resolver( + OutboundPolicy::default(), + Arc::new(RejectingDnsResolver { + allow_origin_can_recover: true, + calls: Some(calls.clone()), + }), + ); + + let error = match discover_provider(&config, &http_client).await { + Ok(_) => panic!("private DNS answer should fail discovery"), + Err(error) => error, + }; + + assert!(error.contains("OIDC provider discovery blocked by outbound policy")); + assert!(error.contains(&format!("add http://keycloak.internal:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}"))); + assert_eq!(calls.load(Ordering::Relaxed), 1, "policy rejection must not be retried"); +} + +#[tokio::test] +async fn workload_config_validation_reports_absent_console_endpoints() { + let (base, handle) = start_mock_oidc_discovery_server_with_jwks( + |base| (base.to_string(), format!("{base}/jwks"), "/jwks".into()), + 2, + "ES256", + true, + |_| serde_json::json!({"keys": []}).to_string(), + ) + .expect("workload discovery mock must bind"); + let mut config = build_mocked_oidc_provider_config("workload", &base); + config.hide_from_ui = true; + // Inferred issuers keep the library's discovery URL construction. + config.config_url = format!("{base}/.well-known/openid-configuration?ignored=1#ignored"); + let result = validate_mocked_oidc_provider_config(&config) + .await + .expect("hidden workload configuration should validate"); + assert_eq!(result.issuer, base); + assert!(result.authorization_endpoint.is_none()); + assert!(result.token_endpoint.is_none()); + handle.join().unwrap(); +} + +#[tokio::test] +async fn test_validate_oidc_provider_config_retries_with_issuer_candidates() { + // Discovery document must advertise the canonical issuer path. The first candidate has no + // trailing slash; openidconnect rejects issuer mismatch, then the second variant succeeds. + let Some((base, handle)) = start_mock_oidc_discovery_server( + |base| (format!("{base}/application/o/rustfs/"), format!("{base}/jwks"), "/jwks".to_string()), + 8, + ) else { + return; + }; + let config_url = format!("{base}/application/o/rustfs"); + let config = build_mocked_oidc_provider_config("default", &config_url); + + let result = validate_mocked_oidc_provider_config(&config).await; + + let validation_result = result.expect("OIDC provider validation should succeed"); + assert_eq!(validation_result.issuer, format!("{base}/application/o/rustfs/")); + assert!(handle.join().is_ok()); +} + +#[tokio::test] +async fn test_validate_oidc_provider_config_fetches_issuer_relative_jwks_from_config_url() { + let Some((base, handle)) = start_mock_oidc_discovery_server( + |_| { + ( + "http://127.0.0.1:1/public/realms/app".to_string(), + "http://127.0.0.1:1/public/realms/app/jwks?version=1".to_string(), + "/internal/realms/app/jwks?version=1".to_string(), + ) + }, + 2, + ) else { + return; + }; + let mut config = + build_mocked_oidc_provider_config("default", &format!("{base}/internal/realms/app/.well-known/openid-configuration")); + config.issuer = Some("http://127.0.0.1:1/public/realms/app".to_string()); + + let validation_result = validate_mocked_oidc_provider_config(&config) + .await + .expect("OIDC provider validation should succeed"); + + assert_eq!(validation_result.issuer, "http://127.0.0.1:1/public/realms/app"); + assert!(handle.join().is_ok()); +} + +#[tokio::test] +async fn test_validate_oidc_provider_config_rejects_separate_issuer_mismatch() { + let Some((base, handle)) = start_mock_oidc_discovery_server( + |base| { + ( + "https://public.example.com/realms/other".to_string(), + format!("{base}/jwks"), + "/jwks".to_string(), + ) + }, + 1, + ) else { + return; + }; + let mut config = + build_mocked_oidc_provider_config("default", &format!("{base}/internal/realms/app/.well-known/openid-configuration")); + config.issuer = Some("https://public.example.com/realms/app".to_string()); + + let err = validate_mocked_oidc_provider_config(&config) + .await + .expect_err("OIDC provider validation should fail"); + + assert!(err.contains("unexpected issuer URI")); + assert!(err.contains("https://public.example.com/realms/app")); + assert!(handle.join().is_ok()); +} + +#[tokio::test] +async fn test_validate_oidc_provider_config_returns_detailed_errors() { + let Some((base, handle)) = start_mock_oidc_discovery_server( + |base| (format!("{base}/application/o/other"), format!("{base}/jwks"), "/jwks".to_string()), + 8, + ) else { + return; + }; + let config_url = format!("{base}/application/o/rustfs"); + let config = build_mocked_oidc_provider_config("default", &config_url); + + let err = validate_mocked_oidc_provider_config(&config) + .await + .expect_err("OIDC provider validation should fail"); + assert!(discovery_error_contains_all_variants(&err, &base)); + assert!(err.contains("issuer '")); + assert!(err.contains(&format!("issuer '{base}/application/o/rustfs'"))); + assert!(err.contains(&format!("issuer '{base}/application/o/rustfs/'"))); + assert!(handle.join().is_ok()); +} diff --git a/crates/iam/src/oidc/runtime.rs b/crates/iam/src/oidc/runtime.rs new file mode 100644 index 000000000..01cff8046 --- /dev/null +++ b/crates/iam/src/oidc/runtime.rs @@ -0,0 +1,897 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use super::{ + config::{ + OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderSummary, SourcedOidcProviderConfig, + load_oidc_config_snapshot, + }, + provider::{ProviderRuntime, discover_provider}, + state::{OidcAuthSession, OidcLogoutSession, OidcStateStore}, + transport::{OidcExtraRootCaProvider, ReqwestHttpClient, inspect_token_response_body, oidc_http_error_diagnostics}, +}; +#[cfg(test)] +use crate::federation::FederatedAuthorizationRuleRef; +use crate::{ + EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC, + federation::{ + CoreFederatedAuthorizationMapper, FederatedAuthorizationRule, FederatedAuthorizationRules, FederatedClaims, + FederatedIdentityService, FederatedProviderQuery, FederatedProviderRef, FederatedProviderView, FederatedRedirectPolicy, + FederationError, OpaqueLogoutContinuation, Result as FederationResult, StandardOidcAuthentication, + VerifiedFederatedCodeExchange, VerifiedFederatedIdentity, + }, +}; +use openidconnect::core::{CoreAuthenticationFlow, CoreClient, CoreIdToken}; +use openidconnect::{ + Audience, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, LogoutRequest, Nonce, PkceCodeChallenge, + PkceCodeVerifier, PostLogoutRedirectUrl, RedirectUrl, RequestTokenError, Scope, +}; +use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive}; +use serde::{Deserialize, Serialize}; +use std::borrow::Cow; +use std::collections::HashMap; +use std::sync::Arc; +use tracing::{debug, error, warn}; + +/// Claims extracted from an OIDC ID token. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub(crate) struct OidcClaims { + pub sub: String, + pub email: String, + pub username: String, + pub groups: Vec, + pub raw: HashMap, +} + +// ---- Core OIDC system ---- + +/// Global OIDC manager for all configured providers. +pub struct OidcSys { + pub(super) configs: HashMap, + pub(super) provider_runtime: ProviderRuntime, + pub(super) state_store: OidcStateStore, +} + +pub(super) fn trusted_aud(other_audiences: &[String], audience: &Audience) -> bool { + for aud in other_audiences { + if audience.as_str() == aud.as_str() { + return true; + } + } + false +} + +impl OidcSys { + pub(crate) async fn new_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result { + let http_client = ReqwestHttpClient::new_with_extra_root_ca_provider(extra_root_ca_provider)?; + http_client.current_extra_root_certs().await.map_err(|err| err.to_string())?; + Self::new_with_http_client(http_client).await + } + + async fn new_with_http_client(http_client: ReqwestHttpClient) -> Result { + let server_config = crate::server_config::current_server_config(); + let parsed_configs = load_oidc_config_snapshot(server_config.as_ref()); + let mut configs = HashMap::new(); + let mut provider_states = HashMap::new(); + + for sourced_config in parsed_configs.into_providers() { + let config = &sourced_config.config; + if !config.enabled { + debug!(provider = %config.id, "OIDC provider disabled"); + continue; + } + + match discover_provider(config, &http_client).await { + Ok(state) => { + debug!(provider = %config.id, "OIDC provider discovered"); + provider_states.insert(config.id.clone(), state); + configs.insert(config.id.clone(), sourced_config); + } + Err(e) => { + error!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "provider_discovery_failed", + provider_id = %config.id, + config_url = %config.config_url, + client_id = %config.client_id, + scopes = ?config.scopes, + redirect_uri = %config.redirect_uri.as_deref().unwrap_or(""), + redirect_uri_dynamic = config.redirect_uri_dynamic, + error = %e, + "oidc provider discovery failed" + ); + } + } + } + + Ok(Self { + configs, + provider_runtime: ProviderRuntime::new(http_client, provider_states), + state_store: OidcStateStore::new(), + }) + } + + /// Create an OidcSys with no providers (useful for when OIDC is not configured). + pub fn empty() -> Result { + Ok(Self { + configs: HashMap::new(), + provider_runtime: ProviderRuntime::new(ReqwestHttpClient::new()?, HashMap::new()), + state_store: OidcStateStore::new(), + }) + } + + /// Return true if any OIDC providers are configured and enabled. + pub(crate) fn has_providers(&self) -> bool { + !self.configs.is_empty() + } + + pub(crate) fn provider_configs(&self) -> impl Iterator { + self.configs.values().map(|provider| &provider.config) + } + + /// List all providers, including providers hidden from the login UI. + pub(crate) fn list_providers(&self) -> Vec { + self.provider_configs() + .map(|config| OidcProviderSummary { + provider_id: config.id.clone(), + display_name: config.display_name.clone(), + }) + .collect() + } + + /// List providers visible in the login UI. + pub(crate) fn list_visible_providers(&self) -> Vec { + self.provider_configs() + .filter(|config| !config.hide_from_ui) + .map(|config| OidcProviderSummary { + provider_id: config.id.clone(), + display_name: config.display_name.clone(), + }) + .collect() + } + + pub(crate) fn config_snapshot(&self) -> OidcConfigSnapshot { + OidcConfigSnapshot::new(self.configs.values().cloned().collect()) + } + + /// Build the PKCE authorization URL for a provider, store state in the state store. + pub(crate) async fn authorize_url( + &self, + provider_id: &str, + redirect_uri: &str, + redirect_after: Option, + ) -> Result { + let config = self + .get_provider_config(provider_id) + .ok_or_else(|| format!("unknown OIDC provider: {provider_id}"))?; + let state = self.provider_runtime.ensure_provider_state(provider_id, config).await?; + + let (pkce_challenge, pkce_verifier) = PkceCodeChallenge::new_random_sha256(); + + let redirect = RedirectUrl::new(redirect_uri.to_string()).map_err(|e| format!("invalid redirect URI: {e}"))?; + + let client = CoreClient::from_provider_metadata( + state.metadata.console()?.clone(), + ClientId::new(config.client_id.clone()), + config.client_secret.as_ref().map(|s| ClientSecret::new(s.clone())), + ) + .set_auth_type(AuthType::RequestBody); + + let mut auth_req = + client.authorize_url(CoreAuthenticationFlow::AuthorizationCode, CsrfToken::new_random, Nonce::new_random); + auth_req = auth_req.set_redirect_uri(Cow::Owned(redirect)); + + for scope in &config.scopes { + auth_req = auth_req.add_scope(Scope::new(scope.clone())); + } + + auth_req = auth_req.set_pkce_challenge(pkce_challenge); + + let (auth_url, csrf_token, nonce) = auth_req.url(); + + // Store the state for callback validation + self.state_store + .insert( + csrf_token.secret().clone(), + OidcAuthSession { + provider_id: provider_id.to_string(), + pkce_verifier: pkce_verifier.secret().clone(), + nonce: nonce.secret().clone(), + redirect_after, + }, + ) + .await; + + Ok(auth_url.to_string()) + } + + /// Exchange an authorization code for tokens and extract claims. + pub(super) async fn exchange_code( + &self, + state: &str, + code: &str, + redirect_uri: &str, + ) -> Result<(OidcClaims, String, OidcAuthSession, String), String> { + // Retrieve and consume the state (single-use) + let session = self + .state_store + .take(state) + .await + .ok_or_else(|| "invalid or expired OIDC state".to_string())?; + + let config = self + .get_provider_config(&session.provider_id) + .ok_or_else(|| format!("unknown provider: {}", session.provider_id))?; + let provider_state = self.provider_runtime.get_provider_state(&session.provider_id)?; + let issuer = provider_state.metadata.issuer().to_string(); + let token_endpoint = provider_state + .metadata + .token_endpoint() + .map(ToString::to_string) + .unwrap_or_default(); + + // Construct CoreClient on-the-fly with JWKS from discovery + let client = CoreClient::from_provider_metadata( + provider_state.metadata.console()?.clone(), + ClientId::new(config.client_id.clone()), + config.client_secret.as_ref().map(|s| ClientSecret::new(s.clone())), + ) + .set_auth_type(AuthType::RequestBody); + + let redirect = RedirectUrl::new(redirect_uri.to_string()).map_err(|e| format!("invalid redirect URI: {e}"))?; + + // Exchange code for tokens + let token_response = client + .exchange_code(AuthorizationCode::new(code.to_string())) + .map_err(|e| { + error!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "token_endpoint_missing", + provider_id = %session.provider_id, + config_url = %config.config_url, + issuer = %issuer, + client_id = %config.client_id, + redirect_uri = %redirect_uri, + scopes = ?config.scopes, + error = %e, + "oidc token exchange failed" + ); + format!( + "token endpoint not configured: {e}: provider_id={}, config_url={}, issuer={}, redirect_uri={}, client_id={}", + session.provider_id, config.config_url, issuer, redirect_uri, config.client_id + ) + })? + .set_pkce_verifier(PkceCodeVerifier::new(session.pkce_verifier.clone())) + .set_redirect_uri(Cow::Owned(redirect)) + .request_async(self.provider_runtime.http_client()) + .await + .map_err(|e| match &e { + RequestTokenError::ServerResponse(response) => { + error!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "token_server_response", + provider_id = %session.provider_id, + config_url = %config.config_url, + issuer = %issuer, + token_endpoint = %token_endpoint, + client_id = %config.client_id, + client_secret_configured = config.client_secret.as_deref().is_some_and(|secret| !secret.is_empty()), + redirect_uri = %redirect_uri, + scopes = ?config.scopes, + oauth_error = %response.error(), + oauth_error_description = %response.error_description().map(String::as_str).unwrap_or(""), + oauth_error_uri = %response.error_uri().map(String::as_str).unwrap_or(""), + error = %e, + "oidc token exchange failed" + ); + format!( + "token exchange failed: {e}: stage=token_server_response, provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}, oauth_error={}, oauth_error_description={}", + session.provider_id, + config.config_url, + issuer, + token_endpoint, + redirect_uri, + config.client_id, + response.error(), + response.error_description().map(String::as_str).unwrap_or("") + ) + } + RequestTokenError::Request(err) => { + let (request_error_kind, request_error_status) = oidc_http_error_diagnostics(err); + error!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "token_request_failed", + provider_id = %session.provider_id, + config_url = %config.config_url, + issuer = %issuer, + token_endpoint = %token_endpoint, + client_id = %config.client_id, + redirect_uri = %redirect_uri, + request_error_kind = %request_error_kind, + request_error_status = %request_error_status, + error = %err, + "oidc token exchange failed" + ); + format!( + "token exchange failed: stage=token_request_failed, provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}, request_error_kind={}, request_error_status={}, request_error={}", + session.provider_id, + config.config_url, + issuer, + token_endpoint, + redirect_uri, + config.client_id, + request_error_kind, + request_error_status, + err + ) + } + RequestTokenError::Parse(parse_err, body) => { + let shape = inspect_token_response_body(body); + error!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "token_response_parse_failed", + provider_id = %session.provider_id, + config_url = %config.config_url, + issuer = %issuer, + token_endpoint = %token_endpoint, + client_id = %config.client_id, + redirect_uri = %redirect_uri, + parse_error_path = %parse_err.path(), + response_body_len = body.len(), + response_json_object = shape.json_object, + response_json_keys = %shape.json_keys, + response_has_access_token = shape.has_access_token, + response_has_id_token = shape.has_id_token, + response_has_token_type = shape.has_token_type, + response_has_expires_in = shape.has_expires_in, + response_has_error = shape.has_error, + response_has_error_description = shape.has_error_description, + response_looks_like_html = shape.looks_like_html, + error = %e, + "oidc token exchange failed" + ); + format!( + "token exchange failed: {e}: stage=token_response_parse_failed, provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}, parse_error_path={}, response_body_len={}, response_json_keys={}, response_has_id_token={}, response_has_error={}, response_looks_like_html={}", + session.provider_id, + config.config_url, + issuer, + token_endpoint, + redirect_uri, + config.client_id, + parse_err.path(), + body.len(), + shape.json_keys, + shape.has_id_token, + shape.has_error, + shape.looks_like_html + ) + } + RequestTokenError::Other(message) => { + error!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "token_exchange_other_error", + provider_id = %session.provider_id, + config_url = %config.config_url, + issuer = %issuer, + token_endpoint = %token_endpoint, + client_id = %config.client_id, + redirect_uri = %redirect_uri, + error = %message, + "oidc token exchange failed" + ); + format!( + "token exchange failed: {e}: stage=token_exchange_other_error, provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}", + session.provider_id, config.config_url, issuer, token_endpoint, redirect_uri, config.client_id + ) + } + })?; + + // Verify the ID token (signature, issuer, audience, expiry, nonce) + let id_token = token_response + .extra_fields() + .id_token() + .ok_or_else(|| { + error!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "token_response_missing_id_token", + provider_id = %session.provider_id, + config_url = %config.config_url, + issuer = %issuer, + token_endpoint = %token_endpoint, + client_id = %config.client_id, + redirect_uri = %redirect_uri, + scopes = ?config.scopes, + "oidc token exchange failed" + ); + format!( + "no id_token in token response: provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}, scopes={}", + session.provider_id, + config.config_url, + issuer, + token_endpoint, + redirect_uri, + config.client_id, + config.scopes.join(",") + ) + })?; + + let verifier = client + .id_token_verifier() + .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); + let verified = id_token.claims(&verifier, &Nonce::new(session.nonce.clone())); + if let Err(e) = verified { + let verification_error = e.to_string(); + warn!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "id_token_verification_retry", + provider_id = %session.provider_id, + config_url = %config.config_url, + issuer = %issuer, + token_endpoint = %token_endpoint, + client_id = %config.client_id, + other_audiences = ?config.other_audiences, + error = %verification_error, + "oidc id token verification failed" + ); + let refreshed_state = self + .provider_runtime + .refresh_provider_state(&session.provider_id, config) + .await + .map_err(|refresh_err| { + format!( + "ID token verification failed: {verification_error}; failed to refresh provider metadata: {refresh_err}" + ) + })?; + + warn!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "jwks_metadata_refreshed", + provider_id = %session.provider_id, + config_url = %config.config_url, + issuer = %issuer, + "oidc provider metadata refreshed" + ); + + let client = CoreClient::from_provider_metadata( + refreshed_state.metadata.console()?.clone(), + ClientId::new(config.client_id.clone()), + config.client_secret.as_ref().map(|s| ClientSecret::new(s.clone())), + ) + .set_auth_type(AuthType::RequestBody); + + let verifier = client + .id_token_verifier() + .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); + id_token + .claims(&verifier, &Nonce::new(session.nonce.clone())) + .map_err(|retry_err| { + error!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "id_token_verification_failed", + provider_id = %session.provider_id, + config_url = %config.config_url, + issuer = %issuer, + token_endpoint = %token_endpoint, + client_id = %config.client_id, + other_audiences = ?config.other_audiences, + original_error = %verification_error, + retry_error = %retry_err, + "oidc id token verification failed" + ); + format!( + "ID token verification failed after JWKS refresh: {retry_err}; original_error={verification_error}; provider_id={}, config_url={}, issuer={}, token_endpoint={}, client_id={}, other_audiences={}", + session.provider_id, + config.config_url, + issuer, + token_endpoint, + config.client_id, + config.other_audiences.join(",") + ) + })?; + } + + // Extract raw claims from the verified JWT for custom claim support + // (the crate verifies signature/expiry/nonce; we decode payload for non-standard claims) + let raw_jwt = id_token.to_string(); + let raw = decode_jwt_payload(&raw_jwt); + + let claims = OidcClaims { + sub: extract_string_claim(&raw, "sub"), + email: extract_string_claim(&raw, &config.email_claim), + username: extract_string_claim(&raw, &config.username_claim), + groups: extract_canonical_group_values(&raw, &config.groups_claim, &config.roles_claim), + raw, + }; + + Ok((claims, session.provider_id.clone(), session, raw_jwt)) + } + + /// Store a one-time logout session keyed by an opaque token so the console can + /// trigger browser logout without persisting the raw ID token. + pub(crate) async fn create_logout_token(&self, provider_id: &str, id_token: &str) -> Result { + if !self.configs.contains_key(provider_id) { + return Err(format!("unknown OIDC provider: {provider_id}")); + } + + let token = CsrfToken::new_random().secret().clone(); + self.state_store + .insert_logout( + token.clone(), + OidcLogoutSession { + provider_id: provider_id.to_string(), + id_token: id_token.to_string(), + }, + ) + .await; + + Ok(token) + } + + /// Build the RP-initiated logout URL for a previously issued logout token. + /// Returns `Ok(None)` when the provider does not advertise an end-session endpoint. + pub(crate) async fn build_logout_url( + &self, + logout_token: &str, + post_logout_redirect_uri: &str, + ) -> Result, String> { + let session = self + .state_store + .take_logout(logout_token) + .await + .ok_or_else(|| "invalid or expired OIDC logout token".to_string())?; + + let config = self + .get_provider_config(&session.provider_id) + .ok_or_else(|| format!("unknown OIDC provider: {}", session.provider_id))?; + let state = self + .provider_runtime + .ensure_provider_state(&session.provider_id, config) + .await?; + let Some(end_session_endpoint) = state.metadata.console()?.additional_metadata().end_session_endpoint.clone() else { + return Ok(None); + }; + + let id_token: CoreIdToken = session + .id_token + .parse() + .map_err(|e: serde_json::Error| format!("failed to parse ID token for logout: {e}"))?; + let post_logout_redirect_uri = PostLogoutRedirectUrl::new(post_logout_redirect_uri.to_string()) + .map_err(|e| format!("invalid post logout redirect URI: {e}"))?; + + let logout_url = LogoutRequest::from(end_session_endpoint) + .set_id_token_hint(&id_token) + .set_client_id(ClientId::new(config.client_id.clone())) + .set_post_logout_redirect_uri(post_logout_redirect_uri) + .http_get_url() + .to_string(); + + Ok(Some(logout_url)) + } + + /// Map OIDC claims to rustfs policy names. + #[cfg(test)] + pub(crate) fn map_claims_to_policies(&self, provider_id: &str, claims: &OidcClaims) -> (Vec, Vec) { + let Some(rule) = self.authorization_rule(provider_id) else { + return (Vec::new(), Vec::new()); + }; + CoreFederatedAuthorizationMapper::map_policies_and_groups(provider_id, rule, &claims.groups, &claims.raw, true) + } + + /// Policy names produced only by the canonical groups in claim-based mode: the groups claim + /// plus the roles claim values that `extract_canonical_group_values` merges into it. + /// + /// Identity providers emit built-in groups and roles that can never have a matching policy + /// (for example `DOMAIN\Domain Users` or Keycloak's `offline_access`), so the session binding + /// may ignore these names when no such policy exists. Names from a fixed role policy or a + /// dedicated policy claim are never included, so those configurations keep requiring every + /// policy to resolve. + #[cfg(test)] + pub(crate) fn group_claim_policy_names(&self, provider_id: &str, claims: &OidcClaims) -> Vec { + let Some(rule) = self.authorization_rule(provider_id) else { + return Vec::new(); + }; + CoreFederatedAuthorizationMapper::group_claim_policy_names(rule, &claims.groups, &claims.raw) + } + + #[cfg(test)] + fn authorization_rule(&self, provider_id: &str) -> Option> { + self.get_provider_config(provider_id).map(|config| { + FederatedAuthorizationRuleRef::new( + &config.claim_name, + &config.claim_prefix, + &config.role_policy, + &config.groups_claim, + &config.roles_claim, + ) + }) + } + + /// Verify a raw JWT (id_token) for the AssumeRoleWithWebIdentity flow. + /// + /// Unlike the authorization code flow, ARWWI receives a raw JWT directly + /// (not via code exchange). This method: + /// 1. Decodes the JWT payload to extract the `iss` claim + /// 2. Finds the OIDC provider whose issuer matches + /// 3. Verifies signature, issuer, audience, and expiry (nonce is skipped) + /// 4. Extracts claims using the provider's claim configuration + pub(crate) async fn verify_web_identity_token(&self, jwt: &str) -> Result<(OidcClaims, String /* provider_id */), String> { + // Decode JWT payload without verification to get the issuer claim + let raw_claims = decode_jwt_payload(jwt); + let issuer = raw_claims + .get("iss") + .and_then(|v| v.as_str()) + .ok_or_else(|| "JWT missing 'iss' claim".to_string())?; + + // Find matching provider by issuer + let (provider_id, config, mut state) = self + .provider_runtime + .find_provider_by_issuer(issuer, &self.configs) + .ok_or_else(|| format!("no OIDC provider configured for issuer: {issuer}"))?; + + state = self + .provider_runtime + .ensure_provider_state_if_stale(&provider_id, &config, &state) + .await?; + + // Parse raw JWT string into CoreIdToken + let id_token: CoreIdToken = jwt + .parse() + .map_err(|e: serde_json::Error| format!("failed to parse JWT as ID token: {e}"))?; + + // Verify the token (signature, issuer, audience, expiry) — skip nonce + // (nonce is only required for the authorization code flow) + let verifier = state + .metadata + .verifier(&config) + .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); + if let Err(e) = id_token.claims(&verifier, |_: Option<&Nonce>| Ok(())) { + state = self + .provider_runtime + .refresh_provider_state(&provider_id, &config) + .await + .map_err(|refresh_err| { + format!("ID token verification failed: {e}; failed to refresh provider metadata: {refresh_err}") + })?; + + let verifier = state + .metadata + .verifier(&config) + .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud)); + id_token + .claims(&verifier, |_: Option<&Nonce>| Ok(())) + .map_err(|retry_err| format!("ID token verification failed after JWKS refresh: {retry_err}"))?; + } + + // Extract claims using the provider's claim configuration + let claims = OidcClaims { + sub: extract_string_claim(&raw_claims, "sub"), + email: extract_string_claim(&raw_claims, &config.email_claim), + username: extract_string_claim(&raw_claims, &config.username_claim), + groups: extract_canonical_group_values(&raw_claims, &config.groups_claim, &config.roles_claim), + raw: raw_claims, + }; + + Ok((claims, provider_id.to_string())) + } + /// Get a provider config by ID. + pub(crate) fn get_provider_config(&self, id: &str) -> Option<&OidcProviderConfig> { + self.configs.get(id).map(|provider| &provider.config) + } +} + +/// Decode the payload section of a JWT without validation (token must already be verified). +pub(crate) fn decode_jwt_payload(token: &str) -> HashMap { + let parts: Vec<&str> = token.split('.').collect(); + if parts.len() < 2 { + return HashMap::new(); + } + let payload_bytes = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(parts[1]); + match payload_bytes { + Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_default(), + Err(_) => HashMap::new(), + } +} + +/// Extract a string claim from raw claims with case-insensitive fallback. +fn extract_string_claim(claims: &HashMap, key: &str) -> String { + match get_claim_case_insensitive(claims, key) { + ClaimLookup::Found(value) => value.as_str().unwrap_or_default().to_string(), + ClaimLookup::Missing | ClaimLookup::Ambiguous => String::new(), + } +} + +/// Extract a groups/array claim from raw claims with case-insensitive fallback. Handles both string arrays and single strings. +fn extract_groups_claim(claims: &HashMap, key: &str) -> Vec { + match get_claim_case_insensitive(claims, key) { + ClaimLookup::Found(serde_json::Value::Array(arr)) => arr.iter().filter_map(|v| v.as_str().map(String::from)).collect(), + ClaimLookup::Found(serde_json::Value::String(s)) => s.split(',').map(|s| s.trim().to_string()).collect(), + _ => vec![], + } +} + +fn extract_canonical_group_values( + claims: &HashMap, + groups_claim: &str, + roles_claim: &str, +) -> Vec { + let mut groups = extract_groups_claim(claims, groups_claim); + if !roles_claim.is_empty() && roles_claim != groups_claim { + groups.extend(extract_groups_claim(claims, roles_claim)); + } + groups.retain(|g| !g.is_empty()); + groups.sort(); + groups.dedup(); + groups +} + +fn verified_identity(provider: FederatedProviderRef, claims: OidcClaims) -> VerifiedFederatedIdentity { + VerifiedFederatedIdentity::from_claims( + provider, + FederatedClaims { + sub: claims.sub, + email: claims.email, + username: claims.username, + groups: claims.groups, + raw: claims.raw, + }, + ) +} + +fn authorization_rules(oidc: &OidcSys) -> FederatedAuthorizationRules { + FederatedAuthorizationRules::new(oidc.provider_configs().map(|config| { + FederatedAuthorizationRule::new( + config.id.clone(), + config.claim_name.clone(), + config.claim_prefix.clone(), + config.role_policy.clone(), + config.groups_claim.clone(), + config.roles_claim.clone(), + ) + })) +} + +pub struct StandardOidcAdapter { + oidc: Arc, + authorization_rules: FederatedAuthorizationRules, +} + +impl StandardOidcAdapter { + pub fn new(oidc: Arc) -> Self { + let authorization_rules = authorization_rules(&oidc); + Self { + oidc, + authorization_rules, + } + } + + pub fn authorization_rules(&self) -> FederatedAuthorizationRules { + self.authorization_rules.clone() + } + + pub fn into_service(self: Arc, mapper: CoreFederatedAuthorizationMapper) -> FederatedIdentityService { + let authentication: Arc = self.clone(); + let provider_query: Arc = self; + FederatedIdentityService::from_standard_oidc_parts(provider_query, authentication, mapper) + } +} + +impl OidcConfigQuery for StandardOidcAdapter { + fn config_snapshot(&self) -> OidcConfigSnapshot { + self.oidc.config_snapshot() + } +} + +#[async_trait::async_trait] +impl StandardOidcAuthentication for StandardOidcAdapter { + async fn authorize_url( + &self, + provider_id: &str, + redirect_uri: &str, + redirect_after: Option, + ) -> FederationResult { + self.oidc + .authorize_url(provider_id, redirect_uri, redirect_after) + .await + .map_err(FederationError::Authorization) + } + + async fn exchange_identity( + &self, + state: &str, + code: &str, + redirect_uri: &str, + ) -> FederationResult { + let (claims, provider_id, session, id_token) = self + .oidc + .exchange_code(state, code, redirect_uri) + .await + .map_err(FederationError::CodeExchange)?; + let provider = FederatedProviderRef::new(provider_id); + Ok(VerifiedFederatedCodeExchange::new( + verified_identity(provider.clone(), claims), + session.redirect_after, + OpaqueLogoutContinuation::new(provider, id_token), + )) + } + + async fn verify_identity(&self, jwt: &str) -> FederationResult { + let (claims, provider_id) = self + .oidc + .verify_web_identity_token(jwt) + .await + .map_err(FederationError::TokenVerification)?; + Ok(verified_identity(FederatedProviderRef::new(provider_id), claims)) + } + + async fn create_logout_token(&self, continuation: OpaqueLogoutContinuation) -> FederationResult { + let (provider, id_token) = continuation.into_parts(); + self.oidc + .create_logout_token(provider.as_str(), &id_token) + .await + .map_err(FederationError::Logout) + } + + async fn build_logout_url(&self, logout_token: &str, post_logout_redirect_uri: &str) -> FederationResult> { + self.oidc + .build_logout_url(logout_token, post_logout_redirect_uri) + .await + .map_err(FederationError::Logout) + } +} + +impl FederatedProviderQuery for StandardOidcAdapter { + fn has_providers(&self) -> bool { + self.oidc.has_providers() + } + + fn list_providers(&self) -> Vec { + self.oidc.list_providers() + } + + fn list_visible_providers(&self) -> Vec { + self.oidc.list_visible_providers() + } + + fn redirect_policy(&self, provider_id: &str) -> Option { + self.oidc + .get_provider_config(provider_id) + .map(|config| FederatedRedirectPolicy { + redirect_uri: config.redirect_uri.clone(), + allow_request_origin: config.redirect_uri_dynamic, + }) + } +} + +#[cfg(test)] +#[path = "runtime_tests.rs"] +mod tests; diff --git a/crates/iam/src/oidc/runtime_tests.rs b/crates/iam/src/oidc/runtime_tests.rs new file mode 100644 index 000000000..afca6f4c3 --- /dev/null +++ b/crates/iam/src/oidc/runtime_tests.rs @@ -0,0 +1,754 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::oidc::{config::*, provider::*, test_support::*}; +use jsonwebtoken::{Algorithm, Header}; +use openidconnect::ProviderMetadataWithLogout; +use rustfs_utils::egress::{ENV_OUTBOUND_ALLOW_ORIGINS, OutboundPolicy}; +use std::time::{Duration as StdDuration, Instant}; +use url::Url; + +#[test] +fn test_extract_string_claim() { + let mut claims = HashMap::new(); + claims.insert("email".to_string(), serde_json::json!("user@example.com")); + claims.insert("sub".to_string(), serde_json::json!("12345")); + + assert_eq!(extract_string_claim(&claims, "email"), "user@example.com"); + assert_eq!(extract_string_claim(&claims, "sub"), "12345"); + assert_eq!(extract_string_claim(&claims, "missing"), ""); +} +#[test] +fn test_extract_groups_claim_array() { + let mut claims = HashMap::new(); + claims.insert("groups".to_string(), serde_json::json!(["admin", "developers", "readonly"])); + + let groups = extract_groups_claim(&claims, "groups"); + assert_eq!(groups, vec!["admin", "developers", "readonly"]); +} +#[test] +fn test_extract_groups_claim_string() { + let mut claims = HashMap::new(); + claims.insert("groups".to_string(), serde_json::json!("admin,developers")); + + let groups = extract_groups_claim(&claims, "groups"); + assert_eq!(groups, vec!["admin", "developers"]); +} +#[test] +fn test_extract_groups_claim_missing() { + let claims = HashMap::new(); + let groups = extract_groups_claim(&claims, "groups"); + assert!(groups.is_empty()); +} +#[test] +fn test_extract_groups_claim_number() { + let mut claims = HashMap::new(); + claims.insert("groups".to_string(), serde_json::json!(42)); + let groups = extract_groups_claim(&claims, "groups"); + assert!(groups.is_empty()); +} +#[test] +fn test_extract_canonical_group_values_merges_groups_and_roles() { + let mut claims = HashMap::new(); + claims.insert("groups".to_string(), serde_json::json!(["devs", "admins"])); + claims.insert("roles".to_string(), serde_json::json!(["admins", "consoleAdmin"])); + + let merged = extract_canonical_group_values(&claims, "groups", "roles"); + assert_eq!(merged, vec!["admins", "consoleAdmin", "devs"]); +} +#[test] +fn test_extract_canonical_group_values_skips_duplicate_claim_name() { + let mut claims = HashMap::new(); + claims.insert("roles".to_string(), serde_json::json!(["consoleAdmin"])); + + let merged = extract_canonical_group_values(&claims, "roles", "roles"); + assert_eq!(merged, vec!["consoleAdmin"]); +} +#[test] +fn test_extract_canonical_group_values_roles_only() { + let mut claims = HashMap::new(); + claims.insert("roles".to_string(), serde_json::json!(["consoleAdmin", "bucket-reader"])); + + let merged = extract_canonical_group_values(&claims, "groups", "roles"); + assert_eq!(merged, vec!["bucket-reader", "consoleAdmin"]); +} +#[test] +fn test_extract_string_claim_case_insensitive() { + let mut claims = HashMap::new(); + claims.insert("policyminio".to_string(), serde_json::json!("consoleAdmin")); + + assert_eq!(extract_string_claim(&claims, "policyMinio"), "consoleAdmin"); + assert_eq!(extract_string_claim(&claims, "POLICYMINIO"), "consoleAdmin"); + assert_eq!(extract_string_claim(&claims, "policyminio"), "consoleAdmin"); +} +#[test] +fn test_extract_groups_claim_case_insensitive() { + let mut claims = HashMap::new(); + claims.insert("policyminio".to_string(), serde_json::json!(["consoleAdmin", "readwrite"])); + + let groups = extract_groups_claim(&claims, "policyMinio"); + assert_eq!(groups, vec!["consoleAdmin", "readwrite"]); + + let groups = extract_groups_claim(&claims, "POLICYMINIO"); + assert_eq!(groups, vec!["consoleAdmin", "readwrite"]); + + let groups = extract_groups_claim(&claims, "policyminio"); + assert_eq!(groups, vec!["consoleAdmin", "readwrite"]); +} +#[test] +fn test_extract_groups_claim_exact_match_preferred() { + let mut claims = HashMap::new(); + claims.insert("Policy".to_string(), serde_json::json!(["exact_match"])); + claims.insert("policy".to_string(), serde_json::json!(["lowercase"])); + + let groups = extract_groups_claim(&claims, "Policy"); + assert_eq!(groups, vec!["exact_match"]); +} +#[test] +fn test_extract_string_claim_ambiguous_case_insensitive_match_returns_empty() { + let mut claims = HashMap::new(); + claims.insert("Policy".to_string(), serde_json::json!("exact_match")); + claims.insert("policy".to_string(), serde_json::json!("lowercase")); + + assert_eq!(extract_string_claim(&claims, "POLICY"), ""); +} +#[test] +fn test_extract_groups_claim_ambiguous_case_insensitive_match_returns_empty() { + let mut claims = HashMap::new(); + claims.insert("Policy".to_string(), serde_json::json!(["exact_match"])); + claims.insert("policy".to_string(), serde_json::json!(["lowercase"])); + + let groups = extract_groups_claim(&claims, "POLICY"); + assert!(groups.is_empty()); +} +#[test] +fn test_decode_jwt_payload() { + let payload = r#"{"sub":"user123","email":"user@example.com"}"#; + let payload_b64 = base64_simd::URL_SAFE_NO_PAD.encode_to_string(payload.as_bytes()); + let token = format!("eyJhbGciOiJSUzI1NiJ9.{payload_b64}.signature"); + + let claims = decode_jwt_payload(&token); + assert_eq!(claims.get("sub").and_then(|v| v.as_str()), Some("user123")); + assert_eq!(claims.get("email").and_then(|v| v.as_str()), Some("user@example.com")); +} +#[tokio::test] +async fn web_identity_verification_refreshes_rotated_jwks() { + let (_, initial_jwk) = oidc_es256_key_and_jwk("initial"); + let (rotated_key, rotated_jwk) = oidc_es256_key_and_jwk("rotated"); + let initial_jwks = serde_json::json!({ "keys": [initial_jwk] }).to_string(); + let rotated_jwks = serde_json::json!({ "keys": [rotated_jwk] }).to_string(); + let Some((base, handle)) = start_mock_oidc_discovery_server_with_jwks( + |base| (base.to_string(), format!("{base}/jwks"), "/jwks".to_string()), + 4, + "ES256", + false, + move |fetch| { + if fetch == 0 { + initial_jwks.clone() + } else { + rotated_jwks.clone() + } + }, + ) else { + return; + }; + + let config = build_mocked_oidc_provider_config("rotating", &base); + let policy = OutboundPolicy::from_allowed_origins(&base).expect("loopback origin should be allowed"); + let http_client = ReqwestHttpClient::with_policy(policy); + let state = discover_provider(&config, &http_client) + .await + .expect("initial OIDC discovery should succeed"); + let sys = OidcSys { + configs: HashMap::from([(config.id.clone(), test_sourced_config(config.clone()))]), + provider_runtime: ProviderRuntime::new(http_client, HashMap::from([(config.id.clone(), state)])), + state_store: OidcStateStore::new(), + }; + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock should be after Unix epoch") + .as_secs(); + let mut header = Header::new(Algorithm::ES256); + header.kid = Some("rotated".to_string()); + let token = jsonwebtoken::encode( + &header, + &serde_json::json!({ + "iss": base, + "sub": "rotated-user", + "aud": config.client_id, + "iat": now, + "exp": now + 300, + "groups": ["readwrite"], + }), + &rotated_key, + ) + .expect("rotated OIDC token should sign"); + + let (claims, provider_id) = sys + .verify_web_identity_token(&token) + .await + .expect("verification should refresh JWKS and accept the rotated key"); + assert_eq!(provider_id, "rotating"); + assert_eq!(claims.sub, "rotated-user"); + assert_eq!(claims.groups, vec!["readwrite"]); + handle.join().expect("rotating JWKS mock server should exit cleanly"); +} +#[tokio::test] +async fn complete_provider_console_login_preserves_hidden_and_issuer_modes() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + for hidden in [false, true] { + for explicit in [false, true] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let issuer = if explicit { + "https://issuer.example.com".to_string() + } else { + base.clone() + }; + let mut config = build_mocked_oidc_provider_config("console", &format!("{base}/.well-known/openid-configuration")); + config.hide_from_ui = hidden; + config.issuer = explicit.then(|| issuer.clone()); + config.client_secret = Some(Nonce::new_random().secret().clone()); + let server_config = config.clone(); + let server_base = base.clone(); + let redirect = "https://console.example.com/oauth_callback"; + let (key, jwk) = oidc_es256_key_and_jwk("console"); + let (auth_tx, auth_rx) = tokio::sync::oneshot::channel::>(); + let server = tokio::spawn(async move { + let mut auth_rx = Some(auth_rx); + for expected_path in ["/.well-known/openid-configuration", "/jwks", "/token"] { + let (mut stream, _) = tokio::time::timeout(StdDuration::from_secs(10), listener.accept()) + .await + .unwrap() + .unwrap(); + let mut bytes = Vec::new(); + let header_end = loop { + bytes.push(stream.read_u8().await.unwrap()); + assert!(bytes.len() < 8192); + if bytes.ends_with(b"\r\n\r\n") { + break bytes.len(); + } + }; + let headers = String::from_utf8(bytes).unwrap(); + let request_line = headers.lines().next().unwrap(); + assert_eq!(request_line.split_whitespace().nth(1), Some(expected_path)); + let headers_map: HashMap<_, _> = headers + .lines() + .skip(1) + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.to_ascii_lowercase(), value.trim().to_string())) + .collect(); + let body = match expected_path { + "/.well-known/openid-configuration" => { + assert!(request_line.starts_with("GET ")); + assert_eq!(headers_map["accept"], "application/json"); + serde_json::json!({ + "issuer": issuer, "authorization_endpoint": format!("{server_base}/authorize"), + "token_endpoint": format!("{server_base}/token"), "jwks_uri": format!("{server_base}/jwks"), + "response_types_supported": ["code"], "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["ES256"] + }) + } + "/jwks" => { + assert!(request_line.starts_with("GET ")); + assert!(headers_map["accept"].contains("application/json")); + serde_json::json!({"keys": [jwk]}) + } + "/token" => { + assert!(request_line.starts_with("POST ")); + assert_eq!(headers_map["accept"], "application/json"); + assert!(headers_map["content-type"].starts_with("application/x-www-form-urlencoded")); + let length: usize = headers_map["content-length"].parse().unwrap(); + assert!(header_end + length < 16384); + let mut body = vec![0; length]; + stream.read_exact(&mut body).await.unwrap(); + let form: HashMap = url::form_urlencoded::parse(&body).into_owned().collect(); + assert_eq!(form["grant_type"], "authorization_code"); + assert_eq!(form["code"], "test-authorization-code"); + assert_eq!(form["client_id"], server_config.client_id); + assert_eq!(Some(&form["client_secret"]), server_config.client_secret.as_ref()); + assert_eq!(form["redirect_uri"], redirect); + let auth = auth_rx.take().unwrap().await.unwrap(); + let challenge = PkceCodeChallenge::from_code_verifier_sha256(&PkceCodeVerifier::new( + form["code_verifier"].clone(), + )); + assert_eq!(challenge.as_str(), auth["code_challenge"]); + let now = time::OffsetDateTime::now_utc().unix_timestamp(); + let mut header = Header::new(Algorithm::ES256); + header.kid = Some("console".into()); + let token = jsonwebtoken::encode( + &header, + &serde_json::json!({ + "iss": issuer, "sub": "existing-user", "aud": server_config.client_id, + "iat": now, "exp": now + 300, "nonce": auth["nonce"], + "email": "user@example.com", "groups": ["readwrite"] + }), + &key, + ) + .unwrap(); + serde_json::json!({"access_token": "test-access-token", "token_type": "Bearer", "id_token": token}) + } + _ => unreachable!(), + } + .to_string(); + stream.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).as_bytes()).await.unwrap(); + } + }); + let http_client = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).unwrap()); + let discovered = discover_provider(&config, &http_client).await.unwrap(); + let sys = OidcSys { + configs: HashMap::from([(config.id.clone(), test_sourced_config(config))]), + provider_runtime: ProviderRuntime::new(http_client, HashMap::from([("console".into(), discovered)])), + state_store: OidcStateStore::new(), + }; + let auth_url = sys.authorize_url("console", redirect, Some("/buckets".into())).await.unwrap(); + let auth_url = Url::parse(&auth_url).unwrap(); + assert_eq!(auth_url.as_str().split('?').next(), Some(format!("{base}/authorize").as_str())); + let auth: HashMap = auth_url.query_pairs().into_owned().collect(); + assert_eq!(auth["response_type"], "code"); + assert_eq!(auth["client_id"], "rustfs-oidc-test"); + assert!(!auth["nonce"].is_empty()); + assert!(!auth["state"].is_empty()); + assert_eq!(auth["redirect_uri"], redirect); + assert_eq!(auth["code_challenge_method"], "S256"); + assert!(auth["scope"].split_whitespace().any(|scope| scope == "openid")); + let state = auth["state"].clone(); + auth_tx.send(auth).unwrap(); + let (claims, provider, session, _) = sys + .exchange_code(&state, "test-authorization-code", redirect) + .await + .unwrap_or_else(|err| panic!("hidden={hidden}, explicit={explicit}: {err}")); + assert_eq!(provider, "console"); + assert_eq!(claims.sub, "existing-user"); + assert_eq!(claims.email, "user@example.com"); + assert_eq!(claims.groups, vec!["readwrite"]); + assert_eq!(session.redirect_after.as_deref(), Some("/buckets")); + assert!(matches!(sys.exchange_code(&state, "test-authorization-code", redirect).await, + Err(error) if error == "invalid or expired OIDC state")); + server.await.unwrap(); + } + } +} +#[test] +fn test_decode_jwt_payload_invalid() { + assert!(decode_jwt_payload("not-a-jwt").is_empty()); + assert!(decode_jwt_payload("").is_empty()); +} +#[test] +fn test_core_token_response_accepts_rfc3339_updated_at() { + // Signature verification happens later; this test covers token response deserialization. + let id_token = "eyJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJodHRwczovL2F1dGguZXhhbXBsZS5jb20vb2lkYyIsInN1YiI6InVzZXItMSIsImF1ZCI6InJ1c3RmcyIsImV4cCI6MTc4NDQzMjc2OSwiaWF0IjoxNzgzMjIzMTY5LCJ1cGRhdGVkX2F0IjoiMjAyNi0wNy0wM1QwNDo1MDo1MC44MTFaIn0.c2ln"; + let body = serde_json::json!({ + "scope": "openid roles profile email", + "token_type": "Bearer", + "access_token": "access-token", + "expires_in": 1209600, + "id_token": id_token, + }) + .to_string(); + + let response: openidconnect::core::CoreTokenResponse = + serde_json::from_str(&body).expect("RFC3339 updated_at should parse in token response"); + + assert!(response.extra_fields().id_token().is_some()); +} +#[test] +fn test_map_claims_to_policies_no_provider() { + let sys = OidcSys::empty().expect("failed to initialize empty OIDC system"); + + let claims = OidcClaims { + sub: "user123".to_string(), + email: "user@example.com".to_string(), + username: "user".to_string(), + groups: vec!["admin".to_string(), "devs".to_string()], + raw: HashMap::new(), + }; + + let (policies, groups) = sys.map_claims_to_policies("nonexistent", &claims); + assert!(policies.is_empty()); + assert!(groups.is_empty()); +} +#[test] +fn test_oidc_claims_default() { + let claims = OidcClaims::default(); + assert!(claims.sub.is_empty()); + assert!(claims.email.is_empty()); + assert!(claims.username.is_empty()); + assert!(claims.groups.is_empty()); + assert!(claims.raw.is_empty()); +} +#[test] +fn test_oidc_claims_serde_roundtrip() { + let claims = OidcClaims { + sub: "user123".to_string(), + email: "user@example.com".to_string(), + username: "testuser".to_string(), + groups: vec!["admin".to_string(), "devs".to_string()], + raw: { + let mut m = HashMap::new(); + m.insert("custom".to_string(), serde_json::json!("value")); + m + }, + }; + + let json = serde_json::to_string(&claims).unwrap(); + let deserialized: OidcClaims = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized.sub, "user123"); + assert_eq!(deserialized.email, "user@example.com"); + assert_eq!(deserialized.groups.len(), 2); +} +#[test] +fn test_oidc_provider_summary_serde() { + let summary = OidcProviderSummary { + provider_id: "okta".to_string(), + display_name: "Okta SSO".to_string(), + }; + + let json = serde_json::to_string(&summary).unwrap(); + let deserialized: OidcProviderSummary = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized.provider_id, "okta"); + assert_eq!(deserialized.display_name, "Okta SSO"); +} +#[test] +fn test_oidc_sys_empty() { + let sys = OidcSys::empty().expect("failed to initialize empty OIDC system"); + assert!(!sys.has_providers()); + assert!(sys.config_snapshot().providers().is_empty()); +} +#[tokio::test] +async fn oidc_token_exchange_reports_forbidden_token_endpoint() { + let provider_id = "default"; + let token_endpoint = "http://192.168.65.254:8080/realms/rustfs/protocol/openid-connect/token"; + let metadata = serde_json::from_value::(serde_json::json!({ + "issuer": "https://idp.example.com/realms/rustfs", + "authorization_endpoint": "https://idp.example.com/realms/rustfs/protocol/openid-connect/auth", + "token_endpoint": token_endpoint, + "jwks_uri": "https://idp.example.com/realms/rustfs/protocol/openid-connect/certs", + "response_types_supported": ["code"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["RS256"] + })) + .expect("provider metadata should parse"); + let config = + build_mocked_oidc_provider_config(provider_id, "https://idp.example.com/realms/rustfs/.well-known/openid-configuration"); + let state_store = OidcStateStore::new(); + state_store + .insert( + "test-state".to_string(), + OidcAuthSession { + provider_id: provider_id.to_string(), + pkce_verifier: "test-pkce-verifier".to_string(), + nonce: "test-nonce".to_string(), + redirect_after: None, + }, + ) + .await; + let sys = OidcSys { + configs: HashMap::from([(provider_id.to_string(), test_sourced_config(config))]), + provider_runtime: ProviderRuntime::new( + ReqwestHttpClient::with_policy(OutboundPolicy::default()), + HashMap::from([( + provider_id.to_string(), + ProviderState { + metadata: DiscoveredProviderMetadata::Console(Box::new(metadata)), + discovered_at: Instant::now(), + }, + )]), + ), + state_store, + }; + + let error = match sys + .exchange_code("test-state", "test-code", "https://console.example.com/oauth_callback") + .await + { + Ok(_) => panic!("private token endpoint should require an explicit allowlist origin"), + Err(error) => error, + }; + + assert!(error.contains("request_error_kind=forbidden_outbound")); + assert!(error.contains(&format!("add http://192.168.65.254:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}"))); +} +#[test] +fn test_list_visible_providers_hides_hidden_provider() { + let visible = test_config("dex"); + let mut hidden = test_config("kubernetes"); + hidden.hide_from_ui = true; + + let sys = make_test_sys(vec![visible, hidden]); + let listed = sys.list_visible_providers(); + + assert_eq!(listed.len(), 1); + assert!(listed.iter().any(|provider| provider.provider_id == "dex")); + assert!(!listed.iter().any(|provider| provider.provider_id == "kubernetes")); +} +#[test] +fn test_hidden_provider_still_resolvable_for_sts() { + let visible = test_config("dex"); + let mut hidden = test_config("kubernetes"); + hidden.hide_from_ui = true; + + let sys = make_test_sys(vec![visible, hidden]); + + assert!(sys.get_provider_config("kubernetes").is_some()); + assert!(sys.get_provider_config("dex").is_some()); +} +#[test] +fn test_config_snapshot_includes_hidden_for_replication() { + let visible = test_config("dex"); + let mut hidden = test_config("kubernetes"); + hidden.hide_from_ui = true; + + let sys = make_test_sys(vec![visible, hidden]); + + assert_eq!(sys.config_snapshot().providers().len(), 2); +} +#[test] +fn test_list_providers_includes_hidden_for_compatibility() { + let visible = test_config("dex"); + let mut hidden = test_config("kubernetes"); + hidden.hide_from_ui = true; + + let sys = make_test_sys(vec![visible, hidden]); + + assert_eq!(sys.list_providers().len(), 2); + assert_eq!(sys.list_visible_providers().len(), 1); +} +#[test] +fn role_policy_does_not_map_groups_as_policies() { + let mut config = test_config("authentik"); + config.role_policy = "consoleAdmin".to_string(); + config.claim_name = "policy".to_string(); + + let sys = make_test_sys(vec![config]); + + let claims = OidcClaims { + groups: vec!["authentik Admins".to_string(), "users".to_string()], + raw: HashMap::from([("policy".to_string(), serde_json::json!(["readonly"]))]), + ..Default::default() + }; + + let (policies, groups) = sys.map_claims_to_policies("authentik", &claims); + assert_eq!(groups, vec!["authentik Admins", "users"]); + assert_eq!(policies, vec!["consoleAdmin"]); +} +#[test] +fn group_claim_policy_names_cover_only_prefixed_group_values() { + let mut config = test_config("ad"); + config.claim_prefix = "oidc-".to_string(); + let sys = make_test_sys(vec![config]); + let claims = OidcClaims { + groups: vec![ + "EXAMPLE\\Domain Users".to_string(), + "admins".to_string(), + "admins".to_string(), + ], + ..Default::default() + }; + + assert_eq!( + sys.group_claim_policy_names("ad", &claims), + vec!["oidc-EXAMPLE\\Domain Users", "oidc-admins"] + ); + assert!(sys.group_claim_policy_names("missing-provider", &claims).is_empty()); +} +#[test] +fn group_claim_policy_names_include_merged_roles_claim_values() { + let mut config = test_config("keycloak"); + config.roles_claim = "roles".to_string(); + let raw = HashMap::from([ + ("groups".to_string(), serde_json::json!(["readonly"])), + ("roles".to_string(), serde_json::json!(["offline_access", "default-roles-corp"])), + ]); + let claims = OidcClaims { + groups: extract_canonical_group_values(&raw, &config.groups_claim, &config.roles_claim), + raw, + ..Default::default() + }; + let sys = make_test_sys(vec![config]); + + assert_eq!( + sys.group_claim_policy_names("keycloak", &claims), + vec!["default-roles-corp", "offline_access", "readonly"] + ); +} +#[test] +fn group_claim_policy_names_exclude_role_policy_and_dedicated_policy_claim() { + let mut role_config = test_config("role"); + role_config.role_policy = "consoleAdmin".to_string(); + let mut claim_config = test_config("claim"); + claim_config.claim_name = "policy".to_string(); + let sys = make_test_sys(vec![role_config, claim_config]); + let claims = OidcClaims { + groups: vec!["readonly".to_string(), "unmapped".to_string()], + raw: HashMap::from([("policy".to_string(), serde_json::json!(["readonly", "writeonly"]))]), + ..Default::default() + }; + + assert!(sys.group_claim_policy_names("role", &claims).is_empty()); + // `readonly` is also requested explicitly through the policy claim, so it must resolve. + assert_eq!(sys.group_claim_policy_names("claim", &claims), vec!["unmapped"]); +} +#[test] +fn test_map_claims_to_policies_with_prefix() { + let mut config = test_config("azure"); + config.claim_prefix = "oidc-".to_string(); + config.display_name = "Azure AD".to_string(); + + let sys = make_test_sys(vec![config]); + + let claims = OidcClaims { + sub: "user456".to_string(), + email: "user@corp.com".to_string(), + username: "user".to_string(), + groups: vec!["engineers".to_string()], + raw: HashMap::new(), + }; + + let (policies, groups) = sys.map_claims_to_policies("azure", &claims); + assert_eq!(groups, vec!["engineers"]); + assert!(policies.contains(&"oidc-engineers".to_string())); + assert_eq!(policies.len(), 1); +} +#[test] +fn blank_role_policy_uses_claim_mapping() { + let mut config = test_config("keycloak"); + config.role_policy = " ".to_string(); + + let sys = make_test_sys(vec![config]); + let claims = OidcClaims { + groups: vec!["readonly".to_string()], + ..Default::default() + }; + + let (policies, groups) = sys.map_claims_to_policies("keycloak", &claims); + assert_eq!(groups, vec!["readonly"]); + assert_eq!(policies, vec!["readonly"]); +} +#[test] +fn claim_mapping_keeps_groups_with_distinct_primary_claim() { + let mut config = test_config("keycloak"); + config.claim_name = "policy".to_string(); + + let sys = make_test_sys(vec![config]); + let claims = OidcClaims { + groups: vec!["developers".to_string()], + raw: HashMap::from([("policy".to_string(), serde_json::json!(["readonly"]))]), + ..Default::default() + }; + + let (policies, groups) = sys.map_claims_to_policies("keycloak", &claims); + assert_eq!(groups, vec!["developers"]); + assert_eq!(policies, vec!["developers", "readonly"]); +} +#[test] +fn test_config_snapshot_lists_providers() { + let mut config = test_config("keycloak"); + config.display_name = "Keycloak SSO".to_string(); + + let sys = make_test_sys(vec![config]); + + assert!(sys.has_providers()); + let snapshot = sys.config_snapshot(); + assert_eq!(snapshot.providers().len(), 1); + assert_eq!(snapshot.providers()[0].config.id, "keycloak"); + assert_eq!(snapshot.providers()[0].config.display_name, "Keycloak SSO"); + assert_eq!(snapshot.providers()[0].source, OidcProviderConfigSource::Persisted); +} +#[test] +fn test_get_provider_config() { + let mut config = test_config("test"); + config.client_id = "my-client".to_string(); + config.client_secret = Some("secret".to_string()); + + let sys = make_test_sys(vec![config]); + + assert!(sys.get_provider_config("test").is_some()); + assert_eq!(sys.get_provider_config("test").unwrap().client_id, "my-client"); + assert!(sys.get_provider_config("nonexistent").is_none()); +} +#[test] +fn verified_identity_preserves_provider_and_claim_values() { + let raw = HashMap::from([ + ("iss".to_string(), serde_json::json!("https://corp.example.test")), + ("department".to_string(), serde_json::json!("engineering")), + ("roles".to_string(), serde_json::json!(["reader", "admin"])), + ]); + let identity = verified_identity( + FederatedProviderRef::new("corp".to_string()), + OidcClaims { + sub: " subject-123 ".to_string(), + email: " user@example.test ".to_string(), + username: " user ".to_string(), + groups: vec![ + "source-ops".to_string(), + "source-developers".to_string(), + "source-ops".to_string(), + ], + raw: raw.clone(), + }, + ); + + assert_eq!(identity.provider().as_str(), "corp"); + assert_eq!(identity.subject(), " subject-123 "); + assert_eq!(identity.email(), " user@example.test "); + assert_eq!(identity.username(), " user "); + assert_eq!(identity.source_groups(), ["source-ops", "source-developers", "source-ops"]); + assert_eq!(identity.attributes(), &raw); +} +#[test] +fn adapter_preserves_provider_views_and_redirect_policy() { + let mut visible = test_config("default"); + visible.display_name = "Company SSO".to_string(); + visible.redirect_uri = Some("https://console.example.test/callback".to_string()); + visible.redirect_uri_dynamic = false; + let mut hidden = test_config("workload"); + hidden.display_name = "Workload Identity".to_string(); + hidden.hide_from_ui = true; + let adapter = StandardOidcAdapter::new(Arc::new(make_test_sys(vec![visible, hidden]))); + + let mut providers = adapter.list_providers(); + providers.sort_by(|left, right| left.provider_id.cmp(&right.provider_id)); + assert_eq!( + providers, + vec![ + FederatedProviderView { + provider_id: "default".to_string(), + display_name: "Company SSO".to_string(), + }, + FederatedProviderView { + provider_id: "workload".to_string(), + display_name: "Workload Identity".to_string(), + }, + ] + ); + assert_eq!( + adapter.list_visible_providers(), + vec![FederatedProviderView { + provider_id: "default".to_string(), + display_name: "Company SSO".to_string(), + }] + ); + assert_eq!( + adapter.redirect_policy("default"), + Some(FederatedRedirectPolicy { + redirect_uri: Some("https://console.example.test/callback".to_string()), + allow_request_origin: false, + }) + ); + assert_eq!(adapter.redirect_policy("missing"), None); +} diff --git a/crates/iam/src/oidc_state.rs b/crates/iam/src/oidc/state.rs similarity index 92% rename from crates/iam/src/oidc_state.rs rename to crates/iam/src/oidc/state.rs index 5cdd962ee..fa9b90f3e 100644 --- a/crates/iam/src/oidc_state.rs +++ b/crates/iam/src/oidc/state.rs @@ -25,32 +25,32 @@ const OIDC_STATE_CAPACITY_LOG_INTERVAL_SECS: u64 = 60; /// Stores the PKCE verifier and nonce for an in-flight OIDC authorization flow. #[derive(Debug, Clone)] -pub struct OidcAuthSession { - pub provider_id: String, - pub pkce_verifier: String, - pub nonce: String, - pub redirect_after: Option, +pub(super) struct OidcAuthSession { + pub(super) provider_id: String, + pub(super) pkce_verifier: String, + pub(super) nonce: String, + pub(super) redirect_after: Option, } /// Stores an ID token behind a one-time opaque handle so the console can trigger /// RP-initiated logout without persisting the raw token in browser storage. #[derive(Debug, Clone)] -pub struct OidcLogoutSession { - pub provider_id: String, - pub id_token: String, +pub(super) struct OidcLogoutSession { + pub(super) provider_id: String, + pub(super) id_token: String, } /// TTL cache for OIDC auth state (PKCE verifiers + nonces) during the authorization flow. /// Entries expire after 5 minutes and are single-use (removed on retrieval). #[derive(Clone)] -pub struct OidcStateStore { +pub(super) struct OidcStateStore { cache: Cache, logout_cache: Cache, last_capacity_log_at: Arc, } impl OidcStateStore { - pub fn new() -> Self { + pub(super) fn new() -> Self { let cache = Cache::builder() .max_capacity(OIDC_STATE_CAPACITY) .time_to_live(Duration::from_secs(300)) // 5 minute TTL @@ -67,7 +67,7 @@ impl OidcStateStore { } /// Store a new auth session keyed by the OAuth2 `state` parameter. - pub async fn insert(&self, state: String, session: OidcAuthSession) { + pub(super) async fn insert(&self, state: String, session: OidcAuthSession) { self.cache.insert(state, session).await; let size = self.cache.entry_count(); @@ -104,27 +104,29 @@ impl OidcStateStore { /// Retrieve and remove an auth session (single-use). Returns None if expired or not found. /// Uses `remove` which returns the value and removes it in a single operation. - pub async fn take(&self, state: &str) -> Option { + pub(super) async fn take(&self, state: &str) -> Option { self.cache.remove(state).await } /// Check if a state key exists (without consuming it). - pub async fn contains(&self, state: &str) -> bool { + #[cfg(test)] + pub(super) async fn contains(&self, state: &str) -> bool { self.cache.get(state).await.is_some() } /// Store a new one-time logout session keyed by an opaque logout token. - pub async fn insert_logout(&self, token: String, session: OidcLogoutSession) { + pub(super) async fn insert_logout(&self, token: String, session: OidcLogoutSession) { self.logout_cache.insert(token, session).await; } /// Retrieve and remove a logout session (single-use). Returns None if expired or not found. - pub async fn take_logout(&self, token: &str) -> Option { + pub(super) async fn take_logout(&self, token: &str) -> Option { self.logout_cache.remove(token).await } /// Check if a logout token exists (without consuming it). - pub async fn contains_logout(&self, token: &str) -> bool { + #[cfg(test)] + pub(super) async fn contains_logout(&self, token: &str) -> bool { self.logout_cache.get(token).await.is_some() } } diff --git a/crates/iam/src/oidc/test_support.rs b/crates/iam/src/oidc/test_support.rs new file mode 100644 index 000000000..e81855537 --- /dev/null +++ b/crates/iam/src/oidc/test_support.rs @@ -0,0 +1,526 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use super::{ + config::{OidcProviderConfig, OidcProviderConfigSource, SourcedOidcProviderConfig}, + provider::ProviderRuntime, + runtime::OidcSys, + state::OidcStateStore, + transport::ReqwestHttpClient, +}; +use std::collections::HashMap; + +#[cfg(any(test, feature = "test-util"))] +#[doc(hidden)] +pub fn make_test_sys(configs: Vec) -> OidcSys { + let configs = configs + .into_iter() + .map(test_sourced_config) + .map(|provider| (provider.config.id.clone(), provider)) + .collect(); + OidcSys { + configs, + provider_runtime: ProviderRuntime::new( + ReqwestHttpClient::new().expect("failed to initialize OIDC HTTP clients"), + HashMap::new(), + ), + state_store: OidcStateStore::new(), + } +} + +#[cfg(any(test, feature = "test-util"))] +pub(super) fn test_sourced_config(config: OidcProviderConfig) -> SourcedOidcProviderConfig { + SourcedOidcProviderConfig { + config, + source: OidcProviderConfigSource::Persisted, + } +} + +#[cfg(any(test, feature = "test-util"))] +#[doc(hidden)] +pub fn test_config(id: &str) -> OidcProviderConfig { + OidcProviderConfig { + id: id.to_string(), + enabled: true, + config_url: format!("https://example.com/{id}/.well-known/openid-configuration"), + issuer: None, + client_id: "client-id".to_string(), + client_secret: None, + scopes: vec!["openid".to_string()], + other_audiences: vec![], + redirect_uri: None, + redirect_uri_dynamic: true, + claim_name: "groups".to_string(), + claim_prefix: String::new(), + role_policy: String::new(), + display_name: id.to_string(), + groups_claim: "groups".to_string(), + roles_claim: String::new(), + email_claim: "email".to_string(), + username_claim: "preferred_username".to_string(), + hide_from_ui: false, + } +} + +#[cfg(test)] +use super::{config::OidcProviderValidationResult, provider::discover_provider, transport::OidcHttpError}; +#[cfg(test)] +use jsonwebtoken::{Algorithm, EncodingKey}; +#[cfg(test)] +use openidconnect::AsyncHttpClient; +#[cfg(test)] +use rustfs_utils::egress::{OutboundDnsPolicyRejection, OutboundPolicy}; +#[cfg(test)] +use std::sync::Arc; +#[cfg(test)] +use std::sync::atomic::{AtomicUsize, Ordering}; +#[cfg(test)] +use url::Url; + +#[cfg(test)] +#[derive(Clone)] +pub(super) struct RejectingDnsResolver { + pub(super) allow_origin_can_recover: bool, + pub(super) calls: Option>, +} + +#[cfg(test)] +impl reqwest::dns::Resolve for RejectingDnsResolver { + fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { + if let Some(calls) = &self.calls { + calls.fetch_add(1, Ordering::Relaxed); + } + let host = name.as_str().to_string(); + let rejection = OutboundDnsPolicyRejection::new(host, self.allow_origin_can_recover); + Box::pin(async move { Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, rejection).into()) }) + } +} + +#[cfg(test)] +pub(super) fn build_mocked_oidc_provider_config(id: &str, config_url: &str) -> OidcProviderConfig { + OidcProviderConfig { + id: id.to_string(), + enabled: true, + config_url: config_url.to_string(), + issuer: None, + client_id: "rustfs-oidc-test".to_string(), + client_secret: None, + scopes: vec!["openid".to_string()], + other_audiences: vec![], + redirect_uri: None, + redirect_uri_dynamic: false, + claim_name: "sub".to_string(), + claim_prefix: "oidc".to_string(), + role_policy: String::new(), + display_name: "mock-oidc".to_string(), + groups_claim: "groups".to_string(), + roles_claim: String::new(), + email_claim: "email".to_string(), + username_claim: "username".to_string(), + hide_from_ui: false, + } +} +#[cfg(test)] +pub(super) fn read_mock_oidc_request(stream: &mut impl std::io::Read) -> String { + let mut request_bytes = Vec::new(); + let mut buffer = [0u8; 4096]; + loop { + match stream.read(&mut buffer) { + Ok(0) => break, + Ok(n) => request_bytes.extend_from_slice(&buffer[..n]), + Err(e) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => break, + Err(_) => break, + } + if request_bytes.windows(4).any(|w| w == b"\r\n\r\n") { + break; + } + if request_bytes.len() >= 8192 { + break; + } + } + String::from_utf8_lossy(&request_bytes).into_owned() +} +#[cfg(test)] +pub(super) fn read_mock_oidc_request_path(stream: &mut impl std::io::Read) -> String { + read_mock_oidc_request(stream) + .lines() + .next() + .unwrap_or("") + .split_whitespace() + .nth(1) + .unwrap_or("") + .to_string() +} +#[cfg(test)] +pub(super) fn mock_oidc_response(path: &str, discovery_body: &str, expected_jwks_path: &str, jwks_body: &str) -> String { + let (status, body) = if path.ends_with("/.well-known/openid-configuration") { + (200, discovery_body) + } else if path == expected_jwks_path { + (200, jwks_body) + } else { + (404, r#"{"error":"not found"}"#) + }; + + format!( + "HTTP/1.1 {status} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + if status == 200 { "OK" } else { "Not Found" }, + body.len() + ) +} +#[cfg(test)] +pub(super) fn start_mock_oidc_discovery_server_with_jwks( + build_discovery_issuer: F, + max_requests: usize, + signing_alg: &'static str, + workload: bool, + jwks_response: J, +) -> Option<(String, std::thread::JoinHandle<()>)> +where + F: Fn(&str) -> (String, String, String) + Send + 'static, + J: Fn(usize) -> String + Send + 'static, +{ + use std::io::Write; + use std::net::{Shutdown, TcpListener}; + use std::sync::mpsc; + use std::time::{Duration, Instant}; + + // After the last completed response, exit if no new connection arrives within this window. + // Keep the mock server alive long enough for slower CI/macOS test environments to finish + // discovery + JWKS requests without racing the shutdown timer. + const IDLE_SHUTDOWN: Duration = Duration::from_secs(1); + const ABSOLUTE_CAP: Duration = Duration::from_secs(5); + + let listener = match TcpListener::bind("127.0.0.1:0") { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, + Err(err) => panic!("test listener should bind: {err}"), + }; + let base = format!("http://{}", listener.local_addr().expect("listener local address should be available")); + let (discovery_issuer, discovery_jwks_uri, expected_jwks_path) = build_discovery_issuer(&base); + let mut discovery_document = serde_json::json!({ + "issuer": discovery_issuer, + "authorization_endpoint": format!("{base}/authorize"), + "token_endpoint": format!("{base}/token"), + "jwks_uri": discovery_jwks_uri, + "response_types_supported": ["code"], + "response_modes_supported": ["query"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": [signing_alg], + }); + if workload { + let fields = discovery_document.as_object_mut().expect("mock metadata is an object"); + fields.remove("authorization_endpoint"); + fields.remove("token_endpoint"); + fields.insert("response_types_supported".into(), serde_json::json!(["id_token"])); + } + let discovery_body = discovery_document.to_string(); + let (ready_tx, ready_rx) = mpsc::channel(); + + let handle = std::thread::spawn(move || { + listener + .set_nonblocking(true) + .expect("failed to set discovery mock listener non-blocking"); + let _ = ready_tx.send(()); + + let mut seen = 0usize; + let mut jwks_fetches = 0usize; + let start = Instant::now(); + let mut last_completed = Instant::now(); + + loop { + if seen > 0 && last_completed.elapsed() >= IDLE_SHUTDOWN { + break; + } + if start.elapsed() >= ABSOLUTE_CAP { + break; + } + + let mut stream = match listener.accept() { + Ok((stream, _)) => stream, + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(5)); + continue; + } + Err(_) => break, + }; + stream + .set_nonblocking(false) + .expect("failed to set discovery mock stream blocking"); + + seen += 1; + stream + .set_nonblocking(false) + .expect("failed to set discovery mock stream blocking"); + stream + .set_read_timeout(Some(Duration::from_secs(1))) + .expect("failed to set discovery mock read timeout"); + + let request = read_mock_oidc_request(&mut stream); + let path = request + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .unwrap_or(""); + let jwks_body = jwks_response(jwks_fetches); + if path == expected_jwks_path { + jwks_fetches += 1; + } + let mut response = mock_oidc_response(path, &discovery_body, &expected_jwks_path, &jwks_body); + if path.contains("/.well-known/openid-configuration") { + assert!( + request + .lines() + .filter_map(|line| line.split_once(':')) + .any(|(name, value)| { name.eq_ignore_ascii_case("accept") && value.trim() == "application/json" }), + "discovery Accept must remain unchanged" + ); + } + if path == expected_jwks_path { + let expected_type = if workload { + "application/jwk-set+json" + } else { + "application/json" + }; + let accepts_type = request.lines().filter_map(|line| line.split_once(':')).any(|(name, value)| { + name.eq_ignore_ascii_case("accept") && value.split(',').any(|item| item.trim() == expected_type) + }); + if !accepts_type { + response = "HTTP/1.1 406 Not Acceptable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".into(); + } else if workload { + response = response.replace("Content-Type: application/json", "Content-Type: application/jwk-set+json"); + } + } + let _ = stream.write_all(response.as_bytes()); + let _ = stream.flush(); + let _ = stream.shutdown(Shutdown::Both); + last_completed = Instant::now(); + + if seen >= max_requests { + break; + } + } + }); + ready_rx + .recv_timeout(Duration::from_millis(100)) + .expect("mock OIDC discovery server should become ready"); + + Some((base, handle)) +} +#[cfg(test)] +pub(super) fn start_mock_oidc_discovery_server( + build_discovery_issuer: F, + max_requests: usize, +) -> Option<(String, std::thread::JoinHandle<()>)> +where + F: Fn(&str) -> (String, String, String) + Send + 'static, +{ + start_mock_oidc_discovery_server_with_jwks(build_discovery_issuer, max_requests, "RS256", false, |_| { + r#"{"keys":[]}"#.to_string() + }) +} +#[cfg(test)] +pub(super) fn oidc_es256_key_and_jwk(kid: &str) -> (EncodingKey, serde_json::Value) { + let certified = rcgen::generate_simple_self_signed(vec![format!("{kid}.invalid")]).expect("OIDC signing key should generate"); + let encoding_key = + EncodingKey::from_ec_pem(certified.signing_key.serialize_pem().as_bytes()).expect("OIDC signing key should encode"); + let mut jwk = jsonwebtoken::jwk::Jwk::from_encoding_key(&encoding_key, Algorithm::ES256) + .expect("OIDC public JWK should derive from signing key"); + jwk.common.key_id = Some(kid.to_string()); + jwk.common.public_key_use = Some(jsonwebtoken::jwk::PublicKeyUse::Signature); + (encoding_key, serde_json::to_value(jwk).expect("OIDC JWK should serialize")) +} +#[cfg(test)] +pub(super) fn start_mock_oidc_tls_discovery_server( + build_discovery_issuer: F, + max_requests: usize, +) -> Option<(String, String, std::thread::JoinHandle<()>)> +where + F: Fn(&str) -> (String, String, String) + Send + 'static, +{ + use std::io::Write; + use std::net::{Shutdown, TcpListener}; + use std::sync::mpsc; + use std::time::{Duration, Instant}; + + const IDLE_SHUTDOWN: Duration = Duration::from_secs(1); + const ABSOLUTE_CAP: Duration = Duration::from_secs(5); + + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let certified = + rcgen::generate_simple_self_signed(vec!["127.0.0.1".to_string()]).expect("generate OIDC TLS test certificate"); + let cert_pem = certified.cert.pem(); + let server_config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert( + vec![certified.cert.der().clone()], + rustls_pki_types::PrivateKeyDer::try_from(certified.signing_key.serialize_der()) + .expect("convert OIDC TLS test private key"), + ) + .expect("build OIDC TLS mock server config"); + + let listener = match TcpListener::bind("127.0.0.1:0") { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, + Err(err) => panic!("test TLS listener should bind: {err}"), + }; + let base = format!("https://{}", listener.local_addr().expect("listener local address should be available")); + let (discovery_issuer, discovery_jwks_uri, expected_jwks_path) = build_discovery_issuer(&base); + let discovery_body = serde_json::json!({ + "issuer": discovery_issuer, + "authorization_endpoint": format!("{base}/authorize"), + "token_endpoint": format!("{base}/token"), + "jwks_uri": discovery_jwks_uri, + "response_types_supported": ["code"], + "response_modes_supported": ["query"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["RS256"], + }) + .to_string(); + let jwks_body = r#"{"keys":[]}"#; + let (ready_tx, ready_rx) = mpsc::channel(); + + let handle = std::thread::spawn(move || { + let server_config = Arc::new(server_config); + listener + .set_nonblocking(true) + .expect("failed to set TLS discovery mock listener non-blocking"); + let _ = ready_tx.send(()); + + let mut seen = 0usize; + let start = Instant::now(); + let mut last_completed = Instant::now(); + + loop { + if seen > 0 && last_completed.elapsed() >= IDLE_SHUTDOWN { + break; + } + if start.elapsed() >= ABSOLUTE_CAP { + break; + } + + let tcp_stream = match listener.accept() { + Ok((stream, _)) => stream, + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(5)); + continue; + } + Err(_) => break, + }; + tcp_stream + .set_nonblocking(false) + .expect("failed to set TLS discovery mock stream blocking"); + tcp_stream + .set_read_timeout(Some(Duration::from_secs(1))) + .expect("failed to set TLS discovery mock read timeout"); + + seen += 1; + let connection = match rustls::ServerConnection::new(server_config.clone()) { + Ok(connection) => connection, + Err(_) => break, + }; + let mut stream = rustls::StreamOwned::new(connection, tcp_stream); + let path = read_mock_oidc_request_path(&mut stream); + let response = mock_oidc_response(&path, &discovery_body, &expected_jwks_path, jwks_body); + let _ = stream.write_all(response.as_bytes()); + let _ = stream.flush(); + let _ = stream.sock.shutdown(Shutdown::Both); + last_completed = Instant::now(); + + if seen >= max_requests { + break; + } + } + }); + ready_rx + .recv_timeout(Duration::from_millis(100)) + .expect("mock TLS OIDC discovery server should become ready"); + + Some((base, cert_pem, handle)) +} +#[cfg(test)] +pub(super) fn discovery_error_contains_all_variants(err: &str, base: &str) -> bool { + err.contains(base) && err.contains(&format!("{base}/")) && err.contains("discovery failed for all issuer variants") +} +#[cfg(test)] +pub(super) async fn validate_mocked_oidc_provider_config( + config: &OidcProviderConfig, +) -> Result { + // The mock discovery/JWKS/token endpoints share the loopback origin of `config_url`. + // Explicitly allow that origin so the egress policy does not reject the loopback mock. + let origin = Url::parse(&config.config_url) + .map_err(|_| "invalid mock config_url".to_string())? + .origin() + .ascii_serialization(); + let policy = OutboundPolicy::from_allowed_origins(&origin).map_err(|err| err.to_string())?; + let http_client = ReqwestHttpClient::with_policy(policy); + let state = discover_provider(config, &http_client).await?; + + Ok(OidcProviderValidationResult { + issuer: state.metadata.issuer().to_string(), + authorization_endpoint: state.metadata.authorization_endpoint(), + token_endpoint: state.metadata.token_endpoint().map(ToString::to_string), + }) +} +#[cfg(test)] +pub(super) fn start_unbounded_body_server(body_len: usize) -> Option<(String, std::thread::JoinHandle<()>)> { + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::sync::mpsc; + use std::time::Duration; + + let listener = match TcpListener::bind("127.0.0.1:0") { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, + Err(err) => panic!("test listener should bind: {err}"), + }; + let base = format!("http://{}", listener.local_addr().expect("listener local address should be available")); + let (ready_tx, ready_rx) = mpsc::channel(); + + let handle = std::thread::spawn(move || { + let _ = ready_tx.send(()); + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let _ = stream.set_read_timeout(Some(Duration::from_secs(1))); + let mut buffer = [0u8; 4096]; + let _ = stream.read(&mut buffer); + let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n"); + + let chunk = vec![b'a'; 64 * 1024]; + let mut written = 0usize; + while written < body_len { + let take = chunk.len().min(body_len - written); + if stream.write_all(&chunk[..take]).is_err() { + break; + } + written += take; + } + let _ = stream.flush(); + }); + ready_rx + .recv_timeout(Duration::from_millis(100)) + .expect("mock body server should become ready"); + + Some((base, handle)) +} +#[cfg(test)] +pub(super) async fn fetch_oidc_mock_body(base: &str) -> Result, OidcHttpError> { + let policy = OutboundPolicy::from_allowed_origins(base).expect("origin should parse"); + let client = ReqwestHttpClient::with_policy(policy); + let request = http::Request::builder() + .method(http::Method::GET) + .uri(base) + .body(Vec::new()) + .expect("request should build"); + client.call(request).await.map(http::Response::into_body) +} diff --git a/crates/iam/src/oidc/transport.rs b/crates/iam/src/oidc/transport.rs new file mode 100644 index 000000000..1fd8706f0 --- /dev/null +++ b/crates/iam/src/oidc/transport.rs @@ -0,0 +1,660 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use crate::{LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC}; +use openidconnect::AsyncHttpClient; +use reqwest::{Certificate, Client}; +use rustfs_config::MAX_OIDC_RESPONSE_SIZE; +use rustfs_utils::egress::{ENV_OUTBOUND_ALLOW_ORIGINS, OutboundPolicy, find_outbound_dns_policy_rejection}; +use std::collections::VecDeque; +use std::future::Future; +use std::net::IpAddr; +use std::pin::Pin; +use std::sync::{Arc, LazyLock, Mutex, MutexGuard, RwLock}; +use std::time::{Duration as StdDuration, Instant}; +use tracing::{debug, error, warn}; +use url::Url; + +pub(super) const EVENT_OIDC_HTTP: &str = "oidc_http"; +const OIDC_HTTP_REQUEST_TIMEOUT: StdDuration = StdDuration::from_secs(10); +const OIDC_HTTP_CONNECT_TIMEOUT: StdDuration = StdDuration::from_secs(3); +const OIDC_PLUGIN_AUTHN_WINDOW: StdDuration = StdDuration::from_secs(60); + +#[derive(Debug, Clone, Copy, Default)] +pub struct OidcPluginAuthnMetricsSnapshot { + pub failed_requests_minute: u64, + pub last_fail_seconds: u64, + pub last_succ_seconds: u64, + pub succ_avg_rtt_ms_minute: u64, + pub succ_max_rtt_ms_minute: u64, + pub total_requests_minute: u64, +} + +#[derive(Debug, Clone)] +struct OidcPluginAuthnSample { + observed_at: Instant, + succeeded: bool, + rtt_ms: u64, +} + +#[derive(Debug, Default)] +struct OidcPluginAuthnMetrics { + samples: Mutex>, + last_fail_at: Mutex>, + last_succ_at: Mutex>, +} + +fn lock_oidc_plugin_authn_metrics<'a, T>(mutex: &'a Mutex, metric: &'static str) -> MutexGuard<'a, T> { + match mutex.lock() { + Ok(guard) => guard, + Err(err) => { + warn!(metric, "Recovering poisoned OIDC authn metrics lock"); + err.into_inner() + } + } +} + +fn seconds_since(now: Instant, observed_at: Option) -> u64 { + observed_at + .map(|instant| now.duration_since(instant).as_secs()) + .unwrap_or_default() +} + +impl OidcPluginAuthnMetrics { + fn record(&self, rtt_ms: u64, succeeded: bool) { + let now = Instant::now(); + let mut samples = lock_oidc_plugin_authn_metrics(&self.samples, "samples"); + samples.push_back(OidcPluginAuthnSample { + observed_at: now, + succeeded, + rtt_ms, + }); + while samples + .front() + .is_some_and(|sample| now.duration_since(sample.observed_at) > OIDC_PLUGIN_AUTHN_WINDOW) + { + samples.pop_front(); + } + drop(samples); + + if succeeded { + *lock_oidc_plugin_authn_metrics(&self.last_succ_at, "last_succ_at") = Some(now); + } else { + *lock_oidc_plugin_authn_metrics(&self.last_fail_at, "last_fail_at") = Some(now); + } + } + + fn snapshot(&self) -> OidcPluginAuthnMetricsSnapshot { + let now = Instant::now(); + let (total_requests_minute, failed_requests_minute, succ_avg_rtt_ms_minute, succ_max_rtt_ms_minute) = { + let mut samples = lock_oidc_plugin_authn_metrics(&self.samples, "samples"); + while samples + .front() + .is_some_and(|sample| now.duration_since(sample.observed_at) > OIDC_PLUGIN_AUTHN_WINDOW) + { + samples.pop_front(); + } + + let mut failed_requests_minute = 0u64; + let mut successful_requests = 0u64; + let mut successful_rtt_sum = 0u64; + let mut succ_max_rtt_ms_minute = 0u64; + + for sample in samples.iter() { + if sample.succeeded { + successful_requests += 1; + successful_rtt_sum += sample.rtt_ms; + succ_max_rtt_ms_minute = succ_max_rtt_ms_minute.max(sample.rtt_ms); + } else { + failed_requests_minute += 1; + } + } + + let succ_avg_rtt_ms_minute = successful_rtt_sum.checked_div(successful_requests).unwrap_or_default(); + + ( + samples.len() as u64, + failed_requests_minute, + succ_avg_rtt_ms_minute, + succ_max_rtt_ms_minute, + ) + }; + + let last_fail_seconds = seconds_since(now, *lock_oidc_plugin_authn_metrics(&self.last_fail_at, "last_fail_at")); + let last_succ_seconds = seconds_since(now, *lock_oidc_plugin_authn_metrics(&self.last_succ_at, "last_succ_at")); + + OidcPluginAuthnMetricsSnapshot { + failed_requests_minute, + last_fail_seconds, + last_succ_seconds, + succ_avg_rtt_ms_minute, + succ_max_rtt_ms_minute, + total_requests_minute, + } + } +} + +static OIDC_PLUGIN_AUTHN_METRICS: LazyLock = LazyLock::new(OidcPluginAuthnMetrics::default); + +pub fn oidc_plugin_authn_metrics_snapshot() -> OidcPluginAuthnMetricsSnapshot { + OIDC_PLUGIN_AUTHN_METRICS.snapshot() +} + +/// Header names whose values may carry OIDC secrets (client credentials, cookies, +/// bearer tokens). Their values are never emitted to logs, only their byte length. +const SENSITIVE_HEADER_NAMES: [&str; 4] = ["authorization", "proxy-authorization", "cookie", "set-cookie"]; + +pub(super) fn is_sensitive_header(name: &str) -> bool { + SENSITIVE_HEADER_NAMES + .iter() + .any(|candidate| name.eq_ignore_ascii_case(candidate)) +} + +pub(super) fn format_http_headers(headers: &http::HeaderMap) -> String { + headers + .iter() + .map(|(name, value)| { + if is_sensitive_header(name.as_str()) { + format!("{}=", name.as_str(), value.as_bytes().len()) + } else { + let value = value.to_str().unwrap_or(""); + format!("{}={}", name.as_str(), value) + } + }) + .collect::>() + .join("; ") +} + +#[derive(Debug, Default)] +pub(super) struct TokenResponseBodyShape { + pub(super) json_object: bool, + pub(super) json_keys: String, + pub(super) has_access_token: bool, + pub(super) has_id_token: bool, + pub(super) has_token_type: bool, + pub(super) has_expires_in: bool, + pub(super) has_error: bool, + pub(super) has_error_description: bool, + pub(super) looks_like_html: bool, +} + +pub(super) fn inspect_token_response_body(body: &[u8]) -> TokenResponseBodyShape { + let mut shape = TokenResponseBodyShape { + looks_like_html: body + .iter() + .copied() + .find(|byte| !byte.is_ascii_whitespace()) + .is_some_and(|byte| byte == b'<'), + ..Default::default() + }; + + let Ok(value) = serde_json::from_slice::(body) else { + return shape; + }; + let Some(object) = value.as_object() else { + return shape; + }; + + shape.json_object = true; + shape.has_access_token = object.contains_key("access_token"); + shape.has_id_token = object.contains_key("id_token"); + shape.has_token_type = object.contains_key("token_type"); + shape.has_expires_in = object.contains_key("expires_in"); + shape.has_error = object.contains_key("error"); + shape.has_error_description = object.contains_key("error_description"); + + let mut keys: Vec<&str> = object.keys().map(String::as_str).collect(); + keys.sort_unstable(); + keys.truncate(16); + shape.json_keys = keys.join(","); + + shape +} + +pub(super) fn oidc_http_error_diagnostics(error: &OidcHttpError) -> (&'static str, String) { + match error { + OidcHttpError::Reqwest(err) if err.is_timeout() => ("timeout", String::new()), + OidcHttpError::Reqwest(err) if err.is_connect() => ("connect", String::new()), + OidcHttpError::Reqwest(err) if err.status().is_some() => { + ("http_status", err.status().map(|status| status.as_u16().to_string()).unwrap_or_default()) + } + OidcHttpError::Reqwest(_) => ("request", String::new()), + OidcHttpError::Http(_) => ("http_build", String::new()), + OidcHttpError::ExtraRootCa(_) => ("extra_root_ca", String::new()), + OidcHttpError::ForbiddenOutbound(_) => ("forbidden_outbound", String::new()), + OidcHttpError::ResponseTooLarge(limit) => ("response_too_large", limit.to_string()), + } +} + +// ---- HTTP Client Adapter ---- + +/// Error type for the OIDC HTTP client adapter. +#[derive(Debug)] +pub(super) enum OidcHttpError { + Reqwest(reqwest::Error), + Http(http::Error), + ExtraRootCa(String), + /// The outbound destination was rejected by the shared egress policy before any + /// connection was attempted (invalid URL, loopback/link-local/metadata/private IP, + /// or a malformed allow-origins configuration). + ForbiddenOutbound(String), + /// The provider response body exceeded [`MAX_OIDC_RESPONSE_SIZE`] and was abandoned + /// instead of being buffered in full. + ResponseTooLarge(usize), +} + +impl std::fmt::Display for OidcHttpError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Reqwest(e) => write!(f, "{e}"), + Self::Http(e) => write!(f, "{e}"), + Self::ExtraRootCa(reason) => write!(f, "failed to load OIDC extra root CA bundle: {reason}"), + Self::ForbiddenOutbound(reason) => write!(f, "outbound request rejected: {reason}"), + Self::ResponseTooLarge(limit) => write!(f, "oidc response body exceeds {limit} bytes"), + } + } +} + +impl std::error::Error for OidcHttpError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Reqwest(e) => Some(e), + Self::Http(e) => Some(e), + Self::ExtraRootCa(_) | Self::ForbiddenOutbound(_) | Self::ResponseTooLarge(_) => None, + } + } +} + +#[derive(Clone, Debug, Default)] +pub struct OidcExtraRootCaMaterial { + pub generation: u64, + pub root_ca_pem: Option>, +} + +type OidcExtraRootCaFuture = Pin> + Send>>; +type OidcExtraRootCaLoader = dyn Fn() -> OidcExtraRootCaFuture + Send + Sync; + +#[derive(Clone)] +pub struct OidcExtraRootCaProvider { + loader: Arc, +} + +impl OidcExtraRootCaProvider { + pub fn new(loader: F) -> Self + where + F: Fn() -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + { + Self { + loader: Arc::new(move || Box::pin(loader())), + } + } + + async fn load(&self) -> Result { + (self.loader)().await + } +} + +#[derive(Clone, Default)] +struct CachedOidcExtraRootCerts { + generation: u64, + initialized: bool, + certs: Vec, +} + +/// HTTP client adapter bridging reqwest 0.13 to the `openidconnect` `AsyncHttpClient` trait. +/// +/// A fresh client is built for every request so the destination is re-validated and the +/// resolved IP is re-classified at connection time. This closes the SSRF / DNS-rebinding +/// gap where a one-shot URL string check is bypassed by a hostname that resolves to an +/// internal address only at connection time, and it also covers endpoints discovered from +/// the provider metadata (JWKS, token) rather than only the operator-configured `config_url`. +pub(super) struct ReqwestHttpClient { + /// `None` in production: the process-cached outbound policy from the environment is used. + /// `Some(..)` only in tests, to explicitly allow a loopback mock endpoint. + policy_override: Option, + extra_root_certs: Arc>, + extra_root_ca_provider: Option, + #[cfg(test)] + dns_resolver_override: Option>, +} + +pub(super) fn parse_oidc_extra_root_certs(source: &str, pem: &[u8]) -> Result, String> { + if pem.iter().all(|byte| byte.is_ascii_whitespace()) { + return Ok(Vec::new()); + } + Certificate::from_pem_bundle(pem).map_err(|err| format!("failed to parse OIDC extra root CA bundle from {source}: {err}")) +} + +pub(super) fn oidc_extra_root_certs(root_ca_pem: Option<&[u8]>) -> Result, String> { + match root_ca_pem { + Some(pem) => parse_oidc_extra_root_certs("RustFS outbound TLS material", pem), + None => Ok(Vec::new()), + } +} + +/// Build a reqwest client pinned to the shared outbound egress policy for a single request. +/// +/// [`OutboundPolicy::resolver_for`] validates the URL shape and rejects loopback, +/// link-local, metadata, multicast and unauthorized private addresses up front, and the +/// returned `OutboundDnsResolver` re-resolves and re-classifies the host on every new +/// connection so DNS rebinding fails closed. Redirects are not followed: a redirect target +/// would otherwise skip URL-shape re-validation. The timeouts bound how long a slow or +/// stalled provider can pin the calling task. +pub(super) fn build_oidc_http_client( + uri: &str, + policy_override: Option<&OutboundPolicy>, + extra_root_certs: &[Certificate], + #[cfg(test)] dns_resolver_override: Option>, +) -> Result<(Client, Url), OidcHttpError> { + let url = Url::parse(uri).map_err(|_| OidcHttpError::ForbiddenOutbound("invalid outbound OIDC URL".to_string()))?; + let resolver = match policy_override { + Some(policy) => policy.resolver_for(&url), + None => OutboundPolicy::from_env_cached() + .map_err(|err| OidcHttpError::ForbiddenOutbound(err.to_string()))? + .resolver_for(&url), + } + .map_err(|err| { + let base = err.to_string(); + let origin = url.origin().ascii_serialization(); + let can_allow_origin = + OutboundPolicy::from_allowed_origins(&origin).is_ok_and(|allowlisted| allowlisted.validate_url(&url).is_ok()); + oidc_forbidden_outbound_error(&url, base, can_allow_origin) + })?; + let bypass_proxy = should_bypass_proxy_for_oidc_uri(uri); + #[cfg(test)] + let bypass_proxy = bypass_proxy || dns_resolver_override.is_some(); + #[cfg(test)] + let resolver: Arc = dns_resolver_override.unwrap_or_else(|| Arc::new(resolver)); + + let mut builder = reqwest::Client::builder() + .dns_resolver(resolver) + .redirect(reqwest::redirect::Policy::none()) + .timeout(OIDC_HTTP_REQUEST_TIMEOUT) + .connect_timeout(OIDC_HTTP_CONNECT_TIMEOUT); + if bypass_proxy { + builder = builder.no_proxy(); + } + if !extra_root_certs.is_empty() { + builder = builder.tls_certs_merge(extra_root_certs.iter().cloned()); + } + builder.build().map(|client| (client, url)).map_err(OidcHttpError::Reqwest) +} + +fn oidc_forbidden_outbound_error(url: &Url, base: String, can_allow_origin: bool) -> OidcHttpError { + let reason = if can_allow_origin { + let origin = url.origin().ascii_serialization(); + format!( + "{base}; add {origin} to {ENV_OUTBOUND_ALLOW_ORIGINS} (comma-separated) and restart RustFS to allow this operator-owned OIDC provider (origin only, no path)" + ) + } else { + base + }; + OidcHttpError::ForbiddenOutbound(reason) +} + +fn oidc_http_error_from_reqwest(url: &Url, error: reqwest::Error) -> OidcHttpError { + if let Some(rejection) = find_outbound_dns_policy_rejection(&error) { + let base = rejection.to_string(); + return oidc_forbidden_outbound_error(url, base, rejection.allow_origin_can_recover()); + } + OidcHttpError::Reqwest(error) +} + +/// Buffer a provider response body, failing closed once `limit` bytes have been seen. +/// +/// `Response::bytes` would buffer the whole body unconditionally, so a hostile or compromised +/// provider endpoint could stream an arbitrarily large (or endless) body into memory. +async fn read_bounded_response_body(response: reqwest::Response, limit: usize) -> Result, OidcHttpError> { + if response.content_length().is_some_and(|len| len > limit as u64) { + return Err(OidcHttpError::ResponseTooLarge(limit)); + } + + let mut response = response; + let mut body = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(OidcHttpError::Reqwest)? { + if body.len() + chunk.len() > limit { + return Err(OidcHttpError::ResponseTooLarge(limit)); + } + body.extend_from_slice(&chunk); + } + Ok(body) +} + +pub(super) fn should_bypass_proxy_for_oidc_uri(uri: &str) -> bool { + let Some(host) = Url::parse(uri).ok().and_then(|url| url.host_str().map(str::to_owned)) else { + return false; + }; + let host = host.trim_matches(['[', ']']); + + host.eq_ignore_ascii_case("localhost") || host.parse::().is_ok_and(|addr| addr.is_loopback()) +} + +impl ReqwestHttpClient { + pub(super) fn new() -> Result { + Self::new_with_extra_root_certs(Vec::new()) + } + + fn extra_root_cert_cache(certs: Vec) -> Arc> { + Arc::new(RwLock::new(CachedOidcExtraRootCerts { + generation: 0, + initialized: true, + certs, + })) + } + + pub(super) fn new_with_extra_root_certs(extra_root_certs: Vec) -> Result { + Ok(Self { + policy_override: None, + extra_root_certs: Self::extra_root_cert_cache(extra_root_certs), + extra_root_ca_provider: None, + #[cfg(test)] + dns_resolver_override: None, + }) + } + + pub(super) fn new_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result { + Ok(Self { + policy_override: None, + extra_root_certs: Arc::new(RwLock::new(CachedOidcExtraRootCerts::default())), + extra_root_ca_provider: Some(extra_root_ca_provider), + #[cfg(test)] + dns_resolver_override: None, + }) + } + + pub(super) async fn current_extra_root_certs(&self) -> Result, OidcHttpError> { + let Some(provider) = self.extra_root_ca_provider.as_ref() else { + return self + .extra_root_certs + .read() + .map(|cache| cache.certs.clone()) + .map_err(|e| OidcHttpError::ExtraRootCa(format!("extra root certificate cache lock poisoned: {e}"))); + }; + + let material = provider.load().await.map_err(OidcHttpError::ExtraRootCa)?; + if let Ok(cache) = self.extra_root_certs.read() + && cache.initialized + && cache.generation == material.generation + { + return Ok(cache.certs.clone()); + } + + let certs = oidc_extra_root_certs(material.root_ca_pem.as_deref()).map_err(OidcHttpError::ExtraRootCa)?; + let mut cache = self + .extra_root_certs + .write() + .map_err(|e| OidcHttpError::ExtraRootCa(format!("extra root certificate cache lock poisoned: {e}")))?; + cache.generation = material.generation; + cache.initialized = true; + cache.certs = certs.clone(); + Ok(certs) + } + + /// Test-only constructor that pins outbound requests to an explicit policy, so a + /// loopback mock server can be reached without depending on process-wide environment. + #[cfg(test)] + pub(super) fn with_policy(policy: OutboundPolicy) -> Self { + Self { + policy_override: Some(policy), + extra_root_certs: Self::extra_root_cert_cache(Vec::new()), + extra_root_ca_provider: None, + dns_resolver_override: None, + } + } + + #[cfg(test)] + pub(super) fn with_policy_and_extra_root_certs(policy: OutboundPolicy, extra_root_certs: Vec) -> Self { + Self { + policy_override: Some(policy), + extra_root_certs: Self::extra_root_cert_cache(extra_root_certs), + extra_root_ca_provider: None, + dns_resolver_override: None, + } + } + + #[cfg(test)] + pub(super) fn with_policy_and_extra_root_ca_provider( + policy: OutboundPolicy, + extra_root_ca_provider: OidcExtraRootCaProvider, + ) -> Self { + Self { + policy_override: Some(policy), + extra_root_certs: Arc::new(RwLock::new(CachedOidcExtraRootCerts::default())), + extra_root_ca_provider: Some(extra_root_ca_provider), + dns_resolver_override: None, + } + } + + #[cfg(test)] + pub(super) fn with_policy_and_dns_resolver(policy: OutboundPolicy, resolver: Arc) -> Self { + Self { + policy_override: Some(policy), + extra_root_certs: Self::extra_root_cert_cache(Vec::new()), + extra_root_ca_provider: None, + dns_resolver_override: Some(resolver), + } + } +} + +impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient { + type Error = OidcHttpError; + type Future = Pin>, Self::Error>> + Send + 'c>>; + + fn call(&'c self, request: http::Request>) -> Self::Future { + Box::pin(async move { + let started_at = Instant::now(); + let (parts, body) = request.into_parts(); + let method = parts.method.clone(); + let uri = parts.uri.to_string(); + if tracing::enabled!(tracing::Level::DEBUG) { + let request_headers = format_http_headers(&parts.headers); + debug!( + event = EVENT_OIDC_HTTP, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "request", + method = %method, + uri = %uri, + request_headers = %request_headers, + request_body_len = body.len(), + "oidc outbound http" + ); + } + + let extra_root_certs = self.current_extra_root_certs().await?; + let (client, url) = build_oidc_http_client( + &uri, + self.policy_override.as_ref(), + &extra_root_certs, + #[cfg(test)] + self.dns_resolver_override.clone(), + )?; + let response = client + .request(parts.method, uri.clone()) + .headers(parts.headers) + .body(body) + .send() + .await; + + let elapsed_ms = started_at.elapsed().as_millis().min(u128::from(u64::MAX)) as u64; + let succeeded = response.as_ref().is_ok_and(|resp| resp.status().is_success()); + OIDC_PLUGIN_AUTHN_METRICS.record(elapsed_ms, succeeded); + + let response = response.map_err(|err| { + let error = oidc_http_error_from_reqwest(&url, err); + error!( + event = EVENT_OIDC_HTTP, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "request_failed", + method = %method, + uri = %uri, + elapsed_ms, + error = %error, + "oidc outbound http" + ); + error + })?; + + let status = response.status(); + let headers = response.headers().clone(); + let body_bytes = read_bounded_response_body(response, MAX_OIDC_RESPONSE_SIZE) + .await + .map_err(|err| { + error!( + event = EVENT_OIDC_HTTP, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "response_body_failed", + method = %method, + uri = %uri, + status = status.as_u16(), + elapsed_ms, + error = %err, + "oidc outbound http" + ); + err + })?; + if tracing::enabled!(tracing::Level::DEBUG) { + let response_headers = format_http_headers(&headers); + debug!( + event = EVENT_OIDC_HTTP, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "response", + method = %method, + uri = %uri, + status = status.as_u16(), + status_success = status.is_success(), + elapsed_ms, + response_headers = %response_headers, + response_body_len = body_bytes.len(), + "oidc outbound http" + ); + } + + let mut http_response = http::Response::builder() + .status(status) + .body(body_bytes) + .map_err(OidcHttpError::Http)?; + *http_response.headers_mut() = headers; + + Ok(http_response) + }) + } +} + +#[cfg(test)] +#[path = "transport_tests.rs"] +mod tests; diff --git a/crates/iam/src/oidc/transport_tests.rs b/crates/iam/src/oidc/transport_tests.rs new file mode 100644 index 000000000..e77fe561a --- /dev/null +++ b/crates/iam/src/oidc/transport_tests.rs @@ -0,0 +1,167 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::oidc::test_support::*; +use openidconnect::AsyncHttpClient; +use rustfs_config::MAX_OIDC_RESPONSE_SIZE; +use rustfs_utils::egress::{ENV_OUTBOUND_ALLOW_ORIGINS, OutboundPolicy}; +use std::sync::Arc; + +#[test] +fn format_http_headers_redacts_sensitive_values() { + let mut headers = http::HeaderMap::new(); + headers.insert(http::header::AUTHORIZATION, "Basic Y2xpZW50OnNlY3JldA==".parse().unwrap()); + headers.insert(http::header::CONTENT_TYPE, "application/json".parse().unwrap()); + headers.insert(http::header::COOKIE, "session=super-secret".parse().unwrap()); + + let rendered = format_http_headers(&headers); + + // Sensitive header values never appear; only their length is emitted. + assert!(!rendered.contains("Y2xpZW50OnNlY3JldA=="), "authorization value leaked: {rendered}"); + assert!(!rendered.contains("super-secret"), "cookie value leaked: {rendered}"); + assert!( + rendered.contains("authorization=`, suffix every provider env var with `_` (for example `RUSTFS_IDENTITY_OPENID_CLIENT_ID_keycloak`) and register the callback `/rustfs/admin/v3/oidc/callback/`. Suffix scanning is `parse_single_provider` in `crates/iam/src/oidc.rs`. +Named providers: to use provider id ``, suffix every provider env var with `_` (for example `RUSTFS_IDENTITY_OPENID_CLIENT_ID_keycloak`) and register the callback `/rustfs/admin/v3/oidc/callback/`. Suffix scanning is `parse_single_provider` in `crates/iam/src/oidc/config.rs`. Restart RustFS after changing any of these settings. diff --git a/docs/operations/oidc-provider-requirements.md b/docs/operations/oidc-provider-requirements.md index 4bc4c6c14..d2dc487b3 100644 --- a/docs/operations/oidc-provider-requirements.md +++ b/docs/operations/oidc-provider-requirements.md @@ -1,7 +1,7 @@ # OIDC provider requirements **Use this when:** evaluating whether an identity provider (or a vendor's "OIDC-like" OAuth product) can back RustFS console SSO, or debugging why a provider fails discovery, login, or policy mapping. -**Source of truth:** `crates/iam/src/oidc.rs` (`OidcProviderConfig`, `discover_provider`, `trusted_aud`, `map_claims_to_policies`); `crates/config/src/constants/oidc.rs` (`OIDC_DEFAULT_*`, `ENV_IDENTITY_OPENID_*`); `rustfs/src/admin/handlers/oidc.rs` (`derive_callback_uri_with_provider_config`, `browser_redirect_url`). +**Source of truth:** `crates/iam/src/oidc/config.rs` (`OidcProviderConfig`), `crates/iam/src/oidc/provider.rs` (`discover_provider`), `crates/iam/src/oidc/runtime.rs` (`trusted_aud`, `map_claims_to_policies`); `crates/config/src/constants/oidc.rs` (`OIDC_DEFAULT_*`, `ENV_IDENTITY_OPENID_*`); `rustfs/src/admin/handlers/oidc.rs` (`derive_callback_uri_with_provider_config`, `browser_redirect_url`). RustFS is a standard OpenID Connect relying party using the authorization-code flow with PKCE. It reads every claim it uses from the ID token; it never calls the UserInfo endpoint. Provider-side setup steps for Keycloak and Authing, and the RustFS-side configuration keys, are in [oidc-console-integration.md](oidc-console-integration.md). @@ -11,16 +11,16 @@ The table below describes Console login. For endpoint-free STS issuers, use the | # | Requirement | Details | Code anchor | | --- | --- | --- | --- | -| 1 | Discovery document | `RUSTFS_IDENTITY_OPENID_CONFIG_URL` names the provider (issuer base or full discovery URL); RustFS fetches `{issuer}/.well-known/openid-configuration` and needs `issuer`, `authorization_endpoint`, `token_endpoint`, `jwks_uri`, and the standard `*_supported` arrays. When `RUSTFS_IDENTITY_OPENID_ISSUER` is set, the document's `issuer` must equal it exactly; otherwise RustFS tries the issuer candidates derived from the config URL. | `crates/iam/src/oidc.rs` `discover_provider`, `discover_provider_from_config_url` | -| 2 | Outbound reachability | Discovery, JWKS, and token requests go through the shared egress policy. A provider on a private or loopback address needs its exact origin in `RUSTFS_OUTBOUND_ALLOW_ORIGINS`; otherwise startup logs `OIDC provider discovery blocked by outbound policy`. | `crates/iam/src/oidc.rs` `OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY`; [Outbound connection policy](outbound-connection-policy.md) | -| 3 | Signed ID token, verifiable via JWKS | The ID token signature is verified against `jwks_uri`; the key set is refreshed after `OIDC_JWKS_REFRESH_INTERVAL` and once more on a verification failure. A token response without `id_token` fails login — an OAuth-only access token is not sufficient. | `crates/iam/src/oidc.rs` `OIDC_JWKS_REFRESH_INTERVAL`, the `no id_token in token response` error | -| 4 | Authorization-code flow with PKCE (S256) | The authorization request carries `response_type=code`, `scope`, `redirect_uri`, `state`, `nonce`, `code_challenge`, `code_challenge_method=S256`; the token request carries the `code_verifier`. Providers that ignore or reject PKCE, `state`, or `nonce` are not supported. | `crates/iam/src/oidc.rs` `PkceCodeChallenge::new_random_sha256`; `crates/iam/src/oidc_state.rs` `pkce_verifier` | -| 5 | Scopes | Default `openid,profile,email` (`OIDC_DEFAULT_SCOPES`); override with `RUSTFS_IDENTITY_OPENID_SCOPES` (comma-separated). The provider must accept every configured scope. | `crates/config/src/constants/oidc.rs` `OIDC_DEFAULT_SCOPES`; `crates/iam/src/oidc.rs` `OidcProviderConfig::scopes` | -| 6 | Callback returns `state` | The callback must carry both `code` and the original `state`; `state` locates the in-flight session that holds the nonce and PKCE verifier. A callback with only `code` fails. | `rustfs/src/admin/handlers/oidc.rs` `OIDC_CALLBACK_SUFFIX`; `crates/iam/src/oidc_state.rs` | -| 7 | ID token claims | `iss`, `aud`, `exp`, and `nonce` are verified; `sub` identifies the user. `aud` must contain the client id or one of `RUSTFS_IDENTITY_OPENID_OTHER_AUDIENCES`. | `crates/iam/src/oidc.rs` `trusted_aud`, `OidcClaims` | -| 8 | Authorization claims in the ID token | Policy mapping reads the claim named by `RUSTFS_IDENTITY_OPENID_CLAIM_NAME` (default `groups`, `OIDC_DEFAULT_CLAIM_NAME`) together with `RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM` and the optional `RUSTFS_IDENTITY_OPENID_ROLES_CLAIM`, as a string or array of strings, optionally prefixed by `RUSTFS_IDENTITY_OPENID_CLAIM_PREFIX`. Values must equal RustFS policy names (for example `consoleAdmin`, `readwrite`, `readonly`) made only of `[A-Za-z0-9_:.-]`. Groups-claim and roles-claim values with no policy of that name, such as the Active Directory group `DOMAIN\Domain Users` or Keycloak's `offline_access` role, are ignored once storage confirms that no such policy exists. Login is rejected when a policy lookup fails, when no value resolves, when a value names an existing policy whose name is outside that character set (dropping it could remove a Deny), or when a name from `RUSTFS_IDENTITY_OPENID_ROLE_POLICY`, a separate `RUSTFS_IDENTITY_OPENID_CLAIM_NAME` claim, or an explicit IAM policy mapping does not exist. Email and username come from `RUSTFS_IDENTITY_OPENID_EMAIL_CLAIM` (default `email`) and `RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM` (default `preferred_username`). A provider that returns only a user id can authenticate but cannot express RustFS authorization. | `crates/iam/src/oidc.rs` `map_claims_to_policies`, `extract_canonical_group_values`; `OidcSys::group_claim_policy_names`; `rustfs/src/admin/service/federated_identity.rs` `select_oidc_policies` | +| 1 | Discovery document | `RUSTFS_IDENTITY_OPENID_CONFIG_URL` names the provider (issuer base or full discovery URL); RustFS fetches `{issuer}/.well-known/openid-configuration` and needs `issuer`, `authorization_endpoint`, `token_endpoint`, `jwks_uri`, and the standard `*_supported` arrays. When `RUSTFS_IDENTITY_OPENID_ISSUER` is set, the document's `issuer` must equal it exactly; otherwise RustFS tries the issuer candidates derived from the config URL. | `crates/iam/src/oidc/provider.rs` `discover_provider`, `discover_provider_from_config_url` | +| 2 | Outbound reachability | Discovery, JWKS, and token requests go through the shared egress policy. A provider on a private or loopback address needs its exact origin in `RUSTFS_OUTBOUND_ALLOW_ORIGINS`; otherwise startup logs `OIDC provider discovery blocked by outbound policy`. | `crates/iam/src/oidc/provider.rs` `OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY`; [Outbound connection policy](outbound-connection-policy.md) | +| 3 | Signed ID token, verifiable via JWKS | The ID token signature is verified against `jwks_uri`; the key set is refreshed after `OIDC_JWKS_REFRESH_INTERVAL` and once more on a verification failure. A token response without `id_token` fails login — an OAuth-only access token is not sufficient. | `crates/iam/src/oidc/provider.rs` `OIDC_JWKS_REFRESH_INTERVAL`; `crates/iam/src/oidc/runtime.rs` `no id_token in token response` | +| 4 | Authorization-code flow with PKCE (S256) | The authorization request carries `response_type=code`, `scope`, `redirect_uri`, `state`, `nonce`, `code_challenge`, `code_challenge_method=S256`; the token request carries the `code_verifier`. Providers that ignore or reject PKCE, `state`, or `nonce` are not supported. | `crates/iam/src/oidc/runtime.rs` `PkceCodeChallenge::new_random_sha256`; `crates/iam/src/oidc/state.rs` `pkce_verifier` | +| 5 | Scopes | Default `openid,profile,email` (`OIDC_DEFAULT_SCOPES`); override with `RUSTFS_IDENTITY_OPENID_SCOPES` (comma-separated). The provider must accept every configured scope. | `crates/config/src/constants/oidc.rs` `OIDC_DEFAULT_SCOPES`; `crates/iam/src/oidc/config.rs` `OidcProviderConfig::scopes` | +| 6 | Callback returns `state` | The callback must carry both `code` and the original `state`; `state` locates the in-flight session that holds the nonce and PKCE verifier. A callback with only `code` fails. | `rustfs/src/admin/handlers/oidc.rs` `OIDC_CALLBACK_SUFFIX`; `crates/iam/src/oidc/state.rs` | +| 7 | ID token claims | `iss`, `aud`, `exp`, and `nonce` are verified; `sub` identifies the user. `aud` must contain the client id or one of `RUSTFS_IDENTITY_OPENID_OTHER_AUDIENCES`. | `crates/iam/src/oidc/runtime.rs` `trusted_aud`, `OidcClaims` | +| 8 | Authorization claims in the ID token | Policy mapping reads the claim named by `RUSTFS_IDENTITY_OPENID_CLAIM_NAME` (default `groups`, `OIDC_DEFAULT_CLAIM_NAME`) together with `RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM` and the optional `RUSTFS_IDENTITY_OPENID_ROLES_CLAIM`, as a string or array of strings, optionally prefixed by `RUSTFS_IDENTITY_OPENID_CLAIM_PREFIX`. Values must equal RustFS policy names (for example `consoleAdmin`, `readwrite`, `readonly`) made only of `[A-Za-z0-9_:.-]`. Groups-claim and roles-claim values with no policy of that name, such as the Active Directory group `DOMAIN\Domain Users` or Keycloak's `offline_access` role, are ignored once storage confirms that no such policy exists. Login is rejected when a policy lookup fails, when no value resolves, when a value names an existing policy whose name is outside that character set (dropping it could remove a Deny), or when a name from `RUSTFS_IDENTITY_OPENID_ROLE_POLICY`, a separate `RUSTFS_IDENTITY_OPENID_CLAIM_NAME` claim, or an explicit IAM policy mapping does not exist. Email and username come from `RUSTFS_IDENTITY_OPENID_EMAIL_CLAIM` (default `email`) and `RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM` (default `preferred_username`). A provider that returns only a user id can authenticate but cannot express RustFS authorization. | `crates/iam/src/oidc/runtime.rs` `map_claims_to_policies`, `extract_canonical_group_values`, `OidcSys::group_claim_policy_names`; `rustfs/src/admin/service/federated_identity.rs` `select_oidc_policies` | | 9 | Registered redirect URI | The provider must accept the callback `{public-origin}/rustfs/admin/v3/oidc/callback/{provider_id}`. RustFS picks the origin in this order: the provider's `redirect_uri` (`RUSTFS_IDENTITY_OPENID_REDIRECT_URI`), then `RUSTFS_BROWSER_REDIRECT_URL`, then the request's own scheme and host — the last only when `RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC` is enabled. | `rustfs/src/admin/handlers/oidc.rs` `derive_callback_uri_with_provider_config`, `browser_redirect_url` | -| 10 | Logout endpoint (optional) | When discovery advertises `end_session_endpoint`, RustFS builds an RP-initiated logout URL with `id_token_hint`, `client_id`, and `post_logout_redirect_uri`. Without it, logout falls back to the console login page. | `crates/iam/src/oidc.rs` `build_logout_url` (reads `end_session_endpoint` from `ProviderMetadataWithLogout`) | +| 10 | Logout endpoint (optional) | When discovery advertises `end_session_endpoint`, RustFS builds an RP-initiated logout URL with `id_token_hint`, `client_id`, and `post_logout_redirect_uri`. Without it, logout falls back to the console login page. | `crates/iam/src/oidc/runtime.rs` `build_logout_url` (reads `end_session_endpoint` from `ProviderMetadataWithLogout`) | ## STS workload providers diff --git a/docs/operations/outbound-connection-policy.md b/docs/operations/outbound-connection-policy.md index f729a2915..b0a094cbe 100644 --- a/docs/operations/outbound-connection-policy.md +++ b/docs/operations/outbound-connection-policy.md @@ -16,7 +16,7 @@ RustFS validates every operator-configured outbound destination to close a serve | --- | --- | --- | | Event-notification webhooks (`RUSTFS_NOTIFY_WEBHOOK_*`) and audit webhooks (`RUSTFS_AUDIT_WEBHOOK_*`) | Full policy | Proxies disabled and redirects not followed, so the endpoint must be reachable directly (`crates/targets/src/target/webhook.rs`) | | Target configuration validation (startup and admin API) | Full policy | `crates/targets/src/config/common.rs` `validate_outbound_http_url`; `rustfs/src/admin/handlers/target_descriptor.rs` | -| OIDC discovery, JWKS, and token requests | Full policy | A blocked provider logs `OIDC provider discovery blocked by outbound policy` naming the origin to allowlist (`crates/iam/src/oidc.rs`) | +| OIDC discovery, JWKS, and token requests | Full policy | A blocked provider logs `OIDC provider discovery blocked by outbound policy` naming the origin to allowlist (`crates/iam/src/oidc/provider.rs`, `crates/iam/src/oidc/transport.rs`) | | Object Lambda targets | Full policy | `rustfs/src/admin/router.rs` `outbound_policy` | | Bucket replication targets | Literal check, relaxed | Private addresses are always allowed; loopback only with `RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET=true` (`crates/ecstore/src/bucket/remote_s3_client.rs` `validate_remote_endpoint`, shared with on-demand migration sources) | | On-demand migration sources | Literal check, relaxed | Same guard and same escape hatch as replication targets (see below) | diff --git a/rustfs/src/app/context.rs b/rustfs/src/app/context.rs index f1373cb46..a9bdcd025 100644 --- a/rustfs/src/app/context.rs +++ b/rustfs/src/app/context.rs @@ -39,9 +39,7 @@ use crate::config::RustFSBufferConfig; use rustfs_config::server_config::Config; use rustfs_credentials::Credentials; use rustfs_iam::{ - error::Error as IamError, - federation::{FederatedIdentityService, oidc::OidcConfigQuery}, - store::object::ObjectStore, + error::Error as IamError, federation::FederatedIdentityService, oidc::OidcConfigQuery, store::object::ObjectStore, sys::IamSys, }; use rustfs_io_metrics::{PerformanceMetrics, internode_metrics::InternodeMetrics}; @@ -544,11 +542,8 @@ mod tests { use crate::config::{RustFSBufferConfig, WorkloadProfile}; use async_trait::async_trait; use rustfs_iam::{ - federation::{ - CoreFederatedAuthorizationMapper, FederatedIdentityService, - oidc::{OidcConfigQuery, StandardOidcAdapter}, - }, - oidc::OidcSys, + federation::{CoreFederatedAuthorizationMapper, FederatedIdentityService}, + oidc::{OidcConfigQuery, OidcSys, StandardOidcAdapter}, store::object::ObjectStore, sys::IamSys, }; diff --git a/rustfs/src/app/context/global.rs b/rustfs/src/app/context/global.rs index fe52e40db..c4ea0edc6 100644 --- a/rustfs/src/app/context/global.rs +++ b/rustfs/src/app/context/global.rs @@ -35,11 +35,7 @@ use super::interfaces::{ }; use crate::app::object_data_cache::ObjectDataCacheAdapter; use crate::app::object_traffic_health::ObjectTrafficHealth; -use rustfs_iam::{ - federation::{FederatedIdentityService, oidc::OidcConfigQuery}, - store::object::ObjectStore, - sys::IamSys, -}; +use rustfs_iam::{federation::FederatedIdentityService, oidc::OidcConfigQuery, store::object::ObjectStore, sys::IamSys}; use rustfs_kms::KmsServiceManager; use std::sync::{Arc, OnceLock}; diff --git a/rustfs/src/app/context/handles.rs b/rustfs/src/app/context/handles.rs index 6fbbddc50..2968af533 100644 --- a/rustfs/src/app/context/handles.rs +++ b/rustfs/src/app/context/handles.rs @@ -32,11 +32,7 @@ use crate::config::RustFSBufferConfig; use async_trait::async_trait; use rustfs_config::server_config::Config; use rustfs_credentials::Credentials; -use rustfs_iam::{ - federation::{FederatedIdentityService, oidc::OidcConfigQuery}, - store::object::ObjectStore, - sys::IamSys, -}; +use rustfs_iam::{federation::FederatedIdentityService, oidc::OidcConfigQuery, store::object::ObjectStore, sys::IamSys}; use rustfs_io_metrics::{PerformanceMetrics, internode_metrics::InternodeMetrics}; use rustfs_kms::KmsServiceManager; use rustfs_lock::LockClient; @@ -654,11 +650,8 @@ mod tests { use crate::app::context::interfaces::{KmsRuntimeInterface, ServerConfigInterface}; use rustfs_config::server_config::Config; use rustfs_iam::{ - federation::{ - CoreFederatedAuthorizationMapper, FederatedIdentityService, - oidc::{OidcConfigQuery, StandardOidcAdapter}, - }, - oidc::OidcSys, + federation::{CoreFederatedAuthorizationMapper, FederatedIdentityService}, + oidc::{OidcConfigQuery, OidcSys, StandardOidcAdapter}, }; use std::collections::HashMap; use std::sync::Arc; diff --git a/rustfs/src/app/context/interfaces.rs b/rustfs/src/app/context/interfaces.rs index 8fd9eb7bf..43052f5ad 100644 --- a/rustfs/src/app/context/interfaces.rs +++ b/rustfs/src/app/context/interfaces.rs @@ -22,11 +22,7 @@ use crate::config::RustFSBufferConfig; use async_trait::async_trait; use rustfs_config::server_config::Config; use rustfs_credentials::Credentials; -use rustfs_iam::{ - federation::{FederatedIdentityService, oidc::OidcConfigQuery}, - store::object::ObjectStore, - sys::IamSys, -}; +use rustfs_iam::{federation::FederatedIdentityService, oidc::OidcConfigQuery, store::object::ObjectStore, sys::IamSys}; use rustfs_io_metrics::{PerformanceMetrics, internode_metrics::InternodeMetrics}; use rustfs_kms::KmsServiceManager; use rustfs_lock::LockClient; diff --git a/rustfs/src/runtime_sources.rs b/rustfs/src/runtime_sources.rs index fc211890b..5fdf451ab 100644 --- a/rustfs/src/runtime_sources.rs +++ b/rustfs/src/runtime_sources.rs @@ -14,7 +14,7 @@ use crate::app::context; use rustfs_config::ENV_RUSTFS_BROWSER_REDIRECT_URL; -use rustfs_iam::federation::{FederatedIdentityService, oidc::OidcConfigQuery}; +use rustfs_iam::{federation::FederatedIdentityService, oidc::OidcConfigQuery}; use std::sync::Arc; use tracing::warn; @@ -104,11 +104,8 @@ pub(crate) fn current_app_context() -> Option> { mod tests { use super::*; use rustfs_iam::{ - federation::{ - CoreFederatedAuthorizationMapper, - oidc::{OidcConfigQuery, StandardOidcAdapter}, - }, - oidc::{OidcSys, make_test_sys, test_config}, + federation::CoreFederatedAuthorizationMapper, + oidc::{OidcConfigQuery, OidcSys, StandardOidcAdapter, make_test_sys, test_config}, }; use std::{ io::{self, Write}, diff --git a/rustfs/src/startup_auth.rs b/rustfs/src/startup_auth.rs index c47afb3b1..ec2cbe6f4 100644 --- a/rustfs/src/startup_auth.rs +++ b/rustfs/src/startup_auth.rs @@ -14,11 +14,8 @@ use rustfs_iam::{ build_oidc_sys_with_extra_root_ca_provider, - federation::{ - CoreFederatedAuthorizationMapper, FederatedIdentityService, - oidc::{OidcConfigQuery, StandardOidcAdapter}, - }, - oidc::{OidcExtraRootCaMaterial, OidcExtraRootCaProvider, OidcSys}, + federation::{CoreFederatedAuthorizationMapper, FederatedIdentityService}, + oidc::{OidcConfigQuery, OidcExtraRootCaMaterial, OidcExtraRootCaProvider, OidcSys, StandardOidcAdapter}, }; use std::{ collections::hash_map::DefaultHasher, From ba2047b18422aebffa5f0686cd304ad746e480fb Mon Sep 17 00:00:00 2001 From: Chris Date: Sat, 10 Oct 2026 22:55:48 -0700 Subject: [PATCH 009/143] ci: reuse workspace test builds for serial ILM checks (#8465) --- .github/workflows/cache-warm.yml | 2 +- .github/workflows/ci.yml | 128 +++++++++++-------------------- docs/testing/ci-gates.md | 9 ++- scripts/check_test_wiring.py | 35 ++++++++- scripts/ci_gate.py | 42 +++++++++- 5 files changed, 126 insertions(+), 90 deletions(-) diff --git a/.github/workflows/cache-warm.yml b/.github/workflows/cache-warm.yml index 699ba2990..c01dee315 100644 --- a/.github/workflows/cache-warm.yml +++ b/.github/workflows/cache-warm.yml @@ -99,7 +99,7 @@ env: RUST_BACKTRACE: 1 jobs: - # Readers: test-and-lint, test-ilm-integration-serial, build-rustfs-debug-binary, + # Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary, # e2e-tests, e2e-full. warm-ci-dev: name: Warm ci-dev diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d02f9966..2e92b226e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,13 +168,10 @@ jobs: echo "started_at=$(date --utc --iso-8601=seconds)" } > artifacts/test-and-lint/run-metadata.txt - # Clippy runs before the test pass: lint failures are the most common - # CI-only breakage and should surface in minutes, not after 20+ minutes - # of tests. - # Sampled too: clippy is the natural control arm for any CARGO_BUILD_JOBS - # experiment, since --all-targets is check-only for workspace members and - # never links the ~100 test binaries the limit exists to throttle. + # Keep full Clippy validation on main, merge groups, and scheduled/manual + # runs. PRs retain the complete default test suite and compile-free checks. - name: Run clippy lints + if: github.event_name != 'pull_request' run: | ./scripts/ci/resource_sampler.sh start clippy trap './scripts/ci/resource_sampler.sh stop' EXIT @@ -283,6 +280,49 @@ jobs: - name: Verify rebalance/decommission migration proofs run: ./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml + # The workspace already compiled these ignored tests. Keep the same Cargo + # package selection to reuse their binaries, then run them serially in + # separate processes because they share fixed ports and global state. + # test_transition_and_restore_flows was re-enabled by rustfs/backlog#1303: + # its "missing xl.meta on disk2" was a test-util bug (open_disk hardcoded + # disk_index 0), not an EC metadata-distribution issue. + # restore_object_usecase_reports_ongoing_conflict_and_completion was + # re-enabled by backlog#1304 (restore accepts serialize on a short CAS + # guard; the copy-back no longer holds the #4877 whole-copy-back lock, + # so the mid-restore ongoing read and fast 409 rejection it asserts are + # the implemented contract). + - name: Run ignored ILM integration tests serially + if: needs.classify-changes.outputs.mode == 'full' + env: + # Match the preceding workspace build and reuse its feature graph. + CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }} + run: | + mkdir -p artifacts/ilm-integration + set +e + NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 80m \ + cargo nextest run --profile default -j1 --run-ignored ignored-only \ + --all --exclude e2e_test \ + -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))' \ + --status-level all --final-status-level all \ + 2>&1 | tee artifacts/ilm-integration/nextest.log + status=${PIPESTATUS[0]} + { + echo "exit_status=${status}" + echo "finished_at=$(date --utc --iso-8601=seconds)" + echo + echo "Remaining test-related processes:" + pgrep -af 'cargo|nextest|target/.*/deps/' || true + } > artifacts/ilm-integration/diagnostics.txt + exit "${status}" + + - name: Upload ILM test diagnostics + if: always() && needs.classify-changes.outputs.mode == 'full' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ilm-integration-${{ github.run_number }}-${{ github.run_attempt }} + path: artifacts/ilm-integration + retention-days: 3 + # This gate builds into a fresh target directory to isolate the E2E root. # Give its cold build a separate budget from workspace tests and migration proofs. offline-enrollment-e2e: @@ -314,80 +354,6 @@ jobs: - name: Check offline enrollment E2E root boundary run: ./scripts/check_offline_enrollment_e2e.sh - # Dedicated serial lane for the ILM / lifecycle integration tests. These tests - # drive the object layer through process-global singletons (the GLOBAL_ENV - # ECStore, the global tier-config manager, background-expiry workers) and bind - # fixed ports (9002 for the scanner suite, 9003 for the app suite), so they are - # marked #[ignore] to keep them out of the default parallel `cargo nextest run - # --all` pass. Run them here explicitly with `--run-ignored ignored-only` and - # `-j1` so no two of them share global state or race for a port at once. - # serial_test's #[serial] does NOT serialize across nextest's process-per-test - # boundary (see .config/nextest.toml); `-j1` is what actually serializes them. - # See rustfs/backlog#1148 (ilm-1) and #1155. - test-ilm-integration-serial: - name: ILM Integration (serial) - if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed') - needs: [ quick-checks, classify-changes ] - runs-on: sm-standard-4 - timeout-minutes: 90 - env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - steps: - - name: Checkout repository - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 - with: - persist-credentials: false - - - name: Setup Rust environment - uses: ./.github/actions/setup - with: - rust-version: stable - cache-shared-key: ci-dev - cache-save-if: 'false' - install-build-packaging-tools: 'false' - - # test_transition_and_restore_flows was re-enabled by rustfs/backlog#1303: - # its "missing xl.meta on disk2" was a test-util bug (open_disk hardcoded - # disk_index 0), not an EC metadata-distribution issue. - # restore_object_usecase_reports_ongoing_conflict_and_completion was - # re-enabled by backlog#1304 (restore accepts serialize on a short CAS - # guard; the copy-back no longer holds the #4877 whole-copy-back lock, - # so the mid-restore ongoing read and fast 409 rejection it asserts are - # the implemented contract). - - name: Run ignored ILM integration tests serially - env: - # Match the measured Test and Lint link budget. The default exposed - # all 14 pod CPUs and a cold cache spent the full 80m compiling - # without starting one ILM test (main run 32982910990). - CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }} - run: | - mkdir -p artifacts/ilm-integration - set +e - NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 80m \ - cargo nextest run -j1 --run-ignored ignored-only \ - -p rustfs-scanner -p rustfs \ - -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))' \ - --status-level all --final-status-level all \ - 2>&1 | tee artifacts/ilm-integration/nextest.log - status=${PIPESTATUS[0]} - { - echo "exit_status=${status}" - echo "finished_at=$(date --utc --iso-8601=seconds)" - echo - echo "Remaining test-related processes:" - pgrep -af 'cargo|nextest|target/.*/deps/' || true - } > artifacts/ilm-integration/diagnostics.txt - exit "${status}" - - - name: Upload ILM test diagnostics - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: ilm-integration-${{ github.run_number }}-${{ github.run_attempt }} - path: | - artifacts/ilm-integration - target/nextest/ci/junit.xml - test-and-lint-rio-v2: name: Test and Lint (rio-v2) # Alternate feature configurations run on main, merge groups, and scheduled @@ -1266,7 +1232,6 @@ jobs: - quick-checks - test-and-lint - offline-enrollment-e2e - - test-ilm-integration-serial - test-and-lint-rio-v2 - connect-short-credential-boundary - test-and-lint-protocols @@ -1300,7 +1265,6 @@ jobs: - quick-checks - test-and-lint - offline-enrollment-e2e - - test-ilm-integration-serial - test-and-lint-rio-v2 - test-and-lint-protocols - build-rustfs-debug-binary diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 389193326..8ea7e09f2 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -21,6 +21,10 @@ PRs changing only Rust sources under `crates/e2e_test/src/` and known E2E select Alternate-feature unit tests and Clippy for rio-v2, Swift, SFTP, and HTTP/3 run on main pushes, merge groups, and scheduled/manual validation. PRs run the default workspace and retain ILM, S3, real io_uring, and Connect security boundaries in their applicable scopes. The aggregate requires the alternate-feature jobs to be skipped on PRs and successful on non-PR events. This reduces repeated builds across PR revisions; feature-specific regressions can be detected after merging when the merge queue is not used. For changes to these features, dispatch `ci.yml` against the candidate branch before merging when that signal is needed, including PRs targeting `gateway/integration`, which has no automatic push trigger in this workflow. Scheduled results supplement the main baseline and do not replace it. +Workspace Clippy uses the same non-PR cadence. Every code PR still builds and runs the complete default workspace test suite, documentation tests, and migration proofs; Quick Checks retain formatting and script/workflow linting. Without a merge queue, Clippy diagnostics may first fail main after merge. A manual `ci.yml` dispatch against the candidate branch runs Clippy before merging when needed. + +Serial ILM tests run at the end of `test-and-lint` for the `full` scope, reusing the workspace's Cargo package and feature selection. The existing ignored-test filter and `-j1` remain; `--profile default` keeps the no-retry execution and preserves the separate core CI JUnit report. Any ILM failure fails the workspace job and required aggregate. Logs remain in the `ilm-integration` artifact. There is no separate ILM runner or cache restore. + `required-checks` runs even after failed or skipped dependencies. `scripts/ci_gate.py verify` rejects missing jobs, unexpected jobs, failure, cancellation, and unexpected skips; optional lanes are required only on their declared events. `Workspace Test and Lint` is the ordinary Rust job, while `Test and Lint` uniquely names the aggregate. New validation jobs must update both its direct dependencies and the script contract. Test this wiring and its failure cases with `python3 scripts/ci_gate.py --self-test`. The `gateway/integration` ruleset (`24663286`) requires the same three contexts with the same review rules for the s3s to gateway integration branch (rustfs/backlog#2734). It omits `required_linear_history` because `gateway-integration-sync.yml` lands `--no-ff` merge commits from `main` on that branch every weekday, and it names the `rustfs-cla-assistant` app as the bypass actor that pushes them. `ci.yml` runs on pull requests targeting either branch. @@ -47,9 +51,10 @@ The four site-replication state-writer concurrency proofs reserve the available | Event | Check name | Workflow / job | Merge status | Reproduce | |---|---|---|---|---| | PR, non-doc change | `Quick Checks` | `ci.yml` `quick-checks` | Required | `make pre-commit` | -| PR, non-doc change | `Workspace Test and Lint` | `ci.yml` `test-and-lint` | Via aggregate | `cargo clippy --all-targets -- -D warnings`; `cargo nextest run --profile ci --all --exclude e2e_test`; `cargo test --all --doc`; migration evidence check described below | +| PR, non-doc change | `Workspace Test and Lint` | `ci.yml` `test-and-lint` | Via aggregate | `cargo nextest run --profile ci --all --exclude e2e_test`; `cargo test --all --exclude e2e_test --doc`; migration evidence check described below | +| Main push, merge group, schedule, manual dispatch | `Workspace Test and Lint` | `ci.yml` `test-and-lint` | Via aggregate on these events | workspace tests plus `cargo clippy --all-targets -- -D warnings` | | PR, non-doc change | `Typos` | `ci.yml` `typos` | Via aggregate | `typos` | -| PR, full selection | `ILM Integration (serial)` | `ci.yml` `test-ilm-integration-serial` | Via aggregate | exact command in the job | +| Full selection | `Workspace Test and Lint` (serial ILM step) | `ci.yml` `test-and-lint` | Via aggregate | exact `Run ignored ILM integration tests serially` command in the job | | Main push, merge group, schedule, manual dispatch | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)`, `Test and Lint (http3)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate on these events; skipped on PRs | `cargo nextest run` with the job's `--features` | | PR, full selection | `Connect Short Credential Boundary` | `ci.yml` `connect-short-credential-boundary` | Via aggregate | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `cargo check -p rustfs --release --features connect-e2e-short-credentials` must fail | | PR, full selection | `Offline Enrollment E2E Root Boundary` | `ci.yml` `offline-enrollment-e2e` | Via aggregate | `scripts/check_offline_enrollment_e2e.sh` | diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index bd9f7013f..3a8251775 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -684,12 +684,12 @@ def check_ilm_build_budget(root: Path) -> list[str]: return [".github/workflows/ci.yml: missing workflow"] workflow = workflow_path.read_text() job = re.search( - r"^ test-ilm-integration-serial:\n(?P.*?)(?=^ [a-z0-9-]+:\n)", + r"^ test-and-lint:\n(?P.*?)(?=^ [a-z0-9-]+:\n)", workflow, re.MULTILINE | re.DOTALL, ) if job is None: - return [".github/workflows/ci.yml: missing test-ilm-integration-serial job"] + return [".github/workflows/ci.yml: ILM integration must share the workspace test job"] step = re.search( r"^ - name: Run ignored ILM integration tests serially\n(?P.*?)(?=^ - name:|\Z)", @@ -705,6 +705,16 @@ def check_ilm_build_budget(root: Path) -> list[str]: ) if budget not in step.group("body"): return [".github/workflows/ci.yml: ILM integration must keep the measured 3/2 Cargo build budget"] + for required in ( + " if: needs.classify-changes.outputs.mode == 'full'", + "cargo nextest run --profile default -j1 --run-ignored ignored-only", + "--all --exclude e2e_test", + "-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'", + 'status=${PIPESTATUS[0]}', + 'exit "${status}"', + ): + if required not in step.group("body"): + return [f".github/workflows/ci.yml: ILM integration is missing {required.strip()}"] return [] @@ -5566,12 +5576,17 @@ class SelfTests(unittest.TestCase): workflow.parent.mkdir(parents=True) valid = ( "jobs:\n" - " test-ilm-integration-serial:\n" + " test-and-lint:\n" " steps:\n" " - name: Run ignored ILM integration tests serially\n" + " if: needs.classify-changes.outputs.mode == 'full'\n" " env:\n" " CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}\n" - " run: cargo nextest run\n" + " run: |\n" + " cargo nextest run --profile default -j1 --run-ignored ignored-only --all --exclude e2e_test \\\n" + " -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'\n" + " status=${PIPESTATUS[0]}\n" + ' exit "${status}"\n' " next-job:\n" " steps: []\n" ) @@ -5580,6 +5595,18 @@ class SelfTests(unittest.TestCase): workflow.write_text(valid.replace(" CARGO_BUILD_JOBS:", " BUILD_JOBS:")) self.assertEqual(len(check_ilm_build_budget(root)), 1) + for old, new in ( + ("test-and-lint:", "test-ilm-integration-serial:"), + ("outputs.mode == 'full'", "outputs.mode == 'docs'"), + ("--profile default", "--profile ci"), + ("-j1", "-j2"), + ("ignored-only", "default"), + ("--all --exclude e2e_test", "-p rustfs-scanner -p rustfs"), + ("binary(lifecycle_integration_test)", "none()"), + ('exit "${status}"', "exit 0"), + ): + workflow.write_text(valid.replace(old, new)) + self.assertTrue(check_ilm_build_budget(root), (old, new)) def test_vault_tests_require_cross_process_serialization(self) -> None: with tempfile.TemporaryDirectory() as tmp: diff --git a/scripts/ci_gate.py b/scripts/ci_gate.py index e8cf52e75..b99c10927 100644 --- a/scripts/ci_gate.py +++ b/scripts/ci_gate.py @@ -16,7 +16,7 @@ import unittest ROOT = Path(__file__).resolve().parent.parent ALWAYS_JOBS = ("classify-changes", "typos", "quick-checks") CODE_JOBS = ( - "test-and-lint", "offline-enrollment-e2e", "test-ilm-integration-serial", "test-and-lint-rio-v2", + "test-and-lint", "offline-enrollment-e2e", "test-and-lint-rio-v2", "connect-short-credential-boundary", "test-and-lint-protocols", "build-rustfs-debug-binary", "uring-integration", "e2e-tests", "s3-implemented-tests", "s3-lifecycle-behavior-tests", @@ -224,6 +224,10 @@ def check_workflow(root: Path) -> list[str]: block = yaml_block(jobs, job, 2) or [] if scalar(block, "if", 4) != "needs.classify-changes.outputs.mode == 'full' && github.event_name != 'pull_request'": errors.append(f"{job} must retain full validation on every non-PR event") + workspace = "\n".join(yaml_block(jobs, "test-and-lint", 2) or []) + clippy = re.search(r"^ - name: Run clippy lints\n(?P.*?)(?=^ - name:|\Z)", workspace, re.MULTILINE | re.DOTALL) + if clippy is None or scalar(clippy["body"].splitlines(), "if", 8) != "github.event_name != 'pull_request'": + errors.append("workspace Clippy must run on every non-PR event") if scalar(gate, "shell", 8) != "bash" or scalar(gate, "run", 8) != "python3 scripts/ci_gate.py verify" or scalar(gate, "CI_NEEDS", 10) != "${{ toJSON(needs) }}": errors.append("required-checks must verify the actual needs results") if any(re.match(r'''\s+(?:- )?(?:["']?continue-on-error["']?\s*:|["']?if["']?\s*:)''', line) and not line.startswith(" if:") for line in gate): @@ -387,6 +391,9 @@ class SelfTests(unittest.TestCase): (root / ".github/workflows").mkdir(parents=True) source = (ROOT / ".github/workflows/ci.yml").read_text() path = root / ".github/workflows/ci.yml" + for condition in ("false", "github.event_name == 'schedule'", "github.event_name == 'pull_request'"): + path.write_text(source.replace(" - name: Run clippy lints\n if: github.event_name != 'pull_request'", f" - name: Run clippy lints\n if: {condition}")) + self.assertIn("workspace Clippy must run on every non-PR event", check_workflow(root)) for job in EXTENDED_FEATURE_JOBS: before, block = source.split(f" {job}:\n", 1) for condition in ("false", "github.event_name == 'schedule'", "github.event_name != 'pull_request'"): @@ -462,6 +469,39 @@ class SelfTests(unittest.TestCase): self.assertEqual(result.returncode, code, result.stderr) self.assertIn("ERROR:" if code else "CI contract passed", result.stderr if code else result.stdout) + def test_merged_ilm_step_preserves_exit_status_and_core_report(self): + from check_test_wiring import yaml_block + from test_security_workflow import named_steps, shell_body + + workflow = (ROOT / ".github/workflows/ci.yml").read_text().splitlines() + step = named_steps(yaml_block(workflow, "test-and-lint", 2))["Run ignored ILM integration tests serially"] + body = shell_body(step) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + commands = root / "bin" + commands.mkdir() + # Exercise the actual pipeline without building the workspace. + fake_timeout = commands / "timeout" + fake_timeout.write_text('#!/bin/sh\nshift 4\nexec "$@"\n') + fake_timeout.chmod(0o755) + cargo = commands / "cargo" + cargo.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > "$ILM_ARGS"\nexit "$ILM_STATUS"\n') + cargo.chmod(0o755) + report = root / "target/nextest/ci/junit.xml" + report.parent.mkdir(parents=True) + report.write_text("existing workspace report") + args = root / "args.txt" + for status in (0, 23, 124): + env = dict(os.environ, PATH=f"{commands}{os.pathsep}{os.environ['PATH']}", + ILM_ARGS=str(args), ILM_STATUS=str(status)) + result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", body], cwd=root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, status, result.stderr) + self.assertEqual(report.read_text(), "existing workspace report") + arguments = args.read_text().splitlines() + self.assertEqual(arguments[:9], ["nextest", "run", "--profile", "default", "-j1", "--run-ignored", "ignored-only", "--all", "--exclude"]) + self.assertEqual(arguments[9], "e2e_test") + self.assertIn(f"exit_status={status}", (root / "artifacts/ilm-integration/diagnostics.txt").read_text()) + def test_actual_selector_bootstrap_uses_base_policy_and_fails_closed(self): from check_test_wiring import yaml_block jobs = yaml_block((ROOT / ".github/workflows/ci.yml").read_text().splitlines(), "jobs", 0) From 38263ba896ea0eec601e91d4b26364416b5033df Mon Sep 17 00:00:00 2001 From: Hauser Date: Sun, 11 Oct 2026 20:46:44 +0800 Subject: [PATCH 010/143] fix(log-analyzer): match structured heal writer failures (#8469) Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/log-analyzer/src/rules/seed/heal.rs | 25 ++++++++++--- crates/log-analyzer/src/rules/seed/tests.rs | 40 +++++++++++++++++++++ 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/crates/log-analyzer/src/rules/seed/heal.rs b/crates/log-analyzer/src/rules/seed/heal.rs index b5564f49e..8957b97d6 100644 --- a/crates/log-analyzer/src/rules/seed/heal.rs +++ b/crates/log-analyzer/src/rules/seed/heal.rs @@ -15,7 +15,7 @@ //! Self-heal rules. use super::super::model::{Rule, Severity::*}; -use super::{any, base, contains, prefix, strings}; +use super::{all, any, base, contains, field, prefix, strings}; pub(super) fn rules() -> Vec { vec![ @@ -33,14 +33,29 @@ pub(super) fn rules() -> Vec { ) }, Rule { - anchors: strings(["all drives had write errors, unable to heal"]), + evidence_fields: strings([ + "bucket", + "object", + "version_id", + "target_count", + "writer_failure_count", + "error", + ]), + anchors: strings(["Set disk object heal writer failures"]), ..base( "heal-all-writes-failed", P0DataRisk, "heal", - "heal 结果无法写入任何盘", - contains("all drives had write errors, unable to heal"), - "heal 结果无法写入任何盘(全盘故障/满盘)。", + "所有自愈目标不可写", + any([ + contains("all drives had write errors, unable to heal"), + all([ + contains("Set disk object heal writer failures"), + field("state", "writer_unavailable"), + field("result", "all_targets_unavailable"), + ]), + ]), + "所有自愈目标不可写,修复无法继续;健康源盘可能仍可读。", "先解决容量/盘故障 finding,再重跑 heal。", ) }, diff --git a/crates/log-analyzer/src/rules/seed/tests.rs b/crates/log-analyzer/src/rules/seed/tests.rs index 88bf61a6a..c4bb4f744 100644 --- a/crates/log-analyzer/src/rules/seed/tests.rs +++ b/crates/log-analyzer/src/rules/seed/tests.rs @@ -297,6 +297,46 @@ fn every_rule_has_a_positive_sample() { } } +#[test] +fn heal_writer_rule_requires_all_targets_and_retains_legacy_messages() { + let engine = engine(); + for (sample, expected) in [ + ( + Sample { + message: "Set disk object heal writer failures", + level: Some(LogLevel::Warn), + fields: &[("state", "writer_unavailable"), ("result", "all_targets_unavailable")], + ..Default::default() + }, + true, + ), + ( + Sample { + message: "Set disk object heal writer failures", + level: Some(LogLevel::Warn), + fields: &[("state", "writer_unavailable"), ("result", "partial_targets_unavailable")], + ..Default::default() + }, + false, + ), + ( + Sample { + message: "Set disk object heal writer failures", + fields: &[("result", "all_targets_unavailable")], + ..Default::default() + }, + false, + ), + (msg("all drives had write errors, unable to heal b/o"), true), + ] { + let matched = engine + .matches(&event(&sample)) + .iter() + .any(|&index| engine.rules()[index].id == "heal-all-writes-failed"); + assert_eq!(matched, expected); + } +} + #[test] fn smoke_samples_hit_exact_rule_sets() { let engine = engine(); From 1324e296f8f1057a4ff9fc77db54386585931916 Mon Sep 17 00:00:00 2001 From: Hauser Date: Sun, 11 Oct 2026 22:43:08 +0800 Subject: [PATCH 011/143] test(scanner): bound expansion fixture preparation I/O (#8473) Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/scanner/src/scanner/backlog.rs | 112 ++++++++++++++++++++------ 1 file changed, 89 insertions(+), 23 deletions(-) diff --git a/crates/scanner/src/scanner/backlog.rs b/crates/scanner/src/scanner/backlog.rs index ba5df0feb..8bc63669a 100644 --- a/crates/scanner/src/scanner/backlog.rs +++ b/crates/scanner/src/scanner/backlog.rs @@ -2192,7 +2192,7 @@ mod tests { }) .expect("native retirement test thread") .join() - .expect("native retirement test completed"); + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)); } async fn native_retirement_store() -> (tempfile::TempDir, Arc) { @@ -2631,13 +2631,13 @@ mod tests { ) .await; eprintln!("native expansion fixture old_pools={old_pool_count} stage=seed_old_authority"); - let fault = unstable_source - .then(|| NativeScannerPauseBacklogWriteFault::fail_before_write(Arc::clone(&old.pools[0].disk_set[0]), "publish", 2)); let now = unix_now(); - let mut controller = ScannerPauseBacklogController::claim(Arc::clone(&old), now) - .await - .expect("the native controller commits the actual old cohort"); - if unstable_source { + let original = if unstable_source { + let fault = + NativeScannerPauseBacklogWriteFault::fail_before_write(Arc::clone(&old.pools[0].disk_set[0]), "publish", 2); + let controller = ScannerPauseBacklogController::claim(Arc::clone(&old), now) + .await + .expect("the native controller commits the actual old cohort"); assert!(controller.loaded.requires_reload); let (bytes, _) = native_replica_bytes(&old.pools[0].disk_set[0]).await; let ScannerPauseBacklogReplicaState::Valid(record) = decode_scanner_pause_backlog_ledger(&bytes) else { @@ -2645,23 +2645,14 @@ mod tests { }; assert!(record.stable.is_none()); assert_eq!(record.committed.unwrap().ledger, controller.loaded.ledger); + let original = controller.loaded.ledger.clone(); + assert!(controller.loaded.durable); + drop(controller); + drop(fault); + original } else { - controller.observe(observation(now + 1, true, 4)).await; - controller.observe(observation(now + 2, false, 4)).await; - let decision = controller.begin_attempt(now + 2).await; - assert!( - matches!(decision, ScannerPauseBacklogAttemptDecision::Tracked(_)), - "expected tracked native expansion setup attempt, got {decision:?}; ledger={:?}; persistence_disabled={}; runtime_error={:?}", - controller.loaded.ledger, - controller.persistence_disabled, - runtime_error() - ); - assert!(controller.loaded.ledger.has_unfinished_attempt()); - } - let original = controller.loaded.ledger.clone(); - assert!(controller.loaded.durable); - drop(controller); - drop(fault); + seed_native_expansion_ledger(&old, now).await + }; old.pool_meta_write_status() .await .expect("healthy pool metadata keeps the background recovery loop read-only"); @@ -2686,6 +2677,81 @@ mod tests { (root, expanded, original) } + async fn seed_native_expansion_ledger(store: &Arc, now: u64) -> ScannerPauseBacklogLedger { + let loaded = load_scanner_pause_backlog(Arc::clone(store)) + .await + .expect("empty native expansion cohort"); + assert!( + loaded + .replicas + .iter() + .all(|replica| matches!(replica.state, ScannerPauseBacklogReplicaState::Missing)) + ); + let mut ledger = claim_scanner_pause_backlog_writer(&loaded.ledger, now).expect("fixture writer generation"); + // The fixture needs the final unfinished attempt, not four separate + // durable controller transitions before the fault under test. Retain + // their generations and timestamps, then publish the final state once + // through the real native commit and stabilization protocol. + ledger.apply_observation(observation(now + 1, true, 4)); + prepare_scanner_pause_backlog_persist(&mut ledger, now + 1).expect("fixture pause generation"); + ledger.apply_observation(observation(now + 2, false, 4)); + prepare_scanner_pause_backlog_persist(&mut ledger, now + 2).expect("fixture resume generation"); + assert!(matches!(ledger.begin_attempt(now + 2), ScannerPauseBacklogAttemptDecision::Tracked(_))); + prepare_scanner_pause_backlog_persist(&mut ledger, now + 2).expect("fixture unfinished attempt generation"); + let published = persist_scanner_pause_backlog(Arc::clone(store), &loaded, ledger.clone()) + .await + .expect("publish the complete native expansion fixture"); + assert!(published.durable && published.stable_matches_ledger && !published.requires_reload); + assert!(ledger.has_unfinished_attempt()); + assert_current_native_writer_ledger(store, &ledger).await; + set_runtime_error(None); + ledger + } + + #[test] + #[serial_test::serial] + fn native_expansion_fixture_retains_controller_state_and_complete_disk_proof() { + run_native_retirement_test(async || { + use crate::storage_api::owner::NativeScannerPauseBacklogWriteFault; + + let (_reference_root, reference) = native_pause_backlog_store(1, 2).await; + let now = unix_now(); + let mut controller = ScannerPauseBacklogController::claim(Arc::clone(&reference), now) + .await + .expect("reference native writer claim"); + controller.observe(observation(now + 1, true, 4)).await; + controller.observe(observation(now + 2, false, 4)).await; + assert!(matches!( + controller.begin_attempt(now + 2).await, + ScannerPauseBacklogAttemptDecision::Tracked(_) + )); + let expected = controller.loaded.ledger.clone(); + assert_current_native_writer_ledger(&reference, &expected).await; + + let (_fixture_root, fixture) = native_pause_backlog_store(1, 2).await; + // A third preparation write would fail: the complete native commit + // and stabilization must suffice, without replaying the unrelated + // pause/resume transitions to disk in every recovery matrix cell. + let fault = + NativeScannerPauseBacklogWriteFault::fail_before_write(Arc::clone(&fixture.pools[0].disk_set[0]), "publish", 3); + let actual = seed_native_expansion_ledger(&fixture, now).await; + assert_eq!(actual, expected, "all generations, timestamps and unfinished work must be retained"); + assert_current_native_writer_ledger(&fixture, &expected).await; + drop(fault); + drop(controller); + shutdown_native_retirement_store(reference).await; + shutdown_native_retirement_store(fixture).await; + }); + } + + #[test] + #[should_panic(expected = "original native scenario failure")] + fn native_retirement_runner_preserves_scenario_failure_payload() { + run_native_retirement_test_with_budget(Duration::from_secs(1), async || { + panic!("original native scenario failure"); + }); + } + async fn native_expanded_retirement_store( old_pool_count: usize, unstable_source: bool, From 257c9f11aee78328a0ca586523b68fac95b34b3a Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 11 Oct 2026 07:43:28 -0700 Subject: [PATCH 012/143] docs: refresh security advisory lessons (#8472) --- .../references/advisory-patterns.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.agents/skills/security-advisory-lessons/references/advisory-patterns.md b/.agents/skills/security-advisory-lessons/references/advisory-patterns.md index fd8521534..98310295e 100644 --- a/.agents/skills/security-advisory-lessons/references/advisory-patterns.md +++ b/.agents/skills/security-advisory-lessons/references/advisory-patterns.md @@ -45,11 +45,13 @@ Update this file only when an advisory adds or changes a reusable lesson, affect - `GHSA-6r96-hmgc-726c`: request headers collided with lowercase server-derived condition keys such as `userid`, `groups`, `versionid`, and JWT/LDAP claims. Lesson: never let caller-controlled headers append to or replace server-derived policy context; reserve trusted condition keys and keep intentional request-header keys separate. - `GHSA-v9cp-qfw9-9pfp`: quantified negated string conditions applied negation after aggregation, transposing `ForAllValues` and `ForAnyValue` semantics. Lesson: push negation into the per-value predicate for quantified operators and test partially overlapping multi-value sets. - `GHSA-5w8r-p896-6vq2`: OPA policy mode skipped `ExistingObjectTag/*` loading, so tagged objects looked untagged to external policies. Lesson: external authorization plugins need the same object-tag and request context as built-in policy evaluation before they decide. +- `GHSA-9wgh-5r68-49p7`: substituted policy-variable values could recreate `${...}` and loop inside a single resolution pass, never reaching the outer iteration limit. Lesson: bound substitutions within each pass and test cycles formed across otherwise literal resolver values; confirm whether attacker-controlled claims can reach those values before assigning remote impact. ### S3 object actions, copy, multipart, and upload policy validation - `GHSA-g8w9-qw9q-fghr` and `GHSA-xm99-m3gq-83g8`: a valid presigned `PutObject` accepted extra `x-amz-*` headers omitted from `SignedHeaders`; `x-amz-copy-source` could turn that upload capability into a cross-bucket read performed as the signer. Lesson: a presigned URL is a bounded capability; inspect all received security-sensitive headers and reject unsigned ones before selecting the S3 operation or reaching storage. - `GHSA-3ppv-fx5m-m749`: explicit `versionId` reads and copy sources authorized `s3:GetObject` instead of `s3:GetObjectVersion`. Lesson: version-specific object access must select version-specific actions for direct reads, `CopyObject`, and `UploadPartCopy`, with tests proving the backend is not reached on denial. +- `GHSA-cg33-pxfp-qcj3` and `GHSA-65wq-gjw8-xr9w`: `HeadObject` and object-tagging paths authorized current-object actions while operating on an explicit historical `versionId`. Lesson: select the corresponding version action for metadata and tag reads and writes too; audit every operation that accepts a version ID. - `GHSA-gjph-xcgw-4p7w`: `PutObject` and browser POST accepted a caller-supplied `versionId` and replaced an unlocked historical version with only `s3:PutObject`. Lesson: only replication-authorized writes may choose an existing version ID; ordinary uploads must not rewrite version history. - `GHSA-x298-9x87-fvjq`: anonymous `ListObjectVersions` fell back to `ListBucket` and returned before public-access-block gates. Lesson: compatibility fallbacks must converge on the same post-authorization checks as direct grants, especially `RestrictPublicBuckets` and anonymous data-plane denies. - `GHSA-mx42-j6wv-px98`: `UploadPartCopy` missed source authorization and allowed cross-bucket object exfiltration. Lesson: multipart copy must enforce the same source and destination contract as `CopyObject`. @@ -110,6 +112,7 @@ Update this file only when an advisory adds or changes a reusable lesson, affect - `GHSA-j548-9grx-fh4f`: Object Lock enforcement treated unreadable, fabricated, or unparsable bucket metadata as absent configuration and allowed retained objects to be deleted or expired. Lesson: retention must fail closed unless Object Lock absence is authoritative, and every delete, lifecycle, scanner, force-delete, and default-retention path needs the same state distinction. - `GHSA-28w4-rp2p-636w` and `GHSA-c7hx-vr97-45rg`: an authorized `DeleteBucket` with `x-rustfs-force-delete: true` removed active COMPLIANCE-retained versions even though ordinary version deletion was denied. Lesson: bucket force-delete must enforce retention across every version, including against root/owner callers. +- `GHSA-q23w-fp7p-472v`: `PutObjectRetention` policy conditions read `x-amz-object-lock-*` headers while the handler applied mode and date from the XML body. Lesson: authorize the parsed retention that will be stored, or reject mismatched header/body values; compare retention dates as instants so offsets cannot change the applied bound. ### Serde deserialization and input validation @@ -143,13 +146,13 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates - IAM fixes: include import/update/list service-account cases with attacker-controlled parent, claims, access key, secret key, and policy. - Copy/upload fixes: include cross-bucket, cross-user, source-denied, destination-denied, copy-source-condition, and multipart completion cases. - Presigned upload fixes: include a valid presign with extra unsigned tagging, redirect, storage-class, and cross-bucket copy-source headers; require rejection before source or destination storage access, and verify explicitly signed equivalents still work. -- Version-action fixes: include historical UUID, explicit current version, `null`, range, partNumber, presigned, STS/session, service-account, anonymous bucket-policy, copy source, and multipart-copy source cases. +- Version-action fixes: include historical UUID, explicit current version, `null`, range, partNumber, presigned, STS/session, service-account, anonymous bucket-policy, copy source, multipart-copy source, `HeadObject`, and tag read/write cases. - Version-write fixes: a `PutObject`-only principal must not replace an existing version through `PutObject` or browser POST `?versionId=`; verify replication-authorized writes and versioned, suspended, and unversioned buckets separately. -- Policy-condition fixes: include reserved-key header collisions, missing keys, partially overlapping multi-value sets, plugin mode, and built-in policy mode. +- Policy-condition fixes: include reserved-key header collisions, missing keys, partially overlapping multi-value sets, cyclic variable substitutions, plugin mode, and built-in policy mode. - Path fixes: include encoded traversal, absolute path, nested traversal, archive entries with `..`, valid object keys that resemble traversal text but should be rejected, and canonical bucket/prefix boundary checks; test per-key `DeleteObjects` errors and prefix-scoped denials on versioned buckets. - Logging fixes: assert redacted output for structs and response bodies that may contain credentials. - IAM export fixes: assert exported archives omit plaintext user and service-account secrets unless the format deliberately encrypts or seals them. - RPC auth fixes: include captured metadata replay across two concrete methods, stale timestamps, wrong path, wrong method surrogate, wrong secret, and valid same-method calls. - Browser/CORS fixes: assert no credentials on reflected/default origins, correct behavior for explicit allowlists, and no same-origin script execution for previewed object content. - SSE fixes: inspect stored bytes and verify API metadata, read-back behavior, and on-disk ciphertext together; for ETags, upload equal plaintext under different keys and assert list-only callers cannot derive or correlate the plaintext digest, including multipart objects. -- Object Lock fixes: include unreadable metadata, fabricated metadata defaults, unparsable config, confirmed absent config, COMPLIANCE/GOVERNANCE retention, lifecycle expiry, scanner sweeps, and force-delete paths; an authorized force-delete must preserve active COMPLIANCE versions. +- Object Lock fixes: include unreadable metadata, fabricated metadata defaults, unparsable config, confirmed absent config, COMPLIANCE/GOVERNANCE retention, lifecycle expiry, scanner sweeps, and force-delete paths; an authorized force-delete must preserve active COMPLIANCE versions. For `PutObjectRetention`, test conflicting header/body values and offset dates against policy bounds and read-back retention. From 2d2679a7106b84fb433c636d1f45b8754dbe47f0 Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Sun, 11 Oct 2026 23:53:29 +0800 Subject: [PATCH 013/143] refactor(oidc): isolate config updates and replication view (#8446) --- crates/iam/src/oidc/config.rs | 443 ++++++++++- crates/iam/src/oidc/config_tests.rs | 146 ++++ crates/iam/src/oidc/mod.rs | 8 +- crates/iam/src/oidc/runtime.rs | 8 +- rustfs/src/admin/handlers/oidc.rs | 701 ++---------------- rustfs/src/admin/handlers/site_replication.rs | 38 +- rustfs/src/admin/service/mod.rs | 1 + rustfs/src/admin/service/oidc_config.rs | 158 ++++ rustfs/src/site_replication/tests.rs | 5 - rustfs/src/site_replication/transport.rs | 10 - 10 files changed, 816 insertions(+), 702 deletions(-) create mode 100644 rustfs/src/admin/service/oidc_config.rs diff --git a/crates/iam/src/oidc/config.rs b/crates/iam/src/oidc/config.rs index 6169ea60b..deceb35e1 100644 --- a/crates/iam/src/oidc/config.rs +++ b/crates/iam/src/oidc/config.rs @@ -11,12 +11,16 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +use base64_simd::URL_SAFE_NO_PAD; use rustfs_config::oidc::*; use rustfs_config::server_config::{Config as ServerConfig, KVS}; use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState}; +use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::fmt; +use url::Url; pub(super) const REDACTED_SECRET: &str = "***redacted***"; @@ -48,6 +52,162 @@ pub struct OidcProviderConfig { pub hide_from_ui: bool, } +#[derive(Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct OidcProviderConfigInput { + pub enabled: bool, + pub display_name: String, + pub config_url: String, + pub issuer: Option, + pub client_id: String, + pub client_secret: Option, + pub scopes: Vec, + pub other_audiences: Vec, + pub redirect_uri: Option, + pub redirect_uri_dynamic: bool, + pub claim_name: String, + pub claim_prefix: String, + pub role_policy: String, + pub groups_claim: String, + pub roles_claim: String, + pub email_claim: String, + pub username_claim: String, + pub hide_from_ui: bool, +} + +impl Default for OidcProviderConfigInput { + fn default() -> Self { + Self { + enabled: true, + display_name: String::new(), + config_url: String::new(), + issuer: None, + client_id: String::new(), + client_secret: None, + scopes: OIDC_DEFAULT_SCOPES.split(',').map(ToString::to_string).collect(), + other_audiences: Vec::new(), + redirect_uri: None, + redirect_uri_dynamic: true, + claim_name: OIDC_DEFAULT_CLAIM_NAME.to_string(), + claim_prefix: String::new(), + role_policy: String::new(), + groups_claim: OIDC_DEFAULT_GROUPS_CLAIM.to_string(), + roles_claim: OIDC_DEFAULT_ROLES_CLAIM.to_string(), + email_claim: OIDC_DEFAULT_EMAIL_CLAIM.to_string(), + username_claim: OIDC_DEFAULT_USERNAME_CLAIM.to_string(), + hide_from_ui: false, + } + } +} + +#[derive(Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct OidcProviderValidationInput { + pub provider_id: String, + pub enabled: bool, + pub display_name: String, + pub config_url: String, + pub issuer: Option, + pub client_id: String, + pub client_secret: Option, + pub scopes: Vec, + pub other_audiences: Vec, + pub redirect_uri: Option, + pub redirect_uri_dynamic: bool, + pub claim_name: String, + pub claim_prefix: String, + pub role_policy: String, + pub groups_claim: String, + pub roles_claim: String, + pub email_claim: String, + pub username_claim: String, + pub hide_from_ui: bool, +} + +impl Default for OidcProviderValidationInput { + fn default() -> Self { + let input = OidcProviderConfigInput::default(); + Self { + provider_id: "default".to_string(), + enabled: input.enabled, + display_name: input.display_name, + config_url: input.config_url, + issuer: input.issuer, + client_id: input.client_id, + client_secret: input.client_secret, + scopes: input.scopes, + other_audiences: input.other_audiences, + redirect_uri: input.redirect_uri, + redirect_uri_dynamic: input.redirect_uri_dynamic, + claim_name: input.claim_name, + claim_prefix: input.claim_prefix, + role_policy: input.role_policy, + groups_claim: input.groups_claim, + roles_claim: input.roles_claim, + email_claim: input.email_claim, + username_claim: input.username_claim, + hide_from_ui: input.hide_from_ui, + } + } +} + +impl From for (String, OidcProviderConfigInput) { + fn from(input: OidcProviderValidationInput) -> Self { + let provider_id = if input.provider_id.trim().is_empty() { + "default".to_string() + } else { + input.provider_id.trim().to_string() + }; + let config = OidcProviderConfigInput { + enabled: input.enabled, + display_name: input.display_name, + config_url: input.config_url, + issuer: input.issuer, + client_id: input.client_id, + client_secret: input.client_secret, + scopes: input.scopes, + other_audiences: input.other_audiences, + redirect_uri: input.redirect_uri, + redirect_uri_dynamic: input.redirect_uri_dynamic, + claim_name: input.claim_name, + claim_prefix: input.claim_prefix, + role_policy: input.role_policy, + groups_claim: input.groups_claim, + roles_claim: input.roles_claim, + email_claim: input.email_claim, + username_claim: input.username_claim, + hide_from_ui: input.hide_from_ui, + }; + (provider_id, config) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum OidcConfigError { + #[error("invalid provider_id")] + InvalidProviderId, + #[error("provider is managed by environment variables")] + EnvironmentManaged, + #[error("provider not found")] + ProviderNotFound, + #[error("config_url is required")] + ConfigUrlRequired, + #[error("client_id is required")] + ClientIdRequired, + #[error("redirect_uri is required when redirect_uri_dynamic is off")] + RedirectUriRequired, + #[error("scopes must include openid")] + OpenidScopeRequired, + #[error("{0} must be an absolute http/https URL")] + InvalidAbsoluteUrl(&'static str), + #[error("{field} is not allowed: {source}")] + ForbiddenOutbound { + field: &'static str, + #[source] + source: OutboundUrlError, + }, +} + impl fmt::Debug for OidcProviderConfig { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("OidcProviderConfig") @@ -122,12 +282,285 @@ pub struct OidcProviderValidationResult { /// Compatibility name for the provider summary returned by OIDC list APIs. pub(crate) type OidcProviderSummary = crate::federation::FederatedProviderView; -/// Read-only access to the active standard OIDC configuration. -/// -/// This interface stays separate from authentication and generic provider -/// views because its snapshots include OIDC-specific fields and secrets. +/// OIDC fields required to compare identity providers across replicated sites. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OidcSiteReplicationProvider { + pub provider_id: String, + pub claim_name: String, + pub role_policy: String, + pub client_id: String, + pub hashed_client_secret: String, +} + +/// Active OIDC settings exposed to site replication without raw credentials. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OidcSiteReplicationSnapshot { + providers: Vec, +} + +impl OidcSiteReplicationSnapshot { + pub fn from_config_snapshot(snapshot: &OidcConfigSnapshot) -> Self { + let providers = snapshot + .providers() + .iter() + .map(|provider| { + let config = &provider.config; + OidcSiteReplicationProvider { + provider_id: config.id.clone(), + claim_name: config.claim_name.clone(), + role_policy: config.role_policy.clone(), + client_id: config.client_id.clone(), + hashed_client_secret: hash_client_secret(config.client_secret.as_deref()), + } + }) + .collect(); + Self { providers } + } + + pub fn providers(&self) -> &[OidcSiteReplicationProvider] { + &self.providers + } +} + +fn hash_client_secret(secret: Option<&str>) -> String { + let Some(secret) = secret.filter(|secret| !secret.is_empty()) else { + return String::new(); + }; + + let mut hasher = Sha256::new(); + hasher.update(secret.as_bytes()); + URL_SAFE_NO_PAD.encode_to_string(hasher.finalize()) +} + +/// Read-only access to the active OIDC settings needed by site replication. pub trait OidcConfigQuery: Send + Sync { - fn config_snapshot(&self) -> OidcConfigSnapshot; + fn site_replication_snapshot(&self) -> OidcSiteReplicationSnapshot; +} + +pub fn validate_mutable_provider_id(provider_id: &str) -> Result<(), OidcConfigError> { + if !is_valid_provider_id(provider_id) { + return Err(OidcConfigError::InvalidProviderId); + } + if load_oidc_provider_configs_from_env() + .iter() + .any(|config| config.id == provider_id) + { + return Err(OidcConfigError::EnvironmentManaged); + } + Ok(()) +} + +fn is_valid_provider_id(id: &str) -> bool { + !id.is_empty() && id.chars().all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') +} + +fn provider_instance_key(provider_id: &str) -> String { + if provider_id == "default" { + DEFAULT_DELIMITER.to_string() + } else { + provider_id.to_string() + } +} + +fn normalize_optional(value: Option) -> Option { + value.map(|value| value.trim().to_string()).filter(|value| !value.is_empty()) +} + +fn or_default(value: &str, default: &str) -> String { + if value.trim().is_empty() { + default.to_string() + } else { + value.trim().to_string() + } +} + +fn validate_absolute_http_url(value: &str, field: &'static str, check_outbound: bool) -> Result<(), OidcConfigError> { + let parsed = Url::parse(value).map_err(|_| OidcConfigError::InvalidAbsoluteUrl(field))?; + if !matches!(parsed.scheme(), "http" | "https") || parsed.host_str().is_none() { + return Err(OidcConfigError::InvalidAbsoluteUrl(field)); + } + if check_outbound { + validate_outbound_url(&parsed).map_err(|source| OidcConfigError::ForbiddenOutbound { field, source })?; + } + Ok(()) +} + +fn normalize_provider_config(mut config: OidcProviderConfig) -> OidcProviderConfig { + config.config_url = config.config_url.trim().to_string(); + config.issuer = normalize_optional(config.issuer); + config.client_id = config.client_id.trim().to_string(); + config.scopes = config + .scopes + .iter() + .map(|scope| scope.trim().to_string()) + .filter(|scope| !scope.is_empty()) + .collect(); + config.redirect_uri = normalize_optional(config.redirect_uri); + config.claim_name = or_default(&config.claim_name, OIDC_DEFAULT_CLAIM_NAME); + config.claim_prefix = config.claim_prefix.trim().to_string(); + config.role_policy = config.role_policy.trim().to_string(); + config.display_name = or_default(&config.display_name, &config.id); + config.groups_claim = or_default(&config.groups_claim, OIDC_DEFAULT_GROUPS_CLAIM); + config.roles_claim = or_default(&config.roles_claim, OIDC_DEFAULT_ROLES_CLAIM); + config.email_claim = or_default(&config.email_claim, OIDC_DEFAULT_EMAIL_CLAIM); + config.username_claim = or_default(&config.username_claim, OIDC_DEFAULT_USERNAME_CLAIM); + config +} + +fn validate_provider_config_fields(config: &OidcProviderConfig) -> Result<(), OidcConfigError> { + if !is_valid_provider_id(&config.id) { + return Err(OidcConfigError::InvalidProviderId); + } + if config.config_url.trim().is_empty() { + return Err(OidcConfigError::ConfigUrlRequired); + } + validate_absolute_http_url(&config.config_url, "config_url", true)?; + if let Some(issuer) = config.issuer.as_deref() { + validate_absolute_http_url(issuer, "issuer", false)?; + } + if config.client_id.trim().is_empty() { + return Err(OidcConfigError::ClientIdRequired); + } + if !config.redirect_uri_dynamic { + let redirect_uri = config.redirect_uri.as_deref().ok_or(OidcConfigError::RedirectUriRequired)?; + validate_absolute_http_url(redirect_uri, "redirect_uri", true)?; + } else if let Some(redirect_uri) = config.redirect_uri.as_deref() { + validate_absolute_http_url(redirect_uri, "redirect_uri", true)?; + } + if !config.scopes.iter().any(|scope| scope == "openid") { + return Err(OidcConfigError::OpenidScopeRequired); + } + Ok(()) +} + +fn build_provider_config( + provider_id: &str, + input: OidcProviderConfigInput, + existing_secret: Option, +) -> Result { + let client_secret = match input.client_secret { + Some(value) if !value.trim().is_empty() => Some(value), + _ => existing_secret.filter(|value| !value.trim().is_empty()), + }; + let config = normalize_provider_config(OidcProviderConfig { + id: provider_id.to_string(), + enabled: input.enabled, + config_url: input.config_url, + issuer: input.issuer, + client_id: input.client_id, + client_secret, + scopes: input.scopes, + other_audiences: input.other_audiences, + redirect_uri: input.redirect_uri, + redirect_uri_dynamic: input.redirect_uri_dynamic, + claim_name: input.claim_name, + claim_prefix: input.claim_prefix, + role_policy: input.role_policy, + display_name: input.display_name, + groups_claim: input.groups_claim, + roles_claim: input.roles_claim, + email_claim: input.email_claim, + username_claim: input.username_claim, + hide_from_ui: input.hide_from_ui, + }); + validate_provider_config_fields(&config)?; + Ok(config) +} + +pub fn build_upsert_provider_config( + provider_id: &str, + input: OidcProviderConfigInput, + existing_secret: Option, +) -> Result { + build_provider_config(provider_id, input, existing_secret) +} + +pub fn build_validation_provider_config(input: OidcProviderValidationInput) -> Result { + let (provider_id, input) = input.into(); + build_provider_config(&provider_id, input, None) +} + +pub fn persisted_provider_secret(config: &ServerConfig, provider_id: &str) -> Option { + config + .0 + .get(IDENTITY_OPENID_SUB_SYS) + .and_then(|subsystem| subsystem.get(&provider_instance_key(provider_id))) + .and_then(|kvs| kvs.lookup(OIDC_CLIENT_SECRET)) + .filter(|value| !value.trim().is_empty()) +} + +fn set_kvs_value(kvs: &mut KVS, key: &str, value: String) { + if let Some(existing) = kvs.0.iter_mut().find(|kv| kv.key == key) { + existing.value = value; + return; + } + kvs.insert(key.to_string(), value); +} + +pub fn upsert_persisted_provider_config(config: &mut ServerConfig, provider: &OidcProviderConfig) { + let mut kvs = ServerConfig::new() + .get_value(IDENTITY_OPENID_SUB_SYS, DEFAULT_DELIMITER) + .unwrap_or_default(); + set_kvs_value( + &mut kvs, + ENABLE_KEY, + if provider.enabled { + EnableState::On.to_string() + } else { + EnableState::Off.to_string() + }, + ); + set_kvs_value(&mut kvs, OIDC_CONFIG_URL, provider.config_url.clone()); + set_kvs_value(&mut kvs, OIDC_ISSUER, provider.issuer.clone().unwrap_or_default()); + set_kvs_value(&mut kvs, OIDC_CLIENT_ID, provider.client_id.clone()); + set_kvs_value(&mut kvs, OIDC_CLIENT_SECRET, provider.client_secret.clone().unwrap_or_default()); + set_kvs_value(&mut kvs, OIDC_SCOPES, provider.scopes.join(",")); + set_kvs_value(&mut kvs, OIDC_OTHER_AUDIENCES, provider.other_audiences.join(",")); + set_kvs_value(&mut kvs, OIDC_REDIRECT_URI, provider.redirect_uri.clone().unwrap_or_default()); + set_kvs_value( + &mut kvs, + OIDC_REDIRECT_URI_DYNAMIC, + if provider.redirect_uri_dynamic { + EnableState::On.to_string() + } else { + EnableState::Off.to_string() + }, + ); + set_kvs_value(&mut kvs, OIDC_CLAIM_NAME, provider.claim_name.clone()); + set_kvs_value(&mut kvs, OIDC_CLAIM_PREFIX, provider.claim_prefix.clone()); + set_kvs_value(&mut kvs, OIDC_ROLE_POLICY, provider.role_policy.clone()); + set_kvs_value(&mut kvs, OIDC_DISPLAY_NAME, provider.display_name.clone()); + set_kvs_value(&mut kvs, OIDC_GROUPS_CLAIM, provider.groups_claim.clone()); + set_kvs_value(&mut kvs, OIDC_ROLES_CLAIM, provider.roles_claim.clone()); + set_kvs_value(&mut kvs, OIDC_EMAIL_CLAIM, provider.email_claim.clone()); + set_kvs_value(&mut kvs, OIDC_USERNAME_CLAIM, provider.username_claim.clone()); + set_kvs_value( + &mut kvs, + OIDC_HIDE_FROM_UI, + if provider.hide_from_ui { + EnableState::On.to_string() + } else { + EnableState::Off.to_string() + }, + ); + config + .0 + .entry(IDENTITY_OPENID_SUB_SYS.to_string()) + .or_default() + .insert(provider_instance_key(&provider.id), kvs); +} + +pub fn delete_persisted_provider_config(config: &mut ServerConfig, provider_id: &str) -> Result<(), OidcConfigError> { + let Some(subsystem) = config.0.get_mut(IDENTITY_OPENID_SUB_SYS) else { + return Err(OidcConfigError::ProviderNotFound); + }; + if subsystem.remove(&provider_instance_key(provider_id)).is_none() { + return Err(OidcConfigError::ProviderNotFound); + } + if subsystem.is_empty() { + config.0.remove(IDENTITY_OPENID_SUB_SYS); + } + Ok(()) } /// Parse all OIDC provider configs from environment variables. diff --git a/crates/iam/src/oidc/config_tests.rs b/crates/iam/src/oidc/config_tests.rs index 70936545a..7f79c7a96 100644 --- a/crates/iam/src/oidc/config_tests.rs +++ b/crates/iam/src/oidc/config_tests.rs @@ -305,3 +305,149 @@ fn test_oidc_provider_config_defaults() { assert_eq!(config.scopes.len(), 3); assert!(config.redirect_uri_dynamic); } + +#[test] +fn config_input_preserves_secret_and_defaults() { + let input = OidcProviderConfigInput { + config_url: " https://example.com/.well-known/openid-configuration ".to_string(), + client_id: " client ".to_string(), + client_secret: Some(String::new()), + scopes: vec![" openid ".to_string(), " profile ".to_string()], + roles_claim: "app_roles".to_string(), + ..Default::default() + }; + let config = build_upsert_provider_config("default", input, Some("existing-secret".to_string())) + .expect("valid config should preserve existing secret"); + + assert_eq!(config.client_secret.as_deref(), Some("existing-secret")); + assert_eq!(config.config_url, "https://example.com/.well-known/openid-configuration"); + assert_eq!(config.client_id, "client"); + assert_eq!(config.scopes, ["openid", "profile"]); + assert_eq!(config.roles_claim, "app_roles"); + assert_eq!(config.claim_name, OIDC_DEFAULT_CLAIM_NAME); + + let raw_secret = build_upsert_provider_config( + "default", + OidcProviderConfigInput { + config_url: config.config_url, + client_id: config.client_id, + client_secret: Some(" raw secret ".to_string()), + ..Default::default() + }, + None, + ) + .expect("a nonempty client secret should be accepted"); + assert_eq!(raw_secret.client_secret.as_deref(), Some(" raw secret ")); +} + +#[test] +fn config_input_rejects_invalid_fields() { + let input = OidcProviderConfigInput { + config_url: "https://example.com/.well-known/openid-configuration".to_string(), + client_id: "client".to_string(), + scopes: vec!["profile".to_string()], + ..Default::default() + }; + assert!(matches!( + build_upsert_provider_config("default", input, None), + Err(OidcConfigError::OpenidScopeRequired) + )); + + let input = OidcProviderConfigInput { + config_url: "https://127.0.0.1/.well-known/openid-configuration".to_string(), + client_id: "client".to_string(), + ..Default::default() + }; + assert!(matches!( + build_upsert_provider_config("default", input, None), + Err(OidcConfigError::ForbiddenOutbound { field: "config_url", .. }) + )); + + assert!(serde_json::from_str::(r#"{"unexpected_field":true}"#).is_err()); + assert!(serde_json::from_str::(r#"{"unexpected_field":true}"#).is_err()); + assert!(matches!( + validate_mutable_provider_id("../escape"), + Err(OidcConfigError::InvalidProviderId) + )); +} + +#[test] +fn validation_input_defaults_to_default_provider() { + let input = OidcProviderValidationInput { + provider_id: " ".to_string(), + config_url: "https://example.com/.well-known/openid-configuration".to_string(), + client_id: "client".to_string(), + ..Default::default() + }; + let config = build_validation_provider_config(input).expect("validation input should use default provider"); + assert_eq!(config.id, "default"); +} + +#[test] +fn persisted_config_codec_keeps_instance_keys_and_fields() { + let mut config = ServerConfig::new(); + let mut provider = build_upsert_provider_config( + "default", + OidcProviderConfigInput { + config_url: "https://example.com/.well-known/openid-configuration".to_string(), + issuer: Some("https://issuer.example.com".to_string()), + client_id: "client".to_string(), + client_secret: Some("secret".to_string()), + hide_from_ui: true, + ..Default::default() + }, + None, + ) + .expect("provider should be valid"); + upsert_persisted_provider_config(&mut config, &provider); + + let default = config + .0 + .get(IDENTITY_OPENID_SUB_SYS) + .and_then(|subsystem| subsystem.get(DEFAULT_DELIMITER)) + .expect("default provider KVS should exist"); + assert_eq!(default.get(OIDC_ISSUER), "https://issuer.example.com"); + assert_eq!(default.get(OIDC_HIDE_FROM_UI), EnableState::On.to_string()); + assert_eq!(persisted_provider_secret(&config, "default").as_deref(), Some("secret")); + + provider.hide_from_ui = false; + upsert_persisted_provider_config(&mut config, &provider); + let default = config + .0 + .get(IDENTITY_OPENID_SUB_SYS) + .and_then(|subsystem| subsystem.get(DEFAULT_DELIMITER)) + .expect("updated default provider KVS should exist"); + assert_eq!(default.get(OIDC_HIDE_FROM_UI), EnableState::Off.to_string()); + + delete_persisted_provider_config(&mut config, "default").expect("default provider should be deleted"); + assert!(matches!( + delete_persisted_provider_config(&mut config, "default"), + Err(OidcConfigError::ProviderNotFound) + )); +} + +#[test] +fn site_replication_snapshot_hashes_secret_without_exposing_config() { + let provider = build_upsert_provider_config( + "default", + OidcProviderConfigInput { + config_url: "https://example.com/.well-known/openid-configuration".to_string(), + client_id: "client".to_string(), + client_secret: Some("secret".to_string()), + ..Default::default() + }, + None, + ) + .expect("provider should be valid"); + let snapshot = OidcSiteReplicationSnapshot::from_config_snapshot(&OidcConfigSnapshot::new(vec![SourcedOidcProviderConfig { + config: provider, + source: OidcProviderConfigSource::Persisted, + }])); + assert_eq!( + snapshot.providers()[0].hashed_client_secret, + "K7gNU3sdo-OL0wNhqoVWhr3g6s1xYv72ol_pe_Unols" + ); + assert!(!format!("{snapshot:?}").contains("\"secret\"")); + assert_eq!(hash_client_secret(None), ""); + assert_eq!(hash_client_secret(Some("")), ""); +} diff --git a/crates/iam/src/oidc/mod.rs b/crates/iam/src/oidc/mod.rs index 4acfb6364..7452dcd19 100644 --- a/crates/iam/src/oidc/mod.rs +++ b/crates/iam/src/oidc/mod.rs @@ -23,9 +23,11 @@ mod transport; mod test_support; pub use config::{ - OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderConfigSource, OidcProviderValidationResult, - SourcedOidcProviderConfig, load_oidc_config_snapshot, load_oidc_provider_configs_from_env, - load_oidc_provider_configs_from_server_config, + OidcConfigError, OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderConfigInput, OidcProviderConfigSource, + OidcProviderValidationInput, OidcProviderValidationResult, OidcSiteReplicationProvider, OidcSiteReplicationSnapshot, + SourcedOidcProviderConfig, build_upsert_provider_config, build_validation_provider_config, delete_persisted_provider_config, + load_oidc_config_snapshot, load_oidc_provider_configs_from_env, load_oidc_provider_configs_from_server_config, + persisted_provider_secret, upsert_persisted_provider_config, validate_mutable_provider_id, }; pub use provider::validate_oidc_provider_config_with_extra_root_ca; pub use runtime::{OidcSys, StandardOidcAdapter}; diff --git a/crates/iam/src/oidc/runtime.rs b/crates/iam/src/oidc/runtime.rs index 01cff8046..3a5797fc1 100644 --- a/crates/iam/src/oidc/runtime.rs +++ b/crates/iam/src/oidc/runtime.rs @@ -13,8 +13,8 @@ // limitations under the License. use super::{ config::{ - OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderSummary, SourcedOidcProviderConfig, - load_oidc_config_snapshot, + OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderSummary, OidcSiteReplicationSnapshot, + SourcedOidcProviderConfig, load_oidc_config_snapshot, }, provider::{ProviderRuntime, discover_provider}, state::{OidcAuthSession, OidcLogoutSession, OidcStateStore}, @@ -806,8 +806,8 @@ impl StandardOidcAdapter { } impl OidcConfigQuery for StandardOidcAdapter { - fn config_snapshot(&self) -> OidcConfigSnapshot { - self.oidc.config_snapshot() + fn site_replication_snapshot(&self) -> OidcSiteReplicationSnapshot { + OidcSiteReplicationSnapshot::from_config_snapshot(&self.oidc.config_snapshot()) } } diff --git a/rustfs/src/admin/handlers/oidc.rs b/rustfs/src/admin/handlers/oidc.rs index b5183ce05..8a095d2e1 100644 --- a/rustfs/src/admin/handlers/oidc.rs +++ b/rustfs/src/admin/handlers/oidc.rs @@ -13,36 +13,22 @@ // limitations under the License. use crate::admin::auth::authorize_admin_request; -use crate::admin::handlers::supervise_admin_mutation; use crate::admin::router::{AdminOperation, Operation, S3Router}; -use crate::admin::runtime_sources::{ - current_app_context, current_federated_identity_service, current_object_store_handle_for_context, - current_server_config_for_context, -}; +use crate::admin::runtime_sources::current_federated_identity_service; use crate::admin::service::federated_identity::DefaultFederatedSessionBinding; -use crate::admin::storage_api::config::{ - read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot, -}; +use crate::admin::service::oidc_config; use crate::admin::utils::json_response; use crate::server::{ADMIN_PREFIX, MINIO_ADMIN_PREFIX, console_prefix}; use http::StatusCode; use hyper::Method; use matchit::Params; -use rustfs_config::oidc::{ - IDENTITY_OPENID_SUB_SYS, OIDC_CLAIM_NAME, OIDC_CLAIM_PREFIX, OIDC_CLIENT_ID, OIDC_CLIENT_SECRET, OIDC_CONFIG_URL, - OIDC_DEFAULT_CLAIM_NAME, OIDC_DEFAULT_EMAIL_CLAIM, OIDC_DEFAULT_GROUPS_CLAIM, OIDC_DEFAULT_ROLES_CLAIM, OIDC_DEFAULT_SCOPES, - OIDC_DEFAULT_USERNAME_CLAIM, OIDC_DISPLAY_NAME, OIDC_EMAIL_CLAIM, OIDC_GROUPS_CLAIM, OIDC_HIDE_FROM_UI, OIDC_ISSUER, - OIDC_OTHER_AUDIENCES, OIDC_REDIRECT_URI, OIDC_REDIRECT_URI_DYNAMIC, OIDC_ROLE_POLICY, OIDC_ROLES_CLAIM, OIDC_SCOPES, - OIDC_USERNAME_CLAIM, -}; -use rustfs_config::server_config::Config as ServerConfig; -use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_RUSTFS_BROWSER_REDIRECT_URL, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE}; +use rustfs_config::{ENV_RUSTFS_BROWSER_REDIRECT_URL, MAX_ADMIN_REQUEST_BODY_SIZE}; use rustfs_iam::federation::{FederatedSessionBindingError, FederationError}; +use rustfs_iam::oidc::{OidcProviderConfigInput, OidcProviderValidationInput}; use rustfs_policy::policy::action::{Action, AdminAction}; -use rustfs_utils::egress::validate_outbound_url; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; +use serde::Serialize; use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use tracing::{debug, error, warn}; use url::Url; @@ -78,6 +64,19 @@ fn callback_federation_error(error: FederationError) -> S3Error { } } +fn config_service_error(error: oidc_config::OidcAdminConfigError) -> S3Error { + use oidc_config::OidcAdminConfigError; + let code = match &error { + OidcAdminConfigError::Config(rustfs_iam::oidc::OidcConfigError::EnvironmentManaged) => S3ErrorCode::AccessDenied, + OidcAdminConfigError::Config(_) | OidcAdminConfigError::Validation(_) => S3ErrorCode::InvalidRequest, + OidcAdminConfigError::StorageUnavailable + | OidcAdminConfigError::Load(_) + | OidcAdminConfigError::Save(_) + | OidcAdminConfigError::Task(_) => S3ErrorCode::InternalError, + }; + S3Error::with_message(code, error.to_string()) +} + /// Validate that a provider ID contains only safe characters (alphanumeric, underscore, hyphen). fn is_valid_provider_id(id: &str) -> bool { !id.is_empty() && id.chars().all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') @@ -198,104 +197,6 @@ struct OidcValidationResponse { token_endpoint: Option, } -#[derive(Debug, Deserialize)] -#[serde(default, deny_unknown_fields)] -struct OidcConfigUpsertRequest { - enabled: bool, - display_name: String, - config_url: String, - issuer: Option, - client_id: String, - client_secret: Option, - scopes: Vec, - other_audiences: Vec, - redirect_uri: Option, - redirect_uri_dynamic: bool, - claim_name: String, - claim_prefix: String, - role_policy: String, - groups_claim: String, - roles_claim: String, - email_claim: String, - username_claim: String, - hide_from_ui: bool, -} - -impl Default for OidcConfigUpsertRequest { - fn default() -> Self { - Self { - enabled: true, - display_name: String::new(), - config_url: String::new(), - issuer: None, - client_id: String::new(), - client_secret: None, - scopes: OIDC_DEFAULT_SCOPES.split(',').map(ToString::to_string).collect(), - other_audiences: Vec::new(), - redirect_uri: None, - redirect_uri_dynamic: true, - claim_name: OIDC_DEFAULT_CLAIM_NAME.to_string(), - claim_prefix: String::new(), - role_policy: String::new(), - groups_claim: OIDC_DEFAULT_GROUPS_CLAIM.to_string(), - roles_claim: OIDC_DEFAULT_ROLES_CLAIM.to_string(), - email_claim: OIDC_DEFAULT_EMAIL_CLAIM.to_string(), - username_claim: OIDC_DEFAULT_USERNAME_CLAIM.to_string(), - hide_from_ui: false, - } - } -} - -#[derive(Debug, Deserialize)] -#[serde(default, deny_unknown_fields)] -struct OidcConfigValidateRequest { - provider_id: String, - enabled: bool, - display_name: String, - config_url: String, - issuer: Option, - client_id: String, - client_secret: Option, - scopes: Vec, - other_audiences: Vec, - redirect_uri: Option, - redirect_uri_dynamic: bool, - claim_name: String, - claim_prefix: String, - role_policy: String, - groups_claim: String, - roles_claim: String, - email_claim: String, - username_claim: String, - hide_from_ui: bool, -} - -impl Default for OidcConfigValidateRequest { - fn default() -> Self { - Self { - provider_id: "default".to_string(), - enabled: true, - display_name: String::new(), - config_url: String::new(), - issuer: None, - client_id: String::new(), - client_secret: None, - scopes: OIDC_DEFAULT_SCOPES.split(',').map(ToString::to_string).collect(), - other_audiences: Vec::new(), - redirect_uri: None, - redirect_uri_dynamic: true, - claim_name: OIDC_DEFAULT_CLAIM_NAME.to_string(), - claim_prefix: String::new(), - role_policy: String::new(), - groups_claim: OIDC_DEFAULT_GROUPS_CLAIM.to_string(), - roles_claim: OIDC_DEFAULT_ROLES_CLAIM.to_string(), - email_claim: OIDC_DEFAULT_EMAIL_CLAIM.to_string(), - username_claim: OIDC_DEFAULT_USERNAME_CLAIM.to_string(), - hide_from_ui: false, - } - } -} - /// Handler: GET /rustfs/admin/v3/oidc/providers /// Returns list of configured OIDC providers for the login page. pub struct ListOidcProvidersHandler {} @@ -323,9 +224,9 @@ impl Operation for GetOidcConfigHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { authorize_oidc_config_request(&req, AdminAction::ServerInfoAdminAction).await?; - let config = load_server_config_from_store().await?; - let restart_required = oidc_restart_required(&config); - let providers = rustfs_iam::oidc::load_oidc_config_snapshot(Some(&config)) + let config = oidc_config::list_config().await.map_err(config_service_error)?; + let providers = config + .snapshot .into_providers() .into_iter() .map(|provider| OidcConfigView { @@ -357,7 +258,7 @@ impl Operation for GetOidcConfigHandler { StatusCode::OK, &OidcConfigListResponse { providers, - restart_required, + restart_required: config.restart_required, }, ) } @@ -373,22 +274,13 @@ impl Operation for PutOidcConfigHandler { let provider_id = params .get("provider_id") .ok_or_else(|| s3_error!(InvalidRequest, "missing provider_id"))?; - if !is_valid_provider_id(provider_id) { - return Err(s3_error!(InvalidRequest, "invalid provider_id")); - } - if is_env_managed_provider(provider_id) { - return Err(s3_error!(AccessDenied, "provider is managed by environment variables")); - } + oidc_config::ensure_mutable_provider_id(provider_id).map_err(config_service_error)?; let provider_id = provider_id.to_owned(); - let request: OidcConfigUpsertRequest = parse_json_body(&mut req).await?; - update_oidc_server_config(move |config| { - let existing_secret = persisted_provider_secret(config, &provider_id); - let provider_config = build_provider_config_from_upsert(&provider_id, request, existing_secret)?; - upsert_persisted_provider_config(config, &provider_config); - Ok(()) - }) - .await?; + let request: OidcProviderConfigInput = parse_json_body(&mut req).await?; + oidc_config::upsert_config(provider_id, request) + .await + .map_err(config_service_error)?; json_response( StatusCode::OK, @@ -411,19 +303,9 @@ impl Operation for DeleteOidcConfigHandler { let provider_id = params .get("provider_id") .ok_or_else(|| s3_error!(InvalidRequest, "missing provider_id"))?; - if !is_valid_provider_id(provider_id) { - return Err(s3_error!(InvalidRequest, "invalid provider_id")); - } - if is_env_managed_provider(provider_id) { - return Err(s3_error!(AccessDenied, "provider is managed by environment variables")); - } + oidc_config::ensure_mutable_provider_id(provider_id).map_err(config_service_error)?; let provider_id = provider_id.to_owned(); - - update_oidc_server_config(move |config| { - delete_persisted_provider_config(config, &provider_id)?; - Ok(()) - }) - .await?; + oidc_config::delete_config(provider_id).await.map_err(config_service_error)?; json_response( StatusCode::OK, @@ -443,22 +325,8 @@ impl Operation for ValidateOidcConfigHandler { async fn call(&self, mut req: S3Request, _params: Params<'_, '_>) -> S3Result> { authorize_oidc_config_request(&req, AdminAction::ServerInfoAdminAction).await?; - let request: OidcConfigValidateRequest = parse_json_body(&mut req).await?; - let provider_id = if request.provider_id.trim().is_empty() { - "default".to_string() - } else { - request.provider_id.trim().to_string() - }; - let provider_config = build_provider_config_from_validate(request, &provider_id)?; - let oidc_extra_root_ca = crate::startup_auth::current_oidc_extra_root_ca_material() - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("validation failed: {e}")))?; - let validation = rustfs_iam::oidc::validate_oidc_provider_config_with_extra_root_ca( - &provider_config, - oidc_extra_root_ca.root_ca_pem.as_deref(), - ) - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("validation failed: {e}")))?; + let request: OidcProviderValidationInput = parse_json_body(&mut req).await?; + let validation = oidc_config::validate_config(request).await.map_err(config_service_error)?; json_response( StatusCode::OK, @@ -877,316 +745,6 @@ async fn parse_json_body(req: &mut S3Request) -> S3Re serde_json::from_slice(&body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e)) } -async fn load_server_config_from_store() -> S3Result { - let store = oidc_config_store()?; - - read_admin_config_without_migrate(store) - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}"))) -} - -fn oidc_config_store() -> S3Result> { - let context = current_app_context(); - current_object_store_handle_for_context(context.as_deref()) - .ok_or_else(|| s3_error!(InternalError, "storage layer not initialized")) -} - -async fn update_oidc_server_config(modifier: F) -> S3Result<()> -where - F: FnOnce(&mut ServerConfig) -> S3Result<()> + Send + 'static, -{ - let store = oidc_config_store()?; - supervise_admin_mutation("OIDC config update", async move { - let snapshot = read_admin_server_config_snapshot(store.clone()) - .await - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))?; - let mut config = snapshot.config.clone(); - modifier(&mut config)?; - save_admin_server_config_snapshot(store, &config, &snapshot) - .await - .map(|_| ()) - .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to save server config: {e}"))) - }) - .await -} - -fn is_env_managed_provider(provider_id: &str) -> bool { - rustfs_iam::oidc::load_oidc_provider_configs_from_env() - .iter() - .any(|config| config.id == provider_id) -} - -fn provider_instance_key(provider_id: &str) -> String { - if provider_id == "default" { - DEFAULT_DELIMITER.to_string() - } else { - provider_id.to_string() - } -} - -fn oidc_restart_required(config: &ServerConfig) -> bool { - let context = current_app_context(); - let active_config = current_server_config_for_context(context.as_deref()); - oidc_restart_required_from_active_config(config, active_config.as_ref()) -} - -fn oidc_restart_required_from_active_config(config: &ServerConfig, active_config: Option<&ServerConfig>) -> bool { - rustfs_iam::oidc::load_oidc_config_snapshot(Some(config)) != rustfs_iam::oidc::load_oidc_config_snapshot(active_config) -} - -fn default_oidc_kvs() -> s3s::S3Result { - ServerConfig::new() - .get_value(IDENTITY_OPENID_SUB_SYS, DEFAULT_DELIMITER) - .ok_or_else(|| s3_error!(InternalError, "default OIDC configuration missing")) -} - -fn set_kvs_value(kvs: &mut rustfs_config::server_config::KVS, key: &str, value: String) { - if let Some(existing) = kvs.0.iter_mut().find(|kv| kv.key == key) { - existing.value = value; - return; - } - - kvs.insert(key.to_string(), value); -} - -fn normalize_scopes(scopes: &[String]) -> Vec { - scopes - .iter() - .map(|scope| scope.trim().to_string()) - .filter(|scope| !scope.is_empty()) - .collect() -} - -fn normalize_optional(value: Option) -> Option { - value.map(|v| v.trim().to_string()).filter(|v| !v.is_empty()) -} - -fn validate_absolute_http_url(value: &str, field_name: &str) -> S3Result<()> { - let parsed = Url::parse(value).map_err(|_| s3_error!(InvalidRequest, "{} must be an absolute http/https URL", field_name))?; - - if !is_valid_scheme(parsed.scheme()) || parsed.host_str().is_none() { - return Err(s3_error!(InvalidRequest, "{} must be an absolute http/https URL", field_name)); - } - - validate_outbound_url(&parsed).map_err(|err| s3_error!(InvalidRequest, "{} is not allowed: {}", field_name, err))?; - - Ok(()) -} - -fn validate_absolute_http_url_without_outbound_check(value: &str, field_name: &str) -> S3Result<()> { - let parsed = Url::parse(value).map_err(|_| s3_error!(InvalidRequest, "{} must be an absolute http/https URL", field_name))?; - - if !is_valid_scheme(parsed.scheme()) || parsed.host_str().is_none() { - return Err(s3_error!(InvalidRequest, "{} must be an absolute http/https URL", field_name)); - } - - Ok(()) -} - -fn validate_provider_config_fields(config: &rustfs_iam::oidc::OidcProviderConfig) -> S3Result<()> { - if !is_valid_provider_id(&config.id) { - return Err(s3_error!(InvalidRequest, "invalid provider_id")); - } - if config.config_url.trim().is_empty() { - return Err(s3_error!(InvalidRequest, "config_url is required")); - } - validate_absolute_http_url(&config.config_url, "config_url")?; - if let Some(issuer) = config.issuer.as_deref() { - validate_absolute_http_url_without_outbound_check(issuer, "issuer")?; - } - - if config.client_id.trim().is_empty() { - return Err(s3_error!(InvalidRequest, "client_id is required")); - } - - if !config.redirect_uri_dynamic { - let redirect_uri = config - .redirect_uri - .as_deref() - .ok_or_else(|| s3_error!(InvalidRequest, "redirect_uri is required when redirect_uri_dynamic is off"))?; - validate_absolute_http_url(redirect_uri, "redirect_uri")?; - } else if let Some(redirect_uri) = config.redirect_uri.as_deref() { - validate_absolute_http_url(redirect_uri, "redirect_uri")?; - } - - if !config.scopes.iter().any(|scope| scope == "openid") { - return Err(s3_error!(InvalidRequest, "scopes must include openid")); - } - - Ok(()) -} - -fn or_default(value: &str, default: &str) -> String { - if value.trim().is_empty() { - default.to_string() - } else { - value.trim().to_string() - } -} - -/// Normalize an `OidcProviderConfig` by trimming strings and applying defaults. -fn normalize_provider_config(mut config: rustfs_iam::oidc::OidcProviderConfig) -> rustfs_iam::oidc::OidcProviderConfig { - config.config_url = config.config_url.trim().to_string(); - config.issuer = normalize_optional(config.issuer); - config.client_id = config.client_id.trim().to_string(); - config.scopes = normalize_scopes(&config.scopes); - config.redirect_uri = normalize_optional(config.redirect_uri); - config.claim_name = or_default(&config.claim_name, OIDC_DEFAULT_CLAIM_NAME); - config.claim_prefix = config.claim_prefix.trim().to_string(); - config.role_policy = config.role_policy.trim().to_string(); - config.display_name = or_default(&config.display_name, &config.id); - config.groups_claim = or_default(&config.groups_claim, OIDC_DEFAULT_GROUPS_CLAIM); - config.roles_claim = or_default(&config.roles_claim, OIDC_DEFAULT_ROLES_CLAIM); - config.email_claim = or_default(&config.email_claim, OIDC_DEFAULT_EMAIL_CLAIM); - config.username_claim = or_default(&config.username_claim, OIDC_DEFAULT_USERNAME_CLAIM); - config -} - -fn build_provider_config_from_upsert( - provider_id: &str, - request: OidcConfigUpsertRequest, - existing_secret: Option, -) -> S3Result { - let client_secret = match request.client_secret { - Some(value) if !value.trim().is_empty() => Some(value), - _ => existing_secret.filter(|value| !value.trim().is_empty()), - }; - - let config = normalize_provider_config(rustfs_iam::oidc::OidcProviderConfig { - id: provider_id.to_string(), - enabled: request.enabled, - config_url: request.config_url, - issuer: request.issuer, - client_id: request.client_id, - client_secret, - scopes: request.scopes, - other_audiences: request.other_audiences, - redirect_uri: request.redirect_uri, - redirect_uri_dynamic: request.redirect_uri_dynamic, - claim_name: request.claim_name, - claim_prefix: request.claim_prefix, - role_policy: request.role_policy, - display_name: request.display_name, - groups_claim: request.groups_claim, - roles_claim: request.roles_claim, - email_claim: request.email_claim, - username_claim: request.username_claim, - hide_from_ui: request.hide_from_ui, - }); - - validate_provider_config_fields(&config)?; - Ok(config) -} - -fn build_provider_config_from_validate( - request: OidcConfigValidateRequest, - provider_id: &str, -) -> S3Result { - let config = normalize_provider_config(rustfs_iam::oidc::OidcProviderConfig { - id: provider_id.to_string(), - enabled: request.enabled, - config_url: request.config_url, - issuer: request.issuer, - client_id: request.client_id, - client_secret: request.client_secret.filter(|value| !value.trim().is_empty()), - scopes: request.scopes, - other_audiences: request.other_audiences, - redirect_uri: request.redirect_uri, - redirect_uri_dynamic: request.redirect_uri_dynamic, - claim_name: request.claim_name, - claim_prefix: request.claim_prefix, - role_policy: request.role_policy, - display_name: request.display_name, - groups_claim: request.groups_claim, - roles_claim: request.roles_claim, - email_claim: request.email_claim, - username_claim: request.username_claim, - hide_from_ui: request.hide_from_ui, - }); - - validate_provider_config_fields(&config)?; - Ok(config) -} - -fn persisted_provider_secret(config: &ServerConfig, provider_id: &str) -> Option { - config - .0 - .get(IDENTITY_OPENID_SUB_SYS) - .and_then(|subsystem| subsystem.get(&provider_instance_key(provider_id))) - .and_then(|kvs| kvs.lookup(OIDC_CLIENT_SECRET)) - .filter(|value| !value.trim().is_empty()) -} - -fn upsert_persisted_provider_config(config: &mut ServerConfig, provider_config: &rustfs_iam::oidc::OidcProviderConfig) { - let instance_key = provider_instance_key(&provider_config.id); - let mut kvs = default_oidc_kvs().unwrap_or_default(); - - set_kvs_value( - &mut kvs, - ENABLE_KEY, - if provider_config.enabled { - EnableState::On.to_string() - } else { - EnableState::Off.to_string() - }, - ); - set_kvs_value(&mut kvs, OIDC_CONFIG_URL, provider_config.config_url.clone()); - set_kvs_value(&mut kvs, OIDC_ISSUER, provider_config.issuer.clone().unwrap_or_default()); - set_kvs_value(&mut kvs, OIDC_CLIENT_ID, provider_config.client_id.clone()); - set_kvs_value(&mut kvs, OIDC_CLIENT_SECRET, provider_config.client_secret.clone().unwrap_or_default()); - set_kvs_value(&mut kvs, OIDC_SCOPES, provider_config.scopes.join(",")); - set_kvs_value(&mut kvs, OIDC_OTHER_AUDIENCES, provider_config.other_audiences.join(",")); - set_kvs_value(&mut kvs, OIDC_REDIRECT_URI, provider_config.redirect_uri.clone().unwrap_or_default()); - set_kvs_value( - &mut kvs, - OIDC_REDIRECT_URI_DYNAMIC, - if provider_config.redirect_uri_dynamic { - EnableState::On.to_string() - } else { - EnableState::Off.to_string() - }, - ); - set_kvs_value(&mut kvs, OIDC_CLAIM_NAME, provider_config.claim_name.clone()); - set_kvs_value(&mut kvs, OIDC_CLAIM_PREFIX, provider_config.claim_prefix.clone()); - set_kvs_value(&mut kvs, OIDC_ROLE_POLICY, provider_config.role_policy.clone()); - set_kvs_value(&mut kvs, OIDC_DISPLAY_NAME, provider_config.display_name.clone()); - set_kvs_value(&mut kvs, OIDC_GROUPS_CLAIM, provider_config.groups_claim.clone()); - set_kvs_value(&mut kvs, OIDC_ROLES_CLAIM, provider_config.roles_claim.clone()); - set_kvs_value(&mut kvs, OIDC_EMAIL_CLAIM, provider_config.email_claim.clone()); - set_kvs_value(&mut kvs, OIDC_USERNAME_CLAIM, provider_config.username_claim.clone()); - set_kvs_value( - &mut kvs, - OIDC_HIDE_FROM_UI, - if provider_config.hide_from_ui { - EnableState::On.to_string() - } else { - EnableState::Off.to_string() - }, - ); - - config - .0 - .entry(IDENTITY_OPENID_SUB_SYS.to_string()) - .or_default() - .insert(instance_key, kvs); -} - -fn delete_persisted_provider_config(config: &mut ServerConfig, provider_id: &str) -> S3Result<()> { - let Some(subsystem) = config.0.get_mut(IDENTITY_OPENID_SUB_SYS) else { - return Err(s3_error!(InvalidRequest, "provider not found")); - }; - - if subsystem.remove(&provider_instance_key(provider_id)).is_none() { - return Err(s3_error!(InvalidRequest, "provider not found")); - } - - if subsystem.is_empty() { - config.0.remove(IDENTITY_OPENID_SUB_SYS); - } - - Ok(()) -} - fn extract_request_scheme(req: &S3Request) -> S3Result { let raw_scheme = req .headers @@ -1739,191 +1297,22 @@ mod tests { } #[test] - fn test_validate_absolute_http_url_rejects_loopback_targets() { - let err = validate_absolute_http_url("https://127.0.0.1/.well-known/openid-configuration", "config_url") - .expect_err("loopback config URL should be rejected"); - assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); - assert!(err.message().unwrap_or_default().contains("not allowed")); - } + fn oidc_config_errors_keep_admin_codes() { + let error = config_service_error(oidc_config::OidcAdminConfigError::Config( + rustfs_iam::oidc::OidcConfigError::EnvironmentManaged, + )); + assert_eq!(error.code(), &S3ErrorCode::AccessDenied); + assert_eq!(error.message(), Some("provider is managed by environment variables")); - #[test] - fn test_provider_instance_key() { - assert_eq!(provider_instance_key("default"), "_"); - assert_eq!(provider_instance_key("okta"), "okta"); - } + let error = config_service_error(oidc_config::OidcAdminConfigError::Config( + rustfs_iam::oidc::OidcConfigError::ProviderNotFound, + )); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + assert_eq!(error.message(), Some("provider not found")); - #[test] - fn test_build_provider_config_requires_openid_scope() { - let req = OidcConfigUpsertRequest { - scopes: vec!["profile".to_string()], - config_url: "https://example.com/.well-known/openid-configuration".to_string(), - client_id: "client-id".to_string(), - ..Default::default() - }; - - assert!(build_provider_config_from_upsert("default", req, None).is_err()); - } - - #[test] - fn test_build_provider_config_preserves_existing_secret_when_request_is_empty() { - let req = OidcConfigUpsertRequest { - config_url: "https://example.com/.well-known/openid-configuration".to_string(), - client_id: "client-id".to_string(), - client_secret: Some("".to_string()), - ..Default::default() - }; - - let config = - build_provider_config_from_upsert("default", req, Some("existing-secret".to_string())).expect("config should build"); - - assert_eq!(config.client_secret.as_deref(), Some("existing-secret")); - assert_eq!(config.roles_claim, OIDC_DEFAULT_ROLES_CLAIM); - } - - #[test] - fn test_oidc_config_upsert_request_rejects_unknown_fields() { - let err = serde_json::from_str::( - r#"{"config_url":"https://example.com/.well-known/openid-configuration","client_id":"client","unexpected_field":true}"#, - ) - .expect_err("unknown upsert field should fail"); - - assert!(err.to_string().contains("unknown field")); - } - - #[test] - fn test_oidc_config_validate_request_rejects_unknown_fields() { - let err = serde_json::from_str::( - r#"{"provider_id":"default","config_url":"https://example.com/.well-known/openid-configuration","client_id":"client","unexpected_field":true}"#, - ) - .expect_err("unknown validate field should fail"); - - assert!(err.to_string().contains("unknown field")); - } - - #[test] - fn test_build_provider_config_uses_custom_roles_claim() { - let req = OidcConfigUpsertRequest { - config_url: "https://example.com/.well-known/openid-configuration".to_string(), - client_id: "client-id".to_string(), - roles_claim: "app_roles".to_string(), - ..Default::default() - }; - - let config = build_provider_config_from_upsert("default", req, None).expect("config should build"); - assert_eq!(config.roles_claim, "app_roles"); - } - - #[test] - fn test_oidc_restart_required_detects_persisted_changes() { - let active_config = ServerConfig::new(); - let mut persisted_config = ServerConfig::new(); - let provider_config = rustfs_iam::oidc::OidcProviderConfig { - id: "default".to_string(), - enabled: true, - config_url: "https://example.com/.well-known/openid-configuration".to_string(), - issuer: None, - client_id: "console".to_string(), - client_secret: Some("secret".to_string()), - scopes: vec!["openid".to_string(), "profile".to_string()], - other_audiences: vec![], - redirect_uri: None, - redirect_uri_dynamic: true, - claim_name: OIDC_DEFAULT_CLAIM_NAME.to_string(), - claim_prefix: String::new(), - role_policy: String::new(), - display_name: "default".to_string(), - groups_claim: OIDC_DEFAULT_GROUPS_CLAIM.to_string(), - roles_claim: OIDC_DEFAULT_ROLES_CLAIM.to_string(), - email_claim: OIDC_DEFAULT_EMAIL_CLAIM.to_string(), - username_claim: OIDC_DEFAULT_USERNAME_CLAIM.to_string(), - hide_from_ui: false, - }; - - upsert_persisted_provider_config(&mut persisted_config, &provider_config); - - assert!(oidc_restart_required_from_active_config(&persisted_config, Some(&active_config))); - assert!(!oidc_restart_required_from_active_config(&persisted_config, Some(&persisted_config))); - } - - #[test] - fn test_upsert_persists_hide_from_ui_on() { - let mut config = ServerConfig::new(); - let mut provider_config = rustfs_iam::oidc::OidcProviderConfig { - id: "kubernetes".to_string(), - enabled: true, - config_url: "https://example.com/.well-known/openid-configuration".to_string(), - issuer: None, - client_id: "test".to_string(), - client_secret: None, - scopes: vec!["openid".to_string()], - other_audiences: vec![], - redirect_uri: None, - redirect_uri_dynamic: true, - claim_name: "sub".to_string(), - claim_prefix: String::new(), - role_policy: String::new(), - display_name: "Kubernetes".to_string(), - groups_claim: OIDC_DEFAULT_GROUPS_CLAIM.to_string(), - roles_claim: OIDC_DEFAULT_ROLES_CLAIM.to_string(), - email_claim: OIDC_DEFAULT_EMAIL_CLAIM.to_string(), - username_claim: OIDC_DEFAULT_USERNAME_CLAIM.to_string(), - hide_from_ui: true, - }; - - upsert_persisted_provider_config(&mut config, &provider_config); - - let kvs = config - .0 - .get(IDENTITY_OPENID_SUB_SYS) - .and_then(|m| m.get("kubernetes")) - .expect("provider KVS should exist"); - assert_eq!(kvs.get(OIDC_HIDE_FROM_UI), EnableState::On.to_string()); - - // Flip to false and verify - provider_config.hide_from_ui = false; - upsert_persisted_provider_config(&mut config, &provider_config); - - let kvs = config - .0 - .get(IDENTITY_OPENID_SUB_SYS) - .and_then(|m| m.get("kubernetes")) - .expect("provider KVS should exist"); - assert_eq!(kvs.get(OIDC_HIDE_FROM_UI), EnableState::Off.to_string()); - } - - #[test] - fn test_upsert_persists_issuer() { - let mut config = ServerConfig::new(); - let provider_config = rustfs_iam::oidc::OidcProviderConfig { - id: "kubernetes".to_string(), - enabled: true, - config_url: "http://keycloak.ns.svc.cluster.local:8080/realms/app/.well-known/openid-configuration".to_string(), - issuer: Some("https://app.local/realms/app".to_string()), - client_id: "test".to_string(), - client_secret: None, - scopes: vec!["openid".to_string()], - other_audiences: vec![], - redirect_uri: None, - redirect_uri_dynamic: true, - claim_name: "sub".to_string(), - claim_prefix: String::new(), - role_policy: String::new(), - display_name: "Kubernetes".to_string(), - groups_claim: OIDC_DEFAULT_GROUPS_CLAIM.to_string(), - roles_claim: OIDC_DEFAULT_ROLES_CLAIM.to_string(), - email_claim: OIDC_DEFAULT_EMAIL_CLAIM.to_string(), - username_claim: OIDC_DEFAULT_USERNAME_CLAIM.to_string(), - hide_from_ui: false, - }; - - upsert_persisted_provider_config(&mut config, &provider_config); - - let kvs = config - .0 - .get(IDENTITY_OPENID_SUB_SYS) - .and_then(|m| m.get("kubernetes")) - .expect("provider KVS should exist"); - assert_eq!(kvs.get(OIDC_ISSUER), "https://app.local/realms/app"); + let error = config_service_error(oidc_config::OidcAdminConfigError::Task("cancelled")); + assert_eq!(error.code(), &S3ErrorCode::InternalError); + assert_eq!(error.message(), Some("OIDC config update task cancelled")); } /// The OIDC config gate now authorizes through the shared admin gate, which diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 7adaf84b6..ec3abcfa8 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -2063,34 +2063,32 @@ fn open_id_settings(providers: Vec<(String, OpenIDProviderSettings)>, region: St settings } -fn open_id_settings_from_snapshot(snapshot: &rustfs_iam::oidc::OidcConfigSnapshot, region: String) -> OpenIDSettings { +fn open_id_settings_from_snapshot(snapshot: &rustfs_iam::oidc::OidcSiteReplicationSnapshot, region: String) -> OpenIDSettings { let providers = snapshot .providers() .iter() .map(|provider| { - let config = &provider.config; - (config.id.clone(), open_id_provider_settings(config)) + ( + provider.provider_id.clone(), + OpenIDProviderSettings { + claim_name: provider.claim_name.clone(), + claim_userinfo_enabled: false, + role_policy: provider.role_policy.clone(), + client_id: provider.client_id.clone(), + hashed_client_secret: provider.hashed_client_secret.clone(), + }, + ) }) .collect(); open_id_settings(providers, region) } -fn open_id_provider_settings(config: &rustfs_iam::oidc::OidcProviderConfig) -> OpenIDProviderSettings { - OpenIDProviderSettings { - claim_name: config.claim_name.clone(), - claim_userinfo_enabled: false, - role_policy: config.role_policy.clone(), - client_id: config.client_id.clone(), - hashed_client_secret: hash_client_secret(config.client_secret.as_deref()), - } -} - fn local_idp_settings() -> IDPSettings { let mut settings = IDPSettings::default(); let region = current_region().map(|region| region.to_string()).unwrap_or_default(); settings.open_id = current_oidc_config_query() - .map(|query| open_id_settings_from_snapshot(&query.config_snapshot(), region)) + .map(|query| open_id_settings_from_snapshot(&query.site_replication_snapshot(), region)) .unwrap_or_default(); let (ldap, ldap_configs) = load_ldap_idp_settings(); @@ -11547,16 +11545,18 @@ mod tests { #[test] fn oidc_config_snapshot_preserves_site_replication_settings() { - let snapshot = rustfs_iam::oidc::OidcConfigSnapshot::new(vec![ - oidc_snapshot_provider("corp", "client-b"), - oidc_snapshot_provider("default", "client-a"), - ]); + let snapshot = rustfs_iam::oidc::OidcSiteReplicationSnapshot::from_config_snapshot( + &rustfs_iam::oidc::OidcConfigSnapshot::new(vec![ + oidc_snapshot_provider("corp", "client-b"), + oidc_snapshot_provider("default", "client-a"), + ]), + ); assert_eq!( snapshot .providers() .iter() - .map(|provider| provider.config.id.as_str()) + .map(|provider| provider.provider_id.as_str()) .collect::>(), ["default", "corp"] ); diff --git a/rustfs/src/admin/service/mod.rs b/rustfs/src/admin/service/mod.rs index 403530f5c..4c1dc8177 100644 --- a/rustfs/src/admin/service/mod.rs +++ b/rustfs/src/admin/service/mod.rs @@ -15,5 +15,6 @@ pub(crate) mod caller_identity; pub mod config; pub(crate) mod federated_identity; +pub(crate) mod oidc_config; pub(crate) mod session_policy; pub mod site_replication; diff --git a/rustfs/src/admin/service/oidc_config.rs b/rustfs/src/admin/service/oidc_config.rs new file mode 100644 index 000000000..df230603f --- /dev/null +++ b/rustfs/src/admin/service/oidc_config.rs @@ -0,0 +1,158 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::admin::runtime_sources::{ + current_app_context, current_object_store_handle_for_context, current_server_config_for_context, +}; +use crate::admin::storage_api::config::{ + read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot, +}; +use crate::admin::storage_api::error::Error as StorageError; +use rustfs_config::server_config::Config as ServerConfig; +use rustfs_iam::oidc::{ + OidcConfigError, OidcConfigSnapshot, OidcProviderConfigInput, OidcProviderValidationInput, OidcProviderValidationResult, + build_upsert_provider_config, build_validation_provider_config, delete_persisted_provider_config, load_oidc_config_snapshot, + persisted_provider_secret, upsert_persisted_provider_config, validate_mutable_provider_id, + validate_oidc_provider_config_with_extra_root_ca, +}; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum OidcAdminConfigError { + #[error("storage layer not initialized")] + StorageUnavailable, + #[error("failed to load server config: {0}")] + Load(#[source] StorageError), + #[error("failed to save server config: {0}")] + Save(#[source] StorageError), + #[error("validation failed: {0}")] + Validation(String), + #[error("OIDC config update task {0}")] + Task(&'static str), + #[error(transparent)] + Config(#[from] OidcConfigError), +} + +type Result = std::result::Result; + +pub(crate) struct OidcConfigList { + pub snapshot: OidcConfigSnapshot, + pub restart_required: bool, +} + +pub(crate) fn ensure_mutable_provider_id(provider_id: &str) -> Result<()> { + validate_mutable_provider_id(provider_id)?; + Ok(()) +} + +fn oidc_config_store() -> Result> { + let context = current_app_context(); + current_object_store_handle_for_context(context.as_deref()).ok_or(OidcAdminConfigError::StorageUnavailable) +} + +async fn load_server_config_from_store() -> Result { + let store = oidc_config_store()?; + read_admin_config_without_migrate(store) + .await + .map_err(OidcAdminConfigError::Load) +} + +fn restart_required_from_active_config(config: &ServerConfig, active_config: Option<&ServerConfig>) -> bool { + load_oidc_config_snapshot(Some(config)) != load_oidc_config_snapshot(active_config) +} + +pub(crate) async fn list_config() -> Result { + let config = load_server_config_from_store().await?; + let context = current_app_context(); + let active_config = current_server_config_for_context(context.as_deref()); + Ok(OidcConfigList { + snapshot: load_oidc_config_snapshot(Some(&config)), + restart_required: restart_required_from_active_config(&config, active_config.as_ref()), + }) +} + +async fn update_server_config(modifier: F) -> Result<()> +where + F: FnOnce(&mut ServerConfig) -> Result<()> + Send + 'static, +{ + let store = oidc_config_store()?; + tokio::spawn(async move { + let snapshot = read_admin_server_config_snapshot(store.clone()) + .await + .map_err(OidcAdminConfigError::Load)?; + let mut config = snapshot.config.clone(); + modifier(&mut config)?; + save_admin_server_config_snapshot(store, &config, &snapshot) + .await + .map(|_| ()) + .map_err(OidcAdminConfigError::Save) + }) + .await + .map_err(|error| OidcAdminConfigError::Task(if error.is_cancelled() { "cancelled" } else { "panicked" }))? +} + +pub(crate) async fn upsert_config(provider_id: String, input: OidcProviderConfigInput) -> Result<()> { + ensure_mutable_provider_id(&provider_id)?; + update_server_config(move |config| { + let existing_secret = persisted_provider_secret(config, &provider_id); + let provider = build_upsert_provider_config(&provider_id, input, existing_secret)?; + upsert_persisted_provider_config(config, &provider); + Ok(()) + }) + .await +} + +pub(crate) async fn delete_config(provider_id: String) -> Result<()> { + ensure_mutable_provider_id(&provider_id)?; + update_server_config(move |config| { + delete_persisted_provider_config(config, &provider_id)?; + Ok(()) + }) + .await +} + +pub(crate) async fn validate_config(input: OidcProviderValidationInput) -> Result { + let provider = build_validation_provider_config(input)?; + let extra_root_ca = crate::startup_auth::current_oidc_extra_root_ca_material() + .await + .map_err(|error| OidcAdminConfigError::Validation(error.to_string()))?; + validate_oidc_provider_config_with_extra_root_ca(&provider, extra_root_ca.root_ca_pem.as_deref()) + .await + .map_err(OidcAdminConfigError::Validation) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn restart_required_detects_persisted_changes() { + let active_config = ServerConfig::new(); + let mut persisted_config = ServerConfig::new(); + let provider = build_upsert_provider_config( + "default", + OidcProviderConfigInput { + config_url: "https://example.com/.well-known/openid-configuration".to_string(), + client_id: "console".to_string(), + client_secret: Some("secret".to_string()), + ..Default::default() + }, + None, + ) + .expect("provider config should be valid"); + upsert_persisted_provider_config(&mut persisted_config, &provider); + + assert!(restart_required_from_active_config(&persisted_config, Some(&active_config))); + assert!(!restart_required_from_active_config(&persisted_config, Some(&persisted_config))); + } +} diff --git a/rustfs/src/site_replication/tests.rs b/rustfs/src/site_replication/tests.rs index 4f20ec2b3..e5c38d77e 100644 --- a/rustfs/src/site_replication/tests.rs +++ b/rustfs/src/site_replication/tests.rs @@ -3569,11 +3569,6 @@ fn test_reconcile_site_replication_bucket_targets_allows_peer_on_same_port_as_lo }); } -#[test] -fn test_hash_client_secret_matches_minio_style_base64url_sha256() { - assert_eq!(hash_client_secret(Some("secret")), "K7gNU3sdo-OL0wNhqoVWhr3g6s1xYv72ol_pe_Unols"); -} - #[test] fn test_site_replication_peer_client_cache_hit_generation_mismatch_returns_none() { let cache = Some(SiteReplicationPeerClientCache { diff --git a/rustfs/src/site_replication/transport.rs b/rustfs/src/site_replication/transport.rs index 18d4850b9..8a33c8585 100644 --- a/rustfs/src/site_replication/transport.rs +++ b/rustfs/src/site_replication/transport.rs @@ -374,16 +374,6 @@ pub(crate) fn runtime_tls_enabled() -> bool { runtime_tls_enabled_with(endpoints.as_ref()) } -pub(crate) fn hash_client_secret(secret: Option<&str>) -> String { - let Some(secret) = secret.filter(|secret| !secret.is_empty()) else { - return String::new(); - }; - - let mut hasher = Sha256::new(); - hasher.update(secret.as_bytes()); - URL_SAFE_NO_PAD.encode_to_string(hasher.finalize()) -} - pub(crate) fn loopback_replication_targets_allowed() -> bool { std::env::var(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV) .map(|value| value.eq_ignore_ascii_case("true") || value == "1") From 5924a55931cc7d709be013125033602450917f8f Mon Sep 17 00:00:00 2001 From: Rafael Kansy <16208524+Pirat83@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:00:24 +0200 Subject: [PATCH 014/143] fix(obs): honor RUSTFS_OBS_USE_STDOUT with per-signal endpoints (#8476) use_stdout was forced to true whenever the root RUSTFS_OBS_ENDPOINT was empty, so a deployment that sets only a per-signal endpoint such as RUSTFS_OBS_TRACE_ENDPOINT could not turn the stdout exporters off: every exported span was also printed to stdout as a multi-line text block. Count a non-empty trace, metric or log endpoint as a configured endpoint. With no endpoint at all the stdout default is still forced on. Add a test covering each per-signal endpoint and an empty value. --- crates/obs/src/config.rs | 57 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/crates/obs/src/config.rs b/crates/obs/src/config.rs index c637719ff..4bc7c7705 100644 --- a/crates/obs/src/config.rs +++ b/crates/obs/src/config.rs @@ -238,8 +238,10 @@ impl OtelConfig { /// that know their display/build version should use /// [`OtelConfig::extract_otel_config_from_env_with_version`]. /// - /// When no endpoint is configured at all, `use_stdout` is forced to `true` - /// so that logs are still visible during development. + /// When no endpoint is configured at all (neither the root endpoint nor any + /// per-signal one), `use_stdout` is forced to `true` so that logs are still + /// visible during development. With any endpoint set, `RUSTFS_OBS_USE_STDOUT` + /// is honoured. /// /// # Example /// ```no_run @@ -268,9 +270,15 @@ impl OtelConfig { _ => env::var(ENV_OBS_ENDPOINT).unwrap_or_default(), }; - // Force stdout when there is no remote endpoint so that operators - // always have *some* log output in the default configuration. - let use_stdout = if endpoint.is_empty() { + // Force stdout only when no OTLP endpoint is configured at all, so that operators + // always have *some* log output in the default configuration. A per-signal endpoint + // (e.g. `RUSTFS_OBS_TRACE_ENDPOINT`) counts as configured: otherwise setting only that + // one forces the stdout exporters on and silently ignores `RUSTFS_OBS_USE_STDOUT=false`, + // printing every exported span to stdout. + let has_signal_endpoint = [ENV_OBS_TRACE_ENDPOINT, ENV_OBS_METRIC_ENDPOINT, ENV_OBS_LOG_ENDPOINT] + .into_iter() + .any(|key| get_env_opt_str(key).is_some_and(|value| !value.is_empty())); + let use_stdout = if endpoint.is_empty() && !has_signal_endpoint { true } else { get_env_bool(ENV_OBS_USE_STDOUT, USE_STDOUT) @@ -538,6 +546,45 @@ mod tests { }); } + #[test] + fn use_stdout_is_forced_only_when_no_endpoint_is_configured() { + with_profiling_env_lock(|| { + temp_env::with_vars_unset( + [ + ENV_OBS_ENDPOINT, + ENV_OBS_TRACE_ENDPOINT, + ENV_OBS_METRIC_ENDPOINT, + ENV_OBS_LOG_ENDPOINT, + ], + || { + temp_env::with_var(ENV_OBS_USE_STDOUT, Some("false"), || { + // No endpoint at all: forced on, the explicit false is ignored. + assert_eq!(OtelConfig::extract_otel_config_from_env(None).use_stdout, Some(true)); + }); + for key in [ENV_OBS_TRACE_ENDPOINT, ENV_OBS_METRIC_ENDPOINT, ENV_OBS_LOG_ENDPOINT] { + temp_env::with_var(key, Some("http://localhost:4318/v1/signal"), || { + temp_env::with_var(ENV_OBS_USE_STDOUT, Some("false"), || { + assert_eq!(OtelConfig::extract_otel_config_from_env(None).use_stdout, Some(false), "{key}"); + }); + temp_env::with_var(ENV_OBS_USE_STDOUT, Some("true"), || { + assert_eq!(OtelConfig::extract_otel_config_from_env(None).use_stdout, Some(true), "{key}"); + }); + temp_env::with_var_unset(ENV_OBS_USE_STDOUT, || { + assert_eq!(OtelConfig::extract_otel_config_from_env(None).use_stdout, Some(USE_STDOUT), "{key}"); + }); + }); + } + temp_env::with_var(ENV_OBS_TRACE_ENDPOINT, Some(""), || { + temp_env::with_var(ENV_OBS_USE_STDOUT, Some("false"), || { + // An empty per-signal endpoint is not a configured endpoint. + assert_eq!(OtelConfig::extract_otel_config_from_env(None).use_stdout, Some(true)); + }); + }); + }, + ); + }); + } + #[test] fn stdout_mirror_environment_preserves_unset_true_and_false() { with_profiling_env_lock(|| { From acd26afd8bb423dcca9f67b5f64f69660bb62dc2 Mon Sep 17 00:00:00 2001 From: Anton Date: Mon, 12 Oct 2026 01:25:14 +0300 Subject: [PATCH 015/143] fix(scanner): skip child prefixes removed before they are opened (#8479) A child prefix can be deleted after its parent listed it and before scan_folder opens it. read_dir then returned NotFound, which propagated through the compacted subtree and dropped the whole top-level child (child_scan_failed), so the cycle never produced a durable usage snapshot under continuous deletes. Treat NotFound on open like a directory that vanished during iteration for non-root folders. A missing bucket root still reports NotFound so namespace removal handling is unchanged. Refs #8478 Co-authored-by: Anton <5527364+basic-system@users.noreply.github.com> --- crates/scanner/src/scanner_folder.rs | 26 ++++++++++- crates/scanner/src/scanner_folder/tests.rs | 53 ++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 7bee115a8..5f664466a 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -1657,8 +1657,26 @@ impl FolderScanner { "Scanner folder state updated" ); + // A child prefix can be removed after its parent listed it and before + // it is opened here (e.g. concurrent object deletes). Treat it like a + // directory that vanished during iteration instead of failing the + // whole parent subtree. The bucket root (no parent) still propagates + // NotFound so namespace removal is detected by the caller. let mut dir_reader = match tokio::fs::read_dir(&dir_path).await { - Ok(dir_reader) => dir_reader, + Ok(dir_reader) => Some(dir_reader), + Err(e) if e.kind() == ErrorKind::NotFound && folder.parent.is_some() => { + debug!( + target: "rustfs::scanner::folder", + event = EVENT_SCANNER_FOLDER_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_FOLDER, + dir_path = %dir_path, + state = "dir_missing_before_open", + error = %e, + "Scanner folder state updated" + ); + None + } Err(e) => return Err(ScannerError::Io(e)), }; let mut pending_entry_progress = 0_u64; @@ -1667,7 +1685,11 @@ impl FolderScanner { let raw_enumeration_committed_entries = self.raw_enumeration_committed_entry_oracle(&folder.name); loop { - let entry = match dir_reader.next_entry().await { + let Some(reader) = dir_reader.as_mut() else { + raw_enumeration_complete = true; + break; + }; + let entry = match reader.next_entry().await { Ok(Some(entry)) => entry, Ok(None) => { raw_enumeration_complete = true; diff --git a/crates/scanner/src/scanner_folder/tests.rs b/crates/scanner/src/scanner_folder/tests.rs index d6f6c9d4f..106169bd1 100644 --- a/crates/scanner/src/scanner_folder/tests.rs +++ b/crates/scanner/src/scanner_folder/tests.rs @@ -2172,6 +2172,59 @@ async fn test_scan_folder_skips_unreadable_child_directory() { assert!(result.is_ok(), "expected unreadable child directory to be skipped"); } +#[tokio::test] +#[serial] +async fn test_scan_folder_treats_child_removed_before_open_as_empty() { + let (mut scanner, temp_dir) = build_test_scanner().await; + let _guard = TestGuard::new(60, 0, &mut scanner, temp_dir.clone()); + + // The parent listed `gone/`, then a concurrent delete removed it before the + // child scan opened it. + std::fs::create_dir_all(temp_dir.join("bucket")).expect("failed to create bucket dir"); + + scanner.old_cache.info.name = "bucket".to_string(); + scanner.new_cache.info.name = "bucket".to_string(); + scanner.update_cache.info.name = "bucket".to_string(); + + let folder = CachedFolder { + name: "bucket/gone".to_string(), + parent: Some(hash_path("bucket")), + object_heal_prob_div: 1, + }; + + let mut into = DataUsageEntry::default(); + let result = scanner.scan_folder(CancellationToken::new(), folder, &mut into).await; + + assert!(result.is_ok(), "expected a child removed before open to be skipped, got {result:?}"); + assert_eq!(into.objects, 0); + assert_eq!(into.size, 0); +} + +#[tokio::test] +#[serial] +async fn test_scan_folder_missing_bucket_root_still_reports_not_found() { + let (mut scanner, temp_dir) = build_test_scanner().await; + let _guard = TestGuard::new(60, 0, &mut scanner, temp_dir.clone()); + + scanner.old_cache.info.name = "bucket".to_string(); + scanner.new_cache.info.name = "bucket".to_string(); + scanner.update_cache.info.name = "bucket".to_string(); + + let folder = CachedFolder { + name: "bucket".to_string(), + parent: None, + object_heal_prob_div: 1, + }; + + let mut into = DataUsageEntry::default(); + let result = scanner.scan_folder(CancellationToken::new(), folder, &mut into).await; + + assert!( + matches!(&result, Err(ScannerError::Io(e)) if e.kind() == ErrorKind::NotFound), + "expected a missing bucket root to keep reporting NotFound, got {result:?}" + ); +} + #[tokio::test] #[serial] async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() { From 1266d4c2701f79a719652834c5cc4cd596cd7177 Mon Sep 17 00:00:00 2001 From: Hauser Date: Mon, 12 Oct 2026 08:53:11 +0800 Subject: [PATCH 016/143] feat(tls): reload and publish external certificate generations (#8467) * feat(tls): reload external certificate snapshots coherently Pin projected certificate generations, validate complete TLS material, coordinate server consumers, and invalidate outbound RPC caches coherently. Add externally managed TLS Secret support and rotation guidance. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * feat(tls): publish server certificates through a fenced admin API Add opt-in server-leaf publication with dedicated IAM actions, actual TLS transport checks, immutable generations, durable captured-token CAS and idempotent receipts. Preserve independent trust and client identity sources and distinguish visible publication state from the full serving chain. Validate 64 TLS tests, 41 TLS-runtime tests, the focused IAM policy test, and eight live local API scenarios including least privilege and restart. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(admin): route certificate handler types through S3 facade Use the existing admin S3 facade to keep direct s3s imports within the footprint ratchet. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(admin): include TLS certificate routes in route matrix Account for the registered certificate-state GET and certificate publication POST without changing route behavior or weakening matrix assertions. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(tls): reconcile startup trust before skipping reloads Compare the initialized listener bundle with coherently published outbound material, retain startup trust and client identity when sources disappear, and cover independent startup rotations through real reload passes. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * refactor(tls): reuse pinned generations and prepared listener state Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(tls): preserve startup trust source requirements --------- Co-authored-by: heihutu Co-authored-by: zhi22915 Co-authored-by: overtrue --- Cargo.lock | 2 + crates/common/Cargo.toml | 1 + crates/common/src/globals.rs | 136 +- crates/ecstore/src/cluster/rpc/client.rs | 68 +- crates/ecstore/src/runtime/sources.rs | 5 +- crates/policy/src/policy/action.rs | 34 + crates/protos/src/lib.rs | 428 +++++- crates/protos/src/runtime_sources.rs | 4 + crates/tls-runtime/Cargo.toml | 1 + crates/tls-runtime/src/certs.rs | 208 ++- crates/tls-runtime/src/error.rs | 32 + crates/tls-runtime/src/lib.rs | 7 +- crates/tls-runtime/src/material.rs | 656 ++++++++- crates/tls-runtime/src/outbound.rs | 26 +- crates/tls-runtime/src/source.rs | 122 ++ docs/README.md | 2 + docs/operations/tls-certificate-rotation.md | 204 +++ helm/rustfs/templates/_helpers.tpl | 23 +- helm/rustfs/templates/configmap.yaml | 11 +- helm/rustfs/templates/deployment.yaml | 80 +- helm/rustfs/templates/statefulset.yaml | 82 +- helm/rustfs/values.yaml | 8 + rustfs/src/admin/handlers/mod.rs | 1 + rustfs/src/admin/handlers/tls_certificate.rs | 177 +++ rustfs/src/admin/mod.rs | 1 + rustfs/src/admin/route_policy.rs | 22 + rustfs/src/admin/route_registration_test.rs | 2 + rustfs/src/server/http.rs | 42 +- rustfs/src/server/mod.rs | 2 + rustfs/src/server/ssec_transport.rs | 7 +- rustfs/src/server/tls_material.rs | 1383 +++++++++++++----- rustfs/src/server/tls_publish.rs | 1341 +++++++++++++++++ rustfs/src/startup_runtime_sources.rs | 1 + rustfs/src/startup_tls_material.rs | 28 + scripts/test_helm_templates.sh | 7 +- scripts/test_helm_tls_reload.sh | 49 + 36 files changed, 4594 insertions(+), 609 deletions(-) create mode 100644 docs/operations/tls-certificate-rotation.md create mode 100644 rustfs/src/admin/handlers/tls_certificate.rs create mode 100644 rustfs/src/server/tls_publish.rs create mode 100644 scripts/test_helm_tls_reload.sh diff --git a/Cargo.lock b/Cargo.lock index 44433ab1e..91b1a5149 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9686,6 +9686,7 @@ dependencies = [ name = "rustfs-common" version = "1.0.1" dependencies = [ + "arc-swap", "hotpath", "metrics", "smallvec", @@ -10957,6 +10958,7 @@ dependencies = [ "thiserror 2.0.21", "tokio", "tracing", + "x509-parser", ] [[package]] diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 3749c929d..b772c459d 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -34,6 +34,7 @@ hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] [dependencies] +arc-swap.workspace = true hotpath.workspace = true tokio = { workspace = true, features = ["fs", "rt-multi-thread"] } tonic = { workspace = true, features = ["gzip", "deflate"] } diff --git a/crates/common/src/globals.rs b/crates/common/src/globals.rs index e4cd89c88..03814d454 100644 --- a/crates/common/src/globals.rs +++ b/crates/common/src/globals.rs @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use arc_swap::ArcSwap; use std::collections::HashMap; -use std::sync::LazyLock; use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, LazyLock}; use tokio::sync::RwLock; use tonic::transport::Channel; @@ -27,6 +28,42 @@ pub static GLOBAL_ROOT_CERT: LazyLock>>> = LazyLock::new(| pub static GLOBAL_MTLS_IDENTITY: LazyLock>> = LazyLock::new(|| RwLock::new(None)); pub static GLOBAL_OUTBOUND_TLS_GENERATION: LazyLock = LazyLock::new(|| AtomicU64::new(0)); +#[derive(Clone, Default)] +pub struct OutboundTlsSnapshot { + pub generation: u64, + pub root_ca_pem: Option>, + pub mtls_identity: Option, +} + +static GLOBAL_OUTBOUND_TLS_STATE: LazyLock> = + LazyLock::new(|| ArcSwap::from_pointee(OutboundTlsSnapshot::default())); + +pub fn get_global_outbound_tls_snapshot() -> Arc { + GLOBAL_OUTBOUND_TLS_STATE.load_full() +} + +/// Publish identity, trust and generation together; the individual globals are legacy views. +pub async fn publish_global_outbound_tls_snapshot(snapshot: OutboundTlsSnapshot) -> u64 { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + if current.generation != 0 + && current.root_ca_pem == snapshot.root_ca_pem + && current.mtls_identity == snapshot.mtls_identity + { + return Arc::clone(current); + } + Arc::new(OutboundTlsSnapshot { + generation: snapshot.generation.max(current.generation.saturating_add(1)), + root_ca_pem: snapshot.root_ca_pem.clone(), + mtls_identity: snapshot.mtls_identity.clone(), + }) + }); + let accepted = get_global_outbound_tls_snapshot(); + *GLOBAL_ROOT_CERT.write().await = accepted.root_ca_pem.clone(); + *GLOBAL_MTLS_IDENTITY.write().await = accepted.mtls_identity.clone(); + GLOBAL_OUTBOUND_TLS_GENERATION.store(accepted.generation, Ordering::Relaxed); + accepted.generation +} + /// Log level to use when reporting cached gRPC connection eviction. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ConnectionEvictionLogLevel { @@ -85,17 +122,27 @@ pub async fn get_global_addr() -> String { /// # Arguments /// * `cert` - A vector of bytes representing the PEM-encoded root CA certificate. pub async fn set_global_root_cert(cert: Vec) { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + let mut next = (**current).clone(); + next.root_ca_pem = Some(cert.clone()); + Arc::new(next) + }); *GLOBAL_ROOT_CERT.write().await = Some(cert); } /// Clear the global root CA certificate for outbound gRPC clients. pub async fn clear_global_root_cert() { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + let mut next = (**current).clone(); + next.root_ca_pem = None; + Arc::new(next) + }); *GLOBAL_ROOT_CERT.write().await = None; } /// Get the global root CA certificate for outbound gRPC clients. pub async fn get_global_root_cert() -> Option> { - GLOBAL_ROOT_CERT.read().await.clone() + get_global_outbound_tls_snapshot().root_ca_pem.clone() } /// Set the global mTLS identity (cert+key PEM) for outbound gRPC clients. @@ -105,22 +152,32 @@ pub async fn get_global_root_cert() -> Option> { /// # Arguments /// * `identity` - An optional MtlsIdentityPem struct containing the cert and key PEM. pub async fn set_global_mtls_identity(identity: Option) { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + let mut next = (**current).clone(); + next.mtls_identity = identity.clone(); + Arc::new(next) + }); *GLOBAL_MTLS_IDENTITY.write().await = identity; } /// Get the global mTLS identity for outbound gRPC clients. pub async fn get_global_mtls_identity() -> Option { - GLOBAL_MTLS_IDENTITY.read().await.clone() + get_global_outbound_tls_snapshot().mtls_identity.clone() } /// Set the global outbound TLS generation. pub fn set_global_outbound_tls_generation(generation: u64) { + GLOBAL_OUTBOUND_TLS_STATE.rcu(|current| { + let mut next = (**current).clone(); + next.generation = generation; + Arc::new(next) + }); GLOBAL_OUTBOUND_TLS_GENERATION.store(generation, Ordering::Relaxed); } /// Get the global outbound TLS generation. pub fn get_global_outbound_tls_generation() -> u64 { - GLOBAL_OUTBOUND_TLS_GENERATION.load(Ordering::Relaxed) + get_global_outbound_tls_snapshot().generation } /// Evict a stale/dead connection from the global connection cache. @@ -194,8 +251,77 @@ pub async fn clear_all_connections() { /// /// When present, gRPC clients will present this identity to servers requesting/requiring mTLS. /// When absent, clients proceed with standard server-authenticated TLS. -#[derive(Clone, Debug)] +#[derive(Clone, PartialEq, Eq)] pub struct MtlsIdentityPem { pub cert_pem: Vec, pub key_pem: Vec, } + +impl std::fmt::Debug for MtlsIdentityPem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MtlsIdentityPem") + .field("cert_bytes", &self.cert_pem.len()) + .field("key_pem", &"[REDACTED]") + .finish() + } +} + +#[cfg(test)] +mod tls_snapshot_tests { + use super::*; + + #[tokio::test] + async fn outbound_snapshot_publication_is_coherent_and_skips_unchanged_material() { + let start = get_global_outbound_tls_generation().saturating_add(1); + let snapshot = OutboundTlsSnapshot { + generation: start, + root_ca_pem: Some(vec![1]), + mtls_identity: Some(MtlsIdentityPem { + cert_pem: vec![1], + key_pem: vec![1], + }), + }; + let first = publish_global_outbound_tls_snapshot(snapshot.clone()).await; + assert_eq!( + publish_global_outbound_tls_snapshot(OutboundTlsSnapshot { + generation: first.saturating_add(1), + ..snapshot + }) + .await, + first + ); + let writer = tokio::spawn(async move { + for tag in 2_u8..=255 { + publish_global_outbound_tls_snapshot(OutboundTlsSnapshot { + generation: start.saturating_add(u64::from(tag)), + root_ca_pem: Some(vec![tag]), + mtls_identity: Some(MtlsIdentityPem { + cert_pem: vec![tag], + key_pem: vec![tag], + }), + }) + .await; + tokio::task::yield_now().await; + } + }); + while !writer.is_finished() { + let state = get_global_outbound_tls_snapshot(); + let identity = state.mtls_identity.as_ref().expect("identity in complete snapshot"); + assert_eq!(state.root_ca_pem.as_ref().expect("roots in complete snapshot"), &identity.cert_pem); + assert_eq!(identity.cert_pem, identity.key_pem); + tokio::task::yield_now().await; + } + writer.await.expect("writer completes"); + } + + #[test] + fn mtls_identity_debug_redacts_private_key() { + let identity = MtlsIdentityPem { + cert_pem: b"public certificate".to_vec(), + key_pem: b"PRIVATE-KEY-MARKER".to_vec(), + }; + let debug = format!("{identity:?}"); + assert!(!debug.contains("PRIVATE-KEY-MARKER")); + assert!(debug.contains("[REDACTED]")); + } +} diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index 76ae3bb36..27bd0542f 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -24,10 +24,11 @@ use crate::cluster::rpc::{gen_tonic_replay_scope_headers, gen_tonic_signature_he #[cfg(test)] use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers}; use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError}; +#[cfg(test)] use crate::runtime::sources as runtime_sources; use http::{Request as HttpRequest, Response as HttpResponse, Uri}; use rustfs_protos::{ - ChannelClass, create_new_channel, get_channel_for_class, + ChannelClass, get_channel_for_class, proto_gen::node_service::{ heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient, scanner_control_service_client::ScannerControlServiceClient, @@ -66,12 +67,9 @@ pub(crate) async fn scanner_control_time_out_client( interceptor: TonicInterceptor, ) -> crate::error::Result>> { let interceptor = interceptor.with_rpc_audience(addr)?; - let channel = match runtime_sources::cached_node_channel(addr).await { - Some(channel) => channel, - None => create_new_channel(addr) - .await - .map_err(|err| crate::error::Error::other(err.to_string()))?, - }; + let channel = get_channel_for_class(addr, ChannelClass::Control) + .await + .map_err(|err| crate::error::Error::other(err.to_string()))?; let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize; Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor) @@ -84,10 +82,7 @@ pub async fn heal_control_time_out_client( interceptor: TonicInterceptor, ) -> Result>, Box> { let interceptor = interceptor.with_rpc_audience(addr)?; - let channel = match runtime_sources::cached_node_channel(addr).await { - Some(channel) => channel, - None => create_new_channel(addr).await?, - }; + let channel = get_channel_for_class(addr, ChannelClass::Control).await?; let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE; let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); Ok(HealControlServiceClient::with_interceptor(channel, interceptor) @@ -100,10 +95,7 @@ pub async fn tier_mutation_control_time_out_client( interceptor: TonicInterceptor, ) -> Result>, Box> { let interceptor = interceptor.with_rpc_audience(addr)?; - let channel = match runtime_sources::cached_node_channel(addr).await { - Some(channel) => channel, - None => create_new_channel(addr).await?, - }; + let channel = get_channel_for_class(addr, ChannelClass::Control).await?; let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE; let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor) @@ -123,17 +115,7 @@ pub async fn node_service_time_out_client_for_class( class: ChannelClass, ) -> Result>, Box> { let interceptor = interceptor.with_rpc_audience(addr)?; - let channel = match class { - ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await { - Some(channel) => { - debug!("Using cached gRPC channel for: {}", addr); - channel - } - // No cached connection, create new one. - None => create_new_channel(addr).await?, - }, - ChannelClass::Bulk => get_channel_for_class(addr, ChannelClass::Bulk).await?, - }; + let channel = get_channel_for_class(addr, class).await?; let max_message_size = rustfs_protos::internode_rpc_max_message_size(); let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); @@ -546,6 +528,40 @@ mod tests { use tracing_opentelemetry::OpenTelemetrySpanExt; use tracing_subscriber::{Registry, layer::SubscriberExt}; + #[tokio::test] + async fn rpc_client_factories_do_not_reuse_https_channels_without_tls_generation() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve local address"); + let addr = format!("https://{}", listener.local_addr().expect("local address")); + drop(listener); + for factory in 0..4 { + let placeholder = tonic::transport::Endpoint::from_static("http://127.0.0.1:1").connect_lazy(); + runtime_sources::cache_test_node_channel(addr.clone(), placeholder).await; + let interceptor = TonicInterceptor::NoOp(NoOpInterceptor); + let _result = match factory { + 0 => scanner_control_time_out_client(&addr, interceptor) + .await + .map(|_| ()) + .map_err(|e| e.to_string()), + 1 => heal_control_time_out_client(&addr, interceptor) + .await + .map(|_| ()) + .map_err(|e| e.to_string()), + 2 => tier_mutation_control_time_out_client(&addr, interceptor) + .await + .map(|_| ()) + .map_err(|e| e.to_string()), + _ => node_service_time_out_client(&addr, interceptor) + .await + .map(|_| ()) + .map_err(|e| e.to_string()), + }; + assert!( + !runtime_sources::test_node_channel_is_cached(&addr).await, + "factory {factory} must evict an unbound HTTPS cache entry before attempting a fresh dial" + ); + } + } + #[derive(Clone)] struct EpochProofService { audience: String, diff --git a/crates/ecstore/src/runtime/sources.rs b/crates/ecstore/src/runtime/sources.rs index f121cffe5..b7db108bf 100644 --- a/crates/ecstore/src/runtime/sources.rs +++ b/crates/ecstore/src/runtime/sources.rs @@ -50,6 +50,7 @@ use s3s::dto::BucketLifecycleConfiguration; use s3s::region::Region; use tokio::sync::{OwnedRwLockReadGuard, RwLock}; use tokio_util::sync::CancellationToken; +#[cfg(test)] use tonic::transport::Channel; use uuid::Uuid; @@ -251,10 +252,6 @@ pub(crate) async fn root_disk_threshold_for_erasure_disk() -> Option { } } -pub(crate) async fn cached_node_channel(addr: &str) -> Option { - rustfs_common::cached_connection(addr).await -} - #[cfg(test)] pub(crate) async fn cache_test_node_channel(addr: String, channel: Channel) { rustfs_common::cache_connection(addr, channel).await; diff --git a/crates/policy/src/policy/action.rs b/crates/policy/src/policy/action.rs index e2c7742d5..d86952566 100644 --- a/crates/policy/src/policy/action.rs +++ b/crates/policy/src/policy/action.rs @@ -447,6 +447,10 @@ pub enum AdminAction { ServiceFreezeAdminAction, #[strum(serialize = "admin:ConfigUpdate")] ConfigUpdateAdminAction, + #[strum(serialize = "admin:TLSCertificatePublish")] + TLSCertificatePublishAdminAction, + #[strum(serialize = "admin:TLSCertificateStatus")] + TLSCertificateStatusAdminAction, #[strum(serialize = "admin:CreateUser")] CreateUserAdminAction, #[strum(serialize = "admin:DeleteUser")] @@ -646,6 +650,8 @@ impl AdminAction { | AdminAction::ServiceStopAdminAction | AdminAction::ServiceFreezeAdminAction | AdminAction::ConfigUpdateAdminAction + | AdminAction::TLSCertificatePublishAdminAction + | AdminAction::TLSCertificateStatusAdminAction | AdminAction::CreateUserAdminAction | AdminAction::DeleteUserAdminAction | AdminAction::ListUsersAdminAction @@ -941,6 +947,34 @@ mod tests { assert!(!Action::AdminAction(get_action).is_match(&Action::AdminAction(set_action))); } + #[test] + fn tls_certificate_actions_preserve_least_privilege() { + let publish = Action::try_from("admin:TLSCertificatePublish").expect("parse certificate publish action"); + let status = Action::try_from("admin:TLSCertificateStatus").expect("parse certificate status action"); + for (action, name) in [ + (&publish, "admin:TLSCertificatePublish"), + (&status, "admin:TLSCertificateStatus"), + ] { + let Action::AdminAction(admin_action) = action else { + panic!("certificate policy must use an admin action"); + }; + assert!(admin_action.is_valid()); + assert_eq!(serde_json::to_string(action).expect("serialize action"), format!("\"{name}\"")); + } + assert!(!status.is_match(&publish)); + assert!(!publish.is_match(&status)); + let config = Action::AdminAction(AdminAction::ConfigUpdateAdminAction); + let server_info = Action::AdminAction(AdminAction::ServerInfoAdminAction); + assert!(!config.is_match(&publish)); + assert!(!server_info.is_match(&publish)); + let admin = Action::AdminAction(AdminAction::AllAdminActions); + assert!(admin.is_match(&publish)); + assert!(admin.is_match(&status)); + let s3 = Action::try_from("s3:*").expect("parse S3 wildcard"); + assert!(!s3.is_match(&publish)); + assert!(!s3.is_match(&status)); + } + #[test] fn test_table_catalog_admin_action_is_valid() { let get_action = AdminAction::try_from("admin:GetTableCatalog").expect("Should parse GetTableCatalog action"); diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index a521c3a9f..451afa21a 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -25,8 +25,8 @@ use rustfs_common::{cache_connection, cached_connection, evict_connection_with_l use std::{ collections::HashMap, error::Error, - sync::LazyLock, sync::atomic::{AtomicU8, AtomicUsize, Ordering}, + sync::{Arc, LazyLock, Mutex as StdMutex, Weak}, time::{Duration, Instant}, }; use tokio::sync::Mutex; @@ -59,6 +59,44 @@ const INTERNODE_RPC_MSGPACK_ONLY_CACHE_UNSET: u8 = 0; const INTERNODE_RPC_MSGPACK_ONLY_CACHE_FALSE: u8 = 1; const INTERNODE_RPC_MSGPACK_ONLY_CACHE_TRUE: u8 = 2; static TLS_GENERATION_CACHE: LazyLock>> = LazyLock::new(|| Mutex::new(HashMap::new())); +// Lock order: generation metadata, then the common connection cache. The dial +// registry's standard mutex is released before waiting on any async lock. +static DIAL_LOCK_REGISTRY: LazyLock>>>> = LazyLock::new(|| StdMutex::new(HashMap::new())); + +struct DialLock { + key: String, + mutex: Arc>, +} + +impl DialLock { + fn for_key(key: &str) -> Self { + let mut registry = DIAL_LOCK_REGISTRY.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + registry.retain(|_, lock| lock.strong_count() != 0); + let mutex = registry.get(key).and_then(Weak::upgrade).unwrap_or_else(|| { + let mutex = Arc::new(Mutex::new(())); + registry.insert(key.to_string(), Arc::downgrade(&mutex)); + mutex + }); + Self { + key: key.to_string(), + mutex, + } + } +} + +impl Drop for DialLock { + fn drop(&mut self) { + let mut registry = DIAL_LOCK_REGISTRY.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if Arc::strong_count(&self.mutex) == 1 + && registry + .get(&self.key) + .is_some_and(|slot| Weak::ptr_eq(slot, &Arc::downgrade(&self.mutex))) + { + registry.remove(&self.key); + } + } +} + static INTERNODE_RPC_MSGPACK_ONLY_CACHE: AtomicU8 = AtomicU8::new(INTERNODE_RPC_MSGPACK_ONLY_CACHE_UNSET); fn enforce_tls_generation_cache_bound(generation_cache: &mut HashMap, generation: u64, addr: &str) { @@ -2805,6 +2843,53 @@ fn bulk_cache_key(addr: &str, idx: usize) -> String { format!("{addr}\u{0}bulk\u{0}{idx}") } +async fn cached_channel_for_tls_generation(dial_addr: &str, cache_key: &str) -> Option { + if !dial_addr.starts_with(RUSTFS_HTTPS_PREFIX) { + return cached_connection(cache_key).await; + } + let mut generations = TLS_GENERATION_CACHE.lock().await; + let Some(channel) = cached_connection(cache_key).await else { + generations.remove(cache_key); + return None; + }; + if generations.get(cache_key).copied() == Some(runtime_sources::outbound_tls_generation()) { + return Some(channel); + } + evict_connection_with_log_level(cache_key, ConnectionEvictionLogLevel::Debug).await; + generations.remove(cache_key); + runtime_sources::record_stale_grpc_channel_tls_generation(); + None +} + +async fn cache_dialed_channel(dial_addr: &str, cache_key: &str, channel: &Channel, generation: Option) -> bool { + if !dial_addr.starts_with(RUSTFS_HTTPS_PREFIX) { + cache_connection(cache_key.to_string(), channel.clone()).await; + return true; + } + let mut generations = TLS_GENERATION_CACHE.lock().await; + if generation != Some(runtime_sources::outbound_tls_generation()) { + runtime_sources::record_stale_grpc_channel_tls_generation(); + return false; + } + cache_connection(cache_key.to_string(), channel.clone()).await; + // Publication can advance while the common cache's async lock is awaited. + if generation != Some(runtime_sources::outbound_tls_generation()) { + evict_connection_with_log_level(cache_key, ConnectionEvictionLogLevel::Debug).await; + generations.remove(cache_key); + runtime_sources::record_stale_grpc_channel_tls_generation(); + return false; + } + if let Some(generation) = generation { + enforce_tls_generation_cache_bound(&mut generations, generation, cache_key); + generations.insert(cache_key.to_string(), generation); + } + true +} + +fn channel_acquisition_timeout(_: tokio::time::error::Elapsed) -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::TimedOut, "internode channel acquisition deadline elapsed") +} + /// Acquire a cached-or-newly-dialed channel for the given peer and channel class. /// /// For [`ChannelClass::Control`] (or whenever isolation is disabled) this is exactly the legacy @@ -2813,22 +2898,27 @@ fn bulk_cache_key(addr: &str, idx: usize) -> String { /// per-peer bulk pool and dialed on demand, physically isolating large transfers from /// control-plane RPCs. pub async fn get_channel_for_class(addr: &str, class: ChannelClass) -> Result> { - if class == ChannelClass::Control || !channel_isolation_enabled() { - if let Some(channel) = cached_connection(addr).await { - debug!("Using cached control gRPC channel for: {}", addr); + let deadline = tokio::time::Instant::now() + internode_connect_timeout(); + tokio::time::timeout_at(deadline, async { + if class == ChannelClass::Control || !channel_isolation_enabled() { + if let Some(channel) = cached_channel_for_tls_generation(addr, addr).await { + debug!("Using cached control gRPC channel for: {}", addr); + return Ok(channel); + } + return build_channel_until(addr, addr, deadline).await; + } + + let pool_size = bulk_channel_pool_size(); + let idx = BULK_CHANNEL_CURSOR.fetch_add(1, Ordering::Relaxed) % pool_size; + let cache_key = bulk_cache_key(addr, idx); + if let Some(channel) = cached_channel_for_tls_generation(addr, &cache_key).await { + debug!("Using cached bulk gRPC channel {} for: {}", idx, addr); return Ok(channel); } - return create_new_channel(addr).await; - } - - let pool_size = bulk_channel_pool_size(); - let idx = BULK_CHANNEL_CURSOR.fetch_add(1, Ordering::Relaxed) % pool_size; - let cache_key = bulk_cache_key(addr, idx); - if let Some(channel) = cached_connection(&cache_key).await { - debug!("Using cached bulk gRPC channel {} for: {}", idx, addr); - return Ok(channel); - } - build_channel(addr, &cache_key).await + build_channel_until(addr, &cache_key, deadline).await + }) + .await + .map_err(channel_acquisition_timeout)? } /// Creates a new gRPC channel with optimized keepalive settings for cluster resilience. @@ -2852,6 +2942,45 @@ pub async fn create_new_channel(addr: &str) -> Result> { /// for isolated bulk channels (see [`bulk_cache_key`]), letting several physically distinct /// channels to the same peer be cached independently. async fn build_channel(dial_addr: &str, cache_key: &str) -> Result> { + let deadline = tokio::time::Instant::now() + internode_connect_timeout(); + build_channel_until(dial_addr, cache_key, deadline).await +} + +async fn build_channel_until( + dial_addr: &str, + cache_key: &str, + deadline: tokio::time::Instant, +) -> Result> { + tokio::time::timeout_at(deadline, async { + if let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { + return Ok(channel); + } + let slot = DialLock::for_key(cache_key); + let _guard = slot.mutex.lock().await; + loop { + // A successful concurrent miss has already populated this exact key. + if let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { + return Ok(channel); + } + let (channel, generation) = dial_channel(dial_addr, cache_key).await?; + if cache_dialed_channel(dial_addr, cache_key, &channel, generation).await + && let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await + { + debug!( + "Successfully created and cached gRPC channel to: {} (cache key: {})", + dial_addr, cache_key + ); + return Ok(channel); + } + // A generation published during dial cannot be returned; rebuilding + // shares the original deadline with lock wait and all earlier dials. + } + }) + .await + .map_err(channel_acquisition_timeout)? +} + +async fn dial_channel(dial_addr: &str, cache_key: &str) -> Result<(Channel, Option), Box> { debug!("Creating new gRPC channel to: {} (cache key: {})", dial_addr, cache_key); let dial_started_at = Instant::now(); let connect_timeout = internode_connect_timeout(); @@ -2887,18 +3016,10 @@ async fn build_channel(dial_addr: &str, cache_key: &str) -> Result Result = tokio::sync::Mutex::const_new(()); + static INTERNODE_RPC_MSGPACK_ONLY_ENV_LOCK: Mutex<()> = Mutex::new(()); #[derive(Clone, PartialEq, prost::Message)] @@ -3144,6 +3253,243 @@ mod tests { assert_eq!(manifest_fields, proto_bin_json_fields("Response")); } + #[tokio::test] + async fn tls02_stale_https_control_and_bulk_cache_entries_are_evicted() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "https://tls02-stale-cache.invalid:9000"; + let generation = rustfs_tls_runtime::load_global_outbound_tls_generation().0; + let stale_generation = generation.checked_add(1).unwrap_or(0); + for key in [addr.to_string(), bulk_cache_key(addr, 0)] { + cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + TLS_GENERATION_CACHE.lock().await.insert(key.clone(), stale_generation); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_none(), + "TLS02_INVARIANT stale HTTPS cache entry cannot be returned" + ); + assert!( + !rustfs_common::has_cached_connection(&key).await, + "TLS02_INVARIANT stale entry is removed from the real connection cache" + ); + } + } + + #[tokio::test] + async fn tls02_missing_https_generation_control_and_bulk_cache_entries_are_evicted() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "https://tls02-missing-generation.invalid:9000"; + for key in [addr.to_string(), bulk_cache_key(addr, 0)] { + cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + TLS_GENERATION_CACHE.lock().await.remove(&key); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_none(), + "TLS02_INVARIANT HTTPS channel without a generation cannot be returned" + ); + assert!( + !rustfs_common::has_cached_connection(&key).await, + "TLS02_INVARIANT unbound entry is removed from the real connection cache" + ); + } + } + + #[tokio::test] + async fn tls02_current_https_control_and_bulk_cache_entries_are_reused() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "https://tls02-current-generation.invalid:9000"; + let generation = rustfs_tls_runtime::load_global_outbound_tls_generation().0; + for key in [addr.to_string(), bulk_cache_key(addr, 0)] { + cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + TLS_GENERATION_CACHE.lock().await.insert(key.clone(), generation); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_some(), + "TLS02_INVARIANT HTTPS channel at the canonical generation is reused" + ); + assert!(rustfs_common::has_cached_connection(&key).await); + evict_connection_with_log_level(&key, ConnectionEvictionLogLevel::Debug).await; + TLS_GENERATION_CACHE.lock().await.remove(&key); + } + } + + #[tokio::test] + async fn tls02_http_control_and_bulk_cache_ignore_tls_generation() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "http://tls02-http-cache.invalid:9000"; + let generation = rustfs_tls_runtime::load_global_outbound_tls_generation().0; + for key in [addr.to_string(), bulk_cache_key(addr, 0)] { + cache_connection(key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + TLS_GENERATION_CACHE + .lock() + .await + .insert(key.clone(), generation.checked_add(1).unwrap_or(0)); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_some(), + "TLS02_INVARIANT HTTP cache is independent of TLS generations" + ); + TLS_GENERATION_CACHE.lock().await.remove(&key); + assert!( + cached_channel_for_tls_generation(addr, &key).await.is_some(), + "TLS02_INVARIANT HTTP cache needs no TLS generation metadata" + ); + evict_connection_with_log_level(&key, ConnectionEvictionLogLevel::Debug).await; + } + } + + #[tokio::test] + async fn tls02_captured_identity_is_immutable_and_late_dial_keeps_its_actual_generation() { + use rustfs_common::MtlsIdentityPem; + use rustfs_tls_runtime::{OutboundTlsMaterial, TlsGeneration, publish_global_outbound_tls_state}; + let _test_guard = TLS02_TEST_LOCK.lock().await; + let previous = runtime_sources::outbound_tls_state().await; + let first_generation = previous.generation.0.checked_add(1).expect("test generation capacity"); + let first = OutboundTlsMaterial { + root_ca_pem: b"root-a".to_vec(), + mtls_identity: Some(MtlsIdentityPem { + cert_pem: b"certificate-a".to_vec(), + key_pem: b"test-key-a".to_vec(), + }), + }; + publish_global_outbound_tls_state(TlsGeneration(first_generation), &first).await; + let captured = runtime_sources::outbound_tls_state().await; + let second = OutboundTlsMaterial { + root_ca_pem: b"root-b".to_vec(), + mtls_identity: Some(MtlsIdentityPem { + cert_pem: b"certificate-b".to_vec(), + key_pem: b"test-key-b".to_vec(), + }), + }; + publish_global_outbound_tls_state(TlsGeneration(first_generation + 1), &second).await; + let current = runtime_sources::outbound_tls_state().await; + let addr = "https://tls02-late-dial.invalid:9000"; + let channel = Endpoint::from_static("http://127.0.0.1:1").connect_lazy(); + let cached = cache_dialed_channel(addr, addr, &channel, Some(captured.generation.0)).await; + let original_key = captured.mtls_identity.as_ref().expect("captured identity").key_pem.clone(); + let current_key = current.mtls_identity.as_ref().expect("current identity").key_pem.clone(); + let restore = OutboundTlsMaterial { + root_ca_pem: previous.root_ca_pem.unwrap_or_default(), + mtls_identity: previous.mtls_identity, + }; + publish_global_outbound_tls_state(TlsGeneration(first_generation + 2), &restore).await; + assert_eq!( + original_key, b"test-key-a", + "TLS02_INVARIANT captured identity cannot change with publication" + ); + assert_eq!(captured.root_ca_pem.as_deref(), Some(b"root-a".as_slice())); + assert_eq!(current_key, b"test-key-b"); + assert!(!cached, "TLS02_INVARIANT late dial cannot be relabeled with the current generation"); + assert!( + !rustfs_common::has_cached_connection(addr).await, + "TLS02_INVARIANT late stale dial is never cached" + ); + assert!(!TLS_GENERATION_CACHE.lock().await.contains_key(addr)); + } + + #[tokio::test] + async fn tls02_concurrent_control_and_bulk_misses_merge_successful_real_dials() { + use tonic::codegen::tokio_stream::{StreamExt, wrappers::TcpListenerStream}; + let _test_guard = TLS02_TEST_LOCK.lock().await; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind HTTP/2 test server"); + let addr = format!("http://{}", listener.local_addr().expect("test server address")); + let accepted = Arc::new(AtomicUsize::new(0)); + let counter = accepted.clone(); + let incoming = TcpListenerStream::new(listener).map(move |stream| { + if stream.is_ok() { + counter.fetch_add(1, Ordering::SeqCst); + } + stream + }); + let server = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_routes(tonic::service::Routes::default()) + .serve_with_incoming(incoming) + .await + }); + for key in [addr.clone(), bulk_cache_key(&addr, 0)] { + let mut tasks = Vec::new(); + for _ in 0..32 { + let dial_addr = addr.clone(); + let cache_key = key.clone(); + tasks.push(tokio::spawn(async move { + build_channel(&dial_addr, &cache_key).await.map_err(|error| error.to_string()) + })); + } + for task in tasks { + let channel = task.await.expect("channel acquisition task").expect("real HTTP/2 dial"); + drop(channel); + } + assert!(rustfs_common::has_cached_connection(&key).await); + assert!( + !DIAL_LOCK_REGISTRY.lock().expect("dial registry").contains_key(&key), + "TLS02_INVARIANT completed miss groups leave no lock entries" + ); + } + // Tonic returns its channel before the server task necessarily polls the + // already-established sockets out of the listener's accept queue. + tokio::time::sleep(Duration::from_millis(20)).await; + let connections = accepted.load(Ordering::SeqCst); + evict_connection_with_log_level(&addr, ConnectionEvictionLogLevel::Debug).await; + evict_connection_with_log_level(&bulk_cache_key(&addr, 0), ConnectionEvictionLogLevel::Debug).await; + server.abort(); + let _ = server.await; + assert_eq!(connections, 2, "TLS02_INVARIANT 32 misses per control/bulk key dial once each"); + } + + #[tokio::test] + async fn tls02_concurrent_waiters_share_bounded_total_deadlines_and_release_registry() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let addr = "http://tls02-contended-dial.invalid:9000"; + let owner = DialLock::for_key(addr); + let guard = owner.mutex.lock().await; + let mut tasks = Vec::new(); + for _ in 0..8 { + tasks.push(tokio::spawn(async move { + let deadline = tokio::time::Instant::now() + Duration::from_millis(40); + let result = build_channel_until(addr, addr, deadline).await; + result.err().is_some_and(|error| { + error + .downcast_ref::() + .is_some_and(|error| error.kind() == std::io::ErrorKind::TimedOut) + }) + })); + } + let completed = tokio::time::timeout(Duration::from_millis(250), async { + for task in tasks { + assert!( + task.await.expect("deadline waiter"), + "TLS02_INVARIANT wait and rebuild share a typed timeout" + ); + } + }) + .await; + drop(guard); + drop(owner); + assert!( + completed.is_ok(), + "TLS02_INVARIANT eight waiters cannot accumulate eight connect timeouts" + ); + assert!( + !DIAL_LOCK_REGISTRY.lock().expect("dial registry").contains_key(addr), + "TLS02_INVARIANT cancelled waiters release the lock registry" + ); + } + + #[test] + fn tls02_dial_registry_reclaims_unique_keys_and_preserves_live_lock_identity() { + for index in 0..1000 { + let key = format!("http://tls02-registry-{index}.invalid"); + let first = DialLock::for_key(&key); + let second = DialLock::for_key(&key); + assert!(Arc::ptr_eq(&first.mutex, &second.mutex), "TLS02_INVARIANT one live lock per cache key"); + drop(first); + assert!(DIAL_LOCK_REGISTRY.lock().expect("dial registry").contains_key(&key)); + drop(second); + assert!( + !DIAL_LOCK_REGISTRY.lock().expect("dial registry").contains_key(&key), + "TLS02_INVARIANT last owner removes its own weak entry" + ); + } + } + #[test] fn enforce_tls_generation_cache_bound_evicts_when_retained_entries_still_full() { let mut cache = HashMap::new(); diff --git a/crates/protos/src/runtime_sources.rs b/crates/protos/src/runtime_sources.rs index 135e3a9f5..ac685d890 100644 --- a/crates/protos/src/runtime_sources.rs +++ b/crates/protos/src/runtime_sources.rs @@ -22,6 +22,10 @@ pub(crate) async fn outbound_tls_state() -> GlobalPublishedOutboundTlsState { load_global_outbound_tls_state().await } +pub(crate) fn outbound_tls_generation() -> u64 { + rustfs_tls_runtime::load_global_outbound_tls_generation().0 +} + pub(crate) fn record_stale_grpc_channel_tls_generation() { record_tls_consumer_stale_generation(PROTOS_GRPC_CHANNEL_TLS_CONSUMER); } diff --git a/crates/tls-runtime/Cargo.toml b/crates/tls-runtime/Cargo.toml index fb8920547..443be5ea0 100644 --- a/crates/tls-runtime/Cargo.toml +++ b/crates/tls-runtime/Cargo.toml @@ -46,6 +46,7 @@ sha2.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["fs", "rt", "sync", "time"] } tracing.workspace = true +x509-parser.workspace = true [dev-dependencies] rcgen.workspace = true diff --git a/crates/tls-runtime/src/certs.rs b/crates/tls-runtime/src/certs.rs index 20260768f..a79c8e9c0 100644 --- a/crates/tls-runtime/src/certs.rs +++ b/crates/tls-runtime/src/certs.rs @@ -429,6 +429,209 @@ pub fn load_all_certs_from_directory( Ok(cert_key_pairs) } +/// Strict runtime loader: a discovered identity must never disappear through +/// best-effort filtering. The diagnostic inspection API remains independent. +pub fn load_all_certs_from_directory_strict( + options: CertDirectoryLoadOptions, +) -> Result>, PrivateKeyDer<'static>)>, crate::error::TlsRuntimeError> { + use crate::error::TlsRuntimeError; + options.validate()?; + let dir = &options.dir_path; + let mut pairs = HashMap::new(); + if let Some(pair) = read_cert_key_pair_strict(dir, &options.cert_filename, &options.key_filename, "default", true)? { + pairs.insert("default".to_string(), pair); + } + for entry in fs::read_dir(dir).map_err(|source| TlsRuntimeError::SourceIo { + path: dir.clone(), + source, + })? { + let entry = entry.map_err(|source| TlsRuntimeError::SourceIo { + path: dir.clone(), + source, + })?; + let path = entry.path(); + let name = entry.file_name(); + let name = name.to_str().ok_or_else(|| TlsRuntimeError::InvalidCertificate { + name: "SNI directory".to_string(), + reason: "directory name is not UTF-8".to_string(), + })?; + if !is_discoverable_cert_domain_dir(name) { + continue; + } + let metadata = fs::metadata(&path).map_err(|source| TlsRuntimeError::SourceIo { + path: path.clone(), + source, + })?; + if !metadata.is_dir() { + continue; + } + if let Some(pair) = read_cert_key_pair_strict(&path, &options.cert_filename, &options.key_filename, name, true)? { + pairs.insert(name.to_string(), pair); + } + } + Ok(pairs) +} + +pub(crate) fn read_optional_tls_file(path: &Path) -> Result>, crate::error::TlsRuntimeError> { + use crate::error::TlsRuntimeError; + match fs::symlink_metadata(path) { + Ok(_) => fs::read(path).map(Some).map_err(|source| TlsRuntimeError::SourceIo { + path: path.to_path_buf(), + source, + }), + Err(error) if error.kind() == io::ErrorKind::NotFound => { + // A disappearing pinned generation is an error, not optional absence. + if let Some(parent) = path.parent() { + fs::read_dir(parent).map_err(|source| TlsRuntimeError::SourceIo { + path: parent.to_path_buf(), + source, + })?; + } + Ok(None) + } + Err(source) => Err(TlsRuntimeError::SourceIo { + path: path.to_path_buf(), + source, + }), + } +} + +pub(crate) fn parse_certificate_pem( + pem: &[u8], + name: &str, +) -> Result>, crate::error::TlsRuntimeError> { + use crate::error::TlsRuntimeError; + let certs = CertificateDer::pem_slice_iter(pem) + .collect::, _>>() + .map_err(|error| TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: error.to_string(), + })?; + if certs.is_empty() { + return Err(TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: "empty certificate PEM".to_string(), + }); + } + for cert in &certs { + let (remaining, _) = + x509_parser::parse_x509_certificate(cert.as_ref()).map_err(|error| TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: error.to_string(), + })?; + if !remaining.is_empty() { + return Err(TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: "trailing certificate DER".to_string(), + }); + } + } + Ok(certs) +} + +pub(crate) fn parse_private_key_pem(pem: &[u8], name: &str) -> Result, crate::error::TlsRuntimeError> { + PrivateKeyDer::from_pem_slice(pem).map_err(|error| crate::error::TlsRuntimeError::InvalidPrivateKey { + name: name.to_string(), + reason: error.to_string(), + }) +} + +pub(crate) fn read_cert_key_pair_strict( + dir: &Path, + cert_filename: &str, + key_filename: &str, + name: &str, + server: bool, +) -> Result>, PrivateKeyDer<'static>)>, crate::error::TlsRuntimeError> { + let cert = read_optional_tls_file(&dir.join(cert_filename))?; + let key = read_optional_tls_file(&dir.join(key_filename))?; + match (cert, key) { + (None, None) => Ok(None), + (Some(cert), Some(key)) => { + let certs = parse_certificate_pem(&cert, name)?; + let key = parse_private_key_pem(&key, name)?; + validate_certified_key(&certs, &key, name, server)?; + Ok(Some((certs, key))) + } + _ => Err(crate::error::TlsRuntimeError::IncompleteIdentity { name: name.to_string() }), + } +} + +/// Validate before publication, including the default certificate that bypasses +/// rustls' SNI name checks. +pub fn validate_certified_key( + certs: &[CertificateDer<'static>], + key: &PrivateKeyDer<'static>, + name: &str, + server: bool, +) -> Result { + use crate::error::TlsRuntimeError; + if certs.is_empty() { + return Err(TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: "empty certificate chain".to_string(), + }); + } + for (index, cert) in certs.iter().enumerate() { + let (remaining, parsed) = + x509_parser::parse_x509_certificate(cert.as_ref()).map_err(|error| TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: error.to_string(), + })?; + if !remaining.is_empty() { + return Err(TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: "trailing certificate DER".to_string(), + }); + } + if index == 0 { + if !parsed.validity().is_valid() { + return Err(TlsRuntimeError::CertificateValidity { name: name.to_string() }); + } + if parsed + .extended_key_usage() + .map_err(|error| TlsRuntimeError::InvalidCertificate { + name: name.to_string(), + reason: error.to_string(), + })? + .is_some_and(|eku| { + if server { + !eku.value.server_auth + } else { + !eku.value.client_auth + } + }) + { + return Err(if server { + TlsRuntimeError::InvalidServerEku { name: name.to_string() } + } else { + TlsRuntimeError::InvalidClientEku { name: name.to_string() } + }); + } + } + } + let certified = CertifiedKey::from_der(certs.to_vec(), key.clone_key(), &rustls::crypto::aws_lc_rs::default_provider()) + .map_err(|source| TlsRuntimeError::InvalidCertifiedKey { + name: name.to_string(), + source, + })?; + certified + .keys_match() + .map_err(|source| TlsRuntimeError::InvalidCertifiedKey { + name: name.to_string(), + source, + })?; + if server && name != "default" { + let mut sni = ResolvesServerCertUsingSni::new(); + sni.add(name, certified.clone()) + .map_err(|source| TlsRuntimeError::InvalidCertifiedKey { + name: name.to_string(), + source, + })?; + } + Ok(certified) +} + fn load_cert_key_pair(cert_path: &str, key_path: &str) -> io::Result<(Vec>, PrivateKeyDer<'static>)> { let certs = load_certs(cert_path)?; let key = load_private_key(key_path)?; @@ -494,10 +697,7 @@ pub fn create_multi_cert_resolver( let mut default_cert = None; for (domain, (certs, key)) in cert_key_pairs { - let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key) - .map_err(|e| certs_error(format!("unsupported private key types:{domain}, err:{e:?}")))?; - - let certified_key = CertifiedKey::new(certs, signing_key); + let certified_key = validate_certified_key(&certs, &key, &domain, true).map_err(io::Error::other)?; if domain == "default" { default_cert = Some(Arc::new(certified_key.clone())); } else { diff --git a/crates/tls-runtime/src/error.rs b/crates/tls-runtime/src/error.rs index ac7d50691..601624d6a 100644 --- a/crates/tls-runtime/src/error.rs +++ b/crates/tls-runtime/src/error.rs @@ -23,6 +23,38 @@ pub enum TlsRuntimeError { DirectoryNotFound { path: PathBuf }, #[error("TLS path is not a directory: {path}")] NotADirectory { path: PathBuf }, + #[error("TLS source I/O error at {path}: {source}")] + SourceIo { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("incomplete TLS certificate/private key pair for {name}")] + IncompleteIdentity { name: String }, + #[error("invalid TLS certificate material for {name}: {reason}")] + InvalidCertificate { name: String, reason: String }, + #[error("invalid TLS private key for {name}: {reason}")] + InvalidPrivateKey { name: String, reason: String }, + #[error("TLS certificate/private key validation failed for {name}: {source}")] + InvalidCertifiedKey { + name: String, + #[source] + source: rustls::Error, + }, + #[error("invalid TLS trust anchor in {name}: {source}")] + InvalidTrustAnchor { + name: String, + #[source] + source: rustls::Error, + }, + #[error("TLS certificate for {name} is outside its validity interval")] + CertificateValidity { name: String }, + #[error("TLS certificate for {name} does not permit server authentication")] + InvalidServerEku { name: String }, + #[error("TLS certificate for {name} does not permit client authentication")] + InvalidClientEku { name: String }, + #[error("TLS source loading task failed: {0}")] + SourceTaskJoin(#[from] tokio::task::JoinError), #[error("TLS material error: {0}")] Material(String), #[error("TLS publication error: {0}")] diff --git a/crates/tls-runtime/src/lib.rs b/crates/tls-runtime/src/lib.rs index 1d5bcf232..ae25ed86b 100644 --- a/crates/tls-runtime/src/lib.rs +++ b/crates/tls-runtime/src/lib.rs @@ -28,7 +28,8 @@ pub mod state; pub use certs::{ CertDirectoryLoadOptions, TlsCertPairInspection, TlsCertPairStatus, TlsDirectoryInspection, TlsDomainInspection, WebPkiClientVerifierOptions, build_webpki_client_verifier, create_multi_cert_resolver, inspect_cert_directory, - load_all_certs_from_directory, load_cert_bundle_der_bytes, load_certs, load_private_key, + load_all_certs_from_directory, load_all_certs_from_directory_strict, load_cert_bundle_der_bytes, load_certs, + load_private_key, validate_certified_key, }; pub use config::{ReloadApplyHint, ReloadDetectMode, TlsReloadOptions}; pub use coordinator::{TlsConsumer, TlsReloadCoordinator}; @@ -45,7 +46,7 @@ pub use outbound::{ load_global_outbound_tls_state, publish_global_outbound_tls_state, summarize_global_outbound_tls_state, }; pub use server::{ReloadableServerCertResolver, spawn_server_cert_reload_loop}; -pub use source::{TlsFileLayout, TlsSource, TlsSourceKind}; +pub use source::{TlsFileLayout, TlsSource, TlsSourceKind, TlsSourceSnapshot}; pub use state::OutboundOnlySnapshotArgs; pub use state::{TlsGeneration, TlsPublishedState, TlsReloadRuntimeState, TlsRuntimeStatusSnapshot}; pub use state::{TlsRuntimeConsumerSection, TlsRuntimeOutboundSection, TlsRuntimeRuntimeSection, TlsRuntimeServerSection}; @@ -111,6 +112,8 @@ mod tests { let coordinator = TlsReloadCoordinator::new(source.clone(), TlsReloadOptions::default()); let snapshot = TlsMaterialSnapshot { source, + server_client_ca_pem: None, + material_files: std::collections::BTreeSet::new(), server: Some(ServerTlsMaterial::MultiCert { cert_key_pairs: HashMap::new(), }), diff --git a/crates/tls-runtime/src/material.rs b/crates/tls-runtime/src/material.rs index f6b3fab99..5f8f200e5 100644 --- a/crates/tls-runtime/src/material.rs +++ b/crates/tls-runtime/src/material.rs @@ -12,15 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::certs::{CertDirectoryLoadOptions, load_all_certs_from_directory, load_certs, load_private_key}; +use crate::certs::{ + CertDirectoryLoadOptions, load_all_certs_from_directory_strict, parse_certificate_pem, parse_private_key_pem, + read_optional_tls_file, validate_certified_key, +}; use crate::error::TlsRuntimeError; use crate::fingerprint::TlsFingerprint; -use crate::source::TlsSource; +use crate::source::{TlsSource, TlsSourceSnapshot}; use rustfs_common::MtlsIdentityPem; -use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject}; -use std::collections::HashMap; -use std::io::Cursor; -use std::io::ErrorKind; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use std::collections::{BTreeSet, HashMap}; use std::path::Path; #[derive(Debug, Clone)] @@ -40,115 +41,196 @@ pub enum ServerTlsMaterial { }, } +impl ServerTlsMaterial { + /// Revalidate an accepted generation without reading the filesystem. + pub fn validate(&self) -> Result<(), TlsRuntimeError> { + match self { + Self::SingleCert { certs, key } => { + crate::certs::validate_certified_key(certs, key, "default", true)?; + } + Self::MultiCert { cert_key_pairs } => { + for (name, (certs, key)) in cert_key_pairs { + crate::certs::validate_certified_key(certs, key, name, true)?; + } + } + } + Ok(()) + } +} + +impl Clone for ServerTlsMaterial { + fn clone(&self) -> Self { + match self { + Self::SingleCert { certs, key } => Self::SingleCert { + certs: certs.clone(), + key: key.clone_key(), + }, + Self::MultiCert { cert_key_pairs } => Self::MultiCert { + cert_key_pairs: cert_key_pairs + .iter() + .map(|(name, (certs, key))| (name.clone(), (certs.clone(), key.clone_key()))) + .collect(), + }, + } + } +} + #[derive(Debug)] pub struct TlsMaterialSnapshot { + /// The resolved directory generation used by every consumer of this snapshot. pub source: TlsSource, pub server: Option, pub outbound: OutboundTlsMaterial, + pub server_client_ca_pem: Option>, + pub material_files: BTreeSet, pub fingerprint: TlsFingerprint, } impl TlsMaterialSnapshot { pub async fn load(source: &TlsSource) -> Result { - let base_dir = source.validate_directory()?.to_path_buf(); + let source = source.clone(); + tokio::task::spawn_blocking(move || Self::load_from_snapshot(&source.snapshot()?)).await? + } - let server = load_server_material(source, &base_dir)?; + /// Load all consumers' material from one previously pinned directory generation. + pub fn load_from_snapshot(snapshot: &TlsSourceSnapshot) -> Result { + Self::load_from_snapshot_with_server_directory(snapshot, None) + } - let public_ca_path = base_dir.join(&source.layout.public_ca_filename); - let client_ca_path = base_dir.join(&source.fallback_ca_filename); - let client_cert_path = base_dir.join(&source.layout.client_cert_filename); - let client_key_path = base_dir.join(&source.layout.client_key_filename); + /// Load a server-only publication without replacing the pinned trust or client identity source. + pub fn load_from_snapshot_with_server_directory( + snapshot: &TlsSourceSnapshot, + server_directory: Option<&Path>, + ) -> Result { + snapshot.validate_available()?; + let source = snapshot.resolved_source(); + let base_dir = snapshot.base_dir(); + let server = load_server_material(source, server_directory.unwrap_or(base_dir))?; + Self::load_from_snapshot_with_loaded_server(snapshot, server) + } - let public_ca_pem = tokio::fs::read(&public_ca_path).await.ok(); - let client_ca_pem = tokio::fs::read(&client_ca_path).await.ok(); - let root_ca_pem = combine_optional_pem(public_ca_pem.as_deref(), client_ca_pem.as_deref()); + /// Validate an already decoded default server chain while reading trust and client identity from the pinned source. + /// Serving validity, EKU and key matching are checked here even when the publication ledger was verified earlier. + pub fn load_from_snapshot_with_server_identity( + snapshot: &TlsSourceSnapshot, + certificates: Vec>, + private_key_pem: &[u8], + ) -> Result { + snapshot.validate_available()?; + let key = parse_private_key_pem(private_key_pem, "default")?; + validate_certified_key(&certificates, &key, "default", true)?; + Self::load_from_snapshot_with_loaded_server( + snapshot, + Some(ServerTlsMaterial::SingleCert { + certs: certificates, + key, + }), + ) + } - let mtls_identity = match ( - tokio::fs::read(&client_cert_path).await.ok(), - tokio::fs::read(&client_key_path).await.ok(), - ) { - (Some(cert_pem), Some(key_pem)) => { - let mut cert_reader = Cursor::new(&cert_pem); - if CertificateDer::pem_reader_iter(&mut cert_reader).next().is_none() { - return Err(TlsRuntimeError::Material("no valid certificate in client cert PEM".to_string())); + fn load_from_snapshot_with_loaded_server( + snapshot: &TlsSourceSnapshot, + server: Option, + ) -> Result { + snapshot.validate_available()?; + let source = snapshot.resolved_source(); + let base_dir = snapshot.base_dir(); + let public_ca_pem = read_optional_tls_file(&base_dir.join(&source.layout.public_ca_filename))?; + let fallback_ca_pem = read_optional_tls_file(&base_dir.join(&source.fallback_ca_filename))?; + let client_ca_pem = read_optional_tls_file(&base_dir.join(&source.layout.client_ca_filename))?; + let mut material_files = BTreeSet::new(); + for (name, present) in [ + (&source.layout.public_ca_filename, public_ca_pem.is_some()), + (&source.fallback_ca_filename, fallback_ca_pem.is_some()), + (&source.layout.client_ca_filename, client_ca_pem.is_some()), + ] { + if present { + material_files.insert(name.clone()); + } + } + for (name, pem) in [ + (&source.layout.public_ca_filename, &public_ca_pem), + (&source.fallback_ca_filename, &fallback_ca_pem), + (&source.layout.client_ca_filename, &client_ca_pem), + ] { + if let Some(pem) = pem { + let certs = parse_certificate_pem(pem, name)?; + let mut roots = rustls::RootCertStore::empty(); + for cert in certs { + roots.add(cert).map_err(|source| TlsRuntimeError::InvalidTrustAnchor { + name: name.clone(), + source, + })?; } - - let mut key_reader = Cursor::new(&key_pem); - PrivateKeyDer::from_pem_reader(&mut key_reader) - .map_err(|e| TlsRuntimeError::Material(format!("invalid client key PEM: {e}")))?; - + } + } + let server_client_ca_pem = client_ca_pem.or_else(|| fallback_ca_pem.clone()); + let root_ca_pem = combine_optional_pem(public_ca_pem.as_deref(), fallback_ca_pem.as_deref()); + let client_cert_pem = read_optional_tls_file(&base_dir.join(&source.layout.client_cert_filename))?; + let client_key_pem = read_optional_tls_file(&base_dir.join(&source.layout.client_key_filename))?; + if client_cert_pem.is_some() { + material_files.insert(source.layout.client_cert_filename.clone()); + } + if client_key_pem.is_some() { + material_files.insert(source.layout.client_key_filename.clone()); + } + let mtls_identity = match (client_cert_pem, client_key_pem) { + (Some(cert_pem), Some(key_pem)) => { + let certs = parse_certificate_pem(&cert_pem, "client identity")?; + let key = parse_private_key_pem(&key_pem, "client identity")?; + validate_certified_key(&certs, &key, "client identity", false)?; Some(MtlsIdentityPem { cert_pem, key_pem }) } - _ => None, + (None, None) => None, + _ => { + return Err(TlsRuntimeError::IncompleteIdentity { + name: "client identity".to_string(), + }); + } }; - + snapshot.validate_available()?; let outbound = OutboundTlsMaterial { - root_ca_pem: root_ca_pem.clone(), + root_ca_pem, mtls_identity: mtls_identity.clone(), }; - - let server_fingerprint_bytes = server.as_ref().map(serialize_server_material_for_fingerprint); + let mut server_fingerprint_bytes = server.as_ref().map(serialize_server_material_for_fingerprint); + if let Some(pem) = &server_client_ca_pem { + let bytes = server_fingerprint_bytes.get_or_insert_with(Vec::new); + bytes.extend_from_slice(b"\0server-client-ca\0"); + bytes.extend_from_slice(pem); + } let fingerprint = TlsFingerprint::from_optional_bytes( server_fingerprint_bytes.as_deref(), public_ca_pem.as_deref(), - client_ca_pem.as_deref(), + fallback_ca_pem.as_deref(), mtls_identity.as_ref().map(|identity| identity.cert_pem.as_slice()), mtls_identity.as_ref().map(|identity| identity.key_pem.as_slice()), ); - Ok(Self { - source: source.clone(), + source: snapshot.resolved_source().clone(), server, outbound, + server_client_ca_pem, + material_files, fingerprint, }) } } fn load_server_material(source: &TlsSource, base_dir: &Path) -> Result, TlsRuntimeError> { - let root_cert = base_dir.join(&source.layout.server_cert_filename); - let root_key = base_dir.join(&source.layout.server_key_filename); - - let has_root_pair = root_cert.exists() && root_key.exists(); - let cert_key_pairs = load_all_certs_from_directory( + let mut cert_key_pairs = load_all_certs_from_directory_strict( CertDirectoryLoadOptions::builder(base_dir, &source.layout.server_cert_filename, &source.layout.server_key_filename) .build(), - ); - - match cert_key_pairs { - Ok(cert_key_pairs) if cert_key_pairs.len() > 1 || cert_key_pairs.keys().any(|key| key != "default") => { - Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs })) - } - Ok(cert_key_pairs) if !cert_key_pairs.is_empty() => { - if let Some((certs, key)) = cert_key_pairs.get("default") { - return Ok(Some(ServerTlsMaterial::SingleCert { - certs: certs.clone(), - key: key.clone_key(), - })); - } - Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs })) - } - Ok(_) => Ok(None), - Err(_err) if has_root_pair => { - let root_cert_path = path_to_utf8_str(&root_cert, "root TLS certificate")?; - let root_key_path = path_to_utf8_str(&root_key, "root TLS private key")?; - let certs = load_certs(root_cert_path) - .map_err(|e| TlsRuntimeError::Material(format!("load root TLS certificate {}: {e}", root_cert.display())))?; - let key = load_private_key(root_key_path) - .map_err(|e| TlsRuntimeError::Material(format!("load root TLS private key {}: {e}", root_key.display())))?; - Ok(Some(ServerTlsMaterial::SingleCert { certs, key })) - } - Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), - Err(err) => Err(TlsRuntimeError::Material(format!( - "discover server TLS certificates under '{}': {err}", - base_dir.display() - ))), + )?; + if cert_key_pairs.is_empty() { + return Ok(None); } -} - -fn path_to_utf8_str<'a>(path: &'a Path, description: &str) -> Result<&'a str, TlsRuntimeError> { - path.to_str() - .ok_or_else(|| TlsRuntimeError::Material(format!("{description} path '{}' is not valid UTF-8", path.display()))) + if cert_key_pairs.len() == 1 { + if let Some((certs, key)) = cert_key_pairs.remove("default") { + return Ok(Some(ServerTlsMaterial::SingleCert { certs, key })); + } + } + Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs })) } fn combine_optional_pem(primary: Option<&[u8]>, fallback: Option<&[u8]>) -> Vec { @@ -198,3 +280,425 @@ pub(crate) fn server_material_fingerprint(material: &ServerTlsMaterial) -> TlsFi let bytes = serialize_server_material_for_fingerprint(material); TlsFingerprint::from_optional_bytes(Some(&bytes), None, None, None, None) } + +#[cfg(test)] +mod tests { + use super::*; + use rcgen::{CertificateParams, ExtendedKeyUsagePurpose, KeyPair}; + use std::fs; + use tempfile::TempDir; + + fn write_pair(dir: &Path, domain: &str) { + let pair = rcgen::generate_simple_self_signed(vec![domain.to_string()]).expect("generate certificate"); + fs::write(dir.join("rustfs_cert.pem"), pair.cert.pem()).expect("write certificate"); + fs::write(dir.join("rustfs_key.pem"), pair.signing_key.serialize_pem()).expect("write private key"); + } + + #[test] + fn provided_server_identity_does_not_reread_server_files_and_preserves_trust() { + let base = TempDir::new().expect("base source"); + let serving = rcgen::generate_simple_self_signed(vec!["example.com".into()]).expect("server identity"); + let trust = rcgen::generate_simple_self_signed(vec!["trust.example".into()]).expect("trust source"); + fs::write(base.path().join("rustfs_cert.pem"), "unusable server certificate").expect("poison base certificate"); + fs::write(base.path().join("rustfs_key.pem"), "unusable server key").expect("poison base key"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), trust.cert.pem()).expect("trust file"); + let source = TlsSource::from_directory(base.path()).snapshot().expect("source"); + let loaded = TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![serving.cert.der().clone()], + serving.signing_key.serialize_pem().as_bytes(), + ) + .expect("provided identity"); + assert_eq!(loaded.outbound.root_ca_pem, trust.cert.pem().as_bytes()); + assert!( + matches!(loaded.server, Some(ServerTlsMaterial::SingleCert { certs, .. }) if certs == vec![serving.cert.der().clone()]) + ); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), "corrupt trust").expect("corrupt independent trust"); + assert!( + TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![serving.cert.der().clone()], + serving.signing_key.serialize_pem().as_bytes() + ) + .is_err(), + "providing server material must not bypass trust validation" + ); + } + + #[test] + fn provided_server_identity_preserves_key_time_and_eku_validation() { + let base = TempDir::new().expect("base source"); + let source = TlsSource::from_directory(base.path()).snapshot().expect("source"); + let serving = rcgen::generate_simple_self_signed(vec!["example.com".into()]).expect("server identity"); + let wrong_key = KeyPair::generate().expect("wrong key"); + assert!( + TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![serving.cert.der().clone()], + wrong_key.serialize_pem().as_bytes() + ) + .is_err(), + "key matching remains mandatory" + ); + let mut expired = CertificateParams::new(vec!["example.com".into()]).expect("parameters"); + expired.not_before = rcgen::date_time_ymd(2000, 1, 1); + expired.not_after = rcgen::date_time_ymd(2001, 1, 1); + let cert = expired.self_signed(&wrong_key).expect("expired certificate"); + assert!( + TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![cert.der().clone()], + wrong_key.serialize_pem().as_bytes() + ) + .is_err(), + "serving validity remains mandatory" + ); + let mut client = CertificateParams::new(vec!["example.com".into()]).expect("parameters"); + client.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth]; + let cert = client.self_signed(&wrong_key).expect("client certificate"); + assert!( + TlsMaterialSnapshot::load_from_snapshot_with_server_identity( + &source, + vec![cert.der().clone()], + wrong_key.serialize_pem().as_bytes() + ) + .is_err(), + "server EKU remains mandatory" + ); + } + + #[tokio::test] + async fn server_overlay_survives_expired_bootstrap_without_replacing_trust() { + let base = TempDir::new().expect("base source"); + let overlay = TempDir::new().expect("server overlay"); + let key = KeyPair::generate().expect("bootstrap key"); + let mut params = CertificateParams::new(vec!["example.com".into()]).expect("bootstrap params"); + params.not_before = rcgen::date_time_ymd(2000, 1, 1); + params.not_after = rcgen::date_time_ymd(2001, 1, 1); + let expired = params.self_signed(&key).expect("expired bootstrap"); + fs::write(base.path().join("rustfs_cert.pem"), expired.pem()).expect("bootstrap certificate"); + fs::write(base.path().join("rustfs_key.pem"), key.serialize_pem()).expect("bootstrap key"); + let ca = rcgen::generate_simple_self_signed(vec!["trust.example".into()]).expect("trust source"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), ca.cert.pem()).expect("trust file"); + write_pair(overlay.path(), "example.com"); + let source = TlsSource::from_directory(base.path()); + assert!( + TlsMaterialSnapshot::load(&source).await.is_err(), + "the expired base alone must remain invalid" + ); + let loaded = TlsMaterialSnapshot::load_from_snapshot_with_server_directory( + &source.snapshot().expect("pinned base"), + Some(overlay.path()), + ) + .expect("valid overlay"); + assert_eq!(loaded.outbound.root_ca_pem, ca.cert.pem().as_bytes()); + assert!(matches!(loaded.server, Some(ServerTlsMaterial::SingleCert { .. }))); + } + + #[tokio::test] + async fn snapshot_rejects_mismatched_default_private_key() { + let dir = TempDir::new().expect("create temp directory"); + write_pair(dir.path(), "example.com"); + let other = KeyPair::generate().expect("generate other key"); + fs::write(dir.path().join("rustfs_key.pem"), other.serialize_pem()).expect("replace key"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_partial_sni_pair_even_with_valid_default() { + let dir = TempDir::new().expect("create temp directory"); + write_pair(dir.path(), "example.com"); + let domain = dir.path().join("sni.example.com"); + fs::create_dir(&domain).expect("create SNI directory"); + fs::write(domain.join("rustfs_cert.pem"), "invalid certificate").expect("write partial SNI pair"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_invalid_existing_ca() { + let dir = TempDir::new().expect("create temp directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + fs::write(dir.path().join(&source.layout.public_ca_filename), "invalid CA").expect("write invalid CA"); + assert!(TlsMaterialSnapshot::load(&source).await.is_err()); + } + + #[tokio::test] + async fn snapshot_rejects_incomplete_client_identity() { + let dir = TempDir::new().expect("create temp directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + fs::write(dir.path().join(&source.layout.client_cert_filename), "invalid client certificate").expect("write client cert"); + assert!(TlsMaterialSnapshot::load(&source).await.is_err()); + } + + #[tokio::test] + async fn snapshot_rejects_expired_server_certificate() { + let dir = TempDir::new().expect("create temp directory"); + let key = KeyPair::generate().expect("generate key"); + let mut params = CertificateParams::new(vec!["example.com".to_string()]).expect("certificate params"); + params.not_before = rcgen::date_time_ymd(2000, 1, 1); + params.not_after = rcgen::date_time_ymd(2001, 1, 1); + let cert = params.self_signed(&key).expect("generate expired certificate"); + fs::write(dir.path().join("rustfs_cert.pem"), cert.pem()).expect("write certificate"); + fs::write(dir.path().join("rustfs_key.pem"), key.serialize_pem()).expect("write key"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_client_only_server_eku() { + let dir = TempDir::new().expect("create temp directory"); + let key = KeyPair::generate().expect("generate key"); + let mut params = CertificateParams::new(vec!["example.com".to_string()]).expect("certificate params"); + params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth]; + let cert = params.self_signed(&key).expect("generate client-only certificate"); + fs::write(dir.path().join("rustfs_cert.pem"), cert.pem()).expect("write certificate"); + fs::write(dir.path().join("rustfs_key.pem"), key.serialize_pem()).expect("write key"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_server_only_client_eku() { + let dir = TempDir::new().expect("client identity directory"); + write_pair(dir.path(), "server.example"); + let key = KeyPair::generate().expect("client key"); + let mut params = CertificateParams::new(vec!["client.example".to_string()]).expect("client params"); + params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + let cert = params.self_signed(&key).expect("server-only certificate"); + fs::write(dir.path().join("client_cert.pem"), cert.pem()).expect("client certificate"); + fs::write(dir.path().join("client_key.pem"), key.serialize_pem()).expect("client key"); + assert!(matches!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())).await, + Err(TlsRuntimeError::InvalidClientEku { .. }) + )); + } + #[tokio::test] + async fn snapshot_rejects_future_server_certificate() { + let dir = TempDir::new().expect("create directory"); + let key = KeyPair::generate().expect("generate key"); + let mut params = CertificateParams::new(vec!["example.com".to_string()]).expect("certificate params"); + params.not_before = rcgen::date_time_ymd(2100, 1, 1); + params.not_after = rcgen::date_time_ymd(2101, 1, 1); + let cert = params.self_signed(&key).expect("generate future certificate"); + fs::write(dir.path().join("rustfs_cert.pem"), cert.pem()).expect("write certificate"); + fs::write(dir.path().join("rustfs_key.pem"), key.serialize_pem()).expect("write key"); + assert!(matches!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())).await, + Err(TlsRuntimeError::CertificateValidity { .. }) + )); + } + + #[tokio::test] + async fn snapshot_rejects_wrong_sni_certificate_name() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let domain = dir.path().join("other.example.com"); + fs::create_dir(&domain).expect("create domain directory"); + write_pair(&domain, "wrong.example.com"); + assert!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .is_err() + ); + } + + #[tokio::test] + async fn snapshot_rejects_malformed_complete_sni_pair() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let domain = dir.path().join("sni.example.com"); + fs::create_dir(&domain).expect("create domain directory"); + write_pair(&domain, "sni.example.com"); + fs::write(domain.join("rustfs_cert.pem"), "invalid certificate").expect("corrupt domain certificate"); + assert!(matches!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())).await, + Err(TlsRuntimeError::InvalidCertificate { .. }) + )); + } + + #[tokio::test] + async fn snapshot_rejects_cert_only_root() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + fs::remove_file(dir.path().join("rustfs_key.pem")).expect("remove key"); + assert!(matches!( + TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())).await, + Err(TlsRuntimeError::IncompleteIdentity { .. }) + )); + } + + #[tokio::test] + async fn snapshot_fingerprint_is_stable_and_certificate_only_renewal_changes_it() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + let initial = TlsMaterialSnapshot::load(&source) + .await + .expect("load valid initial generation"); + let unchanged = TlsMaterialSnapshot::load(&source).await.expect("load unchanged generation"); + assert_eq!(initial.fingerprint, unchanged.fingerprint); + let key = KeyPair::from_pem(&fs::read_to_string(dir.path().join("rustfs_key.pem")).expect("read key")) + .expect("parse existing key"); + let mut params = CertificateParams::new(vec!["example.com".to_string()]).expect("certificate params"); + params.serial_number = Some(123_u64.into()); + let renewed = params.self_signed(&key).expect("renew certificate with existing key"); + fs::write(dir.path().join("rustfs_cert.pem"), renewed.pem()).expect("write renewed certificate"); + let changed = TlsMaterialSnapshot::load(&source).await.expect("load renewed generation"); + assert_ne!(initial.fingerprint, changed.fingerprint); + } + + #[tokio::test] + async fn snapshot_server_client_ca_rotation_changes_fingerprint() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + let first_ca = rcgen::generate_simple_self_signed(vec!["first-ca".to_string()]).expect("first CA"); + fs::write(dir.path().join(&source.layout.client_ca_filename), first_ca.cert.pem()).expect("write first CA"); + let initial = TlsMaterialSnapshot::load(&source).await.expect("load first CA"); + let second_ca = rcgen::generate_simple_self_signed(vec!["second-ca".to_string()]).expect("second CA"); + fs::write(dir.path().join(&source.layout.client_ca_filename), second_ca.cert.pem()).expect("write second CA"); + let changed = TlsMaterialSnapshot::load(&source).await.expect("load second CA"); + assert_ne!(initial.fingerprint, changed.fingerprint); + assert_eq!(changed.server_client_ca_pem.as_deref(), Some(second_ca.cert.pem().as_bytes())); + } + + #[cfg(unix)] + #[test] + fn snapshot_kubernetes_switch_loads_all_material_from_pinned_generation() { + use std::os::unix::fs::symlink; + let dir = TempDir::new().expect("create projection"); + let source = TlsSource::from_directory(dir.path()); + for generation in ["..a", "..b"] { + let path = dir.path().join(generation); + fs::create_dir(&path).expect("create generation"); + write_pair(&path, "example.com"); + let cert = fs::read(path.join(&source.layout.server_cert_filename)).expect("read generated certificate"); + let key = fs::read(path.join(&source.layout.server_key_filename)).expect("read generated key"); + for filename in [ + &source.layout.public_ca_filename, + &source.fallback_ca_filename, + &source.layout.client_ca_filename, + &source.layout.client_cert_filename, + ] { + fs::write(path.join(filename), &cert).expect("write generation CA/client certificate"); + } + fs::write(path.join(&source.layout.client_key_filename), key).expect("write generation client key"); + } + symlink("..a", dir.path().join("..data")).expect("link initial generation"); + for filename in [ + &source.layout.server_cert_filename, + &source.layout.server_key_filename, + &source.layout.public_ca_filename, + &source.fallback_ca_filename, + &source.layout.client_ca_filename, + &source.layout.client_cert_filename, + &source.layout.client_key_filename, + ] { + symlink(format!("..data/{filename}"), dir.path().join(filename)).expect("link projected file"); + } + let pinned = source.snapshot().expect("pin initial generation"); + let expected = TlsMaterialSnapshot::load_from_snapshot(&pinned).expect("load initial generation"); + symlink("..b", dir.path().join("..next")).expect("link next generation"); + fs::rename(dir.path().join("..next"), dir.path().join("..data")).expect("switch projection"); + let actual = TlsMaterialSnapshot::load_from_snapshot(&pinned).expect("load pinned generation after switch"); + assert_eq!(actual.fingerprint, expected.fingerprint); + assert_eq!(actual.source.base_dir, pinned.base_dir()); + assert_eq!(actual.outbound.root_ca_pem, expected.outbound.root_ca_pem); + assert_eq!(actual.server_client_ca_pem, expected.server_client_ca_pem); + assert_eq!( + actual.outbound.mtls_identity.expect("client identity").key_pem, + expected.outbound.mtls_identity.expect("expected identity").key_pem + ); + let current = TlsMaterialSnapshot::load_from_snapshot(&source.snapshot().expect("pin new generation")) + .expect("load new generation"); + assert_ne!(actual.fingerprint, current.fingerprint); + } + + #[cfg(unix)] + #[test] + fn snapshot_rejects_disappeared_projection_source_and_dangling_optional_ca() { + use std::os::unix::fs::symlink; + let dir = TempDir::new().expect("create directory"); + let source = TlsSource::from_directory(dir.path()); + write_pair(dir.path(), "example.com"); + symlink("missing-ca", dir.path().join(&source.layout.public_ca_filename)).expect("link missing CA"); + let pinned = source.snapshot().expect("pin directory"); + assert!(matches!( + TlsMaterialSnapshot::load_from_snapshot(&pinned), + Err(TlsRuntimeError::SourceIo { .. }) + )); + fs::remove_file(dir.path().join(&source.layout.public_ca_filename)).expect("remove CA link"); + fs::remove_dir_all(dir.path()).expect("remove pinned source"); + assert!(matches!( + TlsMaterialSnapshot::load_from_snapshot(&pinned), + Err(TlsRuntimeError::SourceIo { .. }) + )); + } + #[tokio::test] + async fn snapshot_rejects_mismatched_sni_key_and_malformed_chain_member() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let domain = dir.path().join("sni.example.com"); + fs::create_dir(&domain).expect("create SNI directory"); + write_pair(&domain, "sni.example.com"); + let source = TlsSource::from_directory(dir.path()); + let valid = TlsMaterialSnapshot::load(&source) + .await + .expect("load valid multi certificate material"); + assert!(matches!(valid.server, Some(ServerTlsMaterial::MultiCert { ref cert_key_pairs }) if cert_key_pairs.len() == 2)); + fs::write( + domain.join("rustfs_key.pem"), + KeyPair::generate().expect("generate different key").serialize_pem(), + ) + .expect("replace SNI key"); + assert!(matches!( + TlsMaterialSnapshot::load(&source).await, + Err(TlsRuntimeError::InvalidCertifiedKey { .. }) + )); + write_pair(&domain, "sni.example.com"); + let cert_path = dir.path().join("rustfs_cert.pem"); + let mut cert = fs::read_to_string(&cert_path).expect("read leaf certificate"); + cert.push_str("-----BEGIN CERTIFICATE-----\nAQID\n-----END CERTIFICATE-----\n"); + fs::write(cert_path, cert).expect("append malformed chain member"); + assert!(matches!( + TlsMaterialSnapshot::load(&source).await, + Err(TlsRuntimeError::InvalidCertificate { .. }) + )); + } + + #[tokio::test] + async fn snapshot_rejects_invalid_private_key_pem_and_invalid_primary_client_ca() { + let dir = TempDir::new().expect("create directory"); + write_pair(dir.path(), "example.com"); + let source = TlsSource::from_directory(dir.path()); + fs::write(dir.path().join("rustfs_key.pem"), "invalid key").expect("corrupt key"); + assert!(matches!( + TlsMaterialSnapshot::load(&source).await, + Err(TlsRuntimeError::InvalidPrivateKey { .. }) + )); + write_pair(dir.path(), "example.com"); + let valid_ca = fs::read(dir.path().join("rustfs_cert.pem")).expect("read valid certificate"); + fs::write(dir.path().join(&source.fallback_ca_filename), valid_ca).expect("write valid fallback CA"); + fs::write(dir.path().join(&source.layout.client_ca_filename), "invalid primary CA").expect("write invalid primary CA"); + assert!(matches!( + TlsMaterialSnapshot::load(&source).await, + Err(TlsRuntimeError::InvalidCertificate { .. }) + )); + } +} diff --git a/crates/tls-runtime/src/outbound.rs b/crates/tls-runtime/src/outbound.rs index 11c9c5916..fac3a65cb 100644 --- a/crates/tls-runtime/src/outbound.rs +++ b/crates/tls-runtime/src/outbound.rs @@ -16,8 +16,8 @@ use crate::material::OutboundTlsMaterial; use crate::metrics::record_outbound_tls_publication; use crate::state::TlsGeneration; use rustfs_common::{ - MtlsIdentityPem, clear_global_root_cert, get_global_mtls_identity, get_global_outbound_tls_generation, get_global_root_cert, - set_global_mtls_identity, set_global_outbound_tls_generation, set_global_root_cert, + MtlsIdentityPem, OutboundTlsSnapshot, get_global_outbound_tls_generation, get_global_outbound_tls_snapshot, + publish_global_outbound_tls_snapshot, }; #[derive(Debug, Clone)] @@ -35,21 +35,21 @@ pub struct GlobalOutboundTlsStateSummary { } pub async fn publish_global_outbound_tls_state(generation: TlsGeneration, material: &OutboundTlsMaterial) { - if !material.root_ca_pem.is_empty() { - set_global_root_cert(material.root_ca_pem.clone()).await; - } else { - clear_global_root_cert().await; - } - set_global_mtls_identity(material.mtls_identity.clone()).await; - set_global_outbound_tls_generation(generation.0); - record_outbound_tls_publication(generation.0, !material.root_ca_pem.is_empty(), material.mtls_identity.is_some()); + let accepted = publish_global_outbound_tls_snapshot(OutboundTlsSnapshot { + generation: generation.0, + root_ca_pem: (!material.root_ca_pem.is_empty()).then(|| material.root_ca_pem.clone()), + mtls_identity: material.mtls_identity.clone(), + }) + .await; + record_outbound_tls_publication(accepted, !material.root_ca_pem.is_empty(), material.mtls_identity.is_some()); } pub async fn load_global_outbound_tls_state() -> GlobalPublishedOutboundTlsState { + let snapshot = get_global_outbound_tls_snapshot(); GlobalPublishedOutboundTlsState { - generation: TlsGeneration(get_global_outbound_tls_generation()), - root_ca_pem: get_global_root_cert().await, - mtls_identity: get_global_mtls_identity().await, + generation: TlsGeneration(snapshot.generation), + root_ca_pem: snapshot.root_ca_pem.clone(), + mtls_identity: snapshot.mtls_identity.clone(), } } diff --git a/crates/tls-runtime/src/source.rs b/crates/tls-runtime/src/source.rs index dee0b0277..a35b68001 100644 --- a/crates/tls-runtime/src/source.rs +++ b/crates/tls-runtime/src/source.rs @@ -92,3 +92,125 @@ impl TlsSource { Ok(self.base_dir.as_path()) } } + +/// A directory generation resolved once before loading any TLS material. +/// +/// Projected Kubernetes volumes are read through their physical `..data` +/// directory. Ordinary directory and certificate-file symlinks remain supported +/// for layouts such as Certbot. Writers must keep pinned generations immutable. +#[derive(Debug, Clone)] +pub struct TlsSourceSnapshot { + source: TlsSource, + resolved_source: TlsSource, +} + +impl TlsSource { + pub fn snapshot(&self) -> Result { + self.validate_directory()?; + let mut resolved_source = self.clone(); + let root = std::fs::canonicalize(&self.base_dir).map_err(|source| TlsRuntimeError::SourceIo { + path: self.base_dir.clone(), + source, + })?; + let projection = root.join("..data"); + resolved_source.base_dir = match std::fs::symlink_metadata(&projection) { + Ok(metadata) if metadata.file_type().is_symlink() => { + std::fs::canonicalize(&projection).map_err(|source| TlsRuntimeError::SourceIo { + path: projection, + source, + })? + } + Ok(_) => root, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => root, + Err(source) => { + return Err(TlsRuntimeError::SourceIo { + path: projection, + source, + }); + } + }; + resolved_source.validate_directory()?; + Ok(TlsSourceSnapshot { + source: self.clone(), + resolved_source, + }) + } +} + +impl TlsSourceSnapshot { + pub fn source(&self) -> &TlsSource { + &self.source + } + + pub fn resolved_source(&self) -> &TlsSource { + &self.resolved_source + } + + pub fn base_dir(&self) -> &Path { + &self.resolved_source.base_dir + } + + /// Missing optional files are allowed only while the pinned source exists. + pub(crate) fn validate_available(&self) -> Result<(), TlsRuntimeError> { + std::fs::read_dir(self.base_dir()).map_err(|source| TlsRuntimeError::SourceIo { + path: self.base_dir().to_path_buf(), + source, + })?; + Ok(()) + } +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::fs; + use std::os::unix::fs::symlink; + use tempfile::TempDir; + + #[test] + fn source_snapshot_pins_kubernetes_projection_generation() { + let dir = TempDir::new().expect("create projection directory"); + for generation in ["..generation-a", "..generation-b"] { + fs::create_dir(dir.path().join(generation)).expect("create generation"); + fs::write(dir.path().join(generation).join("marker"), generation).expect("write generation marker"); + } + symlink("..generation-a", dir.path().join("..data")).expect("link initial generation"); + symlink("..data/marker", dir.path().join("marker")).expect("link projected file"); + let pinned = TlsSource::from_directory(dir.path()).snapshot().expect("pin projection"); + symlink("..generation-b", dir.path().join("..next-data")).expect("link next generation"); + fs::rename(dir.path().join("..next-data"), dir.path().join("..data")).expect("swap projection"); + assert_eq!( + fs::read_to_string(pinned.base_dir().join("marker")).expect("read pinned file"), + "..generation-a" + ); + assert_eq!( + fs::read_to_string(dir.path().join("marker")).expect("read current projection"), + "..generation-b" + ); + } + + #[test] + fn source_snapshot_pins_tls_path_directory_symlink() { + let dir = TempDir::new().expect("create TLS directory"); + fs::create_dir(dir.path().join("a")).expect("create generation A"); + fs::create_dir(dir.path().join("b")).expect("create generation B"); + symlink("a", dir.path().join("live")).expect("link generation A"); + let pinned = TlsSource::from_directory(dir.path().join("live")) + .snapshot() + .expect("pin TLS source"); + fs::remove_file(dir.path().join("live")).expect("unlink source"); + symlink("b", dir.path().join("live")).expect("link generation B"); + assert_eq!(pinned.base_dir(), fs::canonicalize(dir.path().join("a")).expect("resolve generation A")); + } + + #[test] + fn source_snapshot_rejects_disappeared_generation() { + let dir = TempDir::new().expect("create TLS directory"); + let generation = dir.path().join("..generation"); + fs::create_dir(&generation).expect("create generation"); + symlink("..generation", dir.path().join("..data")).expect("link generation"); + let pinned = TlsSource::from_directory(dir.path()).snapshot().expect("pin TLS source"); + fs::remove_dir(&generation).expect("remove pinned generation"); + assert!(matches!(pinned.validate_available(), Err(TlsRuntimeError::SourceIo { .. }))); + } +} diff --git a/docs/README.md b/docs/README.md index 89549fcd2..2b89a015d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,6 +8,8 @@ collection: ## Operations +- [External certificate issuance and TLS rotation](operations/tls-certificate-rotation.md) — publication, native reload, trust separation and failure handling. + - [Architecture guard troubleshooting](operations/architecture-guard-troubleshooting.md) — diagnose repository guard failures. - [Logging governance](operations/logging-governance.md) — broad event audits, migrations, and guardrail changes. diff --git a/docs/operations/tls-certificate-rotation.md b/docs/operations/tls-certificate-rotation.md new file mode 100644 index 000000000..750f17c6d --- /dev/null +++ b/docs/operations/tls-certificate-rotation.md @@ -0,0 +1,204 @@ +# External certificate issuance and TLS rotation + +Use an independent certificate controller to obtain and renew certificates. +RustFS consumes certificate material; it does not register ACME accounts or +change DNS records. The independent +[rustfs-cert-controller](https://github.com/houseme/rustfs-cert-controller) +owns issuance, renewal scheduling and publication. `plugin-runner` starts its +container; it does not own certificate schedules or CA credentials. Progress +and deployment acceptance are tracked in +[backlog #2806](https://github.com/rustfs/backlog/issues/2806). + +## Deployment boundaries + +For a distributed Kubernetes deployment, prefer public TLS at an +Ingress/Gateway and private TLS/mTLS between the ingress and RustFS. Public +CA certificates cannot cover internal names such as `*.svc.cluster.local`; +DNS-01 can cover names under a controlled public domain even when the service +is private. A listener +that requires mTLS also requires client certificates from browsers and S3 +clients; selecting a different SNI certificate does not change that policy. + +Native server TLS can consume an externally managed `kubernetes.io/tls` +Secret with only `tls.crt` and `tls.key`: + +```yaml +mode: + distributed: + enabled: false + standalone: + enabled: true +ingress: + enabled: false +tls: + enabled: true + existingSecret: rustfs-serving-tls + reloadInterval: 30 +``` + +Create and renew that Secret using cert-manager or an independently managed +publisher. Configure the issuing resource with the actual server DNS/IP SANs +and explicit private-key rotation. The chart does not install cert-manager, +create an ACME account or grant DNS permissions. For distributed native TLS, +the certificate must also cover the configured peer hostnames; explicitly +configure independent outbound trust when a private server CA is used. + +`tls.trustSecret` optionally names a separate Secret with an explicitly chosen +`ca.crt`, mounted as outbound `public.crt`. It does not enable mTLS or add a +public issuing CA to client authentication trust. Do not write a fabricated +`ca.crt` into the Secret managed by an ACME issuer. + +`tls.enabled` and `mtls.enabled` are mutually exclusive. Existing mTLS mode +keeps its separate server/client certificate Secrets and projects their +contents into one read-only directory. The chart enables certificate reload +for either mode and disables TLS key logging. Explicit container environment +variables can override these settings. + +## Material layout and publication + +`RUSTFS_TLS_PATH` contains `rustfs_cert.pem` and `rustfs_key.pem`. SNI +identities can reside in visible domain subdirectories with the same two +filenames. Outbound trust uses `public.crt` and `ca.crt`; inbound mTLS client +trust prefers `client_ca.crt`, falling back to `ca.crt` only at initial load. +The default outbound client identity is `client_cert.pem`/`client_key.pem`. + +Runtime loading validates existing PEM/DER, key pairing, leaf validity and +explicit authentication EKUs. Each named SNI identity must match its domain. +Chain structure validation does not replace issuer/trust-policy verification +by the issuing controller and client PKI verifier. + +Kubernetes mounts must project the entire directory; single-file `subPath` +mounts do not receive Secret updates. RustFS pins one physical `..data` +generation before loading its material. Separate controllers may update the +serving identity and trust independently, so private CA migration still needs +an old-plus-new trust overlap. + +For VMs, publish immutable version directories and atomically change the +directory pointer after syncing the complete candidate and its containing +directories. Keep current/accepted/in-use versions during cleanup. Certbot +file symlinks remain compatible, but independent file replacements do not +provide an atomic directory generation; key mismatch is rejected, and a +publisher should use a complete immutable bundle for coherent publication. + +`RUSTFS_MTLS_CLIENT_CERT`/`RUSTFS_MTLS_CLIENT_KEY` overrides are validated as +one identity. Explicit missing, malformed, mismatched, expired or unsuitable +override material rejects the update; it does not silently select another +identity. Keep default identity files valid when they are present as well. + +## Controller publication API + +For a VM or Docker installation with one default serving certificate, enable +reload and configure `RUSTFS_TLS_PUBLISH_DIR` to a dedicated persistent directory +owned by the RustFS process with mode `0700`. Keep `RUSTFS_TLS_PATH` configured +for the initial serving identity and independent trust/client material. The +publication directory must be separate from the original TLS source. A valid +initial certificate and trusted management credentials must be provisioned +before a controller can connect to the HTTPS API. That bootstrap identity must +cover the names the configured issuer will continue to renew; publication +preserves existing SAN coverage. + +The controller signs requests using the existing S3 SigV4 authentication: + +- `GET /rustfs/admin/v3/tls/certificate-state` requires + `admin:TLSCertificateStatus`. +- `POST /rustfs/admin/v3/tls/certificate` requires + `admin:TLSCertificatePublish` and a connection terminated by RustFS TLS. + +Grant only these two actions to the controller identity. Read permissions, +general configuration permissions and forwarded HTTPS headers do not authorize +private-key publication. Keep API requests and credentials out of access-body +logs. The API does not add MinIO-compatible aliases. + +The `rustfs.tls-publish/v1alpha1` request fixes `resource_id` to `server-default` +and carries an `issuance_id`, the captured `expected_target_revision`, +`certificate_pem` and `private_key_pem`. Read the target token before issuance; +retain it with that immutable candidate. Repeating the same issuance and payload +recovers a lost receipt. A conflict stops that candidate; fetching a new token +and replaying the old candidate would defeat this concurrency protection. +An HTTP 503 or uncertain network result is retryable with the same issuance, +payload and token; HTTP 409 indicates a revision or payload conflict. + +RustFS validates the complete candidate before publishing an immutable leaf +generation and syncing its atomic pointer. It does not copy or overwrite CA +trust or client credentials. This mode rejects projected Kubernetes sources, +multiple SNI identities and configurations that trust the serving leaf as a CA. +Use the existing external-file/cert-manager integration for those deployments. + +The state reports the visible target revision and public leaf fingerprint. A +successful POST receipt proves the publication was synced; a GET during an +uncertain write does not replace that receipt. Retry the identical POST to +confirm acceptance. `observed_certificate_fingerprint` is present only when +this process's serving reload group has the exact current certificate chain; +matching the leaf alone is insufficient. A durable receipt can precede that +observation. Check convergence and a fresh TLS handshake on every relevant +endpoint. One process's response does not prove another node has converged. + +Use a stable local publication volume with one owner. Captured-token CAS and +local process locking do not implement HA leadership, restore ordering or PITR +protection. Do not remove active generation directories; automated generation +cleanup is not part of this interface. + +## Activation and failure handling + +Native `RUSTFS_TLS_RELOAD_ENABLE` defaults to false. Set it to `1` outside the +chart. `RUSTFS_TLS_RELOAD_INTERVAL` defaults to 30 seconds and is clamped to a +minimum of 5 seconds. + +Listeners sharing the same configured TLS directory use one reload group. +The group prepares TCP and enabled HTTP/3 configurations from the same +snapshot before swapping them. A late listener first joins the accepted +configuration, even if the current files have become invalid. TCP, QUIC and +separate processes still have a short activation window; there is no global +atomic certificate switch. +TCP and QUIC derive their protocol settings from one prepared certificate +resolver and client verifier for that generation. Publication state and retries +load each immutable generation once per operation; the next operation validates +its disk contents again. Reload uses the verified serving identity with the +independently pinned trust/client source, without reopening its certificate and +key files. Candidate staging still performs its separate disk validation. + +Unchanged material is skipped. A serving certificate renewal does not advance +the outbound generation unless outbound trust or identity actually changes. +Outbound state is published as one immutable trust/identity/generation +snapshot. Control and bulk RPC cache acquisition checks that generation; +existing in-flight calls retain their channels. + +Reload rejects the disappearance of accepted CA/identity source components, +SNI entries or accepted DNS/IP SAN coverage. Removing or renaming an identity +or trust source requires an explicit configuration change and restart; +ordinary renewal must preserve those requirements. Rejected updates retain +the last accepted in-memory configuration. New handshakes reject certificates +outside their validity interval. Normal renewal does not terminate existing +uploads, downloads or sessions. + +The chart uses TCP listener liveness for TLS deployments so certificate expiry +does not trigger a restart loop. Readiness continues to check the TLS service. +This liveness probe detects listener availability, not complete application +health. Keep ingress traffic admission separate from internal peer routing +and storage quorum. Do not restart every node because a CA is unavailable. + +There is no automatic persisted private-key fallback in Kubernetes. On +restart, invalid explicitly configured TLS material prevents startup. Key +compromise, explicit certificate revocation, session invalidation, per-purpose +profile coordination and deployment-boundary fencing require the separate +operator/control-plane workflow; ordinary reload does not implement them. + +## Verification + +Check a fresh TLS connection to each endpoint and compare the leaf serial, +public certificate fingerprint, SANs and expiry. Include HTTP/2, enabled +HTTP/3, Console and internal RPC endpoints. Do not infer served certificates +from Secret contents or an outbound generation gauge alone. + +Focused local checks: + +```sh +RUSTUP_TOOLCHAIN=stable cargo test -p rustfs-tls-runtime +RUSTUP_TOOLCHAIN=stable cargo test -p rustfs-protos --lib +bash scripts/test_helm_tls_reload.sh +``` + +Production acceptance also requires real Kubernetes projection, CA/DNS +issuance and renewal, outage/expiry, long-transfer continuity, private CA +overlap and distributed convergence tests. These are separate from protocol +and local loader tests. diff --git a/helm/rustfs/templates/_helpers.tpl b/helm/rustfs/templates/_helpers.tpl index 126e07e4f..4c3cac02a 100644 --- a/helm/rustfs/templates/_helpers.tpl +++ b/helm/rustfs/templates/_helpers.tpl @@ -154,6 +154,10 @@ Merges (in order of increasing precedence): }} {{- $annotations = merge $annotations $mtls }} {{- end }} +{{- $tls := .Values.tls | default dict -}} +{{- if and ($tls.enabled | default false) (eq .Values.ingress.className "traefik") -}} +{{- $_ := set $annotations "traefik.ingress.kubernetes.io/service.serversscheme" "https" -}} +{{- end -}} {{- if $annotations }} {{- toYaml $annotations }} {{- end }} @@ -188,6 +192,10 @@ Merges (in order of increasing precedence): {{- end }} {{- $annotations = merge $annotations (default (dict) .Values.ingress.customAnnotations) }} {{- $annotations = merge $annotations (default (dict) .Values.ingress.annotations) }} +{{- $tls := .Values.tls | default dict -}} +{{- if and ($tls.enabled | default false) (eq .Values.ingress.className "nginx") -}} +{{- $_ := set $annotations "nginx.ingress.kubernetes.io/backend-protocol" "HTTPS" -}} +{{- end -}} {{- if $annotations }} {{- toYaml $annotations }} {{- end }} @@ -285,7 +293,8 @@ RUSTFS_VOLUMES on spaces, one pool per expression). */}} {{- define "rustfs.volumes" -}} {{- $protocol := "http" -}} -{{- if .Values.mtls.enabled -}} +{{- $tls := .Values.tls | default dict -}} +{{- if or .Values.mtls.enabled ($tls.enabled | default false) -}} {{- $protocol = "https" -}} {{- end -}} {{- $headless := printf "%s-headless" (include "rustfs.fullname" .) -}} @@ -347,17 +356,17 @@ Render liveness and readiness probe for http and https */}} {{- define "rustfs.probes" -}} +{{- $tls := .Values.tls | default dict -}} {{- if .Values.livenessProbe.enabled }} livenessProbe: - {{- if .Values.mtls.enabled }} - exec: - command: -{{ include "rustfs.probeCommand" (dict "root" . "endpointPath" "/health") | nindent 6 }} + {{- if or .Values.mtls.enabled ($tls.enabled | default false) }} + tcpSocket: + port: {{ .Values.service.endpoint.port | default 9000 }} {{- else }} httpGet: path: /health port: {{ .Values.service.endpoint.port | default 9000 }} - scheme: {{ if .Values.mtls.enabled }}HTTPS{{ else }}HTTP{{ end }} + scheme: {{ if or .Values.mtls.enabled ($tls.enabled | default false) }}HTTPS{{ else }}HTTP{{ end }} {{- end }} initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds | default 60 }} periodSeconds: {{ .Values.livenessProbe.periodSeconds | default 5 }} @@ -376,7 +385,7 @@ readinessProbe: httpGet: path: /health/ready port: {{ .Values.service.endpoint.port | default 9000 }} - scheme: {{ if .Values.mtls.enabled }}HTTPS{{ else }}HTTP{{ end }} + scheme: {{ if or .Values.mtls.enabled ($tls.enabled | default false) }}HTTPS{{ else }}HTTP{{ end }} {{- end }} initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds | default 60 }} periodSeconds: {{ .Values.readinessProbe.periodSeconds | default 5 }} diff --git a/helm/rustfs/templates/configmap.yaml b/helm/rustfs/templates/configmap.yaml index 46e8331f8..e5aaa352f 100644 --- a/helm/rustfs/templates/configmap.yaml +++ b/helm/rustfs/templates/configmap.yaml @@ -1,3 +1,8 @@ +{{- $tls := .Values.tls | default dict -}} +{{- $tlsEnabled := $tls.enabled | default false -}} +{{- if and $tlsEnabled .Values.mtls.enabled -}} +{{- fail "tls.enabled and mtls.enabled are mutually exclusive; use ingress TLS for public access to an mTLS backend" -}} +{{- end -}} apiVersion: v1 kind: ConfigMap metadata: @@ -41,8 +46,12 @@ data: {{- end }} {{- if .Values.mtls.enabled }} RUSTFS_SERVER_MTLS_ENABLE: "1" - RUSTFS_TLS_KEYLOG: "1" + {{- end }} + {{- if or .Values.mtls.enabled $tlsEnabled }} + RUSTFS_TLS_KEYLOG: "0" RUSTFS_TLS_PATH: "/opt/tls" + RUSTFS_TLS_RELOAD_ENABLE: "1" + RUSTFS_TLS_RELOAD_INTERVAL: {{ $tls.reloadInterval | default 30 | quote }} {{- end }} {{- if .Values.config.rustfs.obs_endpoint.enabled }} RUSTFS_OBS_ENDPOINT: {{ .Values.config.rustfs.obs_endpoint.base_endpoint | quote }} diff --git a/helm/rustfs/templates/deployment.yaml b/helm/rustfs/templates/deployment.yaml index 39effeb24..97b663626 100644 --- a/helm/rustfs/templates/deployment.yaml +++ b/helm/rustfs/templates/deployment.yaml @@ -1,3 +1,5 @@ +{{- $tls := .Values.tls | default dict -}} +{{- $tlsEnabled := $tls.enabled | default false -}} {{- $logDir := .Values.config.rustfs.obs_log_directory }} {{- $logDirEnabled := ne $logDir "" }} @@ -123,25 +125,10 @@ spec: {{- toYaml .Values.resources | nindent 12 }} {{- include "rustfs.probes" . | nindent 10 }} volumeMounts: - {{- if .Values.mtls.enabled }} - - name: server-cert - mountPath: /opt/tls/rustfs_cert.pem - subPath: rustfs_cert.pem - - name: server-cert - mountPath: /opt/tls/rustfs_key.pem - subPath: rustfs_key.pem - - name: server-cert - mountPath: /opt/tls/ca.crt - subPath: ca.crt - - name: client-cert - mountPath: /opt/tls/client_cert.pem - subPath: client_cert.pem - - name: client-cert - mountPath: /opt/tls/client_key.pem - subPath: client_key.pem - - name: client-cert - mountPath: /opt/tls/client_ca.crt - subPath: client_ca.crt + {{- if or .Values.mtls.enabled $tlsEnabled }} + - name: tls-material + mountPath: /opt/tls + readOnly: true {{- end }} {{- with .Values.extraVolumeMounts }} {{- toYaml . | nindent 12 }} @@ -157,27 +144,40 @@ spec: {{- with .Values.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.mtls.enabled }} - - name: server-cert - secret: - secretName: {{ include "rustfs.fullname" . }}-server-tls - items: - - key: tls.crt - path: rustfs_cert.pem - - key: tls.key - path: rustfs_key.pem - - key: ca.crt - path: ca.crt - - name: client-cert - secret: - secretName: {{ include "rustfs.fullname" . }}-client-tls - items: - - key: tls.crt - path: client_cert.pem - - key: tls.key - path: client_key.pem - - key: ca.crt - path: client_ca.crt + {{- if or .Values.mtls.enabled $tlsEnabled }} + - name: tls-material + projected: + defaultMode: 0440 + sources: + - secret: + name: {{ if .Values.mtls.enabled }}{{ include "rustfs.fullname" . }}-server-tls{{ else }}{{ required "tls.existingSecret is required when tls.enabled=true" $tls.existingSecret }}{{ end }} + items: + - key: tls.crt + path: rustfs_cert.pem + - key: tls.key + path: rustfs_key.pem + {{- if .Values.mtls.enabled }} + - key: ca.crt + path: ca.crt + {{- end }} + {{- if .Values.mtls.enabled }} + - secret: + name: {{ include "rustfs.fullname" . }}-client-tls + items: + - key: tls.crt + path: client_cert.pem + - key: tls.key + path: client_key.pem + - key: ca.crt + path: client_ca.crt + {{- end }} + {{- if and $tlsEnabled $tls.trustSecret }} + - secret: + name: {{ $tls.trustSecret }} + items: + - key: ca.crt + path: public.crt + {{- end }} {{- end }} {{- if $logDirEnabled }} - name: logs diff --git a/helm/rustfs/templates/statefulset.yaml b/helm/rustfs/templates/statefulset.yaml index f2c90877d..54de97694 100644 --- a/helm/rustfs/templates/statefulset.yaml +++ b/helm/rustfs/templates/statefulset.yaml @@ -1,3 +1,5 @@ +{{- $tls := .Values.tls | default dict -}} +{{- $tlsEnabled := $tls.enabled | default false -}} {{- $logDir := .Values.config.rustfs.obs_log_directory }} {{- $logDirEnabled := ne $logDir "" }} {{- $poolsEnabled := and .Values.pools .Values.pools.enabled }} @@ -217,25 +219,10 @@ spec: {{- toYaml $.Values.resources | nindent 12 }} {{- include "rustfs.probes" $ | nindent 10 }} volumeMounts: - {{- if $.Values.mtls.enabled }} - - name: client-cert - mountPath: /opt/tls/client_cert.pem - subPath: client_cert.pem - - name: client-cert - mountPath: /opt/tls/client_key.pem - subPath: client_key.pem - - name: client-cert - mountPath: /opt/tls/client_ca.crt - subPath: client_ca.crt - - name: server-cert - mountPath: /opt/tls/rustfs_cert.pem - subPath: rustfs_cert.pem - - name: server-cert - mountPath: /opt/tls/rustfs_key.pem - subPath: rustfs_key.pem - - name: server-cert - mountPath: /opt/tls/ca.crt - subPath: ca.crt + {{- if or $.Values.mtls.enabled $tlsEnabled }} + - name: tls-material + mountPath: /opt/tls + readOnly: true {{- end }} {{- with $.Values.extraVolumeMounts }} {{- toYaml . | nindent 12 }} @@ -254,29 +241,42 @@ spec: - name: data mountPath: /data {{- end }} - {{- if or $.Values.mtls.enabled $.Values.extraVolumes }} + {{- if or $.Values.mtls.enabled $tlsEnabled $.Values.extraVolumes }} volumes: - {{- if $.Values.mtls.enabled }} - - name: server-cert - secret: - secretName: {{ include "rustfs.fullname" $ }}-server-tls - items: - - key: tls.crt - path: rustfs_cert.pem - - key: tls.key - path: rustfs_key.pem - - key: ca.crt - path: ca.crt - - name: client-cert - secret: - secretName: {{ include "rustfs.fullname" $ }}-client-tls - items: - - key: tls.crt - path: client_cert.pem - - key: tls.key - path: client_key.pem - - key: ca.crt - path: client_ca.crt + {{- if or $.Values.mtls.enabled $tlsEnabled }} + - name: tls-material + projected: + defaultMode: 0440 + sources: + - secret: + name: {{ if $.Values.mtls.enabled }}{{ include "rustfs.fullname" $ }}-server-tls{{ else }}{{ required "tls.existingSecret is required when tls.enabled=true" $tls.existingSecret }}{{ end }} + items: + - key: tls.crt + path: rustfs_cert.pem + - key: tls.key + path: rustfs_key.pem + {{- if $.Values.mtls.enabled }} + - key: ca.crt + path: ca.crt + {{- end }} + {{- if $.Values.mtls.enabled }} + - secret: + name: {{ include "rustfs.fullname" $ }}-client-tls + items: + - key: tls.crt + path: client_cert.pem + - key: tls.key + path: client_key.pem + - key: ca.crt + path: client_ca.crt + {{- end }} + {{- if and $tlsEnabled $tls.trustSecret }} + - secret: + name: {{ $tls.trustSecret }} + items: + - key: ca.crt + path: public.crt + {{- end }} {{- end }} {{- with $.Values.extraVolumes }} {{- toYaml . | nindent 8 }} diff --git a/helm/rustfs/values.yaml b/helm/rustfs/values.yaml index 88d01e341..de0c4b7f1 100644 --- a/helm/rustfs/values.yaml +++ b/helm/rustfs/values.yaml @@ -388,6 +388,14 @@ gatewayApi: name: "" namespace: "" +# Native TLS identity managed by an external certificate controller. +# Public TLS and an mTLS backend use separate listeners or an ingress terminator. +tls: + enabled: false + existingSecret: "" # kubernetes.io/tls: tls.crt and tls.key only + trustSecret: "" # Optional independent outbound trust bundle: ca.crt + reloadInterval: 30 + mtls: enabled: false clientCertPath: "/opt/tls/client_cert.pem" diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index 1f3f92469..fd9afef19 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -70,6 +70,7 @@ pub mod system; pub mod table_catalog; mod target_descriptor; pub mod tier; +pub mod tls_certificate; pub mod tls_debug; pub mod trace; pub mod usage_prefix; diff --git a/rustfs/src/admin/handlers/tls_certificate.rs b/rustfs/src/admin/handlers/tls_certificate.rs new file mode 100644 index 000000000..a45762625 --- /dev/null +++ b/rustfs/src/admin/handlers/tls_certificate.rs @@ -0,0 +1,177 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::admin::{ + auth::authorize_admin_request, + router::{AdminOperation, Operation, S3Router}, + storage_api::s3::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result}, +}; +use crate::server::{ + ADMIN_PREFIX, NativeTlsTransport, + tls_publish::{self, PublishError, PublishRequest}, +}; +use http::{HeaderMap, HeaderValue, StatusCode}; +use hyper::Method; +use matchit::Params; +use rustfs_policy::policy::{action::Action, action::AdminAction}; + +pub fn register_tls_certificate_routes(router: &mut S3Router) -> std::io::Result<()> { + router.insert( + Method::GET, + &format!("{ADMIN_PREFIX}/v3/tls/certificate-state"), + AdminOperation(&CertificateStateHandler), + )?; + router.insert( + Method::POST, + &format!("{ADMIN_PREFIX}/v3/tls/certificate"), + AdminOperation(&PublishCertificateHandler), + )?; + Ok(()) +} + +pub struct CertificateStateHandler; +pub struct PublishCertificateHandler; + +fn require_native_tls(req: &S3Request) -> S3Result<()> { + if !req + .extensions + .get::() + .is_some_and(|transport| transport.0) + { + return Err(S3Error::with_message( + S3ErrorCode::AccessDenied, + "certificate management requires TLS on the RustFS listener", + )); + } + Ok(()) +} + +async fn authorize_certificate_request(req: &S3Request, action: AdminAction) -> S3Result<()> { + if req.credentials.is_none() { + return Err(S3Error::with_message( + S3ErrorCode::AccessDenied, + "certificate management requires credentials", + )); + } + authorize_admin_request(req, vec![Action::AdminAction(action)]).await?; + require_native_tls(req) +} + +fn tls_path() -> S3Result { + crate::server::tls_material::active_server_tls_path() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "certificate publication is not configured")) +} + +fn publication_error(error: PublishError) -> S3Error { + let (code, status) = match error { + PublishError::Conflict => (S3ErrorCode::InvalidRequest, StatusCode::CONFLICT), + PublishError::Disabled => (S3ErrorCode::InvalidRequest, StatusCode::SERVICE_UNAVAILABLE), + PublishError::Invalid => (S3ErrorCode::InvalidRequest, StatusCode::BAD_REQUEST), + PublishError::Storage | PublishError::Busy => (S3ErrorCode::InternalError, StatusCode::SERVICE_UNAVAILABLE), + }; + let mut result = S3Error::with_message(code, error.to_string()); + result.set_status_code(status); + result +} + +fn json_response(value: &impl serde::Serialize) -> S3Result> { + let bytes = serde_json::to_vec(value) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "certificate status serialization failed"))?; + let mut headers = HeaderMap::new(); + headers.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/json")); + headers.insert(http::header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + Ok(S3Response::with_headers((StatusCode::OK, Body::from(bytes)), headers)) +} + +#[async_trait::async_trait] +impl Operation for CertificateStateHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + authorize_certificate_request(&req, AdminAction::TLSCertificateStatusAdminAction).await?; + json_response(&tls_publish::certificate_state(tls_path()?).await.map_err(publication_error)?) + } +} + +#[async_trait::async_trait] +impl Operation for PublishCertificateHandler { + async fn call(&self, mut req: S3Request, _params: Params<'_, '_>) -> S3Result> { + authorize_certificate_request(&req, AdminAction::TLSCertificatePublishAdminAction).await?; + let body = req.input.store_all_limited(128 * 1024).await.map_err(|_| { + S3Error::with_message( + S3ErrorCode::InvalidRequest, + "certificate publication body exceeds the limit or is unreadable", + ) + })?; + let request: PublishRequest = serde_json::from_slice(&body) + .map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "invalid certificate publication JSON"))?; + json_response(&tls_publish::publish(tls_path()?, request).await.map_err(publication_error)?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn request(method: Method) -> S3Request { + S3Request { + input: Body::empty(), + method, + uri: "/rustfs/admin/v3/tls/certificate".parse().expect("URI"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + } + } + + #[tokio::test] + async fn certificate_handlers_reject_missing_credentials() { + assert_eq!( + CertificateStateHandler + .call(request(Method::GET), Params::new()) + .await + .expect_err("unauthenticated GET") + .code(), + &S3ErrorCode::AccessDenied + ); + assert_eq!( + PublishCertificateHandler + .call(request(Method::POST), Params::new()) + .await + .expect_err("unauthenticated POST") + .code(), + &S3ErrorCode::AccessDenied + ); + } + + #[test] + fn forwarded_protocol_never_authorizes_private_key_upload() { + let mut request = request(Method::POST); + request.headers.insert("x-forwarded-proto", HeaderValue::from_static("https")); + assert!(require_native_tls(&request).is_err()); + request.extensions.insert(NativeTlsTransport(false)); + assert!(require_native_tls(&request).is_err()); + request.extensions.insert(NativeTlsTransport(true)); + assert!(require_native_tls(&request).is_ok()); + } + + #[test] + fn unknown_json_fields_are_rejected_without_echoing_material() { + let error = serde_json::from_str::( + r#"{"schema_version":"x","resource_id":"x","expected_target_revision":"x","issuance_id":"00000000-0000-0000-0000-000000000001","certificate_pem":"cert","private_key_pem":"secret","extra":true}"#, + ); + assert!(error.is_err()); + } +} diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index d330221d6..b9fb0373d 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -103,6 +103,7 @@ fn register_admin_routes(r: &mut S3Router) -> std::io::Result<() inspect_archive::register_inspect_archive_route(r)?; gateway_key_inventory::register_gateway_key_inventory_route(r)?; tls_debug::register_tls_debug_route(r)?; + handlers::tls_certificate::register_tls_certificate_routes(r)?; kms::register_kms_route(r)?; oidc::register_oidc_route(r)?; idp_compat::register_idp_compat_route(r)?; diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index d3baf28d2..8acd5f313 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -21,6 +21,8 @@ const ALL_ADMIN: AdminActionRef = AdminActionRef::new("AllAdminActions"); const ADD_USER_TO_GROUP: AdminActionRef = AdminActionRef::new("AddUserToGroupAdminAction"); const ATTACH_POLICY: AdminActionRef = AdminActionRef::new("AttachPolicyAdminAction"); const CONFIG_UPDATE: AdminActionRef = AdminActionRef::new("ConfigUpdateAdminAction"); +const TLS_CERTIFICATE_PUBLISH: AdminActionRef = AdminActionRef::new("TLSCertificatePublishAdminAction"); +const TLS_CERTIFICATE_STATUS: AdminActionRef = AdminActionRef::new("TLSCertificateStatusAdminAction"); const CONSOLE_LOG: AdminActionRef = AdminActionRef::new("ConsoleLogAdminAction"); const COMMIT_TABLE: AdminActionRef = AdminActionRef::new("CommitTableAction"); const CREATE_POLICY: AdminActionRef = AdminActionRef::new("CreatePolicyAdminAction"); @@ -501,6 +503,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ ), admin(HttpMethod::Get, "/rustfs/admin/v3/config", CONFIG_UPDATE, RouteRiskLevel::High), admin(HttpMethod::Put, "/rustfs/admin/v3/config", CONFIG_UPDATE, RouteRiskLevel::High), + admin( + HttpMethod::Get, + "/rustfs/admin/v3/tls/certificate-state", + TLS_CERTIFICATE_STATUS, + RouteRiskLevel::Sensitive, + ), + admin( + HttpMethod::Post, + "/rustfs/admin/v3/tls/certificate", + TLS_CERTIFICATE_PUBLISH, + RouteRiskLevel::High, + ), admin(HttpMethod::Get, "/rustfs/admin/v3/scanner/status", SERVER_INFO, RouteRiskLevel::Sensitive), admin( HttpMethod::Post, @@ -2233,6 +2247,14 @@ mod tests { } } + #[test] + fn route_policy_records_dedicated_tls_actions() { + assert_action(HttpMethod::Get, "/rustfs/admin/v3/tls/certificate-state", TLS_CERTIFICATE_STATUS); + assert_action(HttpMethod::Post, "/rustfs/admin/v3/tls/certificate", TLS_CERTIFICATE_PUBLISH); + assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/tls/certificate", CONFIG_UPDATE); + assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/tls/certificate-state", TLS_CERTIFICATE_PUBLISH); + } + /// `Critical` marks the routes whose worst case is permanent loss of user /// data, so the inventory is pinned in both directions: the KMS key /// deletion route may not be quietly downgraded, and no other route may be diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index 3d85e3c11..ac24a3d6f 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -302,6 +302,8 @@ fn expected_admin_route_matrix() -> Vec { admin_route(Method::PUT, "/v3/restore-config-history-kv"), admin_route(Method::GET, "/v3/config"), admin_route(Method::PUT, "/v3/config"), + admin_route(Method::GET, "/v3/tls/certificate-state"), + admin_route(Method::POST, "/v3/tls/certificate"), admin_route(Method::GET, "/v3/scanner/status"), admin_route(Method::POST, "/v3/scanner/cycle-state/reset"), admin_route(Method::POST, "/v3/scanner/usage-state/reset"), diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 8feed740a..c93f7bdd1 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -32,12 +32,12 @@ use crate::server::{ ssec_transport::SsecTransportLayer, strip_valid_port_suffix, tls_material::{ - TlsAcceptFailure, TlsAcceptorHolder, TlsHandshakeFailureKind, accept_tls_with_deadline, build_acceptor_from_loaded, + PreparedServerTls, TlsAcceptFailure, TlsAcceptorHolder, TlsHandshakeFailureKind, accept_tls_with_deadline, load_tls_material, spawn_reload_loop, }, }; #[cfg(feature = "http3")] -use crate::server::{http3, hybrid::HybridBody, tls_material::build_http3_server_config_from_loaded}; +use crate::server::{http3, hybrid::HybridBody}; use crate::storage_api::server::http as storage; use crate::storage_api::server::http::rpc::InternodeRpcService; #[cfg(test)] @@ -1448,6 +1448,7 @@ pub async fn start_http_server( // Load TLS materials and build server acceptor in a single pass. // Outbound material (root CAs, mTLS identity) was already published in main.rs; // this load is needed for the server-side TLS acceptor and reload loop. + let mut tls_initial_snapshot = None; let tls_acceptor = if tls_path_configured { let snapshot = load_tls_material(tls_path).await.map_err(|e| { Error::other(format!( @@ -1456,27 +1457,30 @@ pub async fn start_http_server( )) })?; - #[cfg(feature = "http3")] - if http3_enabled { - http3_server_config = build_http3_server_config_from_loaded(snapshot.server.as_ref(), std::path::Path::new(tls_path)) - .await - .map_err(|e| Error::other(format!("HTTP/3 TLS configuration failed: {e}")))?; - } - - let acceptor = build_acceptor_from_loaded(snapshot.server, std::path::Path::new(tls_path)) - .await - .map_err(|e| Error::other(e.to_string()))?; + let prepared = PreparedServerTls::from_snapshot(&snapshot).map_err(|e| Error::other(e.to_string()))?; // Fail closed: if TLS was explicitly configured but no server certificates // were found, refuse to start rather than silently falling back to plain HTTP. - match acceptor { + match prepared { None => { return Err(Error::other(format!( "TLS is explicitly configured via RUSTFS_TLS_PATH/tls_path='{}' but no server certificates were found", tls_path ))); } - Some(a) => Some(a), + Some(prepared) => { + #[cfg(feature = "http3")] + if http3_enabled { + http3_server_config = Some( + prepared + .http3_config() + .map_err(|e| Error::other(format!("HTTP/3 TLS configuration failed: {e}")))?, + ); + } + let acceptor = prepared.tcp_acceptor().map_err(|e| Error::other(e.to_string()))?; + tls_initial_snapshot = Some((snapshot, prepared)); + Some(acceptor) + } } } else { None @@ -1755,14 +1759,20 @@ pub async fn start_http_server( } else { None }; - let tls_reload_task = tls_acceptor.as_ref().and_then(|holder| { + let tls_reload_task = if let (Some(holder), Some((initial, prepared))) = (tls_acceptor.as_ref(), tls_initial_snapshot.take()) + { spawn_reload_loop( tls_path.to_string(), holder.clone(), + initial, + prepared, #[cfg(feature = "http3")] http3_server.as_ref().map(|(endpoint, _)| endpoint.clone()), ) - }); + .map_err(|e| Error::other(e.to_string()))? + } else { + None + }; let is_console = config.console_enable; let server_domains_configured = !config.server_domains.is_empty(); diff --git a/rustfs/src/server/mod.rs b/rustfs/src/server/mod.rs index 7f919d6f7..4841bf4fe 100644 --- a/rustfs/src/server/mod.rs +++ b/rustfs/src/server/mod.rs @@ -29,6 +29,7 @@ pub(crate) mod runtime_sources; mod service_state; mod ssec_transport; pub mod tls_material; +pub(crate) mod tls_publish; #[cfg(feature = "http3")] mod http3; @@ -85,6 +86,7 @@ pub(crate) use readiness::collect_node_readiness_report; pub use readiness::publish_ready_when_runtime_ready; pub(crate) use readiness::snapshot_dependency_readiness_report; pub(crate) use readiness::{collect_cluster_read_health_report, collect_cluster_write_health_report}; +pub(crate) use ssec_transport::NativeTlsTransport; pub use crate::shared_types::RemoteAddr; diff --git a/rustfs/src/server/ssec_transport.rs b/rustfs/src/server/ssec_transport.rs index bcda8f08a..bbe7adaba 100644 --- a/rustfs/src/server/ssec_transport.rs +++ b/rustfs/src/server/ssec_transport.rs @@ -63,6 +63,10 @@ pub(crate) const METRIC_SSEC_PLAINTEXT_REQUESTS_TOTAL: &str = "rustfs_ssec_plain type BoxError = Box; type BoxBody = http_body_util::combinators::UnsyncBoxBody; +/// Set only by the listener stack; proxy protocol assertions do not establish this marker. +#[derive(Clone, Copy)] +pub(crate) struct NativeTlsTransport(pub(crate) bool); + /// Every header that carries, or announces, a customer-provided key: the /// object's own triple and the copy-source triple that CopyObject and /// UploadPartCopy use to read an SSE-C source. @@ -191,7 +195,8 @@ where self.inner.poll_ready(cx) } - fn call(&mut self, req: Request) -> Self::Future { + fn call(&mut self, mut req: Request) -> Self::Future { + req.extensions_mut().insert(NativeTlsTransport(self.connection_is_tls)); if !carries_ssec_headers(req.headers()) || is_secure_transport(self.connection_is_tls, req.extensions().get::()) { diff --git a/rustfs/src/server/tls_material.rs b/rustfs/src/server/tls_material.rs index 4d6239eb2..1790ee2a9 100644 --- a/rustfs/src/server/tls_material.rs +++ b/rustfs/src/server/tls_material.rs @@ -26,19 +26,20 @@ use rustfs_config::{ DEFAULT_SERVER_MTLS_ENABLE, DEFAULT_TLS_KEYLOG, DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, DEFAULT_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_SYSTEM_CA, ENV_MTLS_CLIENT_CERT, ENV_MTLS_CLIENT_KEY, ENV_RUSTFS_EXTRA_CA_CERT, ENV_SERVER_MTLS_ENABLE, ENV_TLS_KEYLOG, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, ENV_TRUST_LEAF_CERT_AS_CA, - ENV_TRUST_SYSTEM_CA, RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, - RUSTFS_TLS_CERT, -}; -use rustfs_tls_runtime::{ - ServerTlsMaterial as RuntimeServerTlsMaterial, TlsGeneration, TlsSource, WebPkiClientVerifierOptions, - build_webpki_client_verifier, create_multi_cert_resolver, + ENV_TRUST_SYSTEM_CA, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, }; +#[cfg(test)] +use rustfs_config::{RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME}; +use rustfs_tls_runtime::{ServerTlsMaterial as RuntimeServerTlsMaterial, TlsGeneration, TlsSource, create_multi_cert_resolver}; +#[cfg(test)] +use rustfs_tls_runtime::{WebPkiClientVerifierOptions, build_webpki_client_verifier}; use rustfs_utils::{get_env_bool, get_env_opt_str}; use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::io::Cursor; use std::path::{Path, PathBuf}; -use std::sync::Arc; use std::sync::RwLock; +use std::sync::{Arc, LazyLock, Mutex, Weak}; use std::time::Duration; use tokio_rustls::TlsAcceptor; use tracing::{debug, info, warn}; @@ -75,11 +76,20 @@ pub async fn load_tls_material(tls_path: &str) -> Result Result, tls_dir: &Path, @@ -100,76 +111,166 @@ pub(crate) async fn build_acceptor_from_loaded( ) .map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?; - match server { - Some(RuntimeServerTlsMaterial::SingleCert { certs, key }) => { - let config = build_server_config(ServerCertSource::SingleCert { certs, key }, mtls_verifier, ServerProtocol::Tcp)?; - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_acceptor_created", - mode = "single_cert", - "TLS acceptor created" - ); - let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config))); - Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor)))) + build_acceptor_with_verifier(server, mtls_verifier) +} + +fn snapshot_client_verifier( + snapshot: &rustfs_tls_runtime::TlsMaterialSnapshot, +) -> Result>, TlsMaterialError> { + if !get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE) { + return Ok(None); + } + let pem = snapshot + .server_client_ca_pem + .as_deref() + .ok_or_else(|| TlsMaterialError::Parse("mTLS client trust is missing".into()))?; + let mut reader = Cursor::new(pem); + let mut roots = rustls::RootCertStore::empty(); + for cert in CertificateDer::pem_reader_iter(&mut reader) { + roots + .add(cert.map_err(|e| TlsMaterialError::Parse(e.to_string()))?) + .map_err(|e| TlsMaterialError::Parse(e.to_string()))?; + } + let verifier = rustls::server::WebPkiClientVerifier::builder(Arc::new(roots)) + .build() + .map_err(|e| TlsMaterialError::Parse(e.to_string()))?; + #[cfg(test)] + TLS_PREPARE_COUNTS.with(|counts| { + let (roots, keys) = counts.get(); + counts.set((roots + 1, keys)); + }); + Ok(Some(verifier)) +} + +#[cfg(test)] +thread_local! { + static TLS_PREPARE_COUNTS: std::cell::Cell<(usize, usize)> = const { std::cell::Cell::new((0, 0)) }; +} + +/// Expensive certificate signing keys and client trust are prepared once per generation. +/// TCP and QUIC retain this shared resolver while owning their protocol settings. +#[derive(Clone)] +pub(crate) struct PreparedServerTls { + resolver: Arc, + verifier: Option>, + provider: Arc, +} + +impl PreparedServerTls { + pub(crate) fn from_snapshot(snapshot: &rustfs_tls_runtime::TlsMaterialSnapshot) -> Result, TlsMaterialError> { + Self::from_material(snapshot.server.as_ref(), snapshot_client_verifier(snapshot)?) + } + + fn from_material( + server: Option<&RuntimeServerTlsMaterial>, + verifier: Option>, + ) -> Result, TlsMaterialError> { + let Some(server) = server else { + return Ok(None); + }; + let provider = rustls::ServerConfig::builder().crypto_provider().clone(); + let (resolver, validity): (Arc, _) = match server { + RuntimeServerTlsMaterial::SingleCert { certs, key } => { + let validity = certificate_validities(std::iter::once(certs.as_slice()))?; + let certified = rustls::sign::CertifiedKey::from_der(certs.clone(), key.clone_key(), &provider) + .map_err(|e| TlsMaterialError::Io(format!("configure single cert: {e}")))?; + (Arc::new(rustls::sign::SingleCertAndKey::from(certified)), validity) + } + RuntimeServerTlsMaterial::MultiCert { cert_key_pairs } => { + let validity = certificate_validities(cert_key_pairs.values().map(|(certs, _)| certs.as_slice()))?; + let pairs = cert_key_pairs + .iter() + .map(|(name, (certs, key))| (name.clone(), (certs.clone(), key.clone_key()))) + .collect(); + let resolver = create_multi_cert_resolver(pairs) + .map_err(|e| TlsMaterialError::Parse(format!("build multi-cert resolver: {e}")))?; + (Arc::new(resolver), validity) + } + }; + #[cfg(test)] + TLS_PREPARE_COUNTS.with(|counts| { + let (roots, keys) = counts.get(); + counts.set((roots, keys + 1)); + }); + Ok(Some(Self { + resolver: Arc::new(ValidityCheckedResolver { + inner: resolver, + validity, + }), + verifier, + provider, + })) + } + + pub(crate) fn tcp_acceptor(&self) -> Result, TlsMaterialError> { + let config = self.server_config(ServerProtocol::Tcp)?; + Ok(Arc::new(TlsAcceptorHolder::new(Arc::new(TlsAcceptor::from(Arc::new(config)))))) + } + + #[cfg(feature = "http3")] + pub(crate) fn http3_config(&self) -> Result { + let crypto = quinn::crypto::rustls::QuicServerConfig::try_from(self.server_config(ServerProtocol::Http3)?) + .map_err(|e| TlsMaterialError::Io(format!("configure QUIC TLS: {e}")))?; + Ok(quinn::ServerConfig::with_crypto(Arc::new(crypto))) + } + + fn server_config(&self, protocol: ServerProtocol) -> Result { + let versions = match protocol { + ServerProtocol::Tcp => rustls::DEFAULT_VERSIONS, + #[cfg(feature = "http3")] + ServerProtocol::Http3 => &[&rustls::version::TLS13], + }; + let builder = rustls::ServerConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(versions) + .map_err(|e| TlsMaterialError::Io(format!("configure TLS protocol: {e}")))?; + let mut config = if let Some(verifier) = &self.verifier { + builder + .with_client_cert_verifier(verifier.clone()) + .with_cert_resolver(self.resolver.clone()) + } else { + builder.with_no_client_auth().with_cert_resolver(self.resolver.clone()) + }; + config.alpn_protocols = match protocol { + ServerProtocol::Tcp => vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()], + #[cfg(feature = "http3")] + ServerProtocol::Http3 => vec![b"h3".to_vec()], + }; + #[cfg(feature = "http3")] + if matches!(protocol, ServerProtocol::Http3) { + config.max_early_data_size = 0; } - Some(RuntimeServerTlsMaterial::MultiCert { cert_key_pairs }) => { - let resolver = create_multi_cert_resolver(cert_key_pairs) - .map_err(|e| TlsMaterialError::Parse(format!("build multi-cert resolver: {e}")))?; - let config = build_server_config(ServerCertSource::Resolver(Arc::new(resolver)), mtls_verifier, ServerProtocol::Tcp)?; - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_acceptor_created", - mode = "sni_resolver", - "TLS acceptor created" - ); - let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config))); - Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor)))) + config.session_storage = rustls::server::ServerSessionMemoryCache::new(10000); + if tls_key_log() { + config.key_log = Arc::new(rustls::KeyLogFile::new()); } - None => Ok(None), + Ok(config) } } -#[cfg(feature = "http3")] +#[cfg(test)] +fn build_acceptor_with_verifier( + server: Option, + mtls_verifier: Option>, +) -> Result>, TlsMaterialError> { + PreparedServerTls::from_material(server.as_ref(), mtls_verifier)? + .map(|prepared| prepared.tcp_acceptor()) + .transpose() +} + +#[cfg(all(test, feature = "http3"))] pub(crate) async fn build_http3_server_config_from_loaded( server: Option<&RuntimeServerTlsMaterial>, tls_dir: &Path, ) -> Result, TlsMaterialError> { - let Some(server) = server else { - return Ok(None); - }; - let mtls_verifier = build_webpki_client_verifier( WebPkiClientVerifierOptions::builder(tls_dir, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CA_CERT) .enabled(get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE)) .build(), ) .map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?; - - let source = match server { - RuntimeServerTlsMaterial::SingleCert { certs, key } => ServerCertSource::SingleCert { - certs: certs.clone(), - key: key.clone_key(), - }, - RuntimeServerTlsMaterial::MultiCert { cert_key_pairs } => { - let pairs = cert_key_pairs - .iter() - .map(|(domain, (certs, key))| (domain.clone(), (certs.clone(), key.clone_key()))) - .collect(); - - let resolver = create_multi_cert_resolver(pairs) - .map_err(|e| TlsMaterialError::Parse(format!("build HTTP/3 SNI resolver: {e}")))?; - - ServerCertSource::Resolver(Arc::new(resolver)) - } - }; - - let tls = build_server_config(source, mtls_verifier, ServerProtocol::Http3)?; - let crypto = quinn::crypto::rustls::QuicServerConfig::try_from(tls) - .map_err(|e| TlsMaterialError::Io(format!("configure QUIC TLS: {e}")))?; - - Ok(Some(quinn::ServerConfig::with_crypto(Arc::new(crypto)))) + PreparedServerTls::from_material(server, mtls_verifier)? + .map(|prepared| prepared.http3_config()) + .transpose() } // ── Outbound Enrichment ── @@ -179,18 +280,30 @@ pub(crate) async fn build_http3_server_config_from_loaded( /// 1. Optional: server leaf cert as root CA (`RUSTFS_TRUST_LEAF_CERT_AS_CA`) /// 2. Optional: system root CAs from platform paths (`RUSTFS_TRUST_SYSTEM_CA`) /// 3. Optional: mTLS identity from env-var-overridden paths -async fn enrich_outbound(outbound: &mut rustfs_tls_runtime::OutboundTlsMaterial, tls_dir: &Path) -> Result<(), TlsMaterialError> { - // 1. Optional: load leaf certs as root CAs - if get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) - && load_cert_file_by_name(tls_dir, RUSTFS_TLS_CERT, &mut outbound.root_ca_pem).await - { - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_root_ca_enriched", - source = "leaf_certificate", - "TLS root CA set enriched" - ); +async fn enrich_outbound(snapshot: &mut rustfs_tls_runtime::TlsMaterialSnapshot) -> Result<(), TlsMaterialError> { + let tls_dir = &snapshot.source.base_dir; + let outbound = &mut snapshot.outbound; + if get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) { + let chains: Vec<_> = match &snapshot.server { + Some(RuntimeServerTlsMaterial::SingleCert { certs, .. }) => vec![certs], + Some(RuntimeServerTlsMaterial::MultiCert { cert_key_pairs }) => { + cert_key_pairs.values().map(|(certs, _)| certs).collect() + } + None => Vec::new(), + }; + let mut certificates = BTreeSet::new(); + for chain in chains { + certificates.extend(chain.iter().map(|cert| cert.as_ref().to_vec())); + } + for certificate in certificates { + outbound.root_ca_pem.extend_from_slice( + format!( + "-----BEGIN CERTIFICATE-----\n{}\n-----END CERTIFICATE-----\n", + base64_simd::STANDARD.encode_to_string(&certificate), + ) + .as_bytes(), + ); + } } // 2. Optional: load system root CAs @@ -260,19 +373,6 @@ async fn load_mtls_identity_from_overridden_paths( None => tls_dir.join(RUSTFS_CLIENT_KEY_FILENAME), }; - if !client_cert_path.exists() || !client_key_path.exists() { - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "mtls_identity_state", - state = "not_configured", - cert_path = ?client_cert_path, - key_path = ?client_key_path, - "mTLS client identity not configured" - ); - return Ok(None); - } - let cert_pem = tokio::fs::read(&client_cert_path) .await .map_err(|e| TlsMaterialError::Io(format!("read client cert {client_cert_path:?}: {e}")))?; @@ -281,11 +381,13 @@ async fn load_mtls_identity_from_overridden_paths( .map_err(|e| TlsMaterialError::Io(format!("read client key {client_key_path:?}: {e}")))?; let mut reader = std::io::Cursor::new(&cert_pem); - if CertificateDer::pem_reader_iter(&mut reader).next().is_none() { - return Err(TlsMaterialError::Parse("no valid certificate in client cert PEM".into())); - } + let certs = CertificateDer::pem_reader_iter(&mut reader) + .collect::, _>>() + .map_err(|e| TlsMaterialError::Parse(format!("invalid client certificate: {e}")))?; let mut reader = std::io::Cursor::new(&key_pem); - PrivateKeyDer::from_pem_reader(&mut reader).map_err(|e| TlsMaterialError::Parse(format!("invalid client key PEM: {e}")))?; + let key = + PrivateKeyDer::from_pem_reader(&mut reader).map_err(|e| TlsMaterialError::Parse(format!("invalid client key: {e}")))?; + rustfs_tls_runtime::validate_certified_key(&certs, &key, "client identity override", false).map_err(map_runtime_tls_error)?; info!( component = LOG_COMPONENT_TLS, @@ -413,54 +515,47 @@ async fn load_cert_file(path: &Path, pem_data: &mut Vec, desc: &str) -> bool /// Search for and load certificate files matching `cert_name` in the directory /// and one level of subdirectories. /// Returns `true` if at least one matching file was loaded. -async fn load_cert_file_by_name(dir: &Path, cert_name: &str, pem_data: &mut Vec) -> bool { - let Ok(mut rd) = tokio::fs::read_dir(dir).await else { - debug!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_material_directory_state", - path = %dir.display(), - state = "missing", - "TLS material directory state changed" + +type CertificateValidities = HashMap, (i64, i64)>; + +fn certificate_validities<'a>( + chains: impl Iterator]>, +) -> Result { + let mut validity = HashMap::new(); + for chain in chains { + let leaf = chain + .first() + .ok_or_else(|| TlsMaterialError::Parse("empty certificate chain".into()))?; + let (_, parsed) = x509_parser::parse_x509_certificate(leaf.as_ref()) + .map_err(|e| TlsMaterialError::Parse(format!("invalid certificate: {e}")))?; + validity.insert( + leaf.as_ref().to_vec(), + (parsed.validity().not_before.timestamp(), parsed.validity().not_after.timestamp()), ); - return false; - }; - - let mut loaded = false; - while let Ok(Some(entry)) = rd.next_entry().await { - let Ok(ft) = entry.file_type().await else { continue }; - - if ft.is_file() { - let fname = entry.file_name().to_string_lossy().to_string(); - if fname == cert_name && load_cert_file(&entry.path(), pem_data, "certificate").await { - loaded = true; - } - } else if ft.is_dir() - && let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await - { - while let Ok(Some(sub_entry)) = sub_rd.next_entry().await { - if let Ok(sub_ft) = sub_entry.file_type().await - && sub_ft.is_file() - { - let fname = sub_entry.file_name().to_string_lossy().to_string(); - if fname == cert_name && load_cert_file(&sub_entry.path(), pem_data, "certificate").await { - loaded = true; - } - } - } - } } - loaded + Ok(validity) } -// ── Server Config Construction ── +#[derive(Debug)] +struct ValidityCheckedResolver { + inner: Arc, + validity: CertificateValidities, +} -enum ServerCertSource { - Resolver(Arc), - SingleCert { - certs: Vec>, - key: PrivateKeyDer<'static>, - }, +impl rustls::server::ResolvesServerCert for ValidityCheckedResolver { + fn resolve(&self, hello: rustls::server::ClientHello<'_>) -> Option> { + let certificate = self.inner.resolve(hello)?; + let leaf = certificate.cert.first()?; + let (not_before, not_after) = self.validity.get(leaf.as_ref())?; + let now = i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .ok()? + .as_secs(), + ) + .ok()?; + (*not_before <= now && now < *not_after).then_some(certificate) + } } enum ServerProtocol { @@ -469,62 +564,6 @@ enum ServerProtocol { Http3, } -fn build_server_config( - cert_source: ServerCertSource, - mtls_verifier: Option>, - protocol: ServerProtocol, -) -> Result { - let builder = match protocol { - ServerProtocol::Tcp => rustls::ServerConfig::builder(), - #[cfg(feature = "http3")] - ServerProtocol::Http3 => rustls::ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]), - }; - - let mut config = match cert_source { - ServerCertSource::Resolver(resolver) => { - if let Some(verifier) = mtls_verifier { - builder.with_client_cert_verifier(verifier).with_cert_resolver(resolver) - } else { - builder.with_no_client_auth().with_cert_resolver(resolver) - } - } - ServerCertSource::SingleCert { certs, key } => { - if let Some(verifier) = mtls_verifier { - builder - .with_client_cert_verifier(verifier) - .with_single_cert(certs, key) - .map_err(|e| TlsMaterialError::Io(format!("configure single cert with mTLS: {e}")))? - } else { - builder - .with_no_client_auth() - .with_single_cert(certs, key) - .map_err(|e| TlsMaterialError::Io(format!("configure single cert: {e}")))? - } - } - }; - - config.alpn_protocols = match protocol { - ServerProtocol::Tcp => { - vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()] - } - #[cfg(feature = "http3")] - ServerProtocol::Http3 => vec![b"h3".to_vec()], - }; - - #[cfg(feature = "http3")] - if matches!(protocol, ServerProtocol::Http3) { - config.max_early_data_size = 0; - } - - config.session_storage = rustls::server::ServerSessionMemoryCache::new(10000); - - if tls_key_log() { - config.key_log = Arc::new(rustls::KeyLogFile::new()); - } - - Ok(config) -} - fn tls_key_log() -> bool { get_env_bool(ENV_TLS_KEYLOG, DEFAULT_TLS_KEYLOG) } @@ -647,26 +686,30 @@ impl TlsAcceptorHolder { } // Build both configurations before publishing either listener's new certificate. +#[cfg(all(test, feature = "http3"))] async fn reload_acceptors( server: Option, tls_dir: &Path, holder: &TlsAcceptorHolder, #[cfg(feature = "http3")] http3_endpoint: Option<&quinn::Endpoint>, ) -> Result { - #[cfg(feature = "http3")] + let verifier = build_webpki_client_verifier( + WebPkiClientVerifierOptions::builder(tls_dir, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CA_CERT) + .enabled(get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE)) + .build(), + ) + .map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?; + let Some(prepared) = PreparedServerTls::from_material(server.as_ref(), verifier)? else { + return Ok(false); + }; let http3_config = if http3_endpoint.is_some() { - let config = build_http3_server_config_from_loaded(server.as_ref(), tls_dir).await?; - let Some(mut config) = config else { - return Ok(false); - }; + let mut config = prepared.http3_config()?; config.migration(false); Some(config) } else { None }; - let Some(new_holder) = build_acceptor_from_loaded(server, tls_dir).await? else { - return Ok(false); - }; + let new_holder = prepared.tcp_acceptor()?; #[cfg(feature = "http3")] if let (Some(endpoint), Some(config)) = (http3_endpoint, http3_config) { endpoint.set_server_config(Some(config)); @@ -680,146 +723,327 @@ async fn reload_acceptors( /// Spawn a background task that periodically checks for TLS certificate changes. /// Single load per tick: loads once via tls-runtime, enriches, publishes outbound, /// and builds acceptor — no double reads. +#[derive(Clone)] +struct ReloadTarget { + holder: Arc, + #[cfg(feature = "http3")] + http3_endpoint: Option, +} + +struct ReloadState { + targets: HashMap, + accepted: Arc, + tcp: Arc, + prepared: PreparedServerTls, + #[cfg(feature = "http3")] + quic: Option, +} + +struct ReloadGroup { + required_startup_files: BTreeSet, + state: Mutex, + changed: tokio::sync::Notify, + next_id: std::sync::atomic::AtomicU64, + #[cfg(test)] + pass_completed: tokio::sync::Notify, +} + +// Lock order is registry, then group state, then acceptor. Neither guard crosses an await. +static RELOAD_GROUPS: LazyLock>>> = LazyLock::new(|| Mutex::new(HashMap::new())); + +pub(crate) fn observed_server_snapshot(tls_path: &str) -> Option> { + let registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + let group = registry.get(tls_path)?.upgrade()?; + let state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + (!state.targets.is_empty()).then(|| state.accepted.clone()) +} + +/// Resolve the source actually registered by listener configuration, including CLI overrides. +pub(crate) fn active_server_tls_path() -> Option { + let registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + let mut paths = registry.iter().filter_map(|(path, group)| { + let group = group.upgrade()?; + let state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + (!state.targets.is_empty()).then(|| path.clone()) + }); + let path = paths.next()?; + paths.next().is_none().then_some(path) +} + +pub(crate) fn notify_server_publication(tls_path: &str) { + let registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(group) = registry.get(tls_path).and_then(Weak::upgrade) { + group.changed.notify_one(); + } +} + +struct ReloadRegistration { + group: Arc, + id: u64, +} + +impl Drop for ReloadRegistration { + fn drop(&mut self) { + self.group + .state + .lock() + .unwrap_or_else(|e| e.into_inner()) + .targets + .remove(&self.id); + self.group.changed.notify_one(); + } +} + +fn server_names(server: &Option) -> Result>, TlsMaterialError> { + let chains: Vec<_> = match server { + Some(RuntimeServerTlsMaterial::SingleCert { certs, .. }) => vec![("default", certs)], + Some(RuntimeServerTlsMaterial::MultiCert { cert_key_pairs }) => cert_key_pairs + .iter() + .map(|(selector, (certs, _))| (selector.as_str(), certs)) + .collect(), + None => Vec::new(), + }; + let mut selectors = BTreeMap::new(); + for (selector, chain) in chains { + let leaf = chain + .first() + .ok_or_else(|| TlsMaterialError::Parse("empty certificate chain".into()))?; + selectors.insert(selector.to_string(), certificate_names(leaf)?); + } + Ok(selectors) +} + +pub(crate) fn certificate_names(leaf: &CertificateDer<'_>) -> Result, TlsMaterialError> { + let mut names = BTreeSet::new(); + let (_, parsed) = x509_parser::parse_x509_certificate(leaf.as_ref()) + .map_err(|e| TlsMaterialError::Parse(format!("invalid certificate: {e}")))?; + if let Some(san) = parsed + .subject_alternative_name() + .map_err(|e| TlsMaterialError::Parse(format!("invalid SAN: {e}")))? + { + for name in &san.value.general_names { + match name { + x509_parser::extensions::GeneralName::DNSName(name) => { + names.insert(format!("dns:{}", name.to_ascii_lowercase())); + } + x509_parser::extensions::GeneralName::IPAddress(bytes) => { + names.insert(format!("ip:{bytes:?}")); + } + _ => {} + } + } + } + Ok(names) +} + +pub(crate) fn validate_required_material_files( + required: &BTreeSet, + candidate: &rustfs_tls_runtime::TlsMaterialSnapshot, +) -> Result<(), TlsMaterialError> { + if !required.is_subset(&candidate.material_files) { + return Err(TlsMaterialError::Parse("required TLS source components disappeared".into())); + } + Ok(()) +} + +pub(crate) fn validate_candidate_replacement( + accepted: &rustfs_tls_runtime::TlsMaterialSnapshot, + candidate: &rustfs_tls_runtime::TlsMaterialSnapshot, +) -> Result<(), TlsMaterialError> { + validate_required_material_files(&accepted.material_files, candidate)?; + let old_names = server_names(&accepted.server)?; + let new_names = server_names(&candidate.server)?; + for (selector, names) in old_names { + if !new_names.get(&selector).is_some_and(|new| names.is_subset(new)) { + return Err(TlsMaterialError::Parse("required server certificate names disappeared".into())); + } + } + Ok(()) +} + +fn outbound_material_matches(a: &rustfs_tls_runtime::OutboundTlsMaterial, b: &rustfs_tls_runtime::OutboundTlsMaterial) -> bool { + a.root_ca_pem == b.root_ca_pem && a.mtls_identity == b.mtls_identity +} + pub(crate) fn spawn_reload_loop( tls_path: String, holder: Arc, + initial: rustfs_tls_runtime::TlsMaterialSnapshot, + prepared: PreparedServerTls, #[cfg(feature = "http3")] http3_endpoint: Option, -) -> Option> { - let enabled = get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE); - if !enabled { - debug!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_state", - state = "disabled", - env_var = ENV_TLS_RELOAD_ENABLE, - "TLS reload state changed" - ); - return None; +) -> Result>, TlsMaterialError> { + if !get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE) { + return Ok(None); } + let required_startup_files = crate::startup_tls_material::required_startup_material_files(&tls_path); + let initial = Arc::new(initial); + let mut registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + registry.retain(|_, group| group.strong_count() != 0); + let existing = registry.get(&tls_path).and_then(Weak::upgrade); + let first = existing.is_none(); + let group = existing.unwrap_or_else(|| { + Arc::new(ReloadGroup { + required_startup_files, + state: Mutex::new(ReloadState { + targets: HashMap::new(), + accepted: initial.clone(), + tcp: holder.get(), + prepared, + #[cfg(feature = "http3")] + quic: None, + }), + changed: tokio::sync::Notify::new(), + next_id: std::sync::atomic::AtomicU64::new(1), + #[cfg(test)] + pass_completed: tokio::sync::Notify::new(), + }) + }); + let id = group.next_id.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + { + let mut state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + #[cfg(feature = "http3")] + if let Some(endpoint) = &http3_endpoint { + if state.quic.is_none() { + let mut config = state.prepared.http3_config()?; + config.migration(false); + state.quic = Some(config); + } + endpoint.set_server_config(state.quic.clone()); + } + holder.swap(&TlsAcceptorHolder::new(state.tcp.clone())); + state.targets.insert( + id, + ReloadTarget { + holder, + #[cfg(feature = "http3")] + http3_endpoint, + }, + ); + } + registry.insert(tls_path.clone(), Arc::downgrade(&group)); + drop(registry); + if first { + tokio::spawn(run_reload_group(tls_path, group.clone())); + } + group.changed.notify_one(); + let registration = ReloadRegistration { group, id }; + Ok(Some(tokio::spawn(async move { + let _registration = registration; + std::future::pending::<()>().await; + }))) +} +async fn run_reload_group(tls_path: String, group: Arc) { let interval_secs = rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5); - let tls_source = TlsSource::from_directory(&tls_path); - - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_state", - state = "enabled", - interval_secs, - "TLS reload state changed" - ); - - Some(tokio::spawn(async move { - let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); - let tls_dir = PathBuf::from(&tls_path); - loop { - interval.tick().await; - - match rustfs_tls_runtime::TlsMaterialSnapshot::load(&tls_source).await { - Ok(mut snapshot) => { - if let Err(e) = enrich_outbound(&mut snapshot.outbound, &tls_dir).await { - startup_runtime_sources::record_tls_reload_result("rustfs_server_reload_loop", "enrich_err", None, None); - warn!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_failed", - stage = "enrich_outbound", - error = %e, - "TLS reload failed" - ); - continue; - } - - let generation = startup_runtime_sources::current_outbound_tls_generation().saturating_add(1); - startup_runtime_sources::publish_outbound_tls_state(TlsGeneration(generation), &snapshot.outbound).await; - startup_runtime_sources::record_tls_generation("rustfs_server_reload_loop", generation); - if !snapshot.outbound.root_ca_pem.is_empty() { - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_root_ca_enriched", - source = "reload_snapshot", - generation, - "TLS root CA set updated" - ); - } - - match reload_acceptors( - snapshot.server, - &tls_dir, - &holder, - #[cfg(feature = "http3")] - http3_endpoint.as_ref(), - ) - .await - { - Ok(true) => { - info!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_state", - state = "reloaded", - generation, - "TLS reload state changed" - ); - startup_runtime_sources::record_tls_reload_result( - "rustfs_server_reload_loop", - "ok", - None, - Some(generation), - ); - } - Ok(false) => { - startup_runtime_sources::record_tls_reload_skipped("rustfs_server_reload_loop", "no_acceptor"); - warn!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_failed", - stage = "build_acceptor", - reason = "no_acceptor", - "TLS reload returned no acceptor" - ) - } - Err(e) => { - startup_runtime_sources::record_tls_reload_result( - "rustfs_server_reload_loop", - "acceptor_err", - None, - Some(generation), - ); - warn!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_failed", - stage = "build_acceptor", - error = %e, - generation, - "TLS reload failed" - ) - } + let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut generation = 1_u64; + loop { + tokio::select! { + _ = interval.tick() => {}, + _ = group.changed.notified() => {}, + } + let (accepted, empty) = { + let state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + (state.accepted.clone(), state.targets.is_empty()) + }; + if empty { + let mut registry = RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()); + if group.state.lock().unwrap_or_else(|e| e.into_inner()).targets.is_empty() { + registry.remove(&tls_path); + return; + } + continue; + } + let result = async { + let candidate = load_tls_material(&tls_path).await?; + validate_candidate_replacement(&accepted, &candidate)?; + validate_required_material_files(&group.required_startup_files, &candidate)?; + // Listener initialization can load a newer bundle than startup published. + let published = startup_runtime_sources::current_outbound_tls_state().await; + let required_roots = + !accepted.outbound.root_ca_pem.is_empty() || published.root_ca_pem.as_ref().is_some_and(|pem| !pem.is_empty()); + let required_identity = accepted.outbound.mtls_identity.is_some() || published.mtls_identity.is_some(); + if (required_roots && candidate.outbound.root_ca_pem.is_empty()) + || (required_identity && candidate.outbound.mtls_identity.is_none()) + { + return Err(TlsMaterialError::Parse("required outbound TLS material disappeared".into())); + } + let outbound_changed = !outbound_material_matches(&accepted.outbound, &candidate.outbound) + || published.root_ca_pem.as_deref().unwrap_or_default() != candidate.outbound.root_ca_pem.as_slice() + || published.mtls_identity != candidate.outbound.mtls_identity; + if accepted.fingerprint == candidate.fingerprint && !outbound_changed { + startup_runtime_sources::record_tls_reload_skipped("rustfs_server_reload_loop", "unchanged"); + return Ok(None); + } + let prepared = PreparedServerTls::from_snapshot(&candidate)? + .ok_or_else(|| TlsMaterialError::Parse("server certificate is missing".into()))?; + let tcp = prepared.tcp_acceptor()?; + #[cfg(feature = "http3")] + let quic = { + let mut config = prepared.http3_config()?; + config.migration(false); + config + }; + let candidate = Arc::new(candidate); + let listeners = { + let mut state = group.state.lock().unwrap_or_else(|e| e.into_inner()); + // Newly registered listeners join the same prepared generation under this lock. + for target in state.targets.values() { + #[cfg(feature = "http3")] + if let Some(endpoint) = &target.http3_endpoint { + endpoint.set_server_config(Some(quic.clone())); } + target.holder.swap(&tcp); } - Err(e) => { - startup_runtime_sources::record_tls_reload_result("rustfs_server_reload_loop", "load_err", None, None); - warn!( - component = LOG_COMPONENT_TLS, - subsystem = LOG_SUBSYSTEM_TLS, - event = "tls_reload_failed", - stage = "load_snapshot", - error = %e, - "TLS reload failed" - ); + state.tcp = tcp.get(); + state.prepared = prepared; + state.accepted = candidate.clone(); + #[cfg(feature = "http3")] + { + state.quic = Some(quic); } + state.targets.len() + }; + if outbound_changed { + let next = startup_runtime_sources::current_outbound_tls_generation().saturating_add(1); + startup_runtime_sources::publish_outbound_tls_state(TlsGeneration(next), &candidate.outbound).await; + } + Ok(Some(listeners)) + } + .await; + match result { + Ok(Some(listeners)) => { + generation = generation.saturating_add(1); + startup_runtime_sources::record_tls_generation("rustfs_server_reload_loop", generation); + startup_runtime_sources::record_tls_reload_result("rustfs_server_reload_loop", "ok", None, Some(generation)); + info!( + event = "tls_reload_state", + component = LOG_COMPONENT_TLS, + subsystem = LOG_SUBSYSTEM_TLS, + state = "reloaded", + generation, + listeners, + "TLS material updated" + ); + } + Ok(None) => {} + Err(error) => { + startup_runtime_sources::record_tls_reload_result("rustfs_server_reload_loop", "rejected", None, None); + warn!(event = "tls_reload_failed", component = LOG_COMPONENT_TLS, subsystem = LOG_SUBSYSTEM_TLS, + result = "rejected", error = %error, "TLS material update rejected"); } } - })) + #[cfg(test)] + group.pass_completed.notify_one(); + } } #[cfg(test)] mod tests { use super::*; use rcgen::CertifiedKey; + use rustfs_config::RUSTFS_TLS_CERT; use std::fs; use std::sync::Once; use tempfile::TempDir; @@ -839,6 +1063,507 @@ mod tests { fs::write(dir.join(rustfs_config::RUSTFS_TLS_KEY), signing_key.serialize_pem()).unwrap(); } + #[tokio::test] + #[serial_test::serial] + async fn prepared_generation_shares_crypto_and_serves_sni_with_mtls() { + ensure_rustls_crypto_provider(); + temp_env::async_with_vars([(ENV_SERVER_MTLS_ENABLE, Some("1"))], async { + let dir = TempDir::new().expect("TLS directory"); + write_test_cert_pair(dir.path(), "default.local"); + let sni_dir = dir.path().join("sni.example"); + fs::create_dir(&sni_dir).expect("SNI directory"); + write_test_cert_pair(&sni_dir, "sni.example"); + let mut ca_params = rcgen::CertificateParams::new(Vec::::new()).expect("CA parameters"); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + let ca_key = rcgen::KeyPair::generate().expect("CA key"); + let ca = ca_params.self_signed(&ca_key).expect("CA certificate"); + fs::write(dir.path().join(RUSTFS_CLIENT_CA_CERT_FILENAME), ca.pem()).expect("client trust"); + let mut client_params = rcgen::CertificateParams::new(vec!["client.example".into()]).expect("client parameters"); + client_params.extended_key_usages = vec![rcgen::ExtendedKeyUsagePurpose::ClientAuth]; + let client_key = rcgen::KeyPair::generate().expect("client key"); + let issuer = rcgen::Issuer::from_params(&ca_params, &ca_key); + let client_cert = client_params.signed_by(&client_key, &issuer).expect("client identity"); + let snapshot = rustfs_tls_runtime::TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .expect("validated TLS snapshot"); + let before = TLS_PREPARE_COUNTS.with(std::cell::Cell::get); + let prepared = PreparedServerTls::from_snapshot(&snapshot) + .expect("prepare generation") + .expect("server TLS"); + let tcp = prepared.server_config(ServerProtocol::Tcp).expect("TCP config"); + let another_tcp = prepared + .server_config(ServerProtocol::Tcp) + .expect("additional listener config"); + assert!(Arc::ptr_eq(&tcp.cert_resolver, &another_tcp.cert_resolver)); + #[cfg(feature = "http3")] + { + let quic_tls = prepared.server_config(ServerProtocol::Http3).expect("QUIC TLS config"); + assert!(Arc::ptr_eq(&tcp.cert_resolver, &quic_tls.cert_resolver)); + } + let after = TLS_PREPARE_COUNTS.with(std::cell::Cell::get); + assert_eq!( + (after.0 - before.0, after.1 - before.1), + (1, 1), + "client verifier and signing resolver must each be prepared once for all protocols and listeners" + ); + + let RuntimeServerTlsMaterial::MultiCert { cert_key_pairs } = snapshot.server.as_ref().expect("SNI material") else { + panic!("multiple selectors expected"); + }; + let mut roots = rustls::RootCertStore::empty(); + for (certificates, _) in cert_key_pairs.values() { + roots.add(certificates[0].clone()).expect("server trust"); + } + for (name, selector) in [("default.local", "default"), ("sni.example", "sni.example")] { + let expected = cert_key_pairs[selector].0[0].clone(); + let client_tls = rustls::ClientConfig::builder() + .with_root_certificates(roots.clone()) + .with_client_auth_cert( + vec![client_cert.der().clone()], + PrivateKeyDer::try_from(client_key.serialize_der()).expect("client private key"), + ) + .expect("mTLS client"); + let (server_io, client_io) = tokio::io::duplex(65536); + let acceptor = prepared.tcp_acceptor().expect("acceptor").get(); + let connector = tokio_rustls::TlsConnector::from(Arc::new(client_tls.clone())); + let (server, client) = tokio::join!( + acceptor.accept(server_io), + connector.connect(rustls::pki_types::ServerName::try_from(name.to_owned()).expect("server name"), client_io) + ); + let server = server.expect("authenticated server handshake"); + let client = client.expect("authenticated client handshake"); + assert_eq!(client.get_ref().1.peer_certificates().expect("server identity")[0], expected); + assert_eq!(server.get_ref().1.peer_certificates().expect("client identity")[0], *client_cert.der()); + + #[cfg(feature = "http3")] + { + let endpoint = quinn::Endpoint::server( + prepared.http3_config().expect("QUIC config"), + "127.0.0.1:0".parse().expect("address"), + ) + .expect("QUIC server"); + let mut client_tls = client_tls; + client_tls.alpn_protocols = vec![b"h3".to_vec()]; + let crypto = quinn::crypto::rustls::QuicClientConfig::try_from(client_tls).expect("QUIC client TLS"); + let mut client_endpoint = + quinn::Endpoint::client("127.0.0.1:0".parse().expect("address")).expect("QUIC client"); + client_endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new(crypto))); + let connection = client_endpoint + .connect(endpoint.local_addr().expect("server address"), name) + .expect("connect"); + let (server, client) = tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!(async { endpoint.accept().await.expect("incoming").await }, connection) + }) + .await + .expect("QUIC handshake deadline"); + let server = server.expect("QUIC mTLS server handshake"); + let client = client.expect("QUIC mTLS client handshake"); + let identity = client + .peer_identity() + .expect("QUIC server identity") + .downcast::>>() + .expect("certificate identity"); + assert_eq!(identity[0], expected); + let identity = server + .peer_identity() + .expect("QUIC client identity") + .downcast::>>() + .expect("certificate identity"); + assert_eq!(identity[0], *client_cert.der()); + client.close(0u32.into(), b"done"); + server.close(0u32.into(), b"done"); + endpoint.close(0u32.into(), b"done"); + client_endpoint.close(0u32.into(), b"done"); + } + } + }) + .await; + } + + #[tokio::test] + async fn candidate_replacement_preserves_ca_components_and_default_names() { + let dir = TempDir::new().expect("TLS directory"); + write_test_cert_pair(dir.path(), "s3.example.com"); + let ca_a = rcgen::generate_simple_self_signed(vec!["ca-a.example".into()]).expect("CA A"); + let ca_b = rcgen::generate_simple_self_signed(vec!["ca-b.example".into()]).expect("CA B"); + fs::write(dir.path().join(RUSTFS_CLIENT_CA_CERT_FILENAME), ca_a.cert.pem()).expect("primary client CA"); + fs::write(dir.path().join(RUSTFS_CA_CERT), ca_b.cert.pem()).expect("fallback CA"); + let source = TlsSource::from_directory(dir.path()); + let accepted = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("accepted generation"); + fs::remove_file(dir.path().join(RUSTFS_CLIENT_CA_CERT_FILENAME)).expect("remove primary client CA"); + let candidate = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("fallback generation parses"); + assert!( + validate_candidate_replacement(&accepted, &candidate).is_err(), + "a fallback must not replace accepted client trust" + ); + + fs::write(dir.path().join(RUSTFS_CLIENT_CA_CERT_FILENAME), ca_a.cert.pem()).expect("restore primary CA"); + write_test_cert_pair(dir.path(), "wrong.example.com"); + let candidate = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("valid but wrong SAN"); + assert!( + validate_candidate_replacement(&accepted, &candidate).is_err(), + "default SAN coverage must survive renewal" + ); + + write_test_cert_pair(dir.path(), "s3.example.com"); + let renewed = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("valid renewal"); + validate_candidate_replacement(&accepted, &renewed).expect("renewal preserving names and trust sources"); + } + + #[tokio::test] + async fn candidate_default_names_cannot_move_into_another_sni_certificate() { + let dir = TempDir::new().expect("TLS directory"); + let sni = dir.path().join("example.com"); + fs::create_dir(&sni).expect("SNI directory"); + write_test_cert_pair(dir.path(), "default.local"); + write_test_cert_pair(&sni, "example.com"); + let source = TlsSource::from_directory(dir.path()); + let accepted = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("accepted selectors"); + write_test_cert_pair(dir.path(), "wrong.local"); + let relocated = rcgen::generate_simple_self_signed(vec!["example.com".into(), "default.local".into()]) + .expect("certificate with relocated SAN"); + fs::write(sni.join(RUSTFS_TLS_CERT), relocated.cert.pem()).expect("SNI certificate"); + fs::write(sni.join(rustfs_config::RUSTFS_TLS_KEY), relocated.signing_key.serialize_pem()).expect("SNI key"); + let candidate = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("individually valid candidate"); + assert!( + validate_candidate_replacement(&accepted, &candidate).is_err(), + "another selector's SAN does not cover the default resolver's hostname" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn snapshot_leaf_trust_is_encoded_in_canonical_der_order() { + let dir = TempDir::new().expect("TLS directory"); + let a = rcgen::generate_simple_self_signed(vec!["a.example".into()]).expect("certificate A"); + let b = rcgen::generate_simple_self_signed(vec!["b.example".into()]).expect("certificate B"); + let (first, second) = if a.cert.der().as_ref() > b.cert.der().as_ref() { + (&a, &b) + } else { + (&b, &a) + }; + fs::write(dir.path().join(RUSTFS_TLS_CERT), format!("{}{}", first.cert.pem(), second.cert.pem())) + .expect("descending DER chain"); + fs::write(dir.path().join(rustfs_config::RUSTFS_TLS_KEY), first.signing_key.serialize_pem()).expect("leaf key"); + let mut snapshot = rustfs_tls_runtime::TlsMaterialSnapshot::load(&TlsSource::from_directory(dir.path())) + .await + .expect("validated material"); + temp_env::async_with_vars( + [ + (ENV_TRUST_LEAF_CERT_AS_CA, Some("1")), + (ENV_TRUST_SYSTEM_CA, Some("0")), + (ENV_MTLS_CLIENT_CERT, None), + (ENV_MTLS_CLIENT_KEY, None), + ], + async { + enrich_outbound(&mut snapshot).await.expect("derive leaf trust"); + let mut reader = Cursor::new(&snapshot.outbound.root_ca_pem); + let certificates = CertificateDer::pem_reader_iter(&mut reader) + .collect::, _>>() + .expect("derived certificate PEM"); + assert_eq!( + certificates, + vec![second.cert.der().clone(), first.cert.der().clone()], + "leaf trust must not depend on source enumeration order" + ); + }, + ) + .await; + } + + #[tokio::test] + async fn client_identity_override_rejects_mismatch_and_missing_files() { + let dir = TempDir::new().expect("identity directory"); + let a = rcgen::generate_simple_self_signed(vec!["client-a.example".into()]).expect("identity A"); + let b = rcgen::generate_simple_self_signed(vec!["client-b.example".into()]).expect("identity B"); + let cert = dir.path().join(RUSTFS_CLIENT_CERT_FILENAME); + let key = dir.path().join(RUSTFS_CLIENT_KEY_FILENAME); + fs::write(&cert, a.cert.pem()).expect("client cert"); + fs::write(&key, b.signing_key.serialize_pem()).expect("mismatched key"); + assert!( + load_mtls_identity_from_overridden_paths(cert.to_str(), key.to_str(), dir.path()) + .await + .is_err() + ); + fs::remove_file(&key).expect("remove override key"); + assert!( + load_mtls_identity_from_overridden_paths(cert.to_str(), key.to_str(), dir.path()) + .await + .is_err() + ); + fs::write(&key, a.signing_key.serialize_pem()).expect("matching key"); + assert!( + load_mtls_identity_from_overridden_paths(cert.to_str(), key.to_str(), dir.path()) + .await + .expect("valid override") + .is_some() + ); + } + + enum StartupRotation { + Outbound, + RootsOnly, + IdentityOnly, + ServerOnly, + MissingRoots, + PartialRoots, + MissingIdentity, + } + + async fn assert_initial_outbound_reconciliation(rotation: StartupRotation) { + ensure_rustls_crypto_provider(); + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TRUST_LEAF_CERT_AS_CA, Some("0")), + (ENV_TRUST_SYSTEM_CA, Some("0")), + (ENV_MTLS_CLIENT_CERT, None), + (ENV_MTLS_CLIENT_KEY, None), + (ENV_RUSTFS_EXTRA_CA_CERT, None), + (super::super::tls_publish::ENV_TLS_PUBLISH_DIR, None), + ], + async { + let previous = startup_runtime_sources::current_outbound_tls_state().await; + let previous_files = crate::startup_tls_material::replace_startup_material_files(None); + let dir = TempDir::new().expect("TLS directory"); + write_test_cert_pair(dir.path(), "server.example"); + let identity_a = rcgen::generate_simple_self_signed(vec!["client-a.example".into()]).expect("identity A"); + fs::write(dir.path().join(RUSTFS_CA_CERT), identity_a.cert.pem()).expect("trust A"); + fs::write(dir.path().join(RUSTFS_CLIENT_CERT_FILENAME), identity_a.cert.pem()).expect("client certificate A"); + fs::write(dir.path().join(RUSTFS_CLIENT_KEY_FILENAME), identity_a.signing_key.serialize_pem()) + .expect("client key A"); + if matches!(rotation, StartupRotation::PartialRoots) { + let public_ca = + rcgen::generate_simple_self_signed(vec!["public-ca.example".into()]).expect("distinct public CA"); + fs::write(dir.path().join(rustfs_config::RUSTFS_PUBLIC_CERT), public_ca.cert.pem()) + .expect("additional public trust"); + } + let path = dir.path().to_str().expect("UTF8 path").to_string(); + let mut config = crate::config::Config::new("127.0.0.1:0", Vec::new()); + config.tls_path = Some(path.clone()); + crate::startup_tls_material::init_outbound_tls_material(&config) + .await + .expect("real startup publication"); + let startup = startup_runtime_sources::current_outbound_tls_state().await; + + write_test_cert_pair(dir.path(), "server.example"); + match rotation { + StartupRotation::Outbound | StartupRotation::RootsOnly | StartupRotation::IdentityOnly => { + let identity_b = rcgen::generate_simple_self_signed(vec!["client-b.example".into()]).expect("identity B"); + if !matches!(rotation, StartupRotation::IdentityOnly) { + fs::write(dir.path().join(RUSTFS_CA_CERT), identity_b.cert.pem()).expect("trust B"); + } + if !matches!(rotation, StartupRotation::RootsOnly) { + fs::write(dir.path().join(RUSTFS_CLIENT_CERT_FILENAME), identity_b.cert.pem()) + .expect("client certificate B"); + fs::write(dir.path().join(RUSTFS_CLIENT_KEY_FILENAME), identity_b.signing_key.serialize_pem()) + .expect("client key B"); + } + } + StartupRotation::ServerOnly => {} + StartupRotation::MissingRoots | StartupRotation::PartialRoots => { + fs::remove_file(dir.path().join(RUSTFS_CA_CERT)).expect("remove startup roots"); + } + StartupRotation::MissingIdentity => { + fs::remove_file(dir.path().join(RUSTFS_CLIENT_CERT_FILENAME)).expect("remove startup client certificate"); + fs::remove_file(dir.path().join(RUSTFS_CLIENT_KEY_FILENAME)).expect("remove startup client key"); + } + } + let initial = load_tls_material(&path).await.expect("HTTP initialization snapshot"); + let incomplete = matches!( + rotation, + StartupRotation::MissingRoots | StartupRotation::PartialRoots | StartupRotation::MissingIdentity + ); + let expected_roots = if incomplete { + startup.root_ca_pem.clone().expect("startup trust") + } else { + initial.outbound.root_ca_pem.clone() + }; + let expected_identity = if incomplete { + startup.mtls_identity.clone() + } else { + initial.outbound.mtls_identity.clone() + }; + let prepared = PreparedServerTls::from_snapshot(&initial).expect("config B").expect("TLS B"); + let holder = prepared.tcp_acceptor().expect("TLS B acceptor"); + let initial_acceptor = holder.get(); + let task = spawn_reload_loop( + path.clone(), + holder.clone(), + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("register HTTP initialization snapshot") + .expect("reload enabled"); + let group = RELOAD_GROUPS + .lock() + .unwrap_or_else(|e| e.into_inner()) + .get(&path) + .and_then(Weak::upgrade) + .expect("registered reload group"); + notify_server_publication(&path); + tokio::time::timeout(Duration::from_secs(10), group.pass_completed.notified()) + .await + .expect("first reload pass completed"); + let first = startup_runtime_sources::current_outbound_tls_state().await; + notify_server_publication(&path); + tokio::time::timeout(Duration::from_secs(10), group.pass_completed.notified()) + .await + .expect("second reload pass completed"); + let second = startup_runtime_sources::current_outbound_tls_state().await; + + task.abort(); + let _ = task.await; + tokio::time::timeout(Duration::from_secs(10), async { + while RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()).contains_key(&path) { + tokio::task::yield_now().await; + } + }) + .await + .expect("reload group exits before restoring global state"); + startup_runtime_sources::publish_outbound_tls_state( + TlsGeneration(second.generation.0.saturating_add(1)), + &rustfs_tls_runtime::OutboundTlsMaterial { + root_ca_pem: previous.root_ca_pem.unwrap_or_default(), + mtls_identity: previous.mtls_identity, + }, + ) + .await; + crate::startup_tls_material::replace_startup_material_files(previous_files); + + assert_eq!(first.root_ca_pem.as_deref(), Some(expected_roots.as_slice())); + assert!( + first.mtls_identity == expected_identity, + "published identity must match the initialized listener snapshot" + ); + let rotate_outbound = matches!( + rotation, + StartupRotation::Outbound | StartupRotation::RootsOnly | StartupRotation::IdentityOnly + ); + let expected_generation = startup.generation.0.saturating_add(u64::from(rotate_outbound)); + assert_eq!(first.generation.0, expected_generation); + assert_eq!(second.generation, first.generation, "unchanged material must not churn outbound clients"); + assert!( + rotate_outbound || Arc::ptr_eq(&holder.get(), &initial_acceptor), + "non-publications must retain the initial server configuration" + ); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_reconciles_initial_outbound_after_startup_rotation() { + for rotation in [ + StartupRotation::RootsOnly, + StartupRotation::IdentityOnly, + StartupRotation::Outbound, + ] { + assert_initial_outbound_reconciliation(rotation).await; + } + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_initial_server_rotation_preserves_matching_outbound_generation() { + assert_initial_outbound_reconciliation(StartupRotation::ServerOnly).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_initial_missing_roots_retains_startup_material() { + assert_initial_outbound_reconciliation(StartupRotation::MissingRoots).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_initial_partial_missing_roots_retains_startup_material() { + assert_initial_outbound_reconciliation(StartupRotation::PartialRoots).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_reload_initial_missing_identity_retains_startup_material() { + assert_initial_outbound_reconciliation(StartupRotation::MissingIdentity).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn late_listener_joins_accepted_generation_when_source_is_invalid() { + ensure_rustls_crypto_provider(); + temp_env::async_with_vars([(ENV_TLS_RELOAD_ENABLE, Some("1")), (ENV_SERVER_MTLS_ENABLE, Some("0"))], async { + let dir = TempDir::new().expect("TLS directory"); + write_test_cert_pair(dir.path(), "server.example"); + let source = TlsSource::from_directory(dir.path()); + let a = rustfs_tls_runtime::TlsMaterialSnapshot::load(&source) + .await + .expect("generation A"); + let prepared_a = PreparedServerTls::from_snapshot(&a).expect("prepare A").expect("TLS A"); + let first = prepared_a.tcp_acceptor().expect("config A"); + let path = dir.path().to_str().expect("UTF8 path").to_string(); + let task_a = spawn_reload_loop( + path.clone(), + first.clone(), + a, + prepared_a, + #[cfg(feature = "http3")] + None, + ) + .expect("register A") + .expect("reload enabled"); + // No async yield between producing B and registration: the actor cannot accept B. + write_test_cert_pair(dir.path(), "server.example"); + let b = rustfs_tls_runtime::TlsMaterialSnapshot::load_from_snapshot(&source.snapshot().expect("pin B")) + .expect("generation B"); + let prepared_b = PreparedServerTls::from_snapshot(&b).expect("prepare B").expect("TLS B"); + let second = prepared_b.tcp_acceptor().expect("config B"); + assert!(!Arc::ptr_eq(&first.get(), &second.get())); + fs::write(dir.path().join(RUSTFS_TLS_CERT), "invalid certificate").expect("break source"); + let task_b = spawn_reload_loop( + path.clone(), + second.clone(), + b, + prepared_b, + #[cfg(feature = "http3")] + None, + ) + .expect("register B") + .expect("reload enabled"); + assert!( + Arc::ptr_eq(&first.get(), &second.get()), + "late listener must use the group's accepted config" + ); + task_a.abort(); + task_b.abort(); + let _ = task_a.await; + let _ = task_b.await; + tokio::time::timeout(Duration::from_secs(5), async { + while RELOAD_GROUPS.lock().unwrap_or_else(|e| e.into_inner()).contains_key(&path) { + tokio::task::yield_now().await; + } + }) + .await + .expect("group exits when all registrations are dropped"); + }) + .await; + } + #[tokio::test] #[serial_test::serial] async fn oidc_extra_ca_cert_loads_configured_bundle() { @@ -1070,15 +1795,17 @@ mod tests { ensure_rustls_crypto_provider(); let cert = rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).expect("certificate should generate"); - let config = build_server_config( - ServerCertSource::SingleCert { + let config = PreparedServerTls::from_material( + Some(&RuntimeServerTlsMaterial::SingleCert { certs: vec![cert.cert.der().clone()], key: rustls::pki_types::PrivateKeyDer::try_from(cert.signing_key.serialize_der()) .expect("private key should convert"), - }, + }), None, - ServerProtocol::Http3, ) + .expect("prepare TLS") + .expect("server certificate") + .server_config(ServerProtocol::Http3) .expect("HTTP/3 TLS config should build"); assert_eq!(config.alpn_protocols, vec![b"h3".to_vec()]); diff --git a/rustfs/src/server/tls_publish.rs b/rustfs/src/server/tls_publish.rs new file mode 100644 index 000000000..ab11787e9 --- /dev/null +++ b/rustfs/src/server/tls_publish.rs @@ -0,0 +1,1341 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Durable server-leaf publication. Trust and client identity remain owned by their original source. + +use super::tls_material; +use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_ENABLE, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; +use rustfs_tls_runtime::{ServerTlsMaterial, TlsMaterialSnapshot, TlsSource}; +use rustls::pki_types::{CertificateDer, pem::PemObject}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + fs::{self, File, OpenOptions}, + io::{Read, Write}, + path::{Component, Path, PathBuf}, +}; +use uuid::Uuid; + +pub(crate) const ENV_TLS_PUBLISH_DIR: &str = "RUSTFS_TLS_PUBLISH_DIR"; +pub(crate) const SCHEMA: &str = "rustfs.tls-publish/v1alpha1"; +pub(crate) const RESOURCE: &str = "server-default"; +const MAX_CERT: usize = 64 * 1024; +const MAX_KEY: usize = 32 * 1024; +const MANIFEST: &str = "manifest.json"; +const CURRENT: &str = "server-current"; +const INITIALIZED: &str = "initialized"; +const INITIALIZED_CONTENT: &[u8] = b"rustfs.tls-publish/v1alpha1\n"; + +#[cfg(test)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct OperationReads { + certificates: usize, + keys: usize, + manifests: usize, + manifest_decodes: usize, + certificate_pem_traversals: usize, + leaf_der_decodes: usize, +} + +#[cfg(test)] +thread_local! { + static OPERATION_READS: std::cell::Cell = const { std::cell::Cell::new(OperationReads { + certificates: 0, keys: 0, manifests: 0, manifest_decodes: 0, + certificate_pem_traversals: 0, leaf_der_decodes: 0, + }) }; +} + +#[cfg(test)] +fn count_operation(action: impl FnOnce(&mut OperationReads)) { + OPERATION_READS.with(|counts| { + let mut value = counts.get(); + action(&mut value); + counts.set(value); + }); +} + +fn certificate_pem_items( + bytes: &[u8], +) -> impl Iterator, rustls::pki_types::pem::Error>> + '_ { + #[cfg(test)] + count_operation(|counts| counts.certificate_pem_traversals += 1); + CertificateDer::pem_slice_iter(bytes) +} + +#[cfg(test)] +#[derive(Clone, Copy)] +enum FailurePoint { + None, + AfterPointerRename, + AfterPointerRenameUnreadable, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct PublishRequest { + pub schema_version: String, + pub resource_id: String, + pub expected_target_revision: String, + pub issuance_id: Uuid, + pub certificate_pem: String, + pub private_key_pem: String, +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Manifest { + schema_version: String, + issuance_id: Uuid, + target_revision: Uuid, + certificate_fingerprint: String, + not_after_unix: i64, + payload_digest: String, +} + +// A disk operation pins one immutable generation and verifies its envelope once. +// This value is never retained across requests or reload ticks. +struct PublishedGeneration { + manifest: Manifest, + chain: Vec>, + leaf: LeafMetadata, + key_pem: Vec, +} + +struct LeafMetadata { + fingerprint: String, + not_after_unix: i64, + names: std::collections::BTreeSet, + is_ca: bool, +} + +#[derive(Serialize)] +pub(crate) struct CertificateState { + pub schema_version: &'static str, + pub resource_id: &'static str, + pub target_revision: String, + pub certificate_fingerprint: String, + pub not_after_unix: i64, + pub last_issuance_id: Option, + pub observed_certificate_fingerprint: Option, +} + +#[derive(Serialize)] +pub(crate) struct PublishReceipt { + pub schema_version: &'static str, + pub resource_id: &'static str, + pub issuance_id: Uuid, + pub target_revision: String, + pub certificate_fingerprint: String, + pub not_after_unix: i64, + pub observed_certificate_fingerprint: Option, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum PublishError { + #[error("certificate publication is disabled or not configured safely")] + Disabled, + #[error("invalid certificate publication request or source")] + Invalid, + #[error("certificate publication revision conflict")] + Conflict, + #[error("certificate publication is busy; retry the identical issuance")] + Busy, + #[error("certificate publication storage failed; retry the identical issuance")] + Storage, +} + +impl From for PublishError { + fn from(_: std::io::Error) -> Self { + Self::Storage + } +} + +fn hex_digest(bytes: &[u8]) -> String { + Sha256::digest(bytes).iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn payload_digest(certificate: &[u8], key: &[u8]) -> String { + let mut digest = Sha256::new(); + // Both inputs have bounded sizes; the fixed-width encoding survives architecture changes. + digest.update((certificate.len() as u128).to_be_bytes()); + digest.update(certificate); + digest.update((key.len() as u128).to_be_bytes()); + digest.update(key); + digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn parse_leaf(leaf: &[u8]) -> Result { + #[cfg(test)] + count_operation(|counts| counts.leaf_der_decodes += 1); + let (remaining, parsed) = x509_parser::parse_x509_certificate(leaf).map_err(|_| PublishError::Invalid)?; + if !remaining.is_empty() { + return Err(PublishError::Invalid); + } + let mut names = std::collections::BTreeSet::new(); + if let Some(san) = parsed.subject_alternative_name().map_err(|_| PublishError::Invalid)? { + for name in &san.value.general_names { + match name { + x509_parser::extensions::GeneralName::DNSName(name) => { + names.insert(format!("dns:{}", name.to_ascii_lowercase())); + } + x509_parser::extensions::GeneralName::IPAddress(bytes) => { + names.insert(format!("ip:{bytes:?}")); + } + _ => {} + } + } + } + Ok(LeafMetadata { + fingerprint: hex_digest(leaf), + not_after_unix: parsed.validity().not_after.timestamp(), + names, + is_ca: parsed + .basic_constraints() + .map_err(|_| PublishError::Invalid)? + .is_some_and(|constraints| constraints.value.ca), + }) +} + +fn snapshot_leaf(snapshot: &TlsMaterialSnapshot) -> Option { + match &snapshot.server { + Some(ServerTlsMaterial::SingleCert { certs, .. }) => certs.first().map(|cert| hex_digest(cert.as_ref())), + _ => None, + } +} + +fn observed(tls_path: &str, expected_chain: &[CertificateDer<'_>]) -> Option { + let snapshot = tls_material::observed_server_snapshot(tls_path)?; + match &snapshot.server { + Some(ServerTlsMaterial::SingleCert { certs, .. }) if certs == expected_chain => snapshot_leaf(&snapshot), + _ => None, + } +} + +fn open_regular(path: &Path, writable: bool, create: bool) -> Result { + let mut options = OpenOptions::new(); + options.read(true).write(writable).create(create); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC); + } + let file = options.open(path)?; + if !file.metadata()?.is_file() { + return Err(PublishError::Invalid); + } + Ok(file) +} + +fn read_limited(path: &Path, limit: usize) -> Result, PublishError> { + let file = open_regular(path, false, false)?; + #[cfg(test)] + count_operation(|counts| match path.file_name().and_then(|name| name.to_str()) { + Some(RUSTFS_TLS_CERT) => counts.certificates += 1, + Some(RUSTFS_TLS_KEY) => counts.keys += 1, + Some(MANIFEST) => counts.manifests += 1, + _ => {} + }); + let mut bytes = Vec::new(); + file.take(u64::try_from(limit).map_err(|_| PublishError::Invalid)? + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > limit { + return Err(PublishError::Invalid); + } + Ok(bytes) +} + +fn validate_root(root: &Path) -> Result<(), PublishError> { + let metadata = fs::symlink_metadata(root)?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(PublishError::Disabled); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if metadata.mode() & 0o077 != 0 || metadata.uid() != rustix::process::geteuid().as_raw() { + return Err(PublishError::Disabled); + } + } + #[cfg(not(unix))] + { + Err(PublishError::Disabled) + } + #[cfg(unix)] + { + Ok(()) + } +} + +fn configured_root() -> Result, PublishError> { + let root = rustfs_utils::get_env_opt_str(ENV_TLS_PUBLISH_DIR).map(PathBuf::from); + if let Some(root) = &root { + validate_root(root)?; + } + Ok(root) +} + +fn single_default_source(tls_path: &str) -> Result<(), PublishError> { + if fs::symlink_metadata(Path::new(tls_path).join("..data")).is_ok() { + return Err(PublishError::Disabled); + } + for entry in fs::read_dir(tls_path)? { + let entry = entry?; + if !entry.file_name().to_string_lossy().starts_with('.') && entry.path().is_dir() { + return Err(PublishError::Invalid); + } + } + Ok(()) +} + +fn current_generation(root: &Path) -> Result, PublishError> { + let initialized = match fs::symlink_metadata(root.join(INITIALIZED)) { + Ok(_) => { + if read_limited(&root.join(INITIALIZED), 128)? != INITIALIZED_CONTENT { + return Err(PublishError::Invalid); + } + true + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Err(error) => return Err(error.into()), + }; + let pointer = root.join(CURRENT); + match fs::symlink_metadata(&pointer) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return if initialized { Err(PublishError::Invalid) } else { Ok(None) }; + } + Err(error) => return Err(error.into()), + Ok(metadata) if !metadata.file_type().is_symlink() => return Err(PublishError::Invalid), + _ => {} + } + let target = fs::read_link(pointer)?; + let mut components = target.components(); + if components.next() != Some(Component::Normal("generations".as_ref())) { + return Err(PublishError::Invalid); + } + let Some(Component::Normal(version)) = components.next() else { + return Err(PublishError::Invalid); + }; + let id = version + .to_str() + .and_then(|text| Uuid::parse_str(text).ok()) + .ok_or(PublishError::Invalid)?; + if components.next().is_some() || version != id.to_string().as_str() { + return Err(PublishError::Invalid); + } + let generations = root.join("generations"); + if fs::symlink_metadata(&generations)?.file_type().is_symlink() { + return Err(PublishError::Invalid); + } + let directory = root.join(target); + if !fs::symlink_metadata(&directory)?.is_dir() { + return Err(PublishError::Invalid); + } + PublishedGeneration::load(directory, id).map(Some) +} + +fn validate_source_separation(root: &Path, tls_path: &str) -> Result<(), PublishError> { + let root = fs::canonicalize(root)?; + let source = fs::canonicalize(tls_path)?; + if root.starts_with(&source) || source.starts_with(&root) { + return Err(PublishError::Disabled); + } + Ok(()) +} + +impl PublishedGeneration { + fn into_snapshot(self, source: &rustfs_tls_runtime::TlsSourceSnapshot) -> Result { + TlsMaterialSnapshot::load_from_snapshot_with_server_identity(source, self.chain, &self.key_pem) + .map_err(|_| PublishError::Invalid) + } + + fn load(directory: PathBuf, revision: Uuid) -> Result { + let manifest_bytes = read_limited(&directory.join(MANIFEST), 4096)?; + #[cfg(test)] + count_operation(|counts| counts.manifest_decodes += 1); + let manifest: Manifest = serde_json::from_slice(&manifest_bytes).map_err(|_| PublishError::Invalid)?; + let certificate = read_limited(&directory.join(RUSTFS_TLS_CERT), MAX_CERT)?; + let key = read_limited(&directory.join(RUSTFS_TLS_KEY), MAX_KEY)?; + let chain = certificate_pem_items(&certificate) + .collect::, _>>() + .map_err(|_| PublishError::Invalid)?; + let leaf = parse_leaf(chain.first().ok_or(PublishError::Invalid)?.as_ref())?; + if manifest.schema_version != SCHEMA + || manifest.issuance_id.is_nil() + || manifest.target_revision.is_nil() + || manifest.target_revision != revision + || manifest.certificate_fingerprint != leaf.fingerprint + || manifest.not_after_unix != leaf.not_after_unix + || manifest.payload_digest != payload_digest(&certificate, &key) + { + return Err(PublishError::Invalid); + } + Ok(Self { + manifest, + chain, + leaf, + key_pem: key, + }) + } +} + +#[cfg(test)] +fn current_directory(root: &Path) -> Result, PublishError> { + current_generation(root).map(|generation| { + generation.map(|generation| root.join("generations").join(generation.manifest.target_revision.to_string())) + }) +} + +pub(crate) fn configured_server_snapshot( + tls_path: &str, + source: &rustfs_tls_runtime::TlsSourceSnapshot, +) -> Result, PublishError> { + let Some(root) = configured_root()? else { + return Ok(None); + }; + validate_source_separation(&root, tls_path)?; + single_default_source(tls_path)?; + current_generation(&root)? + .map(|generation| generation.into_snapshot(source)) + .transpose() +} + +fn state_at(root: &Path, tls_path: &str) -> Result { + let generation = current_generation(root)?; + state_from_generation(generation.as_ref(), tls_path) +} + +fn state_from_generation(generation: Option<&PublishedGeneration>, tls_path: &str) -> Result { + if let Some(generation) = generation { + return Ok(CertificateState { + schema_version: SCHEMA, + resource_id: RESOURCE, + target_revision: generation.manifest.target_revision.to_string(), + certificate_fingerprint: generation.leaf.fingerprint.clone(), + not_after_unix: generation.leaf.not_after_unix, + last_issuance_id: Some(generation.manifest.issuance_id), + observed_certificate_fingerprint: observed(tls_path, &generation.chain), + }); + } + let snapshot = tls_material::observed_server_snapshot(tls_path).ok_or(PublishError::Disabled)?; + let Some(ServerTlsMaterial::SingleCert { certs, .. }) = &snapshot.server else { + return Err(PublishError::Invalid); + }; + let leaf = parse_leaf(certs.first().ok_or(PublishError::Invalid)?.as_ref())?; + Ok(CertificateState { + schema_version: SCHEMA, + resource_id: RESOURCE, + target_revision: format!("bootstrap-{}", leaf.fingerprint), + certificate_fingerprint: leaf.fingerprint, + not_after_unix: leaf.not_after_unix, + last_issuance_id: None, + observed_certificate_fingerprint: observed(tls_path, certs), + }) +} + +pub(crate) async fn certificate_state(tls_path: String) -> Result { + tokio::task::spawn_blocking(move || { + let root = configured_root()?.ok_or(PublishError::Disabled)?; + validate_source_separation(&root, &tls_path)?; + single_default_source(&tls_path)?; + state_at(&root, &tls_path) + }) + .await + .map_err(|_| PublishError::Storage)? +} + +fn write_new(path: &Path, bytes: &[u8]) -> Result<(), PublishError> { + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW); + } + let mut file = options.open(path)?; + file.write_all(bytes)?; + file.sync_all()?; + Ok(()) +} + +fn receipt(state: CertificateState, issuance_id: Uuid) -> PublishReceipt { + PublishReceipt { + schema_version: SCHEMA, + resource_id: RESOURCE, + issuance_id, + target_revision: state.target_revision, + certificate_fingerprint: state.certificate_fingerprint, + not_after_unix: state.not_after_unix, + observed_certificate_fingerprint: state.observed_certificate_fingerprint, + } +} + +fn mark_initialized(root: &Path) -> Result<(), PublishError> { + let marker = root.join(INITIALIZED); + if fs::symlink_metadata(&marker).is_ok() { + if read_limited(&marker, 128)? != INITIALIZED_CONTENT { + return Err(PublishError::Invalid); + } + } else { + // Partial writes must remain unreferenced so an identical issuance can recover after I/O failure. + let temporary = root.join(format!(".initialized-{}", Uuid::new_v4())); + write_new(&temporary, INITIALIZED_CONTENT)?; + fs::rename(temporary, &marker)?; + } + File::open(root)?.sync_all()?; + Ok(()) +} + +fn publish_at(root: &Path, tls_path: &str, request: PublishRequest) -> Result { + publish_at_inner( + root, + tls_path, + request, + #[cfg(test)] + FailurePoint::None, + ) +} + +fn publish_at_inner( + root: &Path, + tls_path: &str, + request: PublishRequest, + #[cfg(test)] failure: FailurePoint, +) -> Result { + validate_root(root)?; + validate_source_separation(root, tls_path)?; + single_default_source(tls_path)?; + if request.schema_version != SCHEMA + || request.resource_id != RESOURCE + || request.issuance_id.is_nil() + || request.certificate_pem.is_empty() + || request.certificate_pem.len() > MAX_CERT + || request.private_key_pem.is_empty() + || request.private_key_pem.len() > MAX_KEY + || request.expected_target_revision.len() > 128 + { + return Err(PublishError::Invalid); + } + // OS lock is shared by processes using this local volume. Captured CAS tokens are never refreshed on retry. + let lock = open_regular(&root.join("publisher.lock"), true, true)?; + lock.try_lock().map_err(|error| match error { + std::fs::TryLockError::WouldBlock => PublishError::Busy, + std::fs::TryLockError::Error(_) => PublishError::Storage, + })?; + let current = current_generation(root)?; + let state = state_from_generation(current.as_ref(), tls_path)?; + let digest = payload_digest(request.certificate_pem.as_bytes(), request.private_key_pem.as_bytes()); + if state.last_issuance_id == Some(request.issuance_id) { + let generation = current.as_ref().ok_or(PublishError::Invalid)?; + if generation.manifest.payload_digest != digest { + return Err(PublishError::Conflict); + } + // A prior response may have been lost after rename or directory sync. + File::open(root)?.sync_all()?; + mark_initialized(root)?; + return Ok(receipt(state, request.issuance_id)); + } + if request.expected_target_revision != state.target_revision { + return Err(PublishError::Conflict); + } + let accepted = tls_material::observed_server_snapshot(tls_path).ok_or(PublishError::Disabled)?; + let Some(ServerTlsMaterial::SingleCert { certs, .. }) = &accepted.server else { + return Err(PublishError::Invalid); + }; + let old_names = if let Some(generation) = current.as_ref() { + generation.leaf.names.clone() + } else { + if snapshot_leaf(&accepted).as_ref() != Some(&state.certificate_fingerprint) { + return Err(PublishError::Conflict); + } + parse_leaf(certs.first().ok_or(PublishError::Invalid)?.as_ref())?.names + }; + let new_chain = certificate_pem_items(request.certificate_pem.as_bytes()) + .collect::, _>>() + .map_err(|_| PublishError::Invalid)?; + let new_leaf = parse_leaf(new_chain.first().ok_or(PublishError::Invalid)?.as_ref())?; + if new_leaf.is_ca || !old_names.is_subset(&new_leaf.names) { + return Err(PublishError::Invalid); + } + let generations = root.join("generations"); + if !generations.exists() { + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(&generations)?; + File::open(root)?.sync_all()?; + } + if !fs::symlink_metadata(&generations)?.is_dir() { + return Err(PublishError::Invalid); + } + let revision = Uuid::new_v4(); + let directory = generations.join(revision.to_string()); + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(&directory)?; + let mut pointer_may_reference = false; + let result = (|| { + write_new(&directory.join(RUSTFS_TLS_CERT), request.certificate_pem.as_bytes())?; + write_new(&directory.join(RUSTFS_TLS_KEY), request.private_key_pem.as_bytes())?; + let source = TlsSource::from_directory(tls_path); + let candidate = TlsMaterialSnapshot::load_from_snapshot_with_server_directory( + &source.snapshot().map_err(|_| PublishError::Invalid)?, + Some(&directory), + ) + .map_err(|_| PublishError::Invalid)?; + if !matches!(&candidate.server, Some(ServerTlsMaterial::SingleCert { certs, .. }) if certs == &new_chain) { + return Err(PublishError::Invalid); + } + tls_material::validate_candidate_replacement(&accepted, &candidate).map_err(|_| PublishError::Invalid)?; + tls_material::validate_required_material_files( + &crate::startup_tls_material::required_startup_material_files(tls_path), + &candidate, + ) + .map_err(|_| PublishError::Invalid)?; + let prepared = tls_material::PreparedServerTls::from_snapshot(&candidate) + .map_err(|_| PublishError::Invalid)? + .ok_or(PublishError::Invalid)?; + prepared.tcp_acceptor().map_err(|_| PublishError::Invalid)?; + #[cfg(feature = "http3")] + prepared.http3_config().map_err(|_| PublishError::Invalid)?; + let ledger = Manifest { + schema_version: SCHEMA.into(), + issuance_id: request.issuance_id, + target_revision: revision, + certificate_fingerprint: new_leaf.fingerprint.clone(), + not_after_unix: new_leaf.not_after_unix, + payload_digest: digest, + }; + write_new( + &directory.join(MANIFEST), + &serde_json::to_vec(&ledger).map_err(|_| PublishError::Invalid)?, + )?; + File::open(&directory)?.sync_all()?; + File::open(&generations)?.sync_all()?; + let temporary = root.join(format!(".pointer-{revision}")); + #[cfg(unix)] + std::os::unix::fs::symlink(Path::new("generations").join(revision.to_string()), &temporary)?; + #[cfg(not(unix))] + let () = Err::<(), _>(PublishError::Disabled)?; + // A rename error can itself have an uncertain outcome on a failing filesystem. + pointer_may_reference = true; + fs::rename(&temporary, root.join(CURRENT))?; + #[cfg(test)] + { + if matches!(failure, FailurePoint::AfterPointerRenameUnreadable) { + fs::remove_file(root.join(CURRENT))?; + write_new(&root.join(CURRENT), b"unreadable pointer")?; + } + if !matches!(failure, FailurePoint::None) { + return Err(PublishError::Storage); + } + } + File::open(root)?.sync_all()?; + mark_initialized(root)?; + let installed = PublishedGeneration { + manifest: ledger, + chain: new_chain, + leaf: new_leaf, + key_pem: request.private_key_pem.into_bytes(), + }; + Ok(receipt(state_from_generation(Some(&installed), tls_path)?, request.issuance_id)) + })(); + // Observation errors must never convert an uncertain publication into deletion of serving material. + if result.is_err() && !pointer_may_reference { + let _ = fs::remove_dir_all(&directory); + } + result +} + +pub(crate) async fn publish(tls_path: String, request: PublishRequest) -> Result { + if !rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE) { + return Err(PublishError::Disabled); + } + if rustfs_utils::get_env_bool(rustfs_config::ENV_TRUST_LEAF_CERT_AS_CA, rustfs_config::DEFAULT_TRUST_LEAF_CERT_AS_CA) { + return Err(PublishError::Disabled); + } + let root = configured_root()?.ok_or(PublishError::Disabled)?; + let notification_path = tls_path.clone(); + let result = tokio::task::spawn_blocking(move || publish_at(&root, &tls_path, request)) + .await + .map_err(|_| PublishError::Storage)?; + tls_material::notify_server_publication(¬ification_path); + result +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + fn pair(subject: &str) -> rcgen::CertifiedKey { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + rcgen::generate_simple_self_signed(vec![subject.into()]).expect("certificate") + } + + fn build_request(expected: &str, certificate: &rcgen::CertifiedKey) -> PublishRequest { + PublishRequest { + schema_version: SCHEMA.into(), + resource_id: RESOURCE.into(), + expected_target_revision: expected.into(), + issuance_id: Uuid::new_v4(), + certificate_pem: certificate.cert.pem(), + private_key_pem: certificate.signing_key.serialize_pem(), + } + } + + fn clone_request(request: &PublishRequest) -> PublishRequest { + PublishRequest { + schema_version: request.schema_version.clone(), + resource_id: request.resource_id.clone(), + expected_target_revision: request.expected_target_revision.clone(), + issuance_id: request.issuance_id, + certificate_pem: request.certificate_pem.clone(), + private_key_pem: request.private_key_pem.clone(), + } + } + + fn measure_operation(operation: impl FnOnce() -> T) -> (T, OperationReads) { + OPERATION_READS.with(|counts| counts.set(OperationReads::default())); + let result = operation(); + (result, OPERATION_READS.with(std::cell::Cell::get)) + } + + #[tokio::test] + #[serial_test::serial] + async fn tls_publication_rejects_missing_startup_ca_source() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (rustfs_config::ENV_TRUST_LEAF_CERT_AS_CA, Some("0")), + (rustfs_config::ENV_TRUST_SYSTEM_CA, Some("0")), + (rustfs_config::ENV_MTLS_CLIENT_CERT, None), + (rustfs_config::ENV_MTLS_CLIENT_KEY, None), + (rustfs_config::ENV_RUSTFS_EXTRA_CA_CERT, None), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let previous = crate::startup_runtime_sources::current_outbound_tls_state().await; + let previous_files = crate::startup_tls_material::replace_startup_material_files(None); + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + fs::write(base.path().join(rustfs_config::RUSTFS_PUBLIC_CERT), pair("public-ca.example").cert.pem()) + .expect("public trust"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), pair("fallback-ca.example").cert.pem()) + .expect("fallback trust"); + let path = base.path().to_str().expect("path"); + let mut config = crate::config::Config::new("127.0.0.1:0", Vec::new()); + config.tls_path = Some(path.into()); + crate::startup_tls_material::init_outbound_tls_material(&config) + .await + .expect("real startup publication"); + fs::remove_file(base.path().join(rustfs_config::RUSTFS_CA_CERT)).expect("remove startup CA before listener"); + let initial = tls_material::load_tls_material(path).await.expect("HTTP initial snapshot"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload enabled"); + let state = state_at(root.path(), path).expect("valid bootstrap CAS state"); + let result = publish_at(root.path(), path, build_request(&state.target_revision, &pair("server.example"))); + let installed = current_directory(root.path()).expect("read durable pointer"); + task.abort(); + let _ = task.await; + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while tls_material::observed_server_snapshot(path).is_some() { + tokio::task::yield_now().await; + } + }) + .await + .expect("reload group exits before restoring startup state"); + crate::startup_runtime_sources::publish_outbound_tls_state( + rustfs_tls_runtime::TlsGeneration( + crate::startup_runtime_sources::current_outbound_tls_generation().saturating_add(1), + ), + &rustfs_tls_runtime::OutboundTlsMaterial { + root_ca_pem: previous.root_ca_pem.unwrap_or_default(), + mtls_identity: previous.mtls_identity, + }, + ) + .await; + crate::startup_tls_material::replace_startup_material_files(previous_files); + assert!( + matches!(result, Err(PublishError::Invalid)), + "source loss must reject an otherwise valid renewal" + ); + assert!(installed.is_none(), "rejected trust must not acquire a durable publication pointer"); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn publication_operation_reads_are_bounded() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + let path = base.path().to_str().expect("path"); + let initial = TlsMaterialSnapshot::load(&TlsSource::from_directory(base.path())) + .await + .expect("initial"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + let bootstrap = state_at(root.path(), path).expect("bootstrap"); + let request = build_request(&bootstrap.target_revision, &pair("server.example")); + let retry = clone_request(&request); + publish_at(root.path(), path, request).expect("first publication"); + let (state, get_counts) = measure_operation(|| state_at(root.path(), path).expect("state")); + let (_, retry_counts) = measure_operation(|| publish_at(root.path(), path, retry).expect("retry")); + let (_, reload_counts) = measure_operation(|| { + let generation = current_generation(root.path()).expect("generation").expect("published"); + generation + .into_snapshot(&TlsSource::from_directory(base.path()).snapshot().expect("source")) + .expect("runtime snapshot") + }); + let (_, publish_counts) = measure_operation(|| { + publish_at(root.path(), path, build_request(&state.target_revision, &pair("server.example"))) + .expect("next publication") + }); + println!("GET={get_counts:?}; RETRY={retry_counts:?}; PUBLISH={publish_counts:?}"); + println!("RELOAD={reload_counts:?}"); + assert_eq!( + reload_counts, + OperationReads { + certificates: 1, + keys: 1, + manifests: 1, + manifest_decodes: 1, + certificate_pem_traversals: 1, + leaf_der_decodes: 1 + } + ); + assert_eq!( + get_counts, + OperationReads { + certificates: 1, + keys: 1, + manifests: 1, + manifest_decodes: 1, + certificate_pem_traversals: 1, + leaf_der_decodes: 1 + } + ); + assert_eq!( + retry_counts, + OperationReads { + certificates: 1, + keys: 1, + manifests: 1, + manifest_decodes: 1, + certificate_pem_traversals: 1, + leaf_der_decodes: 1 + } + ); + assert_eq!( + publish_counts, + OperationReads { + certificates: 1, + keys: 1, + manifests: 1, + manifest_decodes: 1, + certificate_pem_traversals: 2, + leaf_der_decodes: 2 + } + ); + task.abort(); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn publication_preserves_trust_rejects_stale_and_recovers_lost_receipt() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("root permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("original certificate"); + fs::rename(base.path().join(RUSTFS_TLS_CERT), base.path().join("seed-certificate.pem")) + .expect("symlink seed backing"); + std::os::unix::fs::symlink("seed-certificate.pem", base.path().join(RUSTFS_TLS_CERT)) + .expect("Certbot-style seed symlink"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("original key"); + let old_ca = pair("ca-old.example"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), old_ca.cert.pem()).expect("trust file"); + let path = base.path().to_str().expect("base path"); + let initial = TlsMaterialSnapshot::load(&TlsSource::from_directory(base.path())) + .await + .expect("initial"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + let bootstrap = state_at(root.path(), path).expect("bootstrap state"); + let candidate = pair("server.example"); + let request = build_request(&bootstrap.target_revision, &candidate); + let retry = clone_request(&request); + let mutated = clone_request(&request); + let receipt = publish_at(root.path(), path, request).expect("durable receipt"); + assert_ne!(receipt.target_revision, bootstrap.target_revision); + assert_eq!(receipt.certificate_fingerprint, hex_digest(candidate.cert.der().as_ref())); + assert_eq!( + fs::read(base.path().join(RUSTFS_TLS_CERT)).expect("base certificate"), + original.cert.pem().as_bytes() + ); + let new_ca = pair("ca-new.example"); + fs::write(base.path().join(rustfs_config::RUSTFS_CA_CERT), new_ca.cert.pem()) + .expect("independent trust rotation"); + let directory = current_directory(root.path()).expect("pointer").expect("generation"); + let snapshot = TlsMaterialSnapshot::load_from_snapshot_with_server_directory( + &TlsSource::from_directory(base.path()).snapshot().expect("source"), + Some(&directory), + ) + .expect("overlay snapshot"); + assert_eq!(snapshot.outbound.root_ca_pem, new_ca.cert.pem().as_bytes()); + assert_eq!(snapshot_leaf(&snapshot), Some(receipt.certificate_fingerprint.clone())); + let retry_receipt = publish_at(root.path(), path, retry).expect("lost receipt recovery"); + assert_eq!(retry_receipt.target_revision, receipt.target_revision); + let mut mutated = mutated; + mutated.private_key_pem = pair("server.example").signing_key.serialize_pem(); + assert!(matches!(publish_at(root.path(), path, mutated), Err(PublishError::Conflict))); + assert!(matches!( + publish_at(root.path(), path, build_request(&bootstrap.target_revision, &pair("server.example"))), + Err(PublishError::Conflict) + )); + assert_eq!( + state_at(root.path(), path).expect("accepted state").target_revision, + receipt.target_revision + ); + let expanded = rcgen::generate_simple_self_signed(vec!["server.example".into(), "extra.example".into()]) + .expect("expanded names"); + let expanded_receipt = publish_at(root.path(), path, build_request(&receipt.target_revision, &expanded)) + .expect("pending expanded generation"); + assert!( + matches!( + publish_at(root.path(), path, build_request(&expanded_receipt.target_revision, &candidate)), + Err(PublishError::Invalid) + ), + "a later publication must preserve names of the durable generation even before it is observed" + ); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let one = build_request(&expanded_receipt.target_revision, &expanded); + let two = build_request(&expanded_receipt.target_revision, &expanded); + let root_path = root.path(); + let (one, two) = std::thread::scope(|scope| { + let barrier_one = barrier.clone(); + let first = scope.spawn(move || { + barrier_one.wait(); + publish_at(root_path, path, one) + }); + let second = scope.spawn(|| { + barrier.wait(); + publish_at(root_path, path, two) + }); + (first.join().expect("first publisher"), second.join().expect("second publisher")) + }); + assert_eq!(usize::from(one.is_ok()) + usize::from(two.is_ok()), 1, "one concurrent CAS winner"); + assert!( + matches!(one, Err(PublishError::Conflict | PublishError::Busy)) + || matches!(two, Err(PublishError::Conflict | PublishError::Busy)) + ); + let latest = state_at(root.path(), path).expect("concurrent winner"); + let interrupted = build_request(&latest.target_revision, &expanded); + let retry = clone_request(&interrupted); + assert!( + matches!( + publish_at_inner(root.path(), path, interrupted, FailurePoint::AfterPointerRename), + Err(PublishError::Storage) + ), + "failure after rename has an unknown durable outcome" + ); + let recovered = + publish_at(root.path(), path, retry).expect("same issuance recovers and syncs the installed generation"); + assert_ne!(recovered.target_revision, latest.target_revision); + assert!( + current_directory(root.path()) + .expect("installed generation survives error") + .is_some() + ); + let interrupted = build_request(&recovered.target_revision, &expanded); + let before = fs::read_dir(root.path().join("generations")).expect("generations").count(); + assert!(matches!( + publish_at_inner(root.path(), path, interrupted, FailurePoint::AfterPointerRenameUnreadable), + Err(PublishError::Storage) + )); + let after = fs::read_dir(root.path().join("generations")) + .expect("generations after unreadable pointer") + .count(); + assert_eq!(after, before + 1, "an uncertain pointer must not delete the new generation"); + fs::remove_file(root.path().join(CURRENT)).expect("simulate accepted pointer loss"); + assert!( + matches!(current_directory(root.path()), Err(PublishError::Invalid)), + "an acknowledged publisher must never silently revert to bootstrap after pointer loss" + ); + task.abort(); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn expired_generation_can_renew_and_next_operation_detects_corruption() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + let path = base.path().to_str().expect("path"); + let initial = TlsMaterialSnapshot::load(&TlsSource::from_directory(base.path())) + .await + .expect("initial"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + let expired_key = rcgen::KeyPair::generate().expect("expired key"); + let mut params = rcgen::CertificateParams::new(vec!["server.example".into(), "expired-required.example".into()]) + .expect("parameters"); + params.not_before = rcgen::date_time_ymd(2000, 1, 1); + params.not_after = rcgen::date_time_ymd(2001, 1, 1); + let expired = params.self_signed(&expired_key).expect("expired certificate"); + let certificate = expired.pem(); + let key = expired_key.serialize_pem(); + let revision = Uuid::new_v4(); + let generations = root.path().join("generations"); + fs::create_dir(&generations).expect("generations"); + let directory = generations.join(revision.to_string()); + fs::create_dir(&directory).expect("generation"); + write_new(&directory.join(RUSTFS_TLS_CERT), certificate.as_bytes()).expect("expired cert"); + write_new(&directory.join(RUSTFS_TLS_KEY), key.as_bytes()).expect("expired key"); + let leaf = parse_leaf(expired.der().as_ref()).expect("expired metadata"); + let manifest = Manifest { + schema_version: SCHEMA.into(), + issuance_id: Uuid::new_v4(), + target_revision: revision, + certificate_fingerprint: leaf.fingerprint, + not_after_unix: leaf.not_after_unix, + payload_digest: payload_digest(certificate.as_bytes(), key.as_bytes()), + }; + write_new(&directory.join(MANIFEST), &serde_json::to_vec(&manifest).expect("manifest")).expect("manifest file"); + std::os::unix::fs::symlink(Path::new("generations").join(revision.to_string()), root.path().join(CURRENT)) + .expect("pointer"); + mark_initialized(root.path()).expect("initialized"); + let state = state_at(root.path(), path).expect("expired visible head remains readable"); + assert!(state.not_after_unix < 1_000_000_000); + assert!( + current_generation(root.path()) + .expect("expired generation") + .expect("present") + .into_snapshot(&TlsSource::from_directory(base.path()).snapshot().expect("source")) + .is_err(), + "reading an expired ledger must never bypass serving validity checks" + ); + assert!( + matches!( + publish_at(root.path(), path, build_request(&state.target_revision, &pair("server.example"))), + Err(PublishError::Invalid) + ), + "expired durable SAN coverage must survive renewal" + ); + let renewed = + rcgen::generate_simple_self_signed(vec!["server.example".into(), "expired-required.example".into()]) + .expect("renewal"); + let request = build_request(&state.target_revision, &renewed); + let retry = clone_request(&request); + let receipt = publish_at(root.path(), path, request).expect("renew expired generation"); + assert_ne!(receipt.target_revision, state.target_revision); + let generation = current_generation(root.path()).expect("published").expect("generation"); + assert_eq!( + state_at(root.path(), path).expect("first independent read").target_revision, + receipt.target_revision + ); + let directory = root + .path() + .join("generations") + .join(generation.manifest.target_revision.to_string()); + fs::write(directory.join(RUSTFS_TLS_KEY), "corrupted key").expect("corrupt after read"); + let snapshot = generation + .into_snapshot(&TlsSource::from_directory(base.path()).snapshot().expect("source")) + .expect("the pinned operation consumes its already verified material without a second file read"); + assert_eq!(snapshot_leaf(&snapshot), Some(receipt.certificate_fingerprint.clone())); + assert!( + matches!(state_at(root.path(), path), Err(PublishError::Invalid)), + "the next operation must revalidate disk" + ); + assert!( + matches!(publish_at(root.path(), path, retry), Err(PublishError::Invalid)), + "idempotent retry must not bypass disk corruption" + ); + task.abort(); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn invalid_candidate_never_changes_pointer() { + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (ENV_TLS_PUBLISH_DIR, None), + ], + async { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("root permissions"); + let original = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + let path = base.path().to_str().expect("base path"); + let initial = TlsMaterialSnapshot::load(&TlsSource::from_directory(base.path())) + .await + .expect("initial"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let task = tls_material::spawn_reload_loop( + path.into(), + holder, + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + let expected = state_at(root.path(), path).expect("state").target_revision; + assert!(matches!( + publish_at(root.path(), path, build_request(&expected, &pair("wrong.example"))), + Err(PublishError::Invalid) + )); + let valid = pair("server.example"); + let mut mismatch = build_request(&expected, &valid); + mismatch.private_key_pem = pair("server.example").signing_key.serialize_pem(); + assert!(matches!(publish_at(root.path(), path, mismatch), Err(PublishError::Invalid))); + let mut ca_params = rcgen::CertificateParams::new(vec!["server.example".into()]).expect("CA params"); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + let ca_key = rcgen::KeyPair::generate().expect("CA key"); + let ca_cert = ca_params.self_signed(&ca_key).expect("CA certificate"); + let mut ca_candidate = build_request(&expected, &valid); + ca_candidate.certificate_pem = ca_cert.pem(); + ca_candidate.private_key_pem = ca_key.serialize_pem(); + assert!( + matches!(publish_at(root.path(), path, ca_candidate), Err(PublishError::Invalid)), + "a CA certificate must not become the serving leaf" + ); + let mut oversized = build_request(&expected, &valid); + oversized.certificate_pem = "x".repeat(MAX_CERT + 1); + assert!(matches!(publish_at(root.path(), path, oversized), Err(PublishError::Invalid))); + assert!(current_directory(root.path()).expect("pointer absent").is_none()); + assert_eq!(state_at(root.path(), path).expect("unchanged").target_revision, expected); + task.abort(); + }, + ) + .await; + } + + #[test] + fn corrupt_or_escaping_pointer_is_rejected() { + let root = tempfile::TempDir::new().expect("root"); + std::os::unix::fs::symlink("../outside", root.path().join(CURRENT)).expect("pointer"); + assert!(matches!(current_directory(root.path()), Err(PublishError::Invalid))); + fs::remove_file(root.path().join(CURRENT)).expect("remove pointer"); + fs::write(root.path().join(CURRENT), "generations/foo").expect("regular pointer"); + assert!(matches!(current_directory(root.path()), Err(PublishError::Invalid))); + } + + #[tokio::test] + #[serial_test::serial] + async fn publication_is_observed_by_live_tls_handshakes() { + let base = tempfile::TempDir::new().expect("base"); + let root = tempfile::TempDir::new().expect("publication root"); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)).expect("root permissions"); + let original = pair("server.example"); + let renewed = pair("server.example"); + fs::write(base.path().join(RUSTFS_TLS_CERT), original.cert.pem()).expect("certificate"); + fs::write(base.path().join(RUSTFS_TLS_KEY), original.signing_key.serialize_pem()).expect("key"); + temp_env::async_with_vars( + [ + (ENV_TLS_RELOAD_ENABLE, Some("1")), + (ENV_TLS_PUBLISH_DIR, root.path().to_str()), + (rustfs_config::ENV_SERVER_MTLS_ENABLE, Some("0")), + (rustfs_config::ENV_TRUST_SYSTEM_CA, Some("0")), + (rustfs_config::ENV_TRUST_LEAF_CERT_AS_CA, Some("0")), + (rustfs_config::ENV_RUSTFS_TLS_PATH, Some("ignored-env-source")), + ], + async { + let path = base.path().to_str().expect("path"); + let initial = tls_material::load_tls_material(path).await.expect("initial snapshot"); + let prepared = tls_material::PreparedServerTls::from_snapshot(&initial) + .expect("prepared TLS") + .expect("server"); + let holder = prepared.tcp_acceptor().expect("acceptor"); + let registration = tls_material::spawn_reload_loop( + path.into(), + holder.clone(), + initial, + prepared, + #[cfg(feature = "http3")] + None, + ) + .expect("registration") + .expect("reload task"); + assert_eq!( + tls_material::active_server_tls_path(), + Some(path.to_string()), + "registered CLI/listener source wins over environment" + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("listener"); + let address = listener.local_addr().expect("address"); + let server = tokio::spawn(async move { + for _ in 0..2 { + let (socket, _) = listener.accept().await.expect("TCP connection"); + let _tls = holder.get().accept(socket).await.expect("TLS handshake"); + } + }); + let mut roots = rustls::RootCertStore::empty(); + roots.add(original.cert.der().clone()).expect("bootstrap anchor"); + roots.add(renewed.cert.der().clone()).expect("renewal anchor"); + let config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + let connector = tokio_rustls::TlsConnector::from(std::sync::Arc::new(config)); + async fn handshake(connector: &tokio_rustls::TlsConnector, address: std::net::SocketAddr) -> String { + let socket = tokio::net::TcpStream::connect(address).await.expect("connect"); + let stream = connector + .connect(rustls::pki_types::ServerName::try_from("server.example").expect("SNI"), socket) + .await + .expect("verified handshake"); + hex_digest(stream.get_ref().1.peer_certificates().expect("peer chain")[0].as_ref()) + } + assert_eq!(handshake(&connector, address).await, hex_digest(original.cert.der().as_ref())); + let bootstrap = certificate_state(path.into()).await.expect("bootstrap state"); + let receipt = publish(path.into(), build_request(&bootstrap.target_revision, &renewed)) + .await + .expect("publish"); + let expected = hex_digest(renewed.cert.der().as_ref()); + assert_eq!(receipt.certificate_fingerprint, expected); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + if certificate_state(path.into()) + .await + .expect("state") + .observed_certificate_fingerprint + == Some(expected.clone()) + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("publication observed by serving group"); + assert_eq!(handshake(&connector, address).await, expected); + let mut expected_chain = vec![renewed.cert.der().clone()]; + assert_eq!(observed(path, &expected_chain), Some(expected.clone())); + expected_chain.push(original.cert.der().clone()); + assert!( + observed(path, &expected_chain).is_none(), + "matching leaf alone must not acknowledge a different serving chain" + ); + server.await.expect("TLS server"); + registration.abort(); + }, + ) + .await; + } +} diff --git a/rustfs/src/startup_runtime_sources.rs b/rustfs/src/startup_runtime_sources.rs index 30b35458f..73501a592 100644 --- a/rustfs/src/startup_runtime_sources.rs +++ b/rustfs/src/startup_runtime_sources.rs @@ -17,6 +17,7 @@ use crate::runtime_sources::current_outbound_tls_generation as runtime_current_o use crate::storage_api::startup::runtime_sources::{InstanceContext, set_global_rustfs_port}; use rustfs_kms::KmsServiceManager; use rustfs_obs::{GlobalError as ObservabilityError, OtelGuard}; +pub(crate) use rustfs_tls_runtime::load_global_outbound_tls_state as current_outbound_tls_state; use rustfs_tls_runtime::{OutboundTlsMaterial, TlsGeneration}; use std::sync::Arc; use tokio_util::sync::CancellationToken; diff --git a/rustfs/src/startup_tls_material.rs b/rustfs/src/startup_tls_material.rs index 9d770293e..32931c707 100644 --- a/rustfs/src/startup_tls_material.rs +++ b/rustfs/src/startup_tls_material.rs @@ -14,7 +14,9 @@ use crate::config::Config; use crate::startup_runtime_sources; +use std::collections::BTreeSet; use std::io::{Error, Result}; +use std::sync::Mutex; use tracing::{error, info}; const LOG_COMPONENT_MAIN: &str = "main"; @@ -23,6 +25,25 @@ const EVENT_TLS_OUTBOUND_INITIALIZED: &str = "tls_outbound_initialized"; const EVENT_TLS_OUTBOUND_INITIALIZATION_FAILED: &str = "tls_outbound_initialization_failed"; const TLS_STARTUP_GENERATION_CONSUMER: &str = "rustfs_server_startup"; +// Startup completes before listeners register; retain source names independently +// of the later listener snapshot, without pinning renewable certificate bytes. +static STARTUP_MATERIAL_FILES: Mutex)>> = Mutex::new(None); + +pub(crate) fn required_startup_material_files(tls_path: &str) -> BTreeSet { + STARTUP_MATERIAL_FILES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .as_ref() + .filter(|(path, _)| path == tls_path.trim()) + .map(|(_, files)| files.clone()) + .unwrap_or_default() +} + +#[cfg(test)] +pub(crate) fn replace_startup_material_files(files: Option<(String, BTreeSet)>) -> Option<(String, BTreeSet)> { + std::mem::replace(&mut *STARTUP_MATERIAL_FILES.lock().unwrap_or_else(|error| error.into_inner()), files) +} + pub(crate) async fn init_outbound_tls_material(config: &Config) -> Result<()> { crate::server::tls_material::validate_configured_oidc_extra_ca_cert() .await @@ -32,6 +53,8 @@ pub(crate) async fn init_outbound_tls_material(config: &Config) -> Result<()> { match crate::server::tls_material::load_tls_material(tls_path).await { Ok(snapshot) => { publish_outbound_tls_material(&snapshot.outbound, Some(tls_path)).await; + *STARTUP_MATERIAL_FILES.lock().unwrap_or_else(|error| error.into_inner()) = + Some((tls_path.to_string(), snapshot.material_files)); } Err(err) => { error!( @@ -49,6 +72,11 @@ pub(crate) async fn init_outbound_tls_material(config: &Config) -> Result<()> { if startup_runtime_sources::observability_metric_enabled() { startup_runtime_sources::init_tls_metrics(); } + } else { + STARTUP_MATERIAL_FILES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take(); } Ok(()) diff --git a/scripts/test_helm_templates.sh b/scripts/test_helm_templates.sh index 1b21e68cb..f273430b9 100755 --- a/scripts/test_helm_templates.sh +++ b/scripts/test_helm_templates.sh @@ -568,7 +568,7 @@ nondefault_statefulset=$(awk ' nondefault_local_host=$(statefulset_env_value RUSTFS_LOCAL_ENDPOINT_HOST <<<"$nondefault_statefulset") nondefault_namespace=$(yq eval '.metadata.namespace' - <<<"$nondefault_statefulset") nondefault_endpoint_port=$(yq eval '.spec.template.spec.containers[0].ports[] | select(.name == "endpoint") | .containerPort' - <<<"$nondefault_statefulset") -mtls_liveness_command=$(yq eval '.spec.template.spec.containers[0].livenessProbe.exec.command[-1]' - <<<"$nondefault_statefulset") +mtls_liveness_port=$(yq eval '.spec.template.spec.containers[0].livenessProbe.tcpSocket.port' - <<<"$nondefault_statefulset") mtls_readiness_command=$(yq eval '.spec.template.spec.containers[0].readinessProbe.exec.command[-1]' - <<<"$nondefault_statefulset") if [[ "$nondefault_local_host" != '$(RUSTFS_CHART_POD_NAME).tenant-rustfs-headless.object-storage.svc.cluster.corp' ]]; then echo "Combined generated topology produced an unexpected local endpoint host: $nondefault_local_host" >&2 @@ -578,9 +578,8 @@ if [[ "$nondefault_namespace" != "object-storage" || "$nondefault_endpoint_port" echo "Combined generated topology must retain its namespace and custom endpoint port" >&2 exit 1 fi -if ! grep -Fq 'https://127.0.0.1:9443/health' <<<"$mtls_liveness_command" || - grep -Fq '/health/ready' <<<"$mtls_liveness_command"; then - echo "mTLS liveness must use the process health endpoint, not readiness" >&2 +if [[ "$mtls_liveness_port" != 9443 ]]; then + echo "mTLS liveness must check the listener independently of certificate validity" >&2 exit 1 fi if ! grep -Fq 'https://127.0.0.1:9443/health/ready' <<<"$mtls_readiness_command"; then diff --git a/scripts/test_helm_tls_reload.sh b/scripts/test_helm_tls_reload.sh new file mode 100644 index 000000000..e8ad68d86 --- /dev/null +++ b/scripts/test_helm_tls_reload.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +CHART_DIR="$ROOT_DIR/helm/rustfs" + +for distributed in false true; do + rendered=$(helm template tls-reload "$CHART_DIR" \ + --set secret.allowInsecureDefaults=true \ + --set mode.distributed.enabled="$distributed" \ + --set mode.standalone.enabled="$([[ "$distributed" == false ]] && echo true || echo false)" \ + --set tls.enabled=true --set tls.existingSecret=issued-tls) + workload=$(yq eval 'select(.kind == "Deployment" or .kind == "StatefulSet")' - <<<"$rendered") + mount=$(yq eval '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "tls-material")' - <<<"$workload") + [[ $(yq eval '.mountPath' - <<<"$mount") == /opt/tls ]] + [[ $(yq eval '.readOnly' - <<<"$mount") == true ]] + [[ $(yq eval 'has("subPath")' - <<<"$mount") == false ]] + keys=$(yq eval '.spec.template.spec.volumes[] | select(.name == "tls-material") | .projected.sources[].secret.items[].key' - <<<"$workload") + [[ "$keys" == $'tls.crt\ntls.key' ]] + [[ $(yq eval '.spec.template.spec.containers[0].readinessProbe.httpGet.scheme' - <<<"$workload") == HTTPS ]] + [[ $(yq eval '.spec.template.spec.containers[0].livenessProbe.tcpSocket.port' - <<<"$workload") == 9000 ]] + config=$(yq eval 'select(.kind == "ConfigMap") | .data' - <<<"$rendered") + [[ $(yq eval '.RUSTFS_TLS_RELOAD_ENABLE' - <<<"$config") == 1 ]] + [[ $(yq eval '.RUSTFS_TLS_KEYLOG' - <<<"$config") == 0 ]] + if [[ "$distributed" == true ]]; then + [[ $(yq eval '.RUSTFS_VOLUMES' - <<<"$config") == https://* ]] + fi + + mtls=$(helm template tls-reload "$CHART_DIR" --set secret.allowInsecureDefaults=true \ + --set mode.distributed.enabled="$distributed" \ + --set mode.standalone.enabled="$([[ "$distributed" == false ]] && echo true || echo false)" \ + --set mtls.enabled=true) + workload=$(yq eval 'select(.kind == "Deployment" or .kind == "StatefulSet")' - <<<"$mtls") + [[ $(yq eval '.spec.template.spec.volumes[] | select(.name == "tls-material") | .projected.sources | length' - <<<"$workload") == 2 ]] + [[ $(yq eval '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "tls-material") | has("subPath")' - <<<"$workload") == false ]] + [[ $(yq eval '.spec.template.spec.containers[0].livenessProbe.tcpSocket.port' - <<<"$workload") == 9000 ]] +done + +if helm template tls-reload "$CHART_DIR" --set secret.allowInsecureDefaults=true \ + --set tls.enabled=true --set tls.existingSecret=issued-tls --set mtls.enabled=true >/dev/null 2>&1; then + echo "Public TLS must not silently enable global client authentication" >&2 + exit 1 +fi +if helm template tls-reload "$CHART_DIR" --set secret.allowInsecureDefaults=true \ + --set tls.enabled=true >/dev/null 2>&1; then + echo "Native TLS requires an explicitly configured serving Secret" >&2 + exit 1 +fi +echo "TLS Secret projection, rotation settings, topology and probes passed" From 7ad4fa439eebf1ca617797bd7ae45cda1c5ecad8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:09:19 -0700 Subject: [PATCH 017/143] build(deps): bump the dependencies group across 1 directory with 3 updates (#8481) Bumps the dependencies group with 3 updates in the / directory: [datafusion](https://github.com/apache/datafusion), rust-embed and [hotpath](https://github.com/pawurb/hotpath-rs). Updates `datafusion` from 55.1.0 to 55.2.0 - [Changelog](https://github.com/apache/datafusion/blob/main/CHANGELOG.md) - [Commits](https://github.com/apache/datafusion/compare/55.1.0...55.2.0) Updates `rust-embed` from 8.12.0 to 8.13.0 Updates `hotpath` from 0.28.5 to 0.28.6 - [Release notes](https://github.com/pawurb/hotpath-rs/releases) - [Changelog](https://github.com/pawurb/hotpath-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/pawurb/hotpath-rs/compare/v0.28.5...v0.28.6) --- updated-dependencies: - dependency-name: datafusion dependency-version: 55.2.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: dependencies - dependency-name: hotpath dependency-version: 0.28.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: rust-embed dependency-version: 8.13.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: dependencies ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 178 +++++++++++++++++++++++++++-------------------------- 1 file changed, 90 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 91b1a5149..dc20abde7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2839,9 +2839,9 @@ checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" [[package]] name = "datafusion" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14313430439ef858ed5f52ca9c840e9eea9844da34d3716d05a476449634da5d" +checksum = "4cd5317db77ba6bca25c761a8091b9d294a9f4188e4f9dcd0243e6a57de862ac" dependencies = [ "arrow", "arrow-schema", @@ -2888,9 +2888,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c0756ebc770bcebae366dfd273dad50935f6103b4d212253cb17e2e53c5427f" +checksum = "f35d36794509b7bfcbf90e11a1b5e4d047123c34610105e2e80a2048351cdb2f" dependencies = [ "arrow", "async-trait", @@ -2913,9 +2913,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbde2961698705d8402f0f382f08876fc6f1126ffde790ef17833cf0ab87cd59" +checksum = "a6454bfb7f43b83942e2ca2576fd09342eedbd0d323e1eef72c3c8e4d57d7203" dependencies = [ "arrow", "async-trait", @@ -2937,9 +2937,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e06fe711ff815c8cf3143158547aea599702fd43f76f5866c70d5a5b3693a0" +checksum = "d26626e12b224b83351239465dcd00f72d47b02fb5e5b65a643e2dc8ed37f687" dependencies = [ "arrow", "arrow-ipc", @@ -2964,9 +2964,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7a5107acff4e3ed50a7732acec06bba40e74ab49df021397ed250d60f04e42a" +checksum = "a106420cbfa980f242c22243ae9a6cce661d4d86ce96cff87dacabf7754a4470" dependencies = [ "futures", "log", @@ -2975,9 +2975,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f69f45d2d700785be8df9d4b44410e1c8e4a856f10cc728b418a1066c0b594fb" +checksum = "210aa1e9525e9454f2399dbb8b0b95510145bd8c1fdc35de74e9ccbf81bcb699" dependencies = [ "arrow", "async-trait", @@ -3005,9 +3005,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81650af64894ef34b46d2ef818ef244b40a81d90818f3bc642bf8fbf6758f8e0" +checksum = "65b1b041688cad707fa103c3e7f4c6d89b3134ebf67da0129070dad3381851eb" dependencies = [ "arrow", "arrow-ipc", @@ -3029,9 +3029,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e2519bed59a6907c79298b9c21fe807bbc34e4ff8414577d699b0741083593" +checksum = "2a7b34e253c4fa5251f174977af910a2f2ec831ec39dbbb56b29ce505cff278b" dependencies = [ "arrow", "async-trait", @@ -3052,9 +3052,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635e61290e171597f6e50427e0cb486eda468ab47e880fa24d15bbb0fa24ac7e" +checksum = "1663338f43be6a5746473008e2d0fbde282577ddf0bae90d6c7386e988d71b29" dependencies = [ "arrow", "async-trait", @@ -3075,9 +3075,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cd518f85b3a027d913047b4ab0811111a2b734fdf84a51b78234455f10d8fa" +checksum = "7c521ae91b6d899d7190812881f45fe9463c222c11d90149f5332a2e56555904" dependencies = [ "arrow", "arrow-schema", @@ -3107,15 +3107,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d44c5919b560862dd1cb20a5c75b3e48c3d9e49719c91d78da5c38eaad5495c4" +checksum = "0c2569c2e74a34fcae3a369ab3912d61e41a062592510f11e31b1dae2725ecdc" [[package]] name = "datafusion-execution" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7620ad535af8730186bdec05d720e40d9a4584b40c7eb75f72c43aa7978e5484" +checksum = "39cfdd5ac0769d1e74a885450e49fde8f63ca7ab41eb479a068564aaea80b837" dependencies = [ "arrow", "arrow-buffer", @@ -3139,9 +3139,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea229af8b73e566fc6ebb1c8dd42ab250f8855c5304bd173f4b50ead78b64304" +checksum = "86560fd7df4dda924f5474b588c56e7ae8e96dcf158c7327c9cfe9daff448ec5" dependencies = [ "arrow", "arrow-schema", @@ -3156,15 +3156,16 @@ dependencies = [ "indexmap 2.14.2", "itertools 0.15.0", "recursive", + "serde", "serde_json", "sqlparser", ] [[package]] name = "datafusion-expr-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a18baa840f77760dcc9eaad281329bc57536c5c39e65e8649596f2a51216be7" +checksum = "e82e313cf655cf6bc2dce6cbd23bcb0d20236a54a1b09c436615e4bac4d8c49f" dependencies = [ "arrow", "datafusion-common", @@ -3174,9 +3175,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763bbfeec40b55a8a961d7349c0fcfcec4f3f66cc234d6295250b95371b85e34" +checksum = "939ae2771da8a3fdc23ab4de610ae8fef9515476626b49d96d0d5c782b105771" dependencies = [ "arrow", "arrow-buffer", @@ -3202,9 +3203,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c0c2b2e856883ecf6a4fa8b23e7c6041183af8a68d0583dc95d1d21e4eaee0" +checksum = "fd6c3aee9f2bc65716787d651e023743a90b4efbc57a2f679f036632e16fb110" dependencies = [ "arrow", "datafusion-common", @@ -3223,9 +3224,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e35e42c87dfbff06934ea0e99cfed3440ac614c3f3bfb8c33be35ca0587d4140" +checksum = "988b5d3dfb4939d81c2f9fab267063610fe52a843620f2fa6cb2a1598e43369c" dependencies = [ "arrow", "datafusion-common", @@ -3235,9 +3236,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08079b0604cdcc43c7d6cf006cfd5d3ed9c235ade0c1ebe91085a62f875603df" +checksum = "08149d433a97c9ea3cdf80de06480ab5c7f5453485f1e452d5b3583c81d5c0dd" dependencies = [ "arrow", "arrow-ord", @@ -3260,9 +3261,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe2070ecf5638a93c81c8babc7858339ef52165a9f54da1411afd69adf7d85ce" +checksum = "2c767567cce2782d4d0bdfa55fab010aaba0bdbe533509e9e0b9a6a3ee29ed59" dependencies = [ "arrow", "async-trait", @@ -3276,9 +3277,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e9a164ebe1998c5c3f9e15c9a0c4e6b599a0c8739b5471bfb089752441963f1" +checksum = "0a311c1f558d1f792f1f631c3670da12ee72a20fbf5d6ca151325d66191c03c1" dependencies = [ "arrow", "datafusion-common", @@ -3293,9 +3294,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98965b183f92201c16798994820ae032649c4821e089c4c80e7d0f4485d41df5" +checksum = "fffd54b8acf6ecef284cc0677e01e1b8c71be64fec50612ba00a0ff0d9b41a52" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -3303,9 +3304,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a711bd5a06ec85d9683e142d28a98ad0679a7220669f5c1b8589e57e9b29e46" +checksum = "14d6df9763c8fb968b2417d797aef4c55ac2e4a895aaef38c72c16a287aa4a40" dependencies = [ "datafusion-doc", "quote", @@ -3314,9 +3315,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff3256b62ff44616ec83c7f04741d21124970833bb377ed56d9abc5ec6f707ec" +checksum = "acfb8cea6e05e2e75bae183340cf7d53cccb3db0449e2f1fbc07654d980fd19f" dependencies = [ "arrow", "chrono", @@ -3334,9 +3335,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e37a25b5aa1f3a1db114f0ee3605bf01639b83a831dcde35119ee61f0b11d6" +checksum = "9f002cef849e10c0597a217ec7891c2796c0f8b42d739f5d24abc24496373406" dependencies = [ "arrow", "arrow-schema", @@ -3357,9 +3358,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a4d12787d42e88b26b0d3ec006192fa23ceb832ef2a9e87439ce820a8e038cc" +checksum = "82761522c4536e7059d3b185cd99f10525c2bf48f1bd26a0a57e73e19eadc0fb" dependencies = [ "arrow", "datafusion-common", @@ -3372,9 +3373,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d13bf7f50fd902f5afc46d50759f5185b5620c40b24054f80f7c648897d3e5a6" +checksum = "f8c5c67aa252886f4d6b331c8114797f5b140464e774c4da36889269a1bf1bd8" dependencies = [ "arrow", "chrono", @@ -3389,9 +3390,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11fbe7e6665072957d44ed9587676e17a1f9ab3acfadfc287dac23750585f18a" +checksum = "f1a49dd3a76e4879012682c59dc3735282e60640c97483546a4113564a60bf8f" dependencies = [ "arrow", "datafusion-common", @@ -3409,9 +3410,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1265d58e5bce07d154e642a51ff43033b576a6ae40d50a29ac2b9f311004eb52" +checksum = "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97" dependencies = [ "arrow", "arrow-data", @@ -3438,15 +3439,16 @@ dependencies = [ "num-traits", "parking_lot", "pin-project-lite", + "serde", "serde_json", "tokio", ] [[package]] name = "datafusion-pruning" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74b06b333405c05015836ed36cea02cc29d1e2ee8fb414a77532d8346fb56492" +checksum = "f11f0dbe2b5a642f6106e44de8bbd6654fa7220ca14f32313d4efd5514a86968" dependencies = [ "arrow", "datafusion-common", @@ -3460,9 +3462,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7616b17b1d2fee3e9b13c651eac91b768d153265e6ba88e06e0b4af7883f69c" +checksum = "62dc5ce37a5ce1ff5362949776829aded2529a11dbaa13d24b0ffa6a059e106a" dependencies = [ "arrow-schema", "async-trait", @@ -3475,9 +3477,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b4848be66dd008d55adbe1dbc8cbd6a2502588ad0c0b8483bab0f43fd4495d0" +checksum = "816c224a0ef5469a32a562b5d4830544f714f97c5f50142109286d40a288c7db" dependencies = [ "arrow", "bigdecimal", @@ -3947,7 +3949,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4297,7 +4299,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5297,9 +5299,9 @@ dependencies = [ [[package]] name = "hotpath" -version = "0.28.5" +version = "0.28.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34aabdeba3c81cef2075a2961a370ebfe1d9cc3f8d2242fd400ce6ef6f3bbfcb" +checksum = "e2ee4f8d17ebe2400fa845f079acf9ede235be6472512876779af45f7890f934" dependencies = [ "arc-swap", "async-channel", @@ -5332,15 +5334,15 @@ dependencies = [ [[package]] name = "hotpath-drain" -version = "0.28.5" +version = "0.28.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b61a4a77e6563bdc37f831cd6aa5e2089429b62ea377dd759bbabafc207d7a" +checksum = "fd9e65572a429f44a48c360559a3f59028579b53e1df07c5f31dc147d2b94a50" [[package]] name = "hotpath-macros" -version = "0.28.5" +version = "0.28.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c6275c8d1931d280bd505debb7ef91dbeae2c23477aabbfe10487b7430bdeb3" +checksum = "2b90aafbaf5f6af01ee9db2888644b15995f5306e5c280aec8261feb3e5e214a" dependencies = [ "proc-macro2", "quote", @@ -5702,7 +5704,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" dependencies = [ "io-lifetimes 3.0.1", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -6961,7 +6963,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8343,7 +8345,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.14.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -8363,7 +8365,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck 0.5.0", - "itertools 0.14.0", + "itertools 0.10.5", "log", "multimap", "petgraph 0.8.3", @@ -8384,7 +8386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.119", @@ -8397,7 +8399,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.119", @@ -8724,7 +8726,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9430,9 +9432,9 @@ dependencies = [ [[package]] name = "rust-embed" -version = "8.12.0" +version = "8.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e7760e252aaba7b09f4be00e36476cf585bdb68a53552ac954cdf504ab4bc9" +checksum = "19afa5b4b6a611de00bd1bdae6ae6f39084c9399f0679c3f52d8469cf335cc23" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -9441,9 +9443,9 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "8.12.0" +version = "8.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bcfc4d6f53af43755f7a723e4b6b8794fcce052a178dd8c6c1dadc5f5343097" +checksum = "e0d8afda6374eac59e066abee06d265247ebbaf3006cf878e2879e8356e34053" dependencies = [ "mime_guess", "proc-macro2", @@ -9456,9 +9458,9 @@ dependencies = [ [[package]] name = "rust-embed-utils" -version = "8.12.0" +version = "8.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ffa149f6aa81b58a5b3011d01a857c4ed12c7a732d2c51947a4c7c692185f0" +checksum = "0d84e8ba78bd384263e5922f084cbe1b081c3b7e69add59c8fb097b879ba968a" dependencies = [ "sha2 0.11.0", "walkdir", @@ -11129,7 +11131,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11212,7 +11214,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -12483,10 +12485,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -13580,7 +13582,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] From 902c97ce055b1faddb8cd689f0d3a04a0a3e6f35 Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 11 Oct 2026 20:09:55 -0700 Subject: [PATCH 018/143] fix: auto-repair breaking changes from acd26afd8 (#8483) --- rustfs/src/admin/service/oidc_config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rustfs/src/admin/service/oidc_config.rs b/rustfs/src/admin/service/oidc_config.rs index df230603f..a3eea052e 100644 --- a/rustfs/src/admin/service/oidc_config.rs +++ b/rustfs/src/admin/service/oidc_config.rs @@ -125,7 +125,7 @@ pub(crate) async fn validate_config(input: OidcProviderValidationInput) -> Resul let provider = build_validation_provider_config(input)?; let extra_root_ca = crate::startup_auth::current_oidc_extra_root_ca_material() .await - .map_err(|error| OidcAdminConfigError::Validation(error.to_string()))?; + .map_err(OidcAdminConfigError::Validation)?; validate_oidc_provider_config_with_extra_root_ca(&provider, extra_root_ca.root_ca_pem.as_deref()) .await .map_err(OidcAdminConfigError::Validation) From 201b5f62d2c187a8447dbb3faa33bb6e9c179cc0 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 12:55:23 +0800 Subject: [PATCH 019/143] fix(s3): normalize PUT read quorum failures as unavailable (#8484) --- rustfs/src/app/object/put.rs | 121 ++++++++++++++++++++++++++++++- rustfs/src/error.rs | 101 ++++++++++++++++++++++++++ rustfs/src/storage/access.rs | 135 ++++++++++++++++++++++++++--------- 3 files changed, 322 insertions(+), 35 deletions(-) diff --git a/rustfs/src/app/object/put.rs b/rustfs/src/app/object/put.rs index d9a1c3477..9bff39cf2 100644 --- a/rustfs/src/app/object/put.rs +++ b/rustfs/src/app/object/put.rs @@ -18,9 +18,30 @@ use super::*; use crate::app::trailer_adapter::trailer_source; use crate::auth::{RUSTFS_MAX_CONTENT_LENGTH_QUERY, VerifiedPresignedRequest, parse_presigned_put_max_content_length}; -use crate::error::UploadLimitExceeded; +use crate::error::{UploadLimitExceeded, normalize_put_read_quorum_error}; static PUT_FAILURE_LOGS: rustfs_utils::LogThrottle = rustfs_utils::LogThrottle::new(5_000); +#[cfg(test)] +#[derive(Clone, Copy, PartialEq)] +enum PutReadQuorumTestStage { + DefaultSse, + Prelookup, + Commit, +} + +#[cfg(test)] +tokio::task_local! { + static PUT_READ_QUORUM_TEST_STAGE: PutReadQuorumTestStage; +} + +#[cfg(test)] +fn injected_put_read_quorum_failure(stage: PutReadQuorumTestStage) -> Option { + PUT_READ_QUORUM_TEST_STAGE + .try_with(|configured| *configured == stage) + .unwrap_or(false) + .then(|| StorageError::InsufficientReadQuorum("bucket".into(), "object".into())) +} + const DEFAULT_PUT_LARGE_CONCURRENCY_TUNING_MIN_SIZE_BYTES: i64 = 32 * 1024 * 1024; const ENV_ZERO_COPY_EAGER_PUT_MAX_SIZE_BYTES: &str = "RUSTFS_ZERO_COPY_EAGER_PUT_MAX_SIZE_BYTES"; @@ -1098,6 +1119,11 @@ impl PutObjectCompletion { /// Record the failed completion and hand the error back to the caller. fn fail_put_object(completion: PutObjectCompletion, err: S3Error) -> S3Error { + let err = if matches!(&completion, PutObjectCompletion::S3(_)) { + normalize_put_read_quorum_error(err) + } else { + err + }; let result: S3Result> = Err(err); let _ = completion.complete(&result); match result { @@ -1185,7 +1211,9 @@ impl DefaultObjectUsecase { #[instrument(name = "execute_put_object", level = "info", skip(self, _fs, req))] #[hotpath::measure(impl_type = "DefaultObjectUsecase", future = true)] pub async fn execute_put_object(&self, _fs: &FS, req: S3Request) -> S3Result> { - self.execute_put_object_boxed(_fs, req).await + self.execute_put_object_boxed(_fs, req) + .await + .map_err(normalize_put_read_quorum_error) } fn execute_put_object_boxed<'a>( @@ -1549,6 +1577,9 @@ impl DefaultObjectUsecase { let sse_config_stage_start = put_stage_metrics_enabled.then(Instant::now); let bucket_sse_config = load_bucket_default_sse_config(&bucket).await; + #[cfg(test)] + let bucket_sse_config = injected_put_read_quorum_failure(PutReadQuorumTestStage::DefaultSse) + .map_or(bucket_sse_config, |err| Err(ApiError::from(err).into())); rustfs_io_metrics::record_put_object_stage_duration_from("app_sse_config_lookup", sse_config_stage_start); let bucket_sse_config = bucket_sse_config?; debug!( @@ -1699,6 +1730,9 @@ impl DefaultObjectUsecase { // the dst xl.meta that rename_data already reads, saving a full-disk // metadata fanout per PUT. let prelookup_required = version_id.is_some() || object_lock_checks_required_for_state(&object_lock_config_state); + #[cfg(test)] + let prelookup_required = + prelookup_required || injected_put_read_quorum_failure(PutReadQuorumTestStage::Prelookup).is_some(); // Outer None = prelookup skipped (accounting comes from the commit // backfill); Some(inner) = the previous current size as observed by the // lookup, with the pre-#1009 semantics kept bit-for-bit. @@ -1713,6 +1747,9 @@ impl DefaultObjectUsecase { crate::hp_guard!("S3::put_object_prelookup"); store.get_object_info(&bucket, &key, ¤t_opts).await }; + #[cfg(test)] + let previous_current_info = + injected_put_read_quorum_failure(PutReadQuorumTestStage::Prelookup).map_or(previous_current_info, Err); Some(match previous_current_info { Ok(existing_obj_info) => { validate_existing_object_lock_for_write(&object_lock_config_state, &existing_obj_info, &opts)?; @@ -1987,6 +2024,8 @@ impl DefaultObjectUsecase { } else { None }; + #[cfg(test)] + let injected_commit_error = injected_put_read_quorum_failure(PutReadQuorumTestStage::Commit); let put_commit = spawn_traced_join({ let store = Arc::clone(&store); let bucket = bucket.clone(); @@ -2007,6 +2046,11 @@ impl DefaultObjectUsecase { .and_then(ObjectTrafficHealth::track_write_storage); let mut reader = reader; let store_put_stage_start = put_stage_metrics_enabled.then(Instant::now); + #[cfg(test)] + if let Some(err) = injected_commit_error { + store_put_watchdog.cancel(); + return Err(ApiError::from(err).into()); + } let (obj_info, backfilled_old_current_size) = match store .put_object_with_old_current_size(&bucket, &key, &mut reader, &opts) .await @@ -2203,6 +2247,79 @@ mod tests { use super::*; use futures::StreamExt; use http::{HeaderMap, HeaderName, HeaderValue, Method}; + + #[tokio::test] + #[serial_test::serial] + async fn put_read_quorum_failures_are_normalized_at_each_stage() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let (store, context) = real_store_test_context().await; + let bucket = format!("put-quorum-{}", Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("read-quorum PUT bucket must be created"); + let usecase = DefaultObjectUsecase::with_context(Some(context)); + + for (index, stage) in [ + PutReadQuorumTestStage::DefaultSse, + PutReadQuorumTestStage::Prelookup, + PutReadQuorumTestStage::Commit, + ] + .into_iter() + .enumerate() + { + for method in [Method::PUT, Method::POST] { + let key = format!("refused-{index}-{method}"); + let input = PutObjectInput::builder() + .bucket(bucket.clone()) + .key(key.clone()) + .body(Some(StreamingBlob::from_bytes(Bytes::from_static(b"quorum probe")))) + .content_length(Some(12)) + .build() + .expect("quorum probe input must build"); + let mut req = build_request(input, method.clone()); + if method == Method::POST { + req.extensions.insert(PostObjectRequestMarker); + } + let err = PUT_READ_QUORUM_TEST_STAGE + .scope(stage, usecase.execute_put_object(&FS::new(), req)) + .await + .expect_err("read quorum failure must refuse the upload"); + assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable, "stage {index}, method {method}"); + assert_eq!(err.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE)); + assert_eq!(err.message(), Some("The service is unavailable. Please retry.")); + assert!(matches!( + err.source().and_then(|source| source.downcast_ref::()), + Some(StorageError::InsufficientReadQuorum(_, _)) + )); + let response = err.to_http_response().expect("PUT error must serialize"); + assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE); + let xml = http_body_util::BodyExt::collect(response.into_body()) + .await + .expect("PUT error XML must be readable") + .to_bytes(); + let xml = std::str::from_utf8(&xml).expect("PUT error XML must be UTF-8"); + assert!(xml.contains("ServiceUnavailable"), "{xml}"); + assert!(!xml.contains("SlowDownRead"), "{xml}"); + let lookup = store.get_object_info(&bucket, &key, &ObjectOptions::default()).await; + assert!( + lookup.as_ref().is_err_and(is_err_object_not_found), + "failed PUT must leave no object: {lookup:?}" + ); + } + } + } + + #[test] + fn failed_put_completion_normalizes_only_s3_errors() { + let error = || S3Error::from(ApiError::from(StorageError::ErasureReadQuorum)); + let err = fail_put_object(PutObjectCompletion::S3(OperationHelper::Disabled), error()); + assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable); + assert_eq!(err.message(), Some("The service is unavailable. Please retry.")); + let internal = fail_put_object(PutObjectCompletion::Internal(None), error()); + assert_eq!(internal.code(), &S3ErrorCode::Custom("SlowDownRead".into())); + } use s3s::dto::{ DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRule, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index e8cc2ae0b..e1542b685 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -173,6 +173,40 @@ pub(crate) fn slow_down_read_api_error(err: StorageError) -> ApiError { } } +/// Normalize a read-quorum failure that escaped through a PutObject boundary. +/// +/// Storage-level read quorum failures keep their `SlowDownRead` code for GET +/// compatibility. A write request that encounters the same failure must expose +/// the write-side retry contract instead, while retaining the typed source for +/// diagnostics and audit consumers. +pub(crate) fn normalize_put_read_quorum_error(mut err: S3Error) -> S3Error { + let is_slow_down_read = matches!( + err.code(), + S3ErrorCode::Custom(code) if &**code == SLOW_DOWN_READ_CODE + ); + if !is_slow_down_read { + return err; + } + let has_read_quorum_source = err.source().is_some_and(|source| { + error_chain_find(source, |error| { + matches!( + error.downcast_ref::(), + Some(StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _)) + ) + .then_some(()) + }) + .is_some() + }); + + if has_read_quorum_source { + err.set_code(S3ErrorCode::ServiceUnavailable); + err.set_message(ApiError::error_code_to_message(&S3ErrorCode::ServiceUnavailable)); + err.set_status_code(StatusCode::SERVICE_UNAVAILABLE); + } + + err +} + /// Marks a request body that exceeded a presigned upload size capability. /// /// This marker must survive the body-reader and storage layers so the client @@ -1602,6 +1636,73 @@ mod tests { assert_eq!(S3Error::from(api_error).status_code(), Some(StatusCode::SERVICE_UNAVAILABLE)); } + #[test] + fn put_read_quorum_error_preserves_context_and_is_idempotent() { + for source in [ + StorageError::ErasureReadQuorum, + StorageError::InsufficientReadQuorum("bucket".into(), "object".into()), + ] { + for nested in [false, true] { + let mut err: S3Error = ApiError::from(source.clone()).into(); + if nested { + err.set_source(Box::new(IoError::other(source.clone()))); + } + err.set_request_id("put-quorum-request"); + let mut headers = http::HeaderMap::new(); + headers.insert("retry-after", http::HeaderValue::from_static("2")); + err.set_headers(headers); + + let err = normalize_put_read_quorum_error(normalize_put_read_quorum_error(err)); + assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable); + assert_eq!(err.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE)); + assert_eq!(err.message(), Some("The service is unavailable. Please retry.")); + assert_eq!(err.request_id(), Some("put-quorum-request")); + assert_eq!(err.headers().expect("retry headers must survive")["retry-after"], "2"); + let retained = error_chain_find(err.source().expect("typed source must survive"), |error| { + error.downcast_ref::().cloned() + }); + assert_eq!(retained, Some(source.clone())); + } + } + } + + #[test] + fn put_read_quorum_error_preserves_other_error_contracts() { + for code in [ + S3ErrorCode::SlowDown, + S3ErrorCode::AccessDenied, + S3ErrorCode::InvalidRequest, + S3ErrorCode::PreconditionFailed, + S3ErrorCode::BadDigest, + S3ErrorCode::IncompleteBody, + S3ErrorCode::NoSuchKey, + S3ErrorCode::NoSuchBucket, + S3ErrorCode::InternalError, + S3ErrorCode::ServiceUnavailable, + ] { + let mut err = S3Error::with_message(code.clone(), "original message"); + err.set_source(Box::new(StorageError::ErasureReadQuorum)); + let original_status = err.status_code(); + let err = normalize_put_read_quorum_error(err); + assert_eq!(err.code(), &code); + assert_eq!(err.message(), Some("original message")); + assert_eq!(err.status_code(), original_status); + } + for source in [ + None, + Some(StorageError::ErasureWriteQuorum), + Some(StorageError::PartMissingOrCorrupt), + ] { + let mut err = S3Error::with_message(S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into()), "original message"); + if let Some(source) = source { + err.set_source(Box::new(source)); + } + let err = normalize_put_read_quorum_error(err); + assert_eq!(err.code(), &S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into())); + assert_eq!(err.message(), Some("original message")); + } + } + #[test] fn test_unknown_authoritative_quota_usage_maps_to_retryable_error() { let api_error = ApiError::from(QuotaError::UsageUnavailable { diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index 316eee5fc..b601c9333 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -22,7 +22,7 @@ use crate::auth::{ get_session_token, parse_presigned_multipart_max_total_object_size, parse_presigned_put_max_content_length, reject_unsigned_amz_headers_on_sigv4_request, }; -use crate::error::ApiError; +use crate::error::{ApiError, normalize_put_read_quorum_error}; use crate::license::license_check; use crate::shared_types::RemoteAddr; use crate::storage::request_context::RequestContext; @@ -424,6 +424,11 @@ type ObjectGenerationTestHook = (String, tokio::sync::oneshot::Sender<()>, tokio #[cfg(test)] static OBJECT_GENERATION_TEST_HOOK: OnceLock>> = OnceLock::new(); +#[cfg(test)] +tokio::task_local! { + static PUT_GENERATION_READ_QUORUM_FAILURE: (); +} + #[cfg(test)] fn install_object_generation_test_hook( bucket: String, @@ -3296,40 +3301,50 @@ impl S3Access for FS { /// This method returns `Ok(())` by default. async fn put_object(&self, req: &mut S3Request) -> S3Result<()> { crate::hp_guard!("S3Access::put_object"); - let req_info = ext_req_info_mut(&mut req.extensions)?; - req_info.bucket = Some(req.input.bucket.clone()); - req_info.object = Some(req.input.key.clone()); - req_info.version_id = req.input.version_id.clone(); + async { + let req_info = ext_req_info_mut(&mut req.extensions)?; + req_info.bucket = Some(req.input.bucket.clone()); + req_info.object = Some(req.input.key.clone()); + req_info.version_id = req.input.version_id.clone(); - if has_write_offset_bytes_header(&req.headers) { - return Err(S3Error::with_message( - S3ErrorCode::NotImplemented, - ApiError::error_code_to_message(&S3ErrorCode::NotImplemented), - )); + if has_write_offset_bytes_header(&req.headers) { + return Err(S3Error::with_message( + S3ErrorCode::NotImplemented, + ApiError::error_code_to_message(&S3ErrorCode::NotImplemented), + )); + } + + let bucket = req.input.bucket.clone(); + // Snapshot before authorization, but preserve AccessDenied precedence + // by exposing any bucket-state error only after authorization succeeds. + let bucket_generation = load_bucket_generation(self, req, &bucket).await; + #[cfg(test)] + let bucket_generation = if PUT_GENERATION_READ_QUORUM_FAILURE.try_with(|_| ()).is_ok() { + Err(ApiError::from(StorageError::ErasureReadQuorum).into()) + } else { + bucket_generation + }; + let requested_retain_until = req.input.object_lock_retain_until_date.clone(); + set_requested_object_lock_retain_until(req, requested_retain_until.as_ref()); + authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await?; + req.extensions.insert(bucket_generation?); + + if req.method != http::Method::POST { + authorize_replication_only_put_headers(req).await?; + } + + if legal_hold_write_requested(req.input.object_lock_legal_hold_status.as_ref()) { + authorize_request(req, Action::S3Action(S3Action::PutObjectLegalHoldAction)).await?; + } + + if retention_write_requested(req.input.object_lock_mode.as_ref(), req.input.object_lock_retain_until_date.as_ref()) { + authorize_request(req, Action::S3Action(S3Action::PutObjectRetentionAction)).await?; + } + + Ok(()) } - - let bucket = req.input.bucket.clone(); - // Snapshot before authorization, but preserve AccessDenied precedence - // by exposing any bucket-state error only after authorization succeeds. - let bucket_generation = load_bucket_generation(self, req, &bucket).await; - let requested_retain_until = req.input.object_lock_retain_until_date.clone(); - set_requested_object_lock_retain_until(req, requested_retain_until.as_ref()); - authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await?; - req.extensions.insert(bucket_generation?); - - if req.method != http::Method::POST { - authorize_replication_only_put_headers(req).await?; - } - - if legal_hold_write_requested(req.input.object_lock_legal_hold_status.as_ref()) { - authorize_request(req, Action::S3Action(S3Action::PutObjectLegalHoldAction)).await?; - } - - if retention_write_requested(req.input.object_lock_mode.as_ref(), req.input.object_lock_retain_until_date.as_ref()) { - authorize_request(req, Action::S3Action(S3Action::PutObjectRetentionAction)).await?; - } - - Ok(()) + .await + .map_err(normalize_put_read_quorum_error) } /// Checks whether the PutObjectAcl request has accesses to the resources. @@ -5183,6 +5198,60 @@ mod tests { }); } + #[tokio::test] + #[serial] + async fn put_read_quorum_access_keeps_authorization_precedence() { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let server_ctx = ServerContextSlot::new(); + assert!(server_ctx.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms))))); + let fs = FS::with_server_ctx(server_ctx); + + for effect in ["Allow", "Deny"] { + let bucket = format!("put-quorum-access-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("authorization precedence bucket must be created"); + let policy_json = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"{effect}","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"# + ); + let mut metadata = (*crate::storage::get_bucket_metadata(&bucket) + .await + .expect("test bucket metadata must be cached")) + .clone(); + metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy must parse")); + metadata.policy_config_json = policy_json.into_bytes(); + crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata) + .await + .expect("test policy must be installed"); + + let input = PutObjectInput::builder() + .bucket(bucket) + .key("object".into()) + .build() + .expect("access probe input must build"); + let mut req = build_request(input, Method::PUT); + ensure_req_info(&mut req); + req.extensions.insert(fs.server_ctx().clone()); + let err = super::PUT_GENERATION_READ_QUORUM_FAILURE + .scope((), fs.put_object(&mut req)) + .await + .expect_err("a generation failure must never admit a write"); + if effect == "Allow" { + assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable); + assert_eq!(err.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE)); + assert!(matches!( + err.source().and_then(|source| source.downcast_ref::()), + Some(StorageError::ErasureReadQuorum) + )); + } else { + assert_eq!(err.code(), &S3ErrorCode::AccessDenied); + assert_eq!(err.status_code(), Some(http::StatusCode::FORBIDDEN)); + } + assert!(req.extensions.get::().is_none()); + } + } + #[tokio::test] #[serial] async fn put_object_access_captures_authorized_bucket_incarnation() { From 4166b9503f3cc84bed869a3828163952cd359ede Mon Sep 17 00:00:00 2001 From: Hauser Date: Mon, 12 Oct 2026 13:47:37 +0800 Subject: [PATCH 020/143] fix(ecstore): recover interrupted delete transactions (#8468) * fix(ecstore): recover crash-interrupted delete retries Retain set-quorum cleanup proof and restore metadata before dropping rollback markers. Reclaim eligible exact-key residue during explicit delete retries, with bucket identity, version, retention and namespace fencing guards. Preserve prefix children and snapshot ownership through batched physical cleanup, and cover interrupted transactions and failure boundaries. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): route recovery test imports through contracts Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): preserve missing-key deletes at quorum * feat(ecstore): add certified set-wide delete recovery Atomically prepare deletion journals and exact targets, publish durable set receipts, and recover through one fenced disk/RPC core used by startup and scoped administrator healing. Preserve returning generations, ordinary missing-key quorum semantics, and legacy rollback wire compatibility. Retain namespace and volume ownership through late physical operations, order related transactions causally, and keep proof until full convergence. Remove the superseded local-only recovery path and add storage, protocol, RPC, durability and isolated Docker regression coverage. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): reclaim completed multipart delete sidecars * fix(ecstore): recover multipart sidecars with verified proofs Recognize native multipart sidecars at the shared recovery boundary and restore them without treating metadata or proof files as shard payloads. Validate required proof generations and bind every present live/staged shard copy to its metadata-bound Merkle proof before restoring metadata. Cover protected native MPU rollback, split source layouts, missing or corrupt proofs, and payload corruption with recomputed bitrot checksums. Add real multipart Docker regressions in both integrity modes. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): settle superseded committed version deletes Allow a validated committed decision to finish its own data cleanup after a later decision removes current metadata. Prepared copies need an inspected same-transaction receipt matching immutable local intent before promotion, and retain their local journal without importing peer hashes. Reuse the authority matcher for dry-run and settlement. Cover chained explicit-version deletions, partial receipt publication, mismatch rejection and no-resurrection, while retaining the no-decision absence protection. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(ecstore): clarify delete recovery case label Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): restore recovery CI guard coverage Keep production error sites visible to the ratchet by placing test modules last. Preserve native encoding errors, group preparation identity and ownership, and reduce RPC enum size without changing the wire payload. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(admin): preserve owned OIDC validation errors Co-Authored-By: heihutu Co-Authored-By: zhi22915 * refactor(ecstore): unify recovery authority and snapshot work Use one user-bucket legacy settlement boundary and one adoption source predicate for preview and execution. Reuse immutable snapshot digests, consume owned generation metadata, index version membership, and prepare empty witnesses through the canonical journal path. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): preserve delete recovery invariants --------- Co-authored-by: heihutu Co-authored-by: zhi22915 Co-authored-by: overtrue --- .../e2e_test/src/reliant/grpc_lock_server.rs | 7 + crates/ecstore/src/api/mod.rs | 11 +- crates/ecstore/src/cluster/rpc/client.rs | 102 + crates/ecstore/src/cluster/rpc/remote_disk.rs | 395 +- crates/ecstore/src/disk/disk_store.rs | 203 +- crates/ecstore/src/disk/local.rs | 1619 +++++++- .../ecstore/src/disk/local/delete_recovery.rs | 3646 +++++++++++++++++ crates/ecstore/src/disk/mod.rs | 653 +++ crates/ecstore/src/object_api/types.rs | 5 + crates/ecstore/src/set_disk/mod.rs | 1 + .../src/set_disk/ops/delete_recovery.rs | 1992 +++++++++ crates/ecstore/src/set_disk/ops/heal.rs | 183 +- crates/ecstore/src/set_disk/ops/mod.rs | 1 + crates/ecstore/src/set_disk/ops/object.rs | 1047 ++++- crates/ecstore/src/store/heal.rs | 128 + crates/ecstore/src/store/object.rs | 282 +- crates/heal/src/heal/storage.rs | 49 + crates/heal/src/heal/task/heal_bucket.rs | 36 + crates/heal/src/heal/task/heal_erasure_set.rs | 15 + crates/heal/src/heal/task/heal_object.rs | 48 +- .../src/generated/proto_gen/node_service.rs | 70 + crates/protos/src/lib.rs | 136 +- crates/protos/src/node.proto | 16 + docs/README.md | 3 + .../operations/delete-transaction-recovery.md | 137 + rustfs/src/server/http.rs | 45 + rustfs/src/storage/rpc/node_service.rs | 208 +- rustfs/src/storage/rpc/node_service/disk.rs | 179 +- rustfs/src/storage/storage_api.rs | 101 +- rustfs/src/storage_api.rs | 2 + scripts/test/delete_recovery_docker.py | 1257 ++++++ 31 files changed, 12239 insertions(+), 338 deletions(-) create mode 100644 crates/ecstore/src/disk/local/delete_recovery.rs create mode 100644 crates/ecstore/src/set_disk/ops/delete_recovery.rs create mode 100644 docs/operations/delete-transaction-recovery.md create mode 100644 scripts/test/delete_recovery_docker.py diff --git a/crates/e2e_test/src/reliant/grpc_lock_server.rs b/crates/e2e_test/src/reliant/grpc_lock_server.rs index ccc466d23..b97e96555 100644 --- a/crates/e2e_test/src/reliant/grpc_lock_server.rs +++ b/crates/e2e_test/src/reliant/grpc_lock_server.rs @@ -465,6 +465,13 @@ impl NodeService for MinimalLockNodeService { Err(Status::unimplemented("lock-only test server")) } + async fn recover_delete_transaction( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("delete recovery is not supported by the lock fixture")) + } + async fn settle_part_transaction( &self, _request: Request, diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index e5e22995f..a43116a77 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -423,11 +423,12 @@ pub mod disk { pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage}; pub use crate::disk::{ BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, - CheckPartsResp, ConditionalFileUpdate, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, - DiskStore, FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, - PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, - STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, - validate_batch_read_version_item_count, + CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, + DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, + DeleteRecoveryRpcReply, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, + FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, PartTransactionAction, + RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, + SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count, }; pub use bytes::Bytes; pub use endpoint::Endpoint; diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index 27bd0542f..75706254c 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -124,6 +124,22 @@ pub async fn node_service_time_out_client_for_class( .max_encoding_message_size(max_message_size)) } +pub(crate) async fn node_service_delete_recovery_client( + addr: &str, + interceptor: TonicInterceptor, + timeout: std::time::Duration, +) -> Result>, DiskError> { + let interceptor = interceptor.with_rpc_audience(addr).map_err(DiskError::from)?; + let channel = rustfs_protos::create_channel_with_timeout(addr, timeout) + .await + .map_err(|error| DiskError::RemoteClientUnavailable(error.to_string()))?; + let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience()); + let max_message_size = rustfs_protos::internode_rpc_max_message_size(); + Ok(NodeServiceClient::with_interceptor(channel, interceptor) + .max_decoding_message_size(max_message_size) + .max_encoding_message_size(max_message_size)) +} + pub async fn node_service_time_out_client_no_auth( addr: &String, ) -> Result>, Box> { @@ -528,6 +544,92 @@ mod tests { use tracing_opentelemetry::OpenTelemetrySpanExt; use tracing_subscriber::{Registry, layer::SubscriberExt}; + #[tokio::test] + async fn delete_recovery_channel_uses_its_scoped_deadline() { + use bytes::Bytes; + use http_body_util::StreamBody; + use hyper::{body::Frame, service::service_fn}; + use hyper_util::rt::{TokioExecutor, TokioIo}; + use rustfs_protos::proto_gen::node_service::{RecoverDeleteTransactionRequest, RecoverDeleteTransactionResponse}; + use tonic_prost::prost::Message; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind recovery peer"); + let addr = format!("http://{}", listener.local_addr().expect("peer address")); + let server = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.expect("accept recovery connection"); + tokio::spawn(async move { + let service = service_fn(|_request: HttpRequest| async { + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + let payload = RecoverDeleteTransactionResponse { + success: true, + protocol_version: 1, + result: Bytes::new(), + error: None, + } + .encode_to_vec(); + let mut data = vec![0]; + data.extend_from_slice(&u32::try_from(payload.len()).expect("small response").to_be_bytes()); + data.extend_from_slice(&payload); + let mut trailers = http::HeaderMap::new(); + trailers.insert("grpc-status", "0".parse().expect("status")); + let frames: Vec, std::convert::Infallible>> = + vec![Ok(Frame::data(data.into())), Ok(Frame::trailers(trailers))]; + let mut response = HttpResponse::new(StreamBody::new(futures::stream::iter(frames))); + response + .headers_mut() + .insert("content-type", "application/grpc".parse().expect("content type")); + Ok::<_, std::convert::Infallible>(response) + }); + let _ = hyper::server::conn::http2::Builder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(stream), service) + .await; + }); + } + }); + let request = || RecoverDeleteTransactionRequest { + disk: "disk".into(), + volume: "bucket".into(), + protocol_version: 1, + operation: Bytes::new(), + }; + // Seed an existing short-lived control channel to reproduce the inherited deadline. + let control = tonic::transport::Endpoint::from_shared(addr.clone()) + .expect("test endpoint") + .timeout(std::time::Duration::from_millis(50)) + .connect() + .await + .expect("control connect"); + rustfs_common::cache_connection(addr.clone(), control).await; + let mut foreground = node_service_time_out_client(&addr, TonicInterceptor::NoOp(NoOpInterceptor)) + .await + .expect("control client"); + let error = foreground + .recover_delete_transaction(request()) + .await + .expect_err("short control deadline"); + assert!(matches!(error.code(), tonic::Code::Cancelled | tonic::Code::DeadlineExceeded)); + // The 2-second test override bypasses the production range parser, which rejects 2. + let mut recovery = node_service_delete_recovery_client( + &addr, + TonicInterceptor::NoOp(NoOpInterceptor), + std::time::Duration::from_secs(2), + ) + .await + .expect("scoped recovery client"); + assert!( + recovery + .recover_delete_transaction(request()) + .await + .expect("scoped deadline must survive control deadline") + .into_inner() + .success + ); + server.abort(); + rustfs_protos::evict_failed_connection(&addr).await; + } + #[tokio::test] async fn rpc_client_factories_do_not_reuse_https_channels_without_tls_generation() { let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve local address"); diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index 0ba276567..e067609a0 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -14,7 +14,8 @@ use crate::cluster::rpc::client::{ AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, - node_service_time_out_client, node_service_time_out_client_for_class, node_service_time_out_client_no_auth, + node_service_delete_recovery_client, node_service_time_out_client, node_service_time_out_client_for_class, + node_service_time_out_client_no_auth, }; use crate::cluster::rpc::internode_data_transport::{ InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest, @@ -22,10 +23,11 @@ use crate::cluster::rpc::internode_data_transport::{ }; use crate::disk::error::{Error, Result}; use crate::disk::{ - BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DeleteOptions, DiskAPI, DiskInfo, - DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, - ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, - batch_read_version_one_by_one, + BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION, + DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection, + DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, + DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions, + RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one, disk_store::{ DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE, get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout, @@ -59,10 +61,10 @@ use rustfs_protos::proto_gen::node_service::{ CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, - ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest, - SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, - SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, - node_service_client::NodeServiceClient, + ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest, + RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, + SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, + WriteMetadataRequest, node_service_client::NodeServiceClient, }; use serde::{Serialize, de::DeserializeOwned}; use std::{ @@ -142,6 +144,45 @@ fn decode_delete_versions_errors(response: DeleteVersionsResponse, expected_len: .collect() } +fn decode_delete_recovery_batch_errors(codes: Vec>, expected_len: usize) -> Result>> { + if codes.len() != expected_len { + return Err(Error::other("malformed recoverable batch delete response length")); + } + codes + .into_iter() + .map(|code| match code { + None => Ok(None), + Some(code) => DiskError::from_u32(code) + .map(Some) + .ok_or_else(|| Error::other("invalid recoverable batch delete error code")), + }) + .collect() +} + +fn validate_delete_recovery_inspection( + volume: &str, + object: &str, + transaction: Uuid, + inspection: &DeleteRecoveryInspection, +) -> Result<()> { + if inspection.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || inspection.id.object != object + || inspection.id.transaction != transaction + || transaction.is_nil() + { + return Err(Error::other("delete recovery inspection does not match request")); + } + if let Some(journal) = &inspection.journal + && (journal.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || journal.volume != volume + || journal.object != object + || journal.transaction != transaction) + { + return Err(Error::other("delete recovery journal does not match request")); + } + Ok(()) +} + fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result { if !response.success { return Err(response.error.unwrap_or_default().into()); @@ -858,6 +899,75 @@ fn spawn_control_channel_prewarm(addr: String) { } impl RemoteDisk { + async fn recover_delete_transaction( + &self, + volume: &str, + operation: DeleteRecoveryRpcOperation, + ) -> Result { + let payload = rmp_serde::to_vec_named(&operation)?; + if payload.len() > operation.max_wire_payload_bytes() { + return Err(Error::other("delete recovery operation exceeds payload limit")); + } + let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) { + Some(crate::disk::get_delete_recovery_timeout()?) + } else { + None + }; + self.execute_with_timeout( + || async { + let mut client = if let Some(timeout) = settle_timeout { + if let Some(error) = self.offline_bypass_error() { + return Err(error); + } + node_service_delete_recovery_client( + &self.addr, + TonicInterceptor::Signature(gen_tonic_signature_interceptor()), + timeout, + ) + .await? + } else { + self.get_client().await? + }; + let mut request = Request::new(RecoverDeleteTransactionRequest { + disk: self.endpoint.to_string(), + volume: volume.to_owned(), + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + operation: payload.into(), + }); + let canonical_body = rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()) + .map_err(|_| Error::other("recover_delete_transaction request length cannot be represented"))?; + crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body).map_err(Error::other)?; + // UNIMPLEMENTED is deliberately propagated: older peers cannot settle transactions. + let response = client + .recover_delete_transaction(request) + .await + .map_err(|status| { + if status.code() == Code::Unimplemented { + DiskError::UnsupportedDisk + } else { + status.into() + } + })? + .into_inner(); + if response.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION { + return Err(DiskError::UnsupportedDisk); + } + if !response.success { + return Err(response + .error + .ok_or_else(|| Error::other("missing delete recovery error"))? + .into()); + } + if response.error.is_some() { + return Err(Error::other("malformed delete recovery success")); + } + rustfs_protos::decode_delete_recovery_payload(&response.result).map_err(Into::into) + }, + settle_timeout.unwrap_or_else(get_max_timeout_duration), + ) + .await + } + async fn rename_file_with_durability( &self, src_volume: &str, @@ -1920,6 +2030,12 @@ fn encode_msgpack(value: &T) -> Result> { encode_msgpack_with_capacity(value, MSGPACK_ENCODE_CAPACITY_HINT) } +// Optional fields are omitted by serde; named maps keep rollback flags and +// transaction UUIDs independent of field order across peer versions. +fn encode_delete_options_msgpack(value: &DeleteOptions) -> Result> { + Ok(rmp_serde::to_vec_named(value)?) +} + fn encode_file_info_msgpack(value: &FileInfo) -> Result> { encode_msgpack_with_capacity(value, FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT) } @@ -2566,7 +2682,7 @@ impl DiskAPI for RemoteDisk { // `_bin` support for DeleteVersion is new (grpc-optimization P2); always dual-write // JSON + msgpack until its fallback counter has read zero across a release window. let file_info_bin = encode_file_info_msgpack(&fi)?; - let opts_bin = encode_msgpack(&opts)?; + let opts_bin = encode_delete_options_msgpack(&opts)?; let conditional_marker = opts.expected_delete_marker.is_some(); let file_info = serde_json::to_string(&fi)?; let opts = serde_json::to_string(&opts)?; @@ -2638,7 +2754,7 @@ impl DiskAPI for RemoteDisk { // `_bin` support for DeleteVersions is new (grpc-optimization P2); always dual-write JSON + // msgpack until its fallback counter has read zero across a release window. - let opts_bin = match encode_msgpack(&opts) { + let opts_bin = match encode_delete_options_msgpack(&opts) { Ok(opts_bin) => opts_bin, Err(err) => { let mut errors = Vec::with_capacity(versions.len()); @@ -3617,6 +3733,138 @@ impl DiskAPI for RemoteDisk { } #[tracing::instrument(level = "trace", skip_all)] + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> Result<()> { + match self + .recover_delete_transaction( + volume, + DeleteRecoveryRpcOperation::ApplyVersion { + object: object.to_owned(), + file_info: Box::new(file_info), + force_del_marker, + opts, + }, + ) + .await? + { + DeleteRecoveryRpcReply::AppliedVersion => Ok(()), + _ => Err(Error::other("unexpected recoverable delete reply")), + } + } + + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + ) -> Vec> { + let expected_len = versions.len(); + let result = match self + .recover_delete_transaction(volume, DeleteRecoveryRpcOperation::ApplyVersions { versions, opts }) + .await + { + Ok(DeleteRecoveryRpcReply::AppliedVersions(codes)) => decode_delete_recovery_batch_errors(codes, expected_len), + Ok(_) => Err(Error::other("unexpected recoverable batch delete reply")), + Err(error) => Err(error), + }; + match result { + Ok(errors) => errors, + Err(error) => vec![Some(error); expected_len], + } + } + + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + match self + .recover_delete_transaction( + volume, + DeleteRecoveryRpcOperation::Discover { + object: object.map(str::to_owned), + cursor, + limit, + }, + ) + .await? + { + DeleteRecoveryRpcReply::Discovered(page) => { + if object.is_some_and(|object| page.transactions.iter().any(|id| id.object != object)) { + return Err(Error::other("delete recovery discovery escaped requested object")); + } + if page.transactions.len() > usize::try_from(limit).map_err(|_| Error::other("invalid discovery limit"))? + || page + .transactions + .iter() + .any(|id| id.transaction.is_nil() || id.object.is_empty()) + { + return Err(Error::other("malformed delete discovery page")); + } + Ok(page) + } + _ => Err(Error::other("unexpected delete discovery reply")), + } + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + ) -> Result { + match self + .recover_delete_transaction( + volume, + DeleteRecoveryRpcOperation::Inspect { + object: object.to_owned(), + transaction, + }, + ) + .await? + { + DeleteRecoveryRpcReply::Inspected(inspection) => { + validate_delete_recovery_inspection(volume, object, transaction, &inspection)?; + Ok(inspection) + } + _ => Err(Error::other("unexpected delete inspection reply")), + } + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + ) -> Result { + match self + .recover_delete_transaction( + volume, + DeleteRecoveryRpcOperation::Settle { + object: object.to_owned(), + transaction, + expected_fingerprint, + action, + }, + ) + .await? + { + DeleteRecoveryRpcReply::Settled(inspection) => { + validate_delete_recovery_inspection(volume, object, transaction, &inspection)?; + Ok(inspection) + } + _ => Err(Error::other("unexpected delete settlement reply")), + } + } + async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> { self.execute_with_timeout( || async { @@ -4030,6 +4278,45 @@ mod tests { static INIT: Once = Once::new(); + #[test] + fn recoverable_batch_delete_rejects_invalid_error_codes_and_length() { + let errors = decode_delete_recovery_batch_errors(vec![None, Some(DiskError::FileNotFound.to_u32())], 2) + .expect("valid per-item errors"); + assert!(errors[0].is_none()); + assert_eq!(errors[1], Some(DiskError::FileNotFound)); + assert!(decode_delete_recovery_batch_errors(vec![None], 2).is_err()); + assert!(decode_delete_recovery_batch_errors(vec![Some(0)], 1).is_err()); + assert!(decode_delete_recovery_batch_errors(vec![Some(u32::MAX)], 1).is_err()); + } + + #[test] + fn delete_recovery_inspection_rejects_wrong_transaction_object_and_protocol() { + let transaction = Uuid::new_v4(); + let inspection = DeleteRecoveryInspection { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + id: crate::disk::DeleteTransactionId { + object: "object".into(), + transaction, + }, + state: crate::disk::DeleteRecoveryState::Absent, + journal: None, + current_digest: None, + current_versions: Vec::new(), + backup_digest: None, + rollback_data_complete: false, + commit_data_safe: false, + explicit_delete_safe: false, + fingerprint: [0; 32], + }; + validate_delete_recovery_inspection("bucket", "object", transaction, &inspection).expect("matching inspection"); + assert!(validate_delete_recovery_inspection("bucket", "other", transaction, &inspection).is_err()); + assert!(validate_delete_recovery_inspection("bucket", "object", Uuid::new_v4(), &inspection).is_err()); + assert!(validate_delete_recovery_inspection("bucket", "object", Uuid::nil(), &inspection).is_err()); + let mut incompatible = inspection; + incompatible.protocol_version += 1; + assert!(validate_delete_recovery_inspection("bucket", "object", transaction, &incompatible).is_err()); + } + #[test] fn compare_and_update_wire_outcome_fails_closed() { assert_eq!( @@ -4063,32 +4350,19 @@ mod tests { } #[test] - fn retired_marker_options_preserve_legacy_positional_wire_shape() { + fn delete_options_named_wire_preserves_rollback_and_legacy_marker_fields() { #[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq)] struct LegacyDeleteOptions { recursive: bool, immediate: bool, undo_write: bool, + #[serde(default)] undo_delete: bool, old_data_dir: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + expected_delete_marker: Option, } - let legacy = LegacyDeleteOptions { - recursive: false, - immediate: false, - undo_write: false, - undo_delete: false, - old_data_dir: None, - }; - let original = encode_msgpack(&legacy).unwrap(); - let current = encode_msgpack(&DeleteOptions::default()).unwrap(); - assert_eq!(current, original, "ordinary deletes must retain the older peer's positional payload"); - assert_eq!(rmp_serde::from_slice::(¤t).unwrap(), legacy); - assert!( - rmp_serde::from_slice::(&original) - .unwrap() - .expected_delete_marker - .is_none() - ); + let transaction = Uuid::new_v4(); let mut marker = FileInfo { deleted: true, version_id: Some(Uuid::new_v4()), @@ -4097,12 +4371,69 @@ mod tests { }; marker.set_delete_marker_incarnation(Uuid::new_v4()); let marker = rustfs_filemeta::MetaDeleteMarker::from(marker); + for expected_delete_marker in [None, Some(marker)] { + for (undo_write, undo_delete) in [(true, false), (false, true)] { + let legacy = LegacyDeleteOptions { + recursive: false, + immediate: false, + undo_write, + undo_delete, + old_data_dir: Some(transaction), + expected_delete_marker: expected_delete_marker.clone(), + }; + // Older senders use positional arrays. Their unchanged field prefix and + // missing optional fields must also decode on the named-map sender's peer. + for (encoding, payload) in [ + ("positional", rmp_serde::to_vec(&legacy).expect("encode legacy positional options")), + ("named", rmp_serde::to_vec_named(&legacy).expect("encode legacy named options")), + ] { + let decoded: DeleteOptions = rmp_serde::from_slice(&payload).expect("decode legacy options on current peer"); + assert_eq!(decoded.undo_write, undo_write, "{encoding} undo-write flag"); + assert_eq!(decoded.undo_delete, undo_delete, "{encoding} undo-delete flag"); + assert_eq!(decoded.old_data_dir, Some(transaction), "{encoding} transaction UUID"); + assert_eq!(decoded.expected_delete_marker, expected_delete_marker, "{encoding} marker precondition"); + assert!(decoded.delete_recovery_bucket_incarnation.is_none(), "{encoding} omitted incarnation"); + assert!(!decoded.recoverable_delete, "{encoding} omitted recovery flag"); + } + for incarnation in [None, Some(Uuid::new_v4())] { + let current = DeleteOptions { + undo_write, + undo_delete, + old_data_dir: Some(transaction), + expected_delete_marker: expected_delete_marker.clone(), + delete_recovery_bucket_incarnation: incarnation, + recoverable_delete: true, + ..Default::default() + }; + let payload = encode_delete_options_msgpack(¤t).expect("encode current named options"); + let decoded: DeleteOptions = rmp_serde::from_slice(&payload).expect("decode current options on current peer"); + assert_eq!(decoded.undo_write, undo_write); + assert_eq!(decoded.undo_delete, undo_delete); + assert_eq!(decoded.old_data_dir, Some(transaction)); + assert_eq!(decoded.expected_delete_marker, expected_delete_marker); + assert_eq!(decoded.delete_recovery_bucket_incarnation, incarnation); + assert!(decoded.recoverable_delete); + let decoded: LegacyDeleteOptions = + rmp_serde::from_slice(&payload).expect("decode current options on legacy peer"); + assert_eq!(decoded, legacy, "unknown recovery fields cannot shift rollback or marker values"); + } + } + } + } + + #[test] + fn malformed_nonempty_delete_options_do_not_fall_back_to_json() { let options = DeleteOptions { - expected_delete_marker: Some(marker.clone()), + undo_write: true, + old_data_dir: Some(Uuid::new_v4()), ..Default::default() }; - let decoded: DeleteOptions = rmp_serde::from_slice(&encode_msgpack(&options).unwrap()).unwrap(); - assert_eq!(decoded.expected_delete_marker, Some(marker)); + let json = serde_json::to_string(&options).expect("valid compatibility JSON"); + assert!(decode_msgpack_or_json::(&[0xc1], &json, "DeleteOptions").is_err()); + let decoded: DeleteOptions = + decode_msgpack_or_json(&[], &json, "DeleteOptions").expect("absent MessagePack permits JSON compatibility"); + assert!(decoded.undo_write); + assert_eq!(decoded.old_data_dir, options.old_data_dir); } #[test] diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 98de238f5..01091325b 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -466,12 +466,11 @@ impl LocalDiskWrapper { self.track_disk_health_mutation( "delete_version", DiskMetricMutation::Delete, - || async { + || { Box::pin( self.disk .delete_version_with_namespace_owner(volume, path, fi, force_del_marker, opts, namespace_owner), ) - .await }, get_max_timeout_duration(), ) @@ -1816,6 +1815,8 @@ impl LocalDiskWrapper { undo_delete: false, old_data_dir: None, expected_delete_marker: None, + recoverable_delete: false, + delete_recovery_bucket_incarnation: None, }, ) .await?; @@ -2001,10 +2002,86 @@ impl LocalDiskWrapper { } /// Get the current disk ID + pub(crate) async fn delete_versions_with_namespace_owner( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + namespace_owner: Option>, + ) -> Vec> { + let version_count = versions.len(); + self.metrics.record_operation_call("delete_versions"); + // Check if disk is faulty before proceeding + if self.health.is_faulty() { + self.metrics.record_availability_error(); + return vec![Some(DiskError::FaultyDisk); versions.len()]; + } + + // Check if disk is stale + if let Err(e) = self.check_disk_stale().await { + self.metrics.record_availability_error(); + return vec![Some(e); versions.len()]; + } + + let _io_permit = match self.acquire_disk_io_permit().await { + Ok(permit) => permit, + Err(error) => return vec![Some(error); version_count], + }; + + // Record operation start + self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed); + let waiting_guard = self.health.waiting_guard(); + let metric_waiting_guard = self.metrics.waiting_guard(); + let started = Instant::now(); + + // Execute the operation + let result = self + .disk + .delete_versions_with_namespace_owner(volume, versions, opts, namespace_owner) + .await; + self.metrics.record_operation_latency("delete_versions", started.elapsed()); + self.record_batch_delete_error_metrics(&result); + + drop(waiting_guard); + drop(metric_waiting_guard); + let has_err = result.iter().any(|e| e.is_some()); + if !has_err { + // Log success and decrement waiting counter + self.health.record_operation_success(&self.endpoint(), "operation_success"); + self.metrics.record_mutation_success(DiskMetricMutation::Delete); + } + + result + } + pub async fn get_current_disk_id(&self) -> Option { *self.disk_id.read().await } + pub(crate) async fn settle_delete_transaction_with_namespace_owner( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + expected_fingerprint: crate::disk::DeleteRecoveryDigest, + action: crate::disk::DeleteRecoveryAction, + namespace_owner: Option>, + ) -> Result { + self.track_disk_health_mutation( + "settle_delete_transaction", + DiskMetricMutation::Delete, + || { + Box::pin(async { + self.disk + .delete_recovery_settle(volume, object, transaction, expected_fingerprint, action, namespace_owner) + .await + }) + }, + crate::disk::get_delete_recovery_timeout()?, + ) + .await + } + /// Track disk health for an operation. /// This method should wrap disk operations to ensure health checking. pub async fn track_disk_health(&self, operation: F, timeout_duration: Duration) -> Result @@ -2195,6 +2272,87 @@ impl LocalDiskWrapper { #[async_trait::async_trait] impl DiskAPI for LocalDiskWrapper { + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: FileInfo, + force_del_marker: bool, + mut opts: DeleteOptions, + ) -> Result<()> { + crate::disk::validate_delete_recovery_options(&opts)?; + if object.starts_with('/') { + return Err(DiskError::InvalidPath); + } + opts.recoverable_delete = true; + self.delete_version(volume, object, file_info, force_del_marker, opts).await + } + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + mut opts: DeleteOptions, + ) -> Vec> { + if let Err(err) = crate::disk::validate_delete_recovery_options(&opts) { + return vec![Some(err); versions.len()]; + } + if versions.iter().any(|version| version.name.starts_with('/')) { + return vec![Some(DiskError::InvalidPath); versions.len()]; + } + opts.recoverable_delete = true; + self.delete_versions(volume, versions, opts).await + } + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + self.track_disk_health_with_op( + "discover_delete_transactions", + || async { + self.disk + .discover_delete_transactions(volume, object, cursor.clone(), limit) + .await + }, + get_max_timeout_duration(), + ) + .await + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + ) -> Result { + self.track_disk_health_with_op( + "inspect_delete_transaction", + || async { self.disk.inspect_delete_transaction(volume, object, transaction).await }, + get_max_timeout_duration(), + ) + .await + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + expected_fingerprint: crate::disk::DeleteRecoveryDigest, + action: crate::disk::DeleteRecoveryAction, + ) -> Result { + self.track_disk_health_mutation( + "settle_delete_transaction", + DiskMetricMutation::Delete, + || async { + self.disk + .settle_delete_transaction(volume, object, transaction, expected_fingerprint, action) + .await + }, + crate::disk::get_delete_recovery_timeout()?, + ) + .await + } fn has_replacement_mount_lease(&self) -> bool { self.disk.has_replacement_mount_lease() } @@ -2408,46 +2566,7 @@ impl DiskAPI for LocalDiskWrapper { } async fn delete_versions(&self, volume: &str, versions: Vec, opts: DeleteOptions) -> Vec> { - let version_count = versions.len(); - self.metrics.record_operation_call("delete_versions"); - // Check if disk is faulty before proceeding - if self.health.is_faulty() { - self.metrics.record_availability_error(); - return vec![Some(DiskError::FaultyDisk); versions.len()]; - } - - // Check if disk is stale - if let Err(e) = self.check_disk_stale().await { - self.metrics.record_availability_error(); - return vec![Some(e); versions.len()]; - } - - let _io_permit = match self.acquire_disk_io_permit().await { - Ok(permit) => permit, - Err(error) => return vec![Some(error); version_count], - }; - - // Record operation start - self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed); - let waiting_guard = self.health.waiting_guard(); - let metric_waiting_guard = self.metrics.waiting_guard(); - let started = Instant::now(); - - // Execute the operation - let result = self.disk.delete_versions(volume, versions, opts).await; - self.metrics.record_operation_latency("delete_versions", started.elapsed()); - self.record_batch_delete_error_metrics(&result); - - drop(waiting_guard); - drop(metric_waiting_guard); - let has_err = result.iter().any(|e| e.is_some()); - if !has_err { - // Log success and decrement waiting counter - self.health.record_operation_success(&self.endpoint(), "operation_success"); - self.metrics.record_mutation_success(DiskMetricMutation::Delete); - } - - result + self.delete_versions_with_namespace_owner(volume, versions, opts, None).await } async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> { diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index be9497308..11134035f 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -17,6 +17,7 @@ pub(in crate::disk) use self::commit::LocalRenamePreflightRejection; use self::commit::lock_rename_commit_directories; mod commit; +pub(super) mod delete_recovery; #[cfg(all(unix, test))] #[path = "fd_cache_tests.rs"] mod fd_cache_tests; @@ -130,8 +131,10 @@ tokio::task_local! { const RUSTFS_META_TMP_OLD_BUCKET: &str = ".rustfs.sys/tmp-old"; const INLINE_METADATA_ROLLBACK_DIR_XOR: u128 = 0x7275737466735f696e6c696e655f7262; const DELETE_MARKER_ROLLBACK_FILE: &str = "xl.meta.delete-marker.rollback"; +pub(crate) const DELETE_QUORUM_COMMITTED_FILE: &str = "delete-quorum-committed"; pub(crate) const DELETE_DATA_DIR_MARKER_PREFIX: &str = "delete-data."; pub(crate) const RESERVED_DELETE_DATA_DIR_MARKER_PREFIX: &str = "reserve-delete-data."; +pub(crate) const RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX: &str = "recovery-delete-data."; /// Largest directory read the delete-residue probe issues before it must /// fall back to a complete read. Residue holds one or two data dirs, so an /// under-filled batch settles the common case without materializing large @@ -161,6 +164,16 @@ pub(crate) fn metadata_less_part_file(entry: &str) -> bool { .is_some_and(|part_number| part_number.parse::().is_ok_and(|part_number| part_number > 0)) } +/// Native multipart sidecars travel with their part during delete recovery. +/// Callers must additionally require a regular file inside a bound data dir. +fn delete_recovery_part_file(entry: &str) -> bool { + metadata_less_part_file(entry) + || entry.strip_suffix(".meta").is_some_and(metadata_less_part_file) + || entry.split_once(".integrity.").is_some_and(|(part, generation)| { + metadata_less_part_file(part) && Uuid::parse_str(generation).is_ok_and(|id| !id.is_nil()) + }) +} + fn delete_transaction_marker_id(entry: &str, prefix: &str) -> Option { entry .strip_prefix(prefix) @@ -177,10 +190,15 @@ fn is_delete_transaction_marker(entry: &str, prefix: &str) -> bool { /// an `xl.meta`, an unknown file) means the directory is not plain delete residue. fn is_metadata_less_data_dir_entry(entry: &str) -> bool { !entry.ends_with(SLASH_SEPARATOR) - && (metadata_less_part_file(entry) + && (delete_recovery_part_file(entry) || entry == STORAGE_FORMAT_FILE_BACKUP + || entry == DELETE_QUORUM_COMMITTED_FILE + || entry == delete_recovery::PREPARED + || entry == delete_recovery::TARGET + || entry == delete_recovery::FINISHED || is_delete_transaction_marker(entry, DELETE_DATA_DIR_MARKER_PREFIX) - || is_delete_transaction_marker(entry, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX)) + || is_delete_transaction_marker(entry, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX) + || is_delete_transaction_marker(entry, RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX)) } const STARTUP_CLEANUP_WAIT_TIMEOUT: Duration = Duration::from_secs(2); const ENV_BITROT_SIZE_MISMATCH_RETRY_COUNT: &str = "RUSTFS_BITROT_SIZE_MISMATCH_RETRY_COUNT"; @@ -276,10 +294,21 @@ fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, r #[cfg(test)] async fn write_metadata_rollback_backup(object_dir: &Path, rollback_dir: Uuid, data: &[u8]) -> Result<()> { - write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, data, None).await + write_delete_rollback_file( + object_dir.parent().unwrap_or(object_dir), + "test-backup", + object_dir, + rollback_dir, + STORAGE_FORMAT_FILE_BACKUP, + data, + None, + ) + .await } async fn write_delete_rollback_file( + io_root: &Path, + volume: &str, object_dir: &Path, rollback_dir: Uuid, name: &str, @@ -290,16 +319,33 @@ async fn write_delete_rollback_file( let path = backup_dir.join(name); let lease = os::acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await; let data = data.to_vec(); - let object_dir = object_dir.to_path_buf(); + let sync = effective_durability(volume).syncs_commit_metadata(); + let staging_root = io_root.join(RUSTFS_META_TMP_BUCKET); + let io_root = io_root.to_path_buf(); os::run_blocking_namespace_operation(lease, move || { + reject_local_disk_symlink_components(&io_root, &path).map_err(std::io::Error::from)?; + reject_local_disk_symlink_components(&io_root, &staging_root).map_err(std::io::Error::from)?; std::fs::create_dir_all(&backup_dir)?; + std::fs::create_dir_all(&staging_root)?; #[cfg(test)] run_owned_file_write_before_open(&path); - // Rollback must never consume a partially written backup. Stage outside - // the rollback directory so interrupted writes cannot become staged data. - let mut staged = tempfile::Builder::new().prefix(".delete-rollback-").tempfile_in(object_dir)?; + // Private directory staging is reclaimed by existing tmp cleanup after + // a crash and cannot leave a regular artifact in the user's bucket. + let staging_dir = tempfile::Builder::new() + .prefix(".delete-rollback-") + .tempdir_in(&staging_root)?; + let mut staged = tempfile::NamedTempFile::new_in(staging_dir.path())?; std::io::Write::write_all(&mut staged, &data)?; + if sync { + staged.as_file().sync_all()?; + } staged.persist(path).map_err(|err| err.error)?; + if sync { + os::fsync_dir_std(&backup_dir)?; + if let Some(parent) = backup_dir.parent() { + os::fsync_dir_std(parent)?; + } + } Ok(()) }) .await @@ -336,14 +382,35 @@ async fn restore_metadata_backup_with_namespace_owner( } async fn restore_delete_rollback_with_namespace_owner( + volume: &str, object_dir: &Path, xl_path: &Path, rollback_dir: Uuid, publication_root: &os::PublicationRoot, namespace_owner: Option>, ) -> Result<()> { - remove_version_delete_markers(object_dir, rollback_dir, namespace_owner.clone()).await?; let rollback_path = object_dir.join(rollback_dir.to_string()); + match fs::symlink_metadata(rollback_path.join(DELETE_QUORUM_COMMITTED_FILE)).await { + Ok(_) => return Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + let recovery_journal = match fs::symlink_metadata(rollback_path.join(delete_recovery::PREPARED)).await { + Ok(metadata) if metadata.is_file() => { + let journal: super::DeleteRecoveryJournal = serde_json::from_slice( + &fs::read(rollback_path.join(delete_recovery::PREPARED)) + .await + .map_err(to_file_error)?, + )?; + if journal.volume != volume || journal.transaction != rollback_dir { + return Err(DiskError::FileCorrupt); + } + Some(journal) + } + Ok(_) => return Err(DiskError::InvalidPath), + Err(err) if err.kind() == ErrorKind::NotFound => None, + Err(err) => return Err(to_file_error(err).into()), + }; let mut staged_paths = Vec::new(); let mut remove_new_metadata = false; match fs::read_dir(&rollback_path).await { @@ -352,65 +419,166 @@ async fn restore_delete_rollback_with_namespace_owner( let name = entry.file_name(); if name == DELETE_MARKER_ROLLBACK_FILE { remove_new_metadata = true; - } else if name != STORAGE_FORMAT_FILE_BACKUP { + } else if name != STORAGE_FORMAT_FILE_BACKUP + && name != DELETE_QUORUM_COMMITTED_FILE + && name != delete_recovery::PREPARED + && name != delete_recovery::TARGET + && name != delete_recovery::FINISHED + { + if let Some(journal) = &recovery_journal { + let id = name + .to_str() + .and_then(|name| Uuid::parse_str(name).ok()) + .ok_or(DiskError::FileCorrupt)?; + if !journal.data_dirs.contains(&id) || !entry.file_type().await.map_err(to_file_error)?.is_dir() { + return Err(DiskError::FileCorrupt); + } + } staged_paths.push((entry.path(), object_dir.join(name))); } } } - Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) if err.kind() == ErrorKind::NotFound => { + return remove_version_delete_markers(volume, object_dir, rollback_dir, namespace_owner).await; + } Err(err) => return Err(to_file_error(err).into()), } let had_staged_paths = !staged_paths.is_empty(); for (src, dst) in staged_paths { - os::rename_all_with_owner(&src, &dst, object_dir, publication_root, namespace_owner.clone()).await?; + if recovery_journal.is_some() { + let mut entries = fs::read_dir(&src).await.map_err(to_file_error)?; + let mut files = Vec::new(); + while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { + let name = entry.file_name(); + let name = name.to_str().ok_or(DiskError::FileCorrupt)?; + if !entry.file_type().await.map_err(to_file_error)?.is_file() + || (!delete_recovery_part_file(name) + && ![ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .iter() + .any(|prefix| name == format!("{prefix}{rollback_dir}"))) + { + return Err(DiskError::FileCorrupt); + } + files.push((entry.path(), dst.join(name))); + } + os::create_dir_all_with_namespace_owner(&dst, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + for (source, destination) in files { + match fs::symlink_metadata(&destination).await { + Ok(metadata) if metadata.is_file() => { + let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, namespace_owner.clone()).await; + let identical = os::run_blocking_namespace_operation(lease, move || { + let source_meta = std::fs::symlink_metadata(&source)?; + let destination_meta = std::fs::symlink_metadata(&destination)?; + if !source_meta.is_file() + || !destination_meta.is_file() + || source_meta.len() != destination_meta.len() + { + return Ok(false); + } + let mut source_file = std::fs::File::open(&source)?; + let mut destination_file = std::fs::File::open(&destination)?; + let mut left = [0u8; 65536]; + let mut right = [0u8; 65536]; + let mut remaining = source_meta.len(); + while remaining > 0 { + let length = usize::try_from(remaining.min(left.len() as u64)).map_err(std::io::Error::other)?; + std::io::Read::read_exact(&mut source_file, &mut left[..length])?; + std::io::Read::read_exact(&mut destination_file, &mut right[..length])?; + if left[..length] != right[..length] { + return Ok(false); + } + remaining -= length as u64; + } + std::fs::remove_file(source)?; + Ok(true) + }) + .await + .map_err(to_file_error)?; + if !identical { + return Err(DiskError::FileCorrupt); + } + } + Ok(_) => return Err(DiskError::InvalidPath), + Err(err) if err.kind() == ErrorKind::NotFound => { + os::rename_all_with_owner(&source, &destination, object_dir, publication_root, namespace_owner.clone()) + .await?; + } + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&dst, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + os::fsync_dir_with_owner(&src, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + } + let _ = os::remove_dir_with_owner(&src, namespace_owner.clone()).await; + } else { + os::rename_all_with_owner(&src, &dst, object_dir, publication_root, namespace_owner.clone()).await?; + } } let backup_path = rollback_path.join(STORAGE_FORMAT_FILE_BACKUP); match os::rename_all_with_owner(&backup_path, xl_path, object_dir, publication_root, namespace_owner.clone()).await { - Ok(()) => { - let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; - Ok(()) - } - // A missing backup only means "remove the newly-created delete marker" - // when the marker proves there was no old metadata to restore. + Ok(()) => {} Err(DiskError::FileNotFound) if remove_new_metadata => { match os::remove_file_with_owner(xl_path, namespace_owner.clone()).await { - Ok(()) => { - let _ = os::remove_file_with_owner(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), namespace_owner.clone()) - .await; - let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; - Ok(()) - } - Err(err) if err.kind() == ErrorKind::NotFound => { - let _ = os::remove_file_with_owner(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), namespace_owner.clone()) - .await; - let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; - Ok(()) - } - Err(err) => Err(to_file_error(err).into()), + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), } } - Err(DiskError::FileNotFound) if had_staged_paths => Err(DiskError::FileNotFound), + Err(DiskError::FileNotFound) if had_staged_paths => return Err(DiskError::FileNotFound), Err(DiskError::FileNotFound) => match fs::metadata(xl_path).await { - Ok(_) => { - let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; - Ok(()) - } - Err(err) if err.kind() == ErrorKind::NotFound => Err(DiskError::FileNotFound), - Err(err) => Err(to_file_error(err).into()), + Ok(_) => {} + Err(err) if err.kind() == ErrorKind::NotFound => return Err(DiskError::FileNotFound), + Err(err) => return Err(to_file_error(err).into()), }, - Err(err) => Err(err), + Err(err) => return Err(err), } + delete_recovery::restore_barrier(volume, object_dir, rollback_dir).await?; + // Persist both sides of the restore rename before discarding its evidence. + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(object_dir, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + os::fsync_dir_with_owner(&rollback_path, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + } + remove_version_delete_markers(volume, object_dir, rollback_dir, namespace_owner.clone()).await?; + if remove_new_metadata { + let _ = os::remove_file_with_owner(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), namespace_owner.clone()).await; + } + for name in [delete_recovery::TARGET, delete_recovery::PREPARED, delete_recovery::FINISHED] { + match os::remove_file_with_owner(rollback_path.join(name), namespace_owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner).await; + Ok(()) } async fn remove_version_delete_markers( + volume: &str, object_dir: &Path, rollback_dir: Uuid, namespace_owner: Option>, ) -> Result<()> { let reserved_name = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); let committed_name = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); + let recovery_name = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); let mut entries = match fs::read_dir(object_dir).await { Ok(entries) => entries, Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), @@ -422,13 +590,20 @@ async fn remove_version_delete_markers( { continue; } - for marker_name in [&reserved_name, &committed_name] { + for marker_name in [&reserved_name, &committed_name, &recovery_name] { match os::remove_file_with_owner(entry.path().join(marker_name), namespace_owner.clone()).await { Ok(()) => {} Err(err) if err.kind() == ErrorKind::NotFound => {} Err(err) => return Err(to_file_error(err).into()), } } + // Also sync on retry: a previous unlink may have succeeded before its + // directory sync failed. Keep the journal until those removals persist. + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&entry.path(), namespace_owner.clone()) + .await + .map_err(to_file_error)?; + } } Ok(()) } @@ -464,7 +639,8 @@ async fn restore_delete_rollback_after_error( }; if let Err(restore_err) = - restore_delete_rollback_with_namespace_owner(object_dir, xl_path, rollback_dir, publication_root, namespace_owner).await + restore_delete_rollback_with_namespace_owner(volume, object_dir, xl_path, rollback_dir, publication_root, namespace_owner) + .await { warn!( volume, @@ -6300,6 +6476,32 @@ impl LocalDisk { result.map(|()| DataDirDeleteStatus::Deleted) } + pub(crate) async fn delete_versions_with_namespace_owner( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + namespace_owner: Option>, + ) -> Vec> { + let mut errs = Vec::with_capacity(versions.len()); + for _ in 0..versions.len() { + errs.push(None); + } + + for (i, ver) in versions.iter().enumerate() { + if let Err(e) = self + .delete_versions_internal(volume, ver.name.as_str(), &ver.versions, &opts, namespace_owner.clone()) + .await + { + errs[i] = Some(e); + } else { + errs[i] = None; + } + } + + errs + } + async fn delete_version_inner(&self, volume: &str, path: &str, fi: FileInfo, mutation: DeleteVersionMutation) -> Result<()> { let DeleteVersionMutation { force_del_marker, @@ -6347,10 +6549,24 @@ impl LocalDisk { check_path_length(file_path.to_string_lossy().as_ref())?; let xl_path = path_join(&[file_path.as_path(), Path::new(STORAGE_FORMAT_FILE)]); - let namespace_owner: Option> = + let mut namespace_owner: Option> = Some(os::acquire_metadata_mutation_lease(&self.get_object_path(volume, path)?, namespace_owner).await); + if opts.recoverable_delete || (opts.undo_write && opts.undo_delete) { + let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await; + namespace_owner = Some(Arc::new((namespace_owner, volume_guard))); + } if opts.old_data_dir.is_some() && opts.undo_write { - return self.undo_write(file_path.as_path(), &fi, &opts, namespace_owner).await; + if opts.undo_delete + && let Some(transaction) = opts.old_data_dir + && self + .rollback_prepared_delete(volume, path, transaction, namespace_owner.clone()) + .await? + { + return Ok(()); + } + return self + .undo_write(volume, file_path.as_path(), &fi, &opts, namespace_owner) + .await; } let rollback_dir = opts.old_data_dir; @@ -6360,12 +6576,12 @@ impl LocalDisk { if err != DiskError::FileNotFound { return Err(err); } - self.finish_committed_delete_residue(volume, path, namespace_owner.clone()) + self.finish_internal_delete_residue(volume, path, namespace_owner.clone()) .await; if fi.deleted && force_del_marker { return self - .write_missing_delete_marker(volume, path, fi, file_path.as_path(), rollback_dir, namespace_owner.clone()) + .write_missing_delete_marker(volume, path, fi, file_path.as_path(), &opts, namespace_owner.clone()) .await; } @@ -6397,15 +6613,35 @@ impl LocalDisk { } let old_dir = meta.delete_version(&fi)?; let mut reserved_version_delete = false; + let mut recovery_target = None; if let Some(rollback_dir) = rollback_dir { - write_delete_rollback_file( - file_path.as_path(), - rollback_dir, - STORAGE_FORMAT_FILE_BACKUP, - &buf, - namespace_owner.clone(), - ) - .await?; + if opts.recoverable_delete { + recovery_target = self + .prepare_delete_recovery( + volume, + path, + Some(&buf), + std::slice::from_ref(&fi), + None, + delete_recovery::DeletePreparationContext { + transaction: rollback_dir, + bucket_incarnation: opts.delete_recovery_bucket_incarnation, + namespace_owner: namespace_owner.clone(), + }, + ) + .await?; + } else { + write_delete_rollback_file( + self.io_root(), + volume, + file_path.as_path(), + rollback_dir, + STORAGE_FORMAT_FILE_BACKUP, + &buf, + namespace_owner.clone(), + ) + .await?; + } } if let Some(uuid) = old_dir { @@ -6562,7 +6798,7 @@ impl LocalDisk { } let commit_result = if !meta.versions.is_empty() { - let buf = match meta.marshal_msg() { + let buf = match recovery_target.map(Ok).unwrap_or_else(|| meta.marshal_msg()) { Ok(buf) => buf, Err(err) => { let err: DiskError = err.into(); @@ -6642,6 +6878,10 @@ impl LocalDisk { .await); } + if let Some(transaction) = rollback_dir { + self.delete_recovery_barrier(volume, path, transaction, "APPLIED").await?; + } + if reserved_version_delete && let Some(rollback_dir) = rollback_dir && let Err(err) = self @@ -6698,6 +6938,7 @@ impl LocalDisk { async fn undo_write( &self, + volume: &str, file_path: &Path, fi: &FileInfo, opts: &DeleteOptions, @@ -6707,6 +6948,7 @@ impl LocalDisk { let xl_path = path_join(&[file_path, Path::new(STORAGE_FORMAT_FILE)]); if opts.undo_delete { restore_delete_rollback_with_namespace_owner( + volume, file_path, &xl_path, old_data_dir, @@ -6836,13 +7078,29 @@ impl LocalDisk { namespace_owner: Option>, ) -> Result<()> { crate::hp_guard!("LocalDisk::delete"); + let mut opt = opt; + let mut namespace_owner = namespace_owner; let handled_version_delete = if opt.recursive && opt.immediate && let Some((object, transaction_id)) = path.rsplit_once('/') && let Ok(transaction_id) = Uuid::parse_str(transaction_id) + && !transaction_id.is_nil() { - self.finish_version_delete(volume, object, transaction_id, namespace_owner.clone()) - .await? + if opt.old_data_dir == Some(transaction_id) { + let object_path = self.get_object_path(volume, object)?; + namespace_owner = Some(os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await); + let handled = self + .finish_delete_transaction_with_namespace_owner(volume, object, transaction_id, namespace_owner.clone()) + .await?; + // A key below the UUID prefix may be created concurrently with + // cleanup. Never recursively remove newly published descendants. + opt.recursive = false; + handled + } else { + self.finish_version_delete(volume, object, transaction_id, false, namespace_owner.clone()) + .await? + .0 + } } else { false }; @@ -6855,6 +7113,104 @@ impl LocalDisk { } } + // The caller owns the object metadata lease; retry recovery must not + // reacquire it while holding the same mutation domain. + async fn finish_delete_transaction_with_namespace_owner( + &self, + volume: &str, + object: &str, + transaction_id: Uuid, + namespace_owner: Option>, + ) -> Result { + let path = format!("{object}/{transaction_id}"); + let transaction_path = self.io_get_object_path(volume, &path)?; + let mut transaction_cleanup = false; + match fs::read_dir(&transaction_path).await { + Ok(mut entries) => { + while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { + let name = entry.file_name(); + if !entry.file_type().await.map_err(to_file_error)?.is_file() + || (name != STORAGE_FORMAT_FILE_BACKUP + && name != DELETE_MARKER_ROLLBACK_FILE + && name != DELETE_QUORUM_COMMITTED_FILE) + { + return Err(DiskError::FileCorrupt); + } + transaction_cleanup = true; + } + } + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + if transaction_cleanup { + // The set invokes transaction cleanup only after observing write + // quorum. Unlike data-dir markers, this receipt proves that + // decision and also covers inline objects with no data directory. + let receipt_path = transaction_path.join(DELETE_QUORUM_COMMITTED_FILE); + let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, namespace_owner.clone()).await; + let sync = effective_durability(volume).syncs_commit_metadata(); + os::run_blocking_namespace_operation(lease, move || { + let receipt = match std::fs::OpenOptions::new().write(true).create_new(true).open(&receipt_path) { + Ok(receipt) => receipt, + Err(err) if err.kind() == ErrorKind::AlreadyExists => { + if !std::fs::symlink_metadata(&receipt_path)?.is_file() { + return Err(std::io::Error::new(ErrorKind::InvalidData, "delete receipt is not a regular file")); + } + std::fs::File::open(receipt_path)? + } + Err(err) => return Err(err), + }; + if sync { + receipt.sync_all()?; + os::fsync_dir_std(&transaction_path)?; + if let Some(parent) = transaction_path.parent() { + os::fsync_dir_std(parent)?; + } + } + Ok(()) + }) + .await + .map_err(to_file_error)?; + // Retain the receipt until both the backup and marked data are + // gone. Every interrupted cleanup remains recognizable. + for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] { + match os::remove_file_with_owner(self.io_get_object_path(volume, &path)?.join(name), namespace_owner.clone()) + .await + { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&self.io_get_object_path(volume, &path)?, namespace_owner.clone()) + .await + .map_err(to_file_error)?; + } + } + let (found, physically_deleted) = self + .finish_version_delete(volume, object, transaction_id, true, namespace_owner.clone()) + .await?; + if !physically_deleted { + // Snapshot readers still own these parts. Keep the quorum receipt + // until their release finishes the physical data removal. + return Err(DiskError::DiskOngoingReq); + } + if transaction_cleanup { + match os::remove_file_with_owner( + self.io_get_object_path(volume, &path)?.join(DELETE_QUORUM_COMMITTED_FILE), + namespace_owner, + ) + .await + { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + Ok(found || transaction_cleanup) + } + async fn delete_unleased(&self, volume: &str, path: &str, opt: &DeleteOptions) -> Result<()> { self.delete_unleased_with_namespace_owner(volume, path, opt, None).await } @@ -6875,7 +7231,94 @@ impl LocalDisk { let file_path = self.io_get_object_path(volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; - self.delete_file_with_namespace_owner(&volume_dir, &file_path, opt.recursive, opt.immediate, namespace_owner) + let mut recursive = opt.recursive; + if recursive + && let Some(transaction) = opt.old_data_dir + && path + .rsplit_once('/') + .is_some_and(|(_, name)| Uuid::parse_str(name).is_ok_and(|id| id != transaction || opt.recoverable_delete)) + { + // Snapshot release replays these same options. Remove only this + // transaction's regular files; UUID-prefix child keys can appear + // after classification and must never be recursively removed. + let lease = os::acquire_namespace_mutation_lease_with_owner(&file_path, namespace_owner.clone()).await; + let data_path = file_path.clone(); + let sync = effective_durability(volume).syncs_commit_metadata(); + let keep_rollback_witnesses = opt.recoverable_delete; + os::run_blocking_namespace_operation(lease, move || { + let entries = match std::fs::read_dir(&data_path) { + Ok(entries) => entries, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(err), + }; + let committed = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"); + let recovery_committed = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"); + let reserved = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"); + let corrupt = + || std::io::Error::new(ErrorKind::InvalidData, "delete data directory contains unrecognized entries"); + let mut parts = Vec::new(); + let mut markers = Vec::new(); + let mut has_committed_marker = false; + for entry in entries { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_str().ok_or_else(corrupt)?; + if !entry.file_type()?.is_file() { + return Err(corrupt()); + } + if delete_recovery_part_file(name) { + parts.push(entry.path()); + } else if name == committed || name == recovery_committed || name == reserved { + has_committed_marker |= name == committed || name == recovery_committed; + markers.push(entry.path()); + } else if keep_rollback_witnesses + && [ + STORAGE_FORMAT_FILE_BACKUP, + DELETE_MARKER_ROLLBACK_FILE, + DELETE_QUORUM_COMMITTED_FILE, + ] + .contains(&name) + { + // Explicit legacy deletion keeps rollback witnesses until + // every snapshot-protected payload has disappeared. + } else { + return Err(corrupt()); + } + } + if !parts.is_empty() && !has_committed_marker { + return Err(corrupt()); + } + for files in [parts, markers] { + for entry in files { + #[cfg(all(any(test, feature = "test-util"), not(windows)))] + os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &entry); + match std::fs::remove_file(entry) { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(err), + } + } + if sync { + os::fsync_dir_std(&data_path)?; + } + } + match std::fs::remove_dir(data_path) { + Ok(()) => Ok(()), + Err(err) if is_benign_object_rmdir_error(&err) => Ok(()), + Err(err) => Err(err), + } + }) + .await + .map_err(|err| { + if err.kind() == ErrorKind::InvalidData { + DiskError::FileCorrupt + } else { + to_file_error(err).into() + } + })?; + recursive = false; + } + self.delete_file_with_namespace_owner(&volume_dir, &file_path, recursive, opt.immediate, namespace_owner) .await?; // A deleted shard must not remain readable through the io_uring fd cache. self.io_backend.invalidate_cached_fds_under(volume, path); @@ -7245,20 +7688,63 @@ impl LocalDisk { path: &str, fi: FileInfo, object_dir: &Path, - rollback_dir: Option, + opts: &DeleteOptions, namespace_owner: Option>, ) -> Result<()> { + let rollback_dir = opts.old_data_dir; + let incarnation = opts.delete_recovery_bucket_incarnation; let xl_path = object_dir.join(STORAGE_FORMAT_FILE); + let mut recovery_target = None; if let Some(rollback_dir) = rollback_dir { - write_delete_rollback_file(object_dir, rollback_dir, DELETE_MARKER_ROLLBACK_FILE, &[], namespace_owner.clone()) + if opts.recoverable_delete { + recovery_target = self + .prepare_delete_recovery( + volume, + path, + None, + std::slice::from_ref(&fi), + None, + delete_recovery::DeletePreparationContext { + transaction: rollback_dir, + bucket_incarnation: incarnation, + namespace_owner: namespace_owner.clone(), + }, + ) + .await?; + } else { + write_delete_rollback_file( + self.io_root(), + volume, + object_dir, + rollback_dir, + DELETE_MARKER_ROLLBACK_FILE, + &[], + namespace_owner.clone(), + ) .await?; + } } - if let Err(err) = self - .write_metadata_with_namespace_owner(volume, path, fi, namespace_owner.clone()) + let committed = if let Some(target) = recovery_target { + async { + fi.validate_for_metadata_read()?; + self.write_all_meta_with_namespace_owner( + volume, + &format!("{path}/{STORAGE_FORMAT_FILE}"), + &target, + true, + namespace_owner.clone(), + ) + .await + } .await - { + } else { + self.write_metadata_with_namespace_owner(volume, path, fi, namespace_owner.clone()) + .await + }; + if let Err(err) = committed { if let Some(rollback_dir) = rollback_dir && let Err(restore_err) = restore_delete_rollback_with_namespace_owner( + volume, object_dir, &xl_path, rollback_dir, @@ -7281,13 +7767,28 @@ impl LocalDisk { } return Err(err); } + if let Some(transaction) = rollback_dir { + self.delete_recovery_barrier(volume, path, transaction, "APPLIED").await?; + } Ok(()) } - async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> { + async fn delete_versions_internal( + &self, + volume: &str, + path: &str, + fis: &[FileInfo], + opts: &DeleteOptions, + namespace_owner: Option>, + ) -> Result<()> { let volume_dir = self.io_get_bucket_path(volume)?; let object_path = self.get_object_path(volume, path)?; - let namespace_owner: Option> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await); + let mut namespace_owner: Option> = + Some(os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await); + if opts.recoverable_delete || (opts.undo_write && opts.undo_delete) { + let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await; + namespace_owner = Some(Arc::new((namespace_owner, volume_guard))); + } let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?; let object_dir = xlpath .parent() @@ -7297,7 +7798,14 @@ impl LocalDisk { && opts.undo_write { if opts.undo_delete { + if self + .rollback_prepared_delete(volume, path, rollback_dir, namespace_owner.clone()) + .await? + { + return Ok(()); + } return restore_delete_rollback_with_namespace_owner( + volume, object_dir, &xlpath, rollback_dir, @@ -7323,7 +7831,7 @@ impl LocalDisk { { Ok(data) => data, Err(DiskError::FileNotFound) => { - self.finish_committed_delete_residue(volume, path, namespace_owner.clone()) + self.finish_internal_delete_residue(volume, path, namespace_owner.clone()) .await; // `deleted` alone can be an explicit marker purge; only // `mark_deleted` may create metadata that was not present. @@ -7331,14 +7839,7 @@ impl LocalDisk { return Err(DiskError::FileNotFound); }; return self - .write_missing_delete_marker( - volume, - path, - delete_marker, - object_dir, - opts.old_data_dir, - namespace_owner.clone(), - ) + .write_missing_delete_marker(volume, path, delete_marker, object_dir, opts, namespace_owner.clone()) .await; } Err(err) => return Err(err), @@ -7353,9 +7854,35 @@ impl LocalDisk { fm.unmarshal_msg(&data)?; let rollback_dir = opts.old_data_dir; let mut reserved_version_delete = false; + let mut recovery_target = None; if let Some(rollback_dir) = rollback_dir { - write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, &data, namespace_owner.clone()) + if opts.recoverable_delete { + recovery_target = self + .prepare_delete_recovery( + volume, + path, + Some(&data), + fis, + None, + delete_recovery::DeletePreparationContext { + transaction: rollback_dir, + bucket_incarnation: opts.delete_recovery_bucket_incarnation, + namespace_owner: namespace_owner.clone(), + }, + ) + .await?; + } else { + write_delete_rollback_file( + self.io_root(), + volume, + object_dir, + rollback_dir, + STORAGE_FORMAT_FILE_BACKUP, + &data, + namespace_owner.clone(), + ) .await?; + } } for fi in fis.iter() { @@ -7601,6 +8128,9 @@ impl LocalDisk { ) .await); } + if let Some(transaction) = rollback_dir { + self.delete_recovery_barrier(volume, path, transaction, "APPLIED").await?; + } if reserved_version_delete && let Some(rollback_dir) = rollback_dir && let Err(err) = self @@ -7629,7 +8159,7 @@ impl LocalDisk { // Update xl.meta atomically: a concurrent reader or crash mid-write must // never observe a truncated xl.meta for versions that were not deleted. - let buf = match fm.marshal_msg() { + let buf = match recovery_target.map(Ok).unwrap_or_else(|| fm.marshal_msg()) { Ok(buf) => buf, Err(err) => { let err: DiskError = err.into(); @@ -7706,6 +8236,10 @@ impl LocalDisk { .await); } + if let Some(transaction) = rollback_dir { + self.delete_recovery_barrier(volume, path, transaction, "APPLIED").await?; + } + if reserved_version_delete && let Some(rollback_dir) = rollback_dir && let Err(err) = self @@ -9237,7 +9771,16 @@ impl LocalDisk { Err(err) => return Err(to_file_error(err).into()), }; let reserved_name = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); - let committed_name = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); + let recovery_owned = + fs::try_exists(self.io_get_object_path(volume, &format!("{object}/{rollback_dir}/{}", delete_recovery::PREPARED))?) + .await + .map_err(to_file_error)?; + let prefix = if recovery_owned { + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX + } else { + DELETE_DATA_DIR_MARKER_PREFIX + }; + let committed_name = format!("{prefix}{rollback_dir}"); while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { if !entry.file_type().await.map_err(to_file_error)?.is_dir() || !entry.file_name().to_str().is_some_and(|name| Uuid::parse_str(name).is_ok()) @@ -9272,14 +9815,21 @@ impl LocalDisk { /// removed the object's metadata on this disk and committed, so the data /// it marked can no longer be read, healed or rolled back here. Finishing /// removes what `T`'s own cleanup pass would have removed. Reserved - /// markers, rollback backups without a committed marker, and object + /// transaction receipts also prove quorum commitment for inline objects. + /// Reserved markers, rollback backups without either proof, and object /// directories that hold `xl.meta` are left alone. - async fn finish_committed_delete_residue( + async fn finish_internal_delete_residue( &self, volume: &str, path: &str, namespace_owner: Option>, ) -> usize { + // RUSTFS_COMPAT_TODO(backlog#2807): retain legacy cleanup for internal + // maintenance until old system-bucket records drain. User-bucket + // residue requires the explicit set-wide recovery decision. + if !crate::bucket::utils::is_meta_bucketname(volume) { + return 0; + } let transactions = match self.committed_delete_transactions(volume, path).await { Ok(transactions) => transactions, Err(err) => { @@ -9289,7 +9839,7 @@ impl LocalDisk { subsystem = LOG_SUBSYSTEM_DISK_LOCAL, volume, path, - operation = "scan_committed_delete_residue", + operation = "scan_internal_delete_residue", error = ?err, "Disk local delete failed" ); @@ -9303,17 +9853,49 @@ impl LocalDisk { }; let mut finished = 0; for transaction in transactions { - // Unlike the delete's own pass, drop the rollback backup before the - // marked data: if this is cut short, the marker that proves the - // transaction is still in place for the next attempt. - let result = match self - .delete_unleased_with_namespace_owner(volume, &format!("{path}/{transaction}"), &opts, namespace_owner.clone()) - .await - { - Ok(()) | Err(DiskError::FileNotFound) => self - .finish_version_delete(volume, path, transaction, namespace_owner.clone()) - .await - .map(|_| ()), + let transaction_path = format!("{path}/{transaction}"); + let receipt = self + .io_get_object_path(volume, &transaction_path) + .map(|transaction_path| transaction_path.join(DELETE_QUORUM_COMMITTED_FILE)); + let result = match receipt { + Ok(receipt) => match fs::symlink_metadata(receipt).await { + Ok(metadata) if metadata.is_file() => { + match self + .finish_delete_transaction_with_namespace_owner(volume, path, transaction, namespace_owner.clone()) + .await + { + Ok(_) => { + self.delete_unleased_with_namespace_owner( + volume, + &transaction_path, + &DeleteOptions { + recursive: false, + ..opts.clone() + }, + namespace_owner.clone(), + ) + .await + } + Err(err) => Err(err), + } + } + Ok(_) => Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => { + // Preserve the legacy marker-only cleanup policy without + // upgrading disk-local evidence into a quorum receipt. + match self + .delete_unleased_with_namespace_owner(volume, &transaction_path, &opts, namespace_owner.clone()) + .await + { + Ok(()) | Err(DiskError::FileNotFound) => self + .finish_version_delete(volume, path, transaction, false, namespace_owner.clone()) + .await + .map(|_| ()), + Err(err) => Err(err), + } + } + Err(err) => Err(to_file_error(err).into()), + }, Err(err) => Err(err), }; match result { @@ -9325,7 +9907,7 @@ impl LocalDisk { volume, path, transaction = %transaction, - operation = "finish_committed_delete_residue", + operation = "finish_internal_delete_residue", error = ?err, "Disk local delete failed" ), @@ -9385,7 +9967,22 @@ impl LocalDisk { Err(err) if err.kind() == ErrorKind::NotFound => continue, Err(err) => return Err(to_file_error(err).into()), }; + let mut has_quorum_receipt = false; + let mut safe_receipt_dir = true; while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { + let name = entry.file_name(); + let file_type = entry.file_type().await.map_err(to_file_error)?; + if !file_type.is_file() + || (name != STORAGE_FORMAT_FILE_BACKUP + && name != DELETE_MARKER_ROLLBACK_FILE + && name != DELETE_QUORUM_COMMITTED_FILE) + { + safe_receipt_dir = false; + } + if entry.file_name() == DELETE_QUORUM_COMMITTED_FILE { + has_quorum_receipt = file_type.is_file(); + continue; + } let Some(transaction) = entry .file_name() .to_str() @@ -9402,8 +9999,31 @@ impl LocalDisk { Err(err) => return Err(to_file_error(err).into()), } } + if has_quorum_receipt + && safe_receipt_dir + && let Some(transaction) = data_dir + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| Uuid::parse_str(name).ok()) + { + transactions.insert(transaction); + } } - Ok(transactions) + let mut legacy = BTreeSet::new(); + for transaction in transactions { + let rollback = object_path.join(transaction.to_string()); + let mut recovery_owned = false; + for name in [delete_recovery::PREPARED, delete_recovery::TARGET, delete_recovery::FINISHED] { + recovery_owned |= fs::try_exists(rollback.join(name)).await.map_err(to_file_error)?; + } + if let Ok(metadata) = fs::symlink_metadata(rollback.join(DELETE_QUORUM_COMMITTED_FILE)).await { + recovery_owned |= metadata.len() > 0; + } + if !recovery_owned { + legacy.insert(transaction); + } + } + Ok(legacy) } async fn finish_version_delete( @@ -9411,17 +10031,30 @@ impl LocalDisk { volume: &str, object: &str, rollback_dir: Uuid, + preserve_prefix_children: bool, namespace_owner: Option>, - ) -> Result { + ) -> Result<(bool, bool)> { let object_path = self.io_get_object_path(volume, object)?; let mut entries = match fs::read_dir(object_path).await { Ok(entries) => entries, - Err(err) if err.kind() == ErrorKind::NotFound => return Ok(false), + Err(err) if err.kind() == ErrorKind::NotFound => return Ok((false, true)), Err(err) => return Err(to_file_error(err).into()), }; - let marker_name = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); + let recovery_owned = preserve_prefix_children + && fs::try_exists( + self.io_get_object_path(volume, &format!("{object}/{rollback_dir}/{}", delete_recovery::PREPARED))?, + ) + .await + .map_err(to_file_error)?; + let prefix = if recovery_owned { + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX + } else { + DELETE_DATA_DIR_MARKER_PREFIX + }; + let marker_name = format!("{prefix}{rollback_dir}"); let mut first_err = None; let mut found = false; + let mut deferred = false; while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { let Some(data_dir) = entry.file_name().to_str().and_then(|data_dir| Uuid::parse_str(data_dir).ok()) else { continue; @@ -9432,25 +10065,27 @@ impl LocalDisk { Err(err) if err.kind() == ErrorKind::NotFound => continue, Err(err) => return Err(to_file_error(err).into()), } - if let Err(err) = self + match self .delete_data_dir_with_namespace_owner( volume, &format!("{object}/{data_dir}"), DeleteOptions { recursive: true, + old_data_dir: preserve_prefix_children.then_some(rollback_dir), ..Default::default() }, namespace_owner.clone(), ) .await - && first_err.is_none() - && err != DiskError::FileNotFound - && err != DiskError::VolumeNotFound { - first_err = Some(err); + Ok(DataDirDeleteStatus::Deferred) => deferred = true, + Ok(DataDirDeleteStatus::Deleted) => {} + Err(err) if err == DiskError::FileNotFound || err == DiskError::VolumeNotFound => {} + Err(err) if first_err.is_none() => first_err = Some(err), + Err(_) => {} } } - first_err.map_or(Ok(found), Err) + first_err.map_or(Ok((found, !deferred)), Err) } async fn abort_reserved_version_delete_with_failure( @@ -9560,6 +10195,64 @@ pub(crate) async fn batch_shard_pread(requests: Vec<(std::path::PathBuf, usize, #[async_trait::async_trait] impl DiskAPI for LocalDisk { + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: FileInfo, + force_del_marker: bool, + mut opts: DeleteOptions, + ) -> Result<()> { + super::validate_delete_recovery_options(&opts)?; + if object.starts_with('/') { + return Err(DiskError::InvalidPath); + } + opts.recoverable_delete = true; + self.delete_version(volume, object, file_info, force_del_marker, opts).await + } + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + mut opts: DeleteOptions, + ) -> Vec> { + if let Err(err) = super::validate_delete_recovery_options(&opts) { + return vec![Some(err); versions.len()]; + } + if versions.iter().any(|version| version.name.starts_with('/')) { + return vec![Some(DiskError::InvalidPath); versions.len()]; + } + opts.recoverable_delete = true; + self.delete_versions(volume, versions, opts).await + } + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + self.delete_recovery_discover(volume, object, cursor, limit).await + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + ) -> Result { + self.delete_recovery_inspect(volume, object, transaction).await + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: super::DeleteRecoveryDigest, + action: super::DeleteRecoveryAction, + ) -> Result { + self.delete_recovery_settle(volume, object, transaction, expected_fingerprint, action, None) + .await + } fn to_string(&self) -> String { self.root.to_string_lossy().to_string() } @@ -11261,23 +11954,7 @@ impl DiskAPI for LocalDisk { } #[tracing::instrument(level = "trace", skip_all)] async fn delete_versions(&self, volume: &str, versions: Vec, opts: DeleteOptions) -> Vec> { - let mut errs = Vec::with_capacity(versions.len()); - for _ in 0..versions.len() { - errs.push(None); - } - - for (i, ver) in versions.iter().enumerate() { - if let Err(e) = self - .delete_versions_internal(volume, ver.name.as_str(), &ver.versions, &opts) - .await - { - errs[i] = Some(e); - } else { - errs[i] = None; - } - } - - errs + self.delete_versions_with_namespace_owner(volume, versions, opts, None).await } #[tracing::instrument(level = "trace", skip_all)] @@ -11632,7 +12309,7 @@ mod test { } } - fn test_file_info(name: &str, version_id: Uuid, data_dir: Option, data: Option) -> FileInfo { + pub(super) fn test_file_info(name: &str, version_id: Uuid, data_dir: Option, data: Option) -> FileInfo { let size = data .as_ref() .map(|data| i64::try_from(data.len()).expect("test data length should fit i64")) @@ -11694,7 +12371,7 @@ mod test { assert!(!should_read_legacy_inline_part(&ec12, &config)); } - fn test_meta(fi: FileInfo) -> Vec { + pub(super) fn test_meta(fi: FileInfo) -> Vec { let mut meta = FileMeta::default(); meta.add_version(fi).expect("test metadata should accept file info"); meta.marshal_msg().expect("test metadata should encode") @@ -11958,7 +12635,7 @@ mod test { assert!(!child_path.exists(), "the exact locked directory should be removed"); } - async fn ensure_test_volume(disk: &LocalDisk, volume: &str) { + pub(super) async fn ensure_test_volume(disk: &LocalDisk, volume: &str) { match disk.make_volume(volume).await { Ok(()) | Err(DiskError::VolumeExists) => {} Err(err) => panic!("test volume should be available: {err:?}"), @@ -17890,7 +18567,7 @@ mod test { .await .expect("existing metadata should be written"); - disk.delete_versions_internal(bucket, object, &[missing_fi, existing_fi], &DeleteOptions::default()) + disk.delete_versions_internal(bucket, object, &[missing_fi, existing_fi], &DeleteOptions::default(), None) .await .expect("missing non-deleted version should not abort deletion"); @@ -18047,6 +18724,7 @@ mod test { old_data_dir: Some(rollback_dir), ..Default::default() }, + None, ) .await .expect("missing inline data dir should not fail batch deletion"); @@ -18260,8 +18938,7 @@ mod test { } } - /// Leave the state a delete reaches once it committed on this disk but - /// before its cleanup pass: what a cancelled request used to strand. + /// A legacy disk-local mutation without a structured set decision. async fn write_committed_delete_residue(disk: &LocalDisk, bucket: &str, object: &str) -> CommittedDeleteResidue { let version_id = Uuid::new_v4(); let data_dir = Uuid::new_v4(); @@ -18301,6 +18978,56 @@ mod test { residue } + #[tokio::test] + async fn delete_recovery_user_bucket_apply_preserves_ambiguous_legacy_residue() { + for batch in [false, true] { + for recoverable in [false, true] { + let dir = tempfile::tempdir().expect("residue fixture directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 endpoint")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "user-legacy-residue"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let original = fs::read(residue.backup()).await.expect("legacy metadata backup"); + let request = FileInfo { + name: object.to_owned(), + version_id: Some(Uuid::new_v4()), + ..Default::default() + }; + let opts = DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + recoverable_delete: recoverable, + delete_recovery_bucket_incarnation: Some(Uuid::new_v4()), + ..Default::default() + }; + if batch { + let errors = disk + .delete_versions( + bucket, + vec![FileInfoVersions { + name: object.to_owned(), + versions: vec![request], + ..Default::default() + }], + opts, + ) + .await; + assert!(matches!(errors.as_slice(), [Some(DiskError::FileNotFound)]), "{errors:?}"); + } else { + assert!(matches!( + disk.delete_version(bucket, object, request, false, opts).await, + Err(DiskError::FileVersionNotFound) + )); + } + assert_eq!(fs::read(residue.backup()).await.expect("preserved backup"), original); + assert_eq!(fs::read(residue.part()).await.expect("preserved shard"), b"stale"); + assert!(residue.marker().is_file(), "a missing local apply cannot certify historical deletion"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 0); + } + } + } + fn residue_delete_request(object: &str) -> Vec { vec![FileInfoVersions { name: object.to_string(), @@ -18312,14 +19039,14 @@ mod test { }] } - // #6898: repeating the delete of a key whose earlier delete committed but - // never cleaned up reclaims the residue and still reports the key missing. + // Internal maintenance retains its legacy cleanup behavior while user + // buckets require the separate set-wide authorization boundary. #[tokio::test] async fn delete_versions_finishes_committed_delete_residue() { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "prefix/object"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18339,7 +19066,7 @@ mod test { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "object"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18359,14 +19086,13 @@ mod test { assert!(!residue.object_dir.exists(), "the committed residue must be reclaimed"); } - // The rollback backup may already be gone; the committed marker alone - // proves the delete. + // Internal maintenance can finish its legacy marker after backup cleanup. #[tokio::test] async fn delete_versions_finishes_committed_marker_without_rollback_backup() { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "object"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18382,15 +19108,14 @@ mod test { assert!(!residue.object_dir.exists(), "the committed marker alone must be enough"); } - // Only a committed marker on this disk proves the delete. A reserved marker, - // a rollback backup alone, or live metadata beside the marker is left for - // the owning transaction, rollback, or heal. + // Internal maintenance preserves reserved markers, backup-only states and + // live metadata for the owning transaction, rollback or heal. #[tokio::test] async fn committed_delete_residue_finish_leaves_unproven_state_alone() { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; ensure_test_volume(&disk, bucket).await; // A rollback backup without a committed marker, as an inline object leaves it. @@ -18419,7 +19144,7 @@ mod test { for object in ["backup-only", "reserved", "live"] { assert_eq!( - disk.finish_committed_delete_residue(bucket, object, None).await, + disk.finish_internal_delete_residue(bucket, object, None).await, 0, "{object} carries no committed proof" ); @@ -18446,7 +19171,7 @@ mod test { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "logs"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18458,7 +19183,7 @@ mod test { .await .expect("nested metadata should be written"); - assert_eq!(disk.finish_committed_delete_residue(bucket, object, None).await, 0); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 0); assert!(residue.backup().exists() && residue.marker().exists() && residue.part().exists()); assert!(nested.exists()); } @@ -18474,7 +19199,7 @@ mod test { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "object"; ensure_test_volume(&disk, bucket).await; let residue = write_committed_delete_residue(&disk, bucket, object).await; @@ -18492,7 +19217,7 @@ mod test { let _ = release_rx.recv(); }); - let mut finish = Box::pin(disk.finish_committed_delete_residue(bucket, object, None)); + let mut finish = Box::pin(disk.finish_internal_delete_residue(bucket, object, None)); tokio::time::timeout(Duration::from_secs(10), async { tokio::select! { entered = entered_rx => entered.expect("finish should reach the marked data dir"), @@ -18508,7 +19233,7 @@ mod test { // The abandoned rename is still parked; the next attempt recognizes the // residue by its marker and finishes it. - assert_eq!(disk.finish_committed_delete_residue(bucket, object, None).await, 1); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); assert!(!residue.object_dir.exists(), "the retried finish must reclaim the residue"); // Let the abandoned rename run into its missing source and wait until @@ -18530,7 +19255,7 @@ mod test { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); - let bucket = "bucket"; + let bucket = RUSTFS_META_BUCKET; let object = "object"; ensure_test_volume(&disk, bucket).await; let committed = write_committed_delete_residue(&disk, bucket, object).await; @@ -18549,12 +19274,620 @@ mod test { .await .expect("unproven backup should be written"); - assert_eq!(disk.finish_committed_delete_residue(bucket, object, None).await, 1); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); assert!(!committed.object_dir.join(committed.data_dir.to_string()).exists()); assert!(unproven_backup.exists(), "a backup without its own committed marker must remain"); assert_eq!(recorder.counter_value(METRIC_DISK_DELETE_RESIDUE_FINISHED_TOTAL, &[]), 1); } + #[tokio::test] + async fn delete_quorum_receipt_finishes_inline_backup_after_interrupted_cleanup() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let object_dir = disk.io_get_object_path(bucket, object).expect("object path"); + let transaction = Uuid::new_v4(); + let rollback_path = object_dir.join(transaction.to_string()); + fs::create_dir_all(&rollback_path).await.expect("transaction directory"); + fs::write(rollback_path.join(STORAGE_FORMAT_FILE_BACKUP), b"inline metadata") + .await + .expect("inline rollback backup"); + // Backup alone remains ambiguous. Only the set-quorum receipt permits cleanup. + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 0); + fs::write(rollback_path.join(DELETE_QUORUM_COMMITTED_FILE), b"") + .await + .expect("quorum receipt"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!object_dir.exists(), "inline backup and receipt must be reclaimed"); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_quorum_cleanup_removes_backup_before_marked_data() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "quorum-cleanup-order"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let data_path = residue.part(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel::<()>(); + let _hook = hooks::install_at(hooks::Stage::Remove, &data_path, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let transaction_path = format!("{object}/{}", residue.transaction); + let mut cleanup = Box::pin(disk.delete( + bucket, + &transaction_path, + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + )); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + entered = entered_rx => entered.expect("data cleanup starts"), + result = cleanup.as_mut() => panic!("cleanup returned before data removal: {result:?}"), + } + }) + .await + .expect("cleanup reaches data removal"); + assert!(!residue.backup().exists(), "backup must disappear before data"); + assert!(residue.marker().exists(), "data proof must remain during removal"); + assert!( + residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE) + .is_file() + ); + drop(release); + tokio::time::timeout(Duration::from_secs(10), cleanup) + .await + .expect("cleanup completes") + .expect("cleanup succeeds"); + assert!(!residue.object_dir.exists(), "completed cleanup reclaims every transaction file"); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_rollback_keeps_markers_until_metadata_restore() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "rollback-marker-order"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let backup = residue.backup(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel::<()>(); + let _hook = hooks::install_at(hooks::Stage::Rename, &backup, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let request = residue_delete_request(object); + let mut rollback = Box::pin(disk.delete_versions( + bucket, + request, + DeleteOptions { + undo_write: true, + undo_delete: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + )); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + entered = entered_rx => entered.expect("metadata restore starts"), + result = rollback.as_mut() => panic!("rollback returned before restore: {result:?}"), + } + }) + .await + .expect("rollback reaches metadata restore"); + assert!(residue.marker().exists(), "rollback must retain its marker until metadata is restored"); + assert!(residue.backup().exists(), "backup must remain before restore rename"); + drop(release); + let errors = tokio::time::timeout(Duration::from_secs(10), rollback) + .await + .expect("rollback completes"); + assert!(errors.iter().all(Option::is_none), "{errors:?}"); + assert!(residue.object_dir.join(STORAGE_FORMAT_FILE).is_file()); + assert!(!residue.marker().exists()); + assert!(residue.part().exists()); + } + + #[tokio::test] + async fn delete_rollback_rejects_quorum_committed_transaction() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "quorum-rollback-rejection"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + fs::write( + residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE), + b"", + ) + .await + .expect("quorum receipt"); + let errors = disk + .delete_versions( + bucket, + residue_delete_request(object), + DeleteOptions { + undo_write: true, + undo_delete: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(matches!(errors.as_slice(), [Some(DiskError::FileCorrupt)]), "{errors:?}"); + assert!( + !residue.object_dir.join(STORAGE_FORMAT_FILE).exists(), + "a committed delete must not resurrect metadata" + ); + assert!(residue.backup().exists() && residue.marker().exists()); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_quorum_receipt_sync_failure_preserves_rollback_backup() { + let _mode = durability_mode_override::set(DurabilityMode::Strict); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let rollback_path = residue.object_dir.join(residue.transaction.to_string()); + os::fsync_dir_recorder::set_failure(&rollback_path, ErrorKind::Other); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(result.is_err(), "failed receipt durability must stop cleanup"); + assert!(residue.backup().exists() && residue.part().exists() && residue.marker().exists()); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists(), "retry must complete the already-decided deletion"); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_quorum_backup_removal_sync_failure_retains_receipt() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let _mode = durability_mode_override::set(DurabilityMode::Strict); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let transaction_path = residue.object_dir.join(residue.transaction.to_string()); + let receipt = transaction_path.join(DELETE_QUORUM_COMMITTED_FILE); + let _hook = hooks::install_at(hooks::Stage::Remove, &residue.backup(), move || { + // Arm after the receipt's initial fsync to fail the ordering fence + // between backup unlink and eventual receipt removal. + os::fsync_dir_recorder::set_failure(&transaction_path, ErrorKind::Other); + }); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(result.is_err(), "backup unlink must become durable before removing commitment proof"); + assert!(receipt.is_file() && residue.part().is_file() && residue.marker().is_file()); + assert!(!residue.backup().exists()); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists()); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_quorum_payload_removal_sync_failure_retains_markers_and_receipt() { + let _mode = durability_mode_override::set(DurabilityMode::Strict); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + os::fsync_dir_recorder::set_failure(&residue.object_dir.join(residue.data_dir.to_string()), ErrorKind::Other); + let receipt = residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(result.is_err(), "payload unlink sync failure must stop marker cleanup"); + assert!(receipt.is_file() && residue.marker().is_file()); + assert!(!residue.part().exists()); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists()); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_generic_uuid_backup_directory_does_not_publish_quorum_receipt() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "generic-uuid-delete"; + ensure_test_volume(&disk, bucket).await; + let path = format!("object/{}", Uuid::new_v4()); + let directory = disk.io_get_object_path(bucket, &path).expect("UUID directory"); + fs::create_dir_all(&directory).await.expect("UUID directory"); + fs::write(directory.join("part.1"), b"data").await.expect("part"); + fs::write(directory.join(STORAGE_FORMAT_FILE_BACKUP), b"write transaction backup") + .await + .expect("non-delete rollback backup"); + let receipt = directory.join(DELETE_QUORUM_COMMITTED_FILE); + let (receipt_tx, receipt_rx) = tokio::sync::oneshot::channel(); + let _hook = hooks::install_at(hooks::Stage::Rename, &directory, move || { + let _ = receipt_tx.send(receipt.exists()); + }); + disk.delete( + bucket, + &path, + DeleteOptions { + recursive: true, + immediate: true, + ..Default::default() + }, + ) + .await + .expect("generic directory deletion"); + assert!( + !receipt_rx.await.expect("inspect directory before deletion"), + "ordinary UUID data is not a transaction" + ); + } + + #[tokio::test] + async fn delete_quorum_receipt_preserves_uuid_prefix_child() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let transaction = Uuid::new_v4(); + let transaction_path = format!("{object}/{transaction}"); + let rollback = disk.io_get_object_path(bucket, &transaction_path).expect("transaction path"); + let child = rollback.join("child").join(STORAGE_FORMAT_FILE); + fs::create_dir_all(child.parent().expect("child directory")) + .await + .expect("child directory"); + fs::write(&child, b"live child metadata").await.expect("live child"); + fs::write(rollback.join(DELETE_QUORUM_COMMITTED_FILE), b"") + .await + .expect("receipt"); + fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), b"old backup") + .await + .expect("backup"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 0); + let result = disk + .delete( + bucket, + &transaction_path, + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await; + assert!(matches!(result, Err(DiskError::FileCorrupt)), "{result:?}"); + assert_eq!(fs::read(&child).await.expect("preserved child"), b"live child metadata"); + assert!(rollback.join(STORAGE_FORMAT_FILE_BACKUP).is_file()); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_quorum_cleanup_preserves_child_created_after_classification() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "receipt-prefix-race"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let child = residue + .object_dir + .join(residue.transaction.to_string()) + .join("child") + .join(STORAGE_FORMAT_FILE); + let published_child = child.clone(); + let _hook = hooks::install_at(hooks::Stage::Remove, &residue.part(), move || { + std::fs::create_dir_all(published_child.parent().expect("child directory")).expect("concurrent child directory"); + std::fs::write(&published_child, b"concurrent child").expect("concurrent child metadata"); + }); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + result.expect("cleanup preserves a concurrent child and removes only deleted payload"); + assert_eq!(fs::read(child).await.expect("preserved concurrent child"), b"concurrent child"); + assert!(!residue.part().exists(), "the original deleted payload must still be reclaimed"); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_rollback_backup_sync_failure_precedes_metadata_mutation() { + for mode in [DurabilityMode::Strict, DurabilityMode::None] { + let _mode = durability_mode_override::set(mode); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "backup-durability-gate"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let root = disk.io_get_object_path(bucket, object).expect("object path"); + let data_dir = Uuid::new_v4(); + let data_path = root.join(data_dir.to_string()); + fs::create_dir_all(&data_path).await.expect("data directory"); + fs::write(data_path.join("part.1"), b"original payload") + .await + .expect("payload"); + let original = test_meta(test_file_info(object, Uuid::nil(), Some(data_dir), None)); + fs::write(root.join(STORAGE_FORMAT_FILE), &original) + .await + .expect("original metadata"); + let transaction = Uuid::new_v4(); + let rollback = root.join(transaction.to_string()); + os::fsync_dir_recorder::set_failure(&rollback, ErrorKind::Other); + let errors = disk + .delete_versions( + bucket, + residue_delete_request(object), + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await; + if mode == DurabilityMode::Strict { + assert!(errors.iter().any(Option::is_some), "backup sync failure must stop deletion"); + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.expect("intact metadata"), original); + assert_eq!(fs::read(data_path.join("part.1")).await.expect("intact payload"), b"original payload"); + } else { + assert!(errors.iter().all(Option::is_none), "{errors:?}"); + assert!( + !os::fsync_dir_recorder::was_fsynced(&rollback), + "none mode must skip backup directory sync" + ); + assert!( + os::fsync_dir_recorder::take_failure(&rollback).is_some(), + "none mode must not consume a sync failure" + ); + assert!(!root.join(STORAGE_FORMAT_FILE).exists(), "none mode still applies the requested delete"); + } + } + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_rollback_restore_sync_failure_retains_markers() { + let _mode = durability_mode_override::set(DurabilityMode::Strict); + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "restore-durability-gate"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + os::fsync_dir_recorder::set_failure(&residue.object_dir, ErrorKind::Other); + let opts = DeleteOptions { + undo_write: true, + undo_delete: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }; + let errors = disk + .delete_versions(bucket, residue_delete_request(object), opts.clone()) + .await; + assert!(errors.iter().any(Option::is_some), "restore sync failure must remain visible"); + let bytes = fs::read(residue.object_dir.join(STORAGE_FORMAT_FILE)) + .await + .expect("restored metadata is readable"); + assert_eq!(FileMeta::load(&bytes).expect("valid restored metadata").versions.len(), 1); + assert!(residue.marker().exists(), "failed durability must retain transaction evidence"); + assert!(residue.part().exists(), "rollback must retain payload"); + let errors = disk.delete_versions(bucket, residue_delete_request(object), opts).await; + assert!(errors.iter().all(Option::is_none), "retry must finish rollback: {errors:?}"); + assert!(!residue.marker().exists()); + assert!(residue.object_dir.join(STORAGE_FORMAT_FILE).is_file()); + } + + #[tokio::test] + async fn delete_quorum_receipt_survives_deferred_snapshot_cleanup() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let data_path = format!("{object}/{}", residue.data_dir); + let snapshot = disk + .acquire_snapshot_lease(bucket, &data_path) + .await + .expect("snapshot protects deleted payload"); + let receipt = residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE); + let result = disk + .delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await; + assert!(matches!(result, Err(DiskError::DiskOngoingReq)), "{result:?}"); + assert!(receipt.is_file() && residue.marker().is_file() && residue.part().is_file()); + assert!(!residue.backup().exists(), "committed backup is no longer needed"); + disk.release_snapshot_lease(bucket, &data_path, snapshot) + .await + .expect("snapshot release finishes physical cleanup"); + assert!(!residue.part().exists()); + assert!(receipt.is_file(), "receipt remains until transaction cleanup observes physical removal"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists()); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn delete_quorum_data_cleanup_preserves_concurrent_uuid_prefix_child() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = "quorum-data-prefix-race"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let child = residue + .object_dir + .join(residue.data_dir.to_string()) + .join("child") + .join(STORAGE_FORMAT_FILE); + let published = child.clone(); + let _hook = hooks::install_at(hooks::Stage::Remove, &residue.part(), move || { + std::fs::create_dir_all(published.parent().expect("child directory")).expect("concurrent child directory"); + std::fs::write(published, b"new child").expect("concurrent child metadata"); + }); + disk.delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await + .expect("safe cleanup preserves new descendants"); + assert_eq!(fs::read(child).await.expect("preserved child"), b"new child"); + assert!(!residue.part().exists()); + assert!(!residue.marker().exists()); + } + + #[tokio::test] + async fn delete_quorum_snapshot_replay_preserves_live_uuid_prefix_child() { + let dir = tempfile::tempdir().expect("temporary directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + let bucket = RUSTFS_META_BUCKET; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let data_path = format!("{object}/{}", residue.data_dir); + let snapshot = disk.acquire_snapshot_lease(bucket, &data_path).await.expect("snapshot lease"); + let receipt = residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE); + assert!(matches!( + disk.delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + } + ) + .await, + Err(DiskError::DiskOngoingReq) + )); + let child = residue + .object_dir + .join(residue.data_dir.to_string()) + .join("child") + .join(STORAGE_FORMAT_FILE); + fs::create_dir_all(child.parent().expect("child directory")) + .await + .expect("live child directory"); + fs::write(&child, b"live child").await.expect("live child metadata"); + assert!( + disk.release_snapshot_lease(bucket, &data_path, snapshot).await.is_err(), + "replay must reject mixed live contents" + ); + assert_eq!(fs::read(&child).await.expect("preserved child"), b"live child"); + assert!( + receipt.is_file() && residue.part().is_file(), + "blocked replay retains payload and commitment evidence" + ); + fs::remove_file(&child).await.expect("remove test child metadata"); + fs::remove_dir(child.parent().expect("child directory")) + .await + .expect("remove test child directory"); + assert_eq!(disk.finish_internal_delete_residue(bucket, object, None).await, 1); + assert!(!residue.object_dir.exists()); + } + #[tokio::test] async fn test_delete_version_error_after_staging_restores_data_dir() { use tempfile::tempdir; @@ -21825,6 +23158,8 @@ mod test { undo_delete: false, old_data_dir: None, expected_delete_marker: None, + delete_recovery_bucket_incarnation: None, + recoverable_delete: false, }; disk.delete("test-volume", "test-file.txt", delete_opts) .await diff --git a/crates/ecstore/src/disk/local/delete_recovery.rs b/crates/ecstore/src/disk/local/delete_recovery.rs new file mode 100644 index 000000000..c1b90219e --- /dev/null +++ b/crates/ecstore/src/disk/local/delete_recovery.rs @@ -0,0 +1,3646 @@ +use super::*; +use crate::disk::{ + DELETE_RECOVERY_PROTOCOL_VERSION, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection, + DeleteRecoveryJournal, DeleteRecoveryPage, DeleteRecoveryState, DeleteTransactionId, +}; +use sha2::{Digest, Sha256}; + +pub(super) const PREPARED: &str = "delete-prepared"; +pub(super) const TARGET: &str = "delete-target"; +pub(super) const FINISHED: &str = "delete-commit-finished"; +const MAX_METADATA: u64 = 64 * 1024 * 1024; +struct DiscoverySession { + disk_root: PathBuf, + volume: String, + object: Option, + root: PathBuf, + _mount: std::fs::File, + frames: Vec<(String, std::fs::ReadDir)>, + touched: Instant, +} + +static DISCOVERY_SESSIONS: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| ParkingLotMutex::new(HashMap::new())); + +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct CommitReceipt { + journal: DeleteRecoveryJournal, + finished: bool, +} + +pub(super) struct DeletePreparationContext { + pub transaction: Uuid, + pub bucket_incarnation: Option, + pub namespace_owner: Option>, +} + +pub(super) fn digest(bytes: &[u8]) -> DeleteRecoveryDigest { + Sha256::digest(bytes).into() +} + +pub(in crate::disk) fn generation(mut canonical: FileInfo) -> Result { + canonical.data = None; + canonical.erasure.index = 0; + canonical.erasure.checksums.clear(); + canonical.is_latest = false; + canonical.num_versions = 0; + canonical.successor_mod_time = None; + rustfs_filemeta::shard_integrity::clear_integrity_metadata(&mut canonical.metadata); + // Only schema-declared string fields and the inline presence marker are + // aliases here. Unknown suffixes and opaque binary fields stay distinct. + use rustfs_utils::http::metadata_compat as compat; + for suffix in [ + compat::SUFFIX_INLINE_DATA, + compat::SUFFIX_COMPRESSION, + compat::SUFFIX_COMPRESSION_SIZE, + compat::SUFFIX_ACTUAL_SIZE, + compat::SUFFIX_ACTUAL_OBJECT_SIZE, + compat::SUFFIX_ACTUAL_OBJECT_SIZE_CAP, + compat::SUFFIX_TRANSITION_STATUS, + compat::SUFFIX_TRANSITIONED_OBJECTNAME, + compat::SUFFIX_TRANSITION_TIER, + compat::SUFFIX_TRANSITIONED_VERSION_STATE, + compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + compat::SUFFIX_PURGESTATUS, + compat::SUFFIX_REPLICA_STATUS, + compat::SUFFIX_REPLICATION_STATUS, + compat::SUFFIX_REPLICA_TIMESTAMP, + compat::SUFFIX_REPLICATION_TIMESTAMP, + compat::SUFFIX_TAGGING_TIMESTAMP, + compat::SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, + compat::SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, + ] { + if !compat::contains_key_str(&canonical.metadata, suffix) { + continue; + } + // The inline marker is a presence flag whose persisted value is empty. + // Compare both declared aliases explicitly before normalizing it; the + // general compatibility reader rejects empty strings by design. + let value = if suffix == compat::SUFFIX_INLINE_DATA { + let rustfs_key = compat::internal_key_rustfs(suffix); + let minio_key = format!("{}{suffix}", compat::MINIO_INTERNAL_PREFIX); + let mut value = None; + for (key, candidate) in &canonical.metadata { + if !key.eq_ignore_ascii_case(&rustfs_key) && !key.eq_ignore_ascii_case(&minio_key) { + continue; + } + if value.is_some_and(|current: &String| current != candidate) { + return Err(DiskError::FileCorrupt); + } + value = Some(candidate); + } + value.ok_or(DiskError::FileCorrupt)?.clone() + } else { + compat::get_consistent_str(&canonical.metadata, suffix) + .ok_or(DiskError::FileCorrupt)? + .to_owned() + }; + if value.is_empty() && suffix != compat::SUFFIX_INLINE_DATA { + continue; + } + compat::insert_str(&mut canonical.metadata, suffix, value); + } + fn ordered(value: serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(map) => { + let sorted: std::collections::BTreeMap<_, _> = map.into_iter().collect(); + serde_json::Value::Object(sorted.into_iter().map(|(key, value)| (key, ordered(value))).collect()) + } + serde_json::Value::Array(values) => serde_json::Value::Array(values.into_iter().map(ordered).collect()), + other => other, + } + } + Ok(crate::disk::DeleteRecoveryVersionIdentity { + version_id: canonical.version_id, + mod_time: canonical.mod_time, + data_dir: canonical.data_dir, + deleted: canonical.deleted, + metadata_digest: digest(&serde_json::to_vec(&ordered(serde_json::to_value(canonical)?))?), + }) +} + +fn safe_path(path: &str, empty: bool) -> bool { + (empty && path.is_empty()) + || (!path.is_empty() && !path.starts_with('/') && crate::bucket::utils::is_valid_object_prefix(path)) +} + +fn regular_metadata_bytes(path: &Path) -> Result>> { + match std::fs::symlink_metadata(path) { + Ok(meta) if meta.is_file() => Ok(Some(std::fs::read(path).map_err(to_file_error)?)), + Ok(_) => Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), + Err(err) => Err(to_file_error(err).into()), + } +} + +fn regular_bytes(path: &Path) -> Result>> { + if std::fs::symlink_metadata(path).is_ok_and(|meta| meta.len() > MAX_METADATA) { + return Err(DiskError::FileCorrupt); + } + regular_metadata_bytes(path) +} + +fn journal_bytes(journal: &DeleteRecoveryJournal) -> Result> { + let bytes = serde_json::to_vec(journal)?; + // The same journal is carried by adoption requests and decision receipts. + if bytes.len() > usize::try_from(MAX_METADATA).map_err(|_| DiskError::FileCorrupt)? - 128 { + return Err(DiskError::FileCorrupt); + } + let projected = crate::disk::DeleteRecoveryRpcReply::Inspected(DeleteRecoveryInspection { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + id: DeleteTransactionId { + object: journal.object.clone(), + transaction: journal.transaction, + }, + state: DeleteRecoveryState::NewPrepared, + journal: Some(journal.clone()), + current_digest: journal.original_digest, + current_versions: journal.original_versions.clone(), + backup_digest: journal.original_digest, + rollback_data_complete: true, + commit_data_safe: true, + explicit_delete_safe: false, + fingerprint: [0; 32], + }); + if rmp_serde::to_vec_named(&projected)?.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES - 4096 { + return Err(DiskError::FileCorrupt); + } + Ok(bytes) +} + +fn regular_marker(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(meta) if meta.is_file() && meta.len() == 0 => Ok(true), + Ok(_) => Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(false), + Err(err) => Err(to_file_error(err).into()), + } +} + +fn complete_part_sources(live: &Path, staged: &Path, info: &FileInfo) -> Result { + let layout = info.validate(ValidationMode::RequireErasure)?.ok_or(DiskError::FileCorrupt)?; + for part in &info.parts { + let expected = u64::try_from(layout.shard_file_size(part.size).ok_or(DiskError::FileCorrupt)?) + .map_err(|_| DiskError::FileCorrupt)?; + let name = format!("part.{}", part.number); + if ![live, staged] + .iter() + .any(|root| std::fs::symlink_metadata(root.join(&name)).is_ok_and(|meta| meta.is_file() && meta.len() >= expected)) + { + return Ok(false); + } + if let Some(integrity) = &part.integrity { + let name = integrity.file_name(); + let length = u64::try_from(integrity.index_size()?).map_err(|_| DiskError::FileCorrupt)?; + if ![live, staged] + .iter() + .any(|root| std::fs::symlink_metadata(root.join(&name)).is_ok_and(|meta| meta.is_file() && meta.len() == length)) + { + return Ok(false); + } + } + } + Ok(true) +} + +async fn verify_recovery_integrity_sources( + live: &Path, + staged: &Path, + integrity: &rustfs_filemeta::shard_integrity::PartIntegrity, + coding_index: usize, + algorithm: HashAlgorithm, + owner: Option>, +) -> Result<()> { + let lease = os::acquire_namespace_mutation_lease_with_owner(live, owner).await; + let roots = [live.to_path_buf(), staged.to_path_buf()]; + let integrity = integrity.clone(); + os::run_blocking_namespace_operation(lease, move || { + Ok((|| { + integrity.validate()?; + if coding_index >= integrity.layout.shards() + || !matches!(algorithm, HashAlgorithm::HighwayHash256S | HashAlgorithm::HighwayHash256SLegacy) + { + return Err(DiskError::FileCorrupt); + } + let codec = coding::Erasure::try_new_with_options( + usize::from(integrity.layout.data), + usize::from(integrity.layout.parity), + usize::try_from(integrity.layout.block_size).map_err(|_| DiskError::FileCorrupt)?, + integrity.layout.legacy, + ) + .map_err(DiskError::from)?; + let shard_size = codec.shard_size(); + let part_size = + usize::try_from(codec.shard_file_size(i64::try_from(integrity.size).map_err(|_| DiskError::FileCorrupt)?)) + .map_err(|_| DiskError::FileCorrupt)?; + let encoded_size = u64::try_from(coding::bitrot_shard_file_size(part_size, shard_size, algorithm.clone())) + .map_err(|_| DiskError::FileCorrupt)?; + let expected = u64::try_from(integrity.index_size()?).map_err(|_| DiskError::FileCorrupt)?; + let header = integrity.index_header(); + let mut checked = false; + for root in &roots { + let path = root.join(integrity.file_name()); + match std::fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_file() && metadata.len() == expected => {} + Ok(_) => return Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => return Err(to_file_error(err).into()), + } + let mut file = std::fs::File::open(path).map_err(to_file_error)?; + let mut actual = [0u8; rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE]; + std::io::Read::read_exact(&mut file, &mut actual).map_err(to_file_error)?; + if actual != header { + return Err(DiskError::FileCorrupt); + } + // An interrupted restore can split payload and proof between + // live and staged roots. Verify every present payload copy + // against the declared proof, independently of its location. + let mut shards = Vec::new(); + for root in &roots { + let path = root.join(format!("part.{}", integrity.number)); + match std::fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_file() && metadata.len() == encoded_size => { + shards.push(std::fs::File::open(path).map_err(to_file_error)?); + } + Ok(_) => return Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if shards.is_empty() { + return Err(DiskError::FileCorrupt); + } + let mut record = vec![0; integrity.record_size()]; + let mut payload = vec![0; shard_size]; + let mut bitrot_hash = vec![0; algorithm.size()]; + let mut remaining = part_size; + for stripe in 0..integrity.stripes { + std::io::Read::read_exact(&mut file, &mut record).map_err(to_file_error)?; + integrity.verify_record(stripe, &record)?; + let length = remaining.min(shard_size); + if length == 0 { + return Err(DiskError::FileCorrupt); + } + for shard in &mut shards { + std::io::Read::read_exact(shard, &mut bitrot_hash).map_err(to_file_error)?; + std::io::Read::read_exact(shard, &mut payload[..length]).map_err(to_file_error)?; + integrity.verify_shard(stripe, coding_index, &payload[..length], &record)?; + } + remaining -= length; + } + if remaining != 0 { + return Err(DiskError::FileCorrupt); + } + if integrity.stripes == 0 && integrity.root != integrity.root_digest(&integrity.padding_digest(0)) { + return Err(DiskError::FileCorrupt); + } + checked = true; + } + if !checked { + return Err(DiskError::FileCorrupt); + } + Ok(()) + })()) + }) + .await + .map_err(to_file_error)? +} + +struct ExplicitDeletePlan { + data_dirs: Vec, + proof_files: Vec, +} + +fn explicit_delete_plan( + root: &Path, + transaction: Uuid, + original: Option<&FileMeta>, + journal: Option<&DeleteRecoveryJournal>, + current_missing: bool, +) -> Result> { + if !current_missing + || journal + .is_some_and(|journal| journal.target_digest.is_some() || (journal.original_digest.is_some() && original.is_none())) + { + return Ok(None); + } + let mut referenced = HashSet::new(); + if let Some(metadata) = original { + if metadata.versions.len() != 1 || metadata.versions[0].header.version_type != rustfs_filemeta::VersionType::Object { + return Ok(None); + } + let info = metadata.versions[0].into_fileinfo("", "", false)?; + if info.version_id.is_some_and(|id| !id.is_nil()) + || info.deleted + || info.is_remote() + || info.metadata.keys().any(|key| { + let key = key.to_ascii_lowercase(); + key.starts_with("x-amz-object-lock-") + || rustfs_utils::http::metadata_compat::internal_key_strip_suffix_prefix(&key, "objectlock-").is_some() + }) + { + return Ok(None); + } + referenced.extend(metadata.get_data_dirs()?.into_iter().flatten().filter(|id| !id.is_nil())); + } + let rollback = root.join(transaction.to_string()); + let marker_names = [ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .map(|prefix| format!("{prefix}{transaction}")); + let mut data_dirs = Vec::new(); + let mut proof_files = Vec::new(); + let mut candidates = Vec::new(); + match std::fs::read_dir(&rollback) { + Ok(entries) => { + for entry in entries { + let entry = entry.map_err(to_file_error)?; + let kind = entry.file_type().map_err(to_file_error)?; + let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + if kind.is_dir() && Uuid::parse_str(&name).is_ok_and(|id| referenced.contains(&id)) { + candidates.push(entry.path()); + } else if kind.is_file() + && [ + STORAGE_FORMAT_FILE_BACKUP, + DELETE_MARKER_ROLLBACK_FILE, + DELETE_QUORUM_COMMITTED_FILE, + PREPARED, + TARGET, + FINISHED, + ] + .contains(&name.as_str()) + { + proof_files.push(entry.path()); + } else if kind.is_file() + && (marker_names.contains(&name) || (delete_recovery_part_file(&name) && referenced.contains(&transaction))) + { + candidates.push(rollback.clone()); + } else { + return Ok(None); + } + } + } + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + match std::fs::read_dir(root) { + Ok(entries) => { + for entry in entries { + let entry = entry.map_err(to_file_error)?; + let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + let Some(id) = Uuid::parse_str(&name).ok().filter(|id| !id.is_nil()) else { + continue; + }; + if !entry.file_type().map_err(to_file_error)?.is_dir() { + continue; + } + let own_marker = marker_names + .iter() + .any(|name| std::fs::symlink_metadata(entry.path().join(name)).is_ok_and(|metadata| metadata.is_file())); + if referenced.contains(&id) || own_marker { + candidates.push(entry.path()); + } + } + } + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + candidates.sort(); + candidates.dedup(); + for dir in candidates { + let mut has_parts = false; + for entry in std::fs::read_dir(&dir).map_err(to_file_error)? { + let entry = entry.map_err(to_file_error)?; + if !entry.file_type().map_err(to_file_error)?.is_file() { + return Ok(None); + } + let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + if delete_recovery_part_file(&name) { + has_parts = true; + } else if !(marker_names.contains(&name) + || (dir == rollback + && [ + STORAGE_FORMAT_FILE_BACKUP, + DELETE_MARKER_ROLLBACK_FILE, + DELETE_QUORUM_COMMITTED_FILE, + PREPARED, + TARGET, + FINISHED, + ] + .contains(&name.as_str()))) + { + return Ok(None); + } + } + if has_parts && original.is_none() { + return Ok(None); + } + data_dirs.push(dir); + } + data_dirs.sort_by_key(|path| std::cmp::Reverse(path.components().count())); + Ok(Some(ExplicitDeletePlan { data_dirs, proof_files })) +} + +fn inspection(root: &Path, volume: &str, object: &str, transaction: Uuid) -> Result { + let rollback = root.join(transaction.to_string()); + let before = regular_metadata_bytes(&rollback.join(STORAGE_FORMAT_FILE_BACKUP))?; + let current = regular_metadata_bytes(&root.join(STORAGE_FORMAT_FILE))?; + let prepared = regular_bytes(&rollback.join(PREPARED))?; + let target = regular_metadata_bytes(&rollback.join(TARGET))?; + let receipt_bytes = regular_bytes(&rollback.join(DELETE_QUORUM_COMMITTED_FILE))?; + let receipt: Option = receipt_bytes + .as_deref() + .filter(|bytes| !bytes.is_empty()) + .map(serde_json::from_slice) + .transpose()?; + let committed = receipt.is_some(); + let finished = regular_marker(&rollback.join(FINISHED))? || receipt.as_ref().is_some_and(|receipt| receipt.finished); + let journal: Option = prepared + .as_deref() + .map(serde_json::from_slice) + .transpose()? + .or_else(|| receipt.as_ref().map(|receipt| receipt.journal.clone())); + if receipt + .as_ref() + .is_some_and(|receipt| journal.as_ref() != Some(&receipt.journal)) + { + return Err(DiskError::FileCorrupt); + } + if let Some(journal) = &journal + && (journal.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || journal.volume != volume + || journal.object != object + || journal.transaction != transaction + || journal.transaction.is_nil() + || journal.data_dirs.iter().any(|id| id.is_nil() || *id == transaction)) + { + return Err(DiskError::FileCorrupt); + } + let backup_digest = before.as_deref().map(digest); + let current_digest = current.as_deref().map(digest); + let prepared_digest = prepared.as_deref().map(digest); + let target_digest = target.as_deref().map(digest); + let receipt_digest = receipt_bytes.as_deref().map(digest); + if let Some(journal) = &journal { + if (!finished + && journal.target_digest != target_digest + && !(target.is_none() + && !committed + && before.is_none() + && journal.original_digest.is_some() + && current_digest == journal.original_digest)) + || target_digest.is_some_and(|hash| journal.target_digest != Some(hash)) + || backup_digest.is_some_and(|hash| journal.original_digest != Some(hash)) + { + return Err(DiskError::FileCorrupt); + } + if let Some(bytes) = &target { + FileMeta::load(bytes)?; + } + } + let current_meta = current.as_deref().map(FileMeta::load).transpose()?; + let before_meta = before.as_deref().map(FileMeta::load).transpose()?; + let mut fingerprint = Sha256::new(); + for hash in [prepared_digest, target_digest, backup_digest, current_digest, receipt_digest] { + fingerprint.update([u8::from(hash.is_some())]); + if let Some(hash) = hash { + fingerprint.update(hash); + } + } + fingerprint.update([u8::from(committed), u8::from(finished)]); + let mut any_marker = false; + let mut descriptions = Vec::new(); + let entries = match std::fs::read_dir(root) { + Ok(entries) => Some(entries), + Err(err) if err.kind() == ErrorKind::NotFound => None, + Err(err) => return Err(to_file_error(err).into()), + }; + if let Some(entries) = entries { + for entry in entries { + let entry = entry.map_err(to_file_error)?; + let kind = entry.file_type().map_err(to_file_error)?; + if !kind.is_file() && !kind.is_dir() { + return Err(DiskError::InvalidPath); + } + let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + descriptions.push((name.clone(), kind.is_dir(), entry.metadata().map_err(to_file_error)?.len())); + if kind.is_dir() && Uuid::parse_str(&name).is_ok_and(|id| !id.is_nil()) { + for child in std::fs::read_dir(entry.path()).map_err(to_file_error)? { + let child = child.map_err(to_file_error)?; + let kind = child.file_type().map_err(to_file_error)?; + if !kind.is_dir() && !kind.is_file() { + return Err(DiskError::InvalidPath); + } + let child_name = child.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + let metadata = child.metadata().map_err(to_file_error)?; + descriptions.push((format!("{name}/{child_name}"), kind.is_dir(), metadata.len())); + if kind.is_file() + && [ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .iter() + .any(|prefix| child_name == format!("{prefix}{transaction}")) + { + any_marker = true; + } + let staged_id = Uuid::parse_str(&child_name).ok(); + let known_staged = kind.is_dir() + && staged_id.is_some_and(|dir| { + journal.as_ref().is_some_and(|journal| journal.data_dirs.contains(&dir)) + || before_meta + .as_ref() + .is_some_and(|metadata| metadata.get_data_dirs().is_ok_and(|dirs| dirs.contains(&Some(dir)))) + }); + if name == transaction.to_string() + && !(kind.is_dir() && known_staged) + && !(journal.is_none() + && kind.is_file() + && (delete_recovery_part_file(&child_name) + || [DELETE_DATA_DIR_MARKER_PREFIX, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX] + .iter() + .any(|prefix| child_name == format!("{prefix}{transaction}")))) + && (kind.is_dir() + || ![ + PREPARED, + TARGET, + FINISHED, + STORAGE_FORMAT_FILE_BACKUP, + DELETE_MARKER_ROLLBACK_FILE, + DELETE_QUORUM_COMMITTED_FILE, + ] + .contains(&child_name.as_str())) + { + return Err(DiskError::FileCorrupt); + } + if name == transaction.to_string() && kind.is_dir() { + for file in std::fs::read_dir(child.path()).map_err(to_file_error)? { + let file = file.map_err(to_file_error)?; + let kind = file.file_type().map_err(to_file_error)?; + let file_name = file.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?; + if !kind.is_file() + || (!delete_recovery_part_file(&file_name) + && ![ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .iter() + .any(|prefix| file_name == format!("{prefix}{transaction}"))) + { + return Err(DiskError::FileCorrupt); + } + descriptions.push(( + format!("{name}/{child_name}/{file_name}"), + false, + file.metadata().map_err(to_file_error)?.len(), + )); + } + } + } + } + } + } + descriptions.sort(); + fingerprint.update(serde_json::to_vec(&descriptions)?); + let mut rollback_data_complete = false; + let mut commit_data_safe = false; + if let Some(journal) = &journal { + rollback_data_complete = (journal.original_digest == backup_digest || current_digest == journal.original_digest) + && (current_digest == journal.original_digest || current_digest == journal.target_digest); + if let Some(meta) = before_meta + .as_ref() + .or_else(|| current_meta.as_ref().filter(|_| current_digest == journal.original_digest)) + { + let inline: HashMap<&str, &[u8]> = if meta.data.as_slice().is_empty() { + HashMap::new() + } else { + if !meta.data.version_ok() { + return Err(DiskError::FileCorrupt); + } + rmp_serde::from_slice(meta.data.after_version())? + }; + for version in &meta.versions { + if version.header.version_type != rustfs_filemeta::VersionType::Object { + continue; + } + let mut info = version.into_fileinfo(volume, object, true)?; + info.hydrate_shard_integrity()?; + if info.is_remote() { + continue; + } + if info.inline_data() { + let id = info.version_id.unwrap_or_default(); + rollback_data_complete &= + inline.contains_key(id.to_string().as_str()) || (id.is_nil() && inline.contains_key("null")); + continue; + } + let Some(data_dir) = info.data_dir.filter(|id| !id.is_nil()) else { + rollback_data_complete = false; + continue; + }; + rollback_data_complete &= + complete_part_sources(&root.join(data_dir.to_string()), &rollback.join(data_dir.to_string()), &info)?; + } + } + commit_data_safe = current_digest == journal.original_digest || current_digest == journal.target_digest; + if current_digest != journal.original_digest && current_digest != journal.target_digest { + // A later certified delete can remove the remaining metadata of + // this committed version delete. Absence has no live references, + // but cannot itself authorize an undecided transaction. + commit_data_safe = (committed && current.is_none()) + || current_meta.as_ref().is_some_and(|meta| { + meta.get_data_dirs() + .is_ok_and(|dirs| !dirs.into_iter().flatten().any(|dir| journal.data_dirs.contains(&dir))) + && !meta.versions.iter().any(|version| { + version + .header + .version_id + .is_some_and(|id| !id.is_nil() && journal.deleted_versions.contains(&id)) + }) + }); + } + } + // In-place shard repair leaves a backup in the live data directory. + // Preserve it without interpreting it as an unfinished delete. + let live_write_backup = journal.is_none() + && !any_marker + && !regular_marker(&rollback.join(DELETE_MARKER_ROLLBACK_FILE))? + && prepared.is_none() + && target.is_none() + && receipt_bytes.is_none() + && !finished + && current_meta.as_ref().is_some_and(|meta| { + meta.get_data_dirs() + .is_ok_and(|dirs| dirs.into_iter().flatten().any(|dir| dir == transaction)) + }) + && before_meta.as_ref().is_some_and(|meta| { + meta.get_data_dirs() + .is_ok_and(|dirs| dirs.into_iter().flatten().any(|dir| dir == transaction)) + }); + let state = if live_write_backup { + DeleteRecoveryState::Absent + } else if journal.is_some() { + if committed { + if finished { + DeleteRecoveryState::CommitFinished + } else { + DeleteRecoveryState::QuorumCommitted + } + } else { + DeleteRecoveryState::NewPrepared + } + } else if before.is_some() || any_marker || receipt_bytes.is_some() || prepared.is_some() || target.is_some() { + DeleteRecoveryState::LegacyAmbiguous + } else { + DeleteRecoveryState::Absent + }; + let explicit_delete_safe = + explicit_delete_plan(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none())?.is_some(); + Ok(DeleteRecoveryInspection { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + id: DeleteTransactionId { + object: object.to_owned(), + transaction, + }, + state, + journal, + current_digest, + backup_digest, + rollback_data_complete, + commit_data_safe, + explicit_delete_safe, + current_versions: current_meta + .as_ref() + .map(|meta| { + meta.versions + .iter() + .map(|version| generation(version.into_fileinfo(volume, object, true)?)) + .collect::>>() + }) + .transpose()? + .unwrap_or_default(), + fingerprint: fingerprint.finalize().into(), + }) +} + +impl LocalDisk { + async fn delete_recovery_predecessors( + &self, + volume: &str, + object: &str, + transaction: Uuid, + incarnation: Option, + owner: Option>, + ) -> Result> { + let root = self.io_get_object_path(volume, object)?; + let volume = volume.to_owned(); + let object = object.to_owned(); + let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await; + os::run_blocking_namespace_operation(lease, move || { + Ok((|| { + let entries = match std::fs::read_dir(&root) { + Ok(entries) => entries, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Vec::new()), + Err(err) => return Err(to_file_error(err).into()), + }; + let mut predecessors = BTreeSet::new(); + let mut visited = 0usize; + for entry in entries { + visited += 1; + if visited > 65536 { + return Err(DiskError::DiskOngoingReq); + } + let entry = entry.map_err(to_file_error)?; + let Some(id) = entry + .file_name() + .to_str() + .and_then(|name| Uuid::parse_str(name).ok()) + .filter(|id| !id.is_nil() && *id != transaction) + else { + continue; + }; + if !entry.file_type().map_err(to_file_error)?.is_dir() { + return Err(DiskError::InvalidPath); + } + let prepared = regular_bytes(&entry.path().join(PREPARED))?; + let receipt = regular_bytes(&entry.path().join(DELETE_QUORUM_COMMITTED_FILE))?; + let mut records = Vec::new(); + if let Some(bytes) = prepared { + records.push(serde_json::from_slice::(&bytes)?); + } + if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) { + records.push(serde_json::from_slice::(&bytes)?.journal); + } + for record in records { + if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || record.volume != volume + || record.object != object + || record.transaction != id + || (incarnation.is_some() + && record.bucket_incarnation.is_some() + && record.bucket_incarnation != incarnation) + || record.predecessors.iter().any(|before| before.is_nil() || *before == id) + { + return Err(DiskError::FileCorrupt); + } + predecessors.insert(id); + } + } + Ok(predecessors.into_iter().collect()) + })()) + }) + .await + .map_err(to_file_error)? + } + pub(super) async fn delete_recovery_discover( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + if !(1..=256).contains(&limit) + || !safe_path(volume, false) + || volume.contains('/') + || object.is_some_and(|object| !safe_path(object, false)) + { + return Err(DiskError::InvalidPath); + } + let bucket_path = match object { + Some(object) => self.io_get_object_path(volume, object)?, + None => self.io_get_bucket_path(volume)?, + }; + let disk_root = self.io_root().to_path_buf(); + let (token, mut session) = if let Some(cursor) = cursor { + let session = DISCOVERY_SESSIONS + .lock() + .remove(&cursor.token) + .ok_or(DiskError::FileNotFound)?; + if session.disk_root != disk_root + || session.volume != volume + || session.object.as_deref() != object + || session.touched.elapsed() > Duration::from_secs(300) + { + return Err(DiskError::InvalidPath); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let before = session._mount.metadata().map_err(to_file_error)?; + #[cfg(target_os = "linux")] + let current = self.mount_lease.metadata().map_err(to_file_error)?; + #[cfg(not(target_os = "linux"))] + let current = std::fs::metadata(self.io_root()).map_err(to_file_error)?; + if before.dev() != current.dev() || before.ino() != current.ino() { + return Err(DiskError::InvalidPath); + } + } + (cursor.token, session) + } else { + #[cfg(target_os = "linux")] + let (mount, root) = { + use std::os::fd::AsRawFd; + let mount = self.mount_lease.try_clone().map_err(to_file_error)?; + let root = PathBuf::from(format!("/proc/self/fd/{}/.", mount.as_raw_fd())).join(volume); + let root = match object { + Some(object) => root.join(object), + None => root, + }; + (mount, root) + }; + #[cfg(windows)] + let (mount, root) = { + use std::os::windows::fs::OpenOptionsExt; + use windows_sys::Win32::Storage::FileSystem::{ + FILE_FLAG_BACKUP_SEMANTICS, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_TRAVERSE, + }; + // Match the existing publication-root directory open contract. + let mount = std::fs::OpenOptions::new() + .access_mode(FILE_TRAVERSE | FILE_READ_ATTRIBUTES) + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS) + .open(self.io_root()) + .map_err(to_file_error)?; + (mount, bucket_path.clone()) + }; + #[cfg(all(not(target_os = "linux"), not(windows)))] + let (mount, root) = (std::fs::File::open(self.io_root()).map_err(to_file_error)?, bucket_path.clone()); + ( + Uuid::new_v4(), + DiscoverySession { + disk_root, + volume: volume.to_owned(), + object: object.map(str::to_owned), + root, + _mount: mount, + frames: Vec::new(), + touched: Instant::now(), + }, + ) + }; + let lease = os::acquire_namespace_mutation_lease_with_owner(&bucket_path, None).await; + os::run_blocking_namespace_operation(lease, move || { + if session.frames.is_empty() { + match std::fs::read_dir(&session.root) { + Ok(entries) => session.frames.push((String::new(), entries)), + Err(err) if err.kind() == ErrorKind::NotFound => { + return Ok(DeleteRecoveryPage { + transactions: Vec::new(), + next: None, + }); + } + Err(err) => return Err(err), + } + } + let mut transactions = BTreeSet::new(); + let mut visited = 0usize; + while !session.frames.is_empty() + && transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))? + && visited < 4096 + { + let (parent, entries) = session + .frames + .last_mut() + .ok_or_else(|| std::io::Error::from(ErrorKind::InvalidData))?; + let entry = match entries.next() { + Some(entry) => entry?, + None => { + session.frames.pop(); + continue; + } + }; + let name = entry + .file_name() + .into_string() + .map_err(|_| std::io::Error::from(ErrorKind::InvalidData))?; + if !safe_path(&name, false) || name.contains('/') { + return Err(std::io::Error::from(ErrorKind::InvalidData)); + } + let relative = if parent.is_empty() { + name.clone() + } else { + format!("{parent}/{name}") + }; + let path = session.root.join(&relative); + reject_local_disk_symlink_components(&session.root, &path).map_err(std::io::Error::from)?; + let kind = match std::fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => return Err(err), + }; + if !kind.is_dir() && !kind.is_file() { + return Err(std::io::Error::from(ErrorKind::InvalidData)); + } + visited += 1; + if kind.is_file() { + let bound = match session.object.as_deref() { + Some(object) if !parent.is_empty() && !parent.contains('/') => Some((object, parent.as_str())), + Some(_) => None, + None => parent.rsplit_once('/'), + }; + if let Some((object, id)) = bound { + let transaction = + if [PREPARED, STORAGE_FORMAT_FILE_BACKUP, DELETE_QUORUM_COMMITTED_FILE].contains(&name.as_str()) { + Uuid::parse_str(id).ok().filter(|id| !id.is_nil()) + } else { + [ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] + .iter() + .find_map(|prefix| delete_transaction_marker_id(&name, prefix)) + }; + if let Some(transaction) = transaction { + let physical = match session.object.as_ref() { + Some(_) => session.root.clone(), + None => session.root.join(object), + }; + let proof_root = physical.join(transaction.to_string()); + let prepared = regular_bytes(&proof_root.join(PREPARED)).map_err(std::io::Error::from)?; + let receipt = + regular_bytes(&proof_root.join(DELETE_QUORUM_COMMITTED_FILE)).map_err(std::io::Error::from)?; + let record = match prepared { + Some(bytes) => { + Some(serde_json::from_slice::(&bytes).map_err(std::io::Error::other)?) + } + None => receipt + .filter(|bytes| !bytes.is_empty()) + .map(|bytes| serde_json::from_slice::(&bytes).map(|receipt| receipt.journal)) + .transpose() + .map_err(std::io::Error::other)?, + }; + let logical = if let Some(journal) = record { + let bound_path = match session.object.as_ref() { + Some(scope) if scope == &journal.object => physical.clone(), + Some(_) => return Err(std::io::Error::from(ErrorKind::InvalidData)), + None => build_local_disk_object_path(&session.root, "", &journal.object).1, + }; + if journal.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || journal.volume != session.volume + || journal.transaction != transaction + || !safe_path(&journal.object, false) + || bound_path != physical + { + return Err(std::io::Error::from(ErrorKind::InvalidData)); + } + journal.object + } else { + object.to_owned() + }; + transactions.insert((logical, transaction)); + } + } + } else { + if session.object.is_some() && (!parent.is_empty() || !Uuid::parse_str(&name).is_ok_and(|id| !id.is_nil())) { + continue; + } + // A valid 1024-byte key can exceed 256 one-byte path + // components; allow its transaction/data directories too. + if session.frames.len() >= 1024 { + return Err(std::io::Error::from(ErrorKind::InvalidData)); + } + session.frames.push((relative, std::fs::read_dir(path)?)); + } + } + let next = if session.frames.is_empty() { + None + } else { + session.touched = Instant::now(); + let mut sessions = DISCOVERY_SESSIONS.lock(); + sessions.retain(|_, session| session.touched.elapsed() <= Duration::from_secs(300)); + if sessions.len() >= 128 { + return Err(std::io::Error::from(ErrorKind::WouldBlock)); + } + sessions.insert(token, session); + Some(DeleteRecoveryCursor { token }) + }; + Ok(DeleteRecoveryPage { + transactions: transactions + .into_iter() + .map(|(object, transaction)| DeleteTransactionId { object, transaction }) + .collect(), + next, + }) + }) + .await + .map_err(|err| to_file_error(err).into()) + } + + pub(super) async fn delete_recovery_inspect( + &self, + volume: &str, + object: &str, + transaction: Uuid, + ) -> Result { + if !safe_path(object, false) || transaction.is_nil() { + return Err(DiskError::InvalidPath); + } + let object_path = self.get_object_path(volume, object)?; + let owner: Option> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await); + self.delete_recovery_inspect_owned(volume, object, transaction, owner).await + } + + pub(super) async fn delete_recovery_inspect_owned( + &self, + volume: &str, + object: &str, + transaction: Uuid, + owner: Option>, + ) -> Result { + let root = self.io_get_object_path(volume, object)?; + self.io_get_object_path(volume, &format!("{object}/{transaction}"))?; + let volume = volume.to_owned(); + let object = object.to_owned(); + let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await; + os::run_blocking_namespace_operation(lease, move || Ok(inspection(&root, &volume, &object, transaction))) + .await + .map_err(to_file_error)? + } + + async fn activate_delete_preparation( + &self, + journal: &DeleteRecoveryJournal, + original: Option<&[u8]>, + target: Option>, + owner: Option>, + ) -> Result>> { + let volume = journal.volume.as_str(); + let object = journal.object.as_str(); + let transaction = journal.transaction; + let journal = journal_bytes(journal)?; + self.wait_for_startup_cleanup().await; + let temporary = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?; + let root = self.io_get_object_path(volume, object)?; + let destination = root.join(transaction.to_string()); + let original = original.map(<[u8]>::to_vec); + let sync = effective_durability(volume).syncs_commit_metadata(); + #[cfg(test)] + let record_destination = destination.clone(); + let staging_root = temporary.clone(); + let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, owner.clone()).await; + let (staged, target) = os::run_blocking_namespace_operation(lease, move || { + std::fs::create_dir_all(&staging_root)?; + let staged = Arc::new( + tempfile::Builder::new() + .prefix("delete-preparation-") + .tempdir_in(&staging_root)?, + ); + let missing_marker = original.is_none() && target.is_some(); + for (name, bytes) in [ + (STORAGE_FORMAT_FILE_BACKUP, original.as_deref()), + (TARGET, target.as_deref()), + (PREPARED, Some(journal.as_slice())), + (DELETE_MARKER_ROLLBACK_FILE, missing_marker.then_some(&[][..])), + ] { + let Some(bytes) = bytes else { + continue; + }; + #[cfg(test)] + run_owned_file_write_before_open(&record_destination.join(name)); + let mut file = std::fs::File::create(staged.path().join(name))?; + std::io::Write::write_all(&mut file, bytes)?; + if sync { + file.sync_all()?; + } + } + if sync { + os::fsync_dir_std(staged.path())?; + } + Ok((staged, target)) + }) + .await + .map_err(to_file_error)?; + self.delete_recovery_barrier(volume, object, transaction, "PREACTIVATION") + .await?; + match fs::symlink_metadata(&destination).await { + Ok(_) => return Err(DiskError::OutdatedXLMeta), + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + // The directory owner accompanies the blocking rename, so cancelling + // its waiter cannot delete staged files while activation is running. + let publication_owner: Option> = Some(Arc::new((owner.clone(), staged.clone()))); + os::rename_all_with_owner( + staged.path(), + &destination, + &self.io_get_bucket_path(volume)?, + &self.publication_root, + publication_owner, + ) + .await?; + if sync { + let bucket = self.io_get_bucket_path(volume)?; + let mut path = root.as_path(); + loop { + os::fsync_dir_with_owner(path, owner.clone()).await.map_err(to_file_error)?; + if path == bucket { + break; + } + path = path.parent().ok_or(DiskError::InvalidPath)?; + } + os::fsync_dir_with_owner(&temporary, owner.clone()) + .await + .map_err(to_file_error)?; + } + self.delete_recovery_barrier(volume, object, transaction, "PREPARED").await?; + Ok(target) + } + + pub(super) async fn prepare_delete_recovery( + &self, + volume: &str, + object: &str, + original: Option<&[u8]>, + requests: &[FileInfo], + authority: Option<&DeleteRecoveryJournal>, + context: DeletePreparationContext, + ) -> Result>> { + let DeletePreparationContext { + transaction, + bucket_incarnation: incarnation, + namespace_owner: owner, + } = context; + let mut plan = original.map(FileMeta::load).transpose()?.unwrap_or_else(FileMeta::new); + let mut deleted_headers: Vec<_> = plan + .versions + .iter() + .map(|version| (version.header.version_id, std::mem::discriminant(&version.header.version_type))) + .collect(); + let mut original_versions = Vec::new(); + for version in &plan.versions { + original_versions.push(generation(version.into_fileinfo(volume, object, true)?)?); + } + let mut data_dirs = Vec::with_capacity(requests.len()); + for request in requests { + if original.is_none() { + plan.add_version(request.clone())?; + continue; + } + match plan.delete_version(request) { + Ok(Some(dir)) => { + let _ = plan.data.remove(vec![request.version_id.unwrap_or_default(), dir]); + data_dirs.push(dir); + } + Ok(None) => {} + Err(err) => { + let err: DiskError = err.into(); + if request.deleted || (err != DiskError::FileVersionNotFound && err != DiskError::FileNotFound) { + return Err(err); + } + } + } + } + if data_dirs.iter().any(|dir| *dir == transaction || dir.is_nil()) { + return Err(DiskError::FileCorrupt); + } + let target = if plan.versions.is_empty() { + None + } else { + Some(plan.marshal_msg()?) + }; + let predecessors = match authority { + Some(authority) => authority.predecessors.clone(), + None => { + self.delete_recovery_predecessors(volume, object, transaction, incarnation, owner.clone()) + .await? + } + }; + let surviving: HashSet<_> = plan + .versions + .iter() + .map(|version| (version.header.version_id, std::mem::discriminant(&version.header.version_type))) + .collect(); + deleted_headers.retain(|header| !surviving.contains(header)); + let deleted_versions = deleted_headers.into_iter().map(|(id, _)| id.unwrap_or_default()).collect(); + let requests = requests + .iter() + .map(|request| { + let mut sparse = request.clone(); + sparse.data = None; + sparse + }) + .collect(); + let mut journal = DeleteRecoveryJournal { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + volume: volume.to_owned(), + object: object.to_owned(), + transaction, + bucket_incarnation: incarnation, + original_digest: original.map(digest), + target_digest: target.as_deref().map(digest), + data_dirs, + deleted_versions, + requests, + original_versions, + predecessors, + }; + if let Some(authority) = authority { + if authority.volume != volume + || authority.object != object + || authority.transaction != transaction + || authority.bucket_incarnation != incarnation + { + return Err(DiskError::OutdatedXLMeta); + } + journal.original_versions.clone_from(&authority.original_versions); + journal.deleted_versions.clone_from(&authority.deleted_versions); + } + drop(plan); + self.activate_delete_preparation(&journal, original, target, owner).await + } + + pub(crate) async fn delete_recovery_settle( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + namespace_owner: Option>, + ) -> Result { + if !safe_path(object, false) || transaction.is_nil() { + return Err(DiskError::InvalidPath); + } + let object_path = self.get_object_path(volume, object)?; + let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await; + let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await; + let owner: Option> = Some(Arc::new((metadata_owner, volume_guard))); + self.delete_recovery_settle_owned(volume, object, transaction, expected_fingerprint, action, owner) + .await + } + + pub(super) async fn rollback_prepared_delete( + &self, + volume: &str, + object: &str, + transaction: Uuid, + owner: Option>, + ) -> Result { + let prepared = self.io_get_object_path(volume, &format!("{object}/{transaction}/{PREPARED}"))?; + if regular_bytes(&prepared)?.is_none() { + return Ok(false); + } + let state = self + .delete_recovery_inspect_owned(volume, object, transaction, owner.clone()) + .await?; + self.delete_recovery_settle_owned(volume, object, transaction, state.fingerprint, DeleteRecoveryAction::Rollback, owner) + .await?; + Ok(true) + } + + async fn delete_recovery_settle_owned( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + owner: Option>, + ) -> Result { + let observed = self + .delete_recovery_inspect_owned(volume, object, transaction, owner.clone()) + .await?; + if observed.fingerprint != expected_fingerprint { + return Err(DiskError::OutdatedXLMeta); + } + if let DeleteRecoveryAction::AdoptCommit { authority } = &action { + if authority.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || authority.volume != volume + || authority.object != object + || authority.transaction != transaction + { + return Err(DiskError::OutdatedXLMeta); + } + if observed.state == DeleteRecoveryState::NewPrepared { + let local = observed.journal.as_ref().ok_or(DiskError::FileCorrupt)?; + if observed.current_digest.is_some() || !local.matches_prepared_authority(authority) { + return Err(DiskError::OutdatedXLMeta); + } + // The coordinator supplies an inspected same-transaction + // receipt. Retain this drive's immutable hashes, data dirs and + // causal history when completing its partial publication. + let root = self.io_get_object_path(volume, object)?; + let receipt = CommitReceipt { + journal: local.clone(), + finished: false, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?; + return self.delete_recovery_inspect_owned(volume, object, transaction, owner).await; + } + if observed.state != DeleteRecoveryState::Absent { + return Err(DiskError::OutdatedXLMeta); + } + if observed.current_digest.is_none() { + // A replay witness inherits the original decision's causal + // history; observing later records cannot backdate new edges. + let mut witness = authority.as_ref().clone(); + witness.original_digest = None; + witness.target_digest = None; + witness.data_dirs.clear(); + let root = self.io_get_object_path(volume, object)?; + let _ = self.activate_delete_preparation(&witness, None, None, owner.clone()).await?; + let receipt = CommitReceipt { + journal: witness, + finished: false, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?; + return self.delete_recovery_inspect_owned(volume, object, transaction, owner).await; + } + if !authority.matches_adoption_source(&observed.current_versions) { + return Err(DiskError::OutdatedXLMeta); + } + let root = self.io_get_object_path(volume, object)?; + let bytes = self.read_all(volume, &format!("{object}/{STORAGE_FORMAT_FILE}")).await?; + let _ = self + .prepare_delete_recovery( + volume, + object, + Some(&bytes), + &authority.requests, + Some(authority.as_ref()), + DeletePreparationContext { + transaction, + bucket_incarnation: authority.bucket_incarnation, + namespace_owner: owner.clone(), + }, + ) + .await?; + let adopted = self + .delete_recovery_inspect_owned(volume, object, transaction, owner.clone()) + .await?; + let adopted_journal = adopted.journal.ok_or(DiskError::FileCorrupt)?; + let receipt = CommitReceipt { + journal: adopted_journal, + finished: false, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?; + return self.delete_recovery_inspect_owned(volume, object, transaction, owner).await; + } + if observed.state == DeleteRecoveryState::Absent { + if action == DeleteRecoveryAction::PublishCommit { + if observed.current_digest.is_some() { + return Err(DiskError::OutdatedXLMeta); + } + self.prepare_delete_recovery( + volume, + object, + None, + &[], + None, + DeletePreparationContext { + transaction, + bucket_incarnation: None, + namespace_owner: owner.clone(), + }, + ) + .await?; + } else { + return Ok(observed); + } + } + let observed = self + .delete_recovery_inspect_owned(volume, object, transaction, owner.clone()) + .await?; + if action == DeleteRecoveryAction::DiscardLegacy { + if observed.state != DeleteRecoveryState::LegacyAmbiguous || !observed.explicit_delete_safe { + return Err(DiskError::FileCorrupt); + } + let root = self.io_get_object_path(volume, object)?; + let rollback = root.join(transaction.to_string()); + let plan_root = root.clone(); + let missing = observed.current_digest.is_none(); + let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner.clone()).await; + let plan = os::run_blocking_namespace_operation(lease, move || { + Ok((|| { + let before = + regular_metadata_bytes(&plan_root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP))? + .as_deref() + .map(FileMeta::load) + .transpose()?; + explicit_delete_plan(&plan_root, transaction, before.as_ref(), None, missing)?.ok_or(DiskError::FileCorrupt) + })()) + }) + .await + .map_err(to_file_error)??; + for dir in plan.data_dirs { + let relative = dir.strip_prefix(&root).map_err(|_| DiskError::InvalidPath)?; + let data_dir = relative + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| Uuid::parse_str(name).ok()) + .ok_or(DiskError::FileCorrupt)?; + let parent = relative.parent().filter(|parent| !parent.as_os_str().is_empty()); + let staged_object = + parent.map_or_else(|| object.to_owned(), |parent| format!("{object}/{}", parent.to_string_lossy())); + self.reserve_version_delete_with_namespace_owner(volume, &staged_object, data_dir, transaction, owner.clone()) + .await?; + let reserved = dir.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + let private = dir.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + match os::rename_with_namespace_owner(&reserved, &private, owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&dir, owner.clone()).await.map_err(to_file_error)?; + } + if self + .delete_data_dir_with_namespace_owner( + volume, + &format!("{object}/{}", relative.to_string_lossy()), + DeleteOptions { + recursive: true, + old_data_dir: Some(transaction), + recoverable_delete: true, + ..Default::default() + }, + owner.clone(), + ) + .await? + == DataDirDeleteStatus::Deferred + { + return Err(DiskError::DiskOngoingReq); + } + } + for file in plan.proof_files { + match os::remove_file_with_owner(file, owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() && fs::try_exists(&rollback).await.map_err(to_file_error)? { + os::fsync_dir_with_owner(&rollback, owner.clone()) + .await + .map_err(to_file_error)?; + } + self.delete_unleased_with_namespace_owner( + volume, + &format!("{object}/{transaction}"), + &DeleteOptions { + immediate: true, + ..Default::default() + }, + owner.clone(), + ) + .await?; + return self.delete_recovery_inspect_owned(volume, object, transaction, owner).await; + } + let journal = observed.journal.as_ref().ok_or(DiskError::FileCorrupt)?; + let root = self.io_get_object_path(volume, object)?; + let rollback = root.join(transaction.to_string()); + match action { + DeleteRecoveryAction::PublishCommit => { + if !observed.commit_data_safe { + return Err(DiskError::OutdatedXLMeta); + } + if matches!(observed.state, DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished) { + return Ok(observed); + } + let receipt = CommitReceipt { + journal: journal.clone(), + finished: false, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + self.delete_recovery_barrier(volume, object, transaction, "DECISION").await?; + } + DeleteRecoveryAction::FinishCommit => { + if !matches!(observed.state, DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished) + || !observed.commit_data_safe + { + return Err(DiskError::OutdatedXLMeta); + } + if observed.state == DeleteRecoveryState::CommitFinished { + return Ok(observed); + } + for dir in &journal.data_dirs { + self.reserve_version_delete_with_namespace_owner(volume, object, *dir, transaction, owner.clone()) + .await?; + } + if observed.current_digest == journal.original_digest { + if journal.target_digest.is_some() { + let target = self.read_all(volume, &format!("{object}/{transaction}/{TARGET}")).await?; + self.write_all_meta_with_namespace_owner( + volume, + &format!("{object}/{STORAGE_FORMAT_FILE}"), + &target, + true, + owner.clone(), + ) + .await?; + } else { + self.delete_file_with_namespace_owner( + &self.io_get_bucket_path(volume)?, + &root.join(STORAGE_FORMAT_FILE), + false, + false, + owner.clone(), + ) + .await?; + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&root, owner.clone()).await.map_err(to_file_error)?; + } + } + } + self.commit_reserved_version_delete_with_namespace_owner(volume, object, transaction, owner.clone()) + .await?; + for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] { + match os::remove_file_with_owner(rollback.join(name), owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&rollback, owner.clone()) + .await + .map_err(to_file_error)?; + } + let (_, complete) = self + .finish_version_delete(volume, object, transaction, true, owner.clone()) + .await?; + if !complete { + return Err(DiskError::DiskOngoingReq); + } + for dir in &journal.data_dirs { + let staged = rollback.join(dir.to_string()); + match fs::symlink_metadata(&staged).await { + Ok(metadata) if metadata.is_dir() => { + let staged_object = format!("{object}/{transaction}"); + self.reserve_version_delete_with_namespace_owner( + volume, + &staged_object, + *dir, + transaction, + owner.clone(), + ) + .await?; + let source = staged.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + let destination = staged.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + match os::rename_with_namespace_owner(&source, &destination, owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&staged, owner.clone()) + .await + .map_err(to_file_error)?; + } + if self + .delete_data_dir_with_namespace_owner( + volume, + &format!("{staged_object}/{dir}"), + DeleteOptions { + recursive: true, + old_data_dir: Some(transaction), + ..Default::default() + }, + owner.clone(), + ) + .await? + == DataDirDeleteStatus::Deferred + { + return Err(DiskError::DiskOngoingReq); + } + } + Ok(_) => return Err(DiskError::FileCorrupt), + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + write_delete_rollback_file(self.io_root(), volume, &root, transaction, FINISHED, &[], owner.clone()).await?; + let receipt = CommitReceipt { + journal: journal.clone(), + finished: true, + }; + write_delete_rollback_file( + self.io_root(), + volume, + &root, + transaction, + DELETE_QUORUM_COMMITTED_FILE, + &serde_json::to_vec(&receipt)?, + owner.clone(), + ) + .await?; + } + DeleteRecoveryAction::FinalizeCommit => { + if observed.state != DeleteRecoveryState::CommitFinished { + return Err(DiskError::DiskOngoingReq); + } + for name in [TARGET, PREPARED, FINISHED] { + match os::remove_file_with_owner(rollback.join(name), owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + if effective_durability(volume).syncs_commit_metadata() { + os::fsync_dir_with_owner(&rollback, owner.clone()) + .await + .map_err(to_file_error)?; + } + os::remove_file_with_owner(rollback.join(DELETE_QUORUM_COMMITTED_FILE), owner.clone()) + .await + .map_err(to_file_error)?; + self.delete_unleased_with_namespace_owner( + volume, + &format!("{object}/{transaction}"), + &DeleteOptions { + immediate: true, + ..Default::default() + }, + owner.clone(), + ) + .await?; + } + DeleteRecoveryAction::Rollback => { + if observed.state != DeleteRecoveryState::NewPrepared || !observed.rollback_data_complete { + return Err(DiskError::FileCorrupt); + } + let source = if observed.backup_digest.is_some() { + format!("{object}/{transaction}/{STORAGE_FORMAT_FILE_BACKUP}") + } else { + format!("{object}/{STORAGE_FORMAT_FILE}") + }; + if journal.original_digest.is_some() { + let bytes = self.read_all(volume, &source).await?; + let metadata = FileMeta::load(&bytes)?; + for version in &metadata.versions { + if version.header.version_type != rustfs_filemeta::VersionType::Object { + continue; + } + let mut info = version.into_fileinfo(volume, object, true)?; + info.hydrate_shard_integrity()?; + if info.inline_data() || info.is_remote() { + continue; + } + let dir = info.data_dir.filter(|id| !id.is_nil()).ok_or(DiskError::FileCorrupt)?; + let codec = coding::Erasure::try_new_with_options( + info.erasure.data_blocks, + info.erasure.parity_blocks, + info.erasure.block_size, + info.uses_legacy_checksum, + ) + .map_err(DiskError::from)?; + let live = self.io_get_object_path(volume, &format!("{object}/{dir}"))?; + let staged = self.io_get_object_path(volume, &format!("{object}/{transaction}/{dir}"))?; + for part in &info.parts { + let name = format!("part.{}", part.number); + let checksum = info.erasure.get_checksum_info(part.number); + let algo = if info.uses_legacy_checksum && checksum.algorithm == HashAlgorithm::HighwayHash256S { + HashAlgorithm::HighwayHash256SLegacy + } else { + checksum.algorithm + }; + if let Some(integrity) = &part.integrity { + verify_recovery_integrity_sources( + &live, + &staged, + integrity, + info.erasure.index.checked_sub(1).ok_or(DiskError::FileCorrupt)?, + algo.clone(), + owner.clone(), + ) + .await?; + } + let mut checked = false; + for part_root in [&live, &staged] { + let path = part_root.join(&name); + match fs::symlink_metadata(&path).await { + Ok(metadata) if metadata.is_file() => {} + Ok(_) => return Err(DiskError::InvalidPath), + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => return Err(to_file_error(err).into()), + } + self.bitrot_verify( + &path, + usize::try_from( + codec.shard_file_size(i64::try_from(part.size).map_err(|_| DiskError::FileCorrupt)?), + ) + .map_err(|_| DiskError::FileCorrupt)?, + algo.clone(), + codec.shard_size(), + ) + .await?; + checked = true; + } + if !checked { + return Err(DiskError::FileCorrupt); + } + } + } + } + if journal.original_digest.is_none() && journal.target_digest.is_none() { + for name in [TARGET, PREPARED, FINISHED] { + match os::remove_file_with_owner(rollback.join(name), owner.clone()).await { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => return Err(to_file_error(err).into()), + } + } + self.delete_unleased_with_namespace_owner( + volume, + &format!("{object}/{transaction}"), + &DeleteOptions { + immediate: true, + ..Default::default() + }, + owner.clone(), + ) + .await?; + } else { + restore_delete_rollback_with_namespace_owner( + volume, + &root, + &root.join(STORAGE_FORMAT_FILE), + transaction, + &self.publication_root, + owner.clone(), + ) + .await?; + } + } + DeleteRecoveryAction::AdoptCommit { .. } => return Err(DiskError::MethodNotAllowed), + DeleteRecoveryAction::DiscardLegacy => return Err(DiskError::MethodNotAllowed), + } + self.delete_recovery_inspect_owned(volume, object, transaction, owner).await + } + + pub(super) async fn delete_recovery_barrier(&self, volume: &str, object: &str, transaction: Uuid, stage: &str) -> Result<()> { + barrier(volume, object, transaction, stage, &self.root).await + } +} + +#[cfg(not(feature = "e2e-test-hooks"))] +async fn barrier(_volume: &str, _object: &str, _transaction: Uuid, _stage: &str, _drive: &Path) -> Result<()> { + Ok(()) +} + +#[cfg(feature = "e2e-test-hooks")] +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct BarrierArm { + nonce: String, + bucket: String, + object: String, + stage: String, + #[serde(default)] + disk: Option, +} + +#[cfg(feature = "e2e-test-hooks")] +async fn barrier(volume: &str, object: &str, transaction: Uuid, stage: &str, drive: &Path) -> Result<()> { + let Some(directory) = std::env::var_os("RUSTFS_DELETE_RECOVERY_BARRIER_DIR").map(PathBuf::from) else { + return Ok(()); + }; + let bytes = match fs::read(directory.join("arm.json")).await { + Ok(bytes) => bytes, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(to_file_error(err).into()), + }; + let arm: BarrierArm = serde_json::from_slice(&bytes)?; + if arm.bucket != volume + || arm.object != object + || arm.stage != stage + || arm.disk.as_deref().is_some_and(|disk| Path::new(disk) != drive) + { + return Ok(()); + } + if arm.nonce.is_empty() + || arm.nonce.len() > 64 + || !arm + .nonce + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_') + { + return Err(DiskError::InvalidPath); + } + let drive_hash = rustfs_utils::crypto::hex(digest(drive.to_string_lossy().as_bytes())); + let observation = serde_json::json!({"transaction": transaction, "bucket": volume, "object": object, "stage": stage, "pid": std::process::id(), "drive": drive}); + fs::write( + directory.join(format!("reached.{}.{}.{}.json", arm.nonce, std::process::id(), &drive_hash[..16])), + serde_json::to_vec(&observation)?, + ) + .await + .map_err(to_file_error)?; + let deadline = Instant::now() + Duration::from_secs(60); + while !fs::try_exists(directory.join(format!("release.{}", arm.nonce))) + .await + .map_err(to_file_error)? + { + if Instant::now() >= deadline { + return Err(DiskError::Timeout); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + if fs::read(directory.join(format!("release.{}", arm.nonce))) + .await + .map_err(to_file_error)? + == b"io_error" + { + return Err(DiskError::Io(std::io::Error::new( + ErrorKind::Other, + "injected delete recovery barrier failure", + ))); + } + Ok(()) +} + +pub(super) async fn restore_barrier(volume: &str, object_dir: &Path, transaction: Uuid) -> Result<()> { + #[cfg(feature = "e2e-test-hooks")] + if let Some(directory) = std::env::var_os("RUSTFS_DELETE_RECOVERY_BARRIER_DIR").map(PathBuf::from) { + let bytes = match fs::read(directory.join("arm.json")).await { + Ok(bytes) => bytes, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(to_file_error(err).into()), + }; + let arm: BarrierArm = serde_json::from_slice(&bytes)?; + if arm.bucket == volume && arm.stage == "RESTORE" && object_dir.ends_with(Path::new(volume).join(&arm.object)) { + let suffix = Path::new(volume).join(&arm.object); + let canonical = fs::canonicalize(object_dir).await.map_err(to_file_error)?; + let mut drive = canonical; + for _ in suffix.components() { + drive.pop(); + } + return barrier(volume, &arm.object, transaction, "RESTORE", &drive).await; + } + } + let _ = (volume, object_dir, transaction); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::disk::local::test::{ensure_test_volume, test_file_info, test_meta}; + use tokio::io::AsyncReadExt; + + #[test] + fn delete_recovery_native_sidecar_classifier_is_bounded() { + assert!(delete_recovery_part_file("part.1")); + assert!(delete_recovery_part_file("part.1.meta")); + assert!(delete_recovery_part_file(&format!("part.1.integrity.{}", Uuid::new_v4()))); + for name in [ + "part.0.meta", + "part.1.meta.extra", + "part.1.integrity.invalid", + "xl.meta", + "child/", + ] { + assert!(!delete_recovery_part_file(name), "unknown sidecar {name} must stay protected"); + } + assert!(!delete_recovery_part_file(&format!("part.1.integrity.{}", Uuid::nil()))); + assert!(!metadata_less_part_file("part.1.meta"), "the heal shard classifier remains unchanged"); + } + + #[tokio::test] + #[serial_test::serial(shard_integrity_rollout)] + async fn delete_recovery_multipart_rollback_moves_native_sidecars_and_preserves_proofs() { + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::object_api::{ObjectOptions, PutObjReader}; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; + use crate::storage_api_contracts::multipart::{CompletePart, MultipartOperations}; + use crate::storage_api_contracts::object::ObjectIO; + use http::HeaderMap; + + temp_env::async_with_vars( + [ + (rustfs_config::ENV_SHARD_INTEGRITY_WRITE, Some("true")), + (rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED, Some("true")), + ], + async { + for damage in [None, Some("missing"), Some("corrupt"), Some("payload")] { + let (dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "recovery-multipart"; + let object = "object"; + store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap(); + let upload = store + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + .unwrap(); + let payload = vec![63; 6 * 1024 * 1024]; + let mut reader = PutObjReader::from_vec(payload.clone()); + let part = store + .put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default()) + .await + .unwrap(); + store + .clone() + .complete_multipart_upload( + bucket, + object, + &upload.upload_id, + vec![CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }], + &ObjectOptions::default(), + ) + .await + .unwrap(); + let disks = store.pools[0].disk_set[0].disks.read().await.clone(); + let transaction = Uuid::new_v4(); + let mut originals = Vec::new(); + for (dir, disk) in dirs.iter().zip(&disks) { + let root = dir.path().join(bucket).join(object); + let original = fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let metadata = FileMeta::load(&original).unwrap(); + let mut info = metadata.versions[0].into_fileinfo(bucket, object, true).unwrap(); + info.hydrate_shard_integrity().unwrap(); + let data = info.data_dir.unwrap(); + let proof_name = info.parts[0] + .integrity + .as_ref() + .expect("real protected MPU proof") + .file_name(); + assert!(root.join(data.to_string()).join("part.1.meta").is_file()); + disk.as_ref() + .unwrap() + .delete_version_recoverable( + bucket, + object, + FileInfo { + name: object.to_owned(), + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + .unwrap(); + let staged = root.join(transaction.to_string()).join(data.to_string()); + fs::rename(root.join(data.to_string()), &staged).await.unwrap(); + originals.push((root, data, proof_name, original)); + } + let (root, data, proof, _) = &originals[0]; + let proof_path = root.join(transaction.to_string()).join(data.to_string()).join(proof); + if let Some(damage) = damage { + if damage == "missing" { + fs::remove_file(&proof_path).await.unwrap(); + } else if damage == "corrupt" { + let mut bytes = fs::read(&proof_path).await.unwrap(); + bytes[rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE] ^= 1; + fs::write(&proof_path, bytes).await.unwrap(); + } else { + let metadata = FileMeta::load(&originals[0].3).unwrap(); + let info = metadata.versions[0].into_fileinfo(bucket, object, true).unwrap(); + let codec = coding::Erasure::try_new_with_options( + info.erasure.data_blocks, + info.erasure.parity_blocks, + info.erasure.block_size, + info.uses_legacy_checksum, + ) + .unwrap(); + let algorithm = info.erasure.get_checksum_info(1).algorithm; + let path = proof_path.parent().unwrap().join("part.1"); + let mut bytes = fs::read(&path).await.unwrap(); + let hash_size = algorithm.size(); + let part_size = usize::try_from(codec.shard_file_size(info.parts[0].size as i64)).unwrap(); + let length = part_size.min(codec.shard_size()); + bytes[hash_size] ^= 1; + let rewritten = algorithm.hash_encode(&bytes[hash_size..hash_size + length]).as_ref().to_vec(); + bytes[..hash_size].copy_from_slice(&rewritten); + coding::bitrot_verify( + std::io::Cursor::new(bytes.clone()), + bytes.len(), + part_size, + algorithm, + codec.shard_size(), + ) + .await + .expect("recomputed self-described bitrot checksum passes the former rollback gate"); + fs::write(path, bytes).await.unwrap(); + } + let disk = disks[0].as_ref().unwrap(); + let observed = disk.inspect_delete_transaction(bucket, object, transaction).await.unwrap(); + let backup = fs::read(root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP)) + .await + .unwrap(); + assert!(matches!( + disk.settle_delete_transaction( + bucket, + object, + transaction, + observed.fingerprint, + DeleteRecoveryAction::Rollback + ) + .await, + Err(DiskError::FileCorrupt) + )); + assert!(!root.join(STORAGE_FORMAT_FILE).exists()); + assert!(root.join(transaction.to_string()).join(PREPARED).is_file()); + assert_eq!( + fs::read(root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP)) + .await + .unwrap(), + backup + ); + continue; + } + // Both directions of a partially restored native layout: + // proof still staged with live payload, and the inverse. + for (index, (root, data, proof, _)) in originals.iter().take(2).enumerate() { + let live = root.join(data.to_string()); + fs::create_dir(&live).await.unwrap(); + let name = if index == 0 { "part.1" } else { proof.as_str() }; + fs::rename(root.join(transaction.to_string()).join(data.to_string()).join(name), live.join(name)) + .await + .unwrap(); + } + for (disk, (root, data, proof, original)) in disks.iter().zip(originals) { + let disk = disk.as_ref().unwrap(); + let observed = disk.inspect_delete_transaction(bucket, object, transaction).await.unwrap(); + assert!(observed.rollback_data_complete); + disk.settle_delete_transaction( + bucket, + object, + transaction, + observed.fingerprint, + DeleteRecoveryAction::Rollback, + ) + .await + .expect("rollback restores native multipart sidecars"); + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(), original); + assert!(root.join(data.to_string()).join("part.1.meta").is_file()); + assert!(root.join(data.to_string()).join(proof).is_file()); + assert!(!root.join(transaction.to_string()).exists()); + } + let mut reader = store.pools[0].disk_set[0] + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .unwrap(); + let mut restored = Vec::new(); + reader.stream.read_to_end(&mut restored).await.unwrap(); + assert_eq!(restored, payload, "restored protected MPU remains readable through integrity checks"); + } + }, + ) + .await; + } + + async fn fixture() -> (tempfile::TempDir, LocalDisk, FileInfo, Vec) { + let temp = tempfile::tempdir().expect("temporary disk"); + let endpoint = Endpoint::try_from(temp.path().to_str().expect("UTF-8 disk path")).expect("endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk"); + ensure_test_volume(&disk, "recovery").await; + let mut info = test_file_info("object", Uuid::nil(), None, Some(Bytes::from_static(b"complete original inline payload"))); + info.set_inline_data(); + let bytes = test_meta(info.clone()); + let path = disk.io_get_object_path("recovery", "object/xl.meta").expect("metadata path"); + fs::create_dir_all(path.parent().expect("object directory")) + .await + .expect("object directory"); + fs::write(path, &bytes).await.expect("original metadata"); + (temp, disk, info, bytes) + } + + fn request() -> FileInfo { + FileInfo { + name: "object".to_owned(), + version_id: Some(Uuid::nil()), + ..Default::default() + } + } + + #[test] + fn delete_recovery_generation_known_aliases_are_equal_unknowns_stay_distinct() { + use rustfs_utils::http::metadata_compat as compat; + let mut legacy = request(); + legacy + .metadata + .insert(format!("{}{}", compat::MINIO_INTERNAL_PREFIX, compat::SUFFIX_INLINE_DATA), String::new()); + legacy + .metadata + .insert(format!("{}{}", compat::MINIO_INTERNAL_PREFIX, compat::SUFFIX_COMPRESSION), "gzip".into()); + let mut modern = legacy.clone(); + compat::insert_str(&mut modern.metadata, compat::SUFFIX_INLINE_DATA, String::new()); + compat::insert_str(&mut modern.metadata, compat::SUFFIX_COMPRESSION, "gzip".into()); + assert_eq!( + generation(legacy.clone()).expect("legacy identity"), + generation(modern.clone()).expect("modern identity") + ); + modern.metadata.insert( + format!("{}{}", compat::RUSTFS_INTERNAL_PREFIX, compat::SUFFIX_COMPRESSION), + "different".into(), + ); + assert!(matches!(generation(modern), Err(DiskError::FileCorrupt))); + let mut unknown = legacy.clone(); + legacy + .metadata + .insert("x-minio-internal-unknown-extension".into(), "same".into()); + unknown + .metadata + .insert("x-rustfs-internal-unknown-extension".into(), "same".into()); + assert_ne!( + generation(legacy).expect("legacy unknown field"), + generation(unknown).expect("modern unknown field") + ); + } + + #[tokio::test] + async fn delete_recovery_inactive_preparation_artifacts_use_existing_tmp_cleanup() { + let (_temp, disk, _, original) = fixture().await; + let temporary = disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).unwrap(); + fs::create_dir_all(temporary.join("delete-preparation-interrupted")) + .await + .unwrap(); + fs::write(temporary.join("delete-preparation-interrupted").join(TARGET), b"unfinished target") + .await + .unwrap(); + fs::create_dir_all(temporary.join(".delete-rollback-interrupted")) + .await + .unwrap(); + fs::write( + temporary.join(".delete-rollback-interrupted").join("metadata"), + b"unpublished replacement", + ) + .await + .unwrap(); + assert!( + disk.discover_delete_transactions("recovery", Some("object"), None, 16) + .await + .unwrap() + .transactions + .is_empty() + ); + let ready = Arc::new(std::sync::atomic::AtomicU32::new(0)); + let notify = Arc::new(Notify::new()); + LocalDisk::cleanup_tmp_on_startup(disk.io_root(), &disk.publication_root, ready.clone(), notify) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + while ready.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(!temporary.join("delete-preparation-interrupted").exists()); + assert!(!temporary.join(".delete-rollback-interrupted").exists()); + assert!(!disk.io_root().join(RUSTFS_META_TMP_OLD_BUCKET).exists()); + assert!(temporary.join(".trash").is_dir(), "startup recreates the active trash directory"); + assert_eq!(disk.read_all("recovery", "object/xl.meta").await.unwrap().as_ref(), original); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_activation_is_complete_before_metadata_mutation_even_after_cancel() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + use futures::FutureExt; + let (_temp, disk, _, original) = fixture().await; + let disk = Arc::new(disk); + let transaction = Uuid::new_v4(); + let destination = disk.io_get_object_path("recovery", &format!("object/{transaction}")).unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let hook = hooks::install_at(hooks::Stage::Rename, &destination, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let deleting = disk.clone(); + let mut request = request(); + request.expire_restored = true; + let task = tokio::spawn(async move { + deleting + .delete_version_recoverable( + "recovery", + "object", + request, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + }); + let observations = std::panic::AssertUnwindSafe(async { + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .unwrap() + .unwrap(); + assert!(!destination.exists()); + assert_eq!(disk.read_all("recovery", "object/xl.meta").await.unwrap().as_ref(), original); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + }) + .catch_unwind() + .await; + let _ = release_tx.send(()); + drop(hook); + tokio::time::timeout(Duration::from_secs(5), async { + while !destination.exists() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + for name in [PREPARED, TARGET, STORAGE_FORMAT_FILE_BACKUP] { + assert!(destination.join(name).is_file()); + } + let state = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(state.state, DeleteRecoveryState::NewPrepared); + assert_eq!(state.current_digest, Some(digest(&original))); + if let Err(panic) = observations { + std::panic::resume_unwind(panic); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_receiver_volume_guard_outlives_cancelled_restore() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + use futures::FutureExt; + let (_temp, disk, _, _) = fixture().await; + let disk = Arc::new(disk); + let transaction = Uuid::new_v4(); + let state = apply(&disk, transaction).await; + let metadata = disk.io_get_object_path("recovery", "object/xl.meta").unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let hook = hooks::install_at(hooks::Stage::Rename, &metadata, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let restoring = disk.clone(); + let task = tokio::spawn(async move { + restoring + .settle_delete_transaction("recovery", "object", transaction, state.fingerprint, DeleteRecoveryAction::Rollback) + .await + }); + let mut deletion = None; + let observations = std::panic::AssertUnwindSafe(async { + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .unwrap() + .unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + let removing = disk.clone(); + deletion = Some(tokio::spawn(async move { removing.delete_volume("recovery", true).await })); + assert!( + tokio::time::timeout(Duration::from_millis(100), deletion.as_mut().unwrap()) + .await + .is_err() + ); + }) + .catch_unwind() + .await; + let _ = release_tx.send(()); + drop(hook); + if let Some(deletion) = deletion { + tokio::time::timeout(Duration::from_secs(5), deletion) + .await + .unwrap() + .unwrap() + .unwrap(); + } + disk.make_volume("recovery").await.unwrap(); + assert!(!metadata.exists(), "late restore must finish before volume recreation"); + if let Err(panic) = observations { + std::panic::resume_unwind(panic); + } + } + + async fn apply(disk: &LocalDisk, transaction: Uuid) -> DeleteRecoveryInspection { + disk.delete_version_recoverable( + "recovery", + "object", + request(), + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + .expect("prepared metadata application"); + let inspected = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("inspect applied transaction"); + assert_eq!( + inspected.current_digest, + inspected.journal.as_ref().unwrap().target_digest, + "planned target must match the unchanged mutation loop" + ); + inspected + } + + #[tokio::test] + async fn delete_recovery_uncommitted_inline_rolls_back_exact_bytes() { + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let observed = apply(&disk, transaction).await; + assert_eq!(observed.state, DeleteRecoveryState::NewPrepared); + assert!(observed.rollback_data_complete); + let restored = disk + .settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback) + .await + .expect("rollback intact prepared metadata"); + assert_eq!(restored.state, DeleteRecoveryState::Absent); + assert_eq!( + disk.read_all("recovery", "object/xl.meta") + .await + .expect("readable original metadata") + .as_ref(), + original + ); + } + + #[tokio::test] + async fn delete_recovery_preparation_records_existing_decision_before_new_generation() { + let (_temp, disk, info, original) = fixture().await; + let first = Uuid::new_v4(); + let applied = apply(&disk, first).await; + disk.settle_delete_transaction("recovery", "object", first, applied.fingerprint, DeleteRecoveryAction::PublishCommit) + .await + .expect("persist first decision"); + let mut newer = info; + newer.mod_time = newer.mod_time.map(|time| time + time::Duration::seconds(1)); + let newer_bytes = test_meta(newer); + assert_ne!(newer_bytes, original); + fs::write(disk.io_get_object_path("recovery", "object/xl.meta").unwrap(), &newer_bytes) + .await + .unwrap(); + let second = Uuid::new_v4(); + let second_state = apply(&disk, second).await; + assert_eq!(second_state.journal.unwrap().predecessors, vec![first]); + } + + #[tokio::test] + async fn delete_recovery_large_intent_fails_before_metadata_application() { + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let mut large = request(); + large + .metadata + .insert("oversized-intent".to_owned(), "x".repeat(usize::try_from(MAX_METADATA).unwrap())); + assert!( + disk.delete_version_recoverable( + "recovery", + "object", + large, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + } + ) + .await + .is_err() + ); + assert_eq!(disk.read_all("recovery", "object/xl.meta").await.unwrap().as_ref(), original); + assert!( + !disk + .io_get_object_path("recovery", &format!("object/{transaction}/{PREPARED}")) + .unwrap() + .exists() + ); + } + + #[tokio::test] + async fn delete_recovery_metadata_updates_publish_exact_prepared_target_bytes() { + for batch in [false, true] { + for transition in [false, true] { + let (_temp, disk, mut info, _) = fixture().await; + info.metadata.insert("x-amz-meta-first".into(), "retained-a".into()); + info.metadata.insert("x-amz-meta-second".into(), "retained-b".into()); + info.metadata + .insert(rustfs_filemeta::metadata_keys::RESTORE.into(), "ongoing-request=\"false\"".into()); + let original = test_meta(info); + fs::write(disk.io_get_object_path("recovery", "object/xl.meta").unwrap(), &original) + .await + .unwrap(); + let transaction = Uuid::new_v4(); + let mut change = request(); + if transition { + change.transition_status = "complete".into(); + change.transition_tier = "test-tier".into(); + change.transitioned_objname = "remote/object".into(); + } else { + change.expire_restored = true; + } + let opts = DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }; + if batch { + let errors = disk + .delete_versions_recoverable( + "recovery", + vec![FileInfoVersions { + name: "object".into(), + volume: "recovery".into(), + versions: vec![change], + ..Default::default() + }], + opts, + ) + .await; + assert_eq!(errors, vec![None]); + } else { + disk.delete_version_recoverable("recovery", "object", change, false, opts) + .await + .unwrap(); + } + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(observed.current_digest, observed.journal.unwrap().target_digest); + let applied = disk.read_all("recovery", "object/xl.meta").await.unwrap(); + let prepared = disk + .read_all("recovery", &format!("object/{transaction}/{TARGET}")) + .await + .unwrap(); + assert_eq!(applied, prepared, "batch={batch}, transition={transition}"); + assert_ne!(applied.as_ref(), original); + } + } + } + + async fn legacy_fixture(disk: &LocalDisk, original: &[u8], transaction: Uuid) -> PathBuf { + let root = disk.io_get_object_path("recovery", "object").unwrap(); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + fs::create_dir(root.join(transaction.to_string())).await.unwrap(); + fs::write(root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP), original) + .await + .unwrap(); + root + } + + #[tokio::test] + async fn delete_recovery_live_repair_backup_preserves_legacy_delete_guards() { + for shape in ["live", "rollback-marker", "missing-current", "foreign-backup"] { + let (_temp, disk, mut info, _) = fixture().await; + let transaction = Uuid::new_v4(); + info.data_dir = Some(transaction); + let root = disk.io_get_object_path("recovery", "object").unwrap(); + let current = test_meta(info.clone()); + fs::write(root.join(STORAGE_FORMAT_FILE), ¤t).await.unwrap(); + let rollback = root.join(transaction.to_string()); + fs::create_dir(&rollback).await.unwrap(); + if shape == "foreign-backup" { + info.data_dir = Some(Uuid::new_v4()); + } + let backup = test_meta(info); + fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), &backup).await.unwrap(); + if shape == "rollback-marker" { + fs::write(rollback.join(DELETE_MARKER_ROLLBACK_FILE), []).await.unwrap(); + } else if shape == "missing-current" { + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + } + let inspection = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!( + inspection.state, + if shape == "live" { + DeleteRecoveryState::Absent + } else { + DeleteRecoveryState::LegacyAmbiguous + }, + "{shape}" + ); + assert_eq!(fs::read(rollback.join(STORAGE_FORMAT_FILE_BACKUP)).await.unwrap(), backup); + if shape != "missing-current" { + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(), current); + } + } + } + + #[tokio::test] + async fn delete_recovery_explicit_legacy_discard_accepts_old_empty_receipt() { + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let root = legacy_fixture(&disk, &original, transaction).await; + fs::write(root.join(transaction.to_string()).join(DELETE_QUORUM_COMMITTED_FILE), []) + .await + .unwrap(); + let state = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(state.state, DeleteRecoveryState::LegacyAmbiguous); + assert!(state.explicit_delete_safe); + let cleared = disk + .settle_delete_transaction("recovery", "object", transaction, state.fingerprint, DeleteRecoveryAction::DiscardLegacy) + .await + .unwrap(); + assert_eq!(cleared.state, DeleteRecoveryState::Absent); + assert!(!root.join(transaction.to_string()).exists()); + } + + #[tokio::test] + async fn delete_recovery_prepared_explicit_delete_checks_original_header() { + for protected in ["none", "version", "lock", "internal-lock"] { + let (_temp, disk, mut info, _) = fixture().await; + match protected { + "version" => info.version_id = Some(Uuid::new_v4()), + "lock" => { + info.metadata.insert("x-amz-object-lock-mode".into(), "COMPLIANCE".into()); + } + "internal-lock" => rustfs_utils::http::metadata_compat::insert_str( + &mut info.metadata, + rustfs_utils::http::metadata_compat::SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, + "known-lock".to_owned(), + ), + _ => {} + } + fs::write(disk.io_get_object_path("recovery", "object/xl.meta").unwrap(), test_meta(info.clone())) + .await + .unwrap(); + let transaction = Uuid::new_v4(); + disk.delete_version_recoverable( + "recovery", + "object", + FileInfo { + version_id: info.version_id, + ..request() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + .unwrap(); + let state = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(state.state, DeleteRecoveryState::NewPrepared); + assert_eq!(state.explicit_delete_safe, protected == "none", "original header {protected}"); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_cancelled_restore_retains_actual_key_and_bucket_guards() { + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::disk::os::prepared_publication_test_hooks as hooks; + use crate::object_api::{ObjectOptions, PutObjReader}; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; + use crate::storage_api_contracts::object::ObjectIO; + use futures::FutureExt; + use rustfs_heal_contracts::heal_channel::HealOpts; + + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "recover-owned-timeout"; + store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap(); + store + .put_object(bucket, "object", &mut PutObjReader::from_vec(vec![61; 4096]), &ObjectOptions::default()) + .await + .unwrap(); + let disks = store.pools[0].disk_set[0].disk_inventory().await; + let transaction = Uuid::new_v4(); + let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap(); + for disk in disks.iter().flatten() { + disk.delete_version_recoverable( + bucket, + "object", + request(), + false, + DeleteOptions { + old_data_dir: Some(transaction), + delete_recovery_bucket_incarnation: Some(incarnation), + ..Default::default() + }, + ) + .await + .unwrap(); + } + let metadata = disks[0] + .as_ref() + .unwrap() + .get_object_path_for_io_if_local(bucket, "object/xl.meta") + .unwrap() + .unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let hook = hooks::install_at(hooks::Stage::Rename, &metadata, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let before = store.ctx.namespace_commit_generation(); + let recovering = store.clone(); + let mut recovery = + tokio::spawn(async move { recovering.recover_delete_transactions(bucket, &HealOpts::default()).await }); + let mut writer = None; + let mut bucket_writer = None; + let observations = std::panic::AssertUnwindSafe(async { + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .unwrap() + .unwrap(); + assert!(tokio::time::timeout(Duration::from_millis(100), &mut recovery).await.is_err()); + assert!(store.ctx.namespace_commits_pending()); + assert_eq!(store.ctx.namespace_commit_generation(), before + 1); + recovery.abort(); + assert!(recovery.await.unwrap_err().is_cancelled()); + assert!(store.ctx.namespace_commits_pending(), "late restore must retain namespace accounting"); + let writing = store.clone(); + writer = Some(tokio::spawn(async move { + writing + .put_object(bucket, "object", &mut PutObjReader::from_vec(vec![62; 4096]), &ObjectOptions::default()) + .await + })); + assert!( + tokio::time::timeout(Duration::from_millis(100), writer.as_mut().unwrap()) + .await + .is_err(), + "same-key PUT must wait for the late restore" + ); + let removing = store.clone(); + bucket_writer = Some(tokio::spawn(async move { removing.acquire_bucket_lifecycle_write_lock(bucket).await })); + assert!( + tokio::time::timeout(Duration::from_millis(100), bucket_writer.as_mut().unwrap()) + .await + .is_err(), + "bucket incarnation mutation must wait for the late restore" + ); + }) + .catch_unwind() + .await; + let _ = release_tx.send(()); + drop(hook); + if let Some(writer) = writer { + tokio::time::timeout(Duration::from_secs(10), writer) + .await + .unwrap() + .unwrap() + .unwrap(); + } + if let Some(writer) = bucket_writer { + drop( + tokio::time::timeout(Duration::from_secs(10), writer) + .await + .unwrap() + .unwrap() + .unwrap(), + ); + } + if let Err(panic) = observations { + std::panic::resume_unwind(panic); + } + } + + #[tokio::test] + async fn delete_recovery_explicit_legacy_discard_preserves_versions_locks_and_live_children() { + for protected in ["version", "lock", "internal-lock", "child", "unknown"] { + let (_temp, disk, mut info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let root = legacy_fixture(&disk, &original, transaction).await; + let rollback = root.join(transaction.to_string()); + match protected { + "version" => { + info.version_id = Some(Uuid::new_v4()); + } + "lock" => { + info.metadata.insert("X-Amz-Object-Lock-Legal-Hold".into(), "ON".into()); + } + "internal-lock" => { + rustfs_utils::http::metadata_compat::insert_str( + &mut info.metadata, + rustfs_utils::http::metadata_compat::SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, + "old-lock-state".to_owned(), + ); + } + "child" => { + fs::create_dir(rollback.join("live-child")).await.unwrap(); + } + _ => { + fs::write(rollback.join("unknown"), b"preserve").await.unwrap(); + } + } + fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), test_meta(info)) + .await + .unwrap(); + match disk.inspect_delete_transaction("recovery", "object", transaction).await { + Ok(state) => { + assert!(!state.explicit_delete_safe, "protected shape {protected}"); + assert!( + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + state.fingerprint, + DeleteRecoveryAction::DiscardLegacy + ) + .await + .is_err() + ); + } + Err(error) => assert!(matches!(error, DiskError::FileCorrupt | DiskError::InvalidPath)), + } + assert!(rollback.join(STORAGE_FORMAT_FILE_BACKUP).exists()); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_explicit_legacy_discard_rejects_selected_symlink() { + let (temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + let root = legacy_fixture(&disk, &original, transaction).await; + std::os::unix::fs::symlink(temp.path(), root.join(transaction.to_string()).join("linked")).unwrap(); + assert_eq!( + disk.inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap_err(), + DiskError::InvalidPath + ); + assert!(root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP).exists()); + } + + #[tokio::test] + async fn delete_recovery_commit_retains_proof_until_explicit_finalization() { + let (_temp, disk, _info, _original) = fixture().await; + let transaction = Uuid::new_v4(); + let applied = apply(&disk, transaction).await; + let committed = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .expect("durable set decision"); + let finished = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + committed.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .expect("physical cleanup"); + assert_eq!(finished.state, DeleteRecoveryState::CommitFinished); + assert_eq!( + disk.finish_internal_delete_residue("recovery", "object", None).await, + 0, + "legacy retry must not GC new decision proof" + ); + let receipt = disk + .io_get_object_path("recovery", &format!("object/{transaction}/{DELETE_QUORUM_COMMITTED_FILE}")) + .expect("receipt path"); + assert!(receipt.is_file()); + let settled = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + finished.fingerprint, + DeleteRecoveryAction::FinalizeCommit, + ) + .await + .expect("finalize after coordinator convergence proof"); + assert_eq!(settled.state, DeleteRecoveryState::Absent); + assert!(!disk.io_get_object_path("recovery", "object").expect("object path").exists()); + } + + #[tokio::test] + async fn delete_recovery_fingerprint_rejects_new_generation() { + let (_temp, disk, mut info, _original) = fixture().await; + let transaction = Uuid::new_v4(); + let applied = apply(&disk, transaction).await; + info.mod_time = Some(OffsetDateTime::now_utc()); + info.data = Some(Bytes::from_static(b"new generation")); + let newer = test_meta(info); + fs::write(disk.io_get_object_path("recovery", "object/xl.meta").expect("metadata path"), &newer) + .await + .expect("concurrent new generation"); + assert!(matches!( + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit + ) + .await, + Err(DiskError::OutdatedXLMeta) + )); + assert_eq!( + disk.read_all("recovery", "object/xl.meta") + .await + .expect("new generation remains") + .as_ref(), + newer + ); + } + + #[tokio::test] + async fn delete_recovery_untouched_peer_adopts_only_matching_generation() { + let (_temp, source, info, original) = fixture().await; + let (_peer_temp, peer, _peer_info, _) = fixture().await; + fs::write(peer.io_get_object_path("recovery", "object/xl.meta").expect("peer metadata"), &original) + .await + .expect("same original generation"); + let transaction = Uuid::new_v4(); + let applied = apply(&source, transaction).await; + let committed = source + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .expect("source durable decision"); + let authority = committed.journal.expect("source intent"); + let untouched = peer + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("untouched peer"); + let adopted = peer + .settle_delete_transaction( + "recovery", + "object", + transaction, + untouched.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority.clone()), + }, + ) + .await + .expect("generation-bound adoption"); + assert_eq!(adopted.state, DeleteRecoveryState::QuorumCommitted); + let finished = peer + .settle_delete_transaction("recovery", "object", transaction, adopted.fingerprint, DeleteRecoveryAction::FinishCommit) + .await + .expect("converge untouched peer"); + assert_eq!(finished.state, DeleteRecoveryState::CommitFinished); + assert!(finished.current_digest.is_none()); + let (_foreign_temp, foreign, _, _) = fixture().await; + let mut replacement = info; + replacement.mod_time = Some(OffsetDateTime::now_utc()); + let newer = test_meta(replacement); + fs::write( + foreign + .io_get_object_path("recovery", "object/xl.meta") + .expect("foreign metadata"), + &newer, + ) + .await + .expect("foreign generation"); + let observed = foreign + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("inspect foreign generation"); + assert!(matches!( + foreign + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority) + } + ) + .await, + Err(DiskError::OutdatedXLMeta) + )); + assert_eq!( + foreign + .read_all("recovery", "object/xl.meta") + .await + .expect("protected foreign generation") + .as_ref(), + newer + ); + } + + #[tokio::test] + async fn delete_recovery_adoption_preserves_unrelated_new_version() { + let (_source_temp, source, original_info, original) = fixture().await; + let (_peer_temp, peer, _, _) = fixture().await; + let transaction = Uuid::new_v4(); + let applied = apply(&source, transaction).await; + let committed = source + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .unwrap(); + let mut unrelated = original_info; + unrelated.version_id = Some(Uuid::new_v4()); + unrelated.mod_time = unrelated.mod_time.map(|time| time + time::Duration::seconds(1)); + let mut peer_meta = FileMeta::load(&original).unwrap(); + peer_meta.add_version(unrelated.clone()).unwrap(); + fs::write( + peer.io_get_object_path("recovery", "object/xl.meta").unwrap(), + peer_meta.marshal_msg().unwrap(), + ) + .await + .unwrap(); + let observed = peer + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + let adopted = peer + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(committed.journal.unwrap()), + }, + ) + .await + .unwrap(); + peer.settle_delete_transaction( + "recovery", + "object", + transaction, + adopted.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .unwrap(); + let remaining = FileMeta::load(&peer.read_all("recovery", "object/xl.meta").await.unwrap()).unwrap(); + assert_eq!(remaining.versions.len(), 1); + assert_eq!(remaining.versions[0].header.version_id, unrelated.version_id); + } + + #[tokio::test] + async fn delete_recovery_absent_replay_witness_cannot_acquire_future_predecessors() { + let (_source_temp, source, _, _) = fixture().await; + let (_peer_temp, peer, _, _) = fixture().await; + let transaction = Uuid::new_v4(); + let source_state = apply(&source, transaction).await; + let committed = source + .settle_delete_transaction( + "recovery", + "object", + transaction, + source_state.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .unwrap(); + let authority = committed.journal.unwrap(); + assert!(authority.predecessors.is_empty()); + let future = Uuid::new_v4(); + apply(&peer, future).await; + let absent = peer + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(absent.state, DeleteRecoveryState::Absent); + assert!(absent.current_digest.is_none()); + let adopted = peer + .settle_delete_transaction( + "recovery", + "object", + transaction, + absent.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority.clone()), + }, + ) + .await + .unwrap(); + let witness = adopted.journal.unwrap(); + assert_eq!(witness.predecessors, authority.predecessors); + assert!(!witness.predecessors.contains(&future)); + assert_eq!(witness.original_versions, authority.original_versions); + assert_eq!(witness.deleted_versions, authority.deleted_versions); + assert_eq!(witness.bucket_incarnation, authority.bucket_incarnation); + assert!(witness.original_digest.is_none() && witness.target_digest.is_none() && witness.data_dirs.is_empty()); + } + + #[tokio::test] + async fn delete_recovery_raw_discovery_paginates_and_binds_exact_key_scope() { + let (temp, disk, _info, _original) = fixture().await; + for index in 0..512 { + let path = temp + .path() + .join("recovery") + .join(format!("key-{index:04}")) + .join(Uuid::new_v4().to_string()); + fs::create_dir_all(&path).await.expect("hidden transaction directory"); + fs::write(path.join(STORAGE_FORMAT_FILE_BACKUP), b"legacy backup") + .await + .expect("hidden backup"); + } + let mut cursor = None; + let mut found = BTreeSet::new(); + loop { + let page = disk + .discover_delete_transactions("recovery", None, cursor, 13) + .await + .expect("bounded raw discovery"); + found.extend(page.transactions.into_iter().map(|id| id.object)); + cursor = page.next; + if cursor.is_none() { + break; + } + } + assert_eq!(found.len(), 512, "every hidden legacy transaction must be discovered across pages"); + let scoped = disk + .discover_delete_transactions("recovery", Some("key-0000"), None, 1) + .await + .expect("shallow exact-key discovery"); + assert_eq!(scoped.transactions.len(), 1); + assert_eq!(scoped.transactions[0].object, "key-0000"); + if let Some(cursor) = scoped.next { + assert!( + disk.discover_delete_transactions("recovery", Some("key-0001"), Some(cursor), 1) + .await + .is_err(), + "a cursor cannot cross object scopes" + ); + } + } + + #[tokio::test] + async fn delete_recovery_gc_interruption_replays_finished_receipt_without_journal() { + let (_temp, disk, _info, _) = fixture().await; + let transaction = Uuid::new_v4(); + let applied = apply(&disk, transaction).await; + let committed = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .expect("record decision"); + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + committed.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .expect("finish physical cleanup"); + let root = disk + .io_get_object_path("recovery", &format!("object/{transaction}")) + .expect("transaction root"); + fs::remove_file(root.join(PREPARED)) + .await + .expect("simulate partially completed proof GC"); + fs::remove_file(root.join(FINISHED)) + .await + .expect("simulate removed completion marker"); + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("receipt preserves completed intent"); + assert_eq!(observed.state, DeleteRecoveryState::CommitFinished); + let repeated = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .expect("finish is idempotent after interrupted GC"); + let finalized = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + repeated.fingerprint, + DeleteRecoveryAction::FinalizeCommit, + ) + .await + .expect("finish proof GC"); + assert_eq!(finalized.state, DeleteRecoveryState::Absent); + } + + #[tokio::test] + async fn delete_recovery_completed_restore_without_backup_can_finish_rollback() { + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + apply(&disk, transaction).await; + let root = disk.io_get_object_path("recovery", "object").expect("object root"); + fs::rename( + root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP), + root.join(STORAGE_FORMAT_FILE), + ) + .await + .expect("simulate crash after rollback metadata rename"); + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("already restored generation"); + assert_eq!(observed.state, DeleteRecoveryState::NewPrepared); + assert!( + observed.rollback_data_complete, + "restored original replaces consumed backup as rollback evidence" + ); + disk.settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback) + .await + .expect("finish interrupted rollback"); + assert_eq!( + disk.read_all("recovery", "object/xl.meta") + .await + .expect("original remains readable") + .as_ref(), + original + ); + } + + #[tokio::test] + async fn delete_recovery_completed_restore_without_target_can_finish_versioned_rollback() { + let (_temp, disk, info, original) = fixture().await; + let mut remaining = info; + remaining.version_id = Some(Uuid::new_v4()); + remaining.mod_time = remaining.mod_time.map(|time| time + time::Duration::seconds(1)); + let mut metadata = FileMeta::load(&original).unwrap(); + metadata.add_version(remaining).unwrap(); + let original = metadata.marshal_msg().unwrap(); + let root = disk.io_get_object_path("recovery", "object").unwrap(); + fs::write(root.join(STORAGE_FORMAT_FILE), &original).await.unwrap(); + let transaction = Uuid::new_v4(); + let observed = apply(&disk, transaction).await; + assert!(observed.journal.as_ref().unwrap().target_digest.is_some()); + let rollback = root.join(transaction.to_string()); + let target = fs::read(rollback.join(TARGET)).await.unwrap(); + fs::remove_file(rollback.join(TARGET)).await.unwrap(); + assert!(matches!( + disk.inspect_delete_transaction("recovery", "object", transaction).await, + Err(DiskError::FileCorrupt) + )); + fs::write(rollback.join(TARGET), target).await.unwrap(); + fs::rename(rollback.join(STORAGE_FORMAT_FILE_BACKUP), root.join(STORAGE_FORMAT_FILE)) + .await + .unwrap(); + fs::remove_file(rollback.join(TARGET)).await.unwrap(); + let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE); + fs::write( + &receipt_path, + serde_json::to_vec(&CommitReceipt { + journal: observed.journal.unwrap(), + finished: false, + }) + .unwrap(), + ) + .await + .unwrap(); + assert!(matches!( + disk.inspect_delete_transaction("recovery", "object", transaction).await, + Err(DiskError::FileCorrupt) + )); + fs::remove_file(receipt_path).await.unwrap(); + let retry = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(retry.state, DeleteRecoveryState::NewPrepared); + assert!(retry.rollback_data_complete); + disk.settle_delete_transaction("recovery", "object", transaction, retry.fingerprint, DeleteRecoveryAction::Rollback) + .await + .expect("finish cleanup of an exactly restored versioned generation"); + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(), original); + assert!(!rollback.exists()); + } + + #[tokio::test] + async fn delete_recovery_absent_remaining_metadata_requires_existing_commit_receipt() { + let (_temp, disk, info, original) = fixture().await; + let mut remaining = info; + remaining.version_id = Some(Uuid::new_v4()); + remaining.mod_time = remaining.mod_time.map(|time| time + time::Duration::seconds(1)); + let mut metadata = FileMeta::load(&original).unwrap(); + metadata.add_version(remaining).unwrap(); + let root = disk.io_get_object_path("recovery", "object").unwrap(); + fs::write(root.join(STORAGE_FORMAT_FILE), metadata.marshal_msg().unwrap()) + .await + .unwrap(); + let transaction = Uuid::new_v4(); + let applied = apply(&disk, transaction).await; + assert!(applied.journal.as_ref().unwrap().target_digest.is_some()); + let rollback = root.join(transaction.to_string()); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let undecided = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(undecided.state, DeleteRecoveryState::NewPrepared); + assert!( + !undecided.commit_data_safe, + "absence cannot authorize a target-Some undecided transaction" + ); + assert!(matches!( + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + undecided.fingerprint, + DeleteRecoveryAction::PublishCommit + ) + .await, + Err(DiskError::OutdatedXLMeta) + )); + assert!(rollback.join(PREPARED).is_file()); + assert!(rollback.join(STORAGE_FORMAT_FILE_BACKUP).is_file()); + assert!(!rollback.join(DELETE_QUORUM_COMMITTED_FILE).exists()); + fs::copy(rollback.join(TARGET), root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let applied = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .unwrap(); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let certified = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert_eq!(certified.state, DeleteRecoveryState::QuorumCommitted); + assert!(certified.commit_data_safe); + let finished = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + certified.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .expect("certified obsolete version cleanup has no live metadata references"); + assert_eq!(finished.state, DeleteRecoveryState::CommitFinished); + assert!( + !root.join(STORAGE_FORMAT_FILE).exists(), + "cleanup must never republish the old remaining version" + ); + assert!( + rollback.join(DELETE_QUORUM_COMMITTED_FILE).is_file(), + "set-wide GC still owns the final witness" + ); + } + + #[tokio::test] + async fn delete_recovery_partial_publication_adopts_same_intent_without_replacing_local_journal() { + let (_temp, disk, mut original, _) = fixture().await; + let (_peer_temp, peer, _, _) = fixture().await; + let mut remaining = original.clone(); + remaining.version_id = Some(Uuid::new_v4()); + remaining.mod_time = remaining.mod_time.map(|time| time + time::Duration::seconds(1)); + for (index, target) in [(1, &disk), (2, &peer)] { + original.erasure.index = index; + remaining.erasure.index = index; + let mut metadata = FileMeta::load(&test_meta(original.clone())).unwrap(); + metadata.add_version(remaining.clone()).unwrap(); + fs::write( + target.io_get_object_path("recovery", "object/xl.meta").unwrap(), + metadata.marshal_msg().unwrap(), + ) + .await + .unwrap(); + } + let transaction = Uuid::new_v4(); + let local = apply(&disk, transaction).await; + let root = disk.io_get_object_path("recovery", "object").unwrap(); + let rollback = root.join(transaction.to_string()); + let prepared = fs::read(rollback.join(PREPARED)).await.unwrap(); + let target = fs::read(rollback.join(TARGET)).await.unwrap(); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + let applied = apply(&peer, transaction).await; + let committed = peer + .settle_delete_transaction( + "recovery", + "object", + transaction, + applied.fingerprint, + DeleteRecoveryAction::PublishCommit, + ) + .await + .unwrap(); + let authority = committed.journal.unwrap(); + assert_ne!( + local.journal.as_ref().unwrap().original_digest, + authority.original_digest, + "each drive owns distinct raw metadata" + ); + let mut invalid = Vec::new(); + let mut changed = authority.clone(); + changed.bucket_incarnation = Some(Uuid::new_v4()); + invalid.push(changed); + let mut changed = authority.clone(); + changed.deleted_versions.push(Uuid::new_v4()); + invalid.push(changed); + let mut changed = authority.clone(); + changed.requests[0].version_id = Some(Uuid::new_v4()); + invalid.push(changed); + let mut changed = authority.clone(); + changed + .original_versions + .iter_mut() + .find(|version| version.version_id.unwrap_or_default().is_nil()) + .unwrap() + .metadata_digest[0] ^= 1; + invalid.push(changed); + for authority in invalid { + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert!(matches!( + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority) + } + ) + .await, + Err(DiskError::OutdatedXLMeta) + )); + assert!(!rollback.join(DELETE_QUORUM_COMMITTED_FILE).exists()); + } + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + assert!(!observed.commit_data_safe); + let adopted = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(authority), + }, + ) + .await + .expect("same-T certified intent completes partial publication"); + assert_eq!( + adopted.journal, local.journal, + "receipt preserves local hashes and data-directory ownership" + ); + assert_eq!(fs::read(rollback.join(PREPARED)).await.unwrap(), prepared); + assert_eq!(fs::read(rollback.join(TARGET)).await.unwrap(), target); + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + adopted.fingerprint, + DeleteRecoveryAction::FinishCommit, + ) + .await + .unwrap(); + assert!(!root.join(STORAGE_FORMAT_FILE).exists()); + assert!(rollback.join(DELETE_QUORUM_COMMITTED_FILE).is_file()); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_rollback_marker_sync_failure_retains_journal() { + for mode in [DurabilityMode::Strict, DurabilityMode::None] { + let _mode = durability_mode_override::set(mode); + let (_temp, disk, _info, original) = fixture().await; + let transaction = Uuid::new_v4(); + apply(&disk, transaction).await; + let root = disk.io_get_object_path("recovery", "object").unwrap(); + let data = root.join(Uuid::new_v4().to_string()); + fs::create_dir(&data).await.unwrap(); + let marker = data.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")); + fs::write(&marker, []).await.unwrap(); + let prepared = root.join(transaction.to_string()).join(PREPARED); + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + os::fsync_dir_recorder::set_failure(&data, ErrorKind::Other); + let result = disk + .settle_delete_transaction( + "recovery", + "object", + transaction, + observed.fingerprint, + DeleteRecoveryAction::Rollback, + ) + .await; + assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.unwrap(), original); + assert!(!marker.exists(), "the unlink precedes the durability barrier"); + if mode == DurabilityMode::Strict { + assert!(result.is_err(), "a failed marker-directory sync must stop proof removal"); + assert!(prepared.exists(), "rollback evidence survives the sync failure"); + let retry = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap(); + disk.settle_delete_transaction( + "recovery", + "object", + transaction, + retry.fingerprint, + DeleteRecoveryAction::Rollback, + ) + .await + .expect("retry syncs the directory even though its marker is already absent"); + assert!(os::fsync_dir_recorder::was_fsynced(&data)); + } else { + result.expect("none mode does not add a marker durability barrier"); + assert!(!os::fsync_dir_recorder::was_fsynced(&data)); + assert!(os::fsync_dir_recorder::take_failure(&data).is_some()); + } + assert!(!prepared.exists()); + } + } + + #[tokio::test] + async fn delete_recovery_validates_staged_parts_before_restoring_metadata() { + use crate::erasure::coding::BitrotWriter; + let (_temp, disk, _info, _) = fixture().await; + let data_dir = Uuid::new_v4(); + let payload = Bytes::from_static(b"complete staged external payload"); + let mut info = test_file_info("object", Uuid::nil(), Some(data_dir), None); + info.size = i64::try_from(payload.len()).expect("payload length"); + info.erasure.block_size = payload.len(); + info.parts[0].size = payload.len(); + info.parts[0].actual_size = info.size; + let mut second = info.parts[0].clone(); + second.number = 2; + info.parts.push(second); + info.size *= 2; + let original = test_meta(info.clone()); + let root = disk.io_get_object_path("recovery", "object").expect("object root"); + fs::write(root.join(STORAGE_FORMAT_FILE), &original) + .await + .expect("external original metadata"); + let mut writer = BitrotWriter::new( + std::io::Cursor::new(Vec::new()), + info.erasure.shard_size(), + HashAlgorithm::HighwayHash256S, + ); + assert_eq!(writer.write(&payload).await.expect("encode intact external shard"), payload.len()); + writer.shutdown().await.expect("flush shard encoding"); + let encoded = writer.into_inner().into_inner(); + fs::create_dir_all(root.join(data_dir.to_string())) + .await + .expect("external data directory"); + fs::write(root.join(data_dir.to_string()).join("part.1"), &encoded) + .await + .expect("intact external part"); + fs::write(root.join(data_dir.to_string()).join("part.2"), &encoded) + .await + .unwrap(); + let transaction = Uuid::new_v4(); + let _ = disk + .prepare_delete_recovery( + "recovery", + "object", + Some(&original), + &[request()], + None, + DeletePreparationContext { + transaction, + bucket_incarnation: None, + namespace_owner: None, + }, + ) + .await + .expect("prepared intent"); + fs::rename( + root.join(data_dir.to_string()), + root.join(transaction.to_string()).join(data_dir.to_string()), + ) + .await + .expect("simulate staged legacy-compatible data"); + fs::create_dir(root.join(data_dir.to_string())).await.unwrap(); + fs::rename( + root.join(transaction.to_string()).join(data_dir.to_string()).join("part.1"), + root.join(data_dir.to_string()).join("part.1"), + ) + .await + .unwrap(); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)) + .await + .expect("simulate applied metadata removal"); + let observed = disk + .inspect_delete_transaction("recovery", "object", transaction) + .await + .expect("inspect complete staged data"); + assert!(observed.rollback_data_complete); + disk.settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback) + .await + .expect("verified staged rollback"); + assert_eq!( + disk.read_all("recovery", "object/xl.meta") + .await + .expect("restored metadata") + .as_ref(), + original + ); + assert_eq!( + disk.read_all("recovery", &format!("object/{data_dir}/part.1")) + .await + .expect("restored shard") + .as_ref(), + encoded + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn delete_recovery_raw_discovery_rejects_symlink_and_forged_cursor() { + use std::os::unix::fs::symlink; + let (temp, disk, _info, _) = fixture().await; + let outside = tempfile::tempdir().expect("outside sentinel"); + fs::write(outside.path().join("sentinel"), b"outside bytes") + .await + .expect("outside sentinel"); + symlink(outside.path(), temp.path().join("recovery").join("untrusted-prefix")).expect("untrusted symlink"); + assert!(disk.discover_delete_transactions("recovery", None, None, 16).await.is_err()); + assert!( + disk.discover_delete_transactions("recovery", None, Some(DeleteRecoveryCursor { token: Uuid::new_v4() }), 16) + .await + .is_err() + ); + assert_eq!( + fs::read(outside.path().join("sentinel")) + .await + .expect("outside remains intact"), + b"outside bytes" + ); + } +} diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 89fa4c76e..c1166f405 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -230,6 +230,105 @@ pub enum Disk { } impl Disk { + pub(crate) async fn delete_versions_with_namespace_owner( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + namespace_owner: Option>, + ) -> Vec> { + match self { + Self::Local(disk) => { + disk.delete_versions_with_namespace_owner(volume, versions, opts, namespace_owner) + .await + } + Self::Remote(disk) => { + let _owner = namespace_owner; + disk.delete_versions(volume, versions, opts).await + } + } + } + + pub(crate) async fn delete_versions_recoverable_with_namespace_owner( + &self, + volume: &str, + versions: Vec, + mut opts: DeleteOptions, + namespace_owner: Option>, + ) -> Vec> { + if let Err(error) = validate_delete_recovery_options(&opts) { + return vec![Some(error); versions.len()]; + } + if versions.iter().any(|version| version.name.starts_with('/')) { + return vec![Some(DiskError::InvalidPath); versions.len()]; + } + opts.recoverable_delete = true; + match self { + Self::Local(disk) => { + disk.delete_versions_with_namespace_owner(volume, versions, opts, namespace_owner) + .await + } + Self::Remote(disk) => { + let _owner = namespace_owner; + disk.delete_versions_recoverable(volume, versions, opts).await + } + } + } + + pub(crate) async fn settle_delete_transaction_with_namespace_owner( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + namespace_owner: Option>, + ) -> Result { + match self { + Self::Local(disk) => { + disk.settle_delete_transaction_with_namespace_owner( + volume, + object, + transaction, + expected_fingerprint, + action, + namespace_owner, + ) + .await + } + Self::Remote(disk) => { + let _owner = namespace_owner; + disk.settle_delete_transaction(volume, object, transaction, expected_fingerprint, action) + .await + } + } + } + + pub(crate) async fn delete_version_recoverable_with_namespace_owner( + &self, + volume: &str, + path: &str, + fi: FileInfo, + force_del_marker: bool, + mut opts: DeleteOptions, + namespace_owner: Option>, + ) -> Result<()> { + validate_delete_recovery_options(&opts)?; + if path.starts_with('/') { + return Err(DiskError::InvalidPath); + } + opts.recoverable_delete = true; + match self { + Self::Local(disk) => { + disk.delete_version_with_namespace_owner(volume, path, fi, force_del_marker, opts, namespace_owner) + .await + } + Self::Remote(disk) => { + disk.delete_version_recoverable(volume, path, fi, force_del_marker, opts) + .await + } + } + } pub(crate) async fn set_disk_id_state(&self, id: Option) -> Result<()> { match self { Disk::Local(local_disk) => { @@ -257,6 +356,70 @@ impl Disk { #[async_trait::async_trait] impl DiskAPI for Disk { + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> Result<()> { + self.delete_version_recoverable_with_namespace_owner(volume, object, file_info, force_del_marker, opts, None) + .await + } + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + ) -> Vec> { + match self { + Self::Local(disk) => disk.delete_versions_recoverable(volume, versions, opts).await, + Self::Remote(disk) => disk.delete_versions_recoverable(volume, versions, opts).await, + } + } + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> Result { + match self { + Self::Local(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await, + Self::Remote(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await, + } + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + ) -> Result { + match self { + Self::Local(disk) => disk.inspect_delete_transaction(volume, object, transaction).await, + Self::Remote(disk) => disk.inspect_delete_transaction(volume, object, transaction).await, + } + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + ) -> Result { + match self { + Self::Local(disk) => { + disk.settle_delete_transaction(volume, object, transaction, expected_fingerprint, action) + .await + } + Self::Remote(disk) => { + disk.settle_delete_transaction(volume, object, transaction, expected_fingerprint, action) + .await + } + } + } fn to_string(&self) -> String { match self { Disk::Local(local_disk) => local_disk.to_string(), @@ -1246,6 +1409,51 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { Err(DiskError::MethodNotAllowed) } async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; + async fn discover_delete_transactions( + &self, + _volume: &str, + _object: Option<&str>, + _cursor: Option, + _limit: u32, + ) -> Result { + Err(DiskError::MethodNotAllowed) + } + async fn delete_version_recoverable( + &self, + _volume: &str, + _object: &str, + _file_info: FileInfo, + _force_del_marker: bool, + _opts: DeleteOptions, + ) -> Result<()> { + Err(DiskError::MethodNotAllowed) + } + async fn delete_versions_recoverable( + &self, + _volume: &str, + versions: Vec, + _opts: DeleteOptions, + ) -> Vec> { + vec![Some(DiskError::MethodNotAllowed); versions.len()] + } + async fn inspect_delete_transaction( + &self, + _volume: &str, + _object: &str, + _transaction: Uuid, + ) -> Result { + Err(DiskError::MethodNotAllowed) + } + async fn settle_delete_transaction( + &self, + _volume: &str, + _object: &str, + _transaction: Uuid, + _expected_fingerprint: DeleteRecoveryDigest, + _action: DeleteRecoveryAction, + ) -> Result { + Err(DiskError::MethodNotAllowed) + } // VerifyFile async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result; // CheckParts @@ -1582,6 +1790,269 @@ pub struct DeleteOptions { /// Remote calls carrying it must use DeleteRetiredMarker, never DeleteVersion. #[serde(default, skip_serializing_if = "Option::is_none")] pub expected_delete_marker: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delete_recovery_bucket_incarnation: Option, + #[serde(default)] + pub recoverable_delete: bool, +} + +pub const DELETE_RECOVERY_PROTOCOL_VERSION: u32 = 1; +pub(crate) fn get_delete_recovery_timeout() -> Result { + let value = std::env::var("RUSTFS_DELETE_RECOVERY_TIMEOUT_SECS"); + match value { + Ok(value) => parse_delete_recovery_timeout(Some(&value)), + Err(std::env::VarError::NotPresent) => parse_delete_recovery_timeout(None), + Err(std::env::VarError::NotUnicode(_)) => Err(DiskError::other("invalid delete recovery timeout configuration")), + } +} + +fn parse_delete_recovery_timeout(value: Option<&str>) -> Result { + let seconds = match value { + None => 900, + Some(value) => value + .parse::() + .ok() + .filter(|seconds| (30..=86400).contains(seconds)) + .ok_or_else(|| DiskError::other("delete recovery timeout must be an integer between 30 and 86400 seconds"))?, + }; + Ok(std::time::Duration::from_secs(seconds)) +} + +pub(crate) fn validate_delete_recovery_options(opts: &DeleteOptions) -> Result<()> { + if opts.old_data_dir.is_none_or(|id| id.is_nil()) + || opts.undo_write + || opts.undo_delete + || opts.recursive + || opts.immediate + || opts.expected_delete_marker.is_some() + || opts.delete_recovery_bucket_incarnation.is_some_and(|id| id.is_nil()) + { + return Err(DiskError::FileCorrupt); + } + Ok(()) +} +pub type DeleteRecoveryDigest = [u8; 32]; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteTransactionId { + pub object: String, + pub transaction: Uuid, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryCursor { + pub token: Uuid, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryPage { + pub transactions: Vec, + pub next: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryJournal { + pub protocol_version: u32, + pub volume: String, + pub object: String, + pub transaction: Uuid, + pub bucket_incarnation: Option, + pub original_digest: Option, + pub target_digest: Option, + pub data_dirs: Vec, + pub deleted_versions: Vec, + pub requests: Vec, + pub original_versions: Vec, + #[serde(default)] + pub predecessors: Vec, +} + +impl DeleteRecoveryJournal { + /// Checks source generations only; the coordinator must supply a certified + /// decision and the receiver must retain its namespace/incarnation fences. + pub(crate) fn matches_adoption_source(&self, current: &[DeleteRecoveryVersionIdentity]) -> bool { + if self.requests.is_empty() { + return false; + } + let mut originals = std::collections::HashMap::with_capacity(self.original_versions.len()); + for version in &self.original_versions { + originals.entry(version.version_id.unwrap_or_default()).or_insert(version); + } + let mut locals = std::collections::HashMap::with_capacity(current.len()); + for version in current { + locals.entry(version.version_id.unwrap_or_default()).or_insert(version); + } + self.requests.iter().all(|request| { + if request.deleted && request.mark_deleted { + return current.first() == self.original_versions.first(); + } + let id = request.version_id.unwrap_or_default(); + let Some(original) = originals.get(&id) else { + return false; + }; + match locals.get(&id) { + Some(local) => local == original, + None => !id.is_nil(), + } + }) + } + + /// Matches immutable local preparation intent against an independently + /// inspected same-transaction commit receipt. This match is not authority + /// by itself; the coordinator must supply the certified receipt. + pub(crate) fn matches_prepared_authority(&self, authority: &Self) -> bool { + if self.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || authority.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || self.transaction.is_nil() + || self.transaction != authority.transaction + || self.volume != authority.volume + || self.object != authority.object + || self.bucket_incarnation != authority.bucket_incarnation + || self.bucket_incarnation.is_some_and(|id| id.is_nil()) + || self.requests.is_empty() + || self.requests != authority.requests + || self.deleted_versions.iter().collect::>() + != authority.deleted_versions.iter().collect::>() + { + return false; + } + let sources: std::collections::HashMap<_, _> = self + .original_versions + .iter() + .map(|version| (version.version_id.unwrap_or_default(), version)) + .collect(); + let certified: std::collections::HashMap<_, _> = authority + .original_versions + .iter() + .map(|version| (version.version_id.unwrap_or_default(), version)) + .collect(); + if sources.len() != self.original_versions.len() || certified.len() != authority.original_versions.len() { + return false; + } + self.requests.iter().all(|request| { + let (local, remote) = if request.deleted && request.mark_deleted { + (self.original_versions.first(), authority.original_versions.first()) + } else { + let id = request.version_id.unwrap_or_default(); + (sources.get(&id).copied(), certified.get(&id).copied()) + }; + match (local, remote) { + (Some(local), Some(remote)) => local == remote, + (None, None) => self.original_digest.is_none() && authority.original_digest.is_none(), + _ => false, + } + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryVersionIdentity { + pub version_id: Option, + pub mod_time: Option, + pub data_dir: Option, + pub deleted: bool, + pub metadata_digest: DeleteRecoveryDigest, +} + +impl DeleteRecoveryVersionIdentity { + pub(crate) fn from_file_info(info: FileInfo) -> error::Result { + local::delete_recovery::generation(info) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum DeleteRecoveryState { + Absent, + NewPrepared, + QuorumCommitted, + CommitFinished, + LegacyAmbiguous, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteRecoveryInspection { + pub protocol_version: u32, + pub id: DeleteTransactionId, + pub state: DeleteRecoveryState, + pub journal: Option, + pub current_digest: Option, + pub current_versions: Vec, + pub backup_digest: Option, + pub rollback_data_complete: bool, + pub commit_data_safe: bool, + pub explicit_delete_safe: bool, + pub fingerprint: DeleteRecoveryDigest, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum DeleteRecoveryAction { + PublishCommit, + FinishCommit, + FinalizeCommit, + Rollback, + DiscardLegacy, + AdoptCommit { authority: Box }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub enum DeleteRecoveryRpcOperation { + Discover { + object: Option, + cursor: Option, + limit: u32, + }, + Inspect { + object: String, + transaction: Uuid, + }, + Settle { + object: String, + transaction: Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + }, + ApplyVersion { + object: String, + file_info: Box, + force_del_marker: bool, + opts: DeleteOptions, + }, + ApplyVersions { + versions: Vec, + opts: DeleteOptions, + }, +} + +impl DeleteRecoveryRpcOperation { + /// Full metadata application and adoption share the existing mutation envelope. + pub fn max_wire_payload_bytes(&self) -> usize { + match self { + Self::ApplyVersion { .. } + | Self::ApplyVersions { .. } + | Self::Settle { + action: DeleteRecoveryAction::AdoptCommit { .. }, + .. + } => rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES, + _ => rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub enum DeleteRecoveryRpcReply { + Discovered(DeleteRecoveryPage), + Inspected(DeleteRecoveryInspection), + Settled(DeleteRecoveryInspection), + AppliedVersion, + AppliedVersions(Vec>), } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -1872,6 +2343,8 @@ mod tests { undo_delete: false, old_data_dir: Some(Uuid::new_v4()), expected_delete_marker: None, + delete_recovery_bucket_incarnation: None, + recoverable_delete: false, }; assert!(opts.recursive); @@ -2296,3 +2769,183 @@ mod tests { } } } + +#[cfg(test)] +mod delete_recovery_timeout_tests { + use super::*; + #[test] + fn recovery_timeout_defaults_and_validates_range() { + assert_eq!( + parse_delete_recovery_timeout(None).expect("default timeout"), + std::time::Duration::from_secs(900) + ); + for seconds in ["30", "900", "86400"] { + assert!(parse_delete_recovery_timeout(Some(seconds)).is_ok()); + } + for value in ["", "invalid", "0", "2", "29", "86401", "-1", "18446744073709551616"] { + assert!(parse_delete_recovery_timeout(Some(value)).is_err()); + } + } +} + +#[cfg(test)] +mod delete_recovery_wire_limit_tests { + use super::*; + + #[test] + fn boxed_apply_version_preserves_existing_wire_payload() { + #[derive(Serialize)] + enum PriorOperation<'a> { + ApplyVersion { + object: &'a str, + file_info: &'a FileInfo, + force_del_marker: bool, + opts: &'a DeleteOptions, + }, + } + let file_info = FileInfo { + name: "object".into(), + version_id: Some(Uuid::new_v4()), + metadata: std::collections::HashMap::from([("metadata".into(), "value".into())]), + ..Default::default() + }; + let opts = DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + recoverable_delete: true, + delete_recovery_bucket_incarnation: Some(Uuid::new_v4()), + ..Default::default() + }; + let prior = PriorOperation::ApplyVersion { + object: "object", + file_info: &file_info, + force_del_marker: true, + opts: &opts, + }; + let encoded = rmp_serde::to_vec_named(&prior).expect("encode the previous unboxed request shape"); + let operation = DeleteRecoveryRpcOperation::ApplyVersion { + object: "object".into(), + file_info: Box::new(file_info.clone()), + force_del_marker: true, + opts: opts.clone(), + }; + assert_eq!(rmp_serde::to_vec_named(&operation).expect("encode boxed request"), encoded); + assert_eq!( + serde_json::to_value(&operation).expect("boxed request JSON"), + serde_json::to_value(prior).expect("previous request JSON") + ); + let decoded: DeleteRecoveryRpcOperation = + rustfs_protos::decode_delete_recovery_payload(&encoded).expect("decode the previous payload"); + assert_eq!(rmp_serde::to_vec_named(&decoded).expect("re-encode decoded prior request"), encoded); + } + #[test] + fn adoption_and_inspection_reply_support_large_version_journals() { + let transaction = Uuid::new_v4(); + let request = FileInfo { + metadata: std::collections::HashMap::from([("metadata".into(), "x".repeat(2048))]), + ..Default::default() + }; + let journal = DeleteRecoveryJournal { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + volume: "bucket".into(), + object: "object".into(), + transaction, + bucket_incarnation: None, + original_digest: None, + target_digest: None, + data_dirs: Vec::new(), + deleted_versions: Vec::new(), + requests: vec![request; 1000], + original_versions: Vec::new(), + predecessors: Vec::new(), + }; + let operation = DeleteRecoveryRpcOperation::Settle { + object: "object".into(), + transaction, + expected_fingerprint: [0; 32], + action: DeleteRecoveryAction::AdoptCommit { + authority: Box::new(journal.clone()), + }, + }; + let encoded = rmp_serde::to_vec_named(&operation).expect("encode large adoption"); + assert!(encoded.len() > rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES); + assert!(encoded.len() < operation.max_wire_payload_bytes()); + let decoded: DeleteRecoveryRpcOperation = + rustfs_protos::decode_delete_recovery_payload(&encoded).expect("large adoption can decode"); + assert_eq!(decoded.max_wire_payload_bytes(), rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES); + let reply = DeleteRecoveryRpcReply::Inspected(DeleteRecoveryInspection { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + id: DeleteTransactionId { + object: "object".into(), + transaction, + }, + state: DeleteRecoveryState::QuorumCommitted, + journal: Some(journal), + current_digest: None, + current_versions: Vec::new(), + backup_digest: None, + rollback_data_complete: false, + commit_data_safe: true, + explicit_delete_safe: false, + fingerprint: [0; 32], + }); + let encoded = rmp_serde::to_vec_named(&reply).expect("encode large inspection"); + assert!(encoded.len() > rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES); + assert!(rustfs_protos::decode_delete_recovery_payload::(&encoded).is_ok()); + let inspect = DeleteRecoveryRpcOperation::Inspect { + object: "object".into(), + transaction, + }; + assert_eq!(inspect.max_wire_payload_bytes(), rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES); + } +} + +#[cfg(test)] +mod delete_recovery_adoption_tests { + use super::*; + + #[test] + fn source_matching_preserves_non_null_absence_and_rejects_foreign_or_missing_null() { + let id = Uuid::new_v4(); + let source = DeleteRecoveryVersionIdentity { + version_id: Some(id), + mod_time: None, + data_dir: None, + deleted: false, + metadata_digest: [1; 32], + }; + let mut journal = DeleteRecoveryJournal { + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + volume: "bucket".into(), + object: "object".into(), + transaction: Uuid::new_v4(), + bucket_incarnation: None, + original_digest: None, + target_digest: None, + data_dirs: Vec::new(), + deleted_versions: vec![id], + requests: vec![FileInfo { + version_id: Some(id), + ..Default::default() + }], + original_versions: vec![source.clone()], + predecessors: Vec::new(), + }; + assert!(journal.matches_adoption_source(&[]), "certified non-null removal can already be absent"); + assert!(journal.matches_adoption_source(std::slice::from_ref(&source))); + let mut foreign = source.clone(); + foreign.metadata_digest = [2; 32]; + assert!(!journal.matches_adoption_source(std::slice::from_ref(&foreign))); + journal.original_versions.push(foreign.clone()); + assert!( + journal.matches_adoption_source(&[source.clone(), foreign.clone()]), + "retain original first-match semantics" + ); + assert!(!journal.matches_adoption_source(&[foreign, source])); + journal.requests[0].version_id = None; + journal.original_versions[0].version_id = None; + assert!(!journal.matches_adoption_source(&[]), "null absence does not establish its generation"); + assert!(journal.matches_adoption_source(std::slice::from_ref(&journal.original_versions[0]))); + journal.requests.clear(); + assert!(!journal.matches_adoption_source(&journal.original_versions)); + } +} diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 78f8df7a7..1b9048bba 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -938,6 +938,10 @@ pub struct ObjectOptions { /// A genuine target write guard; loss fences alone do not authorize a borrowed write. #[doc(hidden)] pub write_lock_context: Option, + /// In-process delete ownership for late filesystem completion. Never + /// serialized or copied into object metadata. + #[doc(hidden)] + pub delete_mutation_owner: Option>, /// Internal read-only inspection must not enqueue metadata or payload repairs. pub suppress_read_repair: bool, /// Control-plane writers that immediately read or CAS the same namespace @@ -1158,6 +1162,7 @@ impl std::fmt::Debug for ObjectOptions { .field("http_preconditions", &self.http_preconditions.is_some()) .field("delete_replication", &self.delete_replication.is_some()) .field("delete_replication_config_snapshot", &self.delete_replication_config_snapshot) + .field("delete_mutation_owner", &self.delete_mutation_owner.is_some()) .field("namespace_lock_fence", &self.namespace_lock_fence.is_some()) .field("bucket_lifecycle_lock_fence", &self.bucket_lifecycle_lock_fence.is_some()) .field("replication_request", &self.replication_request) diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index bf4bf7a3f..c63244ea8 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -874,6 +874,7 @@ mod ctx; mod metadata; mod ops; pub(crate) use ops::bucket::BucketInfoQuorum; +pub(crate) use ops::delete_recovery::{DeleteRecoveryContext, DeleteRecoverySummary}; pub(crate) use ops::heal::AbsenceProofRequest; #[cfg(test)] pub(crate) use ops::heal::DanglingDeleteFailure; diff --git a/crates/ecstore/src/set_disk/ops/delete_recovery.rs b/crates/ecstore/src/set_disk/ops/delete_recovery.rs new file mode 100644 index 000000000..71d76f8b8 --- /dev/null +++ b/crates/ecstore/src/set_disk/ops/delete_recovery.rs @@ -0,0 +1,1992 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Set-wide delete decisions and replay. Decisions remain until every +//! participant and related transaction for the exact key converges. + +use crate::disk::error::{DiskError, Result, RpcStatusError}; +use crate::disk::{ + DELETE_RECOVERY_PROTOCOL_VERSION, DeleteRecoveryAction, DeleteRecoveryInspection, DeleteRecoveryState, DeleteTransactionId, + DiskAPI, DiskStore, +}; +use crate::set_disk::SetDisks; +use futures::future::join_all; +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::sync::Arc; +use uuid::Uuid; + +const DISCOVERY_PAGE_SIZE: u32 = 128; +const MAX_DISCOVERED_TRANSACTIONS: usize = 1_000_000; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct DeleteRecoverySummary { + pub discovered: usize, + pub committed: usize, + pub rolled_back: usize, + pub blocked: usize, + pub pending: usize, + pub reclaimed: usize, + /// Read-only evidence for ordinary missing-key S3 semantics. This never + /// authorizes settlement while a configured participant is unavailable. + pub no_transactions_at_quorum: bool, +} + +#[derive(Clone)] +pub(crate) struct DeleteRecoveryContext<'a> { + pub incarnation: Option, + pub dry_run: bool, + pub explicit_delete: bool, + /// Only the locked heal path may repair a live marker-creation source. + pub repair_marker_source: bool, + pub namespace_owner: Option>, + pub can_commit: &'a (dyn Fn() -> bool + Sync), +} + +impl<'a> DeleteRecoveryContext<'a> { + pub(crate) fn new(incarnation: Option, can_commit: &'a (dyn Fn() -> bool + Sync)) -> Self { + Self { + incarnation, + dry_run: false, + explicit_delete: false, + repair_marker_source: false, + namespace_owner: None, + can_commit, + } + } +} + +impl DeleteRecoverySummary { + pub(crate) fn add(&mut self, other: Self) { + self.discovered += other.discovered; + self.committed += other.committed; + self.rolled_back += other.rolled_back; + self.blocked += other.blocked; + self.pending += other.pending; + self.reclaimed += other.reclaimed; + } +} + +pub(in crate::set_disk) fn unsupported_recovery(error: &DiskError) -> bool { + matches!(error, DiskError::UnsupportedDisk) + || matches!(error, DiskError::Io(source) if source.get_ref() + .and_then(|source| source.downcast_ref::()) + .is_some_and(|status| status.status().code() == tonic::Code::Unimplemented)) +} + +/// Set authority needs generation identities, not the potentially large +/// per-version apply requests carried by returning-drive adoption. +struct DecisionEvidence { + transaction: Uuid, + deleted_versions: HashSet, + original_versions: HashSet, + predecessors: HashSet, + target_absent: bool, +} + +fn receipt(state: DeleteRecoveryState) -> bool { + matches!(state, DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished) +} + +fn validate(inspection: &DeleteRecoveryInspection, object: &str, transaction: Uuid) -> Result<()> { + if inspection.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION + || inspection.id.object != object + || inspection.id.transaction != transaction + { + return Err(DiskError::FileCorrupt); + } + Ok(()) +} + +async fn inspect( + disks: &[Option], + bucket: &str, + object: &str, + transaction: Uuid, +) -> Vec> { + join_all(disks.iter().map(|disk| async move { + let state = disk + .as_ref() + .ok_or(DiskError::DiskNotFound)? + .inspect_delete_transaction(bucket, object, transaction) + .await?; + validate(&state, object, transaction)?; + Ok(state) + })) + .await +} + +async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Result> { + let mut cursor = None; + let mut ids = BTreeSet::new(); + loop { + let page = disk + .discover_delete_transactions(bucket, object, cursor, DISCOVERY_PAGE_SIZE) + .await?; + if page.transactions.len() > usize::try_from(DISCOVERY_PAGE_SIZE).map_err(|_| DiskError::FileCorrupt)? { + return Err(DiskError::FileCorrupt); + } + for id in page.transactions { + if id.transaction.is_nil() || object.is_some_and(|object| object != id.object) { + return Err(DiskError::FileCorrupt); + } + ids.insert((id.object, id.transaction)); + if ids.len() > MAX_DISCOVERED_TRANSACTIONS { + return Err(DiskError::other("delete recovery transaction limit exceeded")); + } + } + cursor = page.next; + if cursor.is_none() { + break; + } + } + Ok(ids + .into_iter() + .map(|(object, transaction)| DeleteTransactionId { object, transaction }) + .collect()) +} + +async fn settle( + disk: &DiskStore, + bucket: &str, + state: &DeleteRecoveryInspection, + action: DeleteRecoveryAction, + can_commit: &(impl Fn() -> bool + Sync + ?Sized), + namespace_owner: Option<&Arc>, +) -> Result { + if !can_commit() { + return Err(DiskError::other("delete recovery namespace fence was lost")); + } + let result = disk + .settle_delete_transaction_with_namespace_owner( + bucket, + &state.id.object, + state.id.transaction, + state.fingerprint, + action, + namespace_owner.cloned(), + ) + .await; + let next = result?; + validate(&next, &state.id.object, state.id.transaction)?; + Ok(next) +} + +/// Each phase fans out across independent drives, and awaits all started work +/// before evaluating the next decision or cleanup barrier. +async fn settle_group( + disks: &[Option], + bucket: &str, + copies: &mut [DeleteRecoveryInspection], + action: impl Fn(&DeleteRecoveryInspection) -> Option + Sync, + can_commit: &(impl Fn() -> bool + Sync + ?Sized), + namespace_owner: Option<&Arc>, +) -> Result { + let results = join_all(disks.iter().zip(copies.iter()).map(|(disk, copy)| { + let action = action(copy); + async move { + match action { + Some(action) => settle( + disk.as_ref().ok_or(DiskError::DiskNotFound)?, + bucket, + copy, + action, + can_commit, + namespace_owner, + ) + .await + .map(Some), + None => Ok(None), + } + } + })) + .await; + let mut pending = false; + for (copy, result) in copies.iter_mut().zip(results) { + match result { + Ok(Some(next)) => *copy = next, + Ok(None) => {} + Err(DiskError::DiskOngoingReq) => pending = true, + Err(error) => return Err(error), + } + } + Ok(pending) +} + +/// A later decision may require settling its prepared predecessor before an +/// untouched participant can adopt that decision. Ordering provides no commit +/// authority; reject contradictory causal records before any settlement. +fn causal_transaction_order(states: &BTreeMap>) -> Result> { + let mut indegrees: BTreeMap<_, usize> = states.keys().map(|id| (*id, 0)).collect(); + let mut successors: BTreeMap> = BTreeMap::new(); + for (transaction, copies) in states { + for predecessor in copies + .iter() + .filter_map(|copy| copy.journal.as_ref()) + .flat_map(|journal| &journal.predecessors) + { + if states.contains_key(predecessor) && successors.entry(*predecessor).or_default().insert(*transaction) { + let degree = indegrees.get_mut(transaction).ok_or(DiskError::FileCorrupt)?; + *degree = degree.checked_add(1).ok_or(DiskError::FileCorrupt)?; + } + } + } + let mut ready: BTreeSet<_> = indegrees + .iter() + .filter_map(|(id, degree)| (*degree == 0).then_some(*id)) + .collect(); + let mut order = Vec::with_capacity(states.len()); + while let Some(transaction) = ready.pop_first() { + order.push(transaction); + if let Some(children) = successors.get(&transaction) { + for child in children { + let degree = indegrees.get_mut(child).ok_or(DiskError::FileCorrupt)?; + *degree = degree.checked_sub(1).ok_or(DiskError::FileCorrupt)?; + if *degree == 0 { + ready.insert(*child); + } + } + } + } + if order.len() != states.len() { + return Err(DiskError::FileCorrupt); + } + Ok(order) +} + +impl SetDisks { + /// Publication follows the ordinary metadata quorum. Once a publication is + /// attempted, response loss must never trigger blind rollback. + pub(in crate::set_disk) async fn publish_delete_commit( + &self, + bucket: &str, + object: &str, + transaction: Uuid, + disks: &[Option], + write_quorum: usize, + mut context: DeleteRecoveryContext<'_>, + ) -> Result<()> { + if context.namespace_owner.is_none() { + context.namespace_owner = Some(self.ctx.begin_namespace_commit()); + } + let can_commit = context.can_commit; + let states = inspect(disks, bucket, object, transaction).await; + let untouched_absence = |state: &Result| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none()); + if states.iter().filter(|state| untouched_absence(state)).count() >= write_quorum + && states.iter().all(|state| { + untouched_absence(state) || matches!(state, Err(error) if unavailable(error) && !unsupported_recovery(error)) + }) + { + // An ordinary missing-key no-op requires no recovery decision. + // Do not manufacture witness transactions while a drive is offline. + return Ok(()); + } + let published = join_all(disks.iter().zip(&states).map(|(disk, state)| async { + let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?; + let state = state.as_ref().map_err(Clone::clone)?; + if state.state == DeleteRecoveryState::LegacyAmbiguous { + return Err(DiskError::FileCorrupt); + } + settle( + disk, + bucket, + state, + DeleteRecoveryAction::PublishCommit, + can_commit, + context.namespace_owner.as_ref(), + ) + .await + })) + .await; + if published + .iter() + .filter(|state| matches!(state, Ok(state) if receipt(state.state))) + .count() + < write_quorum + { + return Err(DiskError::ErasureWriteQuorum); + } + // Physical cleanup is retryable after the certified decision. An + // unavailable participant retains the decision for its eventual return. + let _ = self.recover_delete_key(bucket, object, context).await; + Ok(()) + } + + pub(crate) async fn recover_delete_bucket( + &self, + bucket: &str, + context: DeleteRecoveryContext<'_>, + ) -> Result { + self.recover_delete_prefix(bucket, "", context).await + } + + pub(crate) async fn recover_delete_prefix( + &self, + bucket: &str, + prefix: &str, + context: DeleteRecoveryContext<'_>, + ) -> Result { + let disks = self.disk_inventory().await; + if disks.len() != self.set_drive_count || disks.iter().any(Option::is_none) { + return Ok(DeleteRecoverySummary { + pending: 1, + ..Default::default() + }); + } + let pages = join_all(disks.iter().flatten().map(|disk| discover(disk, bucket, None))).await; + let mut keys = BTreeSet::new(); + for page in pages { + match page { + Ok(ids) => { + keys.extend( + ids.into_iter() + .map(|id| id.object) + .filter(|object| object.starts_with(prefix)), + ); + } + Err(error) if unavailable(&error) => { + return Ok(DeleteRecoverySummary { + pending: 1, + ..Default::default() + }); + } + Err(error) => return Err(error), + } + } + let mut summary = DeleteRecoverySummary::default(); + // Lock order: caller's bucket lifecycle fence, then one exact key. + for object in keys { + summary.add(self.recover_delete_object(bucket, &object, context.clone()).await?); + } + Ok(summary) + } + + pub(crate) async fn recover_delete_object( + &self, + bucket: &str, + object: &str, + mut context: DeleteRecoveryContext<'_>, + ) -> Result { + // Lock order: caller's bucket lifecycle fence, then one exact key. + let guard = Arc::new( + self.acquire_write_lock_diag("recover_delete", bucket, object) + .await + .map_err(DiskError::other)?, + ); + if !context.dry_run { + let namespace_owner = context + .namespace_owner + .clone() + .unwrap_or_else(|| self.ctx.begin_namespace_commit()); + context.namespace_owner = Some(Arc::new((namespace_owner, guard.clone()))); + } + let can_commit = context.can_commit; + let fence = || can_commit() && !guard.is_lock_lost(); + context.can_commit = &fence; + self.recover_delete_key(bucket, object, context).await + } + + /// Caller retains the exact key's write lock and its lifecycle owner. + pub(in crate::set_disk) async fn recover_delete_key( + &self, + bucket: &str, + object: &str, + context: DeleteRecoveryContext<'_>, + ) -> Result { + let DeleteRecoveryContext { + incarnation, + dry_run, + explicit_delete, + repair_marker_source, + namespace_owner, + can_commit, + } = context; + let disks = self.disk_inventory().await; + let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some); + let pages = join_all(disks.iter().flatten().map(|disk| discover(disk, bucket, Some(object)))).await; + let mut transactions = BTreeSet::new(); + let mut inspected = 0; + let mut offline = !complete_inventory; + let mut unsupported = false; + for page in pages { + match page { + Ok(ids) => { + inspected += 1; + transactions.extend(ids.into_iter().map(|id| id.transaction)); + } + Err(error) if unavailable(&error) => { + offline = true; + unsupported |= unsupported_recovery(&error); + } + Err(error) => return Err(error), + } + } + if offline { + return Ok(DeleteRecoverySummary { + discovered: transactions.len(), + pending: 1, + no_transactions_at_quorum: transactions.is_empty() && !unsupported && inspected > self.set_drive_count / 2, + ..Default::default() + }); + } + let mut summary = DeleteRecoverySummary { + discovered: transactions.len(), + ..Default::default() + }; + let mut all_states = BTreeMap::new(); + for transaction in transactions { + let mut complete = Vec::with_capacity(disks.len()); + for state in inspect(&disks, bucket, object, transaction).await { + match state { + Ok(state) => { + if let Some(journal) = &state.journal { + let witness = journal.original_digest.is_none() + && journal.target_digest.is_none() + && journal.requests.is_empty(); + if incarnation.is_some_and(|expected| journal.bucket_incarnation != Some(expected) && !witness) { + summary.blocked += 1; + return Ok(summary); + } + } + complete.push(state); + } + Err(error) if unavailable(&error) => { + summary.pending += 1; + return Ok(summary); + } + Err(error) => return Err(error), + } + } + all_states.insert(transaction, complete); + } + let prerequisite = if repair_marker_source && !dry_run { + let mut active = all_states + .iter() + .filter(|(_, copies)| copies.iter().any(|copy| copy.state != DeleteRecoveryState::Absent)); + let first = active.next(); + if active.next().is_none() { + first.and_then(|(&transaction, copies)| { + marker_source_to_repair(copies, incarnation).map(|source| (transaction, source)) + }) + } else { + None + } + } else { + None + }; + if let Some((transaction, source)) = prerequisite { + Box::pin(self.heal_delete_recovery_source(bucket, object, &source, can_commit)).await?; + let refreshed = inspect(&disks, bucket, object, transaction) + .await + .into_iter() + .collect::>>()?; + if !can_commit() { + return Err(DiskError::DiskOngoingReq); + } + if refreshed.iter().any(|copy| { + copy.journal + .as_ref() + .is_some_and(|journal| journal.bucket_incarnation != incarnation) + }) { + return Err(DiskError::OutdatedXLMeta); + } + all_states.insert(transaction, refreshed); + } + let transaction_order = causal_transaction_order(&all_states)?; + // Legacy snapshots contain no set decision. A fresh authorized removal + // applies only after the entire key is absent and every raw shape is safe. + let legacy_only = explicit_delete + && all_states.values().flatten().all(|copy| { + copy.current_digest.is_none() + && matches!(copy.state, DeleteRecoveryState::Absent | DeleteRecoveryState::LegacyAmbiguous) + && (copy.state != DeleteRecoveryState::LegacyAmbiguous || copy.explicit_delete_safe) + }); + // This is a fresh authorized delete of an absent unversioned key, + // rather than evidence about the outcome of a historical transaction. + let fresh_absence_decision = explicit_delete + && all_states.values().flatten().all(|copy| { + copy.current_digest.is_none() + && match copy.state { + DeleteRecoveryState::Absent => true, + DeleteRecoveryState::NewPrepared => copy.explicit_delete_safe, + DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished => { + copy.journal.as_ref().is_some_and(|journal| journal.target_digest.is_none()) + } + DeleteRecoveryState::LegacyAmbiguous => false, + } + }); + // Receipt cleanup alone does not mutate xl.meta. Returning peers and + // undecided preparations still need fences before and after recovery. + let metadata_may_change = all_states.values().any(|copies| { + !copies.iter().all(|copy| copy.state == DeleteRecoveryState::Absent) + && copies.iter().any(|copy| { + copy.state == DeleteRecoveryState::NewPrepared + || (copy.state == DeleteRecoveryState::Absent && copy.current_digest.is_some()) + || copy + .journal + .as_ref() + .is_some_and(|journal| copy.current_digest != journal.target_digest) + }) + }); + let namespace_owner = if dry_run { + None + } else { + Some(namespace_owner.unwrap_or_else(|| self.ctx.begin_namespace_commit())) + }; + if !dry_run && metadata_may_change { + self.invalidate_get_object_metadata_cache(bucket, object).await; + } + let authorities = all_states + .values() + .flatten() + .filter(|state| receipt(state.state)) + .filter_map(|state| state.journal.as_ref()) + .map(|journal| DecisionEvidence { + transaction: journal.transaction, + deleted_versions: journal.deleted_versions.iter().copied().collect(), + original_versions: journal.original_versions.iter().cloned().collect(), + predecessors: journal.predecessors.iter().copied().collect(), + target_absent: journal.target_digest.is_none(), + }) + .collect::>(); + let mut unresolved = false; + let mut mutated = false; + for transaction in transaction_order { + let copies = all_states.get_mut(&transaction).ok_or(DiskError::FileCorrupt)?; + if mutated { + // A sibling may have restored metadata or reclaimed shared + // evidence. Refresh fingerprints before acting on this group. + *copies = inspect(&disks, bucket, object, transaction) + .await + .into_iter() + .collect::>>()?; + } + if copies.iter().all(|state| state.state == DeleteRecoveryState::Absent) { + continue; + } + if copies.iter().any(|state| state.state == DeleteRecoveryState::LegacyAmbiguous) { + if legacy_only { + if dry_run { + summary.reclaimed += 1; + continue; + } + mutated = true; + if settle_group( + &disks, + bucket, + copies, + |copy| { + (copy.state == DeleteRecoveryState::LegacyAmbiguous).then_some(DeleteRecoveryAction::DiscardLegacy) + }, + can_commit, + namespace_owner.as_ref(), + ) + .await? + { + summary.pending += 1; + unresolved = true; + } else { + summary.reclaimed += 1; + } + } else { + summary.blocked += 1; + unresolved = true; + } + continue; + } + let authority = copies + .iter() + .filter(|state| receipt(state.state)) + .filter_map(|state| state.journal.as_ref()) + .max_by_key(|journal| !journal.requests.is_empty()) + .cloned(); + let prepared = copies.iter().find_map(|state| state.journal.as_ref()); + let covered = prepared + .filter(|journal| !journal.deleted_versions.is_empty()) + .is_some_and(|journal| { + authorities.iter().any(|authority| { + journal.deleted_versions.iter().all(|version| { + authority.deleted_versions.contains(version) + && (!version.is_nil() + || journal + .original_versions + .iter() + .filter(|source| source.version_id.is_none_or(|id| id.is_nil())) + .all(|source| authority.original_versions.contains(source)) + || (authority.predecessors.contains(&journal.transaction) + && journal.target_digest.is_none() + && authority.target_absent + && copies.iter().all(|copy| { + copy.current_digest.is_none() || copy.current_versions == journal.original_versions + }))) + }) + }) + }); + let original_quorum = prepared.is_some_and(|journal| { + copies + .iter() + .filter(|copy| copy.current_versions == journal.original_versions) + .count() + > self.set_drive_count / 2 + }); + let all_decisions_precede = !authorities.is_empty() + && authorities.iter().all(|authority| { + copies + .iter() + .filter_map(|copy| copy.journal.as_ref()) + .any(|journal| journal.predecessors.contains(&authority.transaction)) + }); + if authority.is_some() || covered || fresh_absence_decision { + let certified_same_transaction = authority.is_some(); + let reference = authority + .or_else(|| copies.iter().find_map(|state| state.journal.clone())) + .ok_or(DiskError::FileCorrupt)?; + if dry_run { + if copies.iter().any(|copy| { + if certified_same_transaction + && copy.state == DeleteRecoveryState::NewPrepared + && copy.current_digest.is_none() + && !copy.commit_data_safe + { + copy.journal + .as_ref() + .is_none_or(|local| !local.matches_prepared_authority(&reference)) + } else if copy.state == DeleteRecoveryState::Absent { + copy.current_digest.is_some() && !reference.matches_adoption_source(©.current_versions) + } else { + !copy.commit_data_safe + } + }) { + summary.blocked += 1; + } else { + summary.committed += 1; + } + continue; + } + mutated = true; + let mut pending = settle_group( + &disks, + bucket, + copies, + |copy| { + Some( + if copy.state == DeleteRecoveryState::Absent + || (certified_same_transaction + && copy.state == DeleteRecoveryState::NewPrepared + && copy.current_digest.is_none() + && !copy.commit_data_safe) + { + DeleteRecoveryAction::AdoptCommit { + authority: Box::new(reference.clone()), + } + } else { + DeleteRecoveryAction::PublishCommit + }, + ) + }, + can_commit, + namespace_owner.as_ref(), + ) + .await?; + if !pending { + pending = settle_group( + &disks, + bucket, + copies, + |_| Some(DeleteRecoveryAction::FinishCommit), + can_commit, + namespace_owner.as_ref(), + ) + .await?; + } + if pending { + summary.pending += 1; + unresolved = true; + } else { + summary.committed += 1; + } + } else { + // A sibling decision or foreign generation can supersede this + // snapshot. Never guess by restoring the whole old backup. + if (!authorities.is_empty() && !original_quorum && !all_decisions_precede) + || copies + .iter() + .any(|state| state.state != DeleteRecoveryState::Absent && !state.rollback_data_complete) + { + summary.blocked += 1; + unresolved = true; + continue; + } + let originals = copies + .iter() + .filter_map(|state| state.journal.as_ref()) + .flat_map(|journal| journal.original_versions.iter()) + .collect::>(); + if copies.iter().any(|state| { + state.state == DeleteRecoveryState::Absent + && state.current_versions.iter().any(|version| !originals.contains(version)) + }) { + summary.blocked += 1; + unresolved = true; + continue; + } + if dry_run { + summary.rolled_back += 1; + continue; + } + mutated = true; + if settle_group( + &disks, + bucket, + copies, + |copy| (copy.state == DeleteRecoveryState::NewPrepared).then_some(DeleteRecoveryAction::Rollback), + can_commit, + namespace_owner.as_ref(), + ) + .await? + { + summary.pending += 1; + unresolved = true; + } else { + summary.rolled_back += 1; + } + } + } + if !unresolved && !dry_run { + // GC is a key-wide barrier: every predecessor must have settled + // before the last surviving commit witness can disappear. + let mut settled = Vec::with_capacity(all_states.len()); + for transaction in all_states.keys() { + let fresh = inspect(&disks, bucket, object, *transaction).await; + if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) { + summary.pending += 1; + return Ok(summary); + } + settled.push(fresh.into_iter().collect::>>()?); + } + for (index, mut copies) in settled.into_iter().enumerate() { + if index > 0 { + copies = inspect(&disks, bucket, object, copies[0].id.transaction) + .await + .into_iter() + .collect::>>()?; + if copies + .iter() + .any(|copy| !matches!(copy.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished)) + { + summary.pending += 1; + return Ok(summary); + } + } + if settle_group( + &disks, + bucket, + &mut copies, + |copy| (copy.state == DeleteRecoveryState::CommitFinished).then_some(DeleteRecoveryAction::FinalizeCommit), + can_commit, + namespace_owner.as_ref(), + ) + .await? + { + summary.pending += 1; + } + } + } + + if !dry_run && metadata_may_change { + self.invalidate_get_object_metadata_cache(bucket, object).await; + } + Ok(summary) + } +} + +// A lagging member may have the complete older suffix of a marker creation's +// live source. Repair that source before applying the original strict matcher. +fn marker_source_to_repair( + copies: &[DeleteRecoveryInspection], + incarnation: Option, +) -> Option { + let expected = incarnation.filter(|id| !id.is_nil())?; + let authority = copies + .iter() + .filter(|copy| receipt(copy.state)) + .find_map(|copy| copy.journal.as_ref())?; + let source = authority.original_versions.first()?; + let request = authority.requests.first()?; + if authority.bucket_incarnation != Some(expected) + || authority.requests.len() != 1 + || !request.deleted + || !request.mark_deleted + || request.version_id.is_none_or(|id| id.is_nil()) + || authority + .original_versions + .iter() + .any(|version| version.version_id == request.version_id) + || !authority.deleted_versions.is_empty() + || !authority.data_dirs.is_empty() + || authority.target_digest.is_none() + || source.version_id.is_none_or(|id| id.is_nil()) + || source.deleted + { + return None; + } + let mut missing_source = false; + for copy in copies { + match copy.state { + DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished => { + let journal = copy.journal.as_ref()?; + if journal.bucket_incarnation != Some(expected) + || journal.original_versions != authority.original_versions + || !journal.matches_prepared_authority(authority) + || copy.current_digest != journal.target_digest + { + return None; + } + } + DeleteRecoveryState::Absent => { + if copy.journal.is_some() + || copy.current_versions.is_empty() + || !authority.original_versions.ends_with(©.current_versions) + { + return None; + } + missing_source |= copy.current_versions.first() != Some(source); + } + _ => return None, + } + } + missing_source.then(|| source.clone()) +} + +fn unavailable(error: &DiskError) -> bool { + if unsupported_recovery(error) { + return true; + } + match error { + DiskError::DiskNotFound + | DiskError::FaultyDisk + | DiskError::DiskOngoingReq + | DiskError::Timeout + | DiskError::RemoteClientUnavailable(_) => true, + DiskError::Io(source) => { + if let Some(status) = crate::cluster::rpc::client::embedded_tonic_status(source) { + return crate::cluster::rpc::client::is_network_like_status(status); + } + matches!( + source.kind(), + std::io::ErrorKind::TimedOut + | std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::NotConnected + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::UnexpectedEof + ) + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bucket::metadata_sys::{self, init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::disk::{DeleteOptions, ReadOptions, STORAGE_FORMAT_FILE}; + use crate::object_api::{ObjectOptions, PutObjReader}; + use crate::storage_api_contracts::bucket::{BucketOperations, DeleteBucketOptions, MakeBucketOptions}; + use crate::storage_api_contracts::object::{ObjectIO, ObjectOperations}; + use crate::store::ECStore; + use rustfs_filemeta::FileInfo; + use rustfs_heal_contracts::heal_channel::HealOpts; + use std::sync::Arc; + use tokio::io::AsyncReadExt; + + async fn fixture(bucket: &str) -> (Vec, Arc, Vec) { + let (dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative recovery bucket"); + let disks = store.pools[0].disk_set[0] + .disk_inventory() + .await + .into_iter() + .map(|disk| disk.expect("fixture drive is online")) + .collect(); + (dirs, store, disks) + } + + async fn put(store: &Arc, bucket: &str, body: &[u8], versioned: bool) -> Option { + let mut reader = PutObjReader::from_vec(body.to_vec()); + store + .put_object( + bucket, + "object", + &mut reader, + &ObjectOptions { + versioned, + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("put recovery source") + .version_id + } + + async fn applied(store: &Arc, disks: &[DiskStore], bucket: &str, version: Option, count: usize) -> Uuid { + applied_at(store, disks, bucket, version, count, Uuid::new_v4()).await + } + + async fn applied_at( + store: &Arc, + disks: &[DiskStore], + bucket: &str, + version: Option, + count: usize, + transaction: Uuid, + ) -> Uuid { + let incarnation = store + .bucket_incarnation_id_from_disk(bucket) + .await + .expect("source bucket identity"); + for disk in disks.iter().take(count) { + disk.delete_version_recoverable( + bucket, + "object", + FileInfo { + name: "object".to_string(), + version_id: version, + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + delete_recovery_bucket_incarnation: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("apply prepared delete without publishing a set decision"); + } + transaction + } + + async fn publish(disk: &DiskStore, bucket: &str, transaction: Uuid) { + let state = disk + .inspect_delete_transaction(bucket, "object", transaction) + .await + .expect("inspect prepared source"); + disk.settle_delete_transaction(bucket, "object", transaction, state.fingerprint, DeleteRecoveryAction::PublishCommit) + .await + .expect("persist a decision after the simulated metadata quorum"); + } + + async fn body(store: &Arc, bucket: &str, version: Option) -> Vec { + let opts = ObjectOptions { + version_id: version.map(|version| version.to_string()), + ..Default::default() + }; + let mut reader = store + .get_object_reader(bucket, "object", None, http::HeaderMap::new(), &opts) + .await + .expect("open recovered object"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("read exact recovered bytes"); + body + } + + #[tokio::test] + async fn delete_recovery_set_rolls_back_undecided_inline_bytes() { + let bucket = "recover-undecided"; + let (_dirs, store, disks) = fixture(bucket).await; + let original = vec![21; 4096]; + put(&store, bucket, &original, false).await; + applied(&store, &disks, bucket, None, disks.len()).await; + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("recover undisputed prepared delete"); + assert!( + report.detail.contains("rolled_back=1"), + "actual rollback must be reported: {}", + report.detail + ); + assert_eq!(body(&store, bucket, None).await, original); + } + + #[tokio::test] + async fn delete_recovery_marker_source_heal_preserves_foreign_history_and_damaged_data() { + fn snapshot(root: &std::path::Path) -> BTreeMap> { + let mut files = BTreeMap::new(); + let mut pending = vec![root.to_path_buf()]; + while let Some(directory) = pending.pop() { + for entry in std::fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + pending.push(path); + } else { + files.insert(path.strip_prefix(root).unwrap().to_path_buf(), std::fs::read(path).unwrap()); + } + } + } + files + } + + for shape in ["live-backup", "foreign", "partial-history", "damaged-data"] { + let bucket = "recover-marker-source-guards"; + let (dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket( + bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .unwrap(); + let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap(); + let set = &store.pools[0].disk_set[0]; + let disks: Vec<_> = set.disk_inventory().await.into_iter().map(Option::unwrap).collect(); + let original = vec![71; 1024 * 1024]; + let oldest = put(&store, bucket, &original, true).await.unwrap(); + let first = put(&store, bucket, &original, true).await.unwrap(); + set.disks.write().await[3] = None; + let payload = vec![72; 1024 * 1024]; + let source = put(&store, bucket, &payload, true).await.unwrap(); + store + .delete_object( + bucket, + "object", + ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("publish marker while one member lacks the source version"); + set.disks.write().await[3] = Some(disks[3].clone()); + let mut first_info = disks[3] + .read_version("", bucket, "object", &first.to_string(), &ReadOptions::default()) + .await + .unwrap(); + match shape { + "live-backup" => { + let root = dirs[3].path().join(bucket).join("object"); + let backup = root.join(first_info.data_dir.unwrap().to_string()).join("xl.meta.bkp"); + tokio::fs::write(backup, tokio::fs::read(root.join("xl.meta")).await.unwrap()) + .await + .unwrap(); + } + "foreign" => { + first_info.version_id = Some(Uuid::new_v4()); + first_info.mod_time = Some(time::OffsetDateTime::now_utc()); + disks[3].write_metadata("", bucket, "object", first_info).await.unwrap(); + } + "partial-history" => { + disks[3] + .delete_version( + bucket, + "object", + FileInfo { + name: "object".into(), + version_id: Some(oldest), + ..Default::default() + }, + false, + DeleteOptions::default(), + ) + .await + .unwrap(); + } + "damaged-data" => { + // Three metadata sources meet write quorum, but corrupting + // two shards leaves fewer than EC2+2's two required readers. + for (index, disk) in disks.iter().take(2).enumerate() { + let info = disk + .read_version("", bucket, "object", &source.to_string(), &ReadOptions::default()) + .await + .unwrap(); + let part = dirs[index] + .path() + .join(bucket) + .join("object") + .join(info.data_dir.unwrap().to_string()) + .join("part.1"); + let mut bytes = tokio::fs::read(&part).await.unwrap(); + *bytes.last_mut().unwrap() ^= 1; + tokio::fs::write(part, bytes).await.unwrap(); + } + } + _ => unreachable!(), + } + let before: Vec<_> = dirs + .iter() + .map(|dir| snapshot(&dir.path().join(bucket).join("object"))) + .collect(); + let result = store + .heal_object_at_incarnation( + bucket, + "object", + &source.to_string(), + incarnation, + &HealOpts { + scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep, + ..Default::default() + }, + ) + .await; + if shape == "live-backup" { + assert!(result.unwrap().error.is_none(), "plain live backups cannot block source repair"); + assert_eq!(body(&store, bucket, Some(source)).await, payload); + assert_eq!(body(&store, bucket, Some(first)).await, original); + } else { + assert!(result.is_err() || result.unwrap().error.is_some(), "{shape} must fail safely"); + for (dir, expected) in dirs.iter().zip(before) { + assert_eq!(snapshot(&dir.path().join(bucket).join("object")), expected, "{shape}"); + } + } + } + } + + #[tokio::test] + async fn delete_recovery_admin_prefix_excludes_other_keys_and_restores_hidden_match() { + let bucket = "recover-prefix-scope"; + let (dirs, store, disks) = fixture(bucket).await; + let original = vec![27; 4096]; + put(&store, bucket, &original, false).await; + let transaction = applied(&store, &disks, bucket, None, disks.len()).await; + let outside = store + .recover_delete_prefix_at_incarnation(bucket, "other/", None, &HealOpts::default()) + .await + .expect("inspect unrelated prefix"); + assert!(outside.is_none()); + for dir in &dirs { + assert!( + dir.path() + .join(bucket) + .join("object") + .join(transaction.to_string()) + .join("xl.meta.bkp") + .exists() + ); + } + let matching = store + .recover_delete_prefix_at_incarnation(bucket, "obj", None, &HealOpts::default()) + .await + .expect("recover matching hidden key") + .expect("report matching transaction"); + assert!(matching.detail.contains("rolled_back=1")); + assert_eq!(body(&store, bucket, None).await, original); + } + + #[tokio::test] + async fn delete_recovery_set_commits_receipt_and_unblocks_bucket() { + let bucket = "recover-committed"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![22; 1024 * 1024], false).await; + let transaction = applied(&store, &disks, bucket, None, disks.len()).await; + publish(&disks[0], bucket, transaction).await; + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("complete certified delete"); + assert!( + report.detail.contains("committed=1"), + "certified replay must be reported: {}", + report.detail + ); + assert!( + store + .get_object_info(bucket, "object", &ObjectOptions::default()) + .await + .is_err() + ); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("non-force bucket deletion must succeed after proof GC"); + } + + #[tokio::test] + async fn delete_recovery_set_handles_prepared_subset() { + let bucket = "recover-subset"; + let (_dirs, store, disks) = fixture(bucket).await; + let original = vec![23; 4096]; + put(&store, bucket, &original, false).await; + applied(&store, &disks, bucket, None, 1).await; + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("untouched peers remain valid evidence"); + assert!(report.detail.contains("rolled_back=1")); + assert_eq!(body(&store, bucket, None).await, original); + } + + #[tokio::test] + async fn delete_recovery_set_adopts_untouched_returning_peer() { + let bucket = "recover-untouched"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![24; 1024 * 1024], false).await; + let transaction = applied(&store, &disks, bucket, None, 3).await; + for disk in disks.iter().take(3) { + publish(disk, bucket, transaction).await; + } + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("matching untouched generation must converge"); + assert!(report.detail.contains("committed=1")); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("returning old replica must not leave bucket residue"); + } + + #[tokio::test] + async fn delete_recovery_set_retains_decisions_until_offline_peer_returns() { + let bucket = "recover-offline"; + let (dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![25; 4096], false).await; + let transaction = applied(&store, &disks, bucket, None, 4).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + let available = set.disk_inventory().await; + set.publish_delete_commit(bucket, "object", transaction, &available, 3, DeleteRecoveryContext::new(None, &|| true)) + .await + .expect("commit quorum remains available"); + for dir in dirs.iter().take(3) { + assert!( + dir.path() + .join(bucket) + .join("object") + .join(transaction.to_string()) + .join(crate::disk::local::DELETE_QUORUM_COMMITTED_FILE) + .exists(), + "decision must survive an unavailable participant" + ); + } + set.disks.write().await[3] = Some(disks[3].clone()); + store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("returning prepared peer must roll forward"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("proof collection follows complete convergence"); + } + + #[tokio::test] + async fn delete_recovery_set_never_resurrects_prepared_old_null_after_new_delete() { + let bucket = "recover-null-history"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![26; 4096], false).await; + applied(&store, &disks, bucket, None, 4).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + put(&store, bucket, &vec![27; 4096], false).await; + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("later visible generation can commit with quorum"); + set.disks.write().await[3] = Some(disks[3].clone()); + store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("later deletion must supersede hidden old null residue"); + assert!( + store + .get_object_info(bucket, "object", &ObjectOptions::default()) + .await + .is_err(), + "old bytes must never become visible again" + ); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("both generations' transactions converge"); + } + + #[tokio::test] + async fn delete_recovery_set_missing_version_does_not_ack_hidden_rollback() { + let bucket = "recover-version-history"; + let (_dirs, store, disks) = fixture(bucket).await; + metadata_sys::update_in(&store.ctx, bucket, crate::bucket::metadata::BUCKET_VERSIONING_CONFIG, + br#"Enabled"#.to_vec()) + .await.expect("enable versioning"); + let first = put(&store, bucket, &vec![28; 4096], true).await.expect("first version"); + let survivor = vec![29; 4096]; + let second = put(&store, bucket, &survivor, true).await.expect("second version"); + applied(&store, &disks, bucket, Some(first), 4).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + let opts = ObjectOptions { + version_id: Some(first.to_string()), + versioned: true, + ..Default::default() + }; + assert!( + store.delete_object(bucket, "object", opts.clone()).await.is_err(), + "hidden prepared version cannot receive an idempotent success while a peer is unavailable" + ); + set.disks.write().await[3] = Some(disks[3].clone()); + store + .delete_object(bucket, "object", opts) + .await + .expect("recover then execute the currently authorized exact-version delete"); + assert!( + store + .get_object_info( + bucket, + "object", + &ObjectOptions { + version_id: Some(first.to_string()), + ..Default::default() + } + ) + .await + .is_err() + ); + assert_eq!(body(&store, bucket, Some(second)).await, survivor); + } + + #[tokio::test] + async fn delete_recovery_set_dry_run_preserves_every_file() { + let bucket = "recover-dry-run"; + let (dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![30; 4096], false).await; + let transaction = applied(&store, &disks, bucket, None, 4).await; + let before = disks[0] + .inspect_delete_transaction(bucket, "object", transaction) + .await + .expect("before inspection"); + let report = store + .recover_delete_transactions( + bucket, + &HealOpts { + dry_run: true, + ..Default::default() + }, + ) + .await + .expect("dry-run report"); + assert!(report.detail.contains("dry run")); + let after = disks[0] + .inspect_delete_transaction(bucket, "object", transaction) + .await + .expect("after inspection"); + assert_eq!(before.fingerprint, after.fingerprint, "dry run may not modify recovery evidence"); + for dir in dirs { + assert!(!dir.path().join(bucket).join("object").join(STORAGE_FORMAT_FILE).exists()); + } + } + #[tokio::test] + async fn delete_recovery_set_admin_object_restores_hidden_metadata() { + let bucket = "recover-admin-object"; + let (_dirs, store, disks) = fixture(bucket).await; + let payload = vec![41; 4096]; + put(&store, bucket, &payload, false).await; + applied(&store, &disks, bucket, None, 4).await; + let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("bucket identity"); + let result = store + .heal_object_at_incarnation(bucket, "object", "", incarnation, &HealOpts::default()) + .await + .expect("admin object recovery"); + assert!(result.error.is_none(), "recovered object should heal cleanly: {:?}", result.error); + assert_eq!(body(&store, bucket, None).await, payload); + } + + #[tokio::test] + async fn delete_recovery_current_version_delete_preserves_legacy_on_missing_participant() { + use crate::storage_api_contracts::object::ObjectToDelete; + for batch in [false, true] { + let bucket = "recover-current-with-legacy"; + let (dirs, store, disks) = fixture(bucket).await; + let original = vec![44; 1024 * 1024]; + let old_version = put(&store, bucket, &original, true).await.expect("historical version"); + let legacy = Uuid::new_v4(); + disks[3] + .delete_version( + bucket, + "object", + FileInfo { + name: "object".into(), + version_id: Some(old_version), + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(legacy), + ..Default::default() + }, + ) + .await + .expect("historical disk-local mutation without a set decision"); + let backup = dirs[3] + .path() + .join(bucket) + .join("object") + .join(legacy.to_string()) + .join("xl.meta.bkp"); + let before = tokio::fs::read(&backup).await.expect("historical backup"); + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + let current = put(&store, bucket, &vec![45; 4096], true) + .await + .expect("current version at quorum"); + set.disks.write().await[3] = Some(disks[3].clone()); + if batch { + let (_, errors) = store + .delete_objects( + bucket, + vec![ObjectToDelete { + object_name: "object".into(), + version_id: Some(current), + ..Default::default() + }], + ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await; + assert!(errors.iter().all(Option::is_none), "current deletion at quorum: {errors:?}"); + } else { + store + .delete_object( + bucket, + "object", + ObjectOptions { + versioned: true, + version_id: Some(current.to_string()), + ..Default::default() + }, + ) + .await + .expect("current deletion at quorum"); + } + assert_eq!(tokio::fs::read(&backup).await.expect("legacy backup retained"), before); + let state = disks[3] + .inspect_delete_transaction(bucket, "object", legacy) + .await + .expect("legacy inspection"); + assert_eq!(state.state, DeleteRecoveryState::LegacyAmbiguous); + assert_eq!(body(&store, bucket, Some(old_version)).await, original); + } + } + + #[tokio::test] + async fn delete_recovery_dry_run_matches_adoption_of_already_absent_non_null_version() { + let bucket = "recover-preview-non-null"; + let (_dirs, store, disks) = fixture(bucket).await; + let first = put(&store, bucket, &vec![46; 4096], true).await.expect("first version"); + let remaining = vec![47; 4096]; + let second = put(&store, bucket, &remaining, true).await.expect("remaining version"); + disks[3] + .delete_version( + bucket, + "object", + FileInfo { + name: "object".into(), + version_id: Some(first), + ..Default::default() + }, + false, + DeleteOptions::default(), + ) + .await + .expect("returning peer already lacks the explicit version"); + let transaction = applied(&store, &disks, bucket, Some(first), 3).await; + publish(&disks[0], bucket, transaction).await; + let preview = store + .recover_delete_transactions( + bucket, + &HealOpts { + dry_run: true, + ..Default::default() + }, + ) + .await + .expect("preview certified recovery"); + assert!( + preview.detail.contains("committed=1") && preview.detail.contains("blocked=0"), + "{}", + preview.detail + ); + let actual = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("recover certified removal"); + assert!( + actual.detail.contains("committed=1") && actual.detail.contains("blocked=0"), + "{}", + actual.detail + ); + assert_eq!(body(&store, bucket, Some(second)).await, remaining); + assert!( + store + .get_object_info( + bucket, + "object", + &ObjectOptions { + version_id: Some(first.to_string()), + ..Default::default() + } + ) + .await + .is_err() + ); + } + + #[tokio::test] + async fn delete_recovery_set_legacy_requires_explicit_removal() { + let bucket = "recover-legacy-remove"; + let (dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![42; 4096], false).await; + let transaction = Uuid::new_v4(); + for disk in &disks { + disk.delete_version( + bucket, + "object", + FileInfo { + name: "object".to_owned(), + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + .expect("create undecided historical delete shape"); + } + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("automatic report"); + assert!(report.detail.contains("blocked=1")); + for dir in &dirs { + assert!( + dir.path() + .join(bucket) + .join("object") + .join(transaction.to_string()) + .join("xl.meta.bkp") + .exists() + ); + } + let dry = store + .recover_delete_transactions( + bucket, + &HealOpts { + dry_run: true, + remove: true, + ..Default::default() + }, + ) + .await + .expect("authorized-removal dry run"); + assert!(dry.detail.contains("reclaimed=1")); + for dir in &dirs { + assert!(dir.path().join(bucket).join("object").exists()); + } + let report = store + .recover_delete_transactions( + bucket, + &HealOpts { + remove: true, + ..Default::default() + }, + ) + .await + .expect("explicit admin removal"); + assert!(report.detail.contains("reclaimed=1")); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("non-force bucket deletion"); + } + + #[tokio::test] + async fn delete_recovery_set_old_null_decision_cannot_commit_new_undecided_generation() { + let bucket = "recover-inverse-null-history"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![51; 4096], false).await; + let old_transaction = applied(&store, &disks, bucket, None, 4).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + set.publish_delete_commit( + bucket, + "object", + old_transaction, + &set.disk_inventory().await, + 3, + DeleteRecoveryContext::new(None, &|| true), + ) + .await + .expect("old generation decision while peer offline"); + let survivor = vec![52; 4096]; + put(&store, bucket, &survivor, false).await; + let new_transaction = applied(&store, &disks, bucket, None, 3).await; + set.disks.write().await[3] = Some(disks[3].clone()); + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("causal recovery across old receipt"); + assert!(report.detail.contains("rolled_back=1")); + assert_eq!( + body(&store, bucket, None).await, + survivor, + "an old null-version receipt cannot authorize deleting a later generation" + ); + for disk in &disks { + assert_eq!( + disk.inspect_delete_transaction(bucket, "object", new_transaction) + .await + .expect("new transaction settled") + .state, + DeleteRecoveryState::Absent + ); + } + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("delete recovered generation explicitly"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("all proof is eventually collected"); + } + + #[tokio::test] + async fn delete_recovery_set_ordinary_missing_single_and_batch_keep_quorum_semantics() { + let bucket = "recover-missing-quorum"; + let (_dirs, store, disks) = fixture(bucket).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + let error = store + .delete_object(bucket, "single-missing", ObjectOptions::default()) + .await + .expect_err("ordinary missing storage result"); + assert!( + crate::error::is_err_object_not_found(&error), + "S3 may translate missing key into idempotent204; not a bucket/quorum error: {error}" + ); + let objects = ["batch-one", "batch-two"] + .into_iter() + .map(|name| crate::storage_api_contracts::object::ObjectToDelete { + object_name: name.to_owned(), + ..Default::default() + }) + .collect(); + let (deleted, errors) = store.delete_objects(bucket, objects, ObjectOptions::default()).await; + assert!( + errors.iter().all(Option::is_none), + "ordinary batch no-op must succeed at available quorum: {errors:?}" + ); + assert_eq!(deleted.len(), 2); + for disk in disks.iter().take(3) { + for key in ["single-missing", "batch-one", "batch-two"] { + assert!( + disk.discover_delete_transactions(bucket, Some(key), None, 128) + .await + .expect("no-op leaves no recovery journals") + .transactions + .is_empty() + ); + } + } + set.disks.write().await[3] = Some(disks[3].clone()); + let payload = vec![53; 4096]; + put(&store, bucket, &payload, false).await; + assert_eq!(body(&store, bucket, None).await, payload); + } + #[tokio::test] + async fn delete_recovery_set_fresh_delete_resolves_unknown_null_predecessor() { + let bucket = "recover-unknown-null-history"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![54; 4096], false).await; + let transaction = Uuid::new_v4(); + let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("bucket identity"); + disks[3] + .delete_version_recoverable( + bucket, + "object", + FileInfo { + name: "object".to_owned(), + ..Default::default() + }, + false, + DeleteOptions { + old_data_dir: Some(transaction), + delete_recovery_bucket_incarnation: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("only future-offline peer prepared old delete"); + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + put(&store, bucket, &vec![55; 4096], false).await; + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("later generation deleted at quorum"); + set.disks.write().await[3] = Some(disks[3].clone()); + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("unknown causal history is reported"); + assert!( + report.detail.contains("blocked=1"), + "automatic recovery cannot invent causality: {}", + report.detail + ); + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("fresh authorized delete certifies current absence"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("fresh intent drains all proven unprotected null history"); + } + #[test] + fn delete_recovery_network_errors_do_not_mask_application_corruption() { + assert!(unavailable(&DiskError::RemoteClientUnavailable( + "transport error: connection refused".to_owned() + ))); + assert!(unavailable(&DiskError::from(tonic::Status::unavailable("peer disconnected")))); + assert!( + unavailable(&DiskError::FaultyDisk), + "a drive disabled by health cannot vote in recovery inspection" + ); + assert!(!unavailable(&DiskError::FileCorrupt)); + assert!(!unavailable(&DiskError::other("invalid journal containing connection refused"))); + assert!(!unavailable(&DiskError::from(tonic::Status::internal("connection refused")))); + assert!(!unavailable(&DiskError::other("bucket unavailable-logs has corrupt metadata"))); + } + + #[tokio::test] + async fn delete_recovery_chained_committed_versions_finish_after_current_metadata_disappears() { + let bucket = "recover-chained-versions"; + let (_dirs, store, disks) = fixture(bucket).await; + let first = put(&store, bucket, &vec![61; 1024 * 1024], true) + .await + .expect("first version"); + let second = put(&store, bucket, &vec![62; 4096], true).await.expect("second version"); + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + for version in [first, second] { + store + .delete_object( + bucket, + "object", + ObjectOptions { + versioned: true, + version_id: Some(version.to_string()), + ..Default::default() + }, + ) + .await + .expect("commit each explicit version at quorum while peer is absent"); + } + assert!( + store + .get_object_info(bucket, "object", &ObjectOptions::default()) + .await + .is_err() + ); + set.disks.write().await[3] = Some(disks[3].clone()); + let report = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("older certified target may be superseded by a later certified absence"); + assert!( + report.detail.contains("committed=2"), + "both retained decisions must converge: {}", + report.detail + ); + for version in [first, second] { + assert!( + store + .get_object_info( + bucket, + "object", + &ObjectOptions { + version_id: Some(version.to_string()), + ..Default::default() + } + ) + .await + .is_err(), + "certified cleanup must not resurrect either deleted version" + ); + } + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("no retained version decision may keep the empty bucket nonempty"); + } + + #[tokio::test] + async fn delete_recovery_partial_receipt_converges_after_later_version_absence() { + let bucket = "recover-partial-chained-versions"; + let (dirs, store, disks) = fixture(bucket).await; + let first = put(&store, bucket, &vec![63; 1024 * 1024], true) + .await + .expect("first version"); + let second = put(&store, bucket, &vec![64; 4096], true).await.expect("second version"); + let transaction = applied(&store, &disks, bucket, Some(first), 4).await; + // A crash after the first durable decision leaves other copies prepared. + publish(&disks[0], bucket, transaction).await; + let later = applied(&store, &disks, bucket, Some(second), 4).await; + publish(&disks[0], bucket, later).await; + let dry = store + .recover_delete_transactions( + bucket, + &HealOpts { + dry_run: true, + ..Default::default() + }, + ) + .await + .expect("dry-run uses the same certified prepared-copy authority"); + assert!( + dry.detail.contains("committed=2") && dry.detail.contains("blocked=0"), + "accurate dry-run: {}", + dry.detail + ); + for dir in &dirs { + for id in [transaction, later] { + assert!( + dir.path() + .join(bucket) + .join("object") + .join(id.to_string()) + .join("delete-prepared") + .exists(), + "dry-run must retain every preparation record" + ); + } + } + store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("same-transaction receipt certifies each prepared absent copy"); + for dir in &dirs { + assert!(!dir.path().join(bucket).join("object").exists(), "all related proof must converge"); + } + for version in [first, second] { + assert!( + store + .get_object_info( + bucket, + "object", + &ObjectOptions { + version_id: Some(version.to_string()), + ..Default::default() + } + ) + .await + .is_err(), + "no erased version may be recreated by receipt adoption" + ); + } + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("non-force bucket removal"); + } + + #[tokio::test] + async fn delete_recovery_causal_cycle_rejects_before_mutation_and_retains_proof() { + let bucket = "recover-causal-cycle"; + let (dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![58; 4096], false).await; + let first = applied(&store, &disks, bucket, None, 4).await; + put(&store, bucket, &vec![59; 4096], false).await; + let second = applied(&store, &disks, bucket, None, 4).await; + let mut evidence = Vec::new(); + for (dir, disk) in dirs.iter().zip(&disks) { + let root = dir.path().join(bucket).join("object"); + let mut journal = disk + .inspect_delete_transaction(bucket, "object", first) + .await + .expect("inspect first preparation") + .journal + .expect("original journal"); + journal.predecessors.push(second); + let path = root.join(first.to_string()).join("delete-prepared"); + std::fs::write(&path, serde_json::to_vec(&journal).expect("encode cyclic fixture")).expect("write cyclic edge"); + for transaction in [first, second] { + for file in ["delete-prepared", "xl.meta.bkp"] { + let path = root.join(transaction.to_string()).join(file); + evidence.push((path.clone(), std::fs::read(path).expect("capture untouched proof"))); + } + } + } + let error = store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect_err("cyclic history cannot settle"); + assert!( + matches!(error, crate::error::StorageError::FileCorrupt), + "typed corruption must reject cyclic evidence: {error:?}" + ); + for (path, bytes) in evidence { + assert_eq!(std::fs::read(path).expect("all proof retained"), bytes); + } + for dir in &dirs { + assert!( + !dir.path().join(bucket).join("object").join(STORAGE_FORMAT_FILE).exists(), + "cycle rejection must not restore or publish metadata" + ); + } + } + + #[tokio::test] + async fn delete_recovery_set_later_null_decision_covers_unapplied_original_peer() { + let bucket = "recover-unapplied-null-history"; + let (_dirs, store, disks) = fixture(bucket).await; + put(&store, bucket, &vec![56; 1024 * 1024], false).await; + // Every generated v4 UUID sorts before MAX, so the former UUID order + // deterministically tried the newer decision before its predecessor. + applied_at(&store, &disks, bucket, None, 3, Uuid::from_u128(u128::MAX)).await; + let set = &store.pools[0].disk_set[0]; + set.disks.write().await[3] = None; + put(&store, bucket, &vec![57; 1024 * 1024], false).await; + store + .delete_object(bucket, "object", ObjectOptions::default()) + .await + .expect("later null decision"); + set.disks.write().await[3] = Some(disks[3].clone()); + store + .recover_delete_transactions(bucket, &HealOpts::default()) + .await + .expect("causal older original can be removed"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("no stale old body or receipts remain"); + } +} diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 07df7f69c..200c13690 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -25,11 +25,14 @@ use super::super::{ use crate::bucket::retirement::MarkerRetirementContext; use crate::diagnostics::object_lock::ObjectLockAttempt; use crate::set_disk::{ObjectLockDiagGuard, is_object_lock_diag_enabled}; +use std::sync::Arc; #[derive(Clone, Copy)] struct ExplicitVersionHeal<'a> { opts: &'a HealOpts, allow_regeneration: bool, + preserve_unrepairable: bool, + commit_fence: Option<&'a (dyn Fn() -> bool + Sync)>, retirement: Option<&'a MarkerRetirementContext<'a>>, } @@ -770,6 +773,8 @@ impl SetDisks { ExplicitVersionHeal { opts, allow_regeneration: true, + preserve_unrepairable: false, + commit_fence: None, retirement: None, }, &mut None, @@ -907,6 +912,8 @@ impl SetDisks { let ExplicitVersionHeal { opts, allow_regeneration: allow_explicit_version_regen, + preserve_unrepairable, + commit_fence, retirement, } = run; trace!( @@ -925,6 +932,9 @@ impl SetDisks { if let Some(scope) = &bucket_heal_scope { scope.check()?; } + if commit_fence.is_some_and(|check| !check()) { + return Err(DiskError::DiskOngoingReq); + } let mut result = HealResultItem { heal_item_type: HealItemType::Object.to_string(), @@ -1248,7 +1258,7 @@ impl SetDisks { // return here and never reach the post-heal reclaim tail // below. Sweep the strays on this path too (issues #3231, // #3191). Skipped on dry-run, like every mutating step. - if !opts.dry_run { + if !opts.dry_run && !preserve_unrepairable { self.reclaim_orphan_data_dirs_best_effort(bucket, object).await; } return Ok((result, None)); @@ -1328,7 +1338,7 @@ impl SetDisks { ); } - if read_repair_uses_shared_lock { + if read_repair_uses_shared_lock || preserve_unrepairable { return Ok((result, Some(cannot_heal_err))); } @@ -1527,6 +1537,9 @@ impl SetDisks { if let Some(scope) = &bucket_heal_scope { scope.check()?; } + if commit_fence.is_some_and(|check| !check()) { + return Err(DiskError::DiskOngoingReq); + } let disk = outdated_disk.as_ref().expect("checked above").clone(); let endpoint = disk.endpoint().to_string(); match disk.make_volume(bucket).await { @@ -1562,6 +1575,9 @@ impl SetDisks { if let Some(scope) = &bucket_heal_scope { scope.check()?; } + if commit_fence.is_some_and(|check| !check()) { + return Err(DiskError::DiskOngoingReq); + } if disks_to_heal_count == 0 { return Ok((result, volume_creation_error.or(Some(DiskError::ErasureWriteQuorum)))); } @@ -1873,6 +1889,10 @@ impl SetDisks { if let Some(scope) = &bucket_heal_scope { scope.check()?; } + if commit_fence.is_some_and(|check| !check()) { + let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await; + return Err(DiskError::DiskOngoingReq); + } rename_attempts += 1; // record the index of the updated disks parts_metadata[index].erasure.index = index + 1; @@ -2004,7 +2024,9 @@ impl SetDisks { // by pre-#3510 unversioned overwrites, which the dangling paths // above never touch (issues #3231, #3191). Best effort — a // failure must not fail the heal. - self.reclaim_orphan_data_dirs_best_effort(bucket, object).await; + if !preserve_unrepairable { + self.reclaim_orphan_data_dirs_best_effort(bucket, object).await; + } Ok((result, None)) } @@ -2070,7 +2092,7 @@ impl SetDisks { .await; } - if opts.dry_run { + if opts.dry_run || preserve_unrepairable { return Ok(( self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id) .await, @@ -3347,6 +3369,74 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks { } impl SetDisks { + /// The recovery coordinator retains the key write lock and lifecycle fence. + /// Only current live metadata can supply this exact source; backups and + /// dangling cleanup cannot authorize prerequisite repair. + pub(in crate::set_disk) async fn heal_delete_recovery_source( + &self, + bucket: &str, + object: &str, + source: &crate::disk::DeleteRecoveryVersionIdentity, + can_commit: &(dyn Fn() -> bool + Sync), + ) -> disk::error::Result<()> { + if !can_commit() { + return Err(DiskError::DiskOngoingReq); + } + let version = source.version_id.ok_or(DiskError::OutdatedXLMeta)?.to_string(); + let disks = self.get_disks_internal().await; + let (metadata, errors, _) = Self::read_metadata(&disks, "", bucket, object, &version, true, false, false) + .await? + .into_legacy(); + let (read_quorum, write_quorum) = Self::object_quorum_from_meta(&metadata, &errors, self.default_parity_count)?; + let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?; + let write_quorum = usize::try_from(write_quorum).map_err(|_| DiskError::ErasureWriteQuorum)?; + let live_sources = metadata + .iter() + .zip(&errors) + .filter(|(info, error)| { + error.is_none() + && !info.deleted + && !info.is_remote() + && crate::disk::DeleteRecoveryVersionIdentity::from_file_info((*info).clone()) + .is_ok_and(|identity| identity == *source) + }) + .count(); + if live_sources < write_quorum || !can_commit() { + return Err(DiskError::OutdatedXLMeta); + } + let (_, mod_time, etag) = Self::list_online_disks(&disks, &metadata, &errors, read_quorum); + let selected = Self::pick_valid_fileinfo(&metadata, mod_time, etag, read_quorum)?; + if crate::disk::DeleteRecoveryVersionIdentity::from_file_info(selected)? != *source { + return Err(DiskError::OutdatedXLMeta); + } + let opts = HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }; + let (_, error) = Box::pin(self.heal_object_with_explicit_version_regen( + bucket, + object, + &version, + ExplicitVersionHeal { + opts: &opts, + allow_regeneration: false, + preserve_unrepairable: true, + commit_fence: Some(can_commit), + retirement: None, + }, + &mut None, + )) + .await?; + if let Some(error) = error { + return Err(error); + } + if !can_commit() { + return Err(DiskError::DiskOngoingReq); + } + Ok(()) + } + pub(crate) async fn heal_object_with_absence( &self, bucket: &str, @@ -3375,11 +3465,11 @@ impl SetDisks { .new_ns_lock(bucket, object) .await .map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?; - Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| { + Some(Arc::new(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| { self.map_namespace_lock_error(bucket, object, "write", e) .narrow_to_disk() .unwrap_or_else(DiskError::other) - })?) + })?)) } else { None }; @@ -3389,6 +3479,75 @@ impl SetDisks { return Ok((result, err.map(|e| e.into()))); } + let mut recovery_removed_version = false; + if !opts.read_repair + && _write_lock_guard.is_some() + && let Some(scope) = crate::store::bucket_heal_scope_for_object(bucket, object) + && !crate::bucket::utils::is_meta_bucketname(bucket) + { + let requested = Uuid::parse_str(version_id).ok(); + if !opts.dry_run && (version_id.is_empty() || requested.is_some()) { + let disks = self.get_disks_internal().await; + let before = Self::read_metadata(&disks, "", bucket, object, version_id, false, false, false) + .await + .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?; + recovery_removed_version = before + .slots + .iter() + .filter_map(|slot| slot.file_info()) + .any(|info| version_id.is_empty() || Some(info.version_id.unwrap_or_default()) == requested); + } + let explicit_delete = opts.remove + && version_id.is_empty() + && crate::bucket::metadata_sys::has_authoritative_never_versioned_state_in(&self.ctx, bucket).await? + && matches!( + crate::bucket::metadata_sys::get_object_lock_config_state_in(&self.ctx, bucket).await?, + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent + ); + let recovery = self + .recover_delete_key( + bucket, + object, + crate::set_disk::DeleteRecoveryContext { + dry_run: opts.dry_run, + explicit_delete, + repair_marker_source: true, + namespace_owner: (!opts.dry_run).then(|| { + Arc::new((scope.clone(), self.ctx.begin_namespace_commit(), _write_lock_guard.clone())) + as Arc + }), + ..crate::set_disk::DeleteRecoveryContext::new(Some(scope.incarnation), &|| { + scope.check().is_ok() && !_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()) + }) + }, + ) + .await?; + if opts.dry_run && recovery.discovered > 0 { + return Ok(( + HealResultItem { + heal_item_type: HealItemType::Object.to_string(), + bucket: bucket.to_owned(), + object: object.to_owned(), + version_id: version_id.to_owned(), + detail: format!( + "delete recovery (dry run): discovered={}, committed={}, rolled_back={}, reclaimed={}, blocked={}, pending={}", + recovery.discovered, + recovery.committed, + recovery.rolled_back, + recovery.reclaimed, + recovery.blocked, + recovery.pending + ), + ..Default::default() + }, + None, + )); + } + if recovery.pending > 0 || recovery.blocked > 0 { + return Err(StorageError::SlowDown); + } + } + // The inner heal and missing-object report read the registry again; // release this snapshot guard before a topology writer can queue between reads. let disks = self.get_disks_internal().await; @@ -3440,12 +3599,12 @@ impl SetDisks { && errs .iter() .all(|err| matches!(err, Some(DiskError::FileNotFound | DiskError::FileVersionNotFound))) - && !_write_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + && !_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()) { *absence = Some(HealedObjectAbsence { pool_index: self.pool_index, set_index: self.set_index, - removed: false, + removed: recovery_removed_version, }); } return Ok((result, Some(err))); @@ -3463,6 +3622,8 @@ impl SetDisks { ExplicitVersionHeal { opts: &inner_opts, allow_regeneration: true, + preserve_unrepairable: false, + commit_fence: None, retirement: proof.retirement, }, absence, @@ -3483,6 +3644,8 @@ impl SetDisks { ExplicitVersionHeal { opts: &inner_opts, allow_regeneration: true, + preserve_unrepairable: false, + commit_fence: None, retirement: proof.retirement, }, absence, @@ -3493,7 +3656,7 @@ impl SetDisks { result, err.map(Into::into), absence, - _write_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()), + _write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()), bucket, object, )); @@ -3505,7 +3668,7 @@ impl SetDisks { result, err.map(Into::into), absence, - _write_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()), + _write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()), bucket, object, )) diff --git a/crates/ecstore/src/set_disk/ops/mod.rs b/crates/ecstore/src/set_disk/ops/mod.rs index 4cea63793..5dfaa66b5 100644 --- a/crates/ecstore/src/set_disk/ops/mod.rs +++ b/crates/ecstore/src/set_disk/ops/mod.rs @@ -19,6 +19,7 @@ pub(crate) mod bitrot_self_verify; pub(crate) mod bucket; +pub(crate) mod delete_recovery; pub(crate) mod heal; pub(crate) mod heal_walk; pub(crate) mod list; diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 791873248..b35eb4cb7 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -64,6 +64,34 @@ use super::super::{ take_prepared_get_object_metadata, to_object_err, try_read_inline_data_shards_direct, warn, }; use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards}; +use super::delete_recovery::DeleteRecoveryContext; + +#[derive(Default)] +struct DeleteCommitContext<'a> { + incarnation: Option, + options: Option<&'a ObjectOptions>, + guard: Option>, +} + +impl<'a> DeleteCommitContext<'a> { + fn request(options: &'a ObjectOptions, guard: Option<&Arc>) -> Self { + Self { + incarnation: options.expected_bucket_incarnation_id, + options: Some(options), + guard: guard.cloned(), + } + } + + fn can_commit(&self, bucket: &str, object: &str) -> bool { + self.options.is_none_or(|options| { + ensure_delete_commit_locks_held(self.guard.as_deref(), bucket, object, options).is_ok() + && options + .scanner_publication_commit_scope + .as_ref() + .is_none_or(|scope| scope.can_commit()) + }) + } +} fn delete_marker_purge_candidate( file_info: &FileInfo, @@ -338,7 +366,7 @@ use crate::storage_api_contracts::object::HTTPPreconditions; use crate::store::ECStore; use crate::store::utils::clean_metadata; use crate::store::{RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence}; -use futures::FutureExt as _; +use futures::{FutureExt as _, StreamExt as _}; use http::HeaderValue; #[cfg(test)] use rustfs_filemeta::FileMeta; @@ -3308,6 +3336,76 @@ pub(in crate::set_disk) fn merge_replication_metadata_lww( } impl SetDisks { + async fn reclaim_missing_delete_residue( + &self, + bucket: &str, + object: &str, + opts: &ObjectOptions, + lock_guard: Option<&Arc>, + ) -> Result { + if is_meta_bucketname(bucket) + || opts.version_id.is_some() + || opts.versioned + || opts.version_suspended + || opts.http_preconditions.is_some() + || opts.data_movement + || opts.transition.expire_restored + || opts.replication_request + || opts.lifecycle_delete_all.is_some() + || opts.delete_prefix + || opts.expected_current_version_id.is_some() + || !metadata_sys::has_authoritative_never_versioned_state_in(&self.ctx, bucket) + .await + .unwrap_or(false) + { + return Ok(false); + } + let lock_state = match opts.object_lock_config_snapshot.as_ref() { + Some(snapshot) => Some(snapshot.state().clone()), + None => metadata_sys::get_object_lock_config_state_in(&self.ctx, bucket).await.ok(), + }; + if !matches!(lock_state, Some(ObjectLockConfigState::ConfirmedAbsent)) { + return Ok(false); + } + if let Some(expected) = opts.expected_bucket_incarnation_id { + self.validate_bucket_incarnation(bucket, expected, opts).await?; + } + ensure_delete_commit_locks_held(lock_guard.map(Arc::as_ref), bucket, object, opts)?; + begin_scanner_publication_delete_mutation(opts.scanner_publication_commit_scope.as_ref())?; + let result = self + .recover_delete_key( + bucket, + object, + DeleteRecoveryContext { + explicit_delete: true, + namespace_owner: Some(Arc::new(( + self.ctx.begin_namespace_commit(), + lock_guard.cloned(), + opts.object_lock_config_snapshot.clone(), + opts.scanner_publication_commit_scope.clone(), + opts.write_lock_context.clone(), + opts.delete_mutation_owner.clone(), + ))), + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| { + ensure_delete_commit_locks_held(lock_guard.map(Arc::as_ref), bucket, object, opts).is_ok() + && opts + .scanner_publication_commit_scope + .as_ref() + .is_none_or(|scope| scope.can_commit()) + }) + }, + ) + .await; + ensure_delete_commit_locks_held(lock_guard.map(Arc::as_ref), bucket, object, opts)?; + let recovered = result.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?; + if recovered.pending > 0 && !recovered.no_transactions_at_quorum { + return Err(StorageError::SlowDown); + } + if recovered.blocked > 0 { + return Err(StorageError::SlowDown); + } + Ok(recovered.reclaimed > 0 || recovered.committed > 0) + } pub(in crate::set_disk) async fn cleanup_rename_tail( &self, targets: Vec, @@ -7612,7 +7710,7 @@ impl SetDisks { &delete_request, false, None, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(opts, None), ) .await?; if let Some((_, deleted_object)) = replication_delete { @@ -7686,7 +7784,7 @@ impl SetDisks { Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended)) } - #[tracing::instrument(skip(self, delete_marker_purge))] + #[tracing::instrument(skip(self, delete_marker_purge, commit_context))] async fn delete_object_version_with_purge( &self, bucket: &str, @@ -7694,12 +7792,28 @@ impl SetDisks { fi: &FileInfo, force_del_marker: bool, delete_marker_purge: Option, - source_bucket_incarnation_id: Option, + commit_context: DeleteCommitContext<'_>, ) -> Result<()> { + let source_bucket_incarnation_id = commit_context.incarnation; let transported = delete_file_info_with_replication_transport_metadata(fi); let fi = &transported; let disks = self.disk_inventory().await; - let namespace_owner = (!is_meta_bucketname(bucket)).then(|| self.ctx.begin_namespace_commit()); + let namespace_owner = (!is_meta_bucketname(bucket)).then(|| { + Arc::new(( + self.ctx.begin_namespace_commit(), + commit_context.guard.clone(), + commit_context + .options + .and_then(|options| options.object_lock_config_snapshot.clone()), + commit_context + .options + .and_then(|options| options.scanner_publication_commit_scope.clone()), + commit_context.options.and_then(|options| options.write_lock_context.clone()), + commit_context + .options + .and_then(|options| options.delete_mutation_owner.clone()), + )) as Arc + }); // Quorum is a property of the configured set, not of the current // online snapshot. A decommission/offline refresh may temporarily // shorten the snapshot; deriving quorum from it would turn a @@ -7714,20 +7828,33 @@ impl SetDisks { let disk_namespace_owner = namespace_owner.clone().map(|owner| owner as Arc); futures.push(async move { if let Some(disk) = disk { - match disk - .delete_version_with_namespace_owner( + let options = DeleteOptions { + old_data_dir: Some(rollback_dir), + delete_recovery_bucket_incarnation: source_bucket_incarnation_id, + ..Default::default() + }; + let result = if is_meta_bucketname(bucket) { + disk.delete_version_with_namespace_owner( bucket, object, fi.clone(), force_del_marker, - DeleteOptions { - old_data_dir: Some(rollback_dir), - ..Default::default() - }, + options, disk_namespace_owner, ) .await - { + } else { + disk.delete_version_recoverable_with_namespace_owner( + bucket, + object, + fi.clone(), + force_del_marker, + options, + disk_namespace_owner, + ) + .await + }; + match result { Ok(r) => Ok(r), Err(e) => Err(e), } @@ -7749,12 +7876,34 @@ impl SetDisks { } } - let quorum_result = resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object); + let mut quorum_result = resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object); + if !is_meta_bucketname(bucket) && errs.iter().flatten().any(super::delete_recovery::unsupported_recovery) { + quorum_result = Err(DiskError::UnsupportedDisk.into()); + } let should_rollback = quorum_result.is_err(); + let structured_cleanup = !is_meta_bucketname(bucket) && !should_rollback; + if structured_cleanup { + self.publish_delete_commit( + bucket, + object, + rollback_dir, + &disks, + write_quorum, + DeleteRecoveryContext { + namespace_owner: namespace_owner.clone().map(|owner| owner as Arc), + ..DeleteRecoveryContext::new(source_bucket_incarnation_id, &|| commit_context.can_commit(bucket, object)) + }, + ) + .await + .map_err(StorageError::from)?; + } #[cfg(test)] pause_delete_before_cleanup(bucket).await; let mut rollback_futures = Vec::new(); for (index, err) in errs.iter().enumerate() { + if structured_cleanup { + break; + } // backlog#1158: when rolling back, fan the idempotent undo out to every // online disk (each self-decides from its staged backup: restore if the // rollback dir is present, no-op otherwise). This covers a disk that @@ -7810,6 +7959,7 @@ impl SetDisks { DeleteOptions { recursive: true, immediate: true, + old_data_dir: Some(rollback_dir), ..Default::default() }, disk_namespace_owner, @@ -8192,8 +8342,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } else { self.bucket_incarnation_id_from_disk(bucket).await.ok() }; - self.delete_object_version_with_purge(bucket, object, fi, force_del_marker, None, source_bucket_incarnation_id) - .await + self.delete_object_version_with_purge( + bucket, + object, + fi, + force_del_marker, + None, + DeleteCommitContext { + incarnation: source_bucket_incarnation_id, + ..Default::default() + }, + ) + .await } #[tracing::instrument(skip(self, objects, opts))] @@ -8313,6 +8473,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let mut tier_reference_leases: Vec<(usize, String, Option)> = Vec::new(); let mut tier_free_version_receipt_candidates: HashMap = HashMap::new(); let mut delete_marker_purge_candidates = vec![None; objects.len()]; + let mut missing_delete_retry_indices = Vec::new(); + let mut deferred_recovery_indices = Vec::new(); for (i, dobj) in objects.iter().enumerate() { if del_errs[i].is_some() { @@ -8343,7 +8505,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id; let replication_needs_source = replicate_delete && (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name)); - let (goi, authoritative_file_info, gerr) = if object_lock_check_required + let (mut goi, mut authoritative_file_info, mut gerr) = if object_lock_check_required || replication_needs_source || opts.tier_delete_journal_api.is_some() || dobj.expected_identity.is_some() @@ -8356,9 +8518,74 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } else { (ObjectInfo::default(), FileInfo::default(), None) }; + if gerr + .as_ref() + .is_some_and(|error| is_err_object_not_found(error) || is_err_version_not_found(error)) + && !is_meta_bucketname(bucket) + && !opts.data_movement + && !opts.replication_request + && opts.http_preconditions.is_none() + { + let recovery = self + .recover_delete_key( + bucket, + &dobj.object_name, + DeleteRecoveryContext { + dry_run: !opts.no_lock, + namespace_owner: opts.delete_mutation_owner.clone(), + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| { + ensure_delete_commit_locks_held(None, bucket, &dobj.object_name, &opts).is_ok() + }) + }, + ) + .await; + match recovery { + Ok(recovery) if !opts.no_lock && recovery.discovered > 0 && recovery.pending == 0 => { + deferred_recovery_indices.push(i); + continue; + } + Ok(recovery) + if (recovery.pending > 0 + && !(recovery.no_transactions_at_quorum + && dobj.version_id.is_none() + && !versioned + && !version_suspended)) + || (recovery.blocked > 0 && (dobj.version_id.is_some() || versioned || version_suspended)) => + { + del_errs[i] = Some(if recovery.pending > 0 { + StorageError::InsufficientWriteQuorum(bucket.to_owned(), dobj.object_name.clone()) + } else { + StorageError::SlowDown + }); + continue; + } + Ok(recovery) if recovery.rolled_back > 0 || recovery.committed > 0 => { + let (fresh, info, _, error) = self + .get_object_info_fileinfo_and_quorum(bucket, &dobj.object_name, &check_opts) + .await; + goi = fresh; + authoritative_file_info = info; + gerr = error; + } + Ok(_) => {} + Err(error) => { + del_errs[i] = Some(to_object_err(error.into(), vec![bucket, &dobj.object_name])); + continue; + } + } + } let source_missing = gerr .as_ref() .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)); + if source_missing + && dobj.version_id.is_none() + && dobj.expected_identity.is_none() + && !versioned + && !version_suspended + && opts.http_preconditions.is_none() + { + missing_delete_retry_indices.push(i); + } // A pool-local miss keeps the existing idempotent batch-delete // semantics. Exact-version deletes fan out to every pool, and a // retry may also arrive after the intended generation is gone. @@ -8596,10 +8823,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } let rollback_dir = Uuid::new_v4(); + let namespace_owner = (!is_meta_bucketname(bucket)).then(|| { + Arc::new(( + self.ctx.begin_namespace_commit(), + opts.delete_mutation_owner.clone(), + opts.object_lock_config_snapshot.clone(), + opts.scanner_publication_commit_scope.clone(), + opts.write_lock_context.clone(), + )) as Arc + }); - let disks = self.disks.read().await; - - let disks = disks.clone(); + let disks = self.disks.read().await.clone(); if opts .namespace_lock_fence @@ -8668,6 +8902,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // let mut errors = Vec::with_capacity(disks.len()); for disk in disks.iter() { + let disk_namespace_owner = namespace_owner.clone(); let vers = vers.clone(); let namespace_lock_fence = opts.namespace_lock_fence.clone(); let lock_lost_during_commit = Arc::clone(&lock_lost_during_commit); @@ -8677,15 +8912,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { return (0..vers.len()).map(|_| Some(DiskError::DiskOngoingReq)).collect(); } if let Some(disk) = disk { - disk.delete_versions( - bucket, - vers, - DeleteOptions { - old_data_dir: Some(rollback_dir), - ..Default::default() - }, - ) - .await + let options = DeleteOptions { + old_data_dir: Some(rollback_dir), + delete_recovery_bucket_incarnation: opts.expected_bucket_incarnation_id, + ..Default::default() + }; + if is_meta_bucketname(bucket) { + disk.delete_versions(bucket, vers, options).await + } else { + disk.delete_versions_recoverable_with_namespace_owner(bucket, vers, options, disk_namespace_owner) + .await + } } else { let mut errs = Vec::with_capacity(vers.len()); for _ in 0..vers.len() { @@ -8721,7 +8958,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } } - let mut has_err = reduce_write_quorum_errs(&disk_err, OBJECT_OP_IGNORED_ERRS, disks.len() / 2 + 1); + let mut has_err = reduce_write_quorum_errs(&disk_err, OBJECT_OP_IGNORED_ERRS, self.set_drive_count / 2 + 1); + if !is_meta_bucketname(bucket) && disk_err.iter().flatten().any(super::delete_recovery::unsupported_recovery) { + has_err = Some(DiskError::UnsupportedDisk); + } if let Some(err) = has_err.clone() { let er = err.into(); if (is_err_object_not_found(&er) || is_err_version_not_found(&er)) && !del_objects[obj_idx].delete_marker { @@ -8765,10 +9005,67 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { #[cfg(test)] pause_delete_before_cleanup(bucket).await; + let mut published_groups = HashSet::new(); + if !is_meta_bucketname(bucket) { + let groups = vers.iter().enumerate().filter_map(|(index, group)| { + group + .versions + .iter() + .all(|version| del_errs[version.idx].is_none()) + .then_some(index) + }); + // Mark all attempted groups before starting publication. A lost + // response can hide a persisted decision and must never roll back. + let groups = groups.collect::>(); + published_groups.extend(groups.iter().map(|index| vers[*index].name.clone())); + let publications = futures::stream::iter(groups.into_iter().map(|index: usize| { + let group = &vers[index]; + let namespace_owner = namespace_owner.clone(); + let disks = &disks; + let opts = &opts; + async move { + let can_commit = || { + ensure_delete_commit_locks_held(None, bucket, &group.name, opts).is_ok() + && opts + .scanner_publication_commit_scope + .as_ref() + .is_none_or(|scope| scope.can_commit()) + }; + let result = self + .publish_delete_commit( + bucket, + &group.name, + rollback_dir, + disks, + self.set_drive_count / 2 + 1, + DeleteRecoveryContext { + namespace_owner, + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &can_commit) + }, + ) + .await; + (index, result) + } + })) + .buffer_unordered(16) + .collect::>() + .await; + for (index, result) in publications { + let group = &vers[index]; + if let Err(error) = result { + for version in &group.versions { + del_errs[version.idx] = Some(to_object_err(error.clone().into(), vec![bucket, &group.name])); + } + } + } + } let mut rollback_futures = Vec::new(); let committed_receipt_indices = committed_tier_free_version_receipt_indices(&vers, &del_errs, &tier_free_version_receipt_candidates); for fi_vers in &vers { + if published_groups.contains(&fi_vers.name) { + continue; + } // delete_versions commits one xl.meta per object group, so rollback must use the same boundary. let should_rollback = fi_vers.versions.iter().any(|fi| del_errs[fi.idx].is_some()); for (disk_idx, disk) in disks.iter().enumerate() { @@ -8788,10 +9085,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let bucket = bucket.to_string(); let object = fi_vers.name.clone(); let versions = fi_vers.clone(); + let disk_namespace_owner = namespace_owner.clone(); rollback_futures.push(async move { if should_rollback { let errs = disk - .delete_versions( + .delete_versions_with_namespace_owner( &bucket, vec![versions], DeleteOptions { @@ -8800,6 +9098,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { old_data_dir: Some(rollback_dir), ..Default::default() }, + disk_namespace_owner, ) .await; if let Some(err) = errs.into_iter().flatten().next() { @@ -8820,6 +9119,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { DeleteOptions { recursive: true, immediate: true, + old_data_dir: Some(rollback_dir), ..Default::default() }, ) @@ -8856,12 +9156,70 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { ); } + // A new explicit DELETE authorizes removal of the exact missing key's + // legacy residue; it does not infer whether the interrupted delete committed. + for index in missing_delete_retry_indices { + if del_errs[index].is_none() { + let object = &objects[index].object_name; + let result = match ensure_delete_commit_locks_held(None, bucket, object, &opts) { + Ok(()) => self.reclaim_missing_delete_residue(bucket, object, &opts, None).await, + Err(err) => Err(err), + }; + if let Err(err) = result { + del_errs[index] = Some(err); + } + } + } + // TODO(backlog): support partial object deletion for multi-part objects if dist_erasure { self.release_dist_delete_object_locks_batch(dist_batch_lock_ids).await; } + // Cold missing-key replay uses the same batch contract after the fast + // batch lock is released; renewable key ownership survives settlement. + for index in deferred_recovery_indices { + let object = &objects[index]; + let explicit_delete = object.version_id.is_none() + && !object.synthetic_version_id + && !opts.versioned + && !opts.version_suspended + && opts.http_preconditions.is_none() + && metadata_sys::has_authoritative_never_versioned_state_in(&self.ctx, bucket) + .await + .unwrap_or(false) + && matches!(object_lock_config.state(), ObjectLockConfigState::ConfirmedAbsent); + let recovery = self + .recover_delete_object( + bucket, + &object.object_name, + DeleteRecoveryContext { + explicit_delete, + namespace_owner: Some(Arc::new(( + self.ctx.begin_namespace_commit(), + opts.delete_mutation_owner.clone(), + opts.object_lock_config_snapshot.clone(), + ))), + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| { + ensure_delete_commit_locks_held(None, bucket, &object.object_name, &opts).is_ok() + }) + }, + ) + .await; + match recovery { + Ok(recovery) if recovery.pending == 0 && recovery.blocked == 0 => { + let (mut deleted, mut errors, mut receipts) = + Box::pin(self.delete_objects_with_accounting(bucket, vec![object.clone()], opts.clone())).await; + del_objects[index] = deleted.remove(0); + del_errs[index] = errors.remove(0); + accounting[index] = receipts.remove(0); + } + Ok(_) => del_errs[index] = Some(StorageError::SlowDown), + Err(error) => del_errs[index] = Some(to_object_err(error.into(), vec![bucket, &object.object_name])), + } + } + let purge_submissions = delete_marker_purge_candidates .into_iter() .enumerate() @@ -8918,7 +9276,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // Guard lock for single object delete let _lock_guard = if (!opts.delete_prefix || opts.delete_prefix_object) && !opts.no_lock { - Some(self.acquire_write_lock_diag("delete_object", bucket, object).await?) + Some(Arc::new(self.acquire_write_lock_diag("delete_object", bucket, object).await?)) } else { None }; @@ -8988,7 +9346,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } }; for version in plan { - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; // The lease closes the gap between the durable // reference proof and publishing either a committed // v6 journal or the legacy free-version fallback. @@ -9022,7 +9380,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &delete_request, false, None, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await?; if let Some((_, deleted_object)) = replication_delete { @@ -9031,7 +9389,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } } else { for version in &versions.versions { - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; let mut delete_request = FileInfo { name: object.to_string(), version_id: version.version_id, @@ -9045,12 +9403,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &delete_request, false, None, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await?; } for version in &versions.free_versions { - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; let mut delete_request = FileInfo { name: object.to_string(), version_id: version.version_id, @@ -9065,7 +9423,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &delete_request, false, None, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await?; } @@ -9081,7 +9439,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { self.validate_bucket_incarnation(bucket, expected_incarnation_id, &opts) .await?; } - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; self.delete_prefix_with_scanner_publication_lease( bucket, @@ -9132,14 +9490,73 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let mut version_found = true; // delete_object_version below derives its own majority quorum from the // disk array, so the object-derived quorum here is unused. - let (mut goi, authoritative_file_info, _write_quorum, gerr) = + let (mut goi, mut authoritative_file_info, _write_quorum, mut gerr) = self.get_object_info_fileinfo_and_quorum(bucket, object, &opts).await; + if gerr + .as_ref() + .is_some_and(|error| is_err_object_not_found(error) || is_err_version_not_found(error)) + && !is_meta_bucketname(bucket) + && !opts.data_movement + && !opts.replication_request + && opts.http_preconditions.is_none() + { + if let Some(expected) = opts.expected_bucket_incarnation_id { + self.validate_bucket_incarnation(bucket, expected, &opts).await?; + } + let recovery = self + .recover_delete_key( + bucket, + object, + DeleteRecoveryContext { + namespace_owner: Some(Arc::new(( + self.ctx.begin_namespace_commit(), + _lock_guard.clone(), + opts.object_lock_config_snapshot.clone(), + opts.scanner_publication_commit_scope.clone(), + opts.write_lock_context.clone(), + opts.delete_mutation_owner.clone(), + ))), + ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| { + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts).is_ok() + }) + }, + ) + .await?; + if (recovery.pending > 0 + && !(recovery.no_transactions_at_quorum + && opts.version_id.is_none() + && !opts.versioned + && !opts.version_suspended)) + || (recovery.blocked > 0 && (opts.version_id.is_some() || opts.versioned || opts.version_suspended)) + { + return Err(if recovery.pending > 0 { + StorageError::InsufficientWriteQuorum(bucket.to_owned(), object.to_owned()) + } else { + StorageError::SlowDown + }); + } + if recovery.rolled_back > 0 || recovery.committed > 0 { + (goi, authoritative_file_info, _, gerr) = self.get_object_info_fileinfo_and_quorum(bucket, object, &opts).await; + } + } if let Some(err) = &gerr && goi.name.is_empty() { if should_force_delete_marker_for_missing_version(&opts) { version_found = false; } else { + if is_err_object_not_found(err) { + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; + if self + .reclaim_missing_delete_residue(bucket, object, &opts, _lock_guard.as_ref()) + .await? + { + if let Some(scope) = scanner_publication_commit_scope.as_ref() { + let _ = scope.mark_committed(); + } + return Ok(ObjectInfo::default()); + } + } return Err(err.clone()); } } @@ -9173,11 +9590,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { expire_restored: true, ..Default::default() }; - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; - self.delete_object_version_with_purge(bucket, object, &dfi, false, None, opts.expected_bucket_incarnation_id) - .await - .map_err(|e| to_object_err(e, vec![bucket, object]))?; + self.delete_object_version_with_purge( + bucket, + object, + &dfi, + false, + None, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), + ) + .await + .map_err(|e| to_object_err(e, vec![bucket, object]))?; self.invalidate_get_object_metadata_cache(bucket, object).await; if let Some(scope) = scanner_publication_commit_scope.as_ref() { let _ = scope.mark_committed(); @@ -9257,7 +9681,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { None }; - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; let _tier_delete_lease = acquire_single_tier_delete_lease(&opts, &goi).await?; if opts.skip_free_version { @@ -9269,7 +9693,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &fi, should_force_delete_marker_for_missing_version(&opts), delete_marker_purge.clone(), - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await .map_err(|e| to_object_err(e, vec![bucket, object]))?; @@ -9316,7 +9740,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { dfi.set_delete_marker_incarnation(incarnation); } - ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?; begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?; let _tier_delete_lease = acquire_single_tier_delete_lease(&opts, &goi).await?; if opts.skip_free_version { @@ -9328,7 +9752,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &dfi, opts.delete_marker, delete_marker_purge, - opts.expected_bucket_incarnation_id, + DeleteCommitContext::request(&opts, _lock_guard.as_ref()), ) .await .map_err(|e| to_object_err(e, vec![bucket, object]))?; @@ -9959,7 +10383,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { pause_transition_transaction_at(bucket, object, TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit).await; upload_cleanup.disarm(); if let Err(err) = self - .delete_object_version_with_purge(bucket, object, &fi, false, None, opts.expected_bucket_incarnation_id) + .delete_object_version_with_purge(bucket, object, &fi, false, None, DeleteCommitContext::request(opts, None)) .await { warn!( @@ -22994,9 +23418,19 @@ mod marker_purge_receipt_tests { .acquire_write_lock_diag("marker_purge_receipt_test", bucket, object) .await .expect("own the deleted object namespace"); - set.delete_object_version_with_purge(bucket, object, &marker, true, None, Some(incarnation)) - .await - .expect("seed the real marker on every disk"); + set.delete_object_version_with_purge( + bucket, + object, + &marker, + true, + None, + DeleteCommitContext { + incarnation: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed the real marker on every disk"); let marker = disks[0] .read_version("", bucket, object, &version.to_string(), &ReadOptions::default()) .await @@ -23014,7 +23448,17 @@ mod marker_purge_receipt_tests { set.disks.write().await[0] = None; tokio::time::timeout( Duration::from_secs(10), - set.delete_object_version_with_purge(bucket, object, &request, false, Some(purge.clone()), Some(incarnation)), + set.delete_object_version_with_purge( + bucket, + object, + &request, + false, + Some(purge.clone()), + DeleteCommitContext { + incarnation: Some(incarnation), + ..Default::default() + }, + ), ) .await .expect("the receipt must not reacquire the deleted object's owner") @@ -23202,6 +23646,54 @@ mod single_delete_namespace_owner_tests { ); } + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn batch_delete_uses_one_namespace_owner_across_object_groups() { + let (dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "batch-delete-namespace"; + for disk in &disks { + disk.make_volume(bucket).await.expect("fixture bucket"); + } + let mut objects = Vec::new(); + for key in ["inline", "external"] { + let version = Uuid::new_v4(); + let body = vec![0x41; if key == "inline" { 4096 } else { 256 * 1024 }]; + seed_version(&set, bucket, key, version, &body).await; + objects.push(ObjectToDelete { + object_name: key.to_owned(), + version_id: Some(version), + ..Default::default() + }); + } + let before = set.ctx.namespace_commit_generation(); + assert!(!set.ctx.namespace_commits_pending()); + let (_, errors, _) = set + .delete_objects_with_accounting( + bucket, + objects, + ObjectOptions { + versioned: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await; + assert!(errors.iter().all(Option::is_none), "batch groups must commit: {errors:?}"); + for dir in &dirs { + for key in ["inline", "external"] { + assert!(!dir.path().join(bucket).join(key).exists(), "all group evidence must converge"); + } + } + assert!(!set.ctx.namespace_commits_pending()); + assert_eq!( + set.ctx.namespace_commit_generation(), + before + 2, + "all batch groups share one complete owner lifetime" + ); + } + #[cfg(not(windows))] async fn assert_single_delete_physical_owner(case: &'static str) { use crate::disk::os::prepared_publication_test_hooks as hooks; @@ -23280,9 +23772,14 @@ mod single_delete_namespace_owner_tests { for (index, path) in metadata_paths.iter().enumerate() { let tx = entered_tx.clone(); let (release, rx) = std::sync::mpsc::channel::<()>(); - if last || cleanup { - let source = if cleanup { &cleanup_sources[index] } else { path }; - destination_guards.push(hooks::observe_rename_destination(source, move |destination| { + if cleanup { + let source = cleanup_sources[index].clone(); + guards.push(hooks::install_at(hooks::Stage::Remove, &source.join("part.1"), move || { + let _ = tx.send((index, source)); + let _ = rx.recv(); + })); + } else if last { + destination_guards.push(hooks::observe_rename_destination(path, move |destination| { let _ = tx.send((index, destination.to_path_buf())); let _ = rx.recv(); })); @@ -23355,6 +23852,12 @@ mod single_delete_namespace_owner_tests { } } } + if cleanup { + for (index, source) in cleanup_sources.iter().enumerate() { + assert_eq!(std::fs::read(source.join("part.1")).expect("paused original shard"), cleanup_parts[index], + "the paused physical cleanup must still own the exact original shard"); + } + } if rollback { tokio::time::pause(); tokio::time::advance(Duration::from_secs(61)).await; @@ -23431,11 +23934,7 @@ mod single_delete_namespace_owner_tests { if cleanup { assert!(!path.exists(), "metadata must remain deleted after cleanup cancellation"); assert!(!cleanup_sources[index].exists(), "physical cleanup must remove the shard directory"); - assert_eq!( - std::fs::read(physical_keys[&index].join("part.1")).expect("actual trashed shard"), - cleanup_parts[index], - "trash must contain the exact original shard" - ); + assert!(!physical_keys[&index].exists(), "the owned physical cleanup must remove its original shard directory"); continue; } if last { @@ -23545,3 +24044,431 @@ mod single_delete_namespace_owner_tests { assert_single_delete_physical_owner("rollback").await; } } + +#[cfg(test)] +mod crash_delete_retry_tests { + use super::*; + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::disk::STORAGE_FORMAT_FILE_BACKUP; + use crate::storage_api_contracts::bucket::{BucketOperations, DeleteBucketOptions, MakeBucketOptions}; + + async fn stage_interrupted_delete(store: &Arc, dirs: &[tempfile::TempDir], bucket: &str, object: &str, size: usize) { + let mut reader = PutObjReader::from_vec(vec![37; size]); + store + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("write the object before simulating an interrupted delete"); + let transaction = Uuid::new_v4(); + for dir in dirs { + let root = dir.path().join(bucket).join(object); + let xl = root.join(STORAGE_FORMAT_FILE); + let bytes = tokio::fs::read(&xl).await.expect("read original object metadata"); + let mut metadata = FileMeta::default(); + metadata.unmarshal_msg(&bytes).expect("decode original object metadata"); + let info = metadata.versions[0] + .into_fileinfo(bucket, object, false) + .expect("read original version"); + let backup = root.join(transaction.to_string()); + tokio::fs::create_dir_all(&backup) + .await + .expect("create transaction directory"); + tokio::fs::write(backup.join(STORAGE_FORMAT_FILE_BACKUP), bytes) + .await + .expect("stage intact rollback backup"); + if let Some(data_dir) = info.data_dir { + let parts = root.join(data_dir.to_string()); + if parts.is_dir() { + tokio::fs::write( + parts.join(format!("{}{}", crate::disk::local::RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, transaction)), + [], + ) + .await + .expect("reserve external deletion before its local commit"); + } + } + tokio::fs::remove_file(&xl) + .await + .expect("simulate crash after removing metadata"); + } + } + + #[tokio::test] + async fn crash_delete_retry_single_reclaims_inline_backup_only() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-single"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + store + .delete_object(bucket, "inline", ObjectOptions::default()) + .await + .expect("retry should reclaim the exact missing key"); + for dir in &dirs { + assert!( + !dir.path().join(bucket).join("inline").exists(), + "inline backup residue must be removed on every disk" + ); + } + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("the empty bucket must be deletable without force"); + } + + #[tokio::test] + async fn crash_delete_retry_batch_reclaims_inline_and_reserved_external() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-batch"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + stage_interrupted_delete(&store, &dirs, bucket, "external", 1024 * 1024).await; + let objects = ["inline", "external"] + .into_iter() + .map(|name| ObjectToDelete { + object_name: name.to_string(), + ..Default::default() + }) + .collect(); + let (_, errors) = store.delete_objects(bucket, objects, ObjectOptions::default()).await; + assert!( + errors.iter().all(Option::is_none), + "retry must complete for both crash shapes: {errors:?}" + ); + for dir in &dirs { + for object in ["inline", "external"] { + assert!(!dir.path().join(bucket).join(object).exists(), "retry must remove {object} on every disk"); + } + } + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("batch retry must unblock non-force bucket deletion"); + } + + #[tokio::test] + async fn crash_delete_retry_preserves_conditional_and_versioned_residue() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-gates"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + for opts in [ + ObjectOptions { + http_preconditions: Some(HTTPPreconditions { + if_match: Some("old-etag".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ObjectOptions { + version_id: Some(Uuid::new_v4().to_string()), + ..Default::default() + }, + ] { + let _ = store.delete_object(bucket, "inline", opts).await; + for dir in &dirs { + assert!( + dir.path().join(bucket).join("inline").exists(), + "conditional/version-specific deletes must retain ambiguous backups" + ); + } + } + } + #[tokio::test] + async fn crash_delete_retry_preserves_suspended_bucket_null_backup() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-suspended"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + metadata_sys::update_in(&store.ctx, bucket, crate::bucket::metadata::BUCKET_VERSIONING_CONFIG, + b"Suspended".to_vec()) + .await.expect("persist suspended versioning state"); + let _ = store.delete_object(bucket, "inline", ObjectOptions::default()).await; + for dir in &dirs { + assert!( + dir.path().join(bucket).join("inline").exists(), + "authoritative suspended state must protect a null-version backup" + ); + } + } + + #[tokio::test] + async fn crash_delete_retry_rejects_stale_bucket_incarnation_at_set_boundary() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "crash-delete-incarnation"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative bucket"); + stage_interrupted_delete(&store, &dirs, bucket, "inline", 4096).await; + let error = store.pools[0].disk_set[0] + .delete_object( + bucket, + "inline", + ObjectOptions { + expected_bucket_incarnation_id: Some(Uuid::new_v4()), + ..Default::default() + }, + ) + .await + .expect_err("the set must reject a stale authorized bucket identity"); + assert!( + matches!(error, StorageError::BucketNotFound(_)), + "identity mismatch must retain its typed error: {error:?}" + ); + for dir in &dirs { + assert!( + dir.path().join(bucket).join("inline").exists(), + "a stale request must leave every backup intact" + ); + } + } + #[tokio::test] + #[serial_test::serial(shard_integrity_rollout)] + async fn batch_delete_completed_multipart_reclaims_native_part_sidecars() { + assert_completed_multipart_delete_reclaims_sidecars(false).await; + } + + #[tokio::test] + #[serial_test::serial(shard_integrity_rollout)] + async fn batch_delete_completed_multipart_reclaims_protected_part_sidecars() { + assert_completed_multipart_delete_reclaims_sidecars(true).await; + } + + async fn assert_completed_multipart_delete_reclaims_sidecars(protected: bool) { + temp_env::async_with_vars( + [ + (rustfs_config::ENV_SHARD_INTEGRITY_WRITE, Some(if protected { "true" } else { "false" })), + ( + rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED, + Some(if protected { "true" } else { "false" }), + ), + ], + assert_completed_multipart_delete_reclaims_sidecars_inner(protected), + ) + .await; + } + + async fn assert_completed_multipart_delete_reclaims_sidecars_inner(protected: bool) { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "multipart-delete-cleanup"; + let object = "completed-object"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let upload = store + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + .expect("create native multipart upload"); + let mut reader = PutObjReader::from_vec(vec![37; 6 * 1024 * 1024]); + let part = store + .put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default()) + .await + .expect("write native part"); + store + .clone() + .complete_multipart_upload( + bucket, + object, + &upload.upload_id, + vec![CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }], + &ObjectOptions::default(), + ) + .await + .expect("complete native multipart upload"); + for dir in &dirs { + let bytes = tokio::fs::read(dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE)) + .await + .expect("completed metadata"); + let mut metadata = FileMeta::default(); + metadata.unmarshal_msg(&bytes).expect("decode completed metadata"); + let mut info = metadata.versions[0] + .into_fileinfo(bucket, object, true) + .expect("completed version"); + info.hydrate_shard_integrity() + .expect("decode native completed integrity descriptor"); + let path = dir + .path() + .join(bucket) + .join(object) + .join(info.data_dir.expect("external multipart directory").to_string()); + assert!(path.join("part.1.meta").exists(), "real multipart completion retains its native sidecar"); + let integrity = info.parts[0].integrity.as_ref(); + assert_eq!(integrity.is_some(), protected, "completion must retain the selected native format"); + if let Some(integrity) = integrity { + assert!( + path.join(integrity.file_name()).exists(), + "real protected completion retains its integrity proof" + ); + } + } + let (_, errors) = store + .delete_objects( + bucket, + vec![ObjectToDelete { + object_name: object.to_string(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await; + assert!(errors.iter().all(Option::is_none), "native batch delete must succeed: {errors:?}"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("successful multipart delete must permit ordinary bucket removal"); + for dir in &dirs { + assert!( + !dir.path().join(bucket).exists(), + "bucket and completed part sidecars must be removed on every disk" + ); + } + } +} + +#[cfg(test)] +mod delete_recovery_inventory_tests { + use super::*; + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; + use std::future::Future; + use std::task::Poll; + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial] + async fn batch_delete_completes_with_queued_disk_renewal() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let set = store.pools[0].disk_set[0].clone(); + let disks = set.disk_inventory().await.into_iter().flatten().collect::>(); + let bucket = "batch-delete-renew"; + let object = "existing-inline-version"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative real bucket"); + store + .put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![0x73; 4096]), + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed actual version"); + let endpoint = set.set_endpoints[0].clone(); + let ctx = crate::runtime::global::current_ctx(); + let was_dist = ctx.is_dist_erasure().await; + if was_dist { + ctx.update_erasure_type(crate::layout::endpoints::SetupType::Erasure).await; + } + let map = ctx.local_disk_map(); + let previous = map.read().await.get(&endpoint.to_string()).cloned(); + + let barrier = DeleteCleanupBarrier::install(bucket); + let delete_store = store.clone(); + let mut deletion = tokio::spawn(async move { + delete_store + .delete_objects( + bucket, + vec![ObjectToDelete { + object_name: object.to_owned(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await + }); + barrier.wait_until_paused().await; + let extra_gate = set.disks.read().await; + let renewal = set.renew_disk(&endpoint); + tokio::pin!(renewal); + tokio::time::timeout( + Duration::from_secs(5), + futures::future::poll_fn(|cx| { + assert!( + std::pin::pin!(tokio::task::unconstrained(renewal.as_mut())) + .poll(cx) + .is_pending(), + "real renewal must reach inventory write" + ); + if set.disks.try_read().is_err() { + Poll::Ready(()) + } else { + Poll::Pending + } + }), + ) + .await + .expect("queue real renewal writer"); + let connected = map + .read() + .await + .get(&endpoint.to_string()) + .cloned() + .flatten() + .expect("renewal registered new handle"); + assert!(!Arc::ptr_eq(&connected, &disks[0])); + drop(extra_gate); + barrier.release(); + let outcome = tokio::time::timeout(Duration::from_secs(3), async { tokio::join!(renewal.as_mut(), &mut deletion) }).await; + if outcome.is_err() { + deletion.abort(); + } + drop(renewal); + if outcome.is_err() { + let _ = deletion.await; + } + { + let mut map = map.write().await; + match previous { + Some(old) => { + map.insert(endpoint.to_string(), old); + } + None => { + map.remove(&endpoint.to_string()); + } + } + } + if was_dist { + ctx.update_erasure_type(crate::layout::endpoints::SetupType::DistErasure) + .await; + } + let (_, result) = + outcome.expect("batch DELETE and real queued disk renewal must finish without nested inventory-read deadlock"); + let (_, errors) = result.expect("delete task must finish"); + assert!(errors.iter().all(Option::is_none), "DELETE result must remain successful: {errors:?}"); + } +} diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index 8e20fa184..fa50a54b4 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -270,6 +270,134 @@ impl ECStore { .map_err(|error| Error::other_with_context("bucket heal owner task failed", error))? } + /// Recover hidden delete journals through the lifecycle-owned heal boundary. + pub async fn recover_delete_transactions(self: &Arc, bucket: &str, opts: &HealOpts) -> Result { + Ok(self + .recover_delete_transactions_at_incarnation(bucket, None, None, opts) + .await? + .unwrap_or_else(|| HealResultItem { + heal_item_type: HealItemType::Bucket.to_string(), + bucket: bucket.to_owned(), + detail: "delete recovery: discovered=0".to_owned(), + ..Default::default() + })) + } + + pub async fn recover_delete_transactions_at_incarnation( + self: &Arc, + bucket: &str, + object: Option<&str>, + expected: Option, + opts: &HealOpts, + ) -> Result> { + self.recover_delete_transactions_in_scope(bucket, object, None, expected, opts) + .await + } + + pub async fn recover_delete_prefix_at_incarnation( + self: &Arc, + bucket: &str, + prefix: &str, + expected: Option, + opts: &HealOpts, + ) -> Result> { + self.recover_delete_transactions_in_scope(bucket, None, Some(prefix), expected, opts) + .await + } + + async fn recover_delete_transactions_in_scope( + self: &Arc, + bucket: &str, + object: Option<&str>, + prefix: Option<&str>, + expected: Option, + opts: &HealOpts, + ) -> Result> { + if opts.no_lock && !opts.dry_run { + return Err(StorageError::InvalidArgument( + "heal".to_string(), + "no_lock".to_string(), + "delete recovery requires namespace locking".to_string(), + )); + } + let expected = match expected { + Some(expected) => expected, + None => self.bucket_incarnation_id_from_disk(bucket).await?, + }; + let object = object.map(str::to_owned); + let prefix = prefix.map(str::to_owned); + self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move { + let scope = bucket_heal_scope(&bucket).ok_or(StorageError::PreconditionFailed)?; + let explicit_delete = opts.remove + && metadata_sys::has_authoritative_never_versioned_state_in(&store.ctx, &bucket).await? + && matches!( + metadata_sys::get_object_lock_config_state_in(&store.ctx, &bucket).await?, + metadata_sys::ObjectLockConfigState::ConfirmedAbsent + ); + let movement_gate = store.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.read().await; + let can_commit = || scope.check().is_ok(); + let recovery_context = crate::set_disk::DeleteRecoveryContext { + dry_run: opts.dry_run, + explicit_delete, + namespace_owner: (!opts.dry_run) + .then(|| Arc::new((scope.clone(), store.ctx.begin_namespace_commit())) as Arc), + ..crate::set_disk::DeleteRecoveryContext::new(Some(scope.incarnation), &can_commit) + }; + let mut summary = crate::set_disk::DeleteRecoverySummary::default(); + for pool in store.get_pools_for_heal_object(&opts)? { + if store.is_suspended(pool.pool_idx).await || store.is_pool_rebalancing(pool.pool_idx).await { + return Err(StorageError::SlowDown); + } + for (set_index, set) in pool.disk_set.iter().enumerate() { + if opts.set.is_some_and(|requested| requested != set_index) { + continue; + } + if let Some(object) = &object { + if !Arc::ptr_eq(set, &pool.get_disks_by_key(object)) { + continue; + } + summary.add(set.recover_delete_object(&bucket, object, recovery_context.clone()).await?); + } else if let Some(prefix) = &prefix { + summary.add(set.recover_delete_prefix(&bucket, prefix, recovery_context.clone()).await?); + } else { + summary.add(set.recover_delete_bucket(&bucket, recovery_context.clone()).await?); + } + } + } + if summary.pending > 0 { + return Err(StorageError::ErasureReadQuorum); + } + if object.is_some() && !opts.dry_run && summary.blocked > 0 { + return Err(StorageError::SlowDown); + } + if summary.discovered == 0 { + return Ok(None); + } + Ok(Some(HealResultItem { + heal_item_type: if object.is_some() { + HealItemType::Object + } else { + HealItemType::Bucket + } + .to_string(), + object: object.or(prefix).unwrap_or_default(), + bucket, + detail: format!( + "delete recovery{}: discovered={}, committed={}, rolled_back={}, reclaimed={}, blocked={}", + if opts.dry_run { " (dry run)" } else { "" }, + summary.discovered, + summary.committed, + summary.rolled_back, + summary.reclaimed, + summary.blocked + ), + ..Default::default() + })) + }) + .await + } + pub async fn heal_bucket_at_incarnation( self: &Arc, bucket: &str, diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 4d287a702..9e16d5b90 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -4849,7 +4849,9 @@ impl ECStore { opts: ObjectOptions, guards: impl IntoIterator, ) -> Result { - let guards: Vec<_> = guards.into_iter().collect(); + let guards = Arc::new(guards.into_iter().collect::>()); + let mut opts = opts; + opts.delete_mutation_owner = Some(Arc::new((opts.delete_mutation_owner.take(), guards.clone()))); let store = Arc::clone(self); let bucket = bucket.to_owned(); let object = object.to_owned(); @@ -4861,7 +4863,7 @@ impl ECStore { if result.is_ok() { list_objects::observe_list_objects_mutation(&store, &bucket).await; } - // Keep: the only use of `guards` here, it moves them into the task so the locks outlive a dropped caller. + // Transient options also retain these guards through late disk I/O. drop(guards); result }) @@ -4930,7 +4932,7 @@ impl ECStore { let mut opts = opts; let receipt_sink = install_tier_free_version_receipt_sink(&mut opts); let result = self - .handle_delete_object_with_journal_inner(bucket, object, opts, tier_journal_api) + .handle_delete_object_with_journal_inner_boxed(bucket, object, opts, tier_journal_api) .await; enqueue_recorded_tier_free_versions(self, receipt_sink).await; result @@ -4938,6 +4940,16 @@ impl ECStore { .await } + fn handle_delete_object_with_journal_inner_boxed<'a>( + &'a self, + bucket: &'a str, + object: &'a str, + opts: ObjectOptions, + tier_journal_api: Option>, + ) -> impl std::future::Future> + Send + 'a { + Box::pin(self.handle_delete_object_with_journal_inner(bucket, object, opts, tier_journal_api)) + } + async fn handle_delete_object_with_journal_inner( &self, bucket: &str, @@ -4955,9 +4967,9 @@ impl ECStore { if is_meta_bucketname(bucket) || (opts.delete_prefix && opts.bucket_lifecycle_lock_fence.is_some()) { None } else if opts.delete_prefix { - Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?) + Some(Arc::new(self.acquire_bucket_lifecycle_write_lock(bucket).await?)) } else { - Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?) + Some(Arc::new(self.acquire_bucket_lifecycle_read_lock(bucket).await?)) }; let object = if opts.delete_prefix && !opts.delete_prefix_object { object.to_owned() @@ -4975,6 +4987,7 @@ impl ECStore { opts.tier_delete_journal_api = tier_journal_api.clone(); if let Some(guard) = _bucket_lifecycle_guard.as_ref() { opts.add_bucket_lifecycle_lock_guard(guard); + opts.delete_mutation_owner = Some(Arc::new((opts.delete_mutation_owner.take(), guard.clone()))); } if !is_meta_bucketname(bucket) { @@ -5227,6 +5240,40 @@ impl ECStore { return Ok(obj); } Err(err) => { + if (is_err_object_not_found(&err) || is_err_version_not_found(&err)) + && (opts.version_id.is_some() || (!opts.versioned && !opts.version_suspended)) + && opts.http_preconditions.is_none() + && !opts.replication_request + { + let mut reclaimed = None; + for pool in &self.pools { + if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await { + return Err(StorageError::SlowDown); + } + let pool_idx = pool.pool_idx; + let pool = pool.clone(); + match self + .run_external_decommission_capacity_object_delete( + pool_idx, + bucket, + object, + object, + opts.clone(), + |opts| async move { pool.delete_object(bucket, object, opts).await }, + ) + .await + { + Ok(result) => reclaimed = Some(result), + Err(missing) if is_err_object_not_found(&missing) || is_err_version_not_found(&missing) => {} + Err(error) => return Err(error), + } + } + if let Some(mut result) = reclaimed { + result.bucket = bucket.to_owned(); + result.name = decode_dir_object(object); + return Ok(result); + } + } // A folder key (`prefix/`) with no object metadata may still exist on // disk as an orphan empty-directory tree (issue #4189): listings show // it as a common prefix, but no regular delete path can remove it. @@ -5422,12 +5469,13 @@ impl ECStore { None } else { match self.acquire_bucket_lifecycle_read_lock(bucket).await { - Ok(guard) => Some(guard), + Ok(guard) => Some(Arc::new(guard)), Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err), } }; if let Some(guard) = _bucket_lifecycle_guard.as_ref() { opts.add_bucket_lifecycle_lock_guard(guard); + opts.delete_mutation_owner = Some(Arc::new((opts.delete_mutation_owner.take(), guard.clone()))); } if opts.delete_replication_config_snapshot.is_none() { match ReplicationObjectBridge::delete_request_config_in(&self.ctx, bucket).await { @@ -5493,10 +5541,11 @@ impl ECStore { if current_bucket_incarnation_id.is_some() { pause_delete_after_object_lock_snapshot(bucket).await; } - let _object_lock_guards = match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await { + let _object_lock_guards = Arc::new(match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await { Ok(guards) => guards, Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err), - }; + }); + opts.delete_mutation_owner = Some(Arc::new((opts.delete_mutation_owner.take(), _object_lock_guards.clone()))); #[cfg(any(test, feature = "test-util"))] if !_object_lock_guards.is_empty() { notify_delete_namespace_acquired(bucket); @@ -9241,8 +9290,31 @@ mod tests { .expect("read original marker"); assert_eq!(info.delete_marker_incarnation(), Some(old_incarnation)); } - // Taking four actual members offline preserves their existing xl.meta. - // All deletion/recreation work below goes through the normal store API. + // Save each stale member's inline versions before converging the old + // bucket's delete receipts, then reintroduce them after recreation. + let mut stale_metadata = Vec::new(); + for disk in before.iter().skip(12).flatten() { + for version in &historical { + let info = disk + .read_version( + "", + bucket, + "history.bin", + &version.to_string(), + &crate::disk::ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read stale inline data version"); + assert!(info.inline_data() && info.data.is_some(), "fixture payload must be preserved in xl.meta"); + } + stale_metadata.push(( + disk.read_all(bucket, "history.bin/xl.meta").await.unwrap(), + disk.read_all(bucket, &format!("{object}/xl.meta")).await.unwrap(), + )); + } for slot in 12..16 { set.disks.write().await[slot] = None; } @@ -9272,6 +9344,29 @@ mod tests { ) .await .expect("delete exact old marker with four drives offline"); + let pending = store + .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default()) + .await + .expect_err("offline participants require retained delete receipts"); + assert!(matches!( + pending, + StorageError::BucketNotEmpty(_) | StorageError::BucketNotEmptyWithDetails { .. } + )); + assert!( + !crate::bucket::retirement::is_retired(store.clone(), bucket, old_incarnation) + .await + .unwrap() + ); + *set.disks.write().await = before.clone(); + store + .recover_delete_transactions_at_incarnation( + bucket, + None, + Some(old_incarnation), + &rustfs_heal_contracts::heal_channel::HealOpts::default(), + ) + .await + .expect("converge old bucket deletes on all participants"); store .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default()) .await @@ -9281,6 +9376,9 @@ mod tests { .await .expect("durable retirement") ); + for slot in 12..16 { + set.disks.write().await[slot] = None; + } store .handle_make_bucket(bucket, &create) .await @@ -9306,6 +9404,13 @@ mod tests { .expect("write new generation"); new_versions.push((key, info.version_id.unwrap(), payload)); } + for (slot, (history, marker)) in (12..16).zip(stale_metadata) { + for (key, bytes) in [("history.bin", history), (object, marker)] { + let root = dirs[slot].path().join(bucket).join(key); + tokio::fs::create_dir_all(&root).await.unwrap(); + tokio::fs::write(root.join("xl.meta"), bytes).await.unwrap(); + } + } *set.disks.write().await = before.clone(); let heal = rustfs_heal_contracts::heal_channel::HealOpts { remove: true, @@ -11477,3 +11582,160 @@ mod tests { .await; } } + +#[cfg(test)] +mod missing_key_delete_quorum_tests { + use super::*; + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::storage_api_contracts::bucket::{BucketOperations, BucketOptions, MakeBucketOptions}; + + async fn degraded_store(bucket: &str) -> (Vec, Arc) { + let (dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create authoritative never-versioned bucket"); + store.pools[0].disk_set[0].disks.write().await[0] = None; + let info = store + .get_bucket_info(bucket, &BucketOptions::default()) + .await + .expect("existing bucket must remain readable at quorum"); + assert_eq!(info.name, bucket); + let lookup = store + .get_object_info_for_delete(bucket, "never-existed", &ObjectOptions::default()) + .await; + assert!( + lookup.as_ref().is_err_and(is_err_object_not_found), + "remaining disks must establish absence through production lookup: {lookup:?}" + ); + (dirs, store) + } + + #[tokio::test] + async fn single_delete_missing_key_preserves_quorum_noop() { + let (_dirs, store) = degraded_store("missing-key-quorum-single").await; + let result = store + .delete_object_with_tier_delete_journal("missing-key-quorum-single", "never-existed", ObjectOptions::default()) + .await; + assert!( + result.as_ref().is_err_and(is_err_object_not_found), + "single missing-key delete must preserve its idempotent not-found outcome at quorum: {result:?}" + ); + } + + #[tokio::test] + async fn batch_delete_missing_key_preserves_quorum_noop() { + let (_dirs, store) = degraded_store("missing-key-quorum-batch").await; + let (_, errors, _) = store + .delete_objects_with_tier_delete_journal_and_accounting( + "missing-key-quorum-batch", + vec![ObjectToDelete { + object_name: "never-existed".to_owned(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await; + assert_eq!(errors.len(), 1, "batch DELETE must return one result per key"); + assert!( + errors.iter().all(Option::is_none), + "batch missing-key delete must preserve its idempotent success at quorum: {errors:?}" + ); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn batch_delete_timeout_retains_physical_key_guard() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + use crate::disk::{DiskAPI, ReadOptions}; + use crate::storage_api_contracts::object::ObjectIO; + temp_env::async_with_vars([("RUSTFS_DELETE_RECOVERY_TIMEOUT_SECS", Some("30"))], async { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "review-batch-late-owner"; + let key = "object"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("fixture bucket"); + let mut reader = PutObjReader::from_vec(vec![0x51; 1024 * 1024]); + store + .put_object( + bucket, + key, + &mut reader, + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed real external object"); + let disks = store.pools[0].disk_set[0].disk_inventory().await; + let disk = disks[0].as_ref().expect("online drive"); + let info = disk + .read_version("", bucket, key, "", &ReadOptions::default()) + .await + .expect("seed metadata"); + assert!(!info.inline_data()); + let crate::disk::Disk::Local(local) = disk.as_ref() else { + panic!("local disk"); + }; + let part = local + .get_disk() + .get_object_path_for_io(bucket, key) + .expect("leased key") + .join(info.data_dir.expect("external data directory").to_string()) + .join("part.1"); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel::<()>(); + let hook = hooks::install_at(hooks::Stage::Remove, &part, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let deleting = store.clone(); + let delete = tokio::spawn(async move { + deleting + .delete_objects_with_tier_delete_journal_and_accounting( + bucket, + vec![ObjectToDelete { + object_name: key.to_owned(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .expect("physical cleanup entry") + .expect("entry hook"); + assert!(part.exists(), "payload unlink has not run"); + let (_, errors, _) = tokio::time::timeout(Duration::from_secs(40), delete) + .await + .expect("real settlement timeout must return") + .expect("delete task"); + assert!(errors.iter().all(Option::is_none), "certified batch response: {errors:?}"); + assert!( + store.ctx.namespace_commits_pending(), + "background physical operation must retain its owner" + ); + let ns = store.handle_new_ns_lock(bucket, key).await.expect("same production key lock"); + let probe = ns.get_write_lock(Duration::from_millis(300)).await; + let acquired_while_pending = probe.is_ok(); + drop(probe); + drop(release); + drop(hook); + assert!( + !acquired_while_pending, + "batch released the exact-key guard while physical cleanup was still paused after its real recovery timeout" + ); + let _released = ns + .get_write_lock(Duration::from_secs(10)) + .await + .expect("physical cleanup must eventually release the exact-key guard"); + }) + .await; + } +} diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index 051159c0e..35facd16b 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -574,6 +574,29 @@ pub trait HealStorageAPI: Send + Sync { /// Heal bucket using ecstore async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result; + /// Recover transactions omitted by ordinary listings. Alternate providers + /// may explicitly expose no transaction-recovery capability. + async fn recover_delete_transactions( + &self, + _bucket: &str, + _object: Option<&str>, + _expected: Option, + _opts: &HealOpts, + ) -> Result> { + Ok(None) + } + + /// Recover hidden transactions only within the requested object prefix. + async fn recover_delete_prefix_transactions( + &self, + _bucket: &str, + _prefix: &str, + _expected: Option, + _opts: &HealOpts, + ) -> Result> { + Ok(None) + } + /// Heal format using ecstore async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)>; @@ -1498,6 +1521,32 @@ impl HealStorageAPI for ECStoreHealStorage { .await) } + async fn recover_delete_transactions( + &self, + bucket: &str, + object: Option<&str>, + expected: Option, + opts: &HealOpts, + ) -> Result> { + self.ecstore + .recover_delete_transactions_at_incarnation(bucket, object, expected, opts) + .await + .map_err(Error::Storage) + } + + async fn recover_delete_prefix_transactions( + &self, + bucket: &str, + prefix: &str, + expected: Option, + opts: &HealOpts, + ) -> Result> { + self.ecstore + .recover_delete_prefix_at_incarnation(bucket, prefix, expected, opts) + .await + .map_err(Error::Storage) + } + async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { debug!( target: "rustfs::heal::storage", diff --git a/crates/heal/src/heal/task/heal_bucket.rs b/crates/heal/src/heal/task/heal_bucket.rs index 73bc20291..cf6b4786f 100644 --- a/crates/heal/src/heal/task/heal_bucket.rs +++ b/crates/heal/src/heal/task/heal_bucket.rs @@ -220,6 +220,22 @@ impl HealTask { set: self.options.set_index, }; + let recovery_opts = HealOpts { + remove: self.source == HealRequestSource::Admin && self.options.remove_corrupted, + ..heal_opts + }; + if let Some(result) = self + .await_with_control(self.storage.recover_delete_transactions( + bucket, + None, + self.bucket_incarnation_id, + &recovery_opts, + )) + .await? + { + self.record_result_item(result).await; + } + let heal_result = match self.bucket_incarnation_id { Some(expected) => { self.await_with_control(self.storage.heal_bucket_at_incarnation(bucket, expected, &heal_opts)) @@ -446,6 +462,26 @@ impl HealTask { "Heal prefix started" ); + let opts = HealOpts { + dry_run: self.options.dry_run, + remove: self.source == HealRequestSource::Admin && self.options.remove_corrupted, + no_lock: self.options.no_lock, + pool: self.options.pool_index, + set: self.options.set_index, + ..Default::default() + }; + if let Some(result) = self + .await_with_control(self.storage.recover_delete_prefix_transactions( + bucket, + prefix, + self.bucket_incarnation_id, + &opts, + )) + .await? + { + self.record_result_item(result).await; + } + self.heal_bucket_objects(bucket, prefix).await } diff --git a/crates/heal/src/heal/task/heal_erasure_set.rs b/crates/heal/src/heal/task/heal_erasure_set.rs index df9b852e9..376fc5314 100644 --- a/crates/heal/src/heal/task/heal_erasure_set.rs +++ b/crates/heal/src/heal/task/heal_erasure_set.rs @@ -460,6 +460,21 @@ impl HealTask { set: self.options.set_index, }; + let recovery_opts = HealOpts { + remove: self.source == HealRequestSource::Admin && self.options.remove_corrupted, + ..bucket_heal_opts + }; + // Include metadata-less transactions for both startup and admin heals. + for bucket in &buckets { + self.check_control_flags().await?; + if let Some(result) = self + .await_with_control(self.storage.recover_delete_transactions(bucket, None, None, &recovery_opts)) + .await? + { + self.record_result_item(result).await; + } + } + for bucket in buckets.iter().filter(|_| self.source != HealRequestSource::Admin) { // Check control flags before starting each bucket heal self.check_control_flags().await?; diff --git a/crates/heal/src/heal/task/heal_object.rs b/crates/heal/src/heal/task/heal_object.rs index 0dc4ec298..e20d4f315 100644 --- a/crates/heal/src/heal/task/heal_object.rs +++ b/crates/heal/src/heal/task/heal_object.rs @@ -99,6 +99,40 @@ impl HealTask { // recreated bucket after the replay anchor was captured. return self.heal_mrf_partial_write_object(bucket, object, version_id).await; } + let heal_opts = HealOpts { + recursive: self.options.recursive, + dry_run: self.options.dry_run, + remove: self.options.remove_corrupted, + recreate: self.options.recreate_missing, + scan_mode: self.options.scan_mode, + update_parity: self.options.update_parity, + no_lock: self.options.no_lock, + read_repair: self.source == HealRequestSource::ReadRepair, + pool: self.options.pool_index, + set: self.options.set_index, + }; + let mut recovery_handled = false; + if matches!(self.source, HealRequestSource::Admin | HealRequestSource::Internal) { + let recovery_opts = HealOpts { + remove: heal_opts.remove && version_id.is_none_or(str::is_empty), + ..heal_opts + }; + if let Some(result) = self + .await_with_control(self.storage.recover_delete_transactions( + bucket, + Some(object), + self.bucket_incarnation_id, + &recovery_opts, + )) + .await? + { + recovery_handled = true; + self.record_result_item(result).await; + if self.options.dry_run { + return Ok(()); + } + } + } let mut object_exists = match self.await_with_control(self.storage.object_exists(bucket, object)).await { Ok(exists) => exists, Err(err @ Error::TransientSkip { .. }) => { @@ -129,7 +163,7 @@ impl HealTask { object }; - if !object_exists { + if !object_exists && !recovery_handled { // Background loops (scanner/MRF/autoheal/read-repair) routinely // race object deletion, so a missing target is per-object noise // for them; only foreground admin/internal requests keep the warn. @@ -197,18 +231,6 @@ impl HealTask { update_parity = self.options.update_parity, "Heal object stage entered" ); - let heal_opts = HealOpts { - recursive: self.options.recursive, - dry_run: self.options.dry_run, - remove: self.options.remove_corrupted, - recreate: self.options.recreate_missing, - scan_mode: self.options.scan_mode, - update_parity: self.options.update_parity, - no_lock: self.options.no_lock, - read_repair: self.source == HealRequestSource::ReadRepair, - pool: self.options.pool_index, - set: self.options.set_index, - }; let mut expected_identity = self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index); expected_identity.bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, self.options.dry_run).await?; diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index 309f485d3..f5c12e5ba 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -527,6 +527,29 @@ pub struct DeletePathsResponse { #[prost(message, optional, tag = "2")] pub error: ::core::option::Option, } +/// Protocol 1 is additive and has no fallback to generic file mutation RPCs. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RecoverDeleteTransactionRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(uint32, tag = "3")] + pub protocol_version: u32, + #[prost(bytes = "bytes", tag = "4")] + pub operation: ::prost::bytes::Bytes, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RecoverDeleteTransactionResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(uint32, tag = "2")] + pub protocol_version: u32, + #[prost(bytes = "bytes", tag = "3")] + pub result: ::prost::bytes::Bytes, + #[prost(message, optional, tag = "4")] + pub error: ::core::option::Option, +} #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SnapshotLeaseRequest { #[prost(string, tag = "1")] @@ -2082,6 +2105,21 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "DeleteAtIncarnation")); self.inner.unary(req, path, codec).await } + pub async fn recover_delete_transaction( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RecoverDeleteTransaction"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "RecoverDeleteTransaction")); + self.inner.unary(req, path, codec).await + } pub async fn acquire_snapshot_lease( &mut self, request: impl tonic::IntoRequest, @@ -3463,6 +3501,10 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn recover_delete_transaction( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn acquire_snapshot_lease( &self, request: tonic::Request, @@ -4237,6 +4279,34 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/RecoverDeleteTransaction" => { + #[allow(non_camel_case_types)] + struct RecoverDeleteTransactionSvc(pub Arc); + impl tonic::server::UnaryService for RecoverDeleteTransactionSvc { + type Response = super::RecoverDeleteTransactionResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::recover_delete_transaction(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = RecoverDeleteTransactionSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/AcquireSnapshotLease" => { #[allow(non_camel_case_types)] struct AcquireSnapshotLeaseSvc(pub Arc); diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index 451afa21a..5b69f697f 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -1259,6 +1259,37 @@ pub fn canonical_prepare_part_transaction_request_body( Ok(body.finish()) } +/// Recovery messages are bounded and must contain exactly one MessagePack value. +pub const DELETE_RECOVERY_MAX_PAYLOAD_BYTES: usize = DEFAULT_GRPC_SERVER_MESSAGE_LEN; +pub const DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES: usize = 1024 * 1024; + +pub fn decode_delete_recovery_payload(payload: &[u8]) -> std::io::Result { + if payload.is_empty() || payload.len() > DELETE_RECOVERY_MAX_PAYLOAD_BYTES { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "invalid delete recovery payload length", + )); + } + let mut cursor = std::io::Cursor::new(payload); + let value = rmp_serde::from_read(&mut cursor) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid delete recovery payload"))?; + if cursor.position() != u64::try_from(payload.len()).map_err(std::io::Error::other)? { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "trailing delete recovery payload")); + } + Ok(value) +} + +pub fn canonical_recover_delete_transaction_request_body( + request: &proto_gen::node_service::RecoverDeleteTransactionRequest, +) -> Result, std::num::TryFromIntError> { + let mut body = CanonicalBodyBuilder::new(b"rustfs-recover-delete-transaction-request-v1\0"); + body.push_str(&request.disk)?; + body.push_str(&request.volume)?; + body.push_u32(request.protocol_version); + body.push_bytes(&request.operation)?; + Ok(body.finish()) +} + pub fn canonical_settle_part_transaction_request_body( request: &proto_gen::node_service::SettlePartTransactionRequest, ) -> Result, std::num::TryFromIntError> { @@ -1313,6 +1344,40 @@ mod disk_mutation_canonical_tests { }; use super::*; + #[test] + fn delete_recovery_canonical_body_binds_all_wire_fields() { + let request = proto_gen::node_service::RecoverDeleteTransactionRequest { + disk: "disk".into(), + volume: "volume".into(), + protocol_version: 1, + operation: vec![0x80].into(), + }; + let mut bodies = vec![canonical_recover_delete_transaction_request_body(&request).expect("canonical body")]; + for mutate in [ + |r: &mut proto_gen::node_service::RecoverDeleteTransactionRequest| r.disk = "other".into(), + |r: &mut proto_gen::node_service::RecoverDeleteTransactionRequest| r.volume = "other".into(), + |r: &mut proto_gen::node_service::RecoverDeleteTransactionRequest| r.protocol_version = 2, + |r: &mut proto_gen::node_service::RecoverDeleteTransactionRequest| r.operation = vec![0x81].into(), + ] { + let mut changed = request.clone(); + mutate(&mut changed); + bodies.push(canonical_recover_delete_transaction_request_body(&changed).expect("changed canonical body")); + } + assert_all_distinct(&bodies); + } + + #[test] + fn delete_recovery_payload_rejects_trailing_malformed_and_oversized_values() { + let payload = rmp_serde::to_vec_named(&vec![1_u32, 2]).expect("encode test payload"); + assert_eq!(decode_delete_recovery_payload::>(&payload).expect("one payload"), vec![1, 2]); + let mut trailing = payload; + trailing.push(0xc0); + assert!(decode_delete_recovery_payload::>(&trailing).is_err()); + assert!(decode_delete_recovery_payload::>(&[]).is_err()); + assert!(decode_delete_recovery_payload::>(&[0xc1]).is_err()); + assert!(decode_delete_recovery_payload::>(&vec![0xc0; DELETE_RECOVERY_MAX_PAYLOAD_BYTES + 1]).is_err()); + } + fn assert_all_distinct(bodies: &[Vec]) { for (i, a) in bodies.iter().enumerate() { for (j, b) in bodies.iter().enumerate() { @@ -2905,7 +2970,7 @@ pub async fn get_channel_for_class(addr: &str, class: ChannelClass) -> Result Result Result> { /// channels to the same peer be cached independently. async fn build_channel(dial_addr: &str, cache_key: &str) -> Result> { let deadline = tokio::time::Instant::now() + internode_connect_timeout(); - build_channel_until(dial_addr, cache_key, deadline).await + build_channel_until(dial_addr, cache_key, deadline, internode_rpc_timeout()).await +} + +/// A separate channel deadline for slow authenticated recovery work. The cache +/// key keeps this deadline from changing foreground lock, read, and write RPCs. +#[derive(Debug)] +pub enum DeleteRecoveryChannelError { + Connection(Box), +} + +impl std::fmt::Display for DeleteRecoveryChannelError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Connection(error) => write!(formatter, "delete recovery channel connection failed: {error}"), + } + } +} + +impl Error for DeleteRecoveryChannelError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Connection(error) => Some(error.as_ref()), + } + } +} + +pub async fn create_channel_with_timeout(addr: &str, rpc_timeout: Duration) -> Result { + let cache_key = format!("delete-recovery:{}:{addr}", rpc_timeout.as_millis()); + let deadline = tokio::time::Instant::now() + internode_connect_timeout(); + build_channel_until(addr, &cache_key, deadline, rpc_timeout) + .await + .map_err(DeleteRecoveryChannelError::Connection) } async fn build_channel_until( dial_addr: &str, cache_key: &str, deadline: tokio::time::Instant, + rpc_timeout: Duration, ) -> Result> { tokio::time::timeout_at(deadline, async { if let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { @@ -2962,7 +3059,7 @@ async fn build_channel_until( if let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { return Ok(channel); } - let (channel, generation) = dial_channel(dial_addr, cache_key).await?; + let (channel, generation) = dial_channel(dial_addr, cache_key, rpc_timeout).await?; if cache_dialed_channel(dial_addr, cache_key, &channel, generation).await && let Some(channel) = cached_channel_for_tls_generation(dial_addr, cache_key).await { @@ -2980,15 +3077,13 @@ async fn build_channel_until( .map_err(channel_acquisition_timeout)? } -async fn dial_channel(dial_addr: &str, cache_key: &str) -> Result<(Channel, Option), Box> { +async fn dial_channel(dial_addr: &str, cache_key: &str, rpc_timeout: Duration) -> Result<(Channel, Option), Box> { debug!("Creating new gRPC channel to: {} (cache key: {})", dial_addr, cache_key); let dial_started_at = Instant::now(); let connect_timeout = internode_connect_timeout(); let tcp_keepalive = internode_tcp_keepalive(); let http2_keepalive_interval = internode_http2_keep_alive_interval(); let http2_keepalive_timeout = internode_http2_keep_alive_timeout(); - let rpc_timeout = internode_rpc_timeout(); - let mut connector = Endpoint::from_shared(dial_addr.to_string())? // Fast connection timeout for dead peer detection .connect_timeout(connect_timeout) @@ -3291,6 +3386,31 @@ mod tests { } } + #[tokio::test] + async fn tls02_delete_recovery_channels_reject_unbound_and_stale_https_cache() { + let _test_guard = TLS02_TEST_LOCK.lock().await; + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve recovery peer address"); + let addr = format!("https://{}", listener.local_addr().expect("recovery peer address")); + drop(listener); + let rpc_timeout = Duration::from_secs(20); + let cache_key = format!("delete-recovery:{}:{addr}", rpc_timeout.as_millis()); + let generation = runtime_sources::outbound_tls_generation(); + for cached_generation in [None, Some(generation.checked_add(1).unwrap_or(0))] { + cache_connection(cache_key.clone(), Endpoint::from_static("http://127.0.0.1:1").connect_lazy()).await; + if let Some(generation) = cached_generation { + TLS_GENERATION_CACHE.lock().await.insert(cache_key.clone(), generation); + } else { + TLS_GENERATION_CACHE.lock().await.remove(&cache_key); + } + assert!( + create_channel_with_timeout(&addr, rpc_timeout).await.is_err(), + "recovery must dial the unavailable HTTPS peer instead of returning an unbound or stale cached channel" + ); + assert!(!rustfs_common::has_cached_connection(&cache_key).await); + assert!(!TLS_GENERATION_CACHE.lock().await.contains_key(&cache_key)); + } + } + #[tokio::test] async fn tls02_current_https_control_and_bulk_cache_entries_are_reused() { let _test_guard = TLS02_TEST_LOCK.lock().await; @@ -3444,7 +3564,7 @@ mod tests { for _ in 0..8 { tasks.push(tokio::spawn(async move { let deadline = tokio::time::Instant::now() + Duration::from_millis(40); - let result = build_channel_until(addr, addr, deadline).await; + let result = build_channel_until(addr, addr, deadline, internode_rpc_timeout()).await; result.err().is_some_and(|error| { error .downcast_ref::() diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index a99f73185..fd56c543c 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -391,6 +391,21 @@ message DeletePathsResponse { optional Error error = 2; } +// Protocol 1 is additive and has no fallback to generic file mutation RPCs. +message RecoverDeleteTransactionRequest { + string disk = 1; + string volume = 2; + uint32 protocol_version = 3; + bytes operation = 4; +} + +message RecoverDeleteTransactionResponse { + bool success = 1; + uint32 protocol_version = 2; + bytes result = 3; + optional Error error = 4; +} + message SnapshotLeaseRequest { string disk = 1; string volume = 2; @@ -1218,6 +1233,7 @@ service NodeService { rpc CompareAndUpdateFile(CompareAndUpdateFileRequest) returns (CompareAndUpdateFileResponse) {}; // auth-policy: body-bound rpc Delete(DeleteRequest) returns (DeleteResponse) {}; // auth-policy: body-bound rpc DeleteAtIncarnation(DeleteRequest) returns (DeleteResponse) {}; // auth-policy: body-bound + rpc RecoverDeleteTransaction(RecoverDeleteTransactionRequest) returns (RecoverDeleteTransactionResponse) {}; // auth-policy: body-bound rpc AcquireSnapshotLease(SnapshotLeaseRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound rpc RenewSnapshotLease(SnapshotLeaseRenewRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound rpc ReleaseSnapshotLease(SnapshotLeaseReleaseRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound diff --git a/docs/README.md b/docs/README.md index 2b89a015d..d30e3758a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -47,6 +47,9 @@ For concurrent bucket creation and deletion, see For persisted administrator bucket tasks and bucket recreation, see [Bucket heal recovery](operations/bucket-heal-recovery.md). +For interrupted object deletions and returning drives, see +[Delete transaction recovery](operations/delete-transaction-recovery.md). + For disk replacement across VM restarts and schema 5/6 maintenance migration, see [Replacement generation recovery](operations/replacement-generation-recovery.md). diff --git a/docs/operations/delete-transaction-recovery.md b/docs/operations/delete-transaction-recovery.md new file mode 100644 index 000000000..175eb84f1 --- /dev/null +++ b/docs/operations/delete-transaction-recovery.md @@ -0,0 +1,137 @@ +# Delete transaction recovery + +An interrupted object deletion can leave metadata backups, reserved shards, or +an unpublished target. Normal S3 listings cannot discover all of these files. +Recovery therefore inspects raw transaction records on every configured drive +in the owning erasure set under the bucket incarnation fence and exact-key +write lock. It does not modify GET or PUT behavior. + +## Decisions and replay + +New deletions assemble the backup, versioned prepared journal, and exact target +in the private system temporary area, then atomically publish the complete +transaction directory before modifying metadata. Interrupted preparation cannot +leave an undiscoverable target-only record or temporary file in the object key. After the metadata write quorum, the coordinator publishes durable +commit receipts to a write quorum before returning success. A failed or lost +publication response is indeterminate: recovery must inspect the persisted +records instead of immediately restoring the backup. + +| Evidence | Action | +| --- | --- | +| New prepared transaction, no decision, intact original shards and matching generation | Restore and sync the original metadata before removing preparation evidence. | +| Structured commit receipt | Publish the certified target to matching returning participants and finish deletion. | +| Missing, unavailable, or unsupported participant | Preserve records and retry when every configured participant can be inspected. | +| Newer generation or partially overlapping version history | Preserve records and report blocked recovery. | +| Historical backup or disk-local marker without a structured set decision | Preserve records during automatic recovery. | + +Missing local metadata does not authorize user-bucket residue cleanup, even +when the current object exists on a metadata quorum. Legacy local cleanup is +restricted to internal system-bucket maintenance. User-bucket legacy residue +uses only the guarded set-wide removal described below. Dry-run and execution +share the same source-generation check; an already-absent explicit non-null +version can converge without touching unrelated live versions. + +Rollback preserves native multipart metadata and integrity sidecars. Protected +parts require their declared proof generation; recovery verifies both the proof +records and actual shard payloads before restoring metadata or removing +recovery evidence. Missing or corrupt required proofs leave the transaction +blocked with its backup intact. + +A disk-local `delete-data.` marker does not prove an erasure-set decision. +Returning drives cannot restore stale data after a certified deletion. Commit +receipts remain until every participant and related transaction for the key +has converged. For consecutive certified version deletions, an older decision +can finish its own data cleanup after later deletion removes current metadata. +A prepared copy needs a matching same-transaction decision before promotion; +absence alone cannot certify it. Cross-transaction null-version supersession also requires causal +predecessor evidence; timestamps alone cannot authorize it. Cleanup respects active snapshots and removes only validated +transaction files, with directory removal performed nonrecursively. + +## Entry points + +Unclean-shutdown erasure-set healing performs raw transaction recovery before +ordinary object listing. Administrator bucket, set, prefix, and object heal use the +same recovery core. Existing heal selection, lifecycle admission, namespace +locking, and movement guards still apply. Prefix recovery settles only matching +keys, including transactions hidden from listings. Dry-run inspects and reports without +settling transactions. + +An unconditional DELETE of a missing exact key, or an administrator heal with +`remove` enabled, can authorize historical residue removal only when the +bucket is authoritatively never-versioned and has no Object Lock. Every +configured drive must be inspected, the current object must be absent, and all +residue must match the guarded legacy layout. Versioned/suspended buckets, +retained versions, conditions, unknown entries, symlinks, and prefix children +are not covered by that authorization. The same fresh intent can certify the +absence of an unprotected new prepared null-version transaction when every +configured drive has no current metadata. This is a new authorized decision; +it does not infer an old commit from missing causal evidence. Mixed or +unrecognized historical layouts remain blocked. + +A missing specific-version DELETE first recovers hidden transactions and then +reads the current version again. It must not acknowledge an unresolved hidden +version as an idempotent no-op. + +## Upgrade and failure handling + +Upgrade all participating nodes before relying on the new protocol. Recovery +uses one authenticated additive RPC; unsupported peers cannot silently accept +new flags through the legacy delete endpoint. An online older participant +causes the modern operation to fail before decision publication. Modern nodes +also reject legacy forward transactional user-bucket deletes. A cluster still +controlled by an older majority retains the older behavior. Do not downgrade +while recovery records remain; old binaries cannot interpret the new protocol. + +Keep unavailable drives and transaction directories intact. Bring the full +set online, run a scoped heal in dry-run mode, inspect its recovery result, +and then run the same scoped heal without dry-run. Use `remove` for ambiguous +historical residue only after verifying that the exact objects are intended +to be deleted and the bucket meets the restrictions above. Do not recursively +remove transaction directories by hand to make a bucket appear empty. + +Recovery reports discovered, committed, rolled-back, reclaimed historical, +and blocked transaction counts. Unavailable participants return a retryable +quorum error; blocked evidence requires operator investigation. A successful +S3 delete response does not imply immediate physical cleanup while snapshots +or returning participants still retain necessary proof. + +## Settlement deadline and durability + +`RUSTFS_DELETE_RECOVERY_TIMEOUT_SECS` controls the settlement RPC deadline, +including full shard verification during rollback. Key and bucket lifecycle +ownership stay attached to late local filesystem operations after cancellation +or timeout. The default is 900 seconds; +valid values are 30 through 86400 seconds. Invalid values fail explicitly. +Increase it when intact large objects need more verification time. It does not +change normal object read or delete-apply deadlines. A timeout can conceal a +completed mutation, so rerun inspection and recovery rather than guessing the +outcome from the RPC error. + +The records and restores follow the configured durability policy. Strict mode +syncs the required metadata and directory changes. Process SIGKILL testing +checks restart behavior; it does not establish power-loss behavior on every +filesystem or storage device. Mode `none` intentionally offers weaker sync +guarantees. Do not treat a throughput comparison as a durability substitute. + +## Reproduce the Docker regression suite + +Build the candidate with the test-only barriers enabled: + +```sh +cargo build --locked --release -p rustfs --bin rustfs --no-default-features --features e2e-test-hooks +python3 scripts/test/delete_recovery_docker.py --help +``` + +Pass a native Linux binary with `--binary`, a Linux image containing Python and +curl with `--image`, and a unique Docker namespace. The optional +`--baseline-binary` enables the mixed-version case. The harness uses synthetic +S3 objects, localhost ports, labeled isolated volumes, and a non-root server. +It stops only its own containers and keeps JSON results and logs in the chosen +artifact directory. Production builds must not enable `e2e-test-hooks`. + +Run the preactivation/prepare/apply/decision process-stop cases, interrupted restore, +returning and untouched peers, both generation orderings, version/retention +and prefix protection, administrator dry-run/removal, response-loss, +permission/corruption, and mixed-version cases. Preserve the source identity +of both binaries with the results. A passing process-stop matrix does not +cover hardware power cuts, Windows runtime behavior, or DELETE throughput. diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index c93f7bdd1..f57c4ac6e 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -4167,6 +4167,51 @@ mod tests { ); } + #[tokio::test] + #[serial_test::serial] + async fn cold_delete_recovery_discovery_uses_production_body_auth() { + use storage::DiskAPI; + temp_env::async_with_vars([("RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT", Some("false"))], async { + let _ = rustfs_credentials::set_global_rpc_secret("rpc-http-test-secret".to_string()); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind recovery RPC peer"); + let addr = listener.local_addr().expect("recovery RPC address"); + let previous_node_name = rustfs_common::get_global_local_node_name().await; + rustfs_common::set_global_local_node_name(&addr.to_string()).await; + let node = InterceptedService::new(NodeServiceServer::new(make_server()), check_auth); + let service = RpcRequestPathService::new(Routes::new(node).prepare()); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.expect("accept recovery RPC connection"); + ConnBuilder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(socket), TowerToHyperService::new(service)) + .await + .expect("serve recovery RPC connection"); + }); + let mut endpoint = + Endpoint::try_from(format!("http://{addr}/cold-recovery-disk").as_str()).expect("remote recovery endpoint"); + endpoint.is_local = false; + let disk = storage::new_disk( + &endpoint, + &storage::DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("production remote disk client"); + // Recovery may race a heal-control peer-state refresh, before another + // authenticated response has repopulated the shared peer state. + let result = disk.discover_delete_transactions("cold-recovery-bucket", None, None, 1).await; + server.abort(); + let _ = server.await; + rustfs_common::set_global_local_node_name(&previous_node_name).await; + assert!( + matches!(result, Err(crate::storage_api::error::DiskError::DiskNotFound)), + "the cold request must pass production authentication and reach the missing-disk handler: {result:?}" + ); + }) + .await; + } + #[tokio::test] #[serial_test::serial] async fn peer_rest_heal_control_uses_production_auth_and_keeps_validation_errors_online() { diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 54f2315b4..255bf1044 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -1536,6 +1536,13 @@ impl Node for NodeService { self.handle_rename_part(request).await } + async fn recover_delete_transaction( + &self, + request: Request, + ) -> Result, Status> { + self.handle_recover_delete_transaction(request).await + } + async fn settle_part_transaction( &self, request: Request, @@ -3073,13 +3080,14 @@ mod tests { LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest, MakeVolumesRequest, Mss, PingRequest, PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, - ReadVersionRequest, ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, - RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerDirtyUsageSnapshotRequest, - ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest, - SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, - StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest, - TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest, - UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest, + ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest, ReloadPoolMetaRequest, + ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, ScannerActivityRequest, + ScannerDirtyUsageSnapshotRequest, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, + ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest, SnapshotLeaseReleaseRequest, + SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, + StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass, TierMutationPeerState, + TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, + WriteMetadataRequest, WriteRequest, heal_control_service_client::HealControlServiceClient, heal_control_service_server::{HealControlService as _, HealControlServiceServer}, node_service_client::NodeServiceClient, @@ -3097,7 +3105,7 @@ mod tests { use tonic::{Request, Response, Status}; use uuid::Uuid; - const DISK_MUTATION_RPC_METHODS: [&str; 24] = [ + const DISK_MUTATION_RPC_METHODS: [&str; 25] = [ "renamedata", "renamedataatincarnation", "writemetadataatincarnation", @@ -3116,6 +3124,7 @@ mod tests { "renamepart", "prepareparttransaction", "settleparttransaction", + "recoverdeletetransaction", "deletevolume", "makevolume", "makevolumes", @@ -4386,6 +4395,177 @@ mod tests { assert!(!signed.into_inner().success, "the unknown test disk cannot resolve"); } + #[tokio::test] + async fn delete_recovery_apply_rejects_excessive_batch_cardinality_before_lookup() { + use crate::storage::storage_api::{DeleteOptions, DeleteRecoveryRpcOperation, FileInfoVersions}; + let service = make_server(); + for versions in [ + vec![FileInfoVersions::default(); 1001], + vec![FileInfoVersions { + name: "object".into(), + versions: vec![Default::default(); 1001], + ..Default::default() + }], + ] { + let operation = DeleteRecoveryRpcOperation::ApplyVersions { + versions, + opts: DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + ..Default::default() + }, + }; + let mut request = Request::new(RecoverDeleteTransactionRequest { + disk: "unknown".into(), + volume: "bucket".into(), + protocol_version: 1, + operation: rmp_serde::to_vec_named(&operation).expect("encode batch").into(), + }); + let body = + rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()).expect("canonical batch"); + set_tonic_canonical_body_digest(&mut request, &body).expect("batch digest"); + mark_v2_authenticated(&mut request); + let error = service + .recover_delete_transaction(request) + .await + .expect_err("oversized batch must reject before disk lookup"); + assert_eq!(error.code(), tonic::Code::InvalidArgument); + } + } + + #[tokio::test] + async fn delete_recovery_apply_rejects_legacy_cleanup_options_and_absolute_keys() { + use crate::storage::storage_api::{DeleteOptions, DeleteRecoveryRpcOperation}; + let service = make_server(); + let valid = DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + ..Default::default() + }; + let mut invalid = vec![DeleteOptions::default()]; + for mutate in [ + |opts: &mut DeleteOptions| opts.old_data_dir = Some(Uuid::nil()), + |opts: &mut DeleteOptions| opts.undo_delete = true, + |opts: &mut DeleteOptions| opts.undo_write = true, + |opts: &mut DeleteOptions| opts.recursive = true, + |opts: &mut DeleteOptions| opts.immediate = true, + ] { + let mut opts = valid.clone(); + mutate(&mut opts); + invalid.push(opts); + } + let mut operations = Vec::new(); + for opts in invalid { + operations.push(DeleteRecoveryRpcOperation::ApplyVersion { + object: "object".into(), + file_info: Default::default(), + force_del_marker: false, + opts: opts.clone(), + }); + operations.push(DeleteRecoveryRpcOperation::ApplyVersions { + versions: Vec::new(), + opts, + }); + } + operations.push(DeleteRecoveryRpcOperation::ApplyVersion { + object: "/object".into(), + file_info: Default::default(), + force_del_marker: false, + opts: valid.clone(), + }); + operations.push(DeleteRecoveryRpcOperation::ApplyVersions { + versions: vec![crate::storage::storage_api::FileInfoVersions { + name: "/object".into(), + ..Default::default() + }], + opts: valid, + }); + for operation in operations { + let mut request = Request::new(RecoverDeleteTransactionRequest { + disk: "unknown".into(), + volume: "bucket".into(), + protocol_version: 1, + operation: rmp_serde::to_vec_named(&operation).expect("encode apply operation").into(), + }); + let body = + rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()).expect("canonical apply"); + set_tonic_canonical_body_digest(&mut request, &body).expect("apply digest"); + mark_v2_authenticated(&mut request); + let error = service + .recover_delete_transaction(request) + .await + .expect_err("invalid apply must reject before disk lookup"); + assert_eq!(error.code(), tonic::Code::InvalidArgument); + } + } + + #[tokio::test] + async fn delete_recovery_requires_digest_and_valid_protocol_payload() { + let service = make_server(); + let operation = crate::storage::storage_api::DeleteRecoveryRpcOperation::Discover { + object: None, + cursor: None, + limit: 1, + }; + let message = RecoverDeleteTransactionRequest { + disk: "unknown".into(), + volume: "bucket".into(), + protocol_version: 1, + operation: rmp_serde::to_vec_named(&operation).expect("encode discovery").into(), + }; + let unsigned = service + .recover_delete_transaction(Request::new(message.clone())) + .await + .expect_err("new RPC must reject unsigned requests"); + assert_eq!(unsigned.code(), tonic::Code::PermissionDenied); + for digest in ["", "UNSIGNED-PAYLOAD"] { + let mut request = Request::new(message.clone()); + request + .metadata_mut() + .insert("x-rustfs-content-sha256", digest.parse().expect("metadata")); + mark_v2_authenticated(&mut request); + let error = service + .recover_delete_transaction(request) + .await + .expect_err("missing or unsigned digest"); + assert_eq!(error.code(), tonic::Code::PermissionDenied); + } + let mut invalid = message.clone(); + invalid.protocol_version = 2; + let mut malformed = message.clone(); + malformed.operation = vec![0xc1].into(); + let mut trailing = message.clone(); + let mut payload = trailing.operation.to_vec(); + payload.push(0xc0); + trailing.operation = payload.into(); + for (message, expected) in [ + (invalid, tonic::Code::Unimplemented), + (malformed, tonic::Code::InvalidArgument), + (trailing, tonic::Code::InvalidArgument), + ] { + let mut request = Request::new(message); + let body = + rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()).expect("canonical recovery"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body digest"); + mark_v2_authenticated(&mut request); + let error = service + .recover_delete_transaction(request) + .await + .expect_err("invalid recovery input"); + assert_eq!(error.code(), expected); + } + let mut request = Request::new(message); + let body = + rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()).expect("canonical recovery"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body digest"); + mark_v2_authenticated(&mut request); + let response = service + .recover_delete_transaction(request) + .await + .expect("valid request reaches disk lookup") + .into_inner(); + assert!(!response.success); + assert!(response.error.is_some()); + } + /// Per-handler wiring check for every mutating disk RPC. A mismatched digest must be rejected /// (catches a handler that omits its `verify_disk_mutation_digest` gate) and a correctly /// body-bound digest must pass the gate (catches a handler wired to the wrong @@ -4659,6 +4839,16 @@ mod tests { }, rustfs_protos::canonical_prepare_part_transaction_request_body ); + assert_gated!( + recover_delete_transaction, + RecoverDeleteTransactionRequest { + disk: disk.clone(), + volume: "dst".into(), + protocol_version: 1, + operation: vec![0x80].into() + }, + rustfs_protos::canonical_recover_delete_transaction_request_body + ); assert_gated!( settle_part_transaction, SettlePartTransactionRequest { @@ -6597,7 +6787,7 @@ mod tests { }; let rollback_dir = Uuid::new_v4(); remote - .delete_version( + .delete_version_recoverable( bucket, object, marker.clone(), diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index c7e73aecc..22c1a88c1 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -19,7 +19,10 @@ use crate::storage::storage_api::rpc_consumer::node_service::{ validate_batch_read_version_item_count, }; use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; -use crate::storage::storage_api::{PartTransactionAction, RenameDataResp, SnapshotLeaseToken, verify_tonic_mutation_body_digest}; +use crate::storage::storage_api::{ + DELETE_RECOVERY_PROTOCOL_VERSION, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, PartTransactionAction, RenameDataResp, + SnapshotLeaseToken, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, +}; use bytes::Bytes; use rustfs_filemeta::FileInfo; use rustfs_io_metrics::internode_metrics::{ @@ -118,6 +121,16 @@ const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512; const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024; const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1; +fn rejects_legacy_delete_transaction(volume: &str, opts: &DeleteOptions) -> bool { + !(volume == ".rustfs.sys" + || volume.starts_with(".rustfs.sys/") + || volume == ".minio.sys" + || volume.starts_with(".minio.sys/")) + && opts.old_data_dir.is_some() + && !opts.recoverable_delete + && !opts.undo_write +} + fn snapshot_lease_response(result: Result) -> Response { match result { Ok(token) => Response::new(SnapshotLeaseResponse { @@ -715,6 +728,14 @@ impl NodeService { } }; + if rejects_legacy_delete_transaction(&request.volume, &opts) { + return Ok(Response::new(DeleteVersionsResponse { + success: false, + errors: Vec::new(), + error: Some(DiskError::UnsupportedDisk.into()), + item_errors: Vec::new(), + })); + } let (errors, item_errors) = encode_delete_versions_errors(disk.delete_versions(&request.volume, versions, opts).await); @@ -782,6 +803,13 @@ impl NodeService { })); } }; + if rejects_legacy_delete_transaction(&request.volume, &opts) { + return Ok(Response::new(DeleteVersionResponse { + success: false, + raw_file_info: String::new(), + error: Some(DiskError::UnsupportedDisk.into()), + })); + } if require_marker_condition && opts.expected_delete_marker.is_none() { return Err(Status::invalid_argument("retired marker deletion requires a marker precondition")); } @@ -1578,6 +1606,131 @@ impl NodeService { } } + pub(super) async fn handle_recover_delete_transaction( + &self, + request: Request, + ) -> Result, Status> { + if request.get_ref().operation.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES { + return Err(Status::resource_exhausted("delete recovery operation exceeds payload limit")); + } + let body = rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref()) + .map_err(|_| Status::invalid_argument("delete recovery request length cannot be represented"))?; + // This additive protocol has no digestless rolling-upgrade exception. + verify_tonic_canonical_body_digest(&request, &body) + .map_err(|_| Status::permission_denied("delete recovery requires body-bound v2 authentication"))?; + let request = request.into_inner(); + if request.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION { + return Err(Status::unimplemented("incompatible delete recovery protocol")); + } + let operation: DeleteRecoveryRpcOperation = rustfs_protos::decode_delete_recovery_payload(&request.operation) + .map_err(|_| Status::invalid_argument("invalid delete recovery operation"))?; + if request.operation.len() > operation.max_wire_payload_bytes() { + return Err(Status::resource_exhausted("delete recovery operation exceeds payload limit")); + } + match &operation { + DeleteRecoveryRpcOperation::ApplyVersions { versions, .. } + if versions.len() > 1000 + || versions + .iter() + .try_fold(0_usize, |count, versions| count.checked_add(versions.versions.len())) + .is_none_or(|count| count > 1000) => + { + return Err(Status::invalid_argument("recoverable batch delete exceeds 1000 object/version requests")); + } + DeleteRecoveryRpcOperation::Discover { limit, .. } if *limit == 0 || *limit > 256 => { + return Err(Status::invalid_argument("delete recovery page limit must be between 1 and 256")); + } + DeleteRecoveryRpcOperation::Inspect { transaction, .. } | DeleteRecoveryRpcOperation::Settle { transaction, .. } + if transaction.is_nil() => + { + return Err(Status::invalid_argument("delete transaction must be a non-nil UUID")); + } + DeleteRecoveryRpcOperation::ApplyVersion { opts, .. } | DeleteRecoveryRpcOperation::ApplyVersions { opts, .. } + if opts.old_data_dir.is_none_or(|transaction| transaction.is_nil()) + || opts.undo_delete + || opts.undo_write + || opts.recursive + || opts.immediate + || opts.expected_delete_marker.is_some() + || opts + .delete_recovery_bucket_incarnation + .is_some_and(|incarnation| incarnation.is_nil()) => + { + return Err(Status::invalid_argument( + "recoverable delete requires a non-nil transaction and valid apply options", + )); + } + DeleteRecoveryRpcOperation::ApplyVersion { object, .. } if object.starts_with('/') => { + return Err(Status::invalid_argument("recoverable delete requires a relative object key")); + } + DeleteRecoveryRpcOperation::ApplyVersions { versions, .. } + if versions.iter().any(|version| version.name.starts_with('/')) => + { + return Err(Status::invalid_argument("recoverable delete requires relative object keys")); + } + _ => {} + } + let result = if let Some(disk) = self.find_disk(&request.disk).await { + match operation { + DeleteRecoveryRpcOperation::ApplyVersion { + object, + file_info, + force_del_marker, + opts, + } => disk + .delete_version_recoverable(&request.volume, &object, *file_info, force_del_marker, opts) + .await + .map(|()| DeleteRecoveryRpcReply::AppliedVersion), + DeleteRecoveryRpcOperation::ApplyVersions { versions, opts } => { + let errors = disk.delete_versions_recoverable(&request.volume, versions, opts).await; + Ok(DeleteRecoveryRpcReply::AppliedVersions( + errors.into_iter().map(|error| error.map(|error| error.to_u32())).collect(), + )) + } + DeleteRecoveryRpcOperation::Discover { object, cursor, limit } => disk + .discover_delete_transactions(&request.volume, object.as_deref(), cursor, limit) + .await + .map(DeleteRecoveryRpcReply::Discovered), + DeleteRecoveryRpcOperation::Inspect { object, transaction } => disk + .inspect_delete_transaction(&request.volume, &object, transaction) + .await + .map(DeleteRecoveryRpcReply::Inspected), + DeleteRecoveryRpcOperation::Settle { + object, + transaction, + expected_fingerprint, + action, + } => disk + .settle_delete_transaction(&request.volume, &object, transaction, expected_fingerprint, action) + .await + .map(DeleteRecoveryRpcReply::Settled), + } + } else { + Err(DiskError::DiskNotFound) + }; + match result { + Ok(reply) => { + let result = encode_msgpack_named(&reply, "delete recovery reply") + .map_err(|_| Status::internal("cannot encode delete recovery reply"))?; + if result.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES { + return Err(Status::resource_exhausted("delete recovery reply exceeds payload limit")); + } + Ok(Response::new(RecoverDeleteTransactionResponse { + success: true, + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + result: result.into(), + error: None, + })) + } + Err(error) => Ok(Response::new(RecoverDeleteTransactionResponse { + success: false, + protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION, + result: Bytes::new(), + error: Some(error.into()), + })), + } + } + pub(super) async fn handle_settle_part_transaction( &self, request: Request, @@ -2007,9 +2160,10 @@ impl NodeService { #[cfg(test)] mod tests { use super::{ - compat_response_json, decode_msgpack_or_json, decode_rename_data_request_file_info, + DeleteOptions, compat_response_json, decode_msgpack_or_json, decode_rename_data_request_file_info, encode_batch_read_version_response_payloads, encode_delete_versions_errors, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named, encode_read_multiple_response_payloads, encode_rename_data_response_payloads, + rejects_legacy_delete_transaction, }; use crate::storage::DiskError; use crate::storage::rpc::node_service::make_server; @@ -2022,6 +2176,7 @@ mod tests { use serde::{Deserialize, Serialize}; use serial_test::serial; use tonic::Request; + use uuid::Uuid; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct SamplePayload { @@ -2029,6 +2184,26 @@ mod tests { count: u32, } + #[test] + fn legacy_delete_transaction_policy_preserves_rollback_and_internal_maintenance() { + let mut opts = DeleteOptions { + old_data_dir: Some(Uuid::new_v4()), + ..Default::default() + }; + assert!(rejects_legacy_delete_transaction("bucket", &opts)); + assert!(!rejects_legacy_delete_transaction(".rustfs.sys", &opts)); + assert!(!rejects_legacy_delete_transaction(".rustfs.sys/tmp", &opts)); + assert!(!rejects_legacy_delete_transaction(".minio.sys", &opts)); + opts.undo_write = true; + assert!(!rejects_legacy_delete_transaction("bucket", &opts)); + opts.undo_write = false; + opts.recoverable_delete = true; + assert!(!rejects_legacy_delete_transaction("bucket", &opts)); + opts.recoverable_delete = false; + opts.old_data_dir = None; + assert!(!rejects_legacy_delete_transaction("bucket", &opts)); + } + #[test] fn response_compat_send_sites_keep_manifest_json_encoders() { // Rolling-upgrade contract (rustfs-protos compat manifest): every diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index adc83492a..3edbe7d4d 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -478,11 +478,12 @@ pub(crate) mod ecstore_data_usage { #[allow(unused_imports)] pub(crate) mod ecstore_disk { pub(crate) use rustfs_ecstore::api::disk::{ - BUCKET_META_PREFIX, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DeleteOptions, - DiskAPI, DiskInfo, DiskInfoOptions, DiskStore, FileInfoVersions, FileReader, FileWriter, OldCurrentSize, - PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, - SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout, - validate_batch_read_version_item_count, + BUCKET_META_PREFIX, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, + DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, + DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo, + DiskInfoOptions, DiskStore, FileInfoVersions, FileReader, FileWriter, OldCurrentSize, PartTransactionAction, + RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, + UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout, validate_batch_read_version_item_count, }; #[cfg(test)] pub(crate) use rustfs_ecstore::api::disk::{DiskOption, new_disk}; @@ -734,6 +735,14 @@ pub(crate) type ReadMultipleReq = ecstore_disk::ReadMultipleReq; pub(crate) type ReadMultipleResp = ecstore_disk::ReadMultipleResp; pub(crate) type ReadOptions = ecstore_disk::ReadOptions; pub(crate) type OldCurrentSize = ecstore_disk::OldCurrentSize; +pub(crate) type DeleteRecoveryCursor = ecstore_disk::DeleteRecoveryCursor; +pub(crate) type DeleteRecoveryPage = ecstore_disk::DeleteRecoveryPage; +pub(crate) type DeleteRecoveryInspection = ecstore_disk::DeleteRecoveryInspection; +pub(crate) type DeleteRecoveryDigest = ecstore_disk::DeleteRecoveryDigest; +pub(crate) type DeleteRecoveryAction = ecstore_disk::DeleteRecoveryAction; +pub(crate) type DeleteRecoveryRpcOperation = ecstore_disk::DeleteRecoveryRpcOperation; +pub(crate) type DeleteRecoveryRpcReply = ecstore_disk::DeleteRecoveryRpcReply; +pub(crate) const DELETE_RECOVERY_PROTOCOL_VERSION: u32 = ecstore_disk::DELETE_RECOVERY_PROTOCOL_VERSION; pub(crate) type PartTransactionAction = ecstore_disk::PartTransactionAction; pub(crate) type RenameDataResp = ecstore_disk::RenameDataResp; pub(crate) type ReplicationStatusType = ecstore_bucket::replication::ReplicationStatusType; @@ -1444,6 +1453,41 @@ pub(crate) trait StorageDiskRpcExt { dst_path: &str, meta: bytes::Bytes, ) -> DiskResult<()>; + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: rustfs_filemeta::FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> DiskResult<()>; + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + ) -> Vec>; + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> DiskResult; + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + ) -> DiskResult; + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + ) -> DiskResult; async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> DiskResult<()>; async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()>; async fn verify_file(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult; @@ -1616,6 +1660,53 @@ where ecstore_disk::DiskAPI::prepare_part_transaction(self, src_volume, src_path, dst_volume, dst_path, meta).await } + async fn delete_version_recoverable( + &self, + volume: &str, + object: &str, + file_info: rustfs_filemeta::FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> DiskResult<()> { + ecstore_disk::DiskAPI::delete_version_recoverable(self, volume, object, file_info, force_del_marker, opts).await + } + async fn delete_versions_recoverable( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + ) -> Vec> { + ecstore_disk::DiskAPI::delete_versions_recoverable(self, volume, versions, opts).await + } + + async fn discover_delete_transactions( + &self, + volume: &str, + object: Option<&str>, + cursor: Option, + limit: u32, + ) -> DiskResult { + ecstore_disk::DiskAPI::discover_delete_transactions(self, volume, object, cursor, limit).await + } + async fn inspect_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + ) -> DiskResult { + ecstore_disk::DiskAPI::inspect_delete_transaction(self, volume, object, transaction).await + } + async fn settle_delete_transaction( + &self, + volume: &str, + object: &str, + transaction: uuid::Uuid, + expected_fingerprint: DeleteRecoveryDigest, + action: DeleteRecoveryAction, + ) -> DiskResult { + ecstore_disk::DiskAPI::settle_delete_transaction(self, volume, object, transaction, expected_fingerprint, action).await + } + async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> DiskResult<()> { ecstore_disk::DiskAPI::settle_part_transaction(self, volume, path, action).await } diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index 2e51ba37e..ecc668e0d 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -180,6 +180,8 @@ pub(crate) mod server { } pub(crate) mod http { + #[cfg(test)] + pub(crate) use crate::storage::storage_api::ecstore_disk::{DiskAPI, DiskOption, new_disk}; pub(crate) use crate::storage::storage_api::{ ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_tonic_rpc_signature_with_bootstrap, diff --git a/scripts/test/delete_recovery_docker.py b/scripts/test/delete_recovery_docker.py new file mode 100644 index 000000000..c9bb96b32 --- /dev/null +++ b/scripts/test/delete_recovery_docker.py @@ -0,0 +1,1257 @@ +#!/usr/bin/env python3 +"""Run delete recovery regressions in disposable local Docker containers. + +Requires host Python 3.9+, Docker, and curl with --aws-sigv4. Supply a native +Linux binary compiled with e2e-test-hooks and a Linux image containing Python 3 +and that binary's runtime libraries. All S3 requests target localhost with +synthetic credentials. Container processes run as UID 10001. The harness labels +its resources, refuses existing volumes, preserves JSON reports/server logs, +and removes only containers, networks, and volumes it created. + +Example: + python3 scripts/test/delete_recovery_docker.py --binary /absolute/path/to/rustfs-candidate --baseline-binary /absolute/path/to/rustfs-old --image local-rustfs-validation:latest --artifact-dir /absolute/path/to/new-results smoke prepared1 applied1 decision1 restore returning untouched mixed + +Available cases include single/multi-node process termination, inline/empty +objects, legacy explicit cleanup, versioned and retained objects, UUID prefix +children, corrupted journals, real permission errors, lost receipt responses, +admin bucket/object repair, and deep keys. Use --help for runtime options. +Logical disks share the Docker host filesystem; this tests process/RPC failure +and namespace safety, not independent hardware failure or power-loss durability. +""" +from pathlib import Path +import base64, hashlib, ipaddress, json, os, subprocess, tempfile, threading, time, uuid, xml.etree.ElementTree as ET +from urllib.parse import quote +ROOT = None +PREFIX = None +IMAGE = None +PORT = 19010 +CANDIDATE = None +BASELINE = None +ACCESS = 'recovery-test' +SECRET = 'recovery-test-secret' + +def run(*args, check=True, **kwargs): + return subprocess.run(args, check=check, capture_output=True, **kwargs) + +def docker(*args, check=True): + return run('docker', *args, check=check) + +class S3: + + def __init__(self, port): + self.url = f'http://127.0.0.1:{port}' + + def request(self, method, path='', body=None, headers=None, expected=None, timeout=60): + call = ['curl', '--noproxy', '*', '--silent', '--show-error', '--max-time', str(timeout), '--aws-sigv4', 'aws:amz:us-east-1:s3', '--user', ACCESS + ':' + SECRET, '--request', method] + for (key, value) in (headers or {}).items(): + call += ['--header', key + ': ' + value] + if body is not None: + call += ['--data-binary', '@-'] + with tempfile.NamedTemporaryFile(dir=ROOT) as header_file: + response = run(*call, self.url + '/' + path, '--dump-header', header_file.name, '--write-out', '\n%{http_code}', input=body, check=False) + self.last_headers = {} + for line in Path(header_file.name).read_text(errors='replace').splitlines(): + if ':' in line: + (key, value) = line.split(':', 1) + self.last_headers[key.lower()] = value.strip() + if response.returncode: + raise RuntimeError(response.stderr.decode(errors='replace')) + (payload, status) = response.stdout.rsplit(b'\n', 1) + code = int(status) + if expected is not None and code not in ((expected,) if isinstance(expected, int) else expected): + raise AssertionError(f'{method} {path}: {code}, expected {expected}: {payload[:1500]!r}') + return (code, payload) + + def put_bucket(self, bucket, lock=False): + headers = {'x-amz-bucket-object-lock-enabled': 'true'} if lock else {} + return self.request('PUT', bucket, headers=headers, expected=200) + + def put(self, bucket, key, body, headers=None): + return self.request('PUT', bucket + '/' + key, body, headers, 200) + + def get(self, bucket, key): + return self.request('GET', bucket + '/' + key, expected=200)[1] + + def delete(self, bucket, key, expected=(204, 200)): + return self.request('DELETE', bucket + '/' + key, expected=expected) + + def delete_bucket(self, bucket, expected=204): + return self.request('DELETE', bucket, expected=expected) + + def list(self, bucket): + (_, body) = self.request('GET', bucket + '?list-type=2', expected=200) + root = ET.fromstring(body) + return [node.text for node in root.iter() if node.tag.rsplit('}', 1)[-1] == 'Key'] + + def versioning(self, bucket, state='Enabled'): + xml = f'{state}'.encode() + return self.request('PUT', bucket + '?versioning', xml, {'Content-Type': 'application/xml'}, 200) + + def batch_delete(self, bucket, keys): + xml = ET.Element('Delete') + for key in keys: + ET.SubElement(ET.SubElement(xml, 'Object'), 'Key').text = key + data = ET.tostring(xml) + (status, body) = self.request('POST', bucket + '?delete', data, {'Content-Type': 'application/xml', 'Content-MD5': base64.b64encode(hashlib.md5(data).digest()).decode()}, 200) + response = ET.fromstring(body) + errors = [node for node in response.iter() if node.tag.rsplit('}', 1)[-1] == 'Error'] + assert not errors, f'DeleteObjects returned per-key errors: {body!r}' + return (status, body) + + def complete_multipart(self, bucket, key, parts): + path = bucket + '/' + key + (_, response) = self.request('POST', path + '?uploads', expected=200) + upload = next(node.text for node in ET.fromstring(response).iter() if node.tag.rsplit('}', 1)[-1] == 'UploadId') + assert upload, 'native multipart initiation must return an upload ID' + query = '?uploadId=' + quote(upload, safe='') + complete = ET.Element('CompleteMultipartUpload') + for number, body in enumerate(parts, 1): + self.request('PUT', path + query + '&partNumber=' + str(number), body, expected=200) + etag = self.last_headers.get('etag') + assert etag, 'native UploadPart must return its ETag' + part = ET.SubElement(complete, 'Part') + ET.SubElement(part, 'PartNumber').text = str(number) + ET.SubElement(part, 'ETag').text = etag + (_, response) = self.request('POST', path + query, ET.tostring(complete), {'Content-Type': 'application/xml'}, 200) + assert ET.fromstring(response).tag.rsplit('}', 1)[-1] == 'CompleteMultipartUploadResult', response + return upload + +class Cluster: + + def __init__(self, case, nodes=1, drives=4, extra_env=None, binary_by_node=None): + self.case = case + self.nodes = nodes + self.drives = drives + self.port = PORT + self.binary = 'candidate' + self.binary_by_node = binary_by_node or {} + self.names = [f'{PREFIX}-{case}-n{i}' for i in range(1, nodes + 1)] + self.network = f'{PREFIX}-{case}-net' + self.env = extra_env or {} + self.created_names = [] + self.created_network = False + self.clients = [S3(self.port + i) for i in range(nodes)] + self.volumes = [] + self.run_args = {} + self.barriers = [ROOT / 'barriers' / case / f'n{i}' for i in range(1, nodes + 1)] + for directory in self.barriers: + directory.mkdir(parents=True, exist_ok=True) + directory.chmod(511) + + def create(self): + docker('network', 'create', '--label', 'rustfs.delete-recovery-test=' + PREFIX, self.network) + self.created_network = True + configuration = json.loads(docker('network', 'inspect', self.network).stdout)[0]['IPAM']['Config'] + subnet = next(ipaddress.ip_network(value['Subnet']) for value in configuration if ':' not in value['Subnet']) + addresses = [str(subnet[index + 2]) for index in range(self.nodes)] + urls = [] + for name in self.names: + for drive in range(1, self.drives + 1): + urls.append(f'http://{name}:9000/data/d{drive}') + for (idx, name) in enumerate(self.names): + args = ['run', '-d', '--user', '10001:10001', '--name', name, '--network', self.network, '--hostname', name, '-p', f'127.0.0.1:{self.port + idx}:9000', '--label', 'rustfs.delete-recovery-test=' + PREFIX, '-v', str(CANDIDATE) + ':/binaries/candidate:ro', '-v', str(self.barriers[idx]) + ':/barrier'] + # Stopped peers must still resolve while the remaining quorum restarts. + args += ['--ip', addresses[idx]] + for peer, address in zip(self.names, addresses): + args += ['--add-host', peer + ':' + address] + if BASELINE is not None: + args += ['-v', str(BASELINE) + ':/binaries/baseline:ro'] + data_mounts = [] + for drive in range(1, self.drives + 1): + volume = f'{name}-d{drive}' + if docker('volume', 'inspect', volume, check=False).returncode == 0: + raise RuntimeError('refusing existing volume: ' + volume) + docker('volume', 'create', '--label', 'rustfs.delete-recovery-test=' + PREFIX, volume) + self.volumes.append(volume) + args += ['-v', f'{volume}:/data/d{drive}'] + data_mounts += ['-v', f'{volume}:/data/d{drive}'] + docker('run', '--rm', *data_mounts, IMAGE, 'python3', '-c', "from pathlib import Path; import os; [os.chown(p,10001,10001) for p in Path('/data').glob('d*')]") + values = {'RUSTFS_VOLUMES': ' '.join(urls) if self.nodes > 1 else ' '.join((f'/data/d{d}' for d in range(1, self.drives + 1))), 'RUSTFS_ADDRESS': '0.0.0.0:9000', 'RUSTFS_CONSOLE_ENABLE': 'false', 'RUSTFS_ACCESS_KEY': ACCESS, 'RUSTFS_SECRET_KEY': SECRET, 'RUSTFS_OBS_LOG_DIRECTORY': '/tmp/logs', 'RUSTFS_OBS_LOGGER_LEVEL': 'info', 'RUSTFS_OBS_LOG_STDOUT_ENABLED': 'true', 'RUSTFS_DELETE_RECOVERY_BARRIER_DIR': '/barrier', 'RUSTFS_UNSAFE_BYPASS_DISK_CHECK': 'true', 'RUSTFS_DURABILITY_MODE': 'strict', **self.env} + for (key, value) in values.items(): + if value is not None: + args += ['-e', key + '=' + value] + self.run_args[idx] = [*args, IMAGE, '/binaries/' + self.binary_by_node.get(idx, self.binary)] + launch = docker(*self.run_args[idx], check=False) + if launch.returncode: + label = docker('inspect', '--format', '{{index .Config.Labels "rustfs.delete-recovery-test"}}', name, check=False) + if label.returncode == 0 and label.stdout.decode().strip() == PREFIX: + self.created_names.append(name) + raise RuntimeError('Docker launch failed: ' + launch.stderr.decode(errors='replace')) + self.created_names.append(name) + self.ready() + + def ready(self, timeout=180, indices=None): + deadline = time.monotonic() + timeout + pending = set(range(self.nodes) if indices is None else indices) + while pending and time.monotonic() < deadline: + for idx in list(pending): + try: + (code, _) = self.clients[idx].request('GET', '', timeout=2) + if code in (200, 403): + pending.remove(idx) + except Exception: + pass + if pending: + for idx in pending: + state = docker('inspect', '--format', '{{.State.Running}}', self.names[idx], check=False) + if state.returncode == 0 and state.stdout.strip() != b'true': + self.logs() + raise RuntimeError('server exited during startup: ' + self.names[idx]) + time.sleep(0.2) + if pending: + self.logs() + raise TimeoutError('cluster not ready: ' + str(pending)) + + def kill(self, index=0): + docker('kill', '--signal', 'KILL', self.names[index]) + + def restart(self, index=None): + for i in range(self.nodes) if index is None else [index]: + docker('start', self.names[i]) + self.ready() + + def replace_node(self, index, binary): + docker('kill', '--signal', 'KILL', self.names[index], check=False) + docker('rm', self.names[index]) + args = self.run_args[index].copy() + args[-1] = '/binaries/' + binary + docker(*args) + self.run_args[index] = args + self.ready(indices=[index]) + + def logs(self): + for name in self.created_names: + response = docker('logs', name, check=False) + (ROOT / (name + '.log')).write_bytes(response.stdout + response.stderr) + + def inspect_data(self, code, indices=None): + """Access only this cluster's named volumes, including stopped peers.""" + result = [] + for index, name in enumerate(self.names): + if indices is not None and index not in indices: + continue + response = docker('run', '--rm', '--volumes-from', name, IMAGE, 'python3', '-c', code) + result.append(json.loads(response.stdout)) + return result + + def remove(self): + """Keep logs and delete only resources successfully created here.""" + self.logs() + for name in self.created_names: + docker('rm', '-f', name, check=False) + for volume in self.volumes: + docker('volume', 'rm', volume, check=False) + if self.created_network: + docker('network', 'rm', self.network, check=False) + +def smoke(): + cluster = Cluster('smoke', drives=1) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'recovery-smoke' + client.put_bucket(bucket) + for (key, size) in [('zero', 0), ('inline', 4096), ('external', 2 * 1024 * 1024)]: + payload = bytes(range(256)) * (size // 256) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + client.batch_delete(bucket, ['zero', 'inline', 'external']) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': 's3-smoke', 'status': 'passed', 'topology': '1 node / 1 named local volume'} + finally: + cluster.remove() + +def seed_legacy(cluster, bucket, key, shape): + transaction = str(uuid.uuid4()) + if shape == 'alias': + directories = {path.split('/')[0] for node in residue_inventory(cluster, bucket, key) for drive in node for path in drive['files'] if path.endswith('/part.1')} + assert len(directories) == 1, 'alias fixture requires one shared data-directory generation' + transaction = directories.pop() + code = "from pathlib import Path\nimport hashlib,json,os,uuid\nbucket=%r; key=%r; shape=%r; transaction=%r\nresult=[]\nfor drive in sorted(Path('/data').glob('d*')):\n root=drive/bucket/key\n original=root/'xl.meta'\n if not original.is_file(): continue\n data=original.read_bytes(); backup=root/transaction/'xl.meta.bkp'\n backup.parent.mkdir(exist_ok=shape=='alias'); backup.write_bytes(data); os.chown(backup.parent,10001,10001); os.chown(backup,10001,10001)\n dirs=[p for p in root.iterdir() if p.is_dir() and (p.name!=transaction or shape=='alias') and any(p.glob('part.*'))]\n if shape in ('reserved','alias'):\n assert dirs, 'external fixture must contain erasure parts'\n for d in dirs:\n marker=d/('reserve-delete-data.'+transaction); marker.write_bytes(b''); os.chown(marker,10001,10001)\n elif shape=='backup-only':\n assert not dirs, 'inline fixture must not contain external parts'\n else: raise AssertionError(shape)\n original.unlink()\n owned=[root,*root.rglob('*')]\n assert all(p.stat().st_uid==10001 and p.stat().st_gid==10001 for p in owned), [(str(p),p.stat().st_uid,p.stat().st_gid) for p in owned]\n result.append({'drive':drive.name,'transaction':transaction,'backup_sha256':hashlib.sha256(data).hexdigest(),'data_dirs':[p.name for p in dirs],'fixture_owner':'10001:10001','fixture_modes':{str(p.relative_to(root)):oct(p.stat().st_mode & 0o777) for p in owned}})\nprint(json.dumps(result))\n" % (bucket, key, shape, transaction) + return cluster.inspect_data(code) + +def residue_inventory(cluster, bucket, key): + return cluster.inspect_data("from pathlib import Path\nimport hashlib,json\nresult=[]\nfor drive in sorted(Path('/data').glob('d*')):\n root=drive/%r/%r\n result.append({'drive':drive.name,'exists':root.exists(),'files':{str(p.relative_to(root)):hashlib.sha256(p.read_bytes()).hexdigest() for p in root.rglob('*') if p.is_file()}})\nprint(json.dumps(result))\n" % (bucket, key)) + +def legacy_retry(nodes=1): + case = 'legacy' + str(nodes) + cluster = Cluster(case, nodes=nodes, drives=1 if nodes == 1 else 2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'legacy-recovery' + client.put_bucket(bucket) + original = {key: bytes(range(256)) * (size // 256) for (key, size) in [('inline', 4096), ('reserved', 2 * 1024 * 1024)]} + for (key, payload) in original.items(): + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + for i in range(nodes): + cluster.kill(i) + fixtures = {key: seed_legacy(cluster, bucket, key, 'backup-only' if key == 'inline' else 'reserved') for key in original} + before = {key: residue_inventory(cluster, bucket, key) for key in original} + cluster.restart() + time.sleep(2) + after = {key: residue_inventory(cluster, bucket, key) for key in original} + assert before == after, 'automatic recovery must not guess an unjournaled historical outcome' + client.batch_delete(bucket, list(original)) + for key in original: + current = residue_inventory(cluster, bucket, key) + assert all((not drive['exists'] for node in current for drive in node)), f'current explicit DELETE must reclaim {key}: {current}' + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': case, 'status': 'passed', 'fixtures': fixtures, 'topology': f'{nodes} nodes'} + finally: + cluster.remove() + +def arm_barrier(cluster, index, bucket, key, stage, disk=None): + nonce = str(uuid.uuid4()) + arm = {'nonce': nonce, 'bucket': bucket, 'object': key, 'stage': stage} + if disk is not None: + arm['disk'] = disk + (cluster.barriers[index] / 'arm.json').write_text(json.dumps(arm)) + return nonce + +def reached_barrier(cluster, index, nonce, timeout=45): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + for path in cluster.barriers[index].glob('reached.' + nonce + '.*.json'): + try: + return json.loads(path.read_text()) + except (json.JSONDecodeError, OSError): + pass + time.sleep(0.02) + cluster.logs() + raise TimeoutError('crash stage not reached: ' + nonce) + +def clear_barriers(cluster): + for directory in cluster.barriers: + (directory / 'arm.json').unlink(missing_ok=True) + +def async_delete(client, bucket, key): + result = {} + + def target(): + try: + result['response'] = client.request('DELETE', bucket + '/' + key, timeout=90) + except Exception as error: + result['exception'] = str(error) + thread = threading.Thread(target=target, daemon=True) + thread.start() + return (thread, result) + +def wait_inventory(cluster, bucket, key, predicate, timeout=60): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = residue_inventory(cluster, bucket, key) + if predicate(last): + return last + time.sleep(0.1) + raise AssertionError('recovery did not converge: ' + json.dumps(last)) + +def process_crash(stage, nodes=1, size=2 * 1024 * 1024): + case = ('inline-' if size == 4096 else 'zero-' if size == 0 else 'kill-') + stage.lower() + '-' + str(nodes) + cluster = Cluster(case, nodes=nodes, drives=4 if nodes == 1 else 2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'journal-crash' + key = 'payload' + client.put_bucket(bucket) + payload = bytes(range(256)) * (size // 256) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + original = residue_inventory(cluster, bucket, key) + nonce = arm_barrier(cluster, 0, bucket, key, stage, disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + for index in range(nodes): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + cluster.restart() + if stage == 'DECISION': + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node))) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + else: + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + assert client.get(bucket, key) == payload + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': case, 'status': 'passed', 'observation': observation, 'request': {key: str(value) for (key, value) in request.items()}} + finally: + cluster.remove() + +def rollback_crash(): + cluster = Cluster('kill-restore', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'rollback-restart' + key = 'payload' + client.put_bucket(bucket) + payload = bytes(range(256)) * 8192 + client.put(bucket, key, payload) + original = residue_inventory(cluster, bucket, key) + first_nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + first = reached_barrier(cluster, 0, first_nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + second_nonce = arm_barrier(cluster, 0, bucket, key, 'RESTORE', disk='/data/d1') + docker('start', cluster.names[0]) + second = reached_barrier(cluster, 0, second_nonce) + cluster.kill() + clear_barriers(cluster) + cluster.restart() + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + assert client.get(bucket, key) == payload + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'rollback-second-process-kill', 'status': 'passed', 'observations': [first, second]} + finally: + cluster.remove() + +def versioned_crash(stage): + cluster = Cluster('versions-' + stage.lower(), drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'versioned-recovery' + key = 'history' + client.put_bucket(bucket) + client.versioning(bucket) + first = b'old-version-' * 200000 + second = b'new-version-' * 300 + client.put(bucket, key, first) + v1 = client.last_headers.get('x-amz-version-id') + assert v1 and v1 != 'null' + client.put(bucket, key, second) + v2 = client.last_headers.get('x-amz-version-id') + assert v2 and v2 != v1 + assert client.get(bucket, key + '?versionId=' + v1) == first + assert client.get(bucket, key) == second + original = residue_inventory(cluster, bucket, key) + nonce = arm_barrier(cluster, 0, bucket, key, stage, disk='/data/d1') + (thread, request) = async_delete(client, bucket, key + '?versionId=' + v1) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + cluster.restart() + if stage == 'APPLIED': + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + assert client.get(bucket, key + '?versionId=' + v1) == first + client.delete(bucket, key + '?versionId=' + v1) + else: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + (code, _) = client.request('GET', bucket + '/' + key + '?versionId=' + v1) + if code == 404: + break + time.sleep(0.1) + else: + raise AssertionError('committed version deletion was resurrected') + assert client.get(bucket, key) == second + assert client.get(bucket, key + '?versionId=' + v2) == second + client.delete(bucket, key + '?versionId=' + v2) + client.delete_bucket(bucket) + return {'case': 'versioned-' + stage.lower(), 'status': 'passed', 'observation': observation, 'deleted_version': v1, 'retained_version': v2} + finally: + cluster.remove() + +def protected_version(): + import datetime + cluster = Cluster('retention', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'protected-recovery' + key = 'retained' + client.put_bucket(bucket, lock=True) + client.versioning(bucket) + until = (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%SZ') + payload = b'compliance-and-legal-hold' * 1024 + client.put(bucket, key, payload, {'x-amz-object-lock-mode': 'COMPLIANCE', 'x-amz-object-lock-retain-until-date': until, 'x-amz-object-lock-legal-hold': 'ON'}) + version = client.last_headers.get('x-amz-version-id') + assert version and version != 'null' + original = residue_inventory(cluster, bucket, key) + client.delete(bucket, key + '?versionId=' + version, expected=403) + assert residue_inventory(cluster, bucket, key) == original + cluster.kill() + cluster.restart() + assert client.get(bucket, key + '?versionId=' + version) == payload + assert residue_inventory(cluster, bucket, key) == original + client.delete_bucket(bucket, expected=409) + return {'case': 'compliance-legal-hold-restart', 'status': 'passed', 'retained_version': version} + finally: + cluster.remove() + +def returning_drive_new_generation(): + cluster = Cluster('returning-new-generation', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'returning-drive' + key = 'payload' + client.put_bucket(bucket) + old = b'old-payload-' * 200000 + new = b'new-payload-' * 180000 + client.put(bucket, key, old) + assert client.get(bucket, key) == old + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + for index in range(4): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + for index in range(3): + docker('start', cluster.names[index]) + cluster.ready(indices=range(3)) + client.put(bucket, key, new) + assert client.get(bucket, key) == new + client.delete(bucket, key) + docker('start', cluster.names[3]) + cluster.ready() + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node)), timeout=90) + client.request('GET', bucket + '/' + key, expected=404) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': 'old-prepared-new-put-new-delete-returning-peer', 'status': 'passed', 'observation': observation} + finally: + cluster.remove() + +def mixed_versions(): + cluster = Cluster('mixed', nodes=4, drives=2, binary_by_node={3: 'baseline'}) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'mixed-recovery' + key = 'payload' + payload = b'mixed-version-data' * 100000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + original = residue_inventory(cluster, bucket, key) + results = [] + for initiator in (0, 3): + (code, body) = cluster.clients[initiator].request('DELETE', bucket + '/' + key) + assert code >= 400, f'mixed initiator{initiator} must fail closed: {code} {body!r}' + after = residue_inventory(cluster, bucket, key) + (ROOT / f'mixed-initiator-{initiator}-inventory.json').write_text(json.dumps({'before': original, 'after': after}, indent=2)) + assert after == original, 'mixed rejection must preserve exact metadata/shards after rollback' + assert client.get(bucket, key) == payload + results.append({'initiator': initiator, 'delete_status': code}) + cluster.replace_node(3, 'candidate') + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'three-current-one-pre-extension-peer-both-initiators', 'status': 'passed', 'results': results} + finally: + cluster.remove() + +def permission_fault(): + cluster = Cluster('permission-error', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'permission-fault' + key = 'payload' + payload = b'permission-test' * 150000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + transaction = str(uuid.UUID(observation['transaction'])) + code = "from pathlib import Path; import json,os; p=Path('/data/d1')/%r/%r/%r; os.chmod(p,0o555); print(json.dumps({'path':str(p),'mode':oct(p.stat().st_mode&0o777)}))" % (bucket, key, transaction) + fault = cluster.inspect_data(code) + (cluster.barriers[0] / ('release.' + nonce)).write_bytes(b'') + thread.join(30) + assert not thread.is_alive(), 'permission failure must return rather than hang' + clear_barriers(cluster) + current = residue_inventory(cluster, bucket, key) + assert any((drive['exists'] for node in current for drive in node)) + client.delete_bucket(bucket, expected=409) + repair = "from pathlib import Path; import json,os; p=Path('/data/d1')/%r/%r/%r; os.chmod(p,0o755); print(json.dumps({'repaired':True}))" % (bucket, key, transaction) + cluster.inspect_data(repair) + cluster.kill() + cluster.restart() + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node))) + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket) + return {'case': 'actual-eacces-decision-publication', 'status': 'passed', 'fault': fault, 'request': {key: str(value) for (key, value) in request.items()}, 'observation': observation} + finally: + cluster.remove() + +def injected_error(stage): + cluster = Cluster('injected-' + stage.lower(), drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'injected-fault' + key = 'payload' + payload = b'injected-error' * 170000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + nonce = arm_barrier(cluster, 0, bucket, key, stage, disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + (cluster.barriers[0] / ('release.' + nonce)).write_bytes(b'io_error') + thread.join(30) + assert not thread.is_alive(), 'injected error must finish the caller' + clear_barriers(cluster) + cluster.kill() + cluster.restart() + if stage == 'PREPARED': + (code, body) = client.request('GET', bucket + '/' + key) + if code == 200: + assert body == payload + client.delete(bucket, key) + else: + assert code == 404 + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node))) + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket) + return {'case': 'post-persist-' + stage.lower() + '-io-error', 'status': 'passed', 'observation': observation, 'request': {key: str(value) for (key, value) in request.items()}} + finally: + cluster.remove() + +def missing_version_returning_peer(): + cluster = Cluster('returning-specific-version', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'missing-version-recovery' + key = 'history' + client.put_bucket(bucket) + client.versioning(bucket) + old = b'first-version' * 160000 + newer = b'latest-version' * 1000 + client.put(bucket, key, old) + v1 = client.last_headers['x-amz-version-id'] + client.put(bucket, key, newer) + v2 = client.last_headers['x-amz-version-id'] + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key + '?versionId=' + v1) + observation = reached_barrier(cluster, 0, nonce) + for index in range(4): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + for index in range(3): + docker('start', cluster.names[index]) + cluster.ready(indices=range(3)) + (code, body) = client.request('DELETE', bucket + '/' + key + '?versionId=' + v1) + initial_delete_status = code + assert code >= 400, f'missing version with unresolved offline journal must not acknowledge204: {code} {body!r}' + docker('start', cluster.names[3]) + cluster.ready() + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + (code, body) = client.request('GET', bucket + '/' + key + '?versionId=' + v1) + if code == 200: + assert body == old + break + time.sleep(0.2) + else: + raise AssertionError('undecided historical version did not recover after peer returned') + client.delete(bucket, key + '?versionId=' + v1) + client.request('GET', bucket + '/' + key + '?versionId=' + v1, expected=404) + assert client.get(bucket, key) == newer + client.delete(bucket, key + '?versionId=' + v2) + client.delete_bucket(bucket) + return {'case': 'missing-specific-version-offline-fence', 'status': 'passed', 'observation': observation, 'initial_delete_status': initial_delete_status} + finally: + cluster.remove() + +def untouched_returning_peer(): + cluster = Cluster('untouched-returning', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'untouched-peer' + key = 'payload' + payload = b'untouched-original' * 150000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + original = residue_inventory(cluster, bucket, key) + cluster.kill(3) + client.delete(bucket, key) + after = residue_inventory(cluster, bucket, key) + assert after[3] == original[3], 'offline unprepared peer must remain byte-for-byte untouched' + client.delete_bucket(bucket, expected=409) + cluster.restart(3) + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node)), timeout=90) + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket) + return {'case': 'acknowledged-delete-untouched-offline-peer-return', 'status': 'passed'} + finally: + cluster.remove() + +def uuid_prefix_crash(): + cluster = Cluster('uuid-prefix', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'uuid-prefix' + key = 'parent' + payload = b'parent-payload' * 160000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + inventory = residue_inventory(cluster, bucket, key) + part = next((name for name in inventory[0][0]['files'] if name.endswith('/part.1'))) + data_dir = str(uuid.UUID(part.split('/')[0])) + child = key + '/' + data_dir + '/child' + child_payload = b'preserve-live-prefix-child' * 800 + client.put(bucket, child, child_payload) + assert client.get(bucket, child) == child_payload + nonce = arm_barrier(cluster, 0, bucket, key, 'DECISION', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + cluster.restart() + assert client.get(bucket, child) == child_payload, 'recovery must preserve actual data-dir UUID-prefix descendants' + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket, expected=409) + client.delete(bucket, child) + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'committed-delete-live-uuid-data-prefix', 'status': 'passed', 'observation': observation, 'protected_child': child} + finally: + cluster.remove() + +def corruption_repair(): + cluster = Cluster('corrupt-journal', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'corrupt-journal' + key = 'payload' + payload = b'original-protected-data' * 100000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + transaction = str(uuid.UUID(observation['transaction'])) + code = "from pathlib import Path; import base64,json; p=Path('/data/d1')/%r/%r/%r/'delete-prepared'; old=p.read_bytes(); p.write_bytes(b'corrupt-record'); print(json.dumps({'original':base64.b64encode(old).decode()}))" % (bucket, key, transaction) + saved = cluster.inspect_data(code)[0]['original'] + before = residue_inventory(cluster, bucket, key) + cluster.restart() + time.sleep(2) + assert residue_inventory(cluster, bucket, key) == before, 'corrupt authority must preserve all backup/payload files' + (code, body) = client.request('DELETE', bucket + '/' + key) + assert code >= 400, f'corrupt hidden transaction must not acknowledge cleanup: {code} {body!r}' + assert residue_inventory(cluster, bucket, key) == before + repair = "from pathlib import Path; import base64,json; p=Path('/data/d1')/%r/%r/%r/'delete-prepared'; p.write_bytes(base64.b64decode(%r)); print(json.dumps({'repaired':True}))" % (bucket, key, transaction, saved) + cluster.inspect_data(repair) + cluster.kill() + cluster.restart() + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + (code, body) = client.request('GET', bucket + '/' + key) + if code == 200: + assert body == payload + break + time.sleep(0.1) + else: + raise AssertionError('valid original journal must recover original bytes after repair') + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'corrupt-journal-fails-closed-then-repair', 'status': 'passed', 'observation': observation} + finally: + cluster.remove() + +def admin_heal(client, bucket, dry_run=False, remove=False, key=None, recursive=None): + path = 'rustfs/admin/v3/heal/' + bucket + ('/' + key if key else '') + options = {'recursive': key is None if recursive is None else recursive, 'dryRun': dry_run, 'remove': remove, 'recreate': False, 'scanMode': 1, 'updateParity': False, 'nolock': False, 'readRepair': False} + (_, body) = client.request('POST', path, json.dumps(options).encode(), {'Content-Type': 'application/json'}, 200) + token = json.loads(body)['clientToken'] + deadline = time.monotonic() + 90 + + def states(value): + if isinstance(value, dict): + for (k, v) in value.items(): + if k == 'state' and isinstance(v, str): + yield v.lower() + yield from states(v) + elif isinstance(value, list): + for v in value: + yield from states(v) + while time.monotonic() < deadline: + (_, body) = client.request('POST', path + '?clientToken=' + token, headers={'Content-Type': 'application/json'}, expected=200) + result = json.loads(body) + with (ROOT / 'admin-responses.jsonl').open('a') as report: + report.write(json.dumps({'bucket': bucket, 'key': key, 'dry_run': dry_run, 'remove': remove, 'recursive': options['recursive'], 'response': result}) + '\n') + status = set(states(result)) + if status & {'failed', 'completed_with_errors', 'blocked'}: + raise AssertionError('admin heal failed: ' + json.dumps(result)) + if 'completed' in status or result.get('summary') == 'finished': + return result + time.sleep(0.1) + raise TimeoutError('admin heal did not complete: ' + token) + +def admin_legacy_dry_run(direct=False): + cluster = Cluster('admin-legacy', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'admin-legacy' + client.put_bucket(bucket) + for (key, payload) in [('inline', b'i' * 4096), ('external', b'e' * (2 * 1024 * 1024))]: + client.put(bucket, key, payload) + cluster.kill() + seed_legacy(cluster, bucket, 'inline', 'backup-only') + seed_legacy(cluster, bucket, 'external', 'reserved') + before = {key: residue_inventory(cluster, bucket, key) for key in ('inline', 'external')} + cluster.restart() + time.sleep(2) + dry = admin_heal(client, bucket, dry_run=True, remove=True) + assert {key: residue_inventory(cluster, bucket, key) for key in before} == before, 'dry run must never mutate ambiguous residue' + actual = [admin_heal(client, bucket, dry_run=False, remove=True, key=key) for key in before] if direct else admin_heal(client, bucket, dry_run=False, remove=True) + for key in before: + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node))) + client.delete_bucket(bucket) + return {'case': 'admin-legacy-dry-run-and-explicit-cleanup', 'status': 'passed', 'dry_run_report': dry, 'cleanup_report': actual} + finally: + cluster.remove() + +def admin_prefix_scoped(): + cluster = Cluster('admin-prefix', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket, prefix = 'admin-prefix', 'selected/' + selected = {'selected/inline': b'i' * 4096, 'selected/external': b'e' * (2 * 1024 * 1024)} + outside = {'selected-neighbor/hidden': b'n' * 4096, 'outside/hidden': b'o' * 4096} + live_key, live_body = 'selected/live', b'live-prefix-object-must-survive' + client.put_bucket(bucket) + for key, body in {**selected, **outside, live_key: live_body}.items(): + client.put(bucket, key, body) + assert client.get(bucket, key) == body + cluster.kill() + for key in {**selected, **outside}: + seed_legacy(cluster, bucket, key, 'reserved' if key == 'selected/external' else 'backup-only') + before = {key: residue_inventory(cluster, bucket, key) for key in {**selected, **outside}} + cluster.restart() + dry = admin_heal(client, bucket, dry_run=True, remove=True, key=prefix, recursive=True) + assert any('delete recovery (dry run)' in item.get('detail', '') and 'discovered=2' in item.get('detail', '') for item in dry.get('items', [])), 'prefix dry run must inspect the two matching hidden transactions' + assert {key: residue_inventory(cluster, bucket, key) for key in before} == before, 'prefix dry run must preserve every hidden transaction' + actual = admin_heal(client, bucket, remove=True, key=prefix, recursive=True) + after = {key: residue_inventory(cluster, bucket, key) for key in before} + (ROOT / 'prefix-inventory.json').write_text(json.dumps({'before': before, 'after': after}, indent=2)) + for key in selected: + wait_inventory(cluster, bucket, key, lambda inventory: all(not drive['exists'] for node in inventory for drive in node)) + for key in outside: + assert residue_inventory(cluster, bucket, key) == before[key], f'prefix recovery must preserve outside transaction {key}' + assert client.get(bucket, live_key) == live_body, 'prefix remove must preserve the live object' + assert client.list(bucket) == [live_key] + client.delete(bucket, live_key) + client.delete_bucket(bucket, expected=409) + for key in outside: + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'admin-prefix-hidden-transactions-only', 'status': 'passed', 'prefix': prefix, 'selected_keys': list(selected), 'protected_keys': list(outside), 'dry_run_report': dry, 'cleanup_report': actual} + finally: + cluster.remove() + +def deep_key_crash(large_stack=False): + extra_env = {'RUSTFS_RUNTIME_THREAD_STACK_SIZE': str(32 * 1024 * 1024)} if large_stack else None + cluster = Cluster('deep-key-large-stack' if large_stack else 'deep-key', drives=4, extra_env=extra_env) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'deep-recovery' + key = '/'.join(['a'] * 300) + '/payload' + payload = b'deep-inline-content' * 200 + client.put_bucket(bucket) + client.put(bucket, key, payload) + original = residue_inventory(cluster, bucket, key) + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + cluster.restart() + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + assert client.get(bucket, key) == payload + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'deep-key-recovery-large-stack' if large_stack else 'deep-key-recovery', 'status': 'passed', 'segments': 301, 'runtime_thread_stack_size': 32 * 1024 * 1024 if large_stack else 'default', 'observation': observation} + finally: + cluster.remove() + +def receipt_response_loss(): + cluster = Cluster('receipt-response-loss', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'receipt-response-loss' + key = 'payload' + payload = b'lost-response-data' * 150000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + nonces = {index: arm_barrier(cluster, index, bucket, key, 'DECISION') for index in (1, 2)} + (thread, request) = async_delete(client, bucket, key) + observations = [] + for (index, nonce) in nonces.items(): + observations.append(reached_barrier(cluster, index, nonce)) + deadline = time.monotonic() + 40 + while len(list(cluster.barriers[index].glob('reached.' + nonce + '.*.json'))) < 2: + if time.monotonic() > deadline: + raise TimeoutError('both drive receipts must persist before peer kill') + time.sleep(0.02) + for index in (1, 2): + cluster.kill(index) + thread.join(75) + assert not thread.is_alive(), 'DELETE must return after peer response loss' + if 'response' in request: + assert request['response'][0] >= 400, 'four online responses cannot satisfy five-drive receipt quorum' + before = residue_inventory(cluster, bucket, key) + assert any(('delete-quorum-committed' in path for node in before for drive in node for path in drive['files'])), 'stored decisions must remain while peers are offline' + clear_barriers(cluster) + for index in (1, 2): + docker('start', cluster.names[index]) + cluster.ready() + wait_inventory(cluster, bucket, key, lambda inventory: all((not drive['exists'] for node in inventory for drive in node)), timeout=90) + client.request('GET', bucket + '/' + key, expected=404) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': 'receipt-publication-response-loss', 'status': 'passed', 'observations': observations, 'request': {key: str(value) for (key, value) in request.items()}} + finally: + cluster.remove() + +def retained_decision_newer_undecided(): + cluster = Cluster('inverse-null-history', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + (bucket, key) = ('inverse-null-history', 'payload') + older = b'older-generation' * 150000 + newer = b'newer-generation' * 150000 + client.put_bucket(bucket) + client.put(bucket, key, older) + cluster.kill(3) + client.delete(bucket, key) + witness = residue_inventory(cluster, bucket, key) + assert any(('delete-quorum-committed' in path for node in witness[:3] for drive in node for path in drive['files'])), 'offline old peer requires retained committed authority' + client.put(bucket, key, newer) + assert client.get(bucket, key) == newer + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + for index in range(3): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + cluster.restart() + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + (status, body) = client.request('GET', bucket + '/' + key) + if status == 200: + assert body == newer, 'older committed nil receipt must never promote a newer undecided generation' + break + time.sleep(0.1) + else: + raise AssertionError('newer undecided generation was not restored after complete peer return') + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'retained-older-decision-newer-undecided-null', 'status': 'passed', 'observation': observation} + finally: + cluster.remove() + + +def completed_multipart_sidecars(protected=False, crash=False): + case = 'multipart-integrity' if protected else 'multipart-sidecars' + if crash: + case += '-applied' + mode = 'true' if protected else 'false' + cluster = Cluster(case, drives=4, extra_env={'RUSTFS_SHARD_INTEGRITY_WRITE': mode, 'RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED': mode}) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'completed-multipart' + key = 'payload' + parts = [bytes(range(256)) * 20480, bytes(reversed(range(256))) * 4096] + payload = b''.join(parts) + client.put_bucket(bucket) + upload = client.complete_multipart(bucket, key, parts) + assert client.get(bucket, key) == payload, 'complete MPU must expose all uploaded bytes' + original = residue_inventory(cluster, bucket, key) + (ROOT / 'multipart-original-inventory.json').write_text(json.dumps(original, indent=2)) + for node in original: + for drive in node: + assert drive['exists'] and 'xl.meta' in drive['files'], 'complete MPU must publish metadata on every drive' + for number in (1, 2): + sidecars = [name for name in drive['files'] if Path(name).name == f'part.{number}.meta'] + assert len(sidecars) == 1, 'native completed MPU must retain each real part metadata sidecar' + prefix = f'part.{number}.integrity.' + proofs = [name for name in drive['files'] if Path(name).name.startswith(prefix)] + assert len(proofs) == (1 if protected else 0), 'native completion must retain the selected integrity format' + for proof in proofs: + assert uuid.UUID(Path(proof).name[len(prefix):]).int != 0, 'integrity proof generation must be non-nil' + observation = None + if crash: + nonce = arm_barrier(cluster, 0, bucket, key, 'APPLIED', disk='/data/d1') + (thread, request) = async_delete(client, bucket, key) + observation = reached_barrier(cluster, 0, nonce) + cluster.kill() + thread.join(5) + clear_barriers(cluster) + cluster.restart() + wait_inventory(cluster, bucket, key, lambda inventory: inventory == original) + restored = residue_inventory(cluster, bucket, key) + assert restored == original, 'rollback must preserve exact metadata, all shards, part sidecars and integrity proofs' + (ROOT / 'multipart-restored-inventory.json').write_text(json.dumps(restored, indent=2)) + assert client.get(bucket, key) == payload, 'rolled-back native MPU must retain the complete readable body' + client.batch_delete(bucket, [key]) + wait_inventory(cluster, bucket, key, lambda inventory: all(not drive['exists'] for node in inventory for drive in node)) + client.request('GET', bucket + '/' + key, expected=404) + assert client.list(bucket) == [] + client.delete_bucket(bucket) + return {'case': case, 'status': 'passed', 'native_upload_id': upload, 'integrity': protected, 'part_sizes': [len(part) for part in parts], 'bytes': len(payload), 'observation': observation, 'bucket_deletion': 'ordinary non-force204'} + finally: + cluster.remove() + +def ordinary_missing_with_offline_peer(): + cluster = Cluster('missing-offline', nodes=4, drives=1) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'missing-offline' + client.put_bucket(bucket) + cluster.kill(3) + client.delete(bucket, 'never-existed-single', expected=204) + client.batch_delete(bucket, ['never-existed-batch-a', 'never-existed-batch-b']) + assert client.list(bucket) == [], 'ordinary missing DELETE must leave the existing bucket valid' + payload = b'bucket-remains-usable' * 200 + client.put(bucket, 'live', payload) + assert client.get(bucket, 'live') == payload + docker('start', cluster.names[3]) + cluster.ready() + assert client.get(bucket, 'live') == payload + client.delete(bucket, 'live') + wait_inventory(cluster, bucket, 'live', lambda inventory: all(not drive['exists'] for node in inventory for drive in node), timeout=90) + client.delete_bucket(bucket) + return {'case': 'ordinary-missing-delete-offline-single-and-batch', 'status': 'passed', 'cleanup_claim': 'no residue recovery was requested'} + finally: + cluster.remove() + +def legacy_transaction_data_dir_alias(): + cluster = Cluster('legacy-data-dir-alias', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket, key = 'legacy-data-dir-alias', 'payload' + payload = b'legacy-data-dir-content' * 100000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + assert client.get(bucket, key) == payload + cluster.kill() + fixture = seed_legacy(cluster, bucket, key, 'alias') + before = residue_inventory(cluster, bucket, key) + cluster.restart() + time.sleep(2) + assert residue_inventory(cluster, bucket, key) == before, 'automatic recovery must preserve ambiguous alias residue' + client.delete(bucket, key) + wait_inventory(cluster, bucket, key, lambda inventory: all(not drive['exists'] for node in inventory for drive in node)) + client.delete_bucket(bucket) + return {'case': 'legacy-transaction-equals-data-directory', 'status': 'passed', 'fixture': fixture} + finally: + cluster.remove() + +def admin_smoke(): + cluster = Cluster('admin-smoke', drives=4) + try: + cluster.create() + client = cluster.clients[0] + bucket, key = 'admin-smoke', 'payload' + payload = b'healthy-admin-object' * 1000 + client.put_bucket(bucket) + client.put(bucket, key, payload) + bucket_report = admin_heal(client, bucket, dry_run=True) + object_report = admin_heal(client, bucket, key=key) + assert client.get(bucket, key) == payload + client.delete(bucket, key) + client.delete_bucket(bucket) + return {'case': 'admin-healthy-bucket-and-object-client', 'status': 'passed', 'bucket_report': bucket_report, 'object_report': object_report} + finally: + cluster.remove() + +def large_batch_delete(): + cluster = Cluster('batch-1000', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket = 'batch-1000' + client.put_bucket(bucket) + payloads = {f'object-{index:04d}': (bytes(range(256)) * 8192 if index % 250 == 0 else f'content-{index:04d}'.encode()) for index in range(1000)} + for key, payload in payloads.items(): + client.put(bucket, key, payload) + for key, payload in payloads.items(): + assert client.get(bucket, key) == payload + client.batch_delete(bucket, list(payloads)) + assert client.list(bucket) == [] + for key in ('object-0000', 'object-0500', 'object-0999'): + assert all(not drive['exists'] for node in residue_inventory(cluster, bucket, key) for drive in node) + client.delete_bucket(bucket) + return {'case': 'maximum-1000-key-remote-delete-batch', 'status': 'passed', 'exact_gets': 1000, 'keys': 1000} + finally: + cluster.remove() + +def large_version_history(): + cluster = Cluster('versions-3000', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket, key = 'versions-3000', 'history' + client.put_bucket(bucket) + client.versioning(bucket) + versions = [] + for index in range(3000): + payload = f'version-content-{index:04d}'.encode() + client.put(bucket, key, payload) + versions.append((client.last_headers['x-amz-version-id'], payload)) + for index in (0, 1500, 2999): + version, payload = versions[index] + assert client.request('GET', bucket + '/' + key + '?versionId=' + version, expected=200)[1] == payload + oldest = versions.pop(0)[0] + client.delete(bucket, key + '?versionId=' + oldest) + client.request('GET', bucket + '/' + key + '?versionId=' + oldest, expected=404) + assert client.get(bucket, key) == versions[-1][1] + for offset in range(0, len(versions), 1000): + group = versions[offset:offset + 1000] + xml = ET.Element('Delete') + for version, payload in group: + item = ET.SubElement(xml, 'Object') + ET.SubElement(item, 'Key').text = key + ET.SubElement(item, 'VersionId').text = version + data = ET.tostring(xml) + status, body = client.request('POST', bucket + '?delete', data, {'Content-Type': 'application/xml', 'Content-MD5': base64.b64encode(hashlib.md5(data).digest()).decode()}, 200, timeout=180) + response = ET.fromstring(body) + assert not [node for node in response.iter() if node.tag.rsplit('}', 1)[-1] == 'Error'], body + deleted = [child.text for node in response.iter() if node.tag.rsplit('}', 1)[-1] == 'Deleted' for child in node if child.tag.rsplit('}', 1)[-1] == 'VersionId'] + assert sorted(deleted) == sorted(version for version, payload in group), body + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket) + return {'case': 'large-version-history-remote-single-and-batch-delete', 'status': 'passed', 'versions': 3000, 'survivor_bytes_verified': True} + finally: + cluster.remove() + +def unknown_predecessor_fresh_delete(): + cluster = Cluster('unknown-predecessor', nodes=4, drives=2) + try: + cluster.create() + client = cluster.clients[0] + bucket, key = 'unknown-predecessor', 'payload' + client.put_bucket(bucket) + client.put(bucket, key, b'older-inline' * 300) + nonce = arm_barrier(cluster, 3, bucket, key, 'APPLIED', disk='/data/d1') + thread, request = async_delete(cluster.clients[3], bucket, key) + observation = reached_barrier(cluster, 3, nonce) + transaction = str(uuid.UUID(observation['transaction'])) + verify = "from pathlib import Path; import json; result=[];\nfor drive in sorted(Path('/data').glob('d*')):\n root=drive/%r/%r; tx=root/%r; result.append(not (root/'xl.meta').exists() and (tx/'xl.meta.bkp').is_file())\nprint(json.dumps(result))" % (bucket, key, transaction) + deadline = time.monotonic() + 30 + while not all(cluster.inspect_data(verify, indices=[3])[0]): + if time.monotonic() >= deadline: + raise TimeoutError('both returning-peer preparations must reach metadata absence') + time.sleep(.05) + cluster.kill(3) + for index in range(3): + cluster.kill(index) + thread.join(5) + clear_barriers(cluster) + restore = "from pathlib import Path; import json,os; result=[];\nfor drive in sorted(Path('/data').glob('d*')):\n root=drive/%r/%r; tx=root/%r; backup=tx/'xl.meta.bkp';\n if backup.exists(): os.replace(backup,root/'xl.meta')\n if tx.exists():\n for path in tx.iterdir():\n assert path.is_file() and path.name in ('delete-prepared','delete-target'), path\n path.unlink()\n tx.rmdir()\n assert (root/'xl.meta').is_file(); result.append(True)\nprint(json.dumps(result))" % (bucket, key, transaction) + cluster.inspect_data(restore, indices=range(3)) + for index in range(3): + docker('start', cluster.names[index]) + cluster.ready(indices=range(3)) + newer = b'newer-inline' * 300 + client.put(bucket, key, newer) + assert client.get(bucket, key) == newer + client.delete(bucket, key) + docker('start', cluster.names[3]) + cluster.ready() + time.sleep(2) + pending = residue_inventory(cluster, bucket, key) + assert all(transaction + '/xl.meta.bkp' in drive['files'] for drive in pending[3]), 'automatic recovery must preserve unknown predecessor evidence' + client.request('GET', bucket + '/' + key, expected=404) + client.delete_bucket(bucket, expected=409) + client.delete(bucket, key, expected=204) + wait_inventory(cluster, bucket, key, lambda inventory: all(not drive['exists'] for node in inventory for drive in node)) + client.delete_bucket(bucket) + return {'case': 'unknown-predecessor-preserved-until-fresh-current-delete', 'status': 'passed', 'observation': observation, 'fixture': 'authentic prepared records retained only on stopped returning peer'} + finally: + cluster.remove() + +def main(): + global ROOT, PREFIX, IMAGE, PORT, CANDIDATE, BASELINE + import argparse, re, sys + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--binary', required=True, type=Path, help='Linux candidate binary built with e2e-test-hooks') + parser.add_argument('--baseline-binary', type=Path, help='Older Linux binary, required by mixed case') + parser.add_argument('--image', required=True, help='Local Linux image with Python 3 and the binary runtime libraries') + parser.add_argument('--artifact-dir', required=True, type=Path, help='New directory for logs and stage handshakes') + parser.add_argument('--namespace', default='rustfs-delete-' + uuid.uuid4().hex[:12]) + parser.add_argument('--port', type=int, default=19010, help='First of four sequential localhost ports') + parser.add_argument('cases', nargs='*', default=['smoke']) + options = parser.parse_args() + if not re.fullmatch('[a-z][a-z0-9-]{0,45}', options.namespace): + parser.error('namespace must be a short lowercase Docker name') + if not 1024 <= options.port <= 65532: + parser.error('port must leave room for four localhost ports') + CANDIDATE = options.binary.resolve(strict=True) + if not CANDIDATE.is_file(): + parser.error('binary must be a regular file') + BASELINE = options.baseline_binary.resolve(strict=True) if options.baseline_binary else None + if BASELINE is not None and (not BASELINE.is_file()): + parser.error('baseline binary must be a regular file') + ROOT = options.artifact_dir.resolve() + ROOT.mkdir(parents=True, exist_ok=False) + PREFIX = options.namespace + IMAGE = options.image + PORT = options.port + for resource in ('container', 'volume', 'network'): + existing = docker(resource, 'ls', '-q', '--filter', 'label=rustfs.delete-recovery-test=' + PREFIX).stdout.strip() + if existing: + parser.error('namespace already owns Docker resources; choose a new namespace') + cases = {'unknown-fresh': unknown_predecessor_fresh_delete, 'batch1000': large_batch_delete, 'versions3000': large_version_history, 'admin-smoke': admin_smoke, 'legacy-alias': legacy_transaction_data_dir_alias, 'missing-offline': ordinary_missing_with_offline_peer, 'inverse': retained_decision_newer_undecided, 'deep': deep_key_crash, 'deep-large-stack': lambda : deep_key_crash(True), 'admin-prefix': admin_prefix_scoped, 'response-loss': receipt_response_loss, 'admin-object': lambda : admin_legacy_dry_run(True), 'smoke': smoke, 'legacy1': lambda : legacy_retry(1), 'legacy4': lambda : legacy_retry(4), 'restore': rollback_crash, 'retention': protected_version, 'version-applied': lambda : versioned_crash('APPLIED'), 'version-decision': lambda : versioned_crash('DECISION'), 'returning': returning_drive_new_generation, 'untouched': untouched_returning_peer, 'missing-version': missing_version_returning_peer, 'mixed': mixed_versions, 'admin': admin_legacy_dry_run, 'prefix': uuid_prefix_crash, 'corruption': corruption_repair, 'permission': permission_fault, 'error-prepared': lambda : injected_error('PREPARED'), 'error-applied': lambda : injected_error('APPLIED'), 'error-decision': lambda : injected_error('DECISION')} + cases.update({'multipart-sidecars': completed_multipart_sidecars, 'multipart-integrity': lambda: completed_multipart_sidecars(True), 'multipart-sidecars-applied': lambda: completed_multipart_sidecars(False, True), 'multipart-integrity-applied': lambda: completed_multipart_sidecars(True, True)}) + for stage in ('PREACTIVATION', 'PREPARED', 'APPLIED', 'DECISION'): + for nodes in (1, 4): + cases[f'{stage.lower()}{nodes}'] = lambda stage=stage, nodes=nodes: process_crash(stage, nodes) + cases[f'inline-{stage.lower()}'] = lambda stage=stage: process_crash(stage, 1, 4096) + cases[f'zero-{stage.lower()}'] = lambda stage=stage: process_crash(stage, 1, 0) + if 'mixed' in options.cases and BASELINE is None: + parser.error('mixed requires --baseline-binary') + unknown = set(options.cases) - set(cases) + if unknown: + parser.error('unknown cases: ' + ','.join(sorted(unknown))) + results = [] + for case in options.cases: + started = time.monotonic() + try: + result = cases[case]() + result['elapsed_seconds'] = round(time.monotonic() - started, 3) + except BaseException as error: + result = {'case': case, 'status': 'failed', 'error': str(error), 'elapsed_seconds': round(time.monotonic() - started, 3)} + (ROOT / (case + '.json')).write_text(json.dumps(result, indent=2, sort_keys=True)) + raise + results.append(result) + (ROOT / (case + '.json')).write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, sort_keys=True), flush=True) + (ROOT / 'results.json').write_text(json.dumps(results, indent=2, sort_keys=True)) +if __name__ == '__main__': + main() From 71c0efa97dde32a697fbb28b4dece06f690979c2 Mon Sep 17 00:00:00 2001 From: Hauser Date: Mon, 12 Oct 2026 15:29:10 +0800 Subject: [PATCH 021/143] fix(scanner): account remote progress and preserve dispatch limits (#8485) * fix(scanner): account remote progress and preserve dispatch limits Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(scanner): isolate remote progress by cycle Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/scanner-metrics/Cargo.toml | 2 +- crates/scanner-metrics/src/metrics.rs | 191 +++++++++++++++++- crates/scanner/src/remote_scanner/stream.rs | 136 +++++++------ .../src/remote_scanner/stream/tests.rs | 92 +++++++-- crates/scanner/src/scanner.rs | 16 +- crates/scanner/src/scanner/tests.rs | 66 ++++-- crates/scanner/src/scanner_budget.rs | 8 + crates/scanner/src/scanner_io/guards.rs | 6 +- crates/scanner/src/scanner_io/io_cache.rs | 13 +- crates/scanner/src/scanner_io/io_cycle.rs | 15 +- crates/scanner/src/scanner_io/tests.rs | 58 ++++-- 11 files changed, 458 insertions(+), 145 deletions(-) diff --git a/crates/scanner-metrics/Cargo.toml b/crates/scanner-metrics/Cargo.toml index cfc0d3e19..eeedf0791 100644 --- a/crates/scanner-metrics/Cargo.toml +++ b/crates/scanner-metrics/Cargo.toml @@ -38,7 +38,7 @@ tokio = { workspace = true, features = ["sync","rt"] } [dev-dependencies] serde_json = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } uuid = { workspace = true, features = ["v4"] } [lib] diff --git a/crates/scanner-metrics/src/metrics.rs b/crates/scanner-metrics/src/metrics.rs index 1f018d51e..8467dbd13 100644 --- a/crates/scanner-metrics/src/metrics.rs +++ b/crates/scanner-metrics/src/metrics.rs @@ -880,6 +880,24 @@ pub struct TierRequestCount { const TIER_REQUEST_COUNTER_SLOTS: usize = TierRequestOperation::ALL.len() * TierRequestOutcome::ALL.len(); +/// Remote work belongs to one cycle; retained worker handles cannot update a later cycle. +#[derive(Default)] +pub struct RemoteScanProgress { + objects: AtomicU64, + directories: AtomicU64, +} + +impl RemoteScanProgress { + /// Record authenticated per-walk deltas without changing process-local operation counters. + pub fn record(&self, objects: u64, directories: u64) { + for (counter, delta) in [(&self.objects, objects), (&self.directories, directories)] { + if delta != 0 { + let _ = counter.try_update(Ordering::Relaxed, Ordering::Relaxed, |value| Some(value.saturating_add(delta))); + } + } + } +} + pub struct Metrics { operations: Vec, latency: Vec, @@ -889,6 +907,10 @@ pub struct Metrics { cycle_info: Arc>>, current_scan_mode: AtomicU8, current_scan_cycle_work_active: AtomicBool, + // Cycle transitions and reports lock cycle_info before remote_scan_progress. + // Cycle transitions take cycle_info before this lock; never hold it across awaits + // or while acquiring another metrics lock. Workers update their own handle directly. + remote_scan_progress: Mutex>, current_scan_cycle_objects_start: AtomicU64, current_scan_cycle_directories_start: AtomicU64, current_scan_cycle_bucket_drive_scans_start: AtomicU64, @@ -1963,6 +1985,7 @@ impl Metrics { cycle_info: Arc::new(RwLock::new(None)), current_scan_mode: AtomicU8::new(HealScanMode::Unknown as u8), current_scan_cycle_work_active: AtomicBool::new(false), + remote_scan_progress: Mutex::new(Arc::new(RemoteScanProgress::default())), current_scan_cycle_objects_start: AtomicU64::new(0), current_scan_cycle_directories_start: AtomicU64::new(0), current_scan_cycle_bucket_drive_scans_start: AtomicU64::new(0), @@ -2794,8 +2817,18 @@ impl Metrics { /// Publish a scanner cycle and its work-accounting baseline as one state transition. pub async fn start_scan_cycle_work_with_cycle(&self, cycle: CurrentCycle) -> ScanCycleWorkSnapshot { + self.start_scan_cycle_work_with_cycle_and_remote_progress(cycle, Arc::new(RemoteScanProgress::default())) + .await + } + + /// Install the cycle's remote work handle together with its cycle identity. + pub async fn start_scan_cycle_work_with_cycle_and_remote_progress( + &self, + cycle: CurrentCycle, + remote_progress: Arc, + ) -> ScanCycleWorkSnapshot { let mut current_cycle = self.cycle_info.write().await; - let snapshot = self.start_scan_cycle_work(); + let snapshot = self.begin_scan_cycle_work(remote_progress); *current_cycle = Some(cycle); snapshot } @@ -2910,6 +2943,15 @@ impl Metrics { } pub fn start_scan_cycle_work(&self) -> ScanCycleWorkSnapshot { + self.begin_scan_cycle_work(Arc::new(RemoteScanProgress::default())) + } + + fn begin_scan_cycle_work(&self, remote_progress: Arc) -> ScanCycleWorkSnapshot { + // The cycle transition serializes replacement; workers keep their own Arc. + *self + .remote_scan_progress + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = remote_progress; let snapshot = self.scan_cycle_work_snapshot(); let source_snapshot = self.scanner_source_work_values(); let replication_repair_snapshot = self.scanner_replication_repair_work_values(); @@ -3027,9 +3069,22 @@ impl Metrics { fn scan_cycle_work_since(&self, start: ScanCycleWorkSnapshot) -> ScanCycleWorkSnapshot { let current = self.scan_cycle_work_snapshot(); + let (remote_objects, remote_directories) = { + let remote = self + .remote_scan_progress + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + (remote.objects.load(Ordering::Relaxed), remote.directories.load(Ordering::Relaxed)) + }; ScanCycleWorkSnapshot { - objects_scanned: current.objects_scanned.saturating_sub(start.objects_scanned), - directories_scanned: current.directories_scanned.saturating_sub(start.directories_scanned), + objects_scanned: current + .objects_scanned + .saturating_sub(start.objects_scanned) + .saturating_add(remote_objects), + directories_scanned: current + .directories_scanned + .saturating_sub(start.directories_scanned) + .saturating_add(remote_directories), bucket_drive_scans: current.bucket_drive_scans.saturating_sub(start.bucket_drive_scans), bucket_drive_failures: current.bucket_drive_failures.saturating_sub(start.bucket_drive_failures), yield_events: current.yield_events.saturating_sub(start.yield_events), @@ -3650,14 +3705,18 @@ pub type CloseDiskFn = Arc Pin + Send>> /// Register a new disk in the global path tracker and return two callbacks: /// one to update the current path and one to deregister the disk when done. pub async fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) { + register_current_path(&global_metrics().current_paths, disk, initial).await +} + +async fn register_current_path( + current_paths: &Arc>>>, + disk: &str, + initial: &str, +) -> (UpdateCurrentPathFn, CloseDiskFn) { let tracker = Arc::new(CurrentPathTracker::new(initial.to_string())); let disk_name = disk.to_string(); - global_metrics() - .current_paths - .write() - .await - .insert(disk_name.clone(), Arc::clone(&tracker)); + current_paths.write().await.insert(disk_name.clone(), Arc::clone(&tracker)); let update_fn: UpdateCurrentPathFn = { let tracker = Arc::clone(&tracker); @@ -3670,10 +3729,17 @@ pub async fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPa let done_fn: CloseDiskFn = { let disk = disk_name; + let current_paths = Arc::clone(current_paths); + let tracker = Arc::clone(&tracker); Arc::new(move || { let disk = disk.clone(); + let current_paths = Arc::clone(¤t_paths); + let tracker = Arc::clone(&tracker); Box::pin(async move { - global_metrics().current_paths.write().await.remove(&disk); + let mut paths = current_paths.write().await; + if paths.get(&disk).is_some_and(|current| Arc::ptr_eq(current, &tracker)) { + paths.remove(&disk); + } }) }) }; @@ -3857,6 +3923,33 @@ mod tests { assert!(!global_metrics().current_paths.read().await.contains_key(&disk)); } + #[tokio::test] + async fn stale_path_cleanup_preserves_a_replacement_scan() { + let metrics = Metrics::new(); + let (_, close_first) = register_current_path(&metrics.current_paths, "remote-node/disk", "first-bucket").await; + let (_, close_second) = register_current_path(&metrics.current_paths, "remote-node/disk", "second-bucket").await; + close_first().await; + assert_eq!(metrics.get_current_paths().await, vec!["remote-node/disk/second-bucket"]); + close_second().await; + assert!(metrics.get_current_paths().await.is_empty()); + } + + #[tokio::test] + async fn cancelled_path_guard_cleans_up_its_remote_scan() { + let metrics = Metrics::new(); + let (_, close_path) = register_current_path(&metrics.current_paths, "remote-node/disk", "bucket").await; + let guard = CloseDiskGuard::new(close_path); + assert_eq!(metrics.report().await.active_paths, vec!["remote-node/disk/bucket"]); + drop(guard); + tokio::time::timeout(Duration::from_secs(1), async { + while !metrics.get_current_paths().await.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("cancelled remote path guard must remove its registration"); + } + #[tokio::test] async fn report_counts_active_scan_paths() { let metrics = Metrics::new(); @@ -5061,6 +5154,86 @@ mod tests { assert_eq!(report.usage_freshness.last_cycle_cleared_dirty_buckets, 0); } + #[test] + fn remote_scan_progress_saturates_instead_of_wrapping() { + let remote = RemoteScanProgress::default(); + remote.record(u64::MAX - 1, u64::MAX - 2); + remote.record(3, 4); + remote.record(0, 0); + assert_eq!(remote.objects.load(Ordering::Relaxed), u64::MAX); + assert_eq!(remote.directories.load(Ordering::Relaxed), u64::MAX); + } + + #[tokio::test] + async fn saturated_remote_cycle_does_not_hide_later_remote_and_local_work() { + let metrics = Metrics::new(); + let old_remote = Arc::new(RemoteScanProgress::default()); + let previous = metrics + .start_scan_cycle_work_with_cycle_and_remote_progress(CurrentCycle::default(), old_remote.clone()) + .await; + old_remote.record(u64::MAX, u64::MAX); + metrics.finish_scan_cycle_work(previous); + + let remote = Arc::new(RemoteScanProgress::default()); + let next = metrics + .start_scan_cycle_work_with_cycle_and_remote_progress(CurrentCycle::default(), remote.clone()) + .await; + old_remote.record(9, 7); + let empty = metrics.report().await; + assert_eq!(empty.current_cycle_objects_scanned, 0); + assert_eq!(empty.current_cycle_directories_scanned, 0); + remote.record(2, 1); + metrics.operations[Metric::ScanObject as usize].fetch_add(3, Ordering::Relaxed); + metrics.operations[Metric::ScanFolder as usize].fetch_add(2, Ordering::Relaxed); + let active = metrics.report().await; + assert_eq!(active.current_cycle_objects_scanned, 5); + assert_eq!(active.current_cycle_directories_scanned, 3); + metrics.finish_scan_cycle_work(next); + let finished = metrics.report().await; + assert_eq!(finished.last_cycle_objects_scanned, 5); + assert_eq!(finished.last_cycle_directories_scanned, 3); + } + + #[tokio::test] + async fn cycle_work_combines_local_and_remote_progress_without_changing_local_operations() { + let metrics = Metrics::new(); + metrics.operations[Metric::ScanObject as usize].store(10, Ordering::Relaxed); + metrics.operations[Metric::ScanFolder as usize].store(5, Ordering::Relaxed); + let remote = Arc::new(RemoteScanProgress::default()); + let start = metrics + .start_scan_cycle_work_with_cycle_and_remote_progress(CurrentCycle::default(), remote.clone()) + .await; + metrics.operations[Metric::ScanObject as usize].fetch_add(3, Ordering::Relaxed); + metrics.operations[Metric::ScanFolder as usize].fetch_add(2, Ordering::Relaxed); + remote.record(7, 4); + let active = metrics.report().await; + assert_eq!(active.current_cycle_objects_scanned, 10); + assert_eq!(active.current_cycle_directories_scanned, 6); + assert_eq!(metrics.lifetime(Metric::ScanObject), 13); + assert_eq!(metrics.lifetime(Metric::ScanFolder), 7); + metrics.finish_scan_cycle_work(start); + let finished = metrics.report().await; + assert_eq!(finished.current_cycle_objects_scanned, 0); + assert_eq!(finished.current_cycle_directories_scanned, 0); + assert_eq!(finished.last_cycle_objects_scanned, 10); + assert_eq!(finished.last_cycle_directories_scanned, 6); + + remote.record(9, 3); + let next_remote = Arc::new(RemoteScanProgress::default()); + let next = metrics + .start_scan_cycle_work_with_cycle_and_remote_progress(CurrentCycle::default(), next_remote.clone()) + .await; + let empty = metrics.report().await; + assert_eq!(empty.current_cycle_objects_scanned, 0); + assert_eq!(empty.current_cycle_directories_scanned, 0); + remote.record(11, 5); + next_remote.record(2, 1); + metrics.finish_scan_cycle_work(next); + let finished = metrics.report().await; + assert_eq!(finished.last_cycle_objects_scanned, 2); + assert_eq!(finished.last_cycle_directories_scanned, 1); + } + #[tokio::test] async fn report_includes_active_scan_cycle_work() { let metrics = Metrics::new(); diff --git a/crates/scanner/src/remote_scanner/stream.rs b/crates/scanner/src/remote_scanner/stream.rs index 3749dc5e4..8dc15dad6 100644 --- a/crates/scanner/src/remote_scanner/stream.rs +++ b/crates/scanner/src/remote_scanner/stream.rs @@ -29,7 +29,7 @@ use crate::{ use hmac::{Hmac, KeyInit, Mac}; use rustfs_credentials::try_get_rpc_token; use rustfs_heal_contracts::heal_channel::HealScanMode; -use rustfs_scanner_metrics::metrics::{Metric, Metrics}; +use rustfs_scanner_metrics::metrics::{CloseDiskGuard, Metric, Metrics, current_path_updater}; use rustfs_utils::path::path_join_buf; use serde::{Deserialize, Serialize}; use sha2::Sha256; @@ -1017,71 +1017,79 @@ pub(crate) async fn scan_remote_bucket( ))); } - let rpc_deadline = Instant::now() + NS_SCANNER_MAX_RPC_LIFETIME; - let open_deadline = (Instant::now() + NS_SCANNER_STALL_TIMEOUT).min(rpc_deadline); - let open_stream = disk.open_ns_scanner_stream(crate::NsScannerOpenRequest { - request_id, - server_epoch, - session_id, - session_sequence, - next_cycle, - leader_epoch, - body, - stall_timeout: Some(NS_SCANNER_STALL_TIMEOUT), - }); - tokio::pin!(open_stream); - let reader = tokio::select! { - _ = ctx.cancelled() => { - budget.cancel_after_unreported_remote_progress(); - return Err(RemoteScannerFailure::transport(StorageError::other( - "remote namespace scanner cancelled while opening stream", - ))); - } - result = tokio::time::timeout_at(open_deadline, &mut open_stream) => { - match result { - Err(_) => { - budget.cancel_after_unreported_remote_progress(); - return Err(RemoteScannerFailure::transport(StorageError::other( - "remote namespace scanner response headers timed out", - ))); - } - Ok(Err(err)) if err.is_internode_http_status(429) => { - return Err(RemoteScannerFailure::retry_bucket(StorageError::other(format!( - "remote namespace scanner worker rejected zero-progress work: {err}" - )))); - } - Ok(Err(err)) => { - budget.cancel_after_unreported_remote_progress(); - return Err(RemoteScannerFailure::transport(StorageError::other(format!( - "failed to open remote namespace scanner stream: {err}" - )))); - } - Ok(Ok(reader)) => reader, - } - } - }; - let authenticator = FrameAuthenticator::from_rpc_secret(request_id).map_err(|err| { - RemoteScannerFailure::transport(StorageError::other(format!( - "failed to authenticate remote namespace scanner stream: {err}" - ))) - })?; - - let stream_result = consume_remote_scanner_stream_until( - reader, - ctx, - budget.clone(), - RemoteScannerResponseExpectation { - bucket, - source: expected_source, + let disk_name = format!("{}{}", disk.host_name(), disk.path().display()); + let (_, close_path) = current_path_updater(&disk_name, bucket).await; + let mut path_guard = CloseDiskGuard::new(close_path); + let result = async { + let rpc_deadline = Instant::now() + NS_SCANNER_MAX_RPC_LIFETIME; + let open_deadline = (Instant::now() + NS_SCANNER_STALL_TIMEOUT).min(rpc_deadline); + let open_stream = disk.open_ns_scanner_stream(crate::NsScannerOpenRequest { + request_id, + server_epoch, + session_id, + session_sequence, next_cycle, - scan_plan_digest, - tier_registry_generation, - }, - authenticator, - rpc_deadline, - ) + leader_epoch, + body, + stall_timeout: Some(NS_SCANNER_STALL_TIMEOUT), + }); + tokio::pin!(open_stream); + let reader = tokio::select! { + _ = ctx.cancelled() => { + budget.cancel_after_unreported_remote_progress(); + return Err(RemoteScannerFailure::transport(StorageError::other( + "remote namespace scanner cancelled while opening stream", + ))); + } + result = tokio::time::timeout_at(open_deadline, &mut open_stream) => { + match result { + Err(_) => { + budget.cancel_after_unreported_remote_progress(); + return Err(RemoteScannerFailure::transport(StorageError::other( + "remote namespace scanner response headers timed out", + ))); + } + Ok(Err(err)) if err.is_internode_http_status(429) => { + return Err(RemoteScannerFailure::retry_bucket(StorageError::other(format!( + "remote namespace scanner worker rejected zero-progress work: {err}" + )))); + } + Ok(Err(err)) => { + budget.cancel_after_unreported_remote_progress(); + return Err(RemoteScannerFailure::transport(StorageError::other(format!( + "failed to open remote namespace scanner stream: {err}" + )))); + } + Ok(Ok(reader)) => reader, + } + } + }; + let authenticator = FrameAuthenticator::from_rpc_secret(request_id).map_err(|err| { + RemoteScannerFailure::transport(StorageError::other(format!( + "failed to authenticate remote namespace scanner stream: {err}" + ))) + })?; + + let stream_result = consume_remote_scanner_stream_until( + reader, + ctx, + budget.clone(), + RemoteScannerResponseExpectation { + bucket, + source: expected_source, + next_cycle, + scan_plan_digest, + tier_registry_generation, + }, + authenticator, + rpc_deadline, + ) + .await; + finish_remote_scanner_stream(stream_result, budget.as_ref()) + } .await; - finish_remote_scanner_stream(stream_result, budget.as_ref()) + path_guard.close().await; + result } fn finish_remote_scanner_stream( diff --git a/crates/scanner/src/remote_scanner/stream/tests.rs b/crates/scanner/src/remote_scanner/stream/tests.rs index b73b1077d..7b5e2df0b 100644 --- a/crates/scanner/src/remote_scanner/stream/tests.rs +++ b/crates/scanner/src/remote_scanner/stream/tests.rs @@ -917,18 +917,20 @@ async fn complete_terminal_frame_reconciles_progress_and_usage() { let (mut writer, reader) = tokio::io::duplex(4096); tokio::spawn(async move { let mut sequence = 0; - write_frame( - &mut writer, - &writer_auth, - &mut sequence, - &RemoteScannerFrame::progress(RemoteScannerProgress { - objects_scanned: 2, - directories_started: 1, - ..Default::default() - }), - ) - .await - .expect("progress should write"); + for _ in 0..2 { + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::progress(RemoteScannerProgress { + objects_scanned: 2, + directories_started: 1, + ..Default::default() + }), + ) + .await + .expect("progress should write"); + } write_frame( &mut writer, &writer_auth, @@ -961,10 +963,26 @@ async fn complete_terminal_frame_reconciles_progress_and_usage() { ..Default::default() }, ); - let outcome = - consume_remote_scanner_stream(reader, parent, budget.clone(), "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) - .await - .expect("stream should complete"); + let metrics = Metrics::new(); + let start = metrics + .start_scan_cycle_work_with_cycle_and_remote_progress(Default::default(), budget.remote_scan_progress()) + .await; + let outcome = consume_remote_scanner_stream_until( + reader, + parent, + budget.clone(), + RemoteScannerResponseExpectation { + bucket: "bucket", + source: TEST_SOURCE, + next_cycle: TEST_NEXT_CYCLE, + scan_plan_digest: TEST_PLAN_DIGEST, + tier_registry_generation: 0, + }, + reader_auth, + Instant::now() + NS_SCANNER_MAX_RPC_LIFETIME, + ) + .await + .expect("stream should complete"); let RemoteScannerOutcome::Complete { usage, @@ -976,6 +994,15 @@ async fn complete_terminal_frame_reconciles_progress_and_usage() { assert_eq!(usage.entry.objects, 3); assert!(pending_maintenance_work); assert_eq!(budget.progress(), (3, 2)); + let active = metrics.report().await; + assert_eq!(active.current_cycle_objects_scanned, 3); + assert_eq!(active.current_cycle_directories_scanned, 2); + assert_eq!(metrics.lifetime(Metric::ScanObject), 0); + assert_eq!(metrics.lifetime(Metric::ScanFolder), 0); + metrics.finish_scan_cycle_work(start); + let finished = metrics.report().await; + assert_eq!(finished.last_cycle_objects_scanned, 3); + assert_eq!(finished.last_cycle_directories_scanned, 2); } #[tokio::test] @@ -1271,6 +1298,39 @@ fn uncertain_stream_failure_cancels_count_budget() { assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects)); } +#[tokio::test] +async fn remote_progress_rejects_regression_and_counts_new_attempts_once() { + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, ScannerCycleBudgetConfig::default()); + let metrics = Metrics::new(); + let _start = metrics + .start_scan_cycle_work_with_cycle_and_remote_progress(Default::default(), budget.remote_scan_progress()) + .await; + let mut last = RemoteScannerProgress::default(); + let progress = RemoteScannerProgress { + objects_scanned: 4, + directories_started: 2, + entries_visited: 6, + }; + assert!(apply_remote_progress(&budget, &mut last, progress).expect("first progress must advance")); + assert!(!apply_remote_progress(&budget, &mut last, progress).expect("duplicate progress must be accepted")); + let regression = RemoteScannerProgress { + objects_scanned: 3, + ..progress + }; + assert!(apply_remote_progress(&budget, &mut last, regression).is_err()); + assert_eq!(last, progress); + assert_eq!(metrics.report().await.current_cycle_objects_scanned, 4); + assert_eq!(metrics.report().await.current_cycle_directories_scanned, 2); + + let mut retry = RemoteScannerProgress::default(); + apply_remote_progress(&budget, &mut retry, progress).expect("new attempt counts repeated work"); + assert_eq!(budget.progress(), (8, 4)); + let report = metrics.report().await; + assert_eq!(report.current_cycle_objects_scanned, 8); + assert_eq!(report.current_cycle_directories_scanned, 4); +} + #[test] fn directory_entry_activity_counts_as_semantic_progress() { let parent = CancellationToken::new(); diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index c41b01176..3624be128 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -51,9 +51,9 @@ use rustfs_data_usage::observed_data_usage_is_newer; use rustfs_heal_contracts::heal_channel::HealScanMode; use rustfs_lock::{NamespaceLockGuard, error::LockError}; use rustfs_scanner_metrics::metrics::{ - CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource, - emit_scan_cycle_complete, emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, - global_metrics, + CurrentCycle, Metric, Metrics, RemoteScanProgress, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, + ScannerWorkSource, emit_scan_cycle_complete, emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, + emit_scan_cycle_superseded, global_metrics, }; use serde::{Deserialize, Serialize}; use sha2::{Digest as _, Sha256}; @@ -1777,7 +1777,7 @@ where let now = Instant::now(); cycle_info.started = Utc::now(); - let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone(), cycle_budget.remote_scan_progress()).await; // Refresh the storage-owned movement snapshot before reading background // heal state. A missing heal object yields an in-memory default; do not @@ -2598,9 +2598,13 @@ struct ScannerCycleMetricsGuard { } impl ScannerCycleMetricsGuard { - async fn new(cycle: CurrentCycle) -> Self { + async fn new(cycle: CurrentCycle, remote_progress: Arc) -> Self { Self { - start: Some(global_metrics().start_scan_cycle_work_with_cycle(cycle).await), + start: Some( + global_metrics() + .start_scan_cycle_work_with_cycle_and_remote_progress(cycle, remote_progress) + .await, + ), } } diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 6cc15580a..4c3c9879c 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -714,7 +714,7 @@ async fn checkpoint_runtime_handoff(store: Arc, expected_cycl assert_eq!((cycle.next, saved_epoch), (expected_cycle, epoch)); cycle.current = cycle.next; let ctx = CancellationToken::new(); - let mut metrics = ScannerCycleMetricsGuard::new(cycle.clone()).await; + let mut metrics = ScannerCycleMetricsGuard::new(cycle.clone(), Arc::new(RemoteScanProgress::default())).await; tokio::time::pause(); let budget = ScannerCycleBudget::new( &ctx, @@ -1289,7 +1289,8 @@ async fn test_mark_scan_cycle_idle_clears_published_cycle_state() { }; global_metrics().set_current_scan_mode(HealScanMode::Deep); - let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + let mut cycle_metrics_guard = + ScannerCycleMetricsGuard::new(cycle_info.clone(), Arc::new(RemoteScanProgress::default())).await; mark_scan_cycle_idle(&mut cycle_info, &mut cycle_metrics_guard).await; @@ -1307,6 +1308,33 @@ async fn test_mark_scan_cycle_idle_clears_published_cycle_state() { global_metrics().set_cycle(None).await; } +#[tokio::test] +#[serial] +async fn scanner_cycle_metrics_guard_installs_budget_remote_progress() { + let parent = CancellationToken::new(); + let old_budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let mut guard = ScannerCycleMetricsGuard::new(CurrentCycle::default(), old_budget.remote_scan_progress()).await; + old_budget.record_remote_progress(3, 2); + guard.finish(CurrentCycle::default()).await; + let previous = global_metrics().report().await; + assert_eq!(previous.last_cycle_objects_scanned, 3); + assert_eq!(previous.last_cycle_directories_scanned, 2); + + let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let mut guard = ScannerCycleMetricsGuard::new(CurrentCycle::default(), budget.remote_scan_progress()).await; + budget.record_remote_progress(2, 1); + old_budget.record_remote_progress(9, 7); + let active = global_metrics().report().await; + assert_eq!(active.current_cycle_objects_scanned, 2); + assert_eq!(active.current_cycle_directories_scanned, 1); + guard.finish(CurrentCycle::default()).await; + old_budget.record_remote_progress(11, 5); + let finished = global_metrics().report().await; + assert_eq!(finished.last_cycle_objects_scanned, 2); + assert_eq!(finished.last_cycle_directories_scanned, 1); + global_metrics().set_cycle(None).await; +} + #[tokio::test] #[serial] async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() { @@ -1317,7 +1345,7 @@ async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() { started: cycle_started, ..Default::default() }; - let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone(), Arc::new(RemoteScanProgress::default())).await; let setup_report = global_metrics().report().await; assert!(setup_report.current_cycle_active); assert_eq!(setup_report.current_cycle, 0); @@ -1343,7 +1371,7 @@ async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finaliz started: Utc::now(), ..Default::default() }; - let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone(), Arc::new(RemoteScanProgress::default())).await; cycle_info.current = 0; tokio::task::yield_now().await; @@ -1362,12 +1390,15 @@ async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finaliz #[tokio::test] #[serial] async fn scanner_cycle_metrics_guard_drop_clears_activity() { - let guard = ScannerCycleMetricsGuard::new(CurrentCycle { - current: 12, - next: 13, - started: Utc::now(), - ..Default::default() - }) + let guard = ScannerCycleMetricsGuard::new( + CurrentCycle { + current: 12, + next: 13, + started: Utc::now(), + ..Default::default() + }, + Arc::new(RemoteScanProgress::default()), + ) .await; assert!(global_metrics().report().await.current_cycle_active); @@ -1639,7 +1670,8 @@ async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() { cycle_completed: vec![], started: Utc::now(), }; - let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + let mut cycle_metrics_guard = + ScannerCycleMetricsGuard::new(cycle_info.clone(), Arc::new(RemoteScanProgress::default())).await; assert!(finalize_partial_scan_cycle(&ctx, store.clone(), &mut cycle_info, &mut revision, 1, &mut cycle_metrics_guard,).await); @@ -1675,7 +1707,8 @@ async fn scanner_cycle_recovers_to_newer_durable_cache_floor() { cycle_completed: vec![], started: Utc::now(), }; - let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + let mut cycle_metrics_guard = + ScannerCycleMetricsGuard::new(cycle_info.clone(), Arc::new(RemoteScanProgress::default())).await; assert!( persist_required_scanner_cycle_floor( @@ -1715,7 +1748,8 @@ async fn scanner_cycle_rejects_invalid_cache_floor() { cycle_completed: vec![], started: Utc::now(), }; - let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + let mut cycle_metrics_guard = + ScannerCycleMetricsGuard::new(cycle_info.clone(), Arc::new(RemoteScanProgress::default())).await; assert!( !persist_required_scanner_cycle_floor( @@ -1736,7 +1770,8 @@ async fn scanner_cycle_rejects_invalid_cache_floor() { next: 12, ..Default::default() }; - let mut max_cycle_metrics_guard = ScannerCycleMetricsGuard::new(max_cycle_info.clone()).await; + let mut max_cycle_metrics_guard = + ScannerCycleMetricsGuard::new(max_cycle_info.clone(), Arc::new(RemoteScanProgress::default())).await; assert!( !persist_required_scanner_cycle_floor( &ctx, @@ -4600,7 +4635,8 @@ async fn test_finalize_partial_scan_cycle_reports_persist_failure() { cycle_completed: vec![], started: Utc::now(), }; - let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + let mut cycle_metrics_guard = + ScannerCycleMetricsGuard::new(cycle_info.clone(), Arc::new(RemoteScanProgress::default())).await; assert!(!finalize_partial_scan_cycle(&ctx, store, &mut cycle_info, &mut revision, 1, &mut cycle_metrics_guard,).await); assert_eq!(cycle_info.next, 13); diff --git a/crates/scanner/src/scanner_budget.rs b/crates/scanner/src/scanner_budget.rs index 65743439f..3a084e90c 100644 --- a/crates/scanner/src/scanner_budget.rs +++ b/crates/scanner/src/scanner_budget.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use rustfs_scanner_metrics::metrics::RemoteScanProgress; use std::sync::{ Arc, atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}, @@ -73,6 +74,7 @@ pub struct ScannerCycleBudget { entries_visited: AtomicU64, last_progress_millis: AtomicU64, cycle_state_persisted: AtomicBool, + remote_scan_progress: Arc, } impl ScannerCycleBudget { @@ -137,6 +139,7 @@ impl ScannerCycleBudget { entries_visited: AtomicU64::new(0), last_progress_millis: AtomicU64::new(0), cycle_state_persisted: AtomicBool::new(false), + remote_scan_progress: Arc::new(RemoteScanProgress::default()), }) } @@ -237,7 +240,12 @@ impl ScannerCycleBudget { } } + pub(crate) fn remote_scan_progress(&self) -> Arc { + Arc::clone(&self.remote_scan_progress) + } + pub(crate) fn record_remote_progress(&self, objects_scanned: u64, directories_started: u64) { + self.remote_scan_progress.record(objects_scanned, directories_started); if self.track_progress || self.max_objects.is_some() { let objects = saturating_fetch_add(&self.objects_scanned, objects_scanned); if self.track_progress { diff --git a/crates/scanner/src/scanner_io/guards.rs b/crates/scanner/src/scanner_io/guards.rs index bc1e0e4a2..3a5cb4fa9 100644 --- a/crates/scanner/src/scanner_io/guards.rs +++ b/crates/scanner/src/scanner_io/guards.rs @@ -546,15 +546,13 @@ pub(super) fn reset_disk_bucket_scan_gauges(pool: &str, set: &str) { record_disk_bucket_scans_active(0, pool, set); } +// Foreground pressure is handled during work by disk admission and the sleeper. +// Dispatch limits must not freeze a momentary request count for an entire cycle. pub(super) fn scanner_concurrency_limit(configured: usize, available: usize) -> usize { if available == 0 { return 0; } - if crate::workload_admission::foreground_workload_activity() > 0 { - return 1; - } - if configured == 0 { available } else { diff --git a/crates/scanner/src/scanner_io/io_cache.rs b/crates/scanner/src/scanner_io/io_cache.rs index aa51abae0..f6a9d4146 100644 --- a/crates/scanner/src/scanner_io/io_cache.rs +++ b/crates/scanner/src/scanner_io/io_cache.rs @@ -469,21 +469,22 @@ impl ScannerIOCache for SetDisks { "Scanner set assigned remote disks without namespace scanner support to coordinator-driven workers" ); } - let disk_scan_limit = scanner_budgeted_concurrency_limit( - scanner_max_concurrent_disk_scans(workers.len()), - budget.requires_serial_progress_accounting(), - ); + let dispatch_limit = scanner_max_concurrent_disk_scans(workers.len()); + let serial_progress_accounting = budget.requires_serial_progress_accounting(); + let disk_scan_limit = scanner_budgeted_concurrency_limit(dispatch_limit, serial_progress_accounting); record_disk_scan_concurrency_limit(&pool_label, &set_label, disk_scan_limit); - debug!( + tracing::info!( target: "rustfs::scanner::io", event = EVENT_SCANNER_SET_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_IO, + state = "disk_concurrency_budget", + reason = if serial_progress_accounting { "count_budget" } else { "configured_or_available_work" }, pool = self.pool_index, set = self.set_index, online_disks = workers.len(), + dispatch_limit, concurrency_limit = disk_scan_limit, - state = "disk_concurrency_budget", "Scanner disk concurrency budget resolved" ); let disk_scan_semaphore = Arc::new(Semaphore::new(disk_scan_limit)); diff --git a/crates/scanner/src/scanner_io/io_cycle.rs b/crates/scanner/src/scanner_io/io_cycle.rs index b77411e09..577cb1094 100644 --- a/crates/scanner/src/scanner_io/io_cycle.rs +++ b/crates/scanner/src/scanner_io/io_cycle.rs @@ -506,22 +506,23 @@ where return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch)); } - let set_scan_limit = scanner_budgeted_concurrency_limit( - scanner_max_concurrent_set_scans(total_results), - budget.requires_serial_progress_accounting(), - ); + let dispatch_limit = scanner_max_concurrent_set_scans(total_results); + let serial_progress_accounting = budget.requires_serial_progress_accounting(); + let set_scan_limit = scanner_budgeted_concurrency_limit(dispatch_limit, serial_progress_accounting); let bucket_failures = ScannerBucketFailureState::default(); let pending_maintenance_work = Arc::new(AtomicBool::new(false)); let cold_zero_walk_reuse_observed = Arc::new(AtomicBool::new(false)); record_set_scan_concurrency_limit(set_scan_limit); - debug!( + tracing::info!( target: "rustfs::scanner::io", event = EVENT_SCANNER_SET_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_IO, - total_sets = total_results, - concurrency_limit = set_scan_limit, state = "concurrency_budget", + reason = if serial_progress_accounting { "count_budget" } else { "configured_or_available_work" }, + total_sets = total_results, + dispatch_limit, + concurrency_limit = set_scan_limit, "Scanner set concurrency budget resolved" ); let set_scan_semaphore = Arc::new(Semaphore::new(set_scan_limit)); diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index 08d6799e6..1a7eef5bb 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -3959,39 +3959,63 @@ fn scanner_concurrency_limit_handles_no_available_work() { #[test] #[serial] -fn scanner_concurrency_limit_yields_to_foreground_reads() { +fn scanner_concurrency_limit_is_stable_across_foreground_read_transitions() { crate::reset_foreground_read_activity_for_test(); crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test(); - crate::set_foreground_read_activity(8); - assert_eq!(scanner_concurrency_limit(0, 4), 1); - assert_eq!(scanner_concurrency_limit(3, 4), 1); + for foreground_activity in [0, 1, 8, 0] { + crate::set_foreground_read_activity(foreground_activity); + assert_eq!(scanner_concurrency_limit(0, 4), 4); + assert_eq!(scanner_concurrency_limit(3, 4), 3); + assert_eq!(scanner_concurrency_limit(1, 4), 1); + assert_eq!(scanner_concurrency_limit(8, 0), 0); + assert_eq!(scanner_budgeted_concurrency_limit(scanner_concurrency_limit(3, 4), true), 1); + assert_eq!(scanner_budgeted_concurrency_limit(scanner_concurrency_limit(3, 4), false), 3); + } crate::reset_foreground_read_activity_for_test(); } #[test] #[serial] -fn scanner_concurrency_limit_yields_to_shared_foreground_pressure() { +fn scanner_concurrency_limit_is_stable_across_shared_foreground_pressure() { crate::reset_foreground_read_activity_for_test(); crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test(); - install_scanner_workload_provider(WorkloadAdmissionRegistrySnapshot::new(vec![ - WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Open).with_counts(Some(2), None, Some(16)), - ])); - - assert_eq!(scanner_concurrency_limit(0, 4), 1); - assert_eq!(scanner_concurrency_limit(3, 4), 1); - + for (state, active) in [ + (AdmissionState::Open, Some(0)), + (AdmissionState::Open, Some(2)), + (AdmissionState::Saturated, None), + (AdmissionState::Open, Some(0)), + ] { + install_scanner_workload_provider(WorkloadAdmissionRegistrySnapshot::new(vec![ + WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, state).with_counts(active, None, Some(16)), + ])); + assert_eq!(scanner_concurrency_limit(0, 4), 4); + assert_eq!(scanner_concurrency_limit(3, 4), 3); + } crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test(); } #[test] #[serial] -fn scanner_concurrency_limit_yields_to_streaming_reads() { +fn scanner_dispatch_limits_preserve_configured_caps_during_streaming_reads() { crate::reset_foreground_read_activity_for_test(); crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test(); - let _guard = crate::ForegroundReadGuard::new(); - - assert_eq!(scanner_concurrency_limit(0, 4), 1); - assert_eq!(scanner_concurrency_limit(3, 4), 1); + temp_env::with_vars( + [ + (ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, Some("3")), + (ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, Some("2")), + ], + || { + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); + for streaming in [false, true, false] { + let _guard = streaming.then(crate::ForegroundReadGuard::new); + assert_eq!(scanner_max_concurrent_set_scans(4), 3); + assert_eq!(scanner_max_concurrent_disk_scans(4), 2); + assert_eq!(scanner_budgeted_concurrency_limit(scanner_max_concurrent_set_scans(4), true), 1); + assert_eq!(scanner_budgeted_concurrency_limit(scanner_max_concurrent_disk_scans(4), true), 1); + } + }, + ); + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); } #[test] From 3dc09c939705a12a09c7a563b869984a75d92e73 Mon Sep 17 00:00:00 2001 From: cppudge Date: Mon, 12 Oct 2026 15:02:33 +0700 Subject: [PATCH 022/143] test(ecstore): deflake bucket sentinel and dispatch rollback tests (#8488) * test(ecstore): bound sentinel tests by the lock wait, not recreation IO Both bucket incarnation sentinel tests gave the bucket recreation that follows the paused delete (force delete_bucket, make_bucket, put_object on four fsync'ing disks) 10 s in total. The delete releases the sentinel milliseconds after the barrier, but on a runner starved for IO the recreation alone can exceed 10 s: delete_objects_holds_bucket_incarnation_sentinel_through_commit failed that way on main in run 37760321449 (job 113259251979). That bound is not what catches a kept sentinel: delete_bucket waits for it on its lifecycle write lock, which fails after the lock acquire timeout, and the recreation's expect reports the lock error. Keep that check and give the recreation 120 s only as a hang guard. The recreation starts that lock wait before the barrier is released, so the wait also has to cover the delete's own commit, and the 5 s default leaves a starved runner little room for it. Run both tests with RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=30 and bound the delete, which was not bounded, by the same 30 s. A kept sentinel still fails the tests, after 30 s instead of 5 s. Signed-off-by: Petr Slavyagin * test(ecstore): keep background recovery out of dispatch rollback tests dispatch_manifest_detached_workers_are_store_bounded_and_shutdown_cancellable seeds six aborting dispatch manifests, installs a pause hook and expects its own recovery passes to park four detached workers on it. Its first store also ran the background tier-delete journal recovery loop, which runs a pass every TIER_DELETE_JOURNAL_RECOVERY_INTERVAL (60 s). When the setup took longer than that, a background pass rolled the manifests back before the hook was installed, no worker paused, and the 30 s wait failed. This happened on main on a runner starved for IO, in run 37760321449 (job 113259251979). Build the first store with background recovery suppressed, as the neighbouring dispatch tests do. The restarted store keeps it: the second half of the test checks that it picks up the retained manifests. dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member and dispatch_manifest_page_timeout_detaches_one_deduplicated_worker_until_convergence have the same exposure: each seeds an aborting manifest on a store with background recovery and only then installs its slow-delete hook. They have not failed in CI, but with the setup stretched past 60 s they fail the same way. Suppress background recovery for them too. Signed-off-by: Petr Slavyagin --------- Signed-off-by: Petr Slavyagin --- crates/ecstore/src/store/init.rs | 99 +++++++++++++++++++++++++++----- 1 file changed, 86 insertions(+), 13 deletions(-) diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 4b5580e71..64656bcb4 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -10823,8 +10823,19 @@ mod tests { const JOURNAL_COUNT: usize = 65; let temp_dir = tempfile::tempdir().expect("create bounded rollback store dir"); - let (ctx, store, _shutdown) = - without_storage_class_env(build_isolated_test_store(temp_dir.path(), "bounded-dispatch-rollback", &[4])).await; + // The test's worker must be the only one to roll back the seeded manifest: if + // the setup outlasts TIER_DELETE_JOURNAL_RECOVERY_INTERVAL, a background pass + // rolls it back before the slow-delete hook is installed. + let mut instance_ctx = crate::runtime::instance::InstanceContext::new(); + instance_ctx.suppress_tier_delete_journal_recovery_for_test(); + let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( + temp_dir.path(), + "bounded-dispatch-rollback", + &[(1, 4)], + CancellationToken::new(), + Some(Arc::new(instance_ctx)), + )) + .await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; let bucket = "bounded-dispatch-rollback-bucket"; store @@ -10980,8 +10991,19 @@ mod tests { #[serial_test::serial(storage_class_env)] async fn dispatch_manifest_page_timeout_detaches_one_deduplicated_worker_until_convergence() { let temp_dir = tempfile::tempdir().expect("create detached rollback store dir"); - let (ctx, store, _shutdown) = - without_storage_class_env(build_isolated_test_store(temp_dir.path(), "detached-dispatch-rollback", &[4])).await; + // The test's own passes must be the only ones to process the seeded manifest: + // if the setup outlasts TIER_DELETE_JOURNAL_RECOVERY_INTERVAL, a background + // pass rolls it back before the slow-delete hook is installed. + let mut instance_ctx = crate::runtime::instance::InstanceContext::new(); + instance_ctx.suppress_tier_delete_journal_recovery_for_test(); + let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( + temp_dir.path(), + "detached-dispatch-rollback", + &[(1, 4)], + CancellationToken::new(), + Some(Arc::new(instance_ctx)), + )) + .await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; let bucket = "detached-dispatch-rollback-bucket"; store @@ -11049,9 +11071,20 @@ mod tests { const WORKER_LIMIT: usize = 4; let temp_dir = tempfile::tempdir().expect("create bounded detached rollback store dir"); - let (ctx, store, shutdown) = - without_storage_class_env(build_isolated_test_store(temp_dir.path(), "bounded-detached-dispatch-rollback", &[4])) - .await; + // The test's passes must be the only ones to process the seeded manifests: if + // the setup outlasts TIER_DELETE_JOURNAL_RECOVERY_INTERVAL, a background pass + // rolls them back before the pause hook is installed. The restarted store + // below keeps background recovery, which its half of the test relies on. + let mut instance_ctx = crate::runtime::instance::InstanceContext::new(); + instance_ctx.suppress_tier_delete_journal_recovery_for_test(); + let (ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( + temp_dir.path(), + "bounded-detached-dispatch-rollback", + &[(1, 4)], + CancellationToken::new(), + Some(Arc::new(instance_ctx)), + )) + .await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; let tier_name = "BOUNDED-DETACHED-DISPATCH-ROLLBACK"; let _backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; @@ -17373,6 +17406,19 @@ mod tests { #[tokio::test] #[serial_test::serial(storage_class_env)] async fn delete_holds_bucket_incarnation_sentinel_through_object_commit() { + // The recreation waits for the delete's sentinel on its lifecycle write lock + // from before `barrier.release()`, so that lock wait also has to cover the + // delete's commit. Raise the lock acquire timeout from the 5 s default to give + // the commit room on a starved runner; a kept sentinel still fails the test, + // after 30 s instead of 5 s. + temp_env::async_with_vars( + [(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("30"))], + delete_holds_bucket_incarnation_sentinel_through_object_commit_body(), + ) + .await; + } + + async fn delete_holds_bucket_incarnation_sentinel_through_object_commit_body() { let temp = tempfile::tempdir().expect("create temp store dir"); let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(temp.path(), "delete-bucket-incarnation", &[4])).await; @@ -17445,13 +17491,19 @@ mod tests { "bucket delete and same-name recreation must wait for the in-flight object delete" ); barrier.release(); - delete + // The recreation's lock wait started before the release; give the delete as long. + tokio::time::timeout(Duration::from_secs(30), delete) .await + .expect("delete should finish once the barrier is released") .expect("delete task should join") .expect("old-generation delete should complete"); - tokio::time::timeout(Duration::from_secs(10), recreate) + // If the delete kept the sentinel, `delete_bucket` would fail on its lifecycle + // write lock after the lock acquire timeout (30 s here). The rest of the + // recreation is fsync-bound IO on four disks, so this bound, above the lock + // timeout, only stops a hang. + tokio::time::timeout(Duration::from_secs(120), recreate) .await - .expect("bucket recreation should finish after delete releases the sentinel") + .expect("bucket recreation should not hang after delete releases the sentinel") .expect("bucket recreation task should join") .expect("bucket recreation should succeed"); @@ -17504,6 +17556,19 @@ mod tests { #[tokio::test] #[serial_test::serial(storage_class_env)] async fn delete_objects_holds_bucket_incarnation_sentinel_through_commit() { + // The recreation waits for the delete's sentinel on its lifecycle write lock + // from before `barrier.release()`, so that lock wait also has to cover the + // delete's commit. Raise the lock acquire timeout from the 5 s default to give + // the commit room on a starved runner; a kept sentinel still fails the test, + // after 30 s instead of 5 s. + temp_env::async_with_vars( + [(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("30"))], + delete_objects_holds_bucket_incarnation_sentinel_through_commit_body(), + ) + .await; + } + + async fn delete_objects_holds_bucket_incarnation_sentinel_through_commit_body() { let temp = tempfile::tempdir().expect("create temp store dir"); let (_ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(temp.path(), "batch-delete-bucket-incarnation", &[4])).await; @@ -17565,14 +17630,22 @@ mod tests { "bucket recreation must wait for the in-flight batch delete" ); barrier.release(); - let (_deleted, errors) = delete.await.expect("batch delete task should join"); + // The recreation's lock wait started before the release; give the delete as long. + let (_deleted, errors) = tokio::time::timeout(Duration::from_secs(30), delete) + .await + .expect("batch delete should finish once the barrier is released") + .expect("batch delete task should join"); assert!( errors.iter().all(Option::is_none), "old-generation batch delete should complete: {errors:?}" ); - tokio::time::timeout(Duration::from_secs(10), recreate) + // If the delete kept the sentinel, `delete_bucket` would fail on its lifecycle + // write lock after the lock acquire timeout (30 s here). The rest of the + // recreation is fsync-bound IO on four disks, so this bound, above the lock + // timeout, only stops a hang. + tokio::time::timeout(Duration::from_secs(120), recreate) .await - .expect("bucket recreation should finish after batch delete releases the sentinel") + .expect("bucket recreation should not hang after batch delete releases the sentinel") .expect("bucket recreation task should join") .expect("bucket recreation should succeed"); store From 13ee76a4a9ddea577bbdf568c819d7c08131ea32 Mon Sep 17 00:00:00 2001 From: Hauser Date: Mon, 12 Oct 2026 16:06:01 +0800 Subject: [PATCH 023/143] fix(ecstore): validate recovery evidence before mutation (#8487) Reject causal cycles before source repair and share conservative backup classification between discovery and inspection. Fence public raw deletion and derive exact-key removals from successful settlement without an extra metadata read. Preserve the existing physical phases and strict source checks. Make outage and renewal fixtures deterministic without weakening their convergence assertions, and complete strict TLS/RPC lint cleanup. Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/ecstore/src/cluster/rpc/client.rs | 6 +- .../ecstore/src/cluster/rpc/network_probe.rs | 10 +- crates/ecstore/src/disk/local.rs | 4 + .../ecstore/src/disk/local/delete_recovery.rs | 183 +++++++++++++++++- .../src/set_disk/ops/delete_recovery.rs | 129 ++++++++++-- crates/ecstore/src/set_disk/ops/heal.rs | 22 +-- crates/ecstore/src/set_disk/ops/object.rs | 50 +++-- crates/tls-runtime/src/material.rs | 8 +- rustfs/src/server/tls_material.rs | 5 +- rustfs/src/server/tls_publish.rs | 2 +- 10 files changed, 345 insertions(+), 74 deletions(-) diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index 75706254c..ca781d6be 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -54,7 +54,7 @@ use super::context_propagation::{inject_request_id_into_metadata, inject_trace_c /// 3. Subsequent calls will attempt fresh connections /// 4. If node is still down, connection will fail fast (3s timeout) pub async fn node_service_time_out_client( - addr: &String, + addr: &str, interceptor: TonicInterceptor, ) -> Result>, Box> { // Default to the latency-sensitive control channel; bulk `bytes` RPCs opt in via the @@ -110,7 +110,7 @@ pub async fn tier_mutation_control_time_out_client( /// from lock/health RPCs; everything else uses [`ChannelClass::Control`]. When isolation is /// disabled the two classes resolve to the same cached channel, i.e. legacy behavior. pub async fn node_service_time_out_client_for_class( - addr: &String, + addr: &str, interceptor: TonicInterceptor, class: ChannelClass, ) -> Result>, Box> { @@ -141,7 +141,7 @@ pub(crate) async fn node_service_delete_recovery_client( } pub async fn node_service_time_out_client_no_auth( - addr: &String, + addr: &str, ) -> Result>, Box> { node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await } diff --git a/crates/ecstore/src/cluster/rpc/network_probe.rs b/crates/ecstore/src/cluster/rpc/network_probe.rs index 75d9049ea..765cc7322 100644 --- a/crates/ecstore/src/cluster/rpc/network_probe.rs +++ b/crates/ecstore/src/cluster/rpc/network_probe.rs @@ -289,13 +289,9 @@ async fn client( >, NetworkPeerProbeError, > { - node_service_time_out_client_for_class( - &address.to_owned(), - TonicInterceptor::Signature(gen_tonic_signature_interceptor()), - class, - ) - .await - .map_err(|_| NetworkPeerProbeError::Unreachable) + node_service_time_out_client_for_class(address, TonicInterceptor::Signature(gen_tonic_signature_interceptor()), class) + .await + .map_err(|_| NetworkPeerProbeError::Unreachable) } fn ping_request(payload: &[u8]) -> PingRequest { diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 11134035f..8b53add0b 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -242,6 +242,10 @@ fn read_all_data_std(path: &Path) -> core::result::Result<(Vec, Option Uuid { let used_data_dirs: HashSet = meta.get_data_dirs().unwrap_or_default().into_iter().flatten().collect(); + inline_metadata_rollback_dir_with_used_dirs(version_id, &used_data_dirs) +} + +fn inline_metadata_rollback_dir_with_used_dirs(version_id: Uuid, used_data_dirs: &HashSet) -> Uuid { let base = version_id.as_u128() ^ INLINE_METADATA_ROLLBACK_DIR_XOR; let mut salt = 0u128; diff --git a/crates/ecstore/src/disk/local/delete_recovery.rs b/crates/ecstore/src/disk/local/delete_recovery.rs index c1b90219e..f8f4e3485 100644 --- a/crates/ecstore/src/disk/local/delete_recovery.rs +++ b/crates/ecstore/src/disk/local/delete_recovery.rs @@ -141,6 +141,73 @@ fn regular_bytes(path: &Path) -> Result>> { regular_metadata_bytes(path) } +/// A write/heal backup is not a delete transaction when the live metadata +/// still owns its directory and retains every backed-up generation. Any delete +/// evidence wins; uncertain backups remain discoverable and fail closed. +fn live_write_backup(root: &Path, transaction: Uuid) -> Result { + let rollback = root.join(transaction.to_string()); + for name in [ + PREPARED, + TARGET, + FINISHED, + DELETE_MARKER_ROLLBACK_FILE, + DELETE_QUORUM_COMMITTED_FILE, + ] { + if regular_bytes(&rollback.join(name))?.is_some() { + return Ok(false); + } + } + for prefix in [ + DELETE_DATA_DIR_MARKER_PREFIX, + RESERVED_DELETE_DATA_DIR_MARKER_PREFIX, + RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX, + ] { + let name = format!("{prefix}{transaction}"); + if regular_bytes(&root.join(&name))?.is_some() || regular_bytes(&rollback.join(name))?.is_some() { + return Ok(false); + } + } + let Some(current) = regular_metadata_bytes(&root.join(STORAGE_FORMAT_FILE))? else { + return Ok(false); + }; + let Some(backup) = regular_metadata_bytes(&rollback.join(STORAGE_FORMAT_FILE_BACKUP))? else { + return Ok(false); + }; + live_write_backup_matches(&FileMeta::load(¤t)?, &FileMeta::load(&backup)?, transaction) +} + +fn live_write_backup_matches(current: &FileMeta, backup: &FileMeta, transaction: Uuid) -> Result { + let mut owns_directory = false; + let used_data_dirs: HashSet<_> = current.get_data_dirs()?.into_iter().flatten().collect(); + let mut live = HashMap::with_capacity(current.versions.len()); + for version in ¤t.versions { + let mut info = version.into_fileinfo("", "", true)?; + let id = info.version_id.unwrap_or_default(); + owns_directory |= + info.data_dir == Some(transaction) || inline_metadata_rollback_dir_with_used_dirs(id, &used_data_dirs) == transaction; + rustfs_utils::http::metadata_compat::remove_str(&mut info.metadata, rustfs_utils::http::metadata_compat::SUFFIX_HEALING); + if live.insert(id, generation(info)?).is_some() { + return Err(DiskError::FileCorrupt); + } + } + if !owns_directory || backup.versions.is_empty() { + return Ok(false); + } + let mut seen = HashSet::with_capacity(backup.versions.len()); + for version in &backup.versions { + let mut info = version.into_fileinfo("", "", true)?; + let id = info.version_id.unwrap_or_default(); + if !seen.insert(id) { + return Err(DiskError::FileCorrupt); + } + rustfs_utils::http::metadata_compat::remove_str(&mut info.metadata, rustfs_utils::http::metadata_compat::SUFFIX_HEALING); + if live.get(&id) != Some(&generation(info)?) { + return Ok(false); + } + } + Ok(true) +} + fn journal_bytes(journal: &DeleteRecoveryJournal) -> Result> { let bytes = serde_json::to_vec(journal)?; // The same journal is carried by adoption requests and decision receipts. @@ -674,14 +741,10 @@ fn inspection(root: &Path, volume: &str, object: &str, transaction: Uuid) -> Res && target.is_none() && receipt_bytes.is_none() && !finished - && current_meta.as_ref().is_some_and(|meta| { - meta.get_data_dirs() - .is_ok_and(|dirs| dirs.into_iter().flatten().any(|dir| dir == transaction)) - }) - && before_meta.as_ref().is_some_and(|meta| { - meta.get_data_dirs() - .is_ok_and(|dirs| dirs.into_iter().flatten().any(|dir| dir == transaction)) - }); + && match (current_meta.as_ref(), before_meta.as_ref()) { + (Some(current), Some(backup)) => live_write_backup_matches(current, backup, transaction)?, + _ => false, + }; let state = if live_write_backup { DeleteRecoveryState::Absent } else if journal.is_some() { @@ -961,6 +1024,11 @@ impl LocalDisk { Some(_) => session.root.clone(), None => session.root.join(object), }; + if name == STORAGE_FORMAT_FILE_BACKUP + && live_write_backup(&physical, transaction).map_err(std::io::Error::from)? + { + continue; + } let proof_root = physical.join(transaction.to_string()); let prepared = regular_bytes(&proof_root.join(PREPARED)).map_err(std::io::Error::from)?; let receipt = @@ -2146,6 +2214,105 @@ mod tests { (temp, disk, info, bytes) } + #[tokio::test] + async fn delete_recovery_live_write_backup_preserves_delete_evidence() { + let (_temp, disk, info, original) = fixture().await; + let root = disk.io_get_object_path("recovery", "object").unwrap(); + let meta = FileMeta::load(&original).unwrap(); + let transaction = inline_metadata_rollback_dir(Uuid::nil(), &meta); + let rollback = root.join(transaction.to_string()); + fs::create_dir_all(&rollback).await.unwrap(); + fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), &original).await.unwrap(); + let mut healing = info.clone(); + healing.set_healing(); + fs::write(root.join(STORAGE_FORMAT_FILE), test_meta(healing)).await.unwrap(); + assert!(live_write_backup(&root, transaction).unwrap()); + let page = disk + .delete_recovery_discover("recovery", Some("object"), None, 128) + .await + .unwrap(); + assert!(page.transactions.is_empty()); + assert_eq!(fs::read(rollback.join(STORAGE_FORMAT_FILE_BACKUP)).await.unwrap(), original); + fs::write(rollback.join(DELETE_QUORUM_COMMITTED_FILE), []).await.unwrap(); + assert!( + !live_write_backup(&root, transaction).unwrap(), + "even an empty legacy receipt is delete evidence" + ); + fs::remove_file(rollback.join(DELETE_QUORUM_COMMITTED_FILE)).await.unwrap(); + let mut changed_parts = info.clone(); + changed_parts.parts[0].actual_size += 1; + fs::write(root.join(STORAGE_FORMAT_FILE), test_meta(changed_parts)) + .await + .unwrap(); + assert!( + !matches!(live_write_backup(&root, transaction), Ok(true)), + "changed part metadata cannot hide recovery evidence" + ); + let mut changed = info; + changed.metadata.insert("x-amz-object-lock-legal-hold".into(), "ON".into()); + fs::write(root.join(STORAGE_FORMAT_FILE), test_meta(changed)).await.unwrap(); + assert!( + !live_write_backup(&root, transaction).unwrap(), + "protected metadata differences remain meaningful" + ); + fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap(); + assert!( + !live_write_backup(&root, transaction).unwrap(), + "hidden legacy deletion remains discoverable" + ); + } + + #[tokio::test] + async fn delete_recovery_live_external_backup_rejects_partial_version_delete() { + let (_temp, disk, mut info, _) = fixture().await; + let root = disk.io_get_object_path("recovery", "object").unwrap(); + let transaction = Uuid::new_v4(); + info.data = None; + rustfs_utils::http::metadata_compat::remove_str( + &mut info.metadata, + rustfs_utils::http::metadata_compat::SUFFIX_INLINE_DATA, + ); + info.data_dir = Some(transaction); + info.version_id = Some(Uuid::new_v4()); + let mut meta = FileMeta::new(); + meta.add_version(info.clone()).unwrap(); + let original = meta.marshal_msg().unwrap(); + let rollback = root.join(transaction.to_string()); + fs::create_dir_all(&rollback).await.unwrap(); + fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), &original).await.unwrap(); + info.set_healing(); + fs::write(root.join(STORAGE_FORMAT_FILE), test_meta(info.clone())) + .await + .unwrap(); + assert!(live_write_backup(&root, transaction).unwrap()); + let mut sibling = info.clone(); + sibling.version_id = Some(Uuid::new_v4()); + sibling.data_dir = Some(Uuid::new_v4()); + meta.add_version(sibling).unwrap(); + fs::write(rollback.join(STORAGE_FORMAT_FILE_BACKUP), meta.marshal_msg().unwrap()) + .await + .unwrap(); + assert!( + !live_write_backup(&root, transaction).unwrap(), + "a missing backed-up sibling retains delete recovery" + ); + assert_eq!( + disk.inspect_delete_transaction("recovery", "object", transaction) + .await + .unwrap() + .state, + DeleteRecoveryState::LegacyAmbiguous + ); + assert!( + disk.discover_delete_transactions("recovery", Some("object"), None, 128) + .await + .unwrap() + .transactions + .iter() + .any(|id| id.transaction == transaction) + ); + } + fn request() -> FileInfo { FileInfo { name: "object".to_owned(), diff --git a/crates/ecstore/src/set_disk/ops/delete_recovery.rs b/crates/ecstore/src/set_disk/ops/delete_recovery.rs index 71d76f8b8..2aef4e65b 100644 --- a/crates/ecstore/src/set_disk/ops/delete_recovery.rs +++ b/crates/ecstore/src/set_disk/ops/delete_recovery.rs @@ -29,7 +29,7 @@ use uuid::Uuid; const DISCOVERY_PAGE_SIZE: u32 = 128; const MAX_DISCOVERED_TRANSACTIONS: usize = 1_000_000; -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Default, Clone, PartialEq, Eq)] pub(crate) struct DeleteRecoverySummary { pub discovered: usize, pub committed: usize, @@ -37,6 +37,8 @@ pub(crate) struct DeleteRecoverySummary { pub blocked: usize, pub pending: usize, pub reclaimed: usize, + /// Observed removals for this exact key, not aggregated across prefixes. + pub removed_versions: HashSet, /// Read-only evidence for ordinary missing-key S3 semantics. This never /// authorizes settlement while a configured participant is unavailable. pub no_transactions_at_quorum: bool, @@ -191,6 +193,7 @@ async fn settle_group( action: impl Fn(&DeleteRecoveryInspection) -> Option + Sync, can_commit: &(impl Fn() -> bool + Sync + ?Sized), namespace_owner: Option<&Arc>, + removed_versions: &mut HashSet, ) -> Result { let results = join_all(disks.iter().zip(copies.iter()).map(|(disk, copy)| { let action = action(copy); @@ -214,7 +217,22 @@ async fn settle_group( let mut pending = false; for (copy, result) in copies.iter_mut().zip(results) { match result { - Ok(Some(next)) => *copy = next, + Ok(Some(next)) => { + if copy.current_digest != next.current_digest { + let retained: HashSet<_> = next + .current_versions + .iter() + .map(|v| v.version_id.unwrap_or_default()) + .collect(); + removed_versions.extend( + copy.current_versions + .iter() + .map(|v| v.version_id.unwrap_or_default()) + .filter(|id| !retained.contains(id)), + ); + } + *copy = next; + } Ok(None) => {} Err(DiskError::DiskOngoingReq) => pending = true, Err(error) => return Err(error), @@ -469,6 +487,8 @@ impl SetDisks { } all_states.insert(transaction, complete); } + // Validate causal authority before prerequisite repair can publish metadata. + let transaction_order = causal_transaction_order(&all_states)?; let prerequisite = if repair_marker_source && !dry_run { let mut active = all_states .iter() @@ -502,7 +522,6 @@ impl SetDisks { } all_states.insert(transaction, refreshed); } - let transaction_order = causal_transaction_order(&all_states)?; // Legacy snapshots contain no set decision. A fresh authorized removal // applies only after the entire key is absent and every raw shape is safe. let legacy_only = explicit_delete @@ -590,6 +609,7 @@ impl SetDisks { }, can_commit, namespace_owner.as_ref(), + &mut summary.removed_versions, ) .await? { @@ -696,6 +716,7 @@ impl SetDisks { }, can_commit, namespace_owner.as_ref(), + &mut summary.removed_versions, ) .await?; if !pending { @@ -706,6 +727,7 @@ impl SetDisks { |_| Some(DeleteRecoveryAction::FinishCommit), can_commit, namespace_owner.as_ref(), + &mut summary.removed_versions, ) .await?; } @@ -752,6 +774,7 @@ impl SetDisks { |copy| (copy.state == DeleteRecoveryState::NewPrepared).then_some(DeleteRecoveryAction::Rollback), can_commit, namespace_owner.as_ref(), + &mut summary.removed_versions, ) .await? { @@ -795,6 +818,7 @@ impl SetDisks { |copy| (copy.state == DeleteRecoveryState::CommitFinished).then_some(DeleteRecoveryAction::FinalizeCommit), can_commit, namespace_owner.as_ref(), + &mut summary.removed_versions, ) .await? { @@ -1030,25 +1054,25 @@ mod tests { assert_eq!(body(&store, bucket, None).await, original); } - #[tokio::test] - async fn delete_recovery_marker_source_heal_preserves_foreign_history_and_damaged_data() { - fn snapshot(root: &std::path::Path) -> BTreeMap> { - let mut files = BTreeMap::new(); - let mut pending = vec![root.to_path_buf()]; - while let Some(directory) = pending.pop() { - for entry in std::fs::read_dir(directory).unwrap() { - let path = entry.unwrap().path(); - if path.is_dir() { - pending.push(path); - } else { - files.insert(path.strip_prefix(root).unwrap().to_path_buf(), std::fs::read(path).unwrap()); - } + fn snapshot(root: &std::path::Path) -> BTreeMap> { + let mut files = BTreeMap::new(); + let mut pending = vec![root.to_path_buf()]; + while let Some(directory) = pending.pop() { + for entry in std::fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + pending.push(path); + } else { + files.insert(path.strip_prefix(root).unwrap().to_path_buf(), std::fs::read(path).unwrap()); } } - files } + files + } - for shape in ["live-backup", "foreign", "partial-history", "damaged-data"] { + #[tokio::test] + async fn delete_recovery_marker_source_heal_preserves_foreign_history_and_damaged_data() { + for shape in ["live-backup", "foreign", "partial-history", "damaged-data", "causal-cycle"] { let bucket = "recover-marker-source-guards"; let (dirs, store) = isolated_store_over_temp_disks().await; init_bucket_metadata_sys(store.clone(), Vec::new()).await; @@ -1135,6 +1159,33 @@ mod tests { tokio::fs::write(part, bytes).await.unwrap(); } } + "causal-cycle" => { + for dir in dirs.iter().take(3) { + let root = dir.path().join(bucket).join("object"); + let mut found = 0; + for entry in std::fs::read_dir(root).unwrap() { + let entry = entry.unwrap(); + let prepared = entry.path().join("delete-prepared"); + if !prepared.is_file() { + continue; + } + let transaction = entry.file_name().to_str().unwrap().to_owned(); + let mut journal: serde_json::Value = + serde_json::from_slice(&std::fs::read(&prepared).unwrap()).unwrap(); + journal["predecessors"] = serde_json::json!([transaction]); + std::fs::write(prepared, serde_json::to_vec(&journal).unwrap()).unwrap(); + let receipt = entry.path().join(crate::disk::local::DELETE_QUORUM_COMMITTED_FILE); + if receipt.is_file() { + let mut value: serde_json::Value = + serde_json::from_slice(&std::fs::read(&receipt).unwrap()).unwrap(); + value["journal"] = journal; + std::fs::write(receipt, serde_json::to_vec(&value).unwrap()).unwrap(); + } + found += 1; + } + assert_eq!(found, 1, "one active marker journal must exercise prerequisite rejection"); + } + } _ => unreachable!(), } let before: Vec<_> = dirs @@ -1260,6 +1311,31 @@ mod tests { .expect("returning old replica must not leave bucket residue"); } + #[tokio::test] + async fn delete_recovery_tracks_only_observed_removed_version() { + let bucket = "recover-actual-removal"; + let (_dirs, store, disks) = fixture(bucket).await; + let target = put(&store, bucket, &[41; 4096], true).await.unwrap(); + let retained = put(&store, bucket, &[42; 4096], true).await.unwrap(); + let transaction = applied(&store, &disks, bucket, Some(target), 3).await; + publish(&disks[0], bucket, transaction).await; + let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap(); + let set = &store.pools[0].disk_set[0]; + let report = set + .recover_delete_object(bucket, "object", DeleteRecoveryContext::new(Some(incarnation), &|| true)) + .await + .unwrap(); + assert_eq!(report.pending, 0); + assert_eq!(report.blocked, 0); + assert_eq!(report.removed_versions, HashSet::from([target])); + assert_eq!(body(&store, bucket, Some(retained)).await, vec![42; 4096]); + let replay = set + .recover_delete_object(bucket, "object", DeleteRecoveryContext::new(Some(incarnation), &|| true)) + .await + .unwrap(); + assert!(replay.removed_versions.is_empty(), "receipt replay cannot claim a second removal"); + } + #[tokio::test] async fn delete_recovery_set_retains_decisions_until_offline_peer_returns() { let bucket = "recover-offline"; @@ -1789,13 +1865,22 @@ mod tests { #[tokio::test] async fn delete_recovery_chained_committed_versions_finish_after_current_metadata_disappears() { let bucket = "recover-chained-versions"; - let (_dirs, store, disks) = fixture(bucket).await; + let (dirs, store, disks) = fixture(bucket).await; let first = put(&store, bucket, &vec![61; 1024 * 1024], true) .await .expect("first version"); let second = put(&store, bucket, &vec![62; 4096], true).await.expect("second version"); let set = &store.pools[0].disk_set[0]; - set.disks.write().await[3] = None; + let returning_key = dirs[3].path().join(bucket).join("object"); + let before_outage = snapshot(&returning_key); + // A missing slot can be renewed by the endpoint monitor. Keep its + // original identity attached while rejecting I/O until explicit return. + disks[3].close().await.expect("stop returning drive's health monitor"); + disks[3].force_offline_for_test(); + set.connect_disks().await; + assert!(Arc::ptr_eq(set.disks.read().await[3].as_ref().unwrap(), &disks[3])); + assert_eq!(disks[3].runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline); + assert!(matches!(discover(&disks[3], bucket, Some("object")).await, Err(DiskError::FaultyDisk))); for version in [first, second] { store .delete_object( @@ -1816,7 +1901,9 @@ mod tests { .await .is_err() ); - set.disks.write().await[3] = Some(disks[3].clone()); + assert_eq!(snapshot(&returning_key), before_outage, "offline peer retains both deleted versions"); + assert_eq!(disks[3].runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline); + disks[3].reset_health_for_store_init_retry(); let report = store .recover_delete_transactions(bucket, &HealOpts::default()) .await diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 200c13690..0cde84a0a 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -3485,18 +3485,6 @@ impl SetDisks { && let Some(scope) = crate::store::bucket_heal_scope_for_object(bucket, object) && !crate::bucket::utils::is_meta_bucketname(bucket) { - let requested = Uuid::parse_str(version_id).ok(); - if !opts.dry_run && (version_id.is_empty() || requested.is_some()) { - let disks = self.get_disks_internal().await; - let before = Self::read_metadata(&disks, "", bucket, object, version_id, false, false, false) - .await - .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?; - recovery_removed_version = before - .slots - .iter() - .filter_map(|slot| slot.file_info()) - .any(|info| version_id.is_empty() || Some(info.version_id.unwrap_or_default()) == requested); - } let explicit_delete = opts.remove && version_id.is_empty() && crate::bucket::metadata_sys::has_authoritative_never_versioned_state_in(&self.ctx, bucket).await? @@ -3522,6 +3510,16 @@ impl SetDisks { }, ) .await?; + recovery_removed_version = if version_id.is_empty() { + !recovery.removed_versions.is_empty() + } else { + let target = if version_id == "null" { + Some(Uuid::nil()) + } else { + Uuid::parse_str(version_id).ok() + }; + target.is_some_and(|id| recovery.removed_versions.contains(&id)) + }; if opts.dry_run && recovery.discovered > 0 { return Ok(( HealResultItem { diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index b35eb4cb7..4c707eeea 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -8342,18 +8342,22 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } else { self.bucket_incarnation_id_from_disk(bucket).await.ok() }; - self.delete_object_version_with_purge( - bucket, - object, - fi, - force_del_marker, - None, - DeleteCommitContext { - incarnation: source_bucket_incarnation_id, - ..Default::default() - }, - ) - .await + self.invalidate_get_object_metadata_cache(bucket, object).await; + let result = self + .delete_object_version_with_purge( + bucket, + object, + fi, + force_del_marker, + None, + DeleteCommitContext { + incarnation: source_bucket_incarnation_id, + ..Default::default() + }, + ) + .await; + self.invalidate_get_object_metadata_cache(bucket, object).await; + result } #[tracing::instrument(skip(self, objects, opts))] @@ -14241,6 +14245,25 @@ mod metadata_mutation_generation_tests { assert_retired(&set_disks, &delete_key).await; drop(delete_probe); + let raw_object = "raw-delete-object"; + let (_, raw_key) = put_and_prime(&set_disks, bucket, raw_object, b"raw DELETE body").await; + let raw_probe = MetadataCacheInvalidationProbe::install(bucket, raw_object); + set_disks + .delete_object_version( + bucket, + raw_object, + &FileInfo { + name: raw_object.to_owned(), + ..Default::default() + }, + false, + ) + .await + .expect("raw DELETE should succeed"); + assert_eq!(raw_probe.count(), 2, "raw DELETE must fence the cache before and after mutation"); + assert_retired(&set_disks, &raw_key).await; + drop(raw_probe); + let copy_object = "copy-object"; let (mut copy_info, copy_key) = put_and_prime(&set_disks, bucket, copy_object, b"COPY body").await; copy_info.metadata_only = true; @@ -24413,8 +24436,7 @@ mod delete_recovery_inventory_tests { }); barrier.wait_until_paused().await; let extra_gate = set.disks.read().await; - let renewal = set.renew_disk(&endpoint); - tokio::pin!(renewal); + let mut renewal = Box::pin(set.renew_disk(&endpoint)); tokio::time::timeout( Duration::from_secs(5), futures::future::poll_fn(|cx| { diff --git a/crates/tls-runtime/src/material.rs b/crates/tls-runtime/src/material.rs index 5f8f200e5..7046f0174 100644 --- a/crates/tls-runtime/src/material.rs +++ b/crates/tls-runtime/src/material.rs @@ -225,10 +225,10 @@ fn load_server_material(source: &TlsSource, base_dir: &Path) -> Result, desc: &str) -> bool } } -/// Search for and load certificate files matching `cert_name` in the directory -/// and one level of subdirectories. -/// Returns `true` if at least one matching file was loaded. - +/// Validity windows indexed by leaf certificate DER. type CertificateValidities = HashMap, (i64, i64)>; fn certificate_validities<'a>( diff --git a/rustfs/src/server/tls_publish.rs b/rustfs/src/server/tls_publish.rs index ab11787e9..afd6dc2fa 100644 --- a/rustfs/src/server/tls_publish.rs +++ b/rustfs/src/server/tls_publish.rs @@ -1148,7 +1148,7 @@ mod tests { let snapshot = generation .into_snapshot(&TlsSource::from_directory(base.path()).snapshot().expect("source")) .expect("the pinned operation consumes its already verified material without a second file read"); - assert_eq!(snapshot_leaf(&snapshot), Some(receipt.certificate_fingerprint.clone())); + assert_eq!(snapshot_leaf(&snapshot), Some(receipt.certificate_fingerprint)); assert!( matches!(state_at(root.path(), path), Err(PublishError::Invalid)), "the next operation must revalidate disk" From 0018bbf5d78f9aef1bc2daac74fc5eedb95057be Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Mon, 12 Oct 2026 16:13:36 +0800 Subject: [PATCH 024/143] ci: guard OIDC architecture boundaries (#8447) * ci: guard OIDC architecture boundaries * fix(ci): guard OIDC session ownership * fix(ci): ignore OIDC authorization destructuring * fix(ci): accept typed authorization destructuring --------- Co-authored-by: overtrue --- .github/workflows/oidc-keycloak.yml | 2 + docs/architecture/crate-boundaries.md | 8 + docs/architecture/runtime-lifecycle.md | 6 + .../architecture-guard-troubleshooting.md | 10 + scripts/check_architecture_migration_rules.sh | 6 + scripts/check_oidc_architecture_boundaries.sh | 365 +++++++++++++++++ .../oidc/cases.json | 381 ++++++++++++++++++ 7 files changed, 778 insertions(+) create mode 100755 scripts/check_oidc_architecture_boundaries.sh create mode 100644 scripts/fixtures/architecture_migration_rules/oidc/cases.json diff --git a/.github/workflows/oidc-keycloak.yml b/.github/workflows/oidc-keycloak.yml index f32287fb2..f40f77859 100644 --- a/.github/workflows/oidc-keycloak.yml +++ b/.github/workflows/oidc-keycloak.yml @@ -22,6 +22,7 @@ on: - "crates/iam/src/federation/**" - "crates/iam/src/oidc/**" - "rustfs/src/admin/handlers/oidc.rs" + - "rustfs/src/admin/service/oidc_config.rs" - "rustfs/src/admin/handlers/sts.rs" - "scripts/test/oidc_keycloak_live.sh" - "scripts/test/fixtures/keycloak-rustfs-ci-realm.json" @@ -33,6 +34,7 @@ on: - "crates/iam/src/federation/**" - "crates/iam/src/oidc/**" - "rustfs/src/admin/handlers/oidc.rs" + - "rustfs/src/admin/service/oidc_config.rs" - "rustfs/src/admin/handlers/sts.rs" - "scripts/test/oidc_keycloak_live.sh" - "scripts/test/fixtures/keycloak-rustfs-ci-realm.json" diff --git a/docs/architecture/crate-boundaries.md b/docs/architecture/crate-boundaries.md index 09a308d8f..7a37c707f 100644 --- a/docs/architecture/crate-boundaries.md +++ b/docs/architecture/crate-boundaries.md @@ -81,6 +81,14 @@ Every temporary compatibility path carries a `RUSTFS_COMPAT_TODO()` source m The server-config model (`Config`, `KV`, `KVS`) and the global server-config snapshot accessors are owned by `rustfs_config::server_config`; ECStore keeps persistence, storage-class state, and startup wiring, and its public facades must not re-export those symbols. See [config-model-boundary-adr.md](config-model-boundary-adr.md). +## OIDC Ownership + +`crates/iam/src/oidc/` owns provider configuration, discovery, HTTP transport, login state, and the OIDC runtime. `crates/iam/src/federation/` owns verified identity transactions and authorization mapping; its production code does not import concrete OIDC runtime or provider configuration types. Production `FederatedAuthorization` values are constructed by `CoreFederatedAuthorizationMapper` in `crates/iam/src/federation/mapper.rs`. + +`rustfs/src/startup_auth.rs` constructs `StandardOidcAdapter` and publishes the federation service with its OIDC query as one runtime pair. `AppContext` stores and publishes that pair through `FederatedIdentityRuntimeSnapshot`; consumers receive interfaces rather than constructing a second OIDC runtime. `scripts/check_oidc_architecture_boundaries.sh` enforces these boundaries through the existing architecture guard, including the Keycloak workflow path for `crates/iam/src/oidc/`. + +`crates/iam/src/oidc/config.rs` owns OIDC configuration input, validation, persisted KVS conversion, and `OidcSiteReplicationSnapshot`. `rustfs/src/admin/service/oidc_config.rs` owns stored configuration reads, compare-and-save updates, restart comparison, and provider discovery validation; the OIDC Admin handler translates HTTP requests and errors. `OidcConfigQuery::site_replication_snapshot()` returns only the provider ID, claim name, role policy, client ID, and hashed client secret needed by site replication. `rustfs/src/admin/handlers/site_replication.rs` maps this narrow view to its existing response format without reading raw OIDC credentials. The Keycloak workflow tracks the Admin configuration service path along with IAM OIDC code. + ## Required Architecture Documents The guard requires the documents and section headings listed in its `require_source_contains` entries (`scripts/check_architecture_migration_rules.sh`); the directory index is [README.md](README.md). diff --git a/docs/architecture/runtime-lifecycle.md b/docs/architecture/runtime-lifecycle.md index 9204a21ad..62ef88bcb 100644 --- a/docs/architecture/runtime-lifecycle.md +++ b/docs/architecture/runtime-lifecycle.md @@ -55,3 +55,9 @@ Embedded startup reuses the same phase owners as the binary: server and storage ## AppContext Foundation AppContext is a context-first facade, not a full replacement for every process global. Resolver files are split and covered by compatibility tests before boot extraction or consumer migration, so the old global fallback path keeps working during transition. New migration work keeps fallback reads inside owner-local runtime-source boundaries and follows the target inventory in [global-state-inventory.md](global-state-inventory.md). + +## OIDC Runtime Publication + +`startup_auth.rs` builds the OIDC runtime and publishes its `FederatedIdentityService` and `OidcConfigQuery` together. `FederatedIdentityRuntimeSnapshot` in `app/context/interfaces.rs` and the owning handle in `app/context/handles.rs` retain this pair. Authentication and configuration consumers resolve their respective interfaces from the same published runtime; the OIDC runtime is created only at IAM startup construction and this startup phase. + +The query exposes `site_replication_snapshot()` for active, narrow replication settings. Admin configuration changes use `admin/service/oidc_config.rs` to read and save persisted configuration and report whether a restart is required; they do not replace the active runtime pair during the request. diff --git a/docs/operations/architecture-guard-troubleshooting.md b/docs/operations/architecture-guard-troubleshooting.md index e70a16107..be23906f8 100644 --- a/docs/operations/architecture-guard-troubleshooting.md +++ b/docs/operations/architecture-guard-troubleshooting.md @@ -34,6 +34,16 @@ source lines. If it fails after a doc edit, you reworded or removed a guarded line — restore the wording or update the script deliberately in the same PR, with rationale. +## OIDC architecture boundaries + +`check_architecture_migration_rules.sh` also runs `scripts/check_oidc_architecture_boundaries.sh`. Its rule identifier and `file:line` point to the source boundary that changed. Keep OIDC provider configuration, discovery, transport, state, and runtime under `crates/iam/src/oidc/`; keep verified identity mapping under `crates/iam/src/federation/mapper.rs`; construct the runtime in IAM startup and `rustfs/src/startup_auth.rs`; publish the federation service and OIDC query together through `AppContext`. The Keycloak workflow must include the OIDC module path for pull requests and pushes. + +OIDC production modules must not reference RustFS `Credentials` or `FederatedSessionTransaction`. Federation orchestration creates the transaction, and the Admin session binding issues credentials. + +The Admin OIDC handler delegates persisted configuration reads, updates, and provider validation to `admin/service/oidc_config.rs`. Site replication uses `OidcConfigQuery::site_replication_snapshot()` and receives only the fields required for its response, including a hashed client secret. The Keycloak workflow also tracks the Admin configuration service path for both events. + +Run `bash scripts/check_oidc_architecture_boundaries.sh --self-test` after editing a rule. Its negative and accepted examples under `scripts/fixtures/architecture_migration_rules/oidc/` must produce the exact expected rule identifiers. Then run the full architecture guard. + ## `check_unsafe_code_allowances.sh` Every `#[allow(unsafe_code)]` needs a `SAFETY:` comment within a few lines. diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 8f3983269..f0b9fad84 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -59,12 +59,14 @@ require_source_contains "docs/architecture/overview.md" "## Phase Order" "archit require_source_contains "docs/architecture/runtime-lifecycle.md" "## Startup And Readiness" "runtime lifecycle startup readiness section" require_source_contains "docs/architecture/runtime-lifecycle.md" "## Shutdown Lifecycle Boundary" "runtime lifecycle shutdown boundary section" require_source_contains "docs/architecture/runtime-lifecycle.md" "## AppContext Foundation" "runtime lifecycle AppContext section" +require_source_contains "docs/architecture/runtime-lifecycle.md" "## OIDC Runtime Publication" "runtime lifecycle OIDC publication section" require_source_contains "docs/architecture/storage-control-data-plane.md" "## Storage API Contracts" "storage control/data plane contracts section" require_source_contains "docs/architecture/storage-control-data-plane.md" "## Cluster Control Plane" "storage control/data plane cluster section" require_source_contains "docs/architecture/storage-control-data-plane.md" "## Background Controllers" "storage control/data plane background controllers section" require_source_contains "docs/architecture/crate-boundaries.md" "## Dependency Direction" "crate boundaries dependency direction section" require_source_contains "docs/architecture/crate-boundaries.md" "storage-api -> ecstore" "crate boundaries storage dependency rule" require_source_contains "docs/architecture/crate-boundaries.md" "extension-schema -> rustfs" "crate boundaries extension dependency rule" +require_source_contains "docs/architecture/crate-boundaries.md" "## OIDC Ownership" "crate boundaries OIDC ownership section" require_source_contains "docs/architecture/readiness-matrix.md" "## Request Behavior Matrix" "readiness matrix request behavior section" require_source_contains "docs/architecture/readiness-matrix.md" "## Runtime Dependency Matrix" "readiness matrix runtime dependency section" require_source_contains "docs/architecture/readiness-matrix.md" "## Probe Semantics" "readiness matrix probe semantics section" @@ -5544,6 +5546,10 @@ else done <"$ECSTORE_LINT_EXPECTED" fi +if ! bash "${ROOT_DIR}/scripts/check_oidc_architecture_boundaries.sh"; then + report_failure "OIDC architecture boundary check" +fi + if (( FAILURES > 0 )); then exit 1 fi diff --git a/scripts/check_oidc_architecture_boundaries.sh b/scripts/check_oidc_architecture_boundaries.sh new file mode 100755 index 000000000..b251492a7 --- /dev/null +++ b/scripts/check_oidc_architecture_boundaries.sh @@ -0,0 +1,365 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +exec python3 - "$ROOT_DIR" "$@" <<'PY' +import json +import re +import sys +from pathlib import Path + + +root = Path(sys.argv[1]) +arguments = sys.argv[2:] +if arguments not in ([], ["--self-test"]): + raise SystemExit("usage: check_oidc_architecture_boundaries.sh [--self-test]") + +fixture_file = root / "scripts/fixtures/architecture_migration_rules/oidc/cases.json" +raw_string = re.compile(r'r(#+)?"') +char_literal = re.compile(r"'(?:\\.|[^'\\\n])'") +required_modules = ( + "crates/iam/src/oidc/mod.rs", + "crates/iam/src/oidc/config.rs", + "crates/iam/src/oidc/provider.rs", + "crates/iam/src/oidc/runtime.rs", + "crates/iam/src/oidc/state.rs", + "crates/iam/src/oidc/transport.rs", +) +removed_modules = ( + "crates/iam/src/oidc.rs", + "crates/iam/src/oidc_state.rs", + "crates/iam/src/federation/registry.rs", + "crates/iam/src/federation/oidc", +) +admin_config_service = "rustfs/src/admin/service/oidc_config.rs" +site_replication_handler = "rustfs/src/admin/handlers/site_replication.rs" +admin_oidc_handler = "rustfs/src/admin/handlers/oidc.rs" +admin_idp_compat_handler = "rustfs/src/admin/handlers/idp_compat.rs" + + +def mask(source, start, end): + return "".join("\n" if character == "\n" else " " for character in source[start:end]) + + +def code_only(source): + """Preserve offsets while ignoring Rust comments, strings, and test items.""" + output = list(source) + index = 0 + while index < len(source): + start = index + if source.startswith("//", index): + end = source.find("\n", index) + index = len(source) if end < 0 else end + elif source.startswith("/*", index): + depth = 1 + index += 2 + while index < len(source) and depth: + if source.startswith("/*", index): + depth += 1 + index += 2 + elif source.startswith("*/", index): + depth -= 1 + index += 2 + else: + index += 1 + else: + raw = raw_string.match(source, index) + if raw: + delimiter = '"' + (raw.group(1) or "") + end = source.find(delimiter, index + len(raw.group())) + index = len(source) if end < 0 else end + len(delimiter) + elif source[index] == '"': + index += 1 + while index < len(source): + if source[index] == "\\": + index += 2 + elif source[index] == '"': + index += 1 + break + else: + index += 1 + elif source[index] == "'" and (literal := char_literal.match(source, index)): + index = literal.end() + else: + index += 1 + continue + output[start:index] = mask(source, start, index) + + production = "".join(output) + test_item = re.compile( + r"#\s*\[\s*cfg\s*\(\s*test\s*\)\s*\]\s*(?:#\s*\[[^]]*\]\s*)*" + r"(?:pub(?:\([^)]*\))?\s+)?(?:async\s+)?(?:mod|fn|impl|use|struct|enum|trait|const|static)\b" + ) + search_from = 0 + while match := test_item.search(production, search_from): + delimiter = re.search(r"[;{]", production[match.end():]) + if not delimiter: + break + end = match.end() + delimiter.end() + if delimiter.group() == "{": + depth = 1 + while end < len(production) and depth: + if production[end] == "{": + depth += 1 + elif production[end] == "}": + depth -= 1 + end += 1 + production = production[: match.start()] + mask(production, match.start(), end) + production[end:] + search_from = end + return production + + +def find_matches(rule, path, source, pattern): + return [ + (rule, path, source.count("\n", 0, match.start()) + 1) + for match in re.finditer(pattern, source, re.MULTILINE | re.DOTALL) + ] + + +def is_struct_pattern(source, opening_brace): + depth = 1 + index = opening_brace + 1 + while index < len(source) and depth: + if source[index] == "{": + depth += 1 + elif source[index] == "}": + depth -= 1 + index += 1 + if depth: + return False + suffix = source[index:].lstrip() + typed_let_pattern = suffix.startswith(":") and re.search( + r"\blet\s+(?:\w+\s*::\s*)*FederatedAuthorization\s*$", source[:opening_brace] + ) + return bool(typed_let_pattern) or suffix.startswith("=>") or (suffix.startswith("=") and not suffix.startswith("==")) + + +def function_body(source, name): + signature = re.search(rf"\bfn\s+{name}\s*\([^)]*\)[^{{]*\{{", source) + if not signature: + return "" + depth = 1 + end = signature.end() + while end < len(source) and depth: + if source[end] == "{": + depth += 1 + elif source[end] == "}": + depth -= 1 + end += 1 + return source[signature.end(): end - 1] + + +def workflow_event_paths(source, event): + section = re.search(rf"^ {event}:\s*\n(.*?)(?=^ [a-z_]+:|\Z)", source, re.MULTILINE | re.DOTALL) + if not section: + return set() + paths = set() + in_paths = False + for line in section.group(1).splitlines(): + if re.fullmatch(r" paths:\s*", line): + in_paths = True + continue + if not in_paths: + continue + if not line.strip() or line.lstrip().startswith("#"): + continue + entry = re.fullmatch(r" -\s+(?:[\"']([^\"']+)[\"']|([^\s#]+))\s*(?:#.*)?", line) + if not entry: + break + paths.add(entry.group(1) or entry.group(2)) + return paths + + +def source_findings(path, source): + findings = [] + if path in removed_modules or path.startswith("crates/iam/src/federation/oidc/"): + return [("oidc-module-owner", path, 1)] + if path == ".github/workflows/oidc-keycloak.yml": + expected_paths = {"crates/iam/src/oidc/**", admin_config_service} + if any(not expected_paths.issubset(workflow_event_paths(source, event)) for event in ("pull_request", "push")): + findings.append(("oidc-live-workflow", path, 1)) + return findings + if not path.endswith(".rs") or path.endswith("_tests.rs") or path.endswith("test_support.rs") or "/tests/" in path: + return findings + if not (path.startswith("crates/iam/src/") or path.startswith("rustfs/src/")): + return findings + + code = code_only(source) + findings += find_matches( + "oidc-legacy-runtime", path, code, + r"\b(?:FederatedIdentityRegistry|FederatedIdentityProvider|FederatedCodeExchange|OIDC_SYS|get_oidc|init_oidc_sys|init_oidc_sys_with_extra_root_ca_provider)\b", + ) + + if path.startswith("crates/iam/src/federation/"): + findings += find_matches( + "federation-oidc-detail", path, code, + r"\b(?:crate|super)::oidc\b|\buse\b[^;]*\boidc\b[^;]*;|\boidc::|\b(?:OidcSys|OidcClaims|OidcStateStore|OidcProviderConfig|OidcConfigSnapshot|OidcExtraRootCaProvider|ProviderRuntime|ReqwestHttpClient|openidconnect|reqwest)\b", + ) + + if path.startswith("crates/iam/src/oidc/"): + findings += find_matches( + "oidc-iam-session-boundary", path, code, + r"\b(?:Credentials|FederatedSessionTransaction)\b", + ) + + if path.startswith("crates/iam/src/") and path != "crates/iam/src/lib.rs" and not path.startswith( + ("crates/iam/src/oidc/", "crates/iam/src/federation/") + ): + findings += find_matches( + "oidc-runtime-owner", path, code, + r"\b(?:OidcSys|StandardOidcAdapter|build_oidc_sys_with_extra_root_ca_provider)\b", + ) + + if path == "crates/iam/src/oidc/config.rs": + findings += find_matches( + "oidc-config-direction", path, code, + r"\b(?:provider|runtime|state|transport)::|\b(?:ProviderRuntime|OidcSys|OidcStateStore|ReqwestHttpClient|CoreClient)\b", + ) + query = re.search(r"pub\s+trait\s+OidcConfigQuery\b[^\{]*\{([^}]*)\}", code, re.DOTALL) + narrow_query = r"\s*fn\s+site_replication_snapshot\s*\(\s*&self\s*\)\s*->\s*OidcSiteReplicationSnapshot\s*;\s*" + if not query or not re.fullmatch(narrow_query, query.group(1)): + findings.append(("oidc-query-shape", path, 1)) + provider = re.search(r"pub\s+struct\s+OidcSiteReplicationProvider\s*\{([^}]*)\}", code, re.DOTALL) + provider_shape = ( + r"\s*pub\s+provider_id\s*:\s*String\s*," + r"\s*pub\s+claim_name\s*:\s*String\s*," + r"\s*pub\s+role_policy\s*:\s*String\s*," + r"\s*pub\s+client_id\s*:\s*String\s*," + r"\s*pub\s+hashed_client_secret\s*:\s*String\s*,?\s*" + ) + if not provider or not re.fullmatch(provider_shape, provider.group(1)): + findings.append(("oidc-site-replication-shape", path, 1)) + snapshot = re.search(r"pub\s+struct\s+OidcSiteReplicationSnapshot\s*\{([^}]*)\}", code, re.DOTALL) + if not snapshot or not re.fullmatch(r"\s*providers\s*:\s*Vec\s*<\s*OidcSiteReplicationProvider\s*>\s*,?\s*", snapshot.group(1)): + findings.append(("oidc-site-replication-shape", path, 1)) + + if path.startswith("rustfs/src/") and path != "rustfs/src/startup_auth.rs": + findings += find_matches( + "oidc-runtime-owner", path, code, + r"\b(?:OidcSys|StandardOidcAdapter|build_oidc_sys_with_extra_root_ca_provider)\b", + ) + + if path == admin_oidc_handler: + findings += find_matches( + "oidc-admin-config-owner", path, code, + r"\b(?:read_admin_config_without_migrate|read_admin_server_config_snapshot|save_admin_server_config_snapshot|current_object_store_handle_for_context|ServerConfig|KVS|build_upsert_provider_config|upsert_persisted_provider_config|delete_persisted_provider_config|persisted_provider_secret|validate_oidc_provider_config_with_extra_root_ca|load_oidc_config_snapshot|current_oidc_extra_root_ca_material)\b", + ) + if path == admin_idp_compat_handler: + findings += find_matches( + "oidc-admin-config-owner", path, code, + r"\b(?:read_admin_config_without_migrate|read_admin_server_config_snapshot|save_admin_server_config_snapshot|current_object_store_handle_for_context|build_upsert_provider_config|upsert_persisted_provider_config|delete_persisted_provider_config|persisted_provider_secret|validate_oidc_provider_config_with_extra_root_ca|current_oidc_extra_root_ca_material)\b", + ) + if path == admin_config_service: + findings += find_matches("oidc-admin-service-layer", path, code, r"\b(?:s3s|S3Error|S3Result|s3_error)\b") + for operation in ("list_config", "upsert_config", "delete_config", "validate_config"): + if not re.search(rf"pub\(crate\)\s+async\s+fn\s+{operation}\s*\(", code): + findings.append(("oidc-admin-config-owner", path, 1)) + for storage_operation in ("read_admin_server_config_snapshot", "save_admin_server_config_snapshot"): + if not re.search(rf"\b{storage_operation}\s*\(", code): + findings.append(("oidc-admin-config-owner", path, 1)) + if path == site_replication_handler: + findings += find_matches( + "oidc-site-replication-consumer", path, code, + r"\b(?:OidcConfigSnapshot|OidcProviderConfig|load_oidc_config_snapshot)\b", + ) + oidc_mapping = function_body(code, "open_id_settings_from_snapshot") + if not oidc_mapping or re.search(r"\b(?:hash_client_secret|Sha256)\b|\.\s*(?:config_snapshot|client_secret)\b", oidc_mapping): + findings.append(("oidc-site-replication-consumer", path, 1)) + if not re.search(r"\.\s*site_replication_snapshot\s*\(", code): + findings.append(("oidc-site-replication-consumer", path, 1)) + + for match in re.finditer(r"\bFederatedAuthorization\s*\{", code): + preceding_word = re.search(r"\b(\w+)\s*$", code[: match.start()]) + if preceding_word and preceding_word.group(1) in ("struct", "impl"): + continue + if is_struct_pattern(code, match.end() - 1): + continue + if path != "crates/iam/src/federation/mapper.rs": + findings.append(("federation-authorization-owner", path, code.count("\n", 0, match.start()) + 1)) + + if path == "rustfs/src/app/context/interfaces.rs": + paired_snapshot = ( + r"pub\s+type\s+FederatedIdentityRuntimeSnapshot\s*=\s*\(\s*" + r"Arc\s*<\s*FederatedIdentityService\s*>\s*,\s*" + r"Arc\s*<\s*dyn\s+OidcConfigQuery\s*>\s*\)\s*;" + ) + if not re.search(paired_snapshot, code): + findings.append(("oidc-runtime-pair", path, 1)) + if path == "rustfs/src/app/context/handles.rs": + runtime = re.search(r"struct\s+FederatedIdentityRuntime\s*\{([^}]*)\}", code, re.DOTALL) + if not runtime or not re.search(r"\bservice\s*:\s*Arc\s*<\s*FederatedIdentityService\s*>", runtime.group(1)) or not re.search( + r"\boidc_config_query\s*:\s*Arc\s*<\s*dyn\s+OidcConfigQuery\s*>", runtime.group(1) + ): + findings.append(("oidc-runtime-pair", path, 1)) + return findings + + +def layout_findings(exists=None, service_modules=None): + if exists is None: + exists = lambda path: (root / path).exists() + if service_modules is None: + service_modules = (root / "rustfs/src/admin/service/mod.rs").read_text() + findings = [] + for path in required_modules: + if not exists(path): + findings.append(("oidc-module-owner", path, 1)) + for path in removed_modules: + if exists(path): + findings.append(("oidc-module-owner", path, 1)) + if not exists(admin_config_service): + findings.append(("oidc-admin-config-owner", admin_config_service, 1)) + if not re.search(r"\bmod\s+oidc_config\s*;", service_modules): + findings.append(("oidc-admin-config-owner", "rustfs/src/admin/service/mod.rs", 1)) + return findings + + +def fixture_findings(): + cases = json.loads(fixture_file.read_text()) + actual = [] + expected = [] + for case in cases: + case_id = case["id"] + expected.extend(f"{rule}|{case_id}" for rule in case["expected"]) + if "layout" in case: + layout = case["layout"] + present = set(required_modules) | {admin_config_service} + present.difference_update(layout.get("missing", [])) + present.update(layout.get("restored", [])) + findings = layout_findings(present.__contains__, layout.get("service_modules", "mod oidc_config;")) + else: + findings = source_findings(case["path"], case["source"]) + actual.extend(f"{rule}|{case_id}" for rule, _, _ in findings) + return sorted(set(actual)), sorted(set(expected)) + + +def repository_findings(): + findings = layout_findings() + for base in ("crates/iam/src", "rustfs/src"): + for file in sorted((root / base).rglob("*.rs")): + path = file.relative_to(root).as_posix() + findings.extend(source_findings(path, file.read_text())) + workflow = ".github/workflows/oidc-keycloak.yml" + findings.extend(source_findings(workflow, (root / workflow).read_text())) + return findings + + +actual, expected = fixture_findings() +if actual != expected: + print("OIDC architecture guard fixture mismatch:", file=sys.stderr) + print(f" expected: {expected}", file=sys.stderr) + print(f" actual: {actual}", file=sys.stderr) + raise SystemExit(1) + +if arguments == ["--self-test"]: + print("OIDC architecture guard fixtures passed.") + raise SystemExit(0) + +findings = repository_findings() +for rule, path, line in findings: + print(f"OIDC architecture boundary failed: {rule}: {path}:{line}", file=sys.stderr) +if findings: + raise SystemExit(1) +print("OIDC architecture boundaries passed.") +PY diff --git a/scripts/fixtures/architecture_migration_rules/oidc/cases.json b/scripts/fixtures/architecture_migration_rules/oidc/cases.json new file mode 100644 index 000000000..9c9772527 --- /dev/null +++ b/scripts/fixtures/architecture_migration_rules/oidc/cases.json @@ -0,0 +1,381 @@ +[ + { + "id": "legacy_registry", + "path": "rustfs/src/admin/handlers/sts.rs", + "source": "let registry = FederatedIdentityRegistry::new(provider);\n", + "expected": ["oidc-legacy-runtime"] + }, + { + "id": "legacy_global_initializer", + "path": "rustfs/src/startup_auth.rs", + "source": "init_oidc_sys_with_extra_root_ca_provider(provider).await?;\n", + "expected": ["oidc-legacy-runtime"] + }, + { + "id": "federation_imports_oidc_runtime", + "path": "crates/iam/src/federation/transaction.rs", + "source": "use crate::oidc::OidcSys;\n", + "expected": ["federation-oidc-detail"] + }, + { + "id": "federation_imports_oidc_module", + "path": "crates/iam/src/federation/provider.rs", + "source": "use crate::oidc::config::OidcProviderConfig;\n", + "expected": ["federation-oidc-detail"] + }, + { + "id": "federation_aliases_oidc_module", + "path": "crates/iam/src/federation/provider.rs", + "source": "use crate::{oidc as oidc_impl};\n", + "expected": ["federation-oidc-detail"] + }, + { + "id": "federation_aliases_oidc_after_nested_import", + "path": "crates/iam/src/federation/provider.rs", + "source": "use crate::{other::{Foo}, oidc as oidc_impl};\n", + "expected": ["federation-oidc-detail"] + }, + { + "id": "federation_constructs_authorization", + "path": "crates/iam/src/federation/transaction.rs", + "source": "let authorization = FederatedAuthorization { policies, groups };\n", + "expected": ["federation-authorization-owner"] + }, + { + "id": "accepted_federation_destructures_authorization", + "path": "crates/iam/src/federation/transaction.rs", + "source": "let FederatedAuthorization { policies, groups, .. } = &authorization;\n", + "expected": [] + }, + { + "id": "accepted_federation_typed_borrow_pattern", + "path": "crates/iam/src/federation/transaction.rs", + "source": "let FederatedAuthorization { policies, groups, .. }: &FederatedAuthorization = &authorization;\n", + "expected": [] + }, + { + "id": "accepted_federation_qualified_typed_borrow_pattern", + "path": "crates/iam/src/federation/transaction.rs", + "source": "let crate::federation::FederatedAuthorization { policies, groups, .. }: &crate::federation::FederatedAuthorization = &authorization;\n", + "expected": [] + }, + { + "id": "federation_constructs_typed_authorization", + "path": "crates/iam/src/federation/transaction.rs", + "source": "let authorization: FederatedAuthorization = FederatedAuthorization { policies, groups };\n", + "expected": ["federation-authorization-owner"] + }, + { + "id": "accepted_federation_matches_authorization", + "path": "crates/iam/src/federation/transaction.rs", + "source": "match authorization { FederatedAuthorization { policies, .. } => log(policies) }\n", + "expected": [] + }, + { + "id": "federation_constructs_authorization_for_comparison", + "path": "crates/iam/src/federation/transaction.rs", + "source": "let matches = FederatedAuthorization { policies, groups } == expected;\n", + "expected": ["federation-authorization-owner"] + }, + { + "id": "oidc_adapter_imports_credentials", + "path": "crates/iam/src/oidc/runtime.rs", + "source": "use rustfs_credentials::Credentials;\n", + "expected": ["oidc-iam-session-boundary"] + }, + { + "id": "oidc_adapter_constructs_qualified_credentials", + "path": "crates/iam/src/oidc/runtime.rs", + "source": "let credentials = rustfs_credentials::Credentials { access_key };\n", + "expected": ["oidc-iam-session-boundary"] + }, + { + "id": "oidc_adapter_imports_binding_transaction", + "path": "crates/iam/src/oidc/runtime.rs", + "source": "use crate::federation::FederatedSessionTransaction as BindingRequest;\n", + "expected": ["oidc-iam-session-boundary"] + }, + { + "id": "oidc_adapter_constructs_binding_transaction", + "path": "crates/iam/src/oidc/runtime.rs", + "source": "let transaction = FederatedSessionTransaction { authorization, duration_seconds, session_policy };\n", + "expected": ["oidc-iam-session-boundary"] + }, + { + "id": "oidc_adapter_test_module_uses_credentials", + "path": "crates/iam/src/oidc/runtime.rs", + "source": "#[cfg(test)]\nmod tests { use rustfs_credentials::Credentials; }\n", + "expected": [] + }, + { + "id": "accepted_federation_transaction_owner", + "path": "crates/iam/src/federation/transaction.rs", + "source": "let transaction = FederatedSessionTransaction { authorization, duration_seconds, session_policy };\n", + "expected": [] + }, + { + "id": "accepted_binding_credentials_owner", + "path": "rustfs/src/admin/service/federated_identity.rs", + "source": "let credentials = Credentials { access_key };\n", + "expected": [] + }, + { + "id": "config_imports_provider_runtime", + "path": "crates/iam/src/oidc/config.rs", + "source": "pub struct OidcSiteReplicationProvider { pub provider_id: String, pub claim_name: String, pub role_policy: String, pub client_id: String, pub hashed_client_secret: String }\npub struct OidcSiteReplicationSnapshot { providers: Vec }\npub trait OidcConfigQuery: Send + Sync { fn site_replication_snapshot(&self) -> OidcSiteReplicationSnapshot; }\nuse super::provider::ProviderRuntime;\n", + "expected": ["oidc-config-direction"] + }, + { + "id": "broad_oidc_config_query", + "path": "crates/iam/src/oidc/config.rs", + "source": "pub struct OidcSiteReplicationProvider { pub provider_id: String, pub claim_name: String, pub role_policy: String, pub client_id: String, pub hashed_client_secret: String }\npub struct OidcSiteReplicationSnapshot { providers: Vec }\npub trait OidcConfigQuery: Send + Sync { fn config_snapshot(&self) -> OidcConfigSnapshot; }\n", + "expected": ["oidc-query-shape"] + }, + { + "id": "replication_view_contains_raw_secret", + "path": "crates/iam/src/oidc/config.rs", + "source": "pub struct OidcSiteReplicationProvider { pub provider_id: String, pub claim_name: String, pub role_policy: String, pub client_id: String, pub hashed_client_secret: String, pub client_secret: String }\npub struct OidcSiteReplicationSnapshot { providers: Vec }\npub trait OidcConfigQuery: Send + Sync { fn site_replication_snapshot(&self) -> OidcSiteReplicationSnapshot; }\n", + "expected": ["oidc-site-replication-shape"] + }, + { + "id": "replication_view_contains_private_secret", + "path": "crates/iam/src/oidc/config.rs", + "source": "pub struct OidcSiteReplicationProvider { pub provider_id: String, pub claim_name: String, pub role_policy: String, pub client_id: String, pub hashed_client_secret: String, client_secret: String }\npub struct OidcSiteReplicationSnapshot { providers: Vec }\npub trait OidcConfigQuery: Send + Sync { fn site_replication_snapshot(&self) -> OidcSiteReplicationSnapshot; }\n", + "expected": ["oidc-site-replication-shape"] + }, + { + "id": "query_adds_full_config_method", + "path": "crates/iam/src/oidc/config.rs", + "source": "pub struct OidcSiteReplicationProvider { pub provider_id: String, pub claim_name: String, pub role_policy: String, pub client_id: String, pub hashed_client_secret: String }\npub struct OidcSiteReplicationSnapshot { providers: Vec }\npub trait OidcConfigQuery: Send + Sync { fn site_replication_snapshot(&self) -> OidcSiteReplicationSnapshot; fn raw_snapshot(&self) -> OidcConfigSnapshot; }\n", + "expected": ["oidc-query-shape"] + }, + { + "id": "admin_constructs_runtime", + "path": "rustfs/src/admin/handlers/oidc.rs", + "source": "let adapter = StandardOidcAdapter::new(oidc);\n", + "expected": ["oidc-runtime-owner"] + }, + { + "id": "admin_builds_runtime", + "path": "rustfs/src/admin/handlers/oidc.rs", + "source": "let oidc = build_oidc_sys_with_extra_root_ca_provider(ca).await?;\n", + "expected": ["oidc-runtime-owner"] + }, + { + "id": "iam_sys_builds_oidc_runtime", + "path": "crates/iam/src/sys/mod.rs", + "source": "let oidc = OidcSys::empty()?;\n", + "expected": ["oidc-runtime-owner"] + }, + { + "id": "handler_reads_config_storage", + "path": "rustfs/src/admin/handlers/oidc.rs", + "source": "let snapshot = read_admin_server_config_snapshot(store).await?;\n", + "expected": ["oidc-admin-config-owner"] + }, + { + "id": "handler_validates_provider_directly", + "path": "rustfs/src/admin/handlers/oidc.rs", + "source": "let result = validate_oidc_provider_config_with_extra_root_ca(&provider, ca).await?;\n", + "expected": ["oidc-admin-config-owner"] + }, + { + "id": "compat_handler_saves_oidc_config", + "path": "rustfs/src/admin/handlers/idp_compat.rs", + "source": "save_admin_server_config_snapshot(store, &config, &snapshot).await?;\n", + "expected": ["oidc-admin-config-owner"] + }, + { + "id": "service_imports_http_error", + "path": "rustfs/src/admin/service/oidc_config.rs", + "source": "use s3s::S3Error;\npub(crate) async fn list_config() {}\npub(crate) async fn upsert_config() {}\npub(crate) async fn delete_config() {}\npub(crate) async fn validate_config() {}\nfn storage() { read_admin_server_config_snapshot(store); save_admin_server_config_snapshot(store); }\n", + "expected": ["oidc-admin-service-layer"] + }, + { + "id": "service_omits_config_write", + "path": "rustfs/src/admin/service/oidc_config.rs", + "source": "pub(crate) async fn list_config() {}\npub(crate) async fn upsert_config() {}\npub(crate) async fn delete_config() {}\npub(crate) async fn validate_config() {}\nfn storage() { read_admin_server_config_snapshot(store); }\n", + "expected": ["oidc-admin-config-owner"] + }, + { + "id": "replication_reads_full_config", + "path": "rustfs/src/admin/handlers/site_replication.rs", + "source": "let providers = query.config_snapshot().providers();\n", + "expected": ["oidc-site-replication-consumer"] + }, + { + "id": "replication_reads_raw_secret", + "path": "rustfs/src/admin/handlers/site_replication.rs", + "source": "fn open_id_settings_from_snapshot(snapshot: &OidcSiteReplicationSnapshot) -> OpenIDSettings { let secret = provider.client_secret.clone(); }\nlet snapshot = query.site_replication_snapshot();\n", + "expected": ["oidc-site-replication-consumer"] + }, + { + "id": "single_runtime_snapshot", + "path": "rustfs/src/app/context/interfaces.rs", + "source": "pub type FederatedIdentityRuntimeSnapshot = Arc;\n", + "expected": ["oidc-runtime-pair"] + }, + { + "id": "single_runtime_handle", + "path": "rustfs/src/app/context/handles.rs", + "source": "struct FederatedIdentityRuntime { service: Arc }\n", + "expected": ["oidc-runtime-pair"] + }, + { + "id": "old_single_file", + "path": "crates/iam/src/oidc.rs", + "source": "pub struct OidcSys;\n", + "expected": ["oidc-module-owner"] + }, + { + "id": "old_federation_directory", + "path": "crates/iam/src/federation/oidc/flow.rs", + "source": "pub fn exchange_code() {}\n", + "expected": ["oidc-module-owner"] + }, + { + "id": "missing_oidc_owner_module", + "layout": { "missing": ["crates/iam/src/oidc/provider.rs"] }, + "expected": ["oidc-module-owner"] + }, + { + "id": "restored_legacy_module", + "layout": { "restored": ["crates/iam/src/federation/oidc"] }, + "expected": ["oidc-module-owner"] + }, + { + "id": "missing_admin_config_service", + "layout": { "missing": ["rustfs/src/admin/service/oidc_config.rs"] }, + "expected": ["oidc-admin-config-owner"] + }, + { + "id": "missing_admin_service_registration", + "layout": { "service_modules": "mod other;" }, + "expected": ["oidc-admin-config-owner"] + }, + { + "id": "missing_live_workflow_path", + "path": ".github/workflows/oidc-keycloak.yml", + "source": "on:\n pull_request:\n paths:\n - \"crates/iam/src/oidc/**\"\n - \"rustfs/src/admin/service/oidc_config.rs\"\n push:\n paths:\n - \"rustfs/src/admin/service/oidc_config.rs\"\n", + "expected": ["oidc-live-workflow"] + }, + { + "id": "service_path_missing_from_push", + "path": ".github/workflows/oidc-keycloak.yml", + "source": "on:\n pull_request:\n paths:\n - \"crates/iam/src/oidc/**\"\n - \"rustfs/src/admin/service/oidc_config.rs\"\n push:\n paths:\n - \"crates/iam/src/oidc/**\"\n", + "expected": ["oidc-live-workflow"] + }, + { + "id": "commented_workflow_path", + "path": ".github/workflows/oidc-keycloak.yml", + "source": "on:\n pull_request:\n paths:\n - \"crates/iam/src/oidc/**\"\n # - \"rustfs/src/admin/service/oidc_config.rs\"\n push:\n paths:\n - \"crates/iam/src/oidc/**\"\n - \"rustfs/src/admin/service/oidc_config.rs\"\n", + "expected": ["oidc-live-workflow"] + }, + { + "id": "workflow_path_in_paths_ignore", + "path": ".github/workflows/oidc-keycloak.yml", + "source": "on:\n pull_request:\n paths-ignore:\n - \"crates/iam/src/oidc/**\"\n - \"rustfs/src/admin/service/oidc_config.rs\"\n push:\n paths:\n - \"crates/iam/src/oidc/**\"\n - \"rustfs/src/admin/service/oidc_config.rs\"\n", + "expected": ["oidc-live-workflow"] + }, + { + "id": "comment_and_string_literals", + "path": "rustfs/src/admin/handlers/oidc.rs", + "source": "// StandardOidcAdapter and OIDC_SYS are inert here.\n/* FederatedIdentityRegistry /* OidcSys */ */\nlet message = r#\"FederatedCodeExchange StandardOidcAdapter\"#;\n", + "expected": [] + }, + { + "id": "test_module_exclusion_and_recovery", + "path": "rustfs/src/admin/handlers/oidc.rs", + "source": "#[cfg(test)]\nmod tests {\n fn test() { let _ = OidcSys::empty(); }\n}\nfn production() { let _ = OidcSys::empty(); }\n", + "expected": ["oidc-runtime-owner"] + }, + { + "id": "test_module_with_extra_attribute", + "path": "rustfs/src/admin/handlers/oidc.rs", + "source": "#[cfg(test)]\n#[allow(dead_code)]\nmod tests { fn test() { let _ = OidcSys::empty(); } }\n", + "expected": [] + }, + { + "id": "test_function_exclusion_and_recovery", + "path": "rustfs/src/app/context.rs", + "source": "#[cfg(test)]\nfn test_runtime() { let _ = OidcSys::empty(); }\nfn production() { let _ = OidcSys::empty(); }\n", + "expected": ["oidc-runtime-owner"] + }, + { + "id": "test_crate_visible_function", + "path": "rustfs/src/app/context.rs", + "source": "#[cfg(test)]\npub(crate) fn test_runtime() { let _ = OidcSys::empty(); }\n", + "expected": [] + }, + { + "id": "accepted_runtime_owner", + "path": "rustfs/src/startup_auth.rs", + "source": "let adapter = StandardOidcAdapter::new(oidc);\n", + "expected": [] + }, + { + "id": "accepted_iam_builder_owner", + "path": "crates/iam/src/lib.rs", + "source": "let oidc = OidcSys::empty()?;\n", + "expected": [] + }, + { + "id": "accepted_mapper_owner", + "path": "crates/iam/src/federation/mapper.rs", + "source": "return FederatedAuthorization { policies, groups };\n", + "expected": [] + }, + { + "id": "accepted_runtime_snapshot", + "path": "rustfs/src/app/context/interfaces.rs", + "source": "pub type FederatedIdentityRuntimeSnapshot = (Arc, Arc);\n", + "expected": [] + }, + { + "id": "accepted_runtime_handle", + "path": "rustfs/src/app/context/handles.rs", + "source": "struct FederatedIdentityRuntime { service: Arc, oidc_config_query: Arc }\n", + "expected": [] + }, + { + "id": "accepted_replication_query", + "path": "crates/iam/src/oidc/config.rs", + "source": "pub struct OidcSiteReplicationProvider { pub provider_id: String, pub claim_name: String, pub role_policy: String, pub client_id: String, pub hashed_client_secret: String }\npub struct OidcSiteReplicationSnapshot { providers: Vec }\npub trait OidcConfigQuery: Send + Sync { fn site_replication_snapshot(&self) -> OidcSiteReplicationSnapshot; }\n", + "expected": [] + }, + { + "id": "accepted_admin_service", + "path": "rustfs/src/admin/service/oidc_config.rs", + "source": "pub(crate) async fn list_config() {}\npub(crate) async fn upsert_config() {}\npub(crate) async fn delete_config() {}\npub(crate) async fn validate_config() {}\nfn storage() { read_admin_server_config_snapshot(store); save_admin_server_config_snapshot(store); }\n", + "expected": [] + }, + { + "id": "accepted_replication_consumer", + "path": "rustfs/src/admin/handlers/site_replication.rs", + "source": "fn open_id_settings_from_snapshot(snapshot: &OidcSiteReplicationSnapshot) -> OpenIDSettings { let secret_hash = provider.hashed_client_secret.clone(); }\nlet snapshot = query.site_replication_snapshot();\n", + "expected": [] + }, + { + "id": "accepted_unrelated_replication_secret", + "path": "rustfs/src/admin/handlers/site_replication.rs", + "source": "fn open_id_settings_from_snapshot(snapshot: &OidcSiteReplicationSnapshot) -> OpenIDSettings { let secret_hash = provider.hashed_client_secret.clone(); }\nfn ldap_settings() { let secret = ldap_provider.client_secret.clone(); }\nlet snapshot = query.site_replication_snapshot();\n", + "expected": [] + }, + { + "id": "accepted_live_workflow_paths", + "path": ".github/workflows/oidc-keycloak.yml", + "source": "on:\n pull_request:\n paths:\n - \"crates/iam/src/oidc/**\"\n - \"rustfs/src/admin/service/oidc_config.rs\"\n push:\n paths:\n - \"crates/iam/src/oidc/**\"\n - \"rustfs/src/admin/service/oidc_config.rs\"\n", + "expected": [] + }, + { + "id": "accepted_workflow_comment_between_paths", + "path": ".github/workflows/oidc-keycloak.yml", + "source": "on:\n pull_request:\n paths:\n - \"crates/iam/src/oidc/**\"\n # Keep configuration changes in the live gate.\n - \"rustfs/src/admin/service/oidc_config.rs\"\n push:\n paths:\n - \"crates/iam/src/oidc/**\"\n - \"rustfs/src/admin/service/oidc_config.rs\"\n", + "expected": [] + }, + { + "id": "accepted_owner_layout", + "layout": {}, + "expected": [] + } +] From dd90bb7488c98931b6db4f7144a52a6c2cc09fe4 Mon Sep 17 00:00:00 2001 From: Hauser Date: Mon, 12 Oct 2026 16:59:21 +0800 Subject: [PATCH 025/143] fix(s3): short-circuit conditional reads at metadata quorum (#8459) * fix(s3): evaluate conditional reads before opening object bodies Reuse locked metadata snapshots for conditional GETs and unify HEAD preconditions and cache headers. Preserve body-cache hooks and validate encryption metadata and key authorization before returning bodyless errors. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * refactor(s3): unify conditional reads at metadata quorum Apply scoped metadata conditions to the existing reader pipeline instead of maintaining parallel prepared-read branches. Preserve quorum checks, early-stop, coalescing, and cache-hook behavior on condition misses. Track metadata-only termination through selection and reject body access or a changed decision. Keep concrete reader futures off request stacks. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(s3): strengthen conditional read acceptance evidence Assert zero shard, decode, cache-fill, and remote-body calls on matching conditions with live positive controls. Exercise GET and HEAD through s3s and Hyper with conditional-header and precedence matrices. Add reproducible paired application-read benchmarks with isolated metadata controls and strict drift, artifact, and resource gates. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * refactor(s3): remove obsolete conditional comparison helper Keep ETag comparison tests on the canonical read-condition evaluator, remove unused date-trait imports, and name the HTTP case type. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(s3): validate full conditional etag fields Reject adjacent tags and malformed trailing list members before accepting a match. Keep the existing request-scoped metadata read and authorization order, while removing redundant validation wrappers and unused benchmark overrides. Replace simulated phase coverage with real 304/412 cache-fill probes and extend HTTP/1.1 and HTTP/2 ingress regressions. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): cache terminal conditional metadata safely Preserve quorum-validated conditional metadata when early termination cancels the remaining fanout. Restrict those entries to terminal GET readers; body and full-layout consumers still obtain their own metadata. Serialize cache publication per key so conditional entries cannot replace complete layouts or renew their TTL. Retain generation fencing and the final selected-metadata predicate check before any body access. Cover real four-disk hedging, nonterminal body and range reads, consumer isolation, generation races, atomic publication and actual cache expiry. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(log-analyzer): align heal writer failure rule with current logs Match the current structured all-targets failure, retain legacy log support, and reject partial or unrelated failures. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/ecstore/src/api/mod.rs | 8 +- crates/ecstore/src/diagnostics/get.rs | 1 + crates/ecstore/src/io_support/bitrot.rs | 4 + crates/ecstore/src/object_api/mod.rs | 4 + .../ecstore/src/object_api/read_condition.rs | 113 ++ crates/ecstore/src/object_api/readers.rs | 19 + .../src/set_disk/conditional_read_bench.rs | 63 + .../src/set_disk/core/io_primitives.rs | 30 + .../src/set_disk/core/metadata_read.rs | 42 +- crates/ecstore/src/set_disk/mod.rs | 261 ++++ crates/ecstore/src/set_disk/ops/object.rs | 59 + crates/ecstore/src/set_disk/read.rs | 1150 ++++++++++++++++- crates/log-analyzer/src/rules/seed/tests.rs | 51 +- docs/testing/README.md | 1 + docs/testing/conditional-read-benchmark.md | 95 ++ rustfs/src/app/metadata_route.rs | 47 +- .../src/app/object/conditional_read_bench.rs | 148 +++ rustfs/src/app/object/get.rs | 600 +++++++-- rustfs/src/app/object/head.rs | 48 +- rustfs/src/app/object/mod.rs | 3 +- rustfs/src/app/object/shared.rs | 28 +- rustfs/src/app/storage_api.rs | 8 +- rustfs/src/server/http.rs | 285 ++++ rustfs/src/server/layer.rs | 68 +- rustfs/src/storage/ecfs_extend.rs | 182 +-- rustfs/src/storage/ecfs_test.rs | 169 ++- rustfs/src/storage/sse.rs | 62 + rustfs/src/storage/storage_api.rs | 26 +- scripts/README.md | 3 + scripts/prepare_conditional_read_baseline.py | 74 ++ scripts/run_conditional_read_abba.py | 205 +++ scripts/test_conditional_read_abba.py | 116 ++ 32 files changed, 3641 insertions(+), 332 deletions(-) create mode 100644 crates/ecstore/src/object_api/read_condition.rs create mode 100644 crates/ecstore/src/set_disk/conditional_read_bench.rs create mode 100644 docs/testing/conditional-read-benchmark.md create mode 100644 rustfs/src/app/object/conditional_read_bench.rs create mode 100644 scripts/prepare_conditional_read_baseline.py create mode 100644 scripts/run_conditional_read_abba.py create mode 100644 scripts/test_conditional_read_abba.py diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index a43116a77..fc26177a9 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -521,13 +521,14 @@ pub mod integrity { pub mod object { pub use crate::object_api::{ BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook, - GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver, - ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission, - RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, + GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReadCondition, GetObjectReadMetadata, GetObjectReader, + NamespaceLockFence, ObjectEncryptionResolver, ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, + PutObjReader, QuotaAdmission, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError, ScannerPublicationCommitState, ShardIntegrityWriteMode, StreamConsumer, WriteCommitGuard, WriteCompletion, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook, + with_get_object_read_condition, }; pub use crate::store::{ PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError, @@ -592,6 +593,7 @@ pub mod set_disk { #[cfg(feature = "test-util")] pub mod test_util { pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test; + pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard; pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause}; /// Keep a namespace commit pending until the returned owner is dropped. diff --git a/crates/ecstore/src/diagnostics/get.rs b/crates/ecstore/src/diagnostics/get.rs index fecb3c701..b0f85383c 100644 --- a/crates/ecstore/src/diagnostics/get.rs +++ b/crates/ecstore/src/diagnostics/get.rs @@ -182,6 +182,7 @@ pub(crate) const GET_METADATA_CACHE_REASON_DELETE_MARKER: &str = "delete_marker" pub(crate) const GET_METADATA_CACHE_REASON_DIST_ERASURE: &str = "dist_erasure"; pub(crate) const GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS: &str = "incl_free_versions"; pub(crate) const GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM: &str = "insufficient_cached_quorum"; +pub(crate) const GET_METADATA_CACHE_REASON_BODY_LAYOUT_REQUIRED: &str = "body_layout_required"; pub(crate) const GET_METADATA_CACHE_REASON_META_BUCKET: &str = "meta_bucket"; pub(crate) const GET_METADATA_CACHE_REASON_NO_LOCK: &str = "no_lock"; pub(crate) const GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED: &str = "not_found_or_expired"; diff --git a/crates/ecstore/src/io_support/bitrot.rs b/crates/ecstore/src/io_support/bitrot.rs index 9d4615f50..081dc963b 100644 --- a/crates/ecstore/src/io_support/bitrot.rs +++ b/crates/ecstore/src/io_support/bitrot.rs @@ -174,10 +174,14 @@ async fn open_reader_source( metrics_path: Option<&'static str>, ) -> disk::error::Result> { if let Some(data) = inline_data { + #[cfg(test)] + crate::set_disk::disk_call_counters::record_shard_open(path, true); let mut reader = Cursor::new(data); reader.set_position(u64::try_from(offset).map_err(|_| DiskError::FileCorrupt)?); Ok(Some(ShardReader::InMemory(reader))) } else if let Some(disk) = disk { + #[cfg(test)] + crate::set_disk::disk_call_counters::record_shard_open(path, false); open_disk_reader(disk, bucket, path, offset, length, use_mmap_read, metrics_path) .await .map(Some) diff --git a/crates/ecstore/src/object_api/mod.rs b/crates/ecstore/src/object_api/mod.rs index 008005547..75b9db65c 100644 --- a/crates/ecstore/src/object_api/mod.rs +++ b/crates/ecstore/src/object_api/mod.rs @@ -102,10 +102,14 @@ mod body_cache_hook; mod encryption; mod hook_slot; mod object_mutation_hook; +mod read_condition; mod readers; mod types; pub(crate) mod write_commit_context; +pub use read_condition::{GetObjectReadCondition, GetObjectReadMetadata, with_get_object_read_condition}; +pub(crate) use read_condition::{get_object_read_condition_is_active, get_object_read_condition_is_terminal}; + #[cfg(test)] mod persisted_metadata_keys_tests; diff --git a/crates/ecstore/src/object_api/read_condition.rs b/crates/ecstore/src/object_api/read_condition.rs new file mode 100644 index 000000000..18754e55c --- /dev/null +++ b/crates/ecstore/src/object_api/read_condition.rs @@ -0,0 +1,113 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_filemeta::{FileInfo, ObjectPartInfo}; +use std::future::Future; +use std::sync::Arc; +use time::OffsetDateTime; + +/// Borrowed fields needed to decide whether an object read needs a body. +pub struct GetObjectReadMetadata<'a> { + pub bucket: &'a str, + pub object: &'a str, + pub etag: Option<&'a str>, + pub mod_time: Option, + pub parts: &'a [ObjectPartInfo], + pub delete_marker: bool, +} + +impl<'a> GetObjectReadMetadata<'a> { + pub(crate) fn from_file_info(bucket: &'a str, object: &'a str, info: &'a FileInfo) -> Self { + Self { + bucket, + object, + etag: info.metadata.get("etag").map(String::as_str), + mod_time: info.mod_time, + parts: &info.parts, + delete_marker: info.deleted, + } + } +} + +/// A pure decision on quorum-validated metadata. Authorization belongs to the caller. +pub trait GetObjectReadCondition: Send + Sync { + fn is_terminal(&self, metadata: GetObjectReadMetadata<'_>) -> bool; +} + +tokio::task_local! { + static READ_CONDITION: Arc; +} + +/// Applies a condition to this reader call, without inheriting it into shared producers. +pub async fn with_get_object_read_condition(condition: Arc, read: F) -> F::Output { + READ_CONDITION.scope(condition, read).await +} + +pub(crate) fn get_object_read_condition_is_terminal(metadata: GetObjectReadMetadata<'_>) -> bool { + READ_CONDITION + .try_with(|condition| condition.is_terminal(metadata)) + .unwrap_or(false) +} + +pub(crate) fn get_object_read_condition_is_active() -> bool { + READ_CONDITION.try_with(|_| ()).is_ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + struct Stop; + impl GetObjectReadCondition for Stop { + fn is_terminal(&self, _: GetObjectReadMetadata<'_>) -> bool { + true + } + } + fn metadata() -> GetObjectReadMetadata<'static> { + GetObjectReadMetadata { + bucket: "bucket", + object: "object", + etag: Some("etag"), + mod_time: None, + parts: &[], + delete_marker: false, + } + } + + #[tokio::test] + async fn read_condition_scope_does_not_escape_or_enter_shared_producers() { + assert!(!get_object_read_condition_is_terminal(metadata())); + with_get_object_read_condition(Arc::new(Stop), async { + assert!(get_object_read_condition_is_terminal(metadata())); + assert!( + !tokio::spawn(async { get_object_read_condition_is_terminal(metadata()) }) + .await + .expect("shared producer") + ); + }) + .await; + assert!(!get_object_read_condition_is_terminal(metadata())); + } + + #[tokio::test] + async fn dropping_a_pending_read_removes_its_condition_scope() { + let mut read = Box::pin(with_get_object_read_condition(Arc::new(Stop), async { + assert!(get_object_read_condition_is_terminal(metadata())); + std::future::pending::<()>().await; + })); + assert!(futures::poll!(&mut read).is_pending()); + drop(read); + assert!(!get_object_read_condition_is_terminal(metadata())); + } +} diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index 1ee72b227..67154621d 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -528,6 +528,14 @@ pub struct GetObjectReader { } impl GetObjectReader { + pub(crate) fn condition_stopped(object_info: ObjectInfo) -> Self { + Self { + stream: Box::new(ConditionStoppedBody), + object_info, + buffered_body: None, + body_source: GetObjectBodySource::Unprobed, + } + } /// Builds a fully materialized reader from a cache-coordinated body. pub fn from_cache_body(mut object_info: ObjectInfo, body: Bytes) -> Result { object_info.size = i64::try_from(body.len()).map_err(|_| Error::other("cached GET body length exceeds i64::MAX"))?; @@ -552,6 +560,17 @@ impl GetObjectReader { } } +struct ConditionStoppedBody; + +impl AsyncRead for ConditionStoppedBody { + fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + Poll::Ready(Err(std::io::Error::other("object read condition must be handled before body access"))) + } +} + enum ReadTransform { // Written but never read by production code: the enclosing struct already // carries the same pair as `storage_offset`/`storage_length`. They survive diff --git a/crates/ecstore/src/set_disk/conditional_read_bench.rs b/crates/ecstore/src/set_disk/conditional_read_bench.rs new file mode 100644 index 000000000..b218d4786 --- /dev/null +++ b/crates/ecstore/src/set_disk/conditional_read_bench.rs @@ -0,0 +1,63 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Test-util-only controls for the paired read benchmark's dedicated bucket. + +use std::sync::atomic::{AtomicBool, Ordering}; + +static ACTIVE: AtomicBool = AtomicBool::new(false); + +/// Keeps metadata off the cache in the benchmark's isolated bucket so GET +/// metadata reads exercise the configured slow disk. Body-cache policy +/// remains intact. +#[must_use] +pub struct ConditionalReadBenchmarkMetadataGuard { + _private: (), +} + +impl ConditionalReadBenchmarkMetadataGuard { + pub fn acquire() -> Self { + assert!( + ACTIVE + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + ); + Self { _private: () } + } +} + +impl Drop for ConditionalReadBenchmarkMetadataGuard { + fn drop(&mut self) { + ACTIVE.store(false, Ordering::Release); + } +} + +pub(super) fn cache_bypass_applies(bucket: &str) -> bool { + bucket == "conditional-bench" && ACTIVE.load(Ordering::Acquire) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn benchmark_metadata_profile_is_scoped_to_its_bucket_and_owner() { + assert!(!cache_bypass_applies("conditional-bench")); + let owner = ConditionalReadBenchmarkMetadataGuard::acquire(); + assert!(cache_bypass_applies("conditional-bench")); + assert!(!cache_bypass_applies("other-bucket")); + drop(owner); + assert!(!cache_bypass_applies("conditional-bench")); + } +} diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index b1ea2d327..72af3fe60 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -1260,6 +1260,10 @@ async fn try_create_bitrot_readers_via_batch_pread( } let requests: Vec<_> = batch_items.iter().map(|(_, p, off, len)| (p.clone(), *off, *len)).collect(); + #[cfg(test)] + for (disk_index, _, _, _) in &batch_items { + disk_call_counters::record(object, disk_call_counters::KIND_DISK_SHARD_OPEN, *disk_index); + } let batch_results = batch_shard_pread(requests).await; let mut setup = BitrotReaderSetup::new(disks.len()); @@ -6078,6 +6082,10 @@ pub(crate) mod disk_call_counters { pub const KIND_READ_VERSION: &str = "read_version"; pub const KIND_BATCH_READ_VERSION: &str = "batch_read_version"; pub const KIND_METADATA_SLOWTAIL_FAULT: &str = "metadata_slowtail_fault"; + pub const KIND_INLINE_SHARD_OPEN: &str = "inline_shard_open"; + pub const KIND_DISK_SHARD_OPEN: &str = "disk_shard_open"; + /// Body verification, direct shard assembly, and erasure decoder entry points. + pub const KIND_BODY_DECODE: &str = "body_decode"; /// Registry key: (object, kind, disk_index). type CountKey = (String, String, usize); @@ -6108,6 +6116,28 @@ pub(crate) mod disk_call_counters { .or_insert(0) += 1; } + /// Matches shard paths to the observed object's namespace path. This also + /// records deferred opens running in spawned decoder tasks. + pub(crate) fn record_shard_open(path: &str, inline: bool) { + let mut reg = registry().lock().expect("disk call-counter registry poisoned"); + let objects: Vec<_> = reg + .observed + .iter() + .filter(|object| { + path == object.as_str() || path.strip_prefix(object.as_str()).is_some_and(|rest| rest.starts_with('/')) + }) + .cloned() + .collect(); + let kind = if inline { + KIND_INLINE_SHARD_OPEN + } else { + KIND_DISK_SHARD_OPEN + }; + for object in objects { + *reg.counts.entry((object, kind.to_owned(), 0)).or_insert(0) += 1; + } + } + /// RAII scope that observes call counts for a single `object`. Counts recorded /// while the scope is alive — including those recorded inside spawned tasks — /// are queryable; the scope clears its own counts on drop. diff --git a/crates/ecstore/src/set_disk/core/metadata_read.rs b/crates/ecstore/src/set_disk/core/metadata_read.rs index ab1700b51..37e727395 100644 --- a/crates/ecstore/src/set_disk/core/metadata_read.rs +++ b/crates/ecstore/src/set_disk/core/metadata_read.rs @@ -66,6 +66,7 @@ use tokio::task::JoinSet; pub(in crate::set_disk) struct MetadataReadResult { pub(in crate::set_disk) slots: Vec, pub(in crate::set_disk) diagnostics: MetadataFanoutDiagnostics, + pub(in crate::set_disk) condition_stopped: bool, } impl MetadataReadResult { @@ -655,6 +656,8 @@ async fn inline_body_miss_reason_from_observations<'a>( return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY); }; + #[cfg(test)] + crate::set_disk::disk_call_counters::record(object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0); match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await { Some(body) if body.len() == object_size => None, _ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY), @@ -1078,7 +1081,11 @@ impl SetDisks { 0, ); } - Ok(MetadataReadResult { slots, diagnostics }) + Ok(MetadataReadResult { + slots, + diagnostics, + condition_stopped: false, + }) } #[allow(clippy::too_many_arguments)] @@ -1121,6 +1128,7 @@ impl SetDisks { let mut next_fanout_index = 0usize; let mut scheduled_count = 0usize; let mut force_full_wait = false; + let read_condition_active = read_data && crate::object_api::get_object_read_condition_is_active(); let mut final_miss_reason_override = None; let mut non_inline_candidate_eligible = None; let mut single_pending_hedge_deadline = None; @@ -1258,12 +1266,28 @@ impl SetDisks { } } - if !force_full_wait + if (!force_full_wait || read_condition_active) && let Some(decision) = accumulator .early_stop_decision() .or_else(|| accumulator.version_early_stop_decision()) { - let should_return_early = if read_data { + let condition_stopped = read_condition_active + && decision.reason == crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM + && accumulator.candidate.as_ref().is_some_and(|candidate| { + !candidate.deleted + && crate::object_api::get_object_read_condition_is_terminal( + crate::object_api::GetObjectReadMetadata::from_file_info( + bucket.as_ref(), + object.as_ref(), + candidate, + ), + ) + }); + let should_return_early = if condition_stopped { + true + } else if force_full_wait { + false + } else if read_data { match accumulator.candidate.as_ref() { Some(_candidate) if non_inline_candidate_eligible == Some(true) => { accumulator.candidate_has_read_reserve() @@ -1332,7 +1356,11 @@ impl SetDisks { cancelled_count, ); let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations); - return Ok(MetadataReadResult { slots, diagnostics }); + return Ok(MetadataReadResult { + slots, + diagnostics, + condition_stopped, + }); } } @@ -1410,7 +1438,11 @@ impl SetDisks { rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0); rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0); let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations); - Ok(MetadataReadResult { slots, diagnostics }) + Ok(MetadataReadResult { + slots, + diagnostics, + condition_stopped: false, + }) } /// Test-only seam that records one per-disk `read_version` metadata RPC for diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index c63244ea8..58d18b5a7 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -793,6 +793,11 @@ const DEFAULT_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE: bool = false; const ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT"; const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = true; +#[cfg(feature = "test-util")] +mod conditional_read_bench; +#[cfg(feature = "test-util")] +pub use conditional_read_bench::ConditionalReadBenchmarkMetadataGuard; + const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS"; const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS"; const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET"; @@ -919,6 +924,7 @@ pub use ops::heal_walk::HealWalkVersion; pub(in crate::set_disk) struct GetObjectFileInfo { owned: Option, shared: Option>, + condition_stopped: bool, } struct OwnedGetObjectFileInfo { @@ -940,6 +946,7 @@ impl GetObjectFileInfo { late_metadata_fanout_disks: None, }), shared: None, + condition_stopped: false, } } @@ -957,13 +964,16 @@ impl GetObjectFileInfo { late_metadata_fanout_disks: Some(late_metadata_fanout_disks), }), shared: None, + condition_stopped: false, } } fn shared(entry: Arc) -> Self { + let condition_stopped = entry.capability == GetObjectMetadataCacheCapability::ConditionOnly; Self { owned: None, shared: Some(entry), + condition_stopped, } } @@ -1144,6 +1154,248 @@ mod prepared_get_object_metadata_tests { assert!(take_prepared_get_object_metadata().is_none()); } + struct StopReadForEtag(String); + impl crate::object_api::GetObjectReadCondition for StopReadForEtag { + fn is_terminal(&self, metadata: crate::object_api::GetObjectReadMetadata<'_>) -> bool { + !metadata.delete_marker && metadata.etag == Some(self.0.as_str()) + } + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn read_condition_stops_on_quorum_before_slow_disk_and_body_verification() { + use crate::object_api::with_get_object_read_condition; + for size in [4096, 1_300_000] { + let (dirs, set) = make_local_set_disks(4, 2).await; + let bucket = "read-condition-quorum"; + let object = object_with_initial_data_shards(bucket, &format!("read-condition-{size}")); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let mut put = PutObjReader::from_vec(vec![b'v'; size]); + let info = set.put_object(bucket, &object, &mut put, &opts).await.expect("write object"); + for dir in &dirs { + let object_path = dir.path().join(bucket).join(&object); + if size == 4096 { + let path = object_path.join("xl.meta"); + let mut bytes = tokio::fs::read(&path).await.expect("read inline metadata"); + *bytes.last_mut().expect("inline payload") ^= 1; + tokio::fs::write(path, bytes).await.expect("damage inline data"); + } else { + tokio::fs::remove_dir_all(object_path.join(info.data_dir.expect("data directory").to_string())) + .await + .expect("remove shards"); + } + } + let slow = bounded_initial_parity_disk_index(bucket, &object); + let barrier = rename_fanout_barrier::arm(&object, slow, rename_fanout_barrier::PHASE_READ_VERSION); + let calls = disk_call_counters::observe(&object); + let read_opts = ObjectOptions { + metadata_cache_safe: true, + ..opts + }; + let followup_opts = read_opts.clone(); + let condition = Arc::new(StopReadForEtag(info.etag.clone().expect("committed ETag"))); + let set_for_read = Arc::clone(&set); + let object_for_read = object.clone(); + let mut read = tokio::spawn(async move { + with_get_object_read_condition( + condition, + set_for_read.get_object_reader(bucket, &object_for_read, None, HeaderMap::new(), &read_opts), + ) + .await + }); + tokio::time::timeout(READ_VERSION_BARRIER_GUARD, barrier.wait_until_paused()) + .await + .expect("slow metadata read is pending"); + let mut reader = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut read) + .await + .expect("condition must not wait for the slow disk or decode damaged data") + .expect("reader task") + .expect("metadata-only result"); + assert_eq!(reader.object_info.etag, info.etag); + assert_eq!( + calls.total(disk_call_counters::KIND_INLINE_SHARD_OPEN), + 0, + "damaged inline data was never opened" + ); + assert_eq!( + calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), + 0, + "missing shards were never opened" + ); + assert_eq!( + calls.total(disk_call_counters::KIND_BODY_DECODE), + 0, + "the body verifier and decoder were never entered" + ); + let mut body = Vec::new(); + assert!( + reader.stream.read_to_end(&mut body).await.is_err(), + "terminal metadata cannot be served as a successful body" + ); + assert!(body.is_empty()); + let first_calls = calls.total(disk_call_counters::KIND_READ_VERSION); + drop(reader); + barrier.release(); + let snapshot = set + .get_object_fileinfo_for_get_object_reader(bucket, &object, &followup_opts, true, true) + .await + .expect("fresh metadata after terminal scope"); + assert!(!snapshot.condition_stopped); + assert!( + calls.total(disk_call_counters::KIND_READ_VERSION) > first_calls, + "an incomplete terminal snapshot must not satisfy the next metadata read from cache" + ); + } + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn read_condition_skips_shard_open_and_decode_with_live_positive_controls() { + for size in [4096, 1_300_000] { + let (_dirs, set) = make_local_set_disks(4, 2).await; + let bucket = "read-condition-body-counters"; + let object = object_with_initial_data_shards(bucket, &format!("body-counters-{size}")); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + let payload: Vec<_> = (0..size) + .map(|index| u8::try_from(index % 251).expect("pattern byte")) + .collect(); + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let mut put = PutObjReader::from_vec(payload.clone()); + let info = set.put_object(bucket, &object, &mut put, &opts).await.expect("write object"); + let calls = disk_call_counters::observe(&object); + let reader = crate::object_api::with_get_object_read_condition( + Arc::new(StopReadForEtag(info.etag.expect("committed ETag"))), + set.get_object_reader(bucket, &object, None, HeaderMap::new(), &opts), + ) + .await + .expect("terminal metadata reader"); + drop(reader); + assert!(calls.total(disk_call_counters::KIND_READ_VERSION) > 0, "metadata really was read"); + assert_eq!( + calls.total(disk_call_counters::KIND_INLINE_SHARD_OPEN), + 0, + "no inline shard source opened" + ); + assert_eq!(calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), 0, "no disk shard source opened"); + assert_eq!( + calls.total(disk_call_counters::KIND_BODY_DECODE), + 0, + "no body verifier or decoder entered" + ); + + let mut reader = crate::object_api::with_get_object_read_condition( + Arc::new(StopReadForEtag("different-etag".to_owned())), + set.get_object_reader(bucket, &object, None, HeaderMap::new(), &opts), + ) + .await + .expect("unmatched condition opens the intact object"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("positive-control body"); + assert_eq!(body, payload); + let source_kind = if size == 4096 { + disk_call_counters::KIND_INLINE_SHARD_OPEN + } else { + disk_call_counters::KIND_DISK_SHARD_OPEN + }; + assert!( + calls.total(source_kind) > 0, + "the same source counter observes the positive control: size={size}, inline={}, disk={}, decode={}", + calls.total(disk_call_counters::KIND_INLINE_SHARD_OPEN), + calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), + calls.total(disk_call_counters::KIND_BODY_DECODE), + ); + assert!( + calls.total(disk_call_counters::KIND_BODY_DECODE) > 0, + "the same decode counter observes the positive control" + ); + } + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn read_condition_rechecks_selected_metadata_before_body_access() { + struct Changing(std::sync::atomic::AtomicUsize); + impl crate::object_api::GetObjectReadCondition for Changing { + fn is_terminal(&self, _: crate::object_api::GetObjectReadMetadata<'_>) -> bool { + self.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed) == 0 + } + } + let (_dirs, set) = make_local_set_disks(4, 2).await; + let bucket = "read-condition-consistency"; + let object = object_with_initial_data_shards(bucket, "read-condition-consistency"); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let mut put = PutObjReader::from_vec(vec![b'x'; 4096]); + set.put_object(bucket, &object, &mut put, &opts).await.expect("write object"); + let result = crate::object_api::with_get_object_read_condition( + Arc::new(Changing(std::sync::atomic::AtomicUsize::new(0))), + set.get_object_reader(bucket, &object, None, HeaderMap::new(), &opts), + ) + .await; + assert!(result.is_err(), "an unverified terminal snapshot must never become a 200 reader"); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn read_condition_miss_preserves_inline_early_stop_and_exact_body() { + let (_dirs, set) = make_local_set_disks(4, 2).await; + let bucket = "read-condition-miss"; + let object = object_with_initial_data_shards(bucket, "read-condition-miss"); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + let payload: Vec = (0..8192) + .map(|index| u8::try_from(index % 251).expect("pattern byte")) + .collect(); + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let mut put = PutObjReader::from_vec(payload.clone()); + set.put_object(bucket, &object, &mut put, &opts) + .await + .expect("write inline object"); + let slow = bounded_initial_parity_disk_index(bucket, &object); + let barrier = rename_fanout_barrier::arm(&object, slow, rename_fanout_barrier::PHASE_READ_VERSION); + let set_for_read = Arc::clone(&set); + let object_for_read = object.clone(); + let mut read = tokio::spawn(async move { + crate::object_api::with_get_object_read_condition( + Arc::new(StopReadForEtag("other".to_owned())), + set_for_read.get_object_reader(bucket, &object_for_read, None, HeaderMap::new(), &opts), + ) + .await + }); + tokio::time::timeout(READ_VERSION_BARRIER_GUARD, barrier.wait_until_paused()) + .await + .expect("slow metadata read is pending"); + let mut reader = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut read) + .await + .expect("condition miss preserves the normal early stop") + .expect("reader task") + .expect("reader"); + let mut bytes = Vec::new(); + reader.stream.read_to_end(&mut bytes).await.expect("normal body"); + assert_eq!(bytes, payload); + barrier.release(); + } + #[test] fn cache_hit_consumers_release_snapshot_at_legacy_ownership() { let fi = FileInfo { @@ -1152,6 +1404,7 @@ mod prepared_get_object_metadata_tests { }; let cached = Arc::new(GetObjectMetadataCacheEntry { created_at: Instant::now(), + capability: GetObjectMetadataCacheCapability::Full, parts_metadata: vec![fi.clone()], fi, online_disks: vec![None], @@ -4434,6 +4687,13 @@ impl Hash for GetObjectMetadataCacheKey { } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum GetObjectMetadataCacheCapability { + Full, + // Quorum-validated metadata without a complete body or write-target layout. + ConditionOnly, +} + #[derive(Debug)] struct GetObjectMetadataCacheEntry { #[allow(dead_code)] // Kept for debugging; moka handles TTL internally @@ -4442,6 +4702,7 @@ struct GetObjectMetadataCacheEntry { parts_metadata: Vec, online_disks: Vec>, read_quorum: usize, + capability: GetObjectMetadataCacheCapability, } #[derive(Clone, Debug)] diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 4c707eeea..c4e25a5da 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -2509,6 +2509,22 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { return Err(to_object_err(Error::MethodNotAllowed, vec![bucket, object])); } + let condition_terminal = + crate::object_api::get_object_read_condition_is_terminal(crate::object_api::GetObjectReadMetadata { + bucket, + object, + etag: object_info.etag.as_deref(), + mod_time: object_info.mod_time, + parts: &object_info.parts, + delete_marker: false, + }); + if snapshot.condition_stopped && !condition_terminal { + return Err(Error::other("conditional metadata snapshot no longer satisfies the read condition")); + } + if condition_terminal { + return Ok(GetObjectReader::condition_stopped(object_info)); + } + // if object_info.size == 0 { // let empty_rd: Box = Box::new(Bytes::new()); @@ -2608,6 +2624,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { // Decode directly let decode_stage_start = rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now); + #[cfg(test)] + crate::set_disk::disk_call_counters::record(object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0); if let Some(body) = try_read_inline_data_shards_direct(&mut readers, data_shards, read_length, object_size).await { if let Some(decode_stage_start) = decode_stage_start { @@ -2677,6 +2695,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { let decode_stage_start = rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now); let mut output = Cursor::new(Vec::with_capacity(object_size)); + #[cfg(test)] + crate::set_disk::disk_call_counters::record(object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0); let (written, err) = erasure.decode(&mut output, readers, 0, object_size, object_size).await; if let Some(e) = err { return Err(to_object_err(e.into(), vec![bucket, object])); @@ -15371,6 +15391,7 @@ mod transition_commit_failure_tests { cache_key.clone(), Arc::new(GetObjectMetadataCacheEntry { created_at: Instant::now(), + capability: crate::set_disk::GetObjectMetadataCacheCapability::Full, fi, parts_metadata, online_disks, @@ -17698,6 +17719,44 @@ mod transition_upload_integrity_tests { /// Plain objects must keep streaming the remote bytes through untouched: /// their plan is `Plain`, so the tiered read stays byte-identical. + #[tokio::test] + #[serial_test::serial] + async fn transitioned_conditional_read_skips_remote_get_with_live_positive_control() { + struct MatchingEtag(String); + impl crate::object_api::GetObjectReadCondition for MatchingEtag { + fn is_terminal(&self, metadata: crate::object_api::GetObjectReadMetadata<'_>) -> bool { + !metadata.delete_marker && metadata.etag == Some(self.0.as_str()) + } + } + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "transitioned-condition-body-get"; + let object = "object.bin"; + let payload = b"conditional remote reads need metadata only".repeat(256); + let original = write_source(&set, &disks, bucket, object, &payload).await; + let tier = format!("CONDITION{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier).await; + set.transition_object(bucket, object, &transition_options(&original, tier)) + .await + .expect("transition object"); + assert_eq!(backend.get_count().await, 0, "transition setup has not fetched the remote body"); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + let reader = crate::object_api::with_get_object_read_condition( + Arc::new(MatchingEtag(original.etag.expect("source ETag"))), + set.get_object_reader(bucket, object, None, HeaderMap::new(), &opts), + ) + .await + .expect("terminal transitioned metadata"); + assert!(reader.object_info.is_remote(), "the condition was evaluated on transitioned metadata"); + drop(reader); + assert_eq!(backend.get_count().await, 0, "a matching condition never calls the warm backend GET"); + let (body, _) = read_transitioned(&set, bucket, object, None, &opts).await; + assert_eq!(body, payload); + assert_eq!(backend.get_count().await, 1, "the same backend counter observes a normal remote GET"); + } + #[tokio::test] #[serial_test::serial] async fn transitioned_plain_object_get_is_unchanged() { diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index c91a16db0..172e40140 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -14,18 +14,20 @@ use super::{ Arc, Bytes, DiskError, DiskOption, DiskStore, Endpoint, ErasureCache, Error, FileInfo, GetCodecStreamingFallbackReason, - GetObjectFileInfo, GetObjectMetadataCacheEntry, GetObjectMetadataCacheGeneration, GetObjectMetadataCacheKey, - GetObjectReadPolicy, HashAlgorithm, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, OBJECT_OP_IGNORED_ERRS, ObjectInfo, - ObjectOptions, RUSTFS_META_BUCKET, ReadOptions, Result, SetDisks, StorageError, adaptive_duplex_buffer_size, - build_get_codec_streaming_decode_engine, build_inline_bitrot_readers_from_refs, collect_inline_data_shard_fileinfos_by_index, - debug, error, get_codec_streaming_metrics_path, get_codec_streaming_multipart_max_parts, get_object_read_policy, - is_codec_streaming_multipart_enabled, is_multipart_reader_setup_prefetch_enabled, object_fits_single_block, - reduce_read_quorum_errs, to_object_err, try_read_inline_data_shards_direct, warn, + GetObjectFileInfo, GetObjectMetadataCacheCapability, GetObjectMetadataCacheEntry, GetObjectMetadataCacheGeneration, + GetObjectMetadataCacheKey, GetObjectReadPolicy, HashAlgorithm, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, + OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectOptions, RUSTFS_META_BUCKET, ReadOptions, Result, SetDisks, StorageError, + adaptive_duplex_buffer_size, build_get_codec_streaming_decode_engine, build_inline_bitrot_readers_from_refs, + collect_inline_data_shard_fileinfos_by_index, debug, error, get_codec_streaming_metrics_path, + get_codec_streaming_multipart_max_parts, get_object_read_policy, is_codec_streaming_multipart_enabled, + is_multipart_reader_setup_prefetch_enabled, object_fits_single_block, reduce_read_quorum_errs, to_object_err, + try_read_inline_data_shards_direct, warn, }; use crate::diagnostics::get::{ GET_DIRECT_MEMORY_SUBPATH_DISK_DATA_BLOCKS, GET_DIRECT_MEMORY_SUBPATH_INLINE_BUFFERED, GET_METADATA_CACHE_DECISION_HIT, GET_METADATA_CACHE_DECISION_MISS, GET_METADATA_CACHE_DECISION_REJECT, GET_METADATA_CACHE_DECISION_SKIP, - GET_METADATA_CACHE_REASON_DATA_MOVEMENT, GET_METADATA_CACHE_REASON_DELETE_MARKER, GET_METADATA_CACHE_REASON_DIST_ERASURE, + GET_METADATA_CACHE_REASON_BODY_LAYOUT_REQUIRED, GET_METADATA_CACHE_REASON_DATA_MOVEMENT, + GET_METADATA_CACHE_REASON_DELETE_MARKER, GET_METADATA_CACHE_REASON_DIST_ERASURE, GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS, GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM, GET_METADATA_CACHE_REASON_META_BUCKET, GET_METADATA_CACHE_REASON_NO_LOCK, GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED, GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_CHECKSUMS, GET_METADATA_CACHE_REASON_PART_NUMBER, @@ -246,6 +248,12 @@ impl AsyncWrite for GetObjectDownstreamWriter { } } +#[derive(Clone, Copy, PartialEq, Eq)] +enum MetadataReadPurpose { + FullLayout, + GetObjectReader, +} + impl SetDisks { /// A startup quorum can leave remote slots unregistered even after their /// peers become ready. Refresh those slots before taking the GET metadata @@ -413,11 +421,12 @@ impl SetDisks { &self, identity: (&str, &str), generation: Option, - fi: &FileInfo, + selected: (&FileInfo, GetObjectMetadataCacheCapability), parts_metadata: &[FileInfo], online_disks: &[Option], read_quorum: usize, ) { + let (fi, capability) = selected; if fi.deleted || !fi.has_valid_erasure_geometry() { return; } @@ -440,6 +449,7 @@ impl SetDisks { parts_metadata: parts_metadata.to_vec(), online_disks: online_disks.to_vec(), read_quorum, + capability, }); self.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {}) .await; @@ -452,7 +462,25 @@ impl SetDisks { entry: Arc, after_insert: impl FnOnce(), ) { - self.get_object_metadata_cache.insert(key.clone(), entry).await; + // Every publisher shares Moka's per-key compute lock. A conditional + // snapshot cannot downgrade a full layout or extend its original TTL. + self.get_object_metadata_cache + .entry(key.clone()) + .and_compute_with(|current| { + std::future::ready( + if !self.is_get_object_metadata_cache_generation_current(generation) + || (entry.capability == GetObjectMetadataCacheCapability::ConditionOnly + && current + .as_ref() + .is_some_and(|existing| existing.value().capability == GetObjectMetadataCacheCapability::Full)) + { + moka::ops::compute::Op::Nop + } else { + moka::ops::compute::Op::Put(entry) + }, + ) + }) + .await; after_insert(); if self.is_get_object_metadata_cache_generation_current(generation) { return; @@ -514,8 +542,15 @@ impl SetDisks { read_data: bool, caller_allows_early_stop: bool, ) -> Result { - self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, caller_allows_early_stop, false) - .await + self.get_object_fileinfo_gated_inner( + bucket, + object, + opts, + read_data, + caller_allows_early_stop, + MetadataReadPurpose::FullLayout, + ) + .await } #[tracing::instrument(level = "debug", skip(self))] @@ -528,15 +563,13 @@ impl SetDisks { read_data: bool, caller_allows_early_stop: bool, ) -> Result { - let allow_read_version_coalescing = !crate::bucket::utils::is_meta_bucketname(bucket) - && crate::runtime::global::get_metadata_read_version_coalescing_service_ready(); self.get_object_fileinfo_gated_inner( bucket, object, opts, read_data, caller_allows_early_stop, - allow_read_version_coalescing, + MetadataReadPurpose::GetObjectReader, ) .await } @@ -554,7 +587,7 @@ impl SetDisks { read_data: bool, allow_early_stop: bool, ) -> Result { - self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, false) + self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, MetadataReadPurpose::FullLayout) .await } @@ -566,7 +599,7 @@ impl SetDisks { opts: &ObjectOptions, read_data: bool, allow_early_stop: bool, - allow_read_version_coalescing: bool, + purpose: MetadataReadPurpose, ) -> Result { let vid = opts.version_id.clone().unwrap_or_default(); let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); @@ -582,7 +615,14 @@ impl SetDisks { ); } else if vid.is_empty() { match self.lookup_cached_get_object_fileinfo(bucket, object).await { - MetadataCacheLookup::Hit(cached) => { + MetadataCacheLookup::Hit(cached) + if cached.capability == GetObjectMetadataCacheCapability::Full + || (purpose == MetadataReadPurpose::GetObjectReader + && allow_early_stop + && crate::object_api::get_object_read_condition_is_terminal( + crate::object_api::GetObjectReadMetadata::from_file_info(bucket, object, &cached.fi), + )) => + { rustfs_io_metrics::record_get_object_metadata_cache_decision( GET_OBJECT_PATH_SET_DISK, GET_METADATA_CACHE_DECISION_HIT, @@ -595,6 +635,15 @@ impl SetDisks { ); return Ok(GetObjectFileInfo::shared(cached)); } + MetadataCacheLookup::Hit(_) => { + // Keep the conditional entry for other requests; this read + // needs a body or a complete mutation target set. + rustfs_io_metrics::record_get_object_metadata_cache_decision( + GET_OBJECT_PATH_SET_DISK, + GET_METADATA_CACHE_DECISION_REJECT, + GET_METADATA_CACHE_REASON_BODY_LAYOUT_REQUIRED, + ); + } MetadataCacheLookup::Miss => { rustfs_io_metrics::record_get_object_metadata_cache_decision( GET_OBJECT_PATH_SET_DISK, @@ -631,6 +680,9 @@ impl SetDisks { // read_metadata_observed (see read_all_fileinfo_early_stop in // core/metadata_read.rs); unsafe requests and callers that opt out // (allow_early_stop=false) fall back to full-wait. + let allow_read_version_coalescing = purpose == MetadataReadPurpose::GetObjectReader + && !crate::bucket::utils::is_meta_bucketname(bucket) + && crate::runtime::global::get_metadata_read_version_coalescing_service_ready(); let metadata_read = if allow_read_version_coalescing { Self::read_metadata_for_get_object( &disks, @@ -660,6 +712,7 @@ impl SetDisks { .await? }; let metadata_fanout_complete = metadata_read.is_complete(); + let condition_stopped = metadata_read.condition_stopped; let (mut parts_metadata, errs, metadata_fanout_diagnostics) = metadata_read.into_legacy(); let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) { GET_OBJECT_PATH_INTERNAL_META @@ -725,13 +778,20 @@ impl SetDisks { "metadata_read_error", ) .await; - } else if use_metadata_cache && metadata_fanout_complete { + } else if use_metadata_cache && (metadata_fanout_complete || condition_stopped) { #[cfg(test)] metadata_cache_tests::wait_before_metadata_cache_publish(bucket, object).await; self.cache_get_object_fileinfo( (bucket, object), metadata_cache_generation, - &fi, + ( + &fi, + if metadata_fanout_complete { + GetObjectMetadataCacheCapability::Full + } else { + GetObjectMetadataCacheCapability::ConditionOnly + }, + ), &parts_metadata, &op_online_disks, read_quorum, @@ -743,20 +803,17 @@ impl SetDisks { // let online_disks: Vec> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect(); - if !metadata_fanout_complete + let mut snapshot = if !metadata_fanout_complete && allow_early_stop && non_inline_data_read_early_stop_allowed(read_data, bucket, object) && late_materialization_candidate_is_safe(&fi) { - Ok(GetObjectFileInfo::owned_with_late_metadata_fanout( - fi, - parts_metadata, - op_online_disks, - disks, - )) + GetObjectFileInfo::owned_with_late_metadata_fanout(fi, parts_metadata, op_online_disks, disks) } else { - Ok(GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks)) - } + GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks) + }; + snapshot.condition_stopped = condition_stopped; + Ok(snapshot) } #[hotpath::measure(impl_type = "SetDisks")] @@ -886,6 +943,8 @@ impl SetDisks { ); let decode_stage_start = Instant::now(); + #[cfg(test)] + crate::set_disk::disk_call_counters::record(object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0); let body = try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await; let decode_elapsed = decode_stage_start.elapsed(); rustfs_io_metrics::record_get_object_decode_duration(decode_elapsed.as_secs_f64()); @@ -943,6 +1002,8 @@ impl SetDisks { } let decode_stage_start = Instant::now(); + #[cfg(test)] + crate::set_disk::disk_call_counters::record(object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0); let body = try_read_inline_data_shards_direct(&mut reader_setup.readers, erasure.data_shards, read_length, object_size).await; let decode_elapsed = decode_stage_start.elapsed(); @@ -1361,6 +1422,8 @@ impl SetDisks { let readers = reader_setup.readers; let deferred_stripe_handles = reader_setup.deferred_stripe_handles; let deferred_reopeners = reader_setup.deferred_reopeners; + #[cfg(test)] + crate::set_disk::disk_call_counters::record(object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0); let coding::decode::DecodeOutcome { written, error: err, @@ -1916,6 +1979,8 @@ impl SetDisks { .with_deferred_parity_handles(deferred_stripe_handles) .with_deferred_parity_reopeners(deferred_reopeners) .with_repair_evidence(Arc::clone(&repair)); + #[cfg(test)] + crate::set_disk::disk_call_counters::record(object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0); let engine = build_get_codec_streaming_decode_engine(erasure.clone())?; let reader = if single_inflight { coding::decode_reader::ErasureDecodeReader::new_single_inflight_with_metrics_path( @@ -2535,6 +2600,10 @@ impl Drop for LegacyPerPartDecodeReader { } fn get_object_metadata_cache_request_bypass_reason(bucket: &str, opts: &ObjectOptions, read_data: bool) -> Option<&'static str> { + #[cfg(feature = "test-util")] + if super::conditional_read_bench::cache_bypass_applies(bucket) { + return Some("benchmark"); + } if !read_data { return Some(GET_METADATA_CACHE_REASON_NOT_READ_DATA); } @@ -3519,8 +3588,15 @@ mod metadata_cache_tests { let online_disks = Vec::new(); let generation = set.get_object_metadata_cache_generation("bucket", "object"); - set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, &parts_metadata, &online_disks, 0) - .await; + set.cache_get_object_fileinfo( + ("bucket", "object"), + generation, + (&fi, GetObjectMetadataCacheCapability::Full), + &parts_metadata, + &online_disks, + 0, + ) + .await; let cached = set .cached_get_object_fileinfo("bucket", "object") @@ -3539,8 +3615,15 @@ mod metadata_cache_tests { let parts_metadata = vec![fi.clone()]; let online_disks = Vec::new(); let generation = set.get_object_metadata_cache_generation("bucket", "object"); - set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, &parts_metadata, &online_disks, 0) - .await; + set.cache_get_object_fileinfo( + ("bucket", "object"), + generation, + (&fi, GetObjectMetadataCacheCapability::Full), + &parts_metadata, + &online_disks, + 0, + ) + .await; let cached = set .cached_get_object_fileinfo("bucket", "object") .await @@ -3803,8 +3886,15 @@ mod metadata_cache_tests { let mut deleted = valid_test_fileinfo("deleted-object"); deleted.deleted = true; let generation = set.get_object_metadata_cache_generation("bucket", "deleted-object"); - set.cache_get_object_fileinfo(("bucket", "deleted-object"), generation, &deleted, &[deleted.clone()], &[], 0) - .await; + set.cache_get_object_fileinfo( + ("bucket", "deleted-object"), + generation, + (&deleted, GetObjectMetadataCacheCapability::Full), + &[deleted.clone()], + &[], + 0, + ) + .await; assert!( set.cached_get_object_fileinfo("bucket", "deleted-object").await.is_none(), "deleted metadata must not be cached" @@ -3812,8 +3902,15 @@ mod metadata_cache_tests { let invalid = FileInfo::default(); let generation = set.get_object_metadata_cache_generation("bucket", "invalid-object"); - set.cache_get_object_fileinfo(("bucket", "invalid-object"), generation, &invalid, std::slice::from_ref(&invalid), &[], 0) - .await; + set.cache_get_object_fileinfo( + ("bucket", "invalid-object"), + generation, + (&invalid, GetObjectMetadataCacheCapability::Full), + std::slice::from_ref(&invalid), + &[], + 0, + ) + .await; assert!( set.cached_get_object_fileinfo("bucket", "invalid-object").await.is_none(), "invalid metadata must not be cached" @@ -3835,6 +3932,7 @@ mod metadata_cache_tests { ), Arc::new(GetObjectMetadataCacheEntry { created_at: Instant::now(), + capability: GetObjectMetadataCacheCapability::Full, fi: fi.clone(), parts_metadata: vec![fi], online_disks: vec![None], @@ -3855,8 +3953,15 @@ mod metadata_cache_tests { let fi = valid_test_fileinfo("object"); let generation = set.get_object_metadata_cache_generation("bucket", "object"); - set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, std::slice::from_ref(&fi), &[], 0) - .await; + set.cache_get_object_fileinfo( + ("bucket", "object"), + generation, + (&fi, GetObjectMetadataCacheCapability::Full), + std::slice::from_ref(&fi), + &[], + 0, + ) + .await; assert!( set.cached_get_object_fileinfo("bucket", "object").await.is_some(), @@ -3881,8 +3986,15 @@ mod metadata_cache_tests { .get_object_metadata_cache_generation("bucket", "object") .expect("metadata cache generation should be active"); let retired_key = GetObjectMetadataCacheKey::new("bucket", "object", generation); - set.cache_get_object_fileinfo(("bucket", "object"), Some(generation), &fi, std::slice::from_ref(&fi), &[], 0) - .await; + set.cache_get_object_fileinfo( + ("bucket", "object"), + Some(generation), + (&fi, GetObjectMetadataCacheCapability::Full), + std::slice::from_ref(&fi), + &[], + 0, + ) + .await; set.get_object_metadata_cache.run_pending_tasks().await; assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_some()); assert_eq!(set.get_object_metadata_cache.entry_count(), 1); @@ -3951,14 +4063,880 @@ mod metadata_cache_tests { ); } + #[tokio::test] + #[serial(metadata_cache_publish_barrier, body_cache_hook)] + async fn conditional_inline_hedge_caches_only_terminal_metadata() { + use crate::object_api::{GetObjectReadCondition, GetObjectReadMetadata, WriteCompletion, with_get_object_read_condition}; + use crate::set_disk::core::io_primitives::{disk_call_counters, rename_fanout_barrier}; + use crate::set_disk::core::metadata_read::bounded_metadata_fanout_order; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; + use http::HeaderMap; + use tokio::io::AsyncReadExt; + + struct MatchingEtag { + bucket: &'static str, + object: &'static str, + etag: String, + reached: Notify, + } + impl GetObjectReadCondition for MatchingEtag { + fn is_terminal(&self, metadata: GetObjectReadMetadata<'_>) -> bool { + let matched = metadata.bucket == self.bucket + && metadata.object == self.object + && !metadata.delete_marker + && metadata.etag == Some(self.etag.as_str()); + if matched { + // Observe the real metadata decision without changing its result. + self.reached.notify_one(); + } + matched + } + } + + temp_env::async_with_vars( + [ + (super::super::ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")), + (super::super::ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true")), + ], + async { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + ctx.update_erasure_type(crate::layout::endpoints::SetupType::Erasure).await; + let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, ctx).await; + let bucket = "conditional-inline-cache-hedge"; + let object = "conditional-inline-cache-hedge.bin"; + let payload: Vec<_> = (0..4096) + .map(|index| u8::try_from(index % 251).expect("pattern byte")) + .collect(); + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("real bucket"); + let info = set + .put_object( + bucket, + object, + &mut PutObjReader::from_vec(payload.clone()), + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("commit all four real inline shards"); + let etag = info.etag.expect("committed ETag"); + let opts = ObjectOptions::default(); + // This metadata-only inspection bypasses publication and discovers the actual layout. + let snapshot = set + .get_object_fileinfo(bucket, object, &opts, false, false) + .await + .expect("actual metadata"); + let distribution = snapshot.fi().erasure.distribution.clone(); + assert_eq!(snapshot.fi().erasure.data_blocks, 2); + assert_eq!(snapshot.fi().erasure.parity_blocks, 2); + assert!(snapshot.fi().inline_data()); + let d1 = distribution.iter().position(|block| *block == 1).expect("D1"); + let d2 = distribution.iter().position(|block| *block == 2).expect("D2"); + let order = bounded_metadata_fanout_order(bucket, object, 4, 2); + // Parity slots retain disk order, not their erasure block-number order. + let p1 = order[2]; + let p2 = order[3]; + assert_eq!(&order[..2], &[d1, d2]); + assert!(distribution[p1] > 2 && distribution[p2] > 2); + drop(snapshot); + assert!(set.cached_get_object_fileinfo(bucket, object).await.is_none(), "cold real object"); + assert!( + set.get_object_metadata_cache_bypass_reason(bucket, &opts, true) + .await + .is_none() + ); + + let calls = disk_call_counters::observe(object); + let data = rename_fanout_barrier::arm(object, d2, rename_fanout_barrier::PHASE_READ_VERSION); + let parity = rename_fanout_barrier::arm(object, p2, rename_fanout_barrier::PHASE_READ_VERSION); + let condition = Arc::new(MatchingEtag { + bucket, + object, + etag: etag.clone(), + reached: Notify::new(), + }); + let reader_set = Arc::clone(&set); + let reader_condition = Arc::clone(&condition); + let mut read = tokio::spawn(async move { + with_get_object_read_condition( + reader_condition, + reader_set.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()), + ) + .await + }); + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let completed = tokio::select! { + _ = data.wait_until_paused_before(deadline) => { + parity.wait_until_paused_before(deadline).await; + assert!(data.is_paused(), "D2 is still unavailable when the spare P2 is scheduled"); + assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, d1), 1); + assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, p1), 1); + parity.release(); + tokio::time::timeout_at(deadline, condition.reached.notified()).await + .expect("actual matching decision on the first three completed responses"); + // The 304 must finish while D2 remains blocked; waiting + // for all four disks would hide the cache starvation. + let result = tokio::time::timeout_at(deadline, &mut read).await.expect("conditional reader deadline"); + data.release(); + result + } + _ = &mut read => panic!("cold read must cross the real hedge barriers"), + }; + let mut reader = completed.expect("reader task").expect("conditional metadata reader"); + assert_eq!(reader.object_info.etag.as_deref(), Some(etag.as_str())); + let mut forbidden_body = Vec::new(); + assert!( + reader.stream.read_to_end(&mut forbidden_body).await.is_err(), + "terminal read is not a successful body" + ); + assert!(forbidden_body.is_empty()); + drop(reader); + assert_eq!(calls.total(disk_call_counters::KIND_INLINE_SHARD_OPEN), 0); + assert_eq!(calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), 0); + assert_eq!( + calls.total(disk_call_counters::KIND_BODY_DECODE), + 0, + "304 metadata setup never verifies or decodes a body" + ); + drop(data); + drop(parity); + let cached = set + .cached_get_object_fileinfo(bucket, object) + .await + .expect("a terminal quorum must remain reusable for another condition"); + assert_eq!(cached.capability, GetObjectMetadataCacheCapability::ConditionOnly); + assert_eq!(cached.online_disks.iter().flatten().count(), 3); + assert_eq!( + cached + .parts_metadata + .iter() + .filter(|metadata| !metadata.name.is_empty()) + .count(), + 3 + ); + + let before = calls.total(disk_call_counters::KIND_READ_VERSION); + let mut hit = with_get_object_read_condition( + Arc::new(MatchingEtag { + bucket, + object, + etag: etag.clone(), + reached: Notify::new(), + }), + set.get_object_reader(bucket, object, None, HeaderMap::new(), &opts), + ) + .await + .expect("warm conditional metadata"); + assert!(hit.stream.read_to_end(&mut Vec::new()).await.is_err()); + drop(hit); + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + before, + "warm conditions reuse metadata without fanout" + ); + let mut ordinary = set + .get_object_reader(bucket, object, None, HeaderMap::new(), &opts) + .await + .expect("ordinary warm GET"); + let mut actual = Vec::new(); + ordinary.stream.read_to_end(&mut actual).await.expect("ordinary body"); + assert_eq!(actual, payload); + assert!( + calls.total(disk_call_counters::KIND_READ_VERSION) > before, + "conditional cached layout cannot supply an ordinary body" + ); + assert!( + calls.total(disk_call_counters::KIND_INLINE_SHARD_OPEN) > 0, + "same source counter sees the ordinary positive control" + ); + assert!( + calls.total(disk_call_counters::KIND_BODY_DECODE) > 0, + "same decoder counter sees the ordinary positive control" + ); + + // Restore the proven conditional quorum for each nonterminal + // consumer, so an earlier ordinary read cannot mask a leak. + for range in [ + None, + Some(HTTPRangeSpec { + is_suffix_length: false, + start: 1, + end: 16, + }), + ] { + set.invalidate_get_object_metadata_cache(bucket, object).await; + set.cache_get_object_fileinfo( + (bucket, object), + set.get_object_metadata_cache_generation(bucket, object), + (&cached.fi, GetObjectMetadataCacheCapability::ConditionOnly), + &cached.parts_metadata, + &cached.online_disks, + cached.read_quorum, + ) + .await; + let before = calls.total(disk_call_counters::KIND_READ_VERSION); + let mut reader = with_get_object_read_condition( + Arc::new(MatchingEtag { + bucket, + object, + etag: "different-etag".to_owned(), + reached: Notify::new(), + }), + set.get_object_reader(bucket, object, range.clone(), HeaderMap::new(), &opts), + ) + .await + .expect("nonterminal condition must read its own body layout"); + let mut actual = Vec::new(); + reader + .stream + .read_to_end(&mut actual) + .await + .expect("complete nonterminal body"); + assert_eq!( + actual.as_slice(), + if range.is_some() { + &payload[1..17] + } else { + payload.as_slice() + } + ); + assert!(calls.total(disk_call_counters::KIND_READ_VERSION) > before); + } + + set.invalidate_get_object_metadata_cache(bucket, object).await; + set.cache_get_object_fileinfo( + (bucket, object), + set.get_object_metadata_cache_generation(bucket, object), + (&cached.fi, GetObjectMetadataCacheCapability::ConditionOnly), + &cached.parts_metadata, + &cached.online_disks, + cached.read_quorum, + ) + .await; + let before = calls.total(disk_call_counters::KIND_READ_VERSION); + let full = with_get_object_read_condition( + Arc::new(MatchingEtag { + bucket, + object, + etag, + reached: Notify::new(), + }), + set.get_object_fileinfo_gated(bucket, object, &opts, true, false), + ) + .await + .expect("full-layout caller under a matching condition"); + assert!(calls.total(disk_call_counters::KIND_READ_VERSION) > before); + assert_eq!(full.online_disks().iter().flatten().count(), 4); + assert_eq!( + set.cached_get_object_fileinfo(bucket, object) + .await + .expect("complete publication") + .capability, + GetObjectMetadataCacheCapability::Full + ); + }, + ) + .await; + } + + fn cache_capability_race_entry( + object: &str, + capability: GetObjectMetadataCacheCapability, + etag: &str, + ) -> Arc { + let mut fi = valid_test_fileinfo(object); + fi.metadata.insert("etag".to_string(), etag.to_string()); + Arc::new(GetObjectMetadataCacheEntry { + created_at: Instant::now(), + parts_metadata: vec![fi.clone()], + fi, + online_disks: Vec::new(), + read_quorum: 0, + capability, + }) + } + + #[tokio::test] + async fn metadata_cache_capability_publication_preserves_full_in_both_orders() { + use GetObjectMetadataCacheCapability::{ConditionOnly, Full}; + + for capabilities in [[Full, ConditionOnly], [ConditionOnly, Full]] { + let set = new_metadata_cache_test_set().await; + let generation = set + .get_object_metadata_cache_generation("bucket", "object") + .expect("active cache generation"); + let key = GetObjectMetadataCacheKey::new("bucket", "object", generation); + let full = cache_capability_race_entry("object", Full, "full-layout"); + let partial = cache_capability_race_entry("object", ConditionOnly, "partial-layout"); + for capability in capabilities { + let incoming = if capability == Full { &full } else { &partial }; + set.insert_get_object_metadata_cache_entry_after_insert(key.clone(), generation, Arc::clone(incoming), || {}) + .await; + } + let cached = set.get_object_metadata_cache.get(&key).await.expect("published cache entry"); + assert_eq!(cached.capability, Full); + assert!(Arc::ptr_eq(&cached, &full), "the complete layout must survive either publication order"); + } + } + + #[tokio::test] + async fn metadata_cache_capability_publisher_waits_for_real_compute_in_both_orders() { + use GetObjectMetadataCacheCapability::{ConditionOnly, Full}; + use moka::ops::compute::Op; + + for held_capability in [Full, ConditionOnly] { + let set = new_metadata_cache_test_set().await; + let generation = set + .get_object_metadata_cache_generation("bucket", "object") + .expect("active cache generation"); + let key = GetObjectMetadataCacheKey::new("bucket", "object", generation); + let full = cache_capability_race_entry("object", Full, "full-layout"); + let partial = cache_capability_race_entry("object", ConditionOnly, "partial-layout"); + let (held_entry, incoming) = if held_capability == Full { + (Arc::clone(&full), Arc::clone(&partial)) + } else { + (Arc::clone(&partial), Arc::clone(&full)) + }; + let release = Arc::new(Notify::new()); + let holder_release = Arc::clone(&release); + // Hold the actual Moka compute waiter across an await. Polling the + // production publisher below proves it queues before any cache write. + let mut holder = Box::pin( + set.get_object_metadata_cache + .entry(key.clone()) + .and_compute_with(|current| async move { + assert!(current.is_none(), "the holder must see an empty key"); + holder_release.notified().await; + Op::Put(held_entry) + }), + ); + assert!(futures::poll!(holder.as_mut()).is_pending()); + let hook_calls = AtomicUsize::new(0); + let mut publisher = + Box::pin( + set.insert_get_object_metadata_cache_entry_after_insert(key.clone(), generation, incoming, || { + hook_calls.fetch_add(1, Ordering::AcqRel); + }), + ); + assert!( + futures::poll!(publisher.as_mut()).is_pending(), + "all production publishers must share Moka's compute waiter" + ); + assert!( + set.get_object_metadata_cache.get(&key).await.is_none(), + "a plain insert must not bypass the holder and expose an entry" + ); + assert_eq!(hook_calls.load(Ordering::Acquire), 0); + release.notify_one(); + holder.await; + publisher.await; + assert_eq!(hook_calls.load(Ordering::Acquire), 1); + let cached = set.get_object_metadata_cache.get(&key).await.expect("published cache entry"); + assert_eq!(cached.capability, Full); + assert!(Arc::ptr_eq(&cached, &full), "a queued conditional publisher must not downgrade Full"); + } + } + + #[tokio::test] + async fn metadata_cache_waiting_stale_compute_cannot_touch_new_generation() { + use GetObjectMetadataCacheCapability::{ConditionOnly, Full}; + use moka::ops::compute::Op; + + let set = new_metadata_cache_test_set().await; + let old_generation = set + .get_object_metadata_cache_generation("bucket", "object") + .expect("active old generation"); + let old_key = GetObjectMetadataCacheKey::new("bucket", "object", old_generation); + let release = Arc::new(Notify::new()); + let holder_release = Arc::clone(&release); + let mut holder = Box::pin( + set.get_object_metadata_cache + .entry(old_key.clone()) + .and_compute_with(|current| async move { + assert!(current.is_none()); + holder_release.notified().await; + Op::Nop + }), + ); + assert!(futures::poll!(holder.as_mut()).is_pending()); + let hook_calls = AtomicUsize::new(0); + let old_partial = cache_capability_race_entry("object", ConditionOnly, "retired-layout"); + let mut old_publisher = Box::pin(set.insert_get_object_metadata_cache_entry_after_insert( + old_key.clone(), + old_generation, + old_partial, + || { + hook_calls.fetch_add(1, Ordering::AcqRel); + }, + )); + assert!(futures::poll!(old_publisher.as_mut()).is_pending()); + assert_eq!(hook_calls.load(Ordering::Acquire), 0); + + set.invalidate_get_object_metadata_cache("bucket", "object").await; + let new_generation = set + .get_object_metadata_cache_generation("bucket", "object") + .expect("active new generation"); + assert_eq!(new_generation.value, old_generation.value + 1); + let new_key = GetObjectMetadataCacheKey::new("bucket", "object", new_generation); + assert_ne!(old_key, new_key); + let new_full = cache_capability_race_entry("object", Full, "current-layout"); + tokio::time::timeout( + Duration::from_secs(1), + set.insert_get_object_metadata_cache_entry_after_insert( + new_key.clone(), + new_generation, + Arc::clone(&new_full), + || {}, + ), + ) + .await + .expect("a retired-generation waiter must not block the new key"); + release.notify_one(); + holder.await; + old_publisher.await; + assert_eq!(hook_calls.load(Ordering::Acquire), 1, "a stale Nop still runs the post-compute hook"); + assert!(set.get_object_metadata_cache.get(&old_key).await.is_none()); + let cached = set + .get_object_metadata_cache + .get(&new_key) + .await + .expect("new generation must survive"); + assert!( + Arc::ptr_eq(&cached, &new_full), + "old publication and its cleanup must not replace or remove the new key" + ); + } + + #[tokio::test] + async fn metadata_cache_put_and_nop_hooks_recheck_generation_exactly_once() { + use GetObjectMetadataCacheCapability::{ConditionOnly, Full}; + + for incoming_capability in [Full, ConditionOnly] { + let set = new_metadata_cache_test_set().await; + let generation = set + .get_object_metadata_cache_generation("bucket", "object") + .expect("active cache generation"); + let key = GetObjectMetadataCacheKey::new("bucket", "object", generation); + let existing = cache_capability_race_entry("object", Full, "existing-full"); + set.insert_get_object_metadata_cache_entry_after_insert(key.clone(), generation, existing, || {}) + .await; + let incoming = cache_capability_race_entry("object", incoming_capability, "incoming"); + let hook_calls = AtomicUsize::new(0); + set.insert_get_object_metadata_cache_entry_after_insert(key.clone(), generation, incoming, || { + assert_eq!(hook_calls.fetch_add(1, Ordering::AcqRel), 0, "hook must run once"); + // The existing production seam invalidates after either a Put or + // Full-preserving Nop; both must still perform the final fence read. + set.get_object_metadata_cache_generations[generation.index].fetch_add(1, Ordering::AcqRel); + }) + .await; + assert_eq!(hook_calls.load(Ordering::Acquire), 1); + assert!( + set.get_object_metadata_cache.get(&key).await.is_none(), + "post-compute generation change must withdraw the retired key for both Put and Nop" + ); + } + } + + #[tokio::test] + async fn metadata_cache_conditional_nop_preserves_actual_moka_expiration() { + use GetObjectMetadataCacheCapability::{ConditionOnly, Full}; + + let set = new_metadata_cache_test_set().await; + let generation = set + .get_object_metadata_cache_generation("bucket", "object") + .expect("active cache generation"); + let key = GetObjectMetadataCacheKey::new("bucket", "object", generation); + let full = cache_capability_race_entry("object", Full, "original-full"); + set.insert_get_object_metadata_cache_entry_after_insert(key.clone(), generation, Arc::clone(&full), || {}) + .await; + let original_insert_finished = Instant::now(); + + // Moka uses its own real clock, so Tokio paused-time advancement would not + // establish this property. Use the actual configured TTL and expiry reads. + tokio::time::sleep(GET_OBJECT_METADATA_CACHE_TTL * 3 / 4).await; + assert!( + set.get_object_metadata_cache.get(&key).await.is_some(), + "the retained entry must be live immediately before the Nop" + ); + let partial = cache_capability_race_entry("object", ConditionOnly, "partial-layout"); + set.insert_get_object_metadata_cache_entry_after_insert(key.clone(), generation, partial, || {}) + .await; + let retained = set.get_object_metadata_cache.get(&key).await.expect("Full must remain live"); + assert!(Arc::ptr_eq(&retained, &full)); + + // A distinct fresh Put is the positive control for the renewed deadline. + let fresh_generation = set + .get_object_metadata_cache_generation("bucket", "fresh-object") + .expect("active positive-control generation"); + let fresh_key = GetObjectMetadataCacheKey::new("bucket", "fresh-object", fresh_generation); + let fresh = cache_capability_race_entry("fresh-object", Full, "fresh-full"); + set.insert_get_object_metadata_cache_entry_after_insert(fresh_key.clone(), fresh_generation, fresh, || {}) + .await; + let renewed_insert_finished = Instant::now(); + let original_expired_by = original_insert_finished + GET_OBJECT_METADATA_CACHE_TTL + Duration::from_millis(200); + tokio::time::sleep_until(tokio::time::Instant::from_std(original_expired_by)).await; + assert!( + renewed_insert_finished.elapsed() < GET_OBJECT_METADATA_CACHE_TTL, + "the measurement must end before a renewed TTL could expire" + ); + assert!( + set.get_object_metadata_cache.get(&fresh_key).await.is_some(), + "a genuinely fresh Put must still be live at the same observation time" + ); + assert!( + set.get_object_metadata_cache.get(&key).await.is_none(), + "Nop must preserve Moka's original expiration, not merely the entry's created_at" + ); + } + + struct CacheBoundaryCondition { + bucket: &'static str, + object: &'static str, + etag: String, + calls: AtomicUsize, + terminal_calls: usize, + } + + impl crate::object_api::GetObjectReadCondition for CacheBoundaryCondition { + fn is_terminal(&self, metadata: crate::object_api::GetObjectReadMetadata<'_>) -> bool { + if metadata.bucket != self.bucket + || metadata.object != self.object + || metadata.delete_marker + || metadata.etag != Some(self.etag.as_str()) + { + return false; + } + self.calls.fetch_add(1, Ordering::AcqRel) < self.terminal_calls + } + } + + async fn condition_only_cache_boundary_fixture( + bucket: &'static str, + object: &'static str, + ) -> (Vec, Arc, Arc) { + use crate::object_api::WriteCompletion; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; + + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + ctx.update_erasure_type(crate::layout::endpoints::SetupType::Erasure).await; + let (dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, ctx).await; + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("real bucket"); + let committed = set + .put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![17; 4096]), + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("commit all four shards"); + let full = set + .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false) + .await + .expect("complete real metadata"); + assert_eq!(full.online_disks().iter().flatten().count(), 4); + let mut metadata = full.parts_metadata().to_vec(); + let mut disks = full.online_disks().to_vec(); + let omitted = metadata + .iter() + .position(|fi| fi.erasure.index == 2) + .expect("actual second data disk"); + metadata[omitted] = FileInfo::default(); + disks[omitted] = None; + // This seam prepares a real, independently validated D1/P1/P2 quorum. + // The hedge test separately exercises its production publication timing. + let errs: Vec> = (0..4).map(|_| None).collect(); + let (online_disks, fi, _) = SetDisks::select_valid_fileinfo(&disks, &metadata, &errs, "", 2, 3) + .expect("three real members satisfy latest-generation selection"); + assert_eq!(online_disks.iter().flatten().count(), 3); + assert_eq!(fi.metadata.get("etag"), committed.etag.as_ref()); + set.invalidate_get_object_metadata_cache(bucket, object).await; + set.cache_get_object_fileinfo( + (bucket, object), + set.get_object_metadata_cache_generation(bucket, object), + (&fi, GetObjectMetadataCacheCapability::ConditionOnly), + &metadata, + &online_disks, + 2, + ) + .await; + let cached = set + .cached_get_object_fileinfo(bucket, object) + .await + .expect("prepared conditional quorum"); + assert_eq!(cached.capability, GetObjectMetadataCacheCapability::ConditionOnly); + (dirs, set, cached) + } + + #[tokio::test] + #[serial(metadata_cache_publish_barrier, body_cache_hook)] + async fn condition_only_cache_final_predicate_flip_fails_before_body() { + use crate::object_api::with_get_object_read_condition; + use crate::set_disk::core::io_primitives::disk_call_counters; + + let bucket = "conditional-cache-final-flip"; + let object = "conditional-cache-final-flip.bin"; + let (_dirs, set, cached) = condition_only_cache_boundary_fixture(bucket, object).await; + let condition = Arc::new(CacheBoundaryCondition { + bucket, + object, + etag: cached.fi.metadata["etag"].clone(), + calls: AtomicUsize::new(0), + // Deliberately violate purity to attack the final fail-closed guard. + terminal_calls: 1, + }); + let calls = disk_call_counters::observe(object); + let result = with_get_object_read_condition( + condition.clone(), + set.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default()), + ) + .await; + let err = result + .err() + .expect("a partial snapshot cannot become a body reader after predicate flip"); + assert!( + err.to_string() + .contains("conditional metadata snapshot no longer satisfies the read condition") + ); + assert_eq!(condition.calls.load(Ordering::Acquire), 2, "lookup and final selected-FI check both ran"); + for kind in [ + disk_call_counters::KIND_READ_VERSION, + disk_call_counters::KIND_INLINE_SHARD_OPEN, + disk_call_counters::KIND_DISK_SHARD_OPEN, + disk_call_counters::KIND_BODY_DECODE, + ] { + assert_eq!(calls.total(kind), 0, "rejected cached snapshot cannot materialize any body"); + } + } + + #[tokio::test] + #[serial(metadata_cache_publish_barrier, body_cache_hook)] + async fn condition_only_cache_scope_and_full_layout_boundaries() { + use crate::object_api::with_get_object_read_condition; + use crate::set_disk::core::io_primitives::disk_call_counters; + + let bucket = "conditional-cache-scope-boundary"; + let object = "conditional-cache-scope-boundary.bin"; + let (_dirs, set, cached) = condition_only_cache_boundary_fixture(bucket, object).await; + for mode in 0..3 { + set.invalidate_get_object_metadata_cache(bucket, object).await; + set.cache_get_object_fileinfo( + (bucket, object), + set.get_object_metadata_cache_generation(bucket, object), + (&cached.fi, GetObjectMetadataCacheCapability::ConditionOnly), + &cached.parts_metadata, + &cached.online_disks, + cached.read_quorum, + ) + .await; + let condition = Arc::new(CacheBoundaryCondition { + bucket, + object, + etag: cached.fi.metadata["etag"].clone(), + calls: AtomicUsize::new(0), + terminal_calls: usize::MAX, + }); + let calls = disk_call_counters::observe(object); + let snapshot = with_get_object_read_condition(condition.clone(), async { + assert!(crate::object_api::get_object_read_condition_is_active()); + match mode { + 0 => { + let producer_set = set.clone(); + tokio::spawn(Box::pin(async move { + assert!( + !crate::object_api::get_object_read_condition_is_active(), + "spawned producers inherit no caller condition" + ); + producer_set + .get_object_fileinfo_for_get_object_reader(bucket, object, &ObjectOptions::default(), true, true) + .await + })) + .await + .expect("producer task") + .expect("producer reads its own layout") + } + 1 => set + .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false) + .await + .expect("generic complete-layout consumer"), + _ => set + .get_object_fileinfo_for_get_object_reader( + bucket, + object, + &ObjectOptions { + no_lock: true, + metadata_cache_safe: false, + ..Default::default() + }, + true, + true, + ) + .await + .expect("unprotected NoLock reads bypass even a terminal cached condition"), + } + }) + .await; + assert!(snapshot.shared.is_none(), "mode {mode} cannot consume a conditional cached layout"); + assert!( + calls.total(disk_call_counters::KIND_READ_VERSION) > 0, + "mode {mode} must independently read metadata" + ); + if mode < 2 { + assert!(!snapshot.condition_stopped); + assert_eq!(condition.calls.load(Ordering::Acquire), 0); + } + } + } + + #[tokio::test] + async fn condition_only_cache_rejects_invalid_deleted_and_insufficient_quorum() { + let set = new_metadata_cache_test_set().await; + let mut deleted = valid_test_fileinfo("conditional-deleted"); + deleted.deleted = true; + let invalid = FileInfo::default(); + for (object, fi) in [("conditional-deleted", deleted), ("conditional-invalid", invalid)] { + set.cache_get_object_fileinfo( + ("bucket", object), + set.get_object_metadata_cache_generation("bucket", object), + (&fi, GetObjectMetadataCacheCapability::ConditionOnly), + std::slice::from_ref(&fi), + &[], + 0, + ) + .await; + assert!(matches!( + set.lookup_cached_get_object_fileinfo("bucket", object).await, + MetadataCacheLookup::Miss + )); + } + for object in ["conditional-deleted", "conditional-invalid"] { + let condition = Arc::new(CacheBoundaryCondition { + bucket: "bucket", + object, + etag: "etag-1".to_owned(), + calls: AtomicUsize::new(0), + terminal_calls: usize::MAX, + }); + let result = crate::object_api::with_get_object_read_condition( + condition.clone(), + set.get_object_fileinfo_for_get_object_reader("bucket", object, &ObjectOptions::default(), true, true), + ) + .await; + assert!(result.is_err(), "invalid or deleted cache metadata cannot terminate the read"); + assert_eq!(condition.calls.load(Ordering::Acquire), 0); + } + let object = "conditional-insufficient"; + let fi = valid_test_fileinfo(object); + let generation = set + .get_object_metadata_cache_generation("bucket", object) + .expect("generation"); + set.get_object_metadata_cache + .insert( + GetObjectMetadataCacheKey::new("bucket", object, generation), + Arc::new(GetObjectMetadataCacheEntry { + created_at: Instant::now(), + capability: GetObjectMetadataCacheCapability::ConditionOnly, + fi: fi.clone(), + parts_metadata: vec![fi], + online_disks: vec![None], + read_quorum: 1, + }), + ) + .await; + assert!(matches!( + set.lookup_cached_get_object_fileinfo("bucket", object).await, + MetadataCacheLookup::RejectedInsufficientQuorum + )); + // With no disks, a rejected entry must produce an actual metadata error, + // even when a caller's predicate would accept the poisoned cached ETag. + let condition = Arc::new(CacheBoundaryCondition { + bucket: "bucket", + object, + etag: "etag-1".to_owned(), + calls: AtomicUsize::new(0), + terminal_calls: usize::MAX, + }); + let result = crate::object_api::with_get_object_read_condition( + condition.clone(), + set.get_object_fileinfo_for_get_object_reader("bucket", object, &ObjectOptions::default(), true, true), + ) + .await; + assert!(result.is_err(), "insufficient cached quorum cannot yield terminal metadata"); + assert_eq!(condition.calls.load(Ordering::Acquire), 0); + } + + #[tokio::test] + #[serial(metadata_cache_publish_barrier, body_cache_hook)] + async fn condition_only_cache_generation_rechecked_after_lookup() { + let bucket = "conditional-cache-generation-boundary"; + let object = "conditional-cache-generation-boundary.bin"; + let (_dirs, set, _) = condition_only_cache_boundary_fixture(bucket, object).await; + let generation = set + .get_object_metadata_cache_generation(bucket, object) + .expect("current generation"); + let key = GetObjectMetadataCacheKey::new(bucket, object, generation); + let condition = Arc::new(CacheBoundaryCondition { + bucket, + object, + etag: set + .cached_get_object_fileinfo(bucket, object) + .await + .expect("cached quorum") + .fi + .metadata["etag"] + .clone(), + calls: AtomicUsize::new(0), + terminal_calls: usize::MAX, + }); + let lookup = crate::object_api::with_get_object_read_condition(condition.clone(), async { + set.lookup_cached_get_object_fileinfo_after_get(bucket, object, || { + set.get_object_metadata_cache_generations[generation.index].fetch_add(1, Ordering::AcqRel); + }) + .await + }) + .await; + assert!( + matches!(lookup, MetadataCacheLookup::Miss), + "a concurrent invalidation rejects old conditional metadata" + ); + assert!( + set.get_object_metadata_cache.get(&key).await.is_none(), + "retired partial entry is withdrawn" + ); + assert_eq!( + condition.calls.load(Ordering::Acquire), + 0, + "generation validation precedes conditional use" + ); + assert!(matches!( + set.lookup_cached_get_object_fileinfo(bucket, object).await, + MetadataCacheLookup::Miss + )); + } + #[tokio::test] async fn metadata_cache_lookup_rechecks_generation_after_get() { let set = new_metadata_cache_test_set().await; let fi = valid_test_fileinfo("object"); let generation = set.get_object_metadata_cache_generation("bucket", "object"); let generation_index = generation.expect("metadata cache generation should be active").index; - set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, std::slice::from_ref(&fi), &[], 0) - .await; + set.cache_get_object_fileinfo( + ("bucket", "object"), + generation, + (&fi, GetObjectMetadataCacheCapability::Full), + std::slice::from_ref(&fi), + &[], + 0, + ) + .await; let lookup = set .lookup_cached_get_object_fileinfo_after_get("bucket", "object", || { @@ -3983,6 +4961,7 @@ mod metadata_cache_tests { let key = GetObjectMetadataCacheKey::new("bucket", "object", generation); let entry = Arc::new(GetObjectMetadataCacheEntry { created_at: Instant::now(), + capability: GetObjectMetadataCacheCapability::Full, fi: fi.clone(), parts_metadata: vec![fi], online_disks: Vec::new(), @@ -4022,8 +5001,15 @@ mod metadata_cache_tests { assert_eq!(first.value, initial.value + 1); assert_eq!(second.value, first.value + 1); - set.cache_get_object_fileinfo(("bucket", "object"), Some(first), &fi, std::slice::from_ref(&fi), &[], 0) - .await; + set.cache_get_object_fileinfo( + ("bucket", "object"), + Some(first), + (&fi, GetObjectMetadataCacheCapability::Full), + std::slice::from_ref(&fi), + &[], + 0, + ) + .await; assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none()); } @@ -4062,10 +5048,24 @@ mod metadata_cache_tests { let first_generation = first.get_object_metadata_cache_generation("bucket", "object"); let second_generation = second.get_object_metadata_cache_generation("bucket", "object"); first - .cache_get_object_fileinfo(("bucket", "object"), first_generation, &fi, std::slice::from_ref(&fi), &[], 0) + .cache_get_object_fileinfo( + ("bucket", "object"), + first_generation, + (&fi, GetObjectMetadataCacheCapability::Full), + std::slice::from_ref(&fi), + &[], + 0, + ) .await; second - .cache_get_object_fileinfo(("bucket", "object"), second_generation, &fi, std::slice::from_ref(&fi), &[], 0) + .cache_get_object_fileinfo( + ("bucket", "object"), + second_generation, + (&fi, GetObjectMetadataCacheCapability::Full), + std::slice::from_ref(&fi), + &[], + 0, + ) .await; first.invalidate_get_object_metadata_cache("bucket", "object").await; @@ -4093,6 +5093,7 @@ mod metadata_cache_tests { let entry = |fi: FileInfo| { Arc::new(GetObjectMetadataCacheEntry { created_at: Instant::now(), + capability: GetObjectMetadataCacheCapability::Full, parts_metadata: vec![fi.clone()], fi, online_disks: Vec::new(), @@ -4140,8 +5141,15 @@ mod metadata_cache_tests { set.invalidate_get_object_metadata_cache("bucket", "object").await; assert_eq!(set.get_object_metadata_cache_generation("bucket", "object"), None); - set.cache_get_object_fileinfo(("bucket", "object"), last_generation, &fi, std::slice::from_ref(&fi), &[], 0) - .await; + set.cache_get_object_fileinfo( + ("bucket", "object"), + last_generation, + (&fi, GetObjectMetadataCacheCapability::Full), + std::slice::from_ref(&fi), + &[], + 0, + ) + .await; assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none()); set.invalidate_get_object_metadata_cache("bucket", "object").await; assert_eq!( @@ -4159,8 +5167,15 @@ mod metadata_cache_tests { set.invalidate_all_get_object_metadata_cache(); - set.cache_get_object_fileinfo(("bucket", "object"), stale_generation, &fi, std::slice::from_ref(&fi), &[], 0) - .await; + set.cache_get_object_fileinfo( + ("bucket", "object"), + stale_generation, + (&fi, GetObjectMetadataCacheCapability::Full), + std::slice::from_ref(&fi), + &[], + 0, + ) + .await; assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none()); } @@ -4174,8 +5189,15 @@ mod metadata_cache_tests { .get_object_metadata_cache_generation("bucket", object) .expect("metadata cache generation should be active"); retired_keys.push(GetObjectMetadataCacheKey::new("bucket", object, generation)); - set.cache_get_object_fileinfo(("bucket", object), Some(generation), &fi, std::slice::from_ref(&fi), &[], 0) - .await; + set.cache_get_object_fileinfo( + ("bucket", object), + Some(generation), + (&fi, GetObjectMetadataCacheCapability::Full), + std::slice::from_ref(&fi), + &[], + 0, + ) + .await; } set.get_object_metadata_cache.run_pending_tasks().await; assert_eq!(set.get_object_metadata_cache.entry_count(), 3); @@ -4200,8 +5222,15 @@ mod metadata_cache_tests { .get_object_metadata_cache_generation("bucket", "object") .expect("metadata cache generation should be active"); let retired_key = GetObjectMetadataCacheKey::new("bucket", "object", generation); - set.cache_get_object_fileinfo(("bucket", "object"), Some(generation), &fi, std::slice::from_ref(&fi), &[], 0) - .await; + set.cache_get_object_fileinfo( + ("bucket", "object"), + Some(generation), + (&fi, GetObjectMetadataCacheCapability::Full), + std::slice::from_ref(&fi), + &[], + 0, + ) + .await; for fence in set.get_object_metadata_cache_generations.iter() { fence.store(u64::MAX, Ordering::Release); } @@ -4228,8 +5257,15 @@ mod metadata_cache_tests { let fresh_fi = valid_test_fileinfo("fresh-object"); let generation = set.get_object_metadata_cache_generation("bucket", "fresh-object"); - set.cache_get_object_fileinfo(("bucket", "fresh-object"), generation, &fresh_fi, std::slice::from_ref(&fresh_fi), &[], 0) - .await; + set.cache_get_object_fileinfo( + ("bucket", "fresh-object"), + generation, + (&fresh_fi, GetObjectMetadataCacheCapability::Full), + std::slice::from_ref(&fresh_fi), + &[], + 0, + ) + .await; assert!( set.cached_get_object_fileinfo("bucket", "fresh-object").await.is_some(), diff --git a/crates/log-analyzer/src/rules/seed/tests.rs b/crates/log-analyzer/src/rules/seed/tests.rs index c4bb4f744..d6a87f094 100644 --- a/crates/log-analyzer/src/rules/seed/tests.rs +++ b/crates/log-analyzer/src/rules/seed/tests.rs @@ -176,7 +176,15 @@ fn every_rule_has_a_positive_sample() { "heal-no-datadir", msg("heal: latest metadata for b/o has no data_dir, cannot heal object data"), ), - ("heal-all-writes-failed", msg("all drives had write errors, unable to heal b/o")), + ( + "heal-all-writes-failed", + Sample { + message: "Set disk object heal writer failures", + level: Some(LogLevel::Warn), + fields: &[("state", "writer_unavailable"), ("result", "all_targets_unavailable")], + ..Default::default() + }, + ), ("heal-rename-failed", msg("heal rename incomplete: 1 of 2 targets committed for b/o")), ( "heal-xlmeta-regen-failed", @@ -412,3 +420,44 @@ fn smoke_samples_hit_exact_rule_sets() { &[], ); } + +#[test] +fn heal_all_writes_failed_matches_current_and_legacy_logs() { + let engine = engine(); + let rule_ids = |sample: &Sample| { + engine + .matches(&event(sample)) + .into_iter() + .map(|index| engine.rules()[index].id.as_str()) + .collect::>() + }; + + assert_eq!( + rule_ids(&Sample { + message: "Set disk object heal writer failures", + level: Some(LogLevel::Warn), + fields: &[("state", "writer_unavailable"), ("result", "all_targets_unavailable")], + ..Default::default() + }), + vec!["heal-all-writes-failed"] + ); + assert_eq!( + rule_ids(&msg("all drives had write errors, unable to heal b/o")), + vec!["heal-all-writes-failed"] + ); + for sample in [ + Sample { + message: "Set disk object heal writer failures", + fields: &[("state", "writer_unavailable"), ("result", "partial_targets_unavailable")], + ..Default::default() + }, + msg("Set disk object heal writer failures"), + Sample { + message: "unrelated operation failed", + fields: &[("result", "all_targets_unavailable")], + ..Default::default() + }, + ] { + assert!(rule_ids(&sample).is_empty(), "unexpected rule match: {}", sample.message); + } +} diff --git a/docs/testing/README.md b/docs/testing/README.md index 208919040..25eb26e1d 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -18,6 +18,7 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh | Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) plus the 4-node kill/fresh-drive/blackhole cases in `crates/e2e_test/src/distributed/chaos_test.rs` | Part of the e2e crate (`e2e-reliability` and `e2e-distributed`) | Reliability cases with `e2e-full`; 4-node chaos on storage-sensitive PRs and nightly via `e2e-distributed` | | Fuzz | `cargo-fuzz` targets over untrusted parsing surfaces; isolated sub-workspace under `fuzz/` | `./scripts/fuzz/run.sh` (see [`fuzz/README.md`](../../fuzz/README.md)) | PR smoke on the paths listed in `.github/workflows/fuzz.yml`, plus nightly corpus | | Benchmarks | Criterion benches under each crate's `benches/` | `cargo bench -p ` | On demand; never a gate | +| Conditional reads | Paired ordinary/hit/miss GET latency, throughput and read-lock measurements | `scripts/run_conditional_read_abba.py` | Local conditional-read acceptance; [guide and scope](conditional-read-benchmark.md) | Every script named above is indexed with status and wiring in [`scripts/README.md`](../../scripts/README.md). Fixed GHSA advisories map to named regression tests in [security-regressions.md](security-regressions.md). diff --git a/docs/testing/conditional-read-benchmark.md b/docs/testing/conditional-read-benchmark.md new file mode 100644 index 000000000..eef1992e5 --- /dev/null +++ b/docs/testing/conditional-read-benchmark.md @@ -0,0 +1,95 @@ +# Conditional object-read benchmark + +**Use this when:** validating the performance of metadata-based conditional GET +changes or reproducing a paired application-read regression experiment. + +Conditional GET acceptance needs paired performance evidence for ordinary reads, +condition hits, and condition misses. Saving body work on a 304 does not establish +that other reads are unaffected. Keep performance acceptance pending if resource +isolation, baseline stability, or regression budgets fail. + +The manually invoked `conditional_read_paired_benchmark` drives the actual +`DefaultObjectUsecase` against four local EC disks. It consumes and verifies the +entire successful body, validates 304 results, and samples existing read-lock +hold histograms. It covers 4 KiB inline and 1.3 MB sharded objects, body cache off +and warmed, and a 20 ms delay on one metadata disk. It excludes network ingress, +authentication, external tiers, cold-fill contention, and distributed capacity. + +The test-util-only metadata guard bypasses metadata caching in the dedicated +`conditional-bench` bucket so cache hits cannot hide the slow-disk scenario. +Both ordinary GET and prepared GET metadata reads already use `read_data=true` +and therefore exercise the existing delay injector without an additional override. +The guard has no effect in ordinary builds or outside its bucket, and ownership +drop restores the default policy. Both compared revisions must include this +identical benchmark control. + +## Build paired artifacts + +Use a clean detached worktree at the full baseline commit, a separate candidate +worktree, and matching stable toolchain, features, and build profile. The adapter +resolves every source anchor before writing. It installs only the test harness, +test-util controls, and their facade exports; it does not install the conditional +read implementation on the baseline. Review its diff before building. + +```bash +python3 scripts/prepare_conditional_read_baseline.py \ + --baseline-worktree "$BASELINE_WORKTREE" --revision "$BASELINE_REVISION" +``` + +In each worktree, format the adapter and build the same artifact: + +```bash +RUSTUP_TOOLCHAIN=stable cargo fmt --all +RUSTUP_TOOLCHAIN=stable cargo test --locked -p rustfs --lib \ + --no-default-features --features rio-v2 \ + conditional_read_paired_benchmark --no-run +``` + +Save each executable printed by Cargo before another build overwrites it. For +deployment acceptance, use matching optimized profiles and an isolated target +environment. A test-profile run is a local application regression experiment, +not a production throughput estimate. + +## Run and interpret + +Stop concurrent builds and other workloads on the measurement resources. The +runner refuses concurrent Cargo/rustc processes or high host load. It executes +fresh processes serially in A1–B1–B2–A2 order, removes ambient RustFS experimental +settings and proxies, uses independent temporary disks, and records binary, +revision, harness, and profile provenance beside raw logs. + +Executable snapshots are read-only and their hashes are rechecked before and +after every case. Cargo/rustc activity is sampled during each process; host load +and isolation are checked again on exit, including the last A2 case. These checks +do not replace exclusive resource ownership for formal deployment acceptance. + +```bash +python3 scripts/run_conditional_read_abba.py \ + --baseline "$BASELINE_BINARY" --candidate "$CANDIDATE_BINARY" \ + --baseline-revision "$BASELINE_REVISION" \ + --candidate-revision "$CANDIDATE_REVISION" \ + --harness rustfs/src/app/object/conditional_read_bench.rs \ + --profile test --out-dir "$RESULT_DIR" +``` + +The default is 1000 requests per case at concurrency 8. Each case records +throughput, p95/p99 latency, read-lock p95/p99, and a zero metadata-cache-hit +assertion. Body-cache-on cases are warmed; they do not measure cold-fill misses. +Inline shard bytes can be carried by an `xl.meta` read even when no separate +body reader, bitrot verification, or decode is needed. + +Budgets are fixed before the run: + +- Baseline drift: throughput within 10%; latency and lock percentiles within + 15%, with a 0.25 ms absolute noise floor. +- Candidate regression: throughput loss at most 5%; latency and lock percentile + increases at most 10%, with a 0.25 ms absolute noise floor. + +A drift failure produces no candidate attribution. After a stable baseline, the +runner compares geometric means of the paired legs. Any workload over budget +keeps acceptance pending; improving 304 cannot compensate for an ordinary-read +or condition-miss regression. Preserve raw logs and investigate before rerunning; +do not relax budgets after seeing the results. + +Run `python3 scripts/test_conditional_read_abba.py` to verify the acceptance +parser and drift/regression gates. diff --git a/rustfs/src/app/metadata_route.rs b/rustfs/src/app/metadata_route.rs index 4cfd6b601..53615a8bb 100644 --- a/rustfs/src/app/metadata_route.rs +++ b/rustfs/src/app/metadata_route.rs @@ -271,7 +271,7 @@ async fn call_website( website_error_with_document(&req, &server_ctx, &target.bucket, config.error_document.as_ref(), StatusCode::NOT_FOUND) .await } - Err(err) if err.code() == &S3ErrorCode::NotModified => Ok(website_error(StatusCode::NOT_MODIFIED, true)), + Err(err) if err.code() == &S3ErrorCode::NotModified => Ok(website_not_modified(&err)), Err(_) if status.is_client_error() => { website_error_with_document(&req, &server_ctx, &target.bucket, config.error_document.as_ref(), status).await } @@ -520,6 +520,18 @@ fn website_error(status: StatusCode, head: bool) -> S3Response { } } +fn website_not_modified(error: &S3Error) -> S3Response { + let mut response = S3Response::new(Body::empty()); + response.status = Some(StatusCode::NOT_MODIFIED); + if let Some(headers) = error.headers() { + response.headers = headers.clone(); + } + response.headers.remove(http::header::CONTENT_LENGTH); + response.headers.remove(http::header::CONTENT_TYPE); + response.headers.remove(http::header::TRANSFER_ENCODING); + response +} + fn website_location(location: String, status: StatusCode) -> S3Result> { let mut response = S3Response::new(Body::empty()); response.status = Some(status); @@ -826,6 +838,39 @@ fn header_value(headers: &HeaderMap, name: &str) -> S3Result> { #[cfg(test)] mod tests { + #[tokio::test] + async fn website_304_preserves_cache_headers_without_content() { + use http_body_util::BodyExt; + let mut error = s3s::S3Error::new(s3s::S3ErrorCode::NotModified); + let mut headers = http::HeaderMap::new(); + for (name, value) in [ + ("etag", "\"abc\""), + ("cache-control", "max-age=60"), + ("expires", "Tue, 14 Nov 2023 22:13:20 GMT"), + ("content-length", "123"), + ("content-type", "application/xml"), + ] { + headers.insert(name, http::HeaderValue::from_static(value)); + } + error.set_headers(headers); + let response = super::website_not_modified(&error); + assert_eq!(response.status, Some(http::StatusCode::NOT_MODIFIED)); + assert_eq!(response.headers["etag"], "\"abc\""); + assert_eq!(response.headers["cache-control"], "max-age=60"); + assert!(response.headers.contains_key("expires")); + assert!(!response.headers.contains_key("content-length")); + assert!(!response.headers.contains_key("content-type")); + assert!( + response + .output + .collect() + .await + .expect("empty website body") + .to_bytes() + .is_empty() + ); + } + use super::{ MetadataOperation, list_object_versions_input, list_objects_v2_input, metadata_operation, website_redirect, website_relative_redirect, website_rule_matches, website_target, diff --git a/rustfs/src/app/object/conditional_read_bench.rs b/rustfs/src/app/object/conditional_read_bench.rs new file mode 100644 index 000000000..e0acc3839 --- /dev/null +++ b/rustfs/src/app/object/conditional_read_bench.rs @@ -0,0 +1,148 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Manually invoked, paired application-read benchmark on real local EC disks. +//! Run the identical harness on both revisions in separate processes. Environment +//! configuration must be set before either process initializes the store. + +use super::*; +use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; +use futures::stream; +use http::{HeaderValue, Method}; +use metrics_util::debugging::{DebugValue, DebuggingRecorder}; +use std::time::Instant; + +fn percentile(samples: &mut [f64], percent: usize) -> f64 { + assert!(!samples.is_empty(), "benchmark must collect samples"); + samples.sort_by(f64::total_cmp); + samples[(samples.len() * percent).div_ceil(100).saturating_sub(1)] +} + +async fn read_once(usecase: &DefaultObjectUsecase, object: &str, etag: &str, kind: &str, expected: &[u8]) -> f64 { + let start = Instant::now(); + let mut req = build_request( + GetObjectInput::builder() + .bucket("conditional-bench".to_owned()) + .key(object.to_owned()) + .build() + .expect("benchmark input"), + Method::GET, + ); + if kind != "unconditional" { + let tag = if kind == "hit" { etag } else { "unmatched-benchmark-etag" }; + req.headers.insert( + http::header::IF_NONE_MATCH, + HeaderValue::from_str(&format!("\"{tag}\"")).expect("benchmark condition"), + ); + } + match usecase.execute_get_object(req).await { + Err(error) => { + assert_eq!(kind, "hit"); + assert_eq!(error.code(), &S3ErrorCode::NotModified); + } + Ok(mut response) => { + assert_ne!(kind, "hit"); + let mut body = response.output.body.take().expect("benchmark body"); + let mut actual = Vec::with_capacity(expected.len()); + while let Some(chunk) = body.next().await { + actual.extend_from_slice(&chunk.expect("benchmark body chunk")); + } + assert_eq!(actual, expected, "benchmark must consume and validate the whole body"); + } + } + start.elapsed().as_secs_f64() * 1000.0 +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "manual benchmark: use the paired runner with --ignored --nocapture"] +async fn conditional_read_paired_benchmark() { + let iterations = std::env::var("RUSTFS_CONDITIONAL_BENCH_ITERATIONS") + .unwrap_or_else(|_| "1000".to_owned()) + .parse::() + .expect("benchmark iteration count"); + let concurrency = std::env::var("RUSTFS_CONDITIONAL_BENCH_CONCURRENCY") + .unwrap_or_else(|_| "8".to_owned()) + .parse::() + .expect("benchmark concurrency"); + assert!(iterations >= 100 && concurrency > 0); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::set_global_recorder(recorder).expect("run the benchmark alone in a fresh process"); + let _metadata_profile = crate::app::storage_api::test::set_disk::ConditionalReadBenchmarkMetadataGuard::acquire(); + let (store, context) = real_store_test_context().await; + rustfs_io_metrics::set_get_stage_metrics_enabled(true); + let cache_enabled = !context.object_data_cache().is_disabled(); + let usecase = DefaultObjectUsecase::with_context(Some(context)); + store + .make_bucket("conditional-bench", &MakeBucketOptions::default()) + .await + .expect("benchmark bucket"); + for size in [4096, 1_300_000] { + let object = format!("bench-{size}"); + let expected = (0..size).map(|index| (index % 251) as u8).collect::>(); + let info = put_real_cold_fill_object(&store, "conditional-bench", &object, &expected).await; + let etag = info.etag.as_deref().expect("benchmark ETag"); + for kind in ["unconditional", "hit", "miss"] { + // Prime the body cache and filesystem for every case, including 304. + for _ in 0..20 { + read_once(&usecase, &object, etag, "unconditional", &expected).await; + } + snapshotter.snapshot(); + let start = Instant::now(); + let mut latencies = stream::iter(0..iterations) + .map(|_| read_once(&usecase, &object, etag, kind, &expected)) + .buffer_unordered(concurrency) + .collect::>() + .await; + let elapsed = start.elapsed().as_secs_f64(); + let mut lock_ms = Vec::new(); + let mut metadata_cache_hits = 0; + for (key, _, _, value) in snapshotter.snapshot().into_vec() { + if key.key().name() == "rustfs_io_get_object_metadata_cache_total" + && key + .key() + .labels() + .any(|label| label.key() == "decision" && label.value() == "hit") + && let DebugValue::Counter(count) = &value + { + metadata_cache_hits += count; + } + if key.key().name() == "rustfs_object_lock_diag_hold_duration_seconds" + && key + .key() + .labels() + .any(|label| label.key() == "mode" && label.value() == "read") + && let DebugValue::Histogram(values) = value + { + lock_ms.extend(values.iter().map(|sample| sample.0 * 1000.0)); + } + } + assert!(!lock_ms.is_empty(), "enable RUSTFS_OBJECT_LOCK_DIAG_ENABLE before startup"); + assert_eq!(metadata_cache_hits, 0, "slow-disk measurements require a metadata-cache bypass"); + println!( + "CONDITIONAL_BENCH {}", + serde_json::json!({ + "size": size, "kind": kind, "cache": cache_enabled, + "iterations": iterations, "concurrency": concurrency, + "metadata_cache_hits": metadata_cache_hits, "metadata_cache_bypassed": true, + "slowtail_ms": std::env::var("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS").unwrap_or_default(), + "ops_per_sec": iterations as f64 / elapsed, + "p95_ms": percentile(&mut latencies, 95), "p99_ms": percentile(&mut latencies, 99), + "lock_samples": lock_ms.len(), "lock_p95_ms": percentile(&mut lock_ms, 95), + "lock_p99_ms": percentile(&mut lock_ms, 99), + }) + ); + } + } +} diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs index bdcb79d26..edbc2e03b 100644 --- a/rustfs/src/app/object/get.rs +++ b/rustfs/src/app/object/get.rs @@ -15,7 +15,30 @@ //! GetObject / GetObjectAttributes read path: cold fill, resume, stream tuning. use super::*; +use crate::app::storage_api::object_usecase::object_cache::{ + GetObjectReadCondition, GetObjectReadMetadata, evaluate_read_preconditions, has_read_preconditions, + with_get_object_read_condition, +}; use crate::on_demand_migration::WriteBackBody; + +struct ConditionalRead { + bucket: String, + object: String, + headers: HeaderMap, + part_number: Option, +} + +impl GetObjectReadCondition for ConditionalRead { + fn is_terminal(&self, metadata: GetObjectReadMetadata<'_>) -> bool { + if metadata.bucket != self.bucket || metadata.object != self.object || metadata.delete_marker { + return false; + } + evaluate_read_preconditions(&self.headers, metadata.etag, metadata.mod_time).is_err() + || self + .part_number + .is_some_and(|number| number > 1 && !metadata.parts.iter().any(|part| part.number == number)) + } +} use crate::on_demand_migration::{ BucketOdmState, OdmLookup, OdmOp, OdmOutcome, OnDemandMigrationSys, PullError, PullLeader, PullOutcome, PullReason, PullSlot, RangeGetPolicy, SourceBody, SourceClient, SourceError, SourceGet, SourceHead, commit_inline, idle_guarded_body, @@ -49,6 +72,39 @@ type ColdFillReaderOpenProbeState = Option<(rustfs_object_data_cache::ObjectData #[cfg(test)] static COLD_FILL_READER_OPEN_PROBE: OnceLock> = OnceLock::new(); +#[cfg(test)] +#[derive(Default)] +struct BodyCacheFillCalls { + reservations: AtomicU64, + fills: AtomicU64, +} + +#[cfg(test)] +type BodyCacheFillProbeState = Option<(rustfs_object_data_cache::ObjectDataCacheKey, Arc)>; + +#[cfg(test)] +static BODY_CACHE_FILL_PROBE: OnceLock> = OnceLock::new(); + +#[cfg(test)] +fn record_body_cache_fill_call_for_test(plan: &rustfs_object_data_cache::ObjectDataCacheGetPlan, reservation: bool) { + let Some(key) = plan.key() else { return }; + let counters = BODY_CACHE_FILL_PROBE + .get_or_init(|| Mutex::new(None)) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .as_ref() + .filter(|(probe_key, _)| probe_key == key) + .map(|(_, counts)| Arc::clone(counts)); + if let Some(counters) = counters { + let counter = if reservation { + &counters.reservations + } else { + &counters.fills + }; + counter.fetch_add(1, Ordering::Relaxed); + } +} + fn adjust_cold_fill_disk_permit_metric(owner: ColdFillDiskPermitOwner, acquired: bool) { macro_rules! adjust_gauge { ($name:literal) => {{ @@ -1075,6 +1131,8 @@ async fn start_cold_fill_producer( + req: &S3Request, + part_number: Option, + read: F, + ) -> impl std::future::Future { + // The concrete reader future is large; keep it out of the request's stack frame. + let read = Box::pin(read); + let condition = if has_read_preconditions(&req.headers) { + let mut headers = HeaderMap::with_capacity(4); + for name in [ + http::header::IF_MATCH, + http::header::IF_UNMODIFIED_SINCE, + http::header::IF_NONE_MATCH, + http::header::IF_MODIFIED_SINCE, + ] { + if let Some(value) = req.headers.get(&name) { + headers.insert(name, value.clone()); + } + } + Some(Arc::new(ConditionalRead { + bucket: req.input.bucket.clone(), + object: rustfs_utils::path::encode_dir_object_ref(&req.input.key).into_owned(), + headers, + part_number, + })) + } else { + None + }; + async move { + if let Some(condition) = condition { + with_get_object_read_condition(condition, read).await + } else { + read.await + } + } + } fn build_memory_bytes_blob( bytes: Bytes, response_content_length: i64, @@ -2279,9 +2373,29 @@ impl DefaultObjectUsecase { Ok(()) } - fn validate_get_object_before_cold_fill(headers: &HeaderMap, part_number: Option, info: &ObjectInfo) -> S3Result<()> { - check_preconditions(headers, info)?; - Self::validate_get_object_part_number(part_number, info) + async fn validate_get_object_before_body( + req: &S3Request, + bucket: &str, + key: &str, + part_number: Option, + info: &ObjectInfo, + ) -> S3Result<()> { + if info.delete_marker { + let code = if req.input.version_id.is_some() { + S3ErrorCode::MethodNotAllowed + } else { + S3ErrorCode::NoSuchKey + }; + return Err(with_delete_marker_read_headers(S3Error::new(code), info, req.input.version_id.as_deref())); + } + let result = + check_preconditions(&req.headers, info).and_then(|()| Self::validate_get_object_part_number(part_number, info)); + if result.is_err() { + let principal = SseKmsPrincipal::from_request(req); + crate::storage::sse::validate_conditional_read(bucket, key, &info.user_defined, &req.headers, principal.as_ref()) + .await?; + } + result } /// The `x-amz-mp-parts-count` value for a response, if one is owed. @@ -2524,6 +2638,7 @@ impl DefaultObjectUsecase { let read_stage_start = rustfs_io_metrics::get_stage_metrics_enabled().then_some(read_start); let store_headers = project_ssec_transport_headers(&req.headers); let cache_adapter = self.object_data_cache(); + let conditional = has_read_preconditions(&req.headers); if cache_adapter.is_disabled() || !cache_adapter.materialize_fill_enabled() { let io_planning = Self::acquire_get_object_io_planning( manager, @@ -2537,7 +2652,11 @@ impl DefaultObjectUsecase { .await?; let reader = track_object_read_setup( object_traffic_health.as_deref(), - store.get_object_reader(bucket, key, rs.clone(), store_headers, opts), + Self::with_get_object_conditions( + req, + part_number, + store.get_object_reader(bucket, key, rs.clone(), store_headers, opts), + ), ) .await .map_err(map_get_object_reader_error)?; @@ -2560,16 +2679,18 @@ impl DefaultObjectUsecase { ) .await?, ); - let mut prepared = Some( - track_object_read_setup( - object_traffic_health.as_deref(), - store.prepare_get_object_reader(bucket, key, rs.clone(), HeaderMap::new(), opts), - ) - .await - .map_err(map_get_object_reader_error)?, - ); + let initial_metadata = track_object_read_setup( + object_traffic_health.as_deref(), + store.prepare_get_object_reader(bucket, key, rs.clone(), HeaderMap::new(), opts), + ) + .await + .map_err(map_get_object_reader_error)?; let mut cache_fill_allowed = true; let mut legacy_hook_missed = false; + if conditional { + Self::validate_get_object_before_body(req, bucket, key, part_number, initial_metadata.object_info()).await?; + } + let mut prepared = Some(initial_metadata); 'snapshot: { let info = prepared .as_ref() @@ -2609,7 +2730,6 @@ impl DefaultObjectUsecase { if matches!(legacy_probe, GetObjectBodyCacheHookLookup::Ineligible) { break 'snapshot; } - Self::validate_get_object_before_cold_fill(&req.headers, part_number, info)?; if let GetObjectBodyCacheHookLookup::Hit(body) = legacy_probe { drop(metadata_admission.take()); let info = prepared @@ -2770,6 +2890,8 @@ impl DefaultObjectUsecase { return; }; + #[cfg(test)] + record_body_cache_fill_call_for_test(&engine_plan, true); let reservation = adapter.reserve_body(&engine_plan); #[cfg(test)] let reader_open_plan = engine_plan.clone(); @@ -2826,10 +2948,12 @@ impl DefaultObjectUsecase { let io_planning = metadata_admission .take() .ok_or_else(|| s3_error!(InternalError, "prepared metadata admission is unavailable"))?; - let reader = - track_object_read_setup(object_traffic_health.as_deref(), prepared.with_headers(store_headers).into_reader()) - .await - .map_err(map_get_object_reader_error)?; + let reader = track_object_read_setup( + object_traffic_health.as_deref(), + Self::with_get_object_conditions(req, part_number, prepared.with_headers(store_headers).into_reader()), + ) + .await + .map_err(map_get_object_reader_error)?; (io_planning, reader) } else { let io_planning = Self::acquire_get_object_io_planning( @@ -2849,13 +2973,20 @@ impl DefaultObjectUsecase { ) .await .map_err(map_get_object_reader_error)?; - track_object_read_setup(object_traffic_health.as_deref(), prepared.with_headers(store_headers).into_reader()) - .await - .map_err(map_get_object_reader_error)? + track_object_read_setup( + object_traffic_health.as_deref(), + Self::with_get_object_conditions(req, part_number, prepared.with_headers(store_headers).into_reader()), + ) + .await + .map_err(map_get_object_reader_error)? } else { track_object_read_setup( object_traffic_health.as_deref(), - store.get_object_reader(bucket, key, rs.clone(), store_headers, opts), + Self::with_get_object_conditions( + req, + part_number, + store.get_object_reader(bucket, key, rs.clone(), store_headers, opts), + ), ) .await .map_err(map_get_object_reader_error)? @@ -2900,6 +3031,7 @@ impl DefaultObjectUsecase { let stream = reader.stream; let buffered_body = reader.buffered_body; + Self::validate_get_object_before_body(req, bucket, key, part_number, &info).await?; let read_duration = read_start.elapsed(); // Conditional metrics recording to reduce overhead @@ -2909,9 +3041,6 @@ impl DefaultObjectUsecase { manager.record_disk_operation(info.size as u64, read_duration, true).await; } - check_preconditions(&req.headers, &info)?; - Self::validate_get_object_part_number(part_number, &info)?; - debug!(object_size = info.size, part_count = info.parts.len(), "GET object metadata snapshot"); for part in info.parts.iter() { debug!( @@ -3411,9 +3540,17 @@ impl DefaultObjectUsecase { let cache_plan = cache_plan.clone(); let fill_bytes = buffered_body.clone(); tokio::spawn(async move { + #[cfg(test)] + if let GetObjectBodyCachePlan::Cacheable(plan) = &cache_plan { + record_body_cache_fill_call_for_test(plan, false); + } let _ = fill_get_object_body_cache_from_buffered_body(&cache_adapter, &cache_plan, &fill_bytes).await; }); } else if cache_fill_allowed { + #[cfg(test)] + if let GetObjectBodyCachePlan::Cacheable(plan) = &cache_plan { + record_body_cache_fill_call_for_test(plan, false); + } let _ = fill_get_object_body_cache_from_buffered_body(cache_adapter, &cache_plan, &buffered_body).await; } @@ -3482,6 +3619,10 @@ impl DefaultObjectUsecase { let cache_plan = cache_plan.clone(); let fill_bytes = bytes.clone(); tokio::spawn(async move { + #[cfg(test)] + if let GetObjectBodyCachePlan::Cacheable(plan) = &cache_plan { + record_body_cache_fill_call_for_test(plan, false); + } let _ = fill_get_object_body_cache_from_materialized_body(&cache_adapter, &cache_plan, &fill_bytes).await; }); @@ -5365,7 +5506,11 @@ mod on_demand_migration_tests { let data = payload(512); let etag = source_head(&data).etag.expect("etag"); - let source = ScriptedSource::for_object(&data); + let head = source_head(&data); + let source = ScriptedSource::new( + vec![Ok(head.clone()), Ok(head.clone()), Ok(head.clone())], + vec![Ok((head, data.clone(), None))], + ); let mut headers = HeaderMap::new(); headers.insert(http::header::IF_NONE_MATCH, HeaderValue::from_str(&format!("\"{etag}\"")).unwrap()); let err = failed(odm_get_from_source(&state, &source, &headers, KEY, None, None).await); @@ -5373,7 +5518,6 @@ mod on_demand_migration_tests { assert_eq!(source.head_calls(), 1); assert_eq!(source.get_calls(), 0, "a 304 never pulls"); - let source = ScriptedSource::for_object(&data); let mut headers = HeaderMap::new(); headers.insert(http::header::IF_MATCH, HeaderValue::from_static("\"another-etag\"")); let err = failed(odm_get_from_source(&state, &source, &headers, KEY, None, None).await); @@ -5382,6 +5526,15 @@ mod on_demand_migration_tests { assert_eq!(get_count(&state, OdmOutcome::SourceHit), 2); assert_eq!(state.inflight_keys(), 0); assert!(rt.write_back.puts().is_empty()); + + let (mut output, _) = served(odm_get_from_source(&state, &source, &HeaderMap::new(), KEY, None, None).await); + assert_eq!( + collect_body(&mut output).await, + data, + "positive control still fetches the source representation" + ); + assert_eq!(source.get_calls(), 1, "the same SourceClient counter observes the positive-control GET"); + assert_eq!(source.head_calls(), 3, "each conditional and ordinary request reads source metadata"); } #[tokio::test] @@ -5671,6 +5824,28 @@ mod tests { use std::task::{Context, Poll}; use tokio::io::{AsyncRead, ReadBuf}; + #[tokio::test] + async fn conditional_reader_scope_keeps_large_futures_off_the_request_stack() { + let mut req = build_request(GetObjectInput::default(), Method::GET); + for conditional in [false, true] { + if conditional { + req.headers + .insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("\"etag\"")); + } + let payload = [0u8; 64 * 1024]; + let read = async move { + std::hint::black_box(payload); + std::future::ready(()).await; + }; + let read = DefaultObjectUsecase::with_get_object_conditions(&req, None, read); + assert!( + std::mem::size_of_val(&read) < 2048, + "reader scope must not embed the concrete reader future" + ); + read.await; + } + } + #[tokio::test(start_paused = true)] async fn cold_fill_disk_admission_preserves_slow_down() { let manager = Box::leak(Box::new(ConcurrencyManager::with_disk_read_caps_for_test(1, 1))); @@ -5690,6 +5865,184 @@ mod tests { drop(primary); } + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn conditional_get_304_uses_metadata_without_decoding_damaged_data() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + let (store, ambient) = real_cold_fill_test_context().await; + let context = temp_env::with_vars([(rustfs_config::ENV_OBJECT_DATA_CACHE_ENABLE, Some("false"))], || { + Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms())) + }); + assert!(context.object_data_cache().is_disabled()); + let usecase = DefaultObjectUsecase::with_context(Some(context)); + let (disk_paths, _) = crate::app::gating_test_env::shared_gating_ecstore_and_disk_paths().await; + let bucket = format!("conditional-metadata-{}", Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create test bucket"); + for (object, length) in [("inline.bin", 4096), ("shards.bin", 1_300_000)] { + let payload = vec![b'c'; length]; + let info = put_real_cold_fill_object(&store, &bucket, object, &payload).await; + let etag = info.etag.as_deref().expect("committed ETag"); + for disk in &disk_paths { + let object_path = disk.join(&bucket).join(object); + if length == 4096 { + let meta_path = object_path.join("xl.meta"); + let mut metadata = tokio::fs::read(&meta_path).await.expect("read inline metadata"); + *metadata.last_mut().expect("inline data bytes") ^= 1; + tokio::fs::write(&meta_path, metadata).await.expect("damage inline payload"); + } else { + let data_path = object_path.join(info.data_dir.expect("sharded object data directory").to_string()); + tokio::fs::remove_dir_all(data_path).await.expect("remove object data shards"); + } + } + let resolved = store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("metadata remains readable"); + assert_eq!(resolved.etag.as_deref(), Some(etag)); + let mut req = build_request( + GetObjectInput::builder() + .bucket(bucket.clone()) + .key(object.to_owned()) + .build() + .expect("GET input"), + Method::GET, + ); + req.headers.insert( + http::header::IF_NONE_MATCH, + HeaderValue::from_str(&format!("W/\"{etag}\"")).expect("ETag header"), + ); + let error = usecase + .execute_get_object(req) + .await + .expect_err("matching condition returns 304"); + assert_eq!(error.code(), &S3ErrorCode::NotModified); + assert_eq!(error.headers().expect("304 validators")["etag"], format!("\"{etag}\"")); + + let mut req = build_request( + HeadObjectInput::builder() + .bucket(bucket.clone()) + .key(object.to_owned()) + .build() + .expect("HEAD input"), + Method::HEAD, + ); + req.headers.insert( + http::header::IF_NONE_MATCH, + HeaderValue::from_str(&format!("\"{etag}\"")).expect("ETag header"), + ); + let error = usecase + .execute_head_object(req) + .await + .expect_err("conditional HEAD returns 304"); + assert_eq!(error.code(), &S3ErrorCode::NotModified); + assert_eq!(error.headers().expect("HEAD validators")["etag"], format!("\"{etag}\"")); + + let req = build_request( + GetObjectInput::builder() + .bucket(bucket.clone()) + .key(object.to_owned()) + .build() + .expect("GET input"), + Method::GET, + ); + if let Ok(mut response) = usecase.execute_get_object(req).await { + let mut body = response.output.body.take().expect("body for damaged GET"); + let mut failed = false; + while let Some(chunk) = body.next().await { + if chunk.is_err() { + failed = true; + break; + } + } + assert!(failed, "ordinary GET must still reject damaged data"); + } + } + } + + #[tokio::test] + async fn conditional_read_checks_encryption_before_not_modified() { + let mut info = ObjectInfo { + etag: Some("abc".to_owned()), + ..Default::default() + }; + info.user_defined = std::collections::HashMap::from([( + "x-amz-server-side-encryption-customer-algorithm".to_owned(), + "AES256".to_owned(), + )]) + .into(); + let mut req = build_request( + GetObjectInput::builder() + .bucket("bucket".to_owned()) + .key("key".to_owned()) + .build() + .expect("GET input"), + Method::GET, + ); + req.headers + .insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("\"abc\"")); + let error = DefaultObjectUsecase::validate_get_object_before_body(&req, "bucket", "key", None, &info) + .await + .expect_err("SSE-C key is mandatory before 304"); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn conditional_get_miss_preserves_full_and_range_bodies() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + let (store, ambient) = real_cold_fill_test_context().await; + let context = temp_env::with_vars([(rustfs_config::ENV_OBJECT_DATA_CACHE_ENABLE, Some("false"))], || { + Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms())) + }); + let usecase = DefaultObjectUsecase::with_context(Some(context)); + let bucket = format!("conditional-miss-{}", Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let payload: Vec = (0..1_300_000) + .map(|index| u8::try_from(index % 251).expect("pattern byte")) + .collect(); + put_real_cold_fill_object(&store, &bucket, "object", &payload).await; + for ranged in [false, true] { + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key("object".to_owned()) + .range(ranged.then_some(Range::Int { + first: 10, + last: Some(29), + })) + .build() + .expect("GET input"); + let mut req = build_request(input, Method::GET); + req.headers + .insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("\"other\"")); + if ranged { + req.headers + .insert(http::header::RANGE, HeaderValue::from_static("bytes=10-29")); + } + let mut response = usecase + .execute_get_object(req) + .await + .expect("unmatched condition serves body"); + let expected = if ranged { &payload[10..30] } else { payload.as_slice() }; + assert_eq!( + response.output.content_length, + Some(i64::try_from(expected.len()).expect("test body length")) + ); + assert_eq!(response.output.content_range.as_deref(), ranged.then_some("bytes 10-29/1300000")); + let mut body = response.output.body.take().expect("GET body"); + let mut bytes = Vec::new(); + while let Some(chunk) = body.next().await { + bytes.extend_from_slice(&chunk.expect("body chunk")); + } + assert_eq!(bytes, expected); + } + } + #[tokio::test] async fn cold_fill_closed_disk_admission_is_not_slow_down() { let manager = Box::leak(Box::new(ConcurrencyManager::with_disk_read_caps_for_test(1, 1))); @@ -6665,6 +7018,100 @@ mod tests { assert_eq!(coordinator.active_session_count_for_test(), 0); } + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn conditional_get_terminal_response_skips_cache_fill_calls_with_live_positive_control() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + struct ResetProbes; + impl Drop for ResetProbes { + fn drop(&mut self) { + *BODY_CACHE_FILL_PROBE + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("cache-fill probe") = None; + *COLD_FILL_READER_OPEN_PROBE + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("reader-open probe") = None; + } + } + let (store, context) = real_cold_fill_test_context().await; + let bucket = format!("conditional-fill-calls-{}", Uuid::new_v4()); + let object = "object.bin"; + let payload: Vec<_> = (0..1_300_000) + .map(|index| u8::try_from(index % 251).expect("pattern byte")) + .collect(); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let info = put_real_cold_fill_object(&store, &bucket, object, &payload).await; + let adapter = context.object_data_cache(); + let plan = real_cold_fill_plan(&adapter, &bucket, object, &info); + let key = plan.key().expect("cacheable key").clone(); + let fills = Arc::new(BodyCacheFillCalls::default()); + let reader_opens = Arc::new(AtomicU64::new(0)); + let _reset = ResetProbes; + *BODY_CACHE_FILL_PROBE + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("cache-fill probe") = Some((key.clone(), Arc::clone(&fills))); + *COLD_FILL_READER_OPEN_PROBE + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("reader-open probe") = Some((key, Arc::clone(&reader_opens))); + let usecase = DefaultObjectUsecase::with_context(Some(context)); + let input = GetObjectInput::builder() + .bucket(bucket) + .key(object.to_owned()) + .build() + .expect("GET input"); + for (header, value, expected) in [ + ( + http::header::IF_NONE_MATCH, + format!("\"{}\"", info.etag.as_deref().expect("ETag")), + S3ErrorCode::NotModified, + ), + (http::header::IF_MATCH, "\"other-etag\"".to_owned(), S3ErrorCode::PreconditionFailed), + ] { + let mut req = build_request(input.clone(), Method::GET); + req.headers.insert(header, HeaderValue::from_str(&value).expect("condition")); + let error = usecase.execute_get_object(req).await.expect_err("terminal condition"); + assert_eq!(error.code(), &expected); + assert_eq!(reader_opens.load(Ordering::Relaxed), 0, "no cold-fill body factory call"); + assert_eq!(fills.reservations.load(Ordering::Relaxed), 0, "no cache-fill reservation call"); + assert_eq!(fills.fills.load(Ordering::Relaxed), 0, "no cache-fill publish call"); + assert_eq!(adapter.cold_fill_coordinator().active_session_count_for_test(), 0); + } + + let mut response = usecase + .execute_get_object(build_request(input, Method::GET)) + .await + .expect("positive-control GET"); + let mut body = response.output.body.take().expect("body"); + let mut actual = Vec::new(); + while let Some(chunk) = body.next().await { + actual.extend_from_slice(&chunk.expect("positive-control chunk")); + } + assert_eq!(actual, payload); + assert!( + reader_opens.load(Ordering::Relaxed) > 0, + "the same factory probe observes the positive control" + ); + assert!( + fills.reservations.load(Ordering::Relaxed) > 0, + "the same reservation probe observes the positive control" + ); + assert!( + fills.fills.load(Ordering::Relaxed) > 0, + "the same fill probe observes the positive control" + ); + assert!( + matches!(adapter.lookup_body(&plan).await, rustfs_object_data_cache::ObjectDataCacheLookup::Hit(_)), + "the positive control really populated the cache" + ); + } + #[tokio::test] #[serial_test::serial(body_cache_hook)] async fn execute_get_object_maps_cold_fill_session_rejection_to_slow_down_without_opening_reader() { @@ -6732,6 +7179,16 @@ mod tests { #[tokio::test] #[serial_test::serial(body_cache_hook)] async fn execute_get_object_generation_change_bypasses_old_cold_fill_plan() { + run_conditional_generation_change(false).await; + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn conditional_get_rechecks_generation_before_fallback_body() { + run_conditional_generation_change(true).await; + } + + async fn run_conditional_generation_change(conditional: bool) { use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; let (store, context) = real_cold_fill_test_context().await; @@ -6743,6 +7200,16 @@ mod tests { .expect("real cold-fill generation bucket must be created"); let initial_body = vec![b'a'; 1_300_000]; let changed_body = vec![b'b'; initial_body.len()]; + let condition_etag = if conditional { + Some( + put_real_cold_fill_object(&store, &bucket, object, &changed_body) + .await + .etag + .expect("replacement ETag"), + ) + } else { + None + }; let initial_info = put_real_cold_fill_object(&store, &bucket, object, &initial_body).await; let adapter = context.object_data_cache(); let initial_plan = real_cold_fill_plan(&adapter, &bucket, object, &initial_info); @@ -6769,7 +7236,14 @@ mod tests { ..GetObjectTimeoutPolicy::default() }, ); - let request = tokio::spawn(async move { usecase.execute_get_object(build_request(input, Method::GET)).await }); + let mut req = build_request(input, Method::GET); + if let Some(etag) = &condition_etag { + req.headers.insert( + http::header::IF_NONE_MATCH, + HeaderValue::from_str(&format!("\"{etag}\"")).expect("replacement condition"), + ); + } + let request = tokio::spawn(async move { usecase.execute_get_object(req).await }); tokio::time::timeout(Duration::from_secs(2), async { while coordinator.global_waiter_count_for_test() != 1 { tokio::task::yield_now().await; @@ -6780,13 +7254,32 @@ mod tests { let changed_info = put_real_cold_fill_object(&store, &bucket, object, &changed_body).await; assert_ne!(initial_info.etag, changed_info.etag); + if conditional { + let (paths, _) = crate::app::gating_test_env::shared_gating_ecstore_and_disk_paths().await; + for path in paths { + tokio::fs::remove_dir_all( + path.join(&bucket) + .join(object) + .join(changed_info.data_dir.expect("replacement data directory").to_string()), + ) + .await + .expect("remove replacement shards before fallback"); + } + } producer.relinquish_or_finish(ColdFillError::Storage(StorageError::Timeout)); - let mut response = tokio::time::timeout(Duration::from_secs(10), request) + let result = tokio::time::timeout(Duration::from_secs(10), request) .await .expect("generation-changing GET must complete") - .expect("generation-changing GET task must join") - .expect("generation-changing GET must fall back successfully"); + .expect("generation-changing GET task must join"); + if conditional { + let error = result.expect_err("fallback must recheck the replacement ETag before body access"); + assert_eq!(error.code(), &S3ErrorCode::NotModified); + assert_eq!(coordinator.global_waiter_count_for_test(), 0); + assert_eq!(coordinator.active_session_count_for_test(), 0); + return; + } + let mut response = result.expect("generation-changing GET must fall back successfully"); let mut response_body = response.output.body.take().expect("GET response must include a body"); let mut actual = Vec::with_capacity(changed_body.len()); while let Some(chunk) = response_body.next().await { @@ -10977,53 +11470,6 @@ mod tests { assert_eq!(DefaultObjectUsecase::get_object_parts_count(Some(1), &single_part_multipart), Some(1)); } - #[test] - fn cold_fill_conditions_fail_before_phase_probe_advances() { - fn run_phase_probe(headers: &HeaderMap, info: &ObjectInfo) -> (S3Result<()>, [usize; 3]) { - let coordination = AtomicUsize::new(0); - let permit = AtomicUsize::new(0); - let reader = AtomicUsize::new(0); - let result = DefaultObjectUsecase::validate_get_object_before_cold_fill(headers, None, info); - if result.is_ok() { - coordination.fetch_add(1, AtomicOrdering::Relaxed); - permit.fetch_add(1, AtomicOrdering::Relaxed); - reader.fetch_add(1, AtomicOrdering::Relaxed); - } - ( - result, - [ - coordination.load(AtomicOrdering::Relaxed), - permit.load(AtomicOrdering::Relaxed), - reader.load(AtomicOrdering::Relaxed), - ], - ) - } - - let info = ObjectInfo { - etag: Some("phase-etag".to_string()), - parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { - number: 1, - ..Default::default() - }]), - ..Default::default() - }; - - let mut not_modified = HeaderMap::new(); - not_modified.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("\"phase-etag\"")); - let (result, phases) = run_phase_probe(¬_modified, &info); - assert_eq!(result.expect_err("matching If-None-Match must reject").code(), &S3ErrorCode::NotModified); - assert_eq!(phases, [0, 0, 0]); - - let mut precondition_failed = HeaderMap::new(); - precondition_failed.insert(http::header::IF_MATCH, HeaderValue::from_static("\"other-etag\"")); - let (result, phases) = run_phase_probe(&precondition_failed, &info); - assert_eq!( - result.expect_err("mismatched If-Match must reject").code(), - &S3ErrorCode::PreconditionFailed - ); - assert_eq!(phases, [0, 0, 0]); - } - #[tokio::test] async fn execute_get_object_rejects_range_with_part_number() { let input = GetObjectInput::builder() diff --git a/rustfs/src/app/object/head.rs b/rustfs/src/app/object/head.rs index 4d53df5ad..35537850b 100644 --- a/rustfs/src/app/object/head.rs +++ b/rustfs/src/app/object/head.rs @@ -312,10 +312,6 @@ impl DefaultObjectUsecase { version_id, part_number, range, - if_none_match, - if_match, - if_modified_since, - if_unmodified_since, .. } = req.input.clone(); @@ -343,6 +339,7 @@ impl DefaultObjectUsecase { let opts: ObjectOptions = get_opts(&bucket, &key, version_id, part_number, &req.headers) .await .map_err(ApiError::from)?; + let opts = internal_object_info_lookup_opts(opts); prepare_odm_read_generation(&store, &mut req, &bucket).await; @@ -413,41 +410,6 @@ impl DefaultObjectUsecase { opts.version_id.as_deref(), )); } - if let Some(match_etag) = if_none_match - && let Some(strong_etag) = match_etag.into_etag() - && info - .etag - .as_ref() - .is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag) - { - return Err(S3Error::new(S3ErrorCode::NotModified)); - } - if let Some(modified_since) = if_modified_since { - // obj_time < givenTime + 1s - if info.mod_time.is_some_and(|mod_time| { - let give_time: OffsetDateTime = modified_since.into(); - mod_time < give_time.add(time::Duration::seconds(1)) - }) { - return Err(S3Error::new(S3ErrorCode::NotModified)); - } - } - if let Some(match_etag) = if_match { - if let Some(strong_etag) = match_etag.into_etag() - && info - .etag - .as_ref() - .is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag) - { - return Err(S3Error::new(S3ErrorCode::PreconditionFailed)); - } - } else if let Some(unmodified_since) = if_unmodified_since - && info.mod_time.is_some_and(|mod_time| { - let give_time: OffsetDateTime = unmodified_since.into(); - mod_time > give_time.add(time::Duration::seconds(1)) - }) - { - return Err(S3Error::new(S3ErrorCode::PreconditionFailed)); - } // An authorized replication convergence check only needs etag/size/mtime // to compare source and replica; it holds no customer key, so the SSE-C // read validation is skipped for it (and only it). @@ -465,6 +427,14 @@ impl DefaultObjectUsecase { )?; } + let conditions = check_preconditions(&req.headers, &info); + if conditions.is_err() && !replication_check { + let principal = SseKmsPrincipal::from_request(&req); + crate::storage::sse::validate_conditional_read(&bucket, &key, &info.user_defined, &req.headers, principal.as_ref()) + .await?; + } + conditions?; + // Compute x-amz-expiration header from lifecycle prediction (before info is partially moved) let expiration_header = resolve_put_object_expiration(&bucket, &info).await; // Clone ObjectInfo for event notification only when an event will diff --git a/rustfs/src/app/object/mod.rs b/rustfs/src/app/object/mod.rs index e6e450436..070de596b 100644 --- a/rustfs/src/app/object/mod.rs +++ b/rustfs/src/app/object/mod.rs @@ -205,6 +205,8 @@ fn remove_source_replication_bookkeeping(user_defined: &mut HashMap, + } + + #[derive(Debug)] + struct FixtureTransportError(String); + + impl std::fmt::Display for FixtureTransportError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.0) + } + } + + impl std::error::Error for FixtureTransportError {} + + type HttpConditionCase<'a> = (&'a str, &'a [(&'a str, &'a str)], StatusCode, bool); + + impl<'a> From> for FixtureTransportError { + fn from(error: Box) -> Self { + Self(error.to_string()) + } + } + + fn object_info() -> StorageObjectInfo { + StorageObjectInfo { + etag: Some("abc".to_owned()), + mod_time: Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixture timestamp")), + user_defined: HashMap::from([("cache-control".to_owned(), "max-age=60".to_owned())]).into(), + ..Default::default() + } + } + + #[async_trait::async_trait] + impl s3s::S3 for ConditionalObjectS3 { + async fn get_object( + &self, + req: s3s::S3Request, + ) -> s3s::S3Result> { + self.calls.fetch_add(1, Ordering::Relaxed); + assert_eq!(req.input.bucket, "bucket"); + assert_eq!(req.input.key, "object"); + assert_eq!(req.input.if_match.is_some(), req.headers.contains_key(http::header::IF_MATCH)); + assert_eq!(req.input.if_none_match.is_some(), req.headers.contains_key(http::header::IF_NONE_MATCH)); + check_preconditions(&req.headers, &object_info())?; + Ok(s3s::S3Response::new(s3s::dto::GetObjectOutput { + body: Some(s3s::dto::StreamingBlob::from_bytes(Bytes::from_static(b"payload"))), + content_length: Some(7), + e_tag: Some(s3s::dto::ETag::Strong("abc".to_owned())), + ..Default::default() + })) + } + + async fn head_object( + &self, + req: s3s::S3Request, + ) -> s3s::S3Result> { + self.calls.fetch_add(1, Ordering::Relaxed); + assert_eq!(req.input.bucket, "bucket"); + assert_eq!(req.input.key, "object"); + assert_eq!(req.input.if_match.is_some(), req.headers.contains_key(http::header::IF_MATCH)); + assert_eq!(req.input.if_none_match.is_some(), req.headers.contains_key(http::header::IF_NONE_MATCH)); + check_preconditions(&req.headers, &object_info())?; + Ok(s3s::S3Response::new(s3s::dto::HeadObjectOutput { + content_length: Some(7), + e_tag: Some(s3s::dto::ETag::Strong("abc".to_owned())), + ..Default::default() + })) + } + } + + #[tokio::test] + async fn conditional_get_head_http1_http2_parse_headers_and_apply_precedence() { + let cases: &[HttpConditionCase<'_>] = &[ + ("strong list", &[("if-none-match", "\"other\", \"abc\"")], StatusCode::NOT_MODIFIED, true), + ("weak list", &[("if-none-match", "\"other\", W/\"abc\"")], StatusCode::NOT_MODIFIED, true), + ("weak tag", &[("if-none-match", "W/\"abc\"")], StatusCode::NOT_MODIFIED, true), + ("wildcard", &[("if-none-match", "*")], StatusCode::NOT_MODIFIED, true), + ("strong match list", &[("if-match", "\"other\", \"abc\"")], StatusCode::OK, true), + ("weak match fails", &[("if-match", "W/\"abc\"")], StatusCode::PRECONDITION_FAILED, true), + ("wildcard match", &[("if-match", "*")], StatusCode::OK, true), + ( + "If-Match precedes matching If-None-Match", + &[("if-match", "\"other\""), ("if-none-match", "\"abc\"")], + StatusCode::PRECONDITION_FAILED, + true, + ), + ( + "matching If-Match does not suppress If-None-Match", + &[("if-match", "\"abc\""), ("if-none-match", "W/\"abc\"")], + StatusCode::NOT_MODIFIED, + true, + ), + ( + "If-Match suppresses If-Unmodified-Since", + &[ + ("if-match", "\"abc\""), + ("if-unmodified-since", "Tue, 14 Nov 2023 22:13:19 GMT"), + ], + StatusCode::OK, + true, + ), + ( + "If-None-Match suppresses If-Modified-Since", + &[ + ("if-none-match", "\"other\""), + ("if-modified-since", "Tue, 14 Nov 2023 22:13:21 GMT"), + ], + StatusCode::OK, + true, + ), + ( + "If-Unmodified-Since precedes wildcard If-None-Match", + &[ + ("if-unmodified-since", "Tue, 14 Nov 2023 22:13:19 GMT"), + ("if-none-match", "*"), + ], + StatusCode::PRECONDITION_FAILED, + true, + ), + ( + "date matches at same second", + &[("if-modified-since", "Tue, 14 Nov 2023 22:13:20 GMT")], + StatusCode::NOT_MODIFIED, + true, + ), + ("unmatched list", &[("if-none-match", "\"other\", W/\"different\"")], StatusCode::OK, true), + ( + "empty list member is allowed", + &[("if-none-match", "\"other\", , \"abc\"")], + StatusCode::NOT_MODIFIED, + true, + ), + // s3s accepts these quoted fields; the canonical matcher must validate every member. + ( + "adjacent tags cannot validate cache", + &[("if-none-match", "\"other\"\"abc\"")], + StatusCode::OK, + true, + ), + ( + "adjacent tags cannot satisfy If-Match", + &[("if-match", "\"other\"\"abc\"")], + StatusCode::PRECONDITION_FAILED, + true, + ), + ( + "matching prefix with trailing garbage", + &[("if-none-match", "\"abc\"garbage\"")], + StatusCode::OK, + true, + ), + ( + "matching prefix cannot satisfy If-Match", + &[("if-match", "\"abc\"garbage\"")], + StatusCode::PRECONDITION_FAILED, + true, + ), + ( + "matching first tag with malformed later member", + &[("if-none-match", "\"abc\", \"other\"\"different\"")], + StatusCode::OK, + true, + ), + ( + "If-Match validates malformed later member", + &[("if-match", "\"abc\", \"other\"\"different\"")], + StatusCode::PRECONDITION_FAILED, + true, + ), + // These existing s3s parse boundaries reject the request before the handler. + ("malformed ETag", &[("if-none-match", "\"abc")], StatusCode::BAD_REQUEST, false), + ("invalid date", &[("if-modified-since", "not-a-date")], StatusCode::BAD_REQUEST, false), + ( + "obsolete date", + &[("if-modified-since", "Tuesday, 14-Nov-23 22:13:20 GMT")], + StatusCode::BAD_REQUEST, + false, + ), + ( + "duplicate condition", + &[("if-none-match", "\"other\""), ("if-none-match", "\"abc\"")], + StatusCode::BAD_REQUEST, + false, + ), + ]; + for http2 in [false, true] { + for method in [Method::GET, Method::HEAD] { + for (name, headers, expected_status, reaches_handler) in cases { + let calls = Arc::new(AtomicUsize::new(0)); + let inner = S3ServiceBuilder::new(ConditionalObjectS3 { + calls: Arc::clone(&calls), + }) + .build(); + let service = ServiceBuilder::new() + .layer(BodylessStatusFixLayer) + .layer(HeadRequestBodyFixLayer) + .map_response(|response: Response| { + response.map(|rest_body| HybridBody::>::Rest { rest_body }) + }) + .map_err(|error: s3s::HttpError| FixtureTransportError(error.to_string())) + .service(inner); + let (client_io, server_io) = tokio::io::duplex(64 * 1024); + let server = tokio::spawn(async move { + let service = TowerToHyperService::new(service); + if http2 { + hyper::server::conn::http2::Builder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(server_io), service) + .await + .expect("HTTP/2 server"); + } else { + hyper::server::conn::http1::Builder::new() + .serve_connection(TokioIo::new(server_io), service) + .await + .expect("HTTP/1 server"); + } + }); + let mut request = HttpRequest::builder() + .method(method.clone()) + .uri("http://localhost/bucket/object"); + for (header, value) in *headers { + request = request.header(*header, *value); + } + let request = request.body(Empty::::new()).expect("conditional request"); + let (response, client) = if http2 { + let (mut sender, connection) = + hyper::client::conn::http2::handshake(TokioExecutor::new(), TokioIo::new(client_io)) + .await + .expect("HTTP/2 handshake"); + let client = tokio::spawn(async move { + connection.await.expect("HTTP/2 client"); + }); + let response = tokio::time::timeout(Duration::from_secs(5), sender.send_request(request)) + .await + .expect("HTTP/2 response deadline") + .expect("HTTP/2 response"); + (response, client) + } else { + let (mut sender, connection) = hyper::client::conn::http1::handshake(TokioIo::new(client_io)) + .await + .expect("HTTP/1 handshake"); + let client = tokio::spawn(async move { + connection.await.expect("HTTP/1 client"); + }); + let response = tokio::time::timeout(Duration::from_secs(5), sender.send_request(request)) + .await + .expect("HTTP/1 response deadline") + .expect("HTTP/1 response"); + (response, client) + }; + assert_eq!(response.status(), *expected_status, "{name}: {method}, HTTP/2={http2}"); + assert_eq!(calls.load(Ordering::Relaxed), usize::from(*reaches_handler), "{name}: parser routing"); + if *expected_status == StatusCode::NOT_MODIFIED { + assert_eq!(response.headers()[http::header::ETAG], "\"abc\"", "{name}"); + assert_eq!(response.headers()[http::header::CACHE_CONTROL], "max-age=60", "{name}"); + assert!(response.headers().contains_key(http::header::LAST_MODIFIED), "{name}"); + assert!(response.headers().contains_key(http::header::DATE), "{name}"); + assert!(!response.headers().contains_key(CONTENT_LENGTH), "{name}"); + assert!(!response.headers().contains_key(http::header::TRANSFER_ENCODING), "{name}"); + } + let bytes = tokio::time::timeout(Duration::from_secs(5), response.into_body().collect()) + .await + .expect("body deadline") + .expect("response body") + .to_bytes(); + if method == Method::HEAD || *expected_status == StatusCode::NOT_MODIFIED { + assert!(bytes.is_empty(), "{name}: {method}, HTTP/2={http2}"); + } else if *expected_status == StatusCode::OK { + assert_eq!(&bytes[..], b"payload", "{name}"); + } else if *expected_status == StatusCode::PRECONDITION_FAILED { + assert!(String::from_utf8_lossy(&bytes).contains("PreconditionFailed"), "{name}"); + } + client.abort(); + server.abort(); + } + } + } + } + } + #[derive(Clone, Copy)] struct UploadPartTimeoutS3 { timeout: Duration, diff --git a/rustfs/src/server/layer.rs b/rustfs/src/server/layer.rs index c6657cddf..cff0e72a8 100644 --- a/rustfs/src/server/layer.rs +++ b/rustfs/src/server/layer.rs @@ -1175,15 +1175,9 @@ where /// responses whose HTTP status code MUST NOT carry a body per RFC 9110 §6.4.1 /// and §15 (1xx, 204, 205, 304). /// -/// The inner s3s layer serializes every `S3Error` — including 304 `NotModified` -/// preconditions — as an XML body. Returning that body for a 304 is a protocol -/// violation: hyper's HTTP/1.1 encoder forces the body to zero length but -/// preserves the response, while the HTTP/2 path fills in `content-length` -/// from the body's size hint and writes DATA frames after a HEADERS frame that -/// should have carried END_STREAM. h2 clients (curl, browsers) and proxies see -/// the malformed response as a connection-level failure — in the wild this -/// surfaces as `GOAWAY error=0` on h2 and as an upstream-disconnect 5xx from -/// reverse proxies like ngrok (`ERR_NGROK_3004`). +/// s3s already suppresses bodyless error responses. This layer also protects +/// REST routes and custom responses: unlike the HTTP/1.1 encoder, hyper's +/// HTTP/2 sender relies on the body rather than the status to end the stream. #[derive(Clone)] pub struct BodylessStatusFixLayer; @@ -5312,6 +5306,62 @@ mod tests { assert!(bytes.is_empty(), "304 response body must be empty"); } + #[tokio::test] + async fn bodyless_304_is_valid_on_http1_and_http2() { + use hyper_util::rt::{TokioExecutor, TokioIo}; + use hyper_util::service::TowerToHyperService; + for http2 in [false, true] { + let (client_io, server_io) = tokio::io::duplex(4096); + let service = TowerToHyperService::new(BodylessStatusFixLayer.layer(FixedResponse { + status: StatusCode::NOT_MODIFIED, + body: Bytes::from_static(b"NotModified"), + content_type: Some("application/xml"), + })); + let server = tokio::spawn(async move { + if http2 { + hyper::server::conn::http2::Builder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(server_io), service) + .await + .expect("HTTP/2 server"); + } else { + hyper::server::conn::http1::Builder::new() + .serve_connection(TokioIo::new(server_io), service) + .await + .expect("HTTP/1 server"); + } + }); + let request = Request::builder() + .uri("http://localhost/object") + .body(Empty::::new()) + .expect("request"); + let (response, client) = if http2 { + let (mut sender, connection) = + hyper::client::conn::http2::handshake(TokioExecutor::new(), TokioIo::new(client_io)) + .await + .expect("HTTP/2 handshake"); + let client = tokio::spawn(async move { + connection.await.expect("HTTP/2 client"); + }); + (sender.send_request(request).await.expect("HTTP/2 response"), client) + } else { + let (mut sender, connection) = hyper::client::conn::http1::handshake(TokioIo::new(client_io)) + .await + .expect("HTTP/1 handshake"); + let client = tokio::spawn(async move { + connection.await.expect("HTTP/1 client"); + }); + (sender.send_request(request).await.expect("HTTP/1 response"), client) + }; + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + assert_eq!(response.headers()[http::header::ETAG], "\"abc123\""); + assert!(!response.headers().contains_key(http::header::CONTENT_LENGTH)); + assert!(!response.headers().contains_key(http::header::CONTENT_TYPE)); + assert!(collect_body(response.into_body()).await.is_empty()); + client.abort(); + server.abort(); + } + } + #[tokio::test] async fn strips_body_for_204() { let mut svc = BodylessStatusFixLayer.layer(FixedResponse { diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index d6b622c3f..382bf6451 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -37,7 +37,6 @@ use s3s::dto::{ use s3s::{S3Error, S3ErrorCode, S3Response, S3Result}; use serde_urlencoded::from_bytes; use std::collections::HashMap; -use std::ops::Add; use std::sync::{Arc, OnceLock, RwLock}; use std::time::{Duration, Instant}; use time::OffsetDateTime; @@ -529,22 +528,29 @@ pub(crate) fn validate_bucket_object_lock_enabled_state(bucket: &str, state: &Ob /// has not changed), allowing the caller to translate this into the correct /// HTTP response. pub(crate) fn check_preconditions(headers: &HeaderMap, info: &ObjectInfo) -> S3Result<()> { - let mod_time = info.mod_time; - let etag = info.etag.as_deref(); + match evaluate_read_preconditions(headers, info.etag.as_deref(), info.mod_time) { + Ok(()) => Ok(()), + Err(S3ErrorCode::NotModified) => Err(not_modified_error(info)?), + Err(code) => Err(S3Error::new(code)), + } +} + +pub(crate) fn evaluate_read_preconditions( + headers: &HeaderMap, + etag: Option<&str>, + mod_time: Option, +) -> Result<(), S3ErrorCode> { let if_match = non_empty_header_value(headers, IF_MATCH); let if_none_match = non_empty_header_value(headers, IF_NONE_MATCH); let if_modified_since = non_empty_header_value(headers, IF_MODIFIED_SINCE); let if_unmodified_since = non_empty_header_value(headers, IF_UNMODIFIED_SINCE); - if mod_time.is_none() && etag.is_none() { - return Ok(()); - } - - // If-Match: requires ETag to exist + // The caller has resolved an existing object, so '*' does not require an ETag. if let Some(if_match_val) = if_match { match etag { - Some(e) if is_etag_equal(e, if_match_val) => {} - _ => return Err(S3Error::new(S3ErrorCode::PreconditionFailed)), + _ if if_match_val == "*" => {} + Some(e) if read_etag_matches(e, if_match_val, false) => {} + _ => return Err(S3ErrorCode::PreconditionFailed), } } @@ -553,32 +559,16 @@ pub(crate) fn check_preconditions(headers: &HeaderMap, info: &ObjectInfo) -> S3R && let Some(t) = mod_time && let Some(if_unmodified_since) = if_unmodified_since && let Ok(given_time) = time::PrimitiveDateTime::parse(if_unmodified_since, &RFC1123).map(|dt| dt.assume_utc()) - && t > given_time.add(time::Duration::seconds(1)) + && t.unix_timestamp() > given_time.unix_timestamp() { - return Err(S3Error::new(S3ErrorCode::PreconditionFailed)); + return Err(S3ErrorCode::PreconditionFailed); } // If-None-Match if let Some(if_none_match) = if_none_match - && let Some(e) = etag - && is_etag_equal(e, if_none_match) + && (if_none_match == "*" || etag.is_some_and(|e| read_etag_matches(e, if_none_match, true))) { - let mut error_headers = HeaderMap::new(); - if let Ok(etag_header) = parse_etag(e) { - error_headers.insert("etag", etag_header); - } - if let Some(t) = mod_time - && let Ok(last_modified_str) = t.format(&RFC1123) - && let Ok(last_modified_header) = HeaderValue::from_str(&last_modified_str) - { - error_headers.insert("last-modified", last_modified_header); - } - - let mut s3_error = S3Error::new(S3ErrorCode::NotModified); - s3_error.set_message("Not Modified".to_string()); - s3_error.set_status_code(StatusCode::NOT_MODIFIED); - s3_error.set_headers(error_headers); - return Err(s3_error); + return Err(S3ErrorCode::NotModified); } // If-Modified-Since (only when If-None-Match is absent — semantics per RFC 7232; dates use RFC 1123 format) @@ -586,31 +576,94 @@ pub(crate) fn check_preconditions(headers: &HeaderMap, info: &ObjectInfo) -> S3R && let Some(t) = mod_time && let Some(if_modified_since) = if_modified_since && let Ok(given_time) = time::PrimitiveDateTime::parse(if_modified_since, &RFC1123).map(|dt| dt.assume_utc()) - && t < given_time.add(time::Duration::seconds(1)) + && t.unix_timestamp() <= given_time.unix_timestamp() { - let mut error_headers = HeaderMap::new(); - if let Some(e) = etag - && let Ok(etag_header) = parse_etag(e) - { - error_headers.insert("etag", etag_header); - } - if let Ok(last_modified_str) = t.format(&RFC1123) - && let Ok(last_modified_header) = HeaderValue::from_str(&last_modified_str) - { - error_headers.insert("last-modified", last_modified_header); - } - - let mut s3_error = S3Error::new(S3ErrorCode::NotModified); - s3_error.set_message("Not Modified".to_string()); - s3_error.set_status_code(StatusCode::NOT_MODIFIED); - s3_error.set_headers(error_headers); - - return Err(s3_error); + return Err(S3ErrorCode::NotModified); } Ok(()) } +fn not_modified_error(info: &ObjectInfo) -> S3Result { + let mut headers = HeaderMap::new(); + if let Some(etag) = info.etag.as_deref() { + headers.insert(http::header::ETAG, parse_etag(etag).map_err(S3Error::internal_error)?); + } + if let Some(mod_time) = info.mod_time { + let value = mod_time.format(&RFC1123).map_err(S3Error::internal_error)?; + headers.insert( + http::header::LAST_MODIFIED, + HeaderValue::from_str(&value).map_err(S3Error::internal_error)?, + ); + } + for name in ["cache-control", "content-location", "vary", "expires"] { + if let Some(value) = info.user_defined.get(name) { + headers.insert(name, HeaderValue::from_str(value).map_err(S3Error::internal_error)?); + } + } + if let Some(expires) = info.expires { + let value = expires.format(&RFC1123).map_err(S3Error::internal_error)?; + headers.insert(http::header::EXPIRES, HeaderValue::from_str(&value).map_err(S3Error::internal_error)?); + } + if let Some(version) = info.version_id { + let value = if version.is_nil() { + "null".to_owned() + } else { + version.to_string() + }; + headers.insert("x-amz-version-id", HeaderValue::from_str(&value).map_err(S3Error::internal_error)?); + } + let mut error = S3Error::new(S3ErrorCode::NotModified); + error.set_status_code(StatusCode::NOT_MODIFIED); + error.set_message("Not Modified".to_owned()); + error.set_headers(headers); + Ok(error) +} + +/// GET/HEAD use weak comparison for If-None-Match and strong comparison for If-Match. +fn read_etag_matches(object_etag: &str, condition: &str, allow_weak: bool) -> bool { + let object_etag = object_etag.trim(); + let object_weak = object_etag.starts_with("W/"); + let object_value = object_etag.strip_prefix("W/").unwrap_or(object_etag).trim_matches('"'); + let mut remaining = condition.trim(); + let mut matched = false; + while !remaining.is_empty() { + // Recipients ignore empty list members, but tags still need commas. + if let Some(rest) = remaining.strip_prefix(',') { + remaining = rest.trim(); + continue; + } + let weak = remaining.starts_with("W/"); + let value = remaining.strip_prefix("W/").unwrap_or(remaining); + let (value, rest) = if let Some(quoted) = value.strip_prefix('"') { + let Some(pair) = quoted.split_once('"') else { return false }; + pair + } else { + // Preserve bare tags used by legacy internal callers. + if weak { + return false; + } + let (value, rest) = value.split_at(value.find(',').unwrap_or(value.len())); + let value = value.trim(); + if value.is_empty() || value == "*" { + return false; + } + (value, rest) + }; + if !value.bytes().all(|byte| byte == b'!' || (byte >= b'#' && byte != 0x7f)) { + return false; + } + matched |= value == object_value && (allow_weak || (!weak && !object_weak)); + let rest = rest.trim(); + if rest.is_empty() { + return matched; + } + let Some(rest) = rest.strip_prefix(',') else { return false }; + remaining = rest.trim(); + } + matched +} + fn non_empty_header_value(headers: &HeaderMap, name: http::header::HeaderName) -> Option<&str> { headers .get(name) @@ -619,33 +672,10 @@ fn non_empty_header_value(headers: &HeaderMap, name: http::header::HeaderName) - .filter(|v| !v.is_empty()) } -/// Compares an object ETag with an ETag value from an HTTP header. -/// -/// This helper implements HTTP ETag comparison semantics for headers such as -/// `If-Match` and `If-None-Match`: -/// - Supports the wildcard `*`, which matches any `object_etag`. -/// - Supports comma-separated ETag lists (e.g., `"etag1", "etag2"`), returning -/// `true` if any entry matches `object_etag`. -/// - Automatically trims surrounding whitespace and double quotes from both the -/// header entries and `object_etag` before comparison. -/// -/// # Parameters -/// - `object_etag`: The ETag associated with the stored object. -/// - `header_etag`: The raw ETag header value received in the request, which may -/// be a wildcard, a single ETag, or a comma-separated list of ETags. -/// -/// # Returns -/// `true` if the header value matches the object ETag according to the above -/// HTTP ETag comparison rules, otherwise `false`. -pub(crate) fn is_etag_equal(object_etag: &str, header_etag: &str) -> bool { - let header_etag = header_etag.trim(); - if header_etag == "*" { - return true; - } - header_etag - .split(',') - .map(|s| s.trim().trim_matches('"')) - .any(|e| e == object_etag.trim_matches('"')) +pub(crate) fn has_read_preconditions(headers: &HeaderMap) -> bool { + [IF_MATCH, IF_UNMODIFIED_SINCE, IF_NONE_MATCH, IF_MODIFIED_SINCE] + .into_iter() + .any(|name| non_empty_header_value(headers, name).is_some()) } /// Converts an object ETag string into an HTTP `HeaderValue` for use in response headers. diff --git a/rustfs/src/storage/ecfs_test.rs b/rustfs/src/storage/ecfs_test.rs index 18f96abfa..149ccf387 100644 --- a/rustfs/src/storage/ecfs_test.rs +++ b/rustfs/src/storage/ecfs_test.rs @@ -27,11 +27,11 @@ mod tests { use crate::storage::storage_api::test_consumer::{ BucketMetadata, DEFAULT_READ_BUFFER_SIZE, StorageObjectInfo as ObjectInfo, apply_cors_headers, apply_default_lock_retention_metadata, bucket_metadata_sys_initialized, check_preconditions, decode_tags_to_map, - get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, get_global_bucket_metadata_sys, is_etag_equal, - matches_origin_pattern, parse_etag, parse_object_lock_legal_hold, parse_object_lock_retention, - process_lambda_configurations, process_queue_configurations, process_topic_configurations, - remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata, set_bucket_metadata, - validate_bucket_object_lock_enabled, validate_list_object_unordered_with_delimiter, + get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, get_global_bucket_metadata_sys, matches_origin_pattern, + parse_etag, parse_object_lock_legal_hold, parse_object_lock_retention, process_lambda_configurations, + process_queue_configurations, process_topic_configurations, remove_object_lock_metadata_for_copy, + remove_object_lock_retention_metadata, set_bucket_metadata, validate_bucket_object_lock_enabled, + validate_list_object_unordered_with_delimiter, }; use http::{Extensions, HeaderMap, HeaderValue, Method, StatusCode, Uri}; use rustfs_config::MI_B; @@ -1191,30 +1191,72 @@ mod tests { } #[test] - fn test_is_etag_equal() { - // [1] Header ETag is "*", should return true (match any object ETag) - assert!(is_etag_equal("\"d41d8cd98f00b204e9800998ecf8427e\"", "*")); + fn test_read_etag_comparison() { + for (object_etag, condition, matches) in [ + ("\"d41d8cd98f00b204e9800998ecf8427e\"", "*", true), + ("\"d41d8cd98f00b204e9800998ecf8427e\"", "\"d41d8cd98f00b204e9800998ecf8427e\"", true), + ("\"d41d8cd98f00b204e9800998ecf8427e\"", "d41d8cd98f00b204e9800998ecf8427e", true), + ("\"12345\"", "\"67890\", \"12345\", \"abcde\"", true), + ("\"12345\"", " \"67890\" , \"12345\" , \"abcde\" ", true), + ("\"12345\"", "\"67890\"", false), + ("\"12345\"", "\"67890\", \"abcde\"", false), + ] { + let info = ObjectInfo { + etag: Some(object_etag.to_owned()), + ..Default::default() + }; + let mut headers = HeaderMap::new(); + headers.insert(http::header::IF_NONE_MATCH, HeaderValue::from_str(condition).unwrap()); + let result = check_preconditions(&headers, &info); + if matches { + assert_eq!(result.unwrap_err().code(), &S3ErrorCode::NotModified); + } else { + assert!(result.is_ok()); + } + } + } - // [2] Exact match (both with double quotes) - assert!(is_etag_equal( - "\"d41d8cd98f00b204e9800998ecf8427e\"", - "\"d41d8cd98f00b204e9800998ecf8427e\"" - )); - - // [3] Exact match (object ETag with quotes, header ETag without) - assert!(is_etag_equal("\"d41d8cd98f00b204e9800998ecf8427e\"", "d41d8cd98f00b204e9800998ecf8427e")); - - // [4] Header ETag has multiple values (comma-separated), one matches - assert!(is_etag_equal("\"12345\"", "\"67890\", \"12345\", \"abcde\"")); - - // [5] Header ETag has multiple values with spaces, one matches after trim - assert!(is_etag_equal("\"12345\"", " \"67890\" , \"12345\" , \"abcde\" ")); - - // [6] No match (different ETag) - assert!(!is_etag_equal("\"12345\"", "\"67890\"")); - - // [7] No match in multiple values - assert!(!is_etag_equal("\"12345\"", "\"67890\", \"abcde\"")); + #[test] + fn test_read_etag_comparison_requires_complete_field() { + for (etag, condition, matches) in [ + ("abc", "\"other\"\"abc\"", false), + ("abc", "\"abc\"garbage\"", false), + ("abc", "\"abc\", \"other\"\"different\"", false), + ("abc", "\"abc\", W/ \"other\"", false), + ("abc", "\"abc\", \"contains space\"", false), + ("abc", "\"abc\", *", false), + ("abc", "\"other\", , \"abc\"", true), + ("abc", ", \"abc\",,", true), + ("a,b", "\"other\", \"a,b\"", true), + ("", ",,,", false), + ("", "\"\"", true), + ] { + let info = ObjectInfo { + etag: Some(etag.to_owned()), + ..Default::default() + }; + for name in [http::header::IF_MATCH, http::header::IF_NONE_MATCH] { + let mut headers = HeaderMap::new(); + headers.insert(name.clone(), HeaderValue::from_str(condition).unwrap()); + let result = check_preconditions(&headers, &info); + if name == http::header::IF_MATCH { + if matches { + assert!(result.is_ok(), "{condition}"); + } else { + assert_eq!(result.unwrap_err().code(), &S3ErrorCode::PreconditionFailed, "{condition}"); + } + } else if matches { + // The empty ETag case validates matching via If-Match above; + // emitting that invalid stored response validator fails closed. + assert!(result.is_err(), "{condition}"); + if !etag.is_empty() { + assert_eq!(result.unwrap_err().code(), &S3ErrorCode::NotModified, "{condition}"); + } + } else { + assert!(result.is_ok(), "{condition}"); + } + } + } } #[test] @@ -1409,6 +1451,77 @@ mod tests { assert!(check_preconditions(&headers18, &info18).is_ok()); } + #[test] + fn test_check_preconditions_rfc304() { + let modified = OffsetDateTime::from_unix_timestamp(1700000000).expect("valid timestamp"); + let mut info = ObjectInfo { + etag: Some("abc".to_owned()), + mod_time: Some(modified), + expires: Some(modified + time::Duration::days(1)), + version_id: Some(uuid::Uuid::nil()), + ..Default::default() + }; + info.user_defined = std::collections::HashMap::from([("cache-control".to_owned(), "max-age=60".to_owned())]).into(); + let mut headers = HeaderMap::new(); + headers.insert("if-none-match", HeaderValue::from_static("W/\"abc\"")); + let error = check_preconditions(&headers, &info).expect_err("weak ETag must validate the cache"); + assert_eq!(error.code(), &S3ErrorCode::NotModified); + let response = error.headers().expect("304 carries cache headers"); + assert_eq!(response["etag"], "\"abc\""); + assert_eq!(response["cache-control"], "max-age=60"); + assert!(response.contains_key("expires")); + assert!(response.contains_key("last-modified")); + assert_eq!(response["x-amz-version-id"], "null"); + + headers.insert("if-match", HeaderValue::from_static("\"other\"")); + assert_eq!( + check_preconditions(&headers, &info).expect_err("If-Match is first").code(), + &S3ErrorCode::PreconditionFailed + ); + headers.remove("if-none-match"); + headers.insert("if-match", HeaderValue::from_static("W/\"abc\"")); + assert_eq!( + check_preconditions(&headers, &info) + .expect_err("If-Match requires strong comparison") + .code(), + &S3ErrorCode::PreconditionFailed + ); + + headers.clear(); + headers.insert("if-none-match", HeaderValue::from_static("\"other\"")); + let date = modified.format(&crate::storage::RFC1123).expect("format timestamp"); + headers.insert("if-modified-since", HeaderValue::from_str(&date).expect("valid date header")); + assert!(check_preconditions(&headers, &info).is_ok(), "If-None-Match suppresses the date"); + headers.clear(); + headers.insert("if-unmodified-since", HeaderValue::from_str(&date).expect("valid date header")); + info.mod_time = Some(modified + time::Duration::seconds(1)); + assert_eq!( + check_preconditions(&headers, &info) + .expect_err("next second was modified") + .code(), + &S3ErrorCode::PreconditionFailed + ); + + headers.clear(); + headers.insert("if-none-match", HeaderValue::from_static("*")); + info.etag = None; + assert_eq!( + check_preconditions(&headers, &info) + .expect_err("existing object wildcard needs no ETag") + .code(), + &S3ErrorCode::NotModified + ); + headers.clear(); + headers.insert("if-match", HeaderValue::from_static("*")); + assert!(check_preconditions(&headers, &info).is_ok()); + headers.insert("if-match", HeaderValue::from_static("\"other\", \"abc,def\"")); + info.etag = Some("abc,def".to_owned()); + assert!( + check_preconditions(&headers, &info).is_ok(), + "commas inside an opaque tag are not list delimiters" + ); + } + #[test] fn test_parse_etag() { // [1] ETag without quotes → adds quotes diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index b6119c9f1..ee081e25b 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -1391,6 +1391,39 @@ fn stored_envelope_master_key_version(metadata: &HashMap) -> Opt pub(crate) struct SseObjectEncryptionResolver; +/// Checks metadata and key access before a conditional read returns without a body. +pub(crate) async fn validate_conditional_read( + bucket: &str, + object: &str, + metadata: &HashMap, + headers: &HeaderMap, + principal: Option<&SseKmsPrincipal>, +) -> Result<(), ApiError> { + let metadata = normalize_encryption_metadata_case(metadata) + .map_err(|error| map_get_object_reader_error(StorageError::Io(std::io::Error::other(error))))?; + validate_sse_headers_for_read(&metadata, headers)?; + let (_, customer_key, customer_key_md5) = extract_ssec_params_from_headers(headers)?; + classify_sse_read_response(DecryptionRequest { + bucket, + key: object, + metadata: &metadata, + sse_customer_key: customer_key.as_ref(), + sse_customer_key_md5: customer_key_md5.as_ref(), + principal, + }) + .await?; + SseObjectEncryptionResolver + .resolve_read_material(ReadEncryptionRequest { + bucket, + object, + metadata: &metadata, + headers, + }) + .await + .map_err(|error| map_get_object_reader_error(StorageError::Io(std::io::Error::other(error))))?; + Ok(()) +} + #[async_trait] impl ObjectEncryptionResolver for SseObjectEncryptionResolver { async fn resolve_read_material( @@ -8588,6 +8621,35 @@ mod tests { assert_eq!(audit_tag(&tags, "kmsErrorClass").as_deref(), Some("access_denied")); } + #[tokio::test] + async fn conditional_read_denies_kms_before_unwrap_with_normalized_metadata() { + let (principal, audit) = audited_principal(true, false); + let metadata = sse_kms_object_metadata() + .into_iter() + .map(|(name, value)| (name.to_ascii_uppercase(), value)) + .collect(); + let error = super::validate_conditional_read("finance", "ledger.csv", &metadata, &HeaderMap::new(), Some(&principal)) + .await + .expect_err("conditional read must authorize the real key before unwrap"); + assert_eq!(error.code, S3ErrorCode::AccessDenied); + assert_eq!(audit_tag(&audit.audit_tags(), "kmsKeyId").as_deref(), Some("finance-key")); + } + + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn conditional_read_denies_minio_only_kms_before_unwrap() { + let (principal, authorizer) = enforcing_principal(false); + let metadata = HashMap::from([ + (super::MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_owned(), "sealed-key".to_owned()), + (super::MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_owned(), "finance-key".to_owned()), + ]); + let error = super::validate_conditional_read("finance", "ledger.csv", &metadata, &HeaderMap::new(), Some(&principal)) + .await + .expect_err("MinIO metadata must not bypass KMS authorization"); + assert_eq!(error.code, S3ErrorCode::AccessDenied); + assert_eq!(authorizer.calls(), vec![(KmsAction::DecryptAction, "finance-key".to_owned())]); + } + #[test] fn read_response_projection_uses_shared_managed_metadata_compatibility() { let context_json = r#"{"tenant":"s3-select"}"#; diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 3edbe7d4d..67c0e586a 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -109,11 +109,12 @@ pub(crate) type StorageSelectObjectSnapshot = ecstore_object::SelectObjectSnapsh pub(crate) type StorageObjectToDelete = contract::object::ObjectToDelete; pub(crate) type StoragePutObjReader = super::PutObjReader; pub(crate) use super::ecfs_extend::{ - RFC1123, apply_bucket_default_lock_retention, apply_cors_headers, check_preconditions, get_buffer_size_opt_in, - get_validated_store, load_bucket_object_lock_config_state, parse_object_lock_legal_hold, parse_object_lock_retention, - parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations, process_topic_configurations, - remove_object_lock_metadata_for_copy, validate_bucket_exists, validate_bucket_object_lock_enabled_state, - validate_list_object_unordered_with_delimiter, validate_object_key, wrap_response_with_cors, + RFC1123, apply_bucket_default_lock_retention, apply_cors_headers, check_preconditions, evaluate_read_preconditions, + get_buffer_size_opt_in, get_validated_store, has_read_preconditions, load_bucket_object_lock_config_state, + parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, process_lambda_configurations, + process_queue_configurations, process_topic_configurations, remove_object_lock_metadata_for_copy, validate_bucket_exists, + validate_bucket_object_lock_enabled_state, validate_list_object_unordered_with_delimiter, validate_object_key, + wrap_response_with_cors, }; pub(crate) use super::sse::{ DecryptionRequest, EncryptionRequest, ObjectDekRewrapOutcome, PrepareEncryptionRequest, SseKmsPrincipal, @@ -400,7 +401,7 @@ pub(crate) mod tonic_service_consumer { pub(crate) mod test_consumer { pub(crate) use super::super::{ apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions, decode_tags_to_map, - get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal, matches_origin_pattern, parse_etag, + get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, matches_origin_pattern, parse_etag, parse_object_lock_legal_hold, parse_object_lock_retention, process_lambda_configurations, process_queue_configurations, process_topic_configurations, remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata, validate_bucket_object_lock_enabled, validate_list_object_unordered_with_delimiter, @@ -579,10 +580,11 @@ pub(crate) mod ecstore_object { pub(crate) use rustfs_ecstore::api::object::GetObjectBodySource; pub(crate) use rustfs_ecstore::api::object::{ EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, - ObjectEncryptionResolver, ObjectMutationHook, PrepareSelectObjectSnapshotError, ReadEncryptionMaterial, - ReadEncryptionMode, ReadEncryptionRequest, SelectObjectSnapshot, WriteCommitGuard, WriteCompletion, - get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, - register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook, + GetObjectReadCondition, GetObjectReadMetadata, ObjectEncryptionResolver, ObjectMutationHook, + PrepareSelectObjectSnapshotError, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, + SelectObjectSnapshot, WriteCommitGuard, WriteCompletion, get_object_body_cache_plaintext_len, + lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook, + unregister_get_object_body_cache_hook, unregister_object_mutation_hook, with_get_object_read_condition, }; } @@ -597,8 +599,8 @@ pub(crate) mod ecstore_test_support { pub(crate) mod ecstore_set_disk { #[cfg(test)] pub(crate) use rustfs_ecstore::api::set_disk::test_util::{ - MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause, - fail_next_quota_ledger_save_for_test, + ConditionalReadBenchmarkMetadataGuard, MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, + PutObjectCommitPause, fail_next_quota_ledger_save_for_test, }; pub(crate) use rustfs_ecstore::api::set_disk::{ DEFAULT_READ_BUFFER_SIZE, file_info_quorum_hash, get_lock_acquire_timeout, is_valid_storage_class, diff --git a/scripts/README.md b/scripts/README.md index 1ea493f2c..78b21a70e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -108,6 +108,9 @@ their issue closes. | `run_object_batch_bench.sh` | dev-tool | Batch object benchmark runner (warp/s3bench) | internode + scanner runbooks | | `run_object_batch_bench_enhanced.sh` | dev-tool | Enhanced batch benchmark runner; hub used by the smoke rigs | hotpath runbook | | `run_pinned_paired_abba_bench.sh` | dev-tool | Pinned RustFS/MinIO paired ABBA benchmark orchestrator for backlog#1432 | `test_pinned_paired_abba_bench.sh` | +| `prepare_conditional_read_baseline.py` | dev-tool | Installs only the matching conditional-read benchmark harness in a clean detached baseline | `docs/testing/conditional-read-benchmark.md` | +| `run_conditional_read_abba.py` | dev-tool | Serial ordinary/hit/miss GET ABBA with cache/slow-disk controls and drift/regression gates | `docs/testing/conditional-read-benchmark.md` | +| `test_conditional_read_abba.py` | dev-tool | Parser, drift rejection, and throughput/lock regression tests for conditional-read evidence | `run_conditional_read_abba.py` | | `run_get_codec_streaming_smoke.sh` | dev-tool | Local GET benchmark harness for the codec streaming read path | `docs/testing/ecstore-validation-suite-design.md` | | `run_get_1mib_abba_stage_metrics.sh` | dev-tool | Exact-1MiB isolated-host GET ABBA/stage-metrics harness for backlog#1434 | `test_get_1mib_abba_stage_metrics.sh` | | `issue_2007_coalescer_prometheus_report.py` | dev-tool | Read-only Prometheus report for GET metadata coalescer delay cost validation; usage in the module docstring | `test_issue_2007_coalescer_prometheus_report.sh` | diff --git a/scripts/prepare_conditional_read_baseline.py b/scripts/prepare_conditional_read_baseline.py new file mode 100644 index 000000000..ce25889b0 --- /dev/null +++ b/scripts/prepare_conditional_read_baseline.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Install only the paired benchmark harness in a clean detached baseline.""" + +import argparse +from pathlib import Path +import re +import subprocess + + +def replace_once(source, old, new, path): + if source.count(old) != 1: + raise ValueError(f"baseline anchor changed in {path}; inspect it before measuring") + return source.replace(old, new) + + +def prepare(root, baseline, revision): + if baseline.resolve() == root.resolve(): + raise ValueError("baseline must be a separate worktree") + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise ValueError("supply the full baseline commit SHA") + def git(*args): + return subprocess.check_output(["git", "-C", str(baseline), *args], text=True).strip() + if git("rev-parse", "HEAD") != revision or git("status", "--porcelain"): + raise ValueError("baseline must be clean at the requested commit") + if subprocess.run(["git", "-C", str(baseline), "symbolic-ref", "-q", "HEAD"], + stdout=subprocess.DEVNULL).returncode == 0: + raise ValueError("use a detached baseline worktree") + edits = {} + def edit(path, old, new): + previous = edits.get(path, (baseline / path).read_text()) + edits[path] = replace_once(previous, old, new, path) + edit("rustfs/src/app/object/mod.rs", "mod test_support;", + "mod test_support;\n#[cfg(test)]\nmod conditional_read_bench;") + edit("crates/ecstore/src/set_disk/mod.rs", + 'const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS";', + '#[cfg(feature = "test-util")]\nmod conditional_read_bench;\n' + '#[cfg(feature = "test-util")]\npub use conditional_read_bench::ConditionalReadBenchmarkMetadataGuard;\n\n' + 'const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS";') + edit("crates/ecstore/src/set_disk/read.rs", + "fn get_object_metadata_cache_request_bypass_reason(bucket: &str, opts: &ObjectOptions, read_data: bool) -> Option<&'static str> {", + "fn get_object_metadata_cache_request_bypass_reason(bucket: &str, opts: &ObjectOptions, read_data: bool) -> Option<&'static str> {\n" + ' #[cfg(feature = "test-util")]\n if super::conditional_read_bench::cache_bypass_applies(bucket) {\n' + ' return Some("benchmark");\n }') + edit("crates/ecstore/src/api/mod.rs", + "pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;", + "pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;\n" + " pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;") + for path in ("rustfs/src/storage/storage_api.rs", "rustfs/src/app/storage_api.rs"): + # The facades also expose these types elsewhere; target the test-util block. + source = (baseline / path).read_text() + marker = ("pub(crate) use rustfs_ecstore::api::set_disk::test_util::{" if "/storage/" in path else + "pub(crate) use crate::storage::storage_api::ecstore_set_disk::{") + replacement = marker + "\n ConditionalReadBenchmarkMetadataGuard," + edits[path] = replace_once(source, marker, replacement, path) + for path in ("rustfs/src/app/object/conditional_read_bench.rs", + "crates/ecstore/src/set_disk/conditional_read_bench.rs"): + if (baseline / path).exists(): + raise ValueError(f"baseline already contains {path}") + edits[path] = (root / path).read_text() + # Resolve every anchor before writing; never apply a partial adapter. + for path, source in edits.items(): + (baseline / path).write_text(source) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline-worktree", type=Path, required=True) + parser.add_argument("--revision", required=True) + args = parser.parse_args() + prepare(Path(__file__).resolve().parent.parent, args.baseline_worktree, args.revision) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_conditional_read_abba.py b/scripts/run_conditional_read_abba.py new file mode 100644 index 000000000..ab7a3e57c --- /dev/null +++ b/scripts/run_conditional_read_abba.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Run the identical application-read test harness in serial ABBA processes. + +This measures local EC reads, not network/authentication or deployment capacity. +Use matching toolchain, features and profiles for both executable artifacts. +""" + +import argparse +import hashlib +import json +import math +import os +from pathlib import Path +import shutil +import subprocess +import time + + +def digest(path): + result = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + result.update(chunk) + return result.hexdigest() + + +def parse_rows(output): + rows = {} + prefix = "CONDITIONAL_BENCH " + for line in output.splitlines(): + if prefix in line: + row = json.loads(line.split(prefix, 1)[1]) + key = (row["size"], row["kind"], row["cache"], row["slowtail_ms"]) + if key in rows: + raise ValueError(f"duplicate benchmark row {key}") + rows[key] = row + if len(rows) != 6: + raise ValueError(f"expected six benchmark rows, received {len(rows)}") + if {(key[0], key[1]) for key in rows} != { + (size, kind) for size in (4096, 1_300_000) for kind in ("unconditional", "hit", "miss") + }: + raise ValueError("benchmark workload matrix is incomplete") + for row in rows.values(): + for metric in ("ops_per_sec", "p95_ms", "p99_ms", "lock_p95_ms", "lock_p99_ms"): + if not math.isfinite(row[metric]) or row[metric] <= 0: + raise ValueError(f"invalid benchmark metric {metric}") + return rows + + +def check_resource_isolation(check_load=True): + commands = subprocess.check_output(["ps", "-A", "-o", "comm="], text=True) + if any(Path(command.strip()).name in ("cargo", "rustc") for command in commands.splitlines()): + raise RuntimeError("stop concurrent Cargo builds before measuring") + load = os.getloadavg() + if check_load and load[0] > (os.cpu_count() or 1) * 0.5: + raise RuntimeError(f"host load too high for paired measurements: {load[0]}") + return load + + +def run_case(executable, expected_digest, env, log_path=None): + if digest(executable) != expected_digest: + raise RuntimeError("benchmark executable changed before execution") + # Poll for concurrent builds without attributing this benchmark's own CPU + # use to external host load. Load gates run before and after the process. + with subprocess.Popen([str(executable.resolve()), "conditional_read_paired_benchmark", + "--ignored", "--nocapture", "--test-threads=1"], + env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) as run: + try: + while True: + try: + output, _ = run.communicate(timeout=1) + break + except subprocess.TimeoutExpired: + check_resource_isolation(check_load=False) + except BaseException: + run.kill() + output, _ = run.communicate() + if log_path is not None: + log_path.write_text(output) + raise + returncode = run.returncode + if log_path is not None: + log_path.write_text(output) + check_resource_isolation() + if digest(executable) != expected_digest: + raise RuntimeError("benchmark executable changed during execution") + return returncode, output + + +def drift_passes(first, last): + if abs(last["ops_per_sec"] / first["ops_per_sec"] - 1) > 0.10: + return False + return all( + abs(last[metric] - first[metric]) <= max(first[metric] * 0.15, 0.25) + for metric in ("p95_ms", "p99_ms", "lock_p95_ms", "lock_p99_ms") + ) + + +def compare(rows): + if not all(rows[0].keys() == leg.keys() for leg in rows): + raise ValueError("ABBA legs have different workload matrices") + unstable = [key for key in rows[0] if not drift_passes(rows[0][key], rows[3][key])] + if unstable: + return {"accepted": False, "reason": "baseline_drift", "unstable_cases": unstable} + results = [] + for key, first in rows[0].items(): + baseline = {metric: math.sqrt(first[metric] * rows[3][key][metric]) for metric in + ("ops_per_sec", "p95_ms", "p99_ms", "lock_p95_ms", "lock_p99_ms")} + candidate = {metric: math.sqrt(rows[1][key][metric] * rows[2][key][metric]) for metric in baseline} + # Protect ordinary reads and condition misses; 304 savings alone do not pass. + passed = candidate["ops_per_sec"] >= baseline["ops_per_sec"] * 0.95 and all( + candidate[metric] <= baseline[metric] + max(baseline[metric] * 0.10, 0.25) + for metric in ("p95_ms", "p99_ms", "lock_p95_ms", "lock_p99_ms") + ) + results.append({"case": key, "baseline": baseline, "candidate": candidate, + "within_regression_budget": passed}) + return {"accepted": all(row["within_regression_budget"] for row in results), "cases": results} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline", type=Path, required=True) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--baseline-revision", required=True) + parser.add_argument("--candidate-revision", required=True) + parser.add_argument("--harness", type=Path, required=True) + parser.add_argument("--profile", required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--iterations", type=int, default=1000) + parser.add_argument("--concurrency", type=int, default=8) + parser.add_argument("--cooldown-seconds", type=float, default=5) + args = parser.parse_args() + if args.iterations < 100 or args.concurrency < 1 or args.cooldown_seconds < 0: + parser.error("iterations must be >=100, concurrency >0, cooldown >=0") + args.out_dir.mkdir(parents=True, exist_ok=False) + executables = {} + for name, source in (("baseline", args.baseline), ("candidate", args.candidate)): + original_digest = digest(source) + snapshot = args.out_dir / f"{name}.binary" + shutil.copyfile(source, snapshot) + snapshot.chmod(0o555) + if digest(snapshot) != original_digest: + raise RuntimeError("source executable changed while taking its snapshot") + executables[name] = snapshot + manifest = { + "baseline_revision": args.baseline_revision, "candidate_revision": args.candidate_revision, + "baseline_sha256": digest(executables["baseline"]), "candidate_sha256": digest(executables["candidate"]), + "harness_sha256": digest(args.harness), "profile": args.profile, + "iterations": args.iterations, "concurrency": args.concurrency, + "order": ["A1", "B1", "B2", "A2"], "slowtail_ms": [0, 20], "cache": [False, True], + "drift_budget": "throughput 10%; latency and lock percentiles 15% or 0.25ms", + "regression_budget": "throughput -5%; latency and lock percentiles +10% or 0.25ms", + "cpu_count": os.cpu_count(), "load_at_start": check_resource_isolation(), + } + (args.out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + legs = [] + for name, artifact in (("A1", "baseline"), ("B1", "candidate"), + ("B2", "candidate"), ("A2", "baseline")): + executable = executables[artifact] + cases = {} + for cache in (False, True): + for delay in (0, 20): + check_resource_isolation() + env = os.environ.copy() + # Keep ambient experiments and proxies out of localhost-sensitive runs. + for variable in tuple(env): + if variable.startswith("RUSTFS_") or variable.lower().endswith("_proxy"): + del env[variable] + env.update({ + "NO_PROXY": "*", "no_proxy": "*", "RUSTFS_OBJECT_LOCK_DIAG_ENABLE": "true", + "RUSTFS_OBJECT_DATA_CACHE_ENABLE": str(cache).lower(), + "RUSTFS_OBJECT_DATA_CACHE_MODE": "fill_materialize_enabled", + "RUSTFS_OBJECT_DATA_CACHE_MAX_BYTES": "8388608", + "RUSTFS_OBJECT_DATA_CACHE_MAX_ENTRY_BYTES": "2097152", + "RUSTFS_OBJECT_DATA_CACHE_MIN_FREE_MEMORY_PERCENT": "0", + "RUSTFS_CONDITIONAL_BENCH_ITERATIONS": str(args.iterations), + "RUSTFS_CONDITIONAL_BENCH_CONCURRENCY": str(args.concurrency), + "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS": str(delay), + "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS": "3", + "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET": "conditional-bench", + "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX": "bench-", + }) + returncode, output = run_case(executable, manifest[f"{artifact}_sha256"], env, + args.out_dir / f"{name}-cache{int(cache)}-delay{delay}.log") + if returncode: + raise RuntimeError(f"{name} cache={cache} delay={delay} failed; see saved log") + for key, row in parse_rows(output).items(): + if (row["cache"] != cache or row["slowtail_ms"] != str(delay) + or row["iterations"] != args.iterations or row["concurrency"] != args.concurrency + or row["metadata_cache_hits"] != 0 or not row["metadata_cache_bypassed"] + or row["lock_samples"] <= 0): + raise ValueError("benchmark did not observe requested environment") + cases[key] = row + print(f"{name}: cache={cache} slowtail={delay}ms complete", flush=True) + legs.append(cases) + time.sleep(args.cooldown_seconds) + result = compare(legs) + result["load_at_end"] = os.getloadavg() + (args.out_dir / "summary.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result, indent=2)) + return 0 if result["accepted"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_conditional_read_abba.py b/scripts/test_conditional_read_abba.py new file mode 100644 index 000000000..c7b53568b --- /dev/null +++ b/scripts/test_conditional_read_abba.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Verify that paired conditional-read evidence cannot bypass acceptance gates.""" + +import copy +import json +from pathlib import Path +import subprocess +import unittest +from unittest.mock import MagicMock, patch + +from run_conditional_read_abba import compare, parse_rows, run_case + + +def row(): + return {"size": 4096, "kind": "miss", "cache": False, "slowtail_ms": "0", + "ops_per_sec": 1000, "p95_ms": 2, "p99_ms": 3, + "lock_p95_ms": 1, "lock_p99_ms": 2} + + +class AcceptanceTests(unittest.TestCase): + def test_stable_equivalent_legs_pass(self): + self.assertTrue(compare([{("case",): row()} for _ in range(4)])["accepted"]) + + def test_baseline_drift_blocks_candidate_attribution(self): + legs = [{("case",): row()} for _ in range(4)] + legs[3][("case",)]["ops_per_sec"] = 800 + result = compare(legs) + self.assertFalse(result["accepted"]) + self.assertEqual(result["reason"], "baseline_drift") + self.assertNotIn("cases", result) + + def test_miss_throughput_and_lock_regressions_fail(self): + for metric, value in [("ops_per_sec", 900), ("lock_p99_ms", 4)]: + legs = [{("case",): row()} for _ in range(4)] + for index in (1, 2): + legs[index][("case",)][metric] = value + self.assertFalse(compare(legs)["accepted"], metric) + + def test_missing_workload_is_rejected(self): + legs = [{("case",): row()} for _ in range(4)] + legs[1] = {} + with self.assertRaises(ValueError): + compare(legs) + + def test_complete_rows_include_first_libtest_output_line(self): + rows = [] + for size in (4096, 1_300_000): + for kind in ("unconditional", "hit", "miss"): + value = copy.copy(row()) + value.update(size=size, kind=kind) + rows.append("CONDITIONAL_BENCH " + json.dumps(value)) + output = "test app::benchmark ... " + "\n".join(rows) + self.assertEqual(len(parse_rows(output)), 6) + with self.assertRaises(ValueError): + parse_rows(output + "\n" + rows[0]) + + def test_wrong_shape_or_nonfinite_metrics_are_rejected(self): + rows = [] + for size in (4096, 1_300_000): + for kind in ("unconditional", "hit", "miss"): + value = copy.copy(row()) + value.update(size=size, kind=kind) + rows.append(value) + for field, value in [("kind", "unknown"), ("p99_ms", float("nan"))]: + changed = copy.deepcopy(rows) + changed[0][field] = value + output = "\n".join("CONDITIONAL_BENCH " + json.dumps(item) for item in changed) + with self.assertRaises(ValueError): + parse_rows(output) + + +class IsolationTests(unittest.TestCase): + def process(self): + process = MagicMock() + process.__enter__.return_value = process + process.communicate.return_value = ("saved output", None) + process.returncode = 0 + return process + + def test_changed_executable_is_rejected_before_launch(self): + with patch("run_conditional_read_abba.digest", return_value="changed"), \ + patch("run_conditional_read_abba.subprocess.Popen") as launch: + with self.assertRaisesRegex(RuntimeError, "before execution"): + run_case(Path("binary"), "expected", {}) + launch.assert_not_called() + + def test_changed_executable_is_rejected_after_launch(self): + with patch("run_conditional_read_abba.digest", side_effect=["expected", "changed"]), \ + patch("run_conditional_read_abba.check_resource_isolation"), \ + patch("run_conditional_read_abba.subprocess.Popen", return_value=self.process()): + with self.assertRaisesRegex(RuntimeError, "during execution"): + run_case(Path("binary"), "expected", {}) + + def test_final_case_checks_resource_isolation_after_exit(self): + with patch("run_conditional_read_abba.digest", return_value="expected"), \ + patch("run_conditional_read_abba.check_resource_isolation", side_effect=RuntimeError("concurrent build")), \ + patch("run_conditional_read_abba.subprocess.Popen", return_value=self.process()): + with self.assertRaisesRegex(RuntimeError, "concurrent build"): + run_case(Path("binary"), "expected", {}) + + def test_concurrent_build_during_execution_kills_case_and_saves_log(self): + process = self.process() + process.communicate.side_effect = [subprocess.TimeoutExpired("binary", 1), ("saved output", None)] + log = MagicMock() + with patch("run_conditional_read_abba.digest", return_value="expected"), \ + patch("run_conditional_read_abba.check_resource_isolation", side_effect=RuntimeError("concurrent build")) as check, \ + patch("run_conditional_read_abba.subprocess.Popen", return_value=process): + with self.assertRaisesRegex(RuntimeError, "concurrent build"): + run_case(Path("binary"), "expected", {}, log) + check.assert_called_once_with(check_load=False) + process.kill.assert_called_once() + log.write_text.assert_called_once_with("saved output") + + +if __name__ == "__main__": + unittest.main() From 5228463a1c294c2efe41d5af8f6db9e1d77bcd4a Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Mon, 12 Oct 2026 18:40:07 +0800 Subject: [PATCH 026/143] fix(oidc): recover providers after startup discovery failures (#8475) * fix(oidc): recover providers after startup discovery failures * fix(oidc): report recovery failures with throttled diagnostics --- crates/iam/src/oidc/provider.rs | 139 ++++++-- crates/iam/src/oidc/provider_tests.rs | 1 + crates/iam/src/oidc/recovery_tests.rs | 489 ++++++++++++++++++++++++++ crates/iam/src/oidc/runtime.rs | 105 +++++- crates/iam/src/oidc/runtime_tests.rs | 68 +++- crates/iam/src/oidc/test_support.rs | 1 + crates/iam/src/oidc/transport.rs | 55 +-- rustfs/src/startup_auth.rs | 72 +++- rustfs/src/startup_lifecycle.rs | 13 + rustfs/src/startup_services.rs | 6 +- 10 files changed, 874 insertions(+), 75 deletions(-) create mode 100644 crates/iam/src/oidc/recovery_tests.rs diff --git a/crates/iam/src/oidc/provider.rs b/crates/iam/src/oidc/provider.rs index 6f0c72a32..6b4f083f8 100644 --- a/crates/iam/src/oidc/provider.rs +++ b/crates/iam/src/oidc/provider.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::{ config::{OidcProviderConfig, OidcProviderValidationResult, SourcedOidcProviderConfig}, - transport::{OidcHttpError, ReqwestHttpClient, oidc_extra_root_certs}, + transport::{DiscoveryRequest, OidcHttpError, ReqwestHttpClient, oidc_extra_root_certs, oidc_http_error_diagnostics}, }; use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC}; use openidconnect::core::{CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm}; @@ -25,7 +25,7 @@ use std::collections::HashMap; use std::sync::RwLock; use std::time::{Duration as StdDuration, Instant}; use tokio::time::sleep; -use tracing::warn; +use tracing::{debug, warn}; use url::Url; pub(super) const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60); @@ -34,6 +34,55 @@ pub(super) const OIDC_DISCOVERY_TRANSPORT_RETRY_DELAY: StdDuration = StdDuration pub(super) const OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY: &str = "JWKS request blocked by outbound policy"; pub(super) const OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY: &str = "OIDC provider discovery blocked by outbound policy"; +/// Preserve recovery eligibility before discovery errors become public diagnostic strings. +#[derive(Debug, thiserror::Error)] +#[error("{message}")] +pub(super) struct DiscoveryFailure { + message: String, + pub(super) retryable: bool, + pub(super) diagnostics: (&'static str, String), +} + +impl From for DiscoveryFailure { + fn from(message: String) -> Self { + Self { + message, + retryable: false, + diagnostics: ("validation", String::new()), + } + } +} + +fn retryable_discovery_status(status: http::StatusCode) -> bool { + status.is_server_error() || matches!(status, http::StatusCode::REQUEST_TIMEOUT | http::StatusCode::TOO_MANY_REQUESTS) +} + +fn retryable_discovery_transport(error: &OidcHttpError) -> bool { + matches!(error, OidcHttpError::Reqwest(error) if !error.is_builder()) +} + +fn discovery_failure(error: DiscoveryError, context: &str, blocked_context: &str) -> DiscoveryFailure { + let retryable = match &error { + DiscoveryError::Request(error) => retryable_discovery_transport(error), + DiscoveryError::Response(status, _, _) => retryable_discovery_status(*status), + _ => false, + }; + let diagnostics = match &error { + DiscoveryError::Request(error) => oidc_http_error_diagnostics(error), + DiscoveryError::Response(status, _, _) => ("http_status", status.as_u16().to_string()), + _ => ("validation", String::new()), + }; + let message = match error { + DiscoveryError::Request(OidcHttpError::ForbiddenOutbound(reason)) => format!("{blocked_context}: {reason}"), + error => format!("{context}: {error}"), + }; + DiscoveryFailure { + message, + retryable, + diagnostics, + } +} + // ---- Internal provider state ---- /// Discovered OIDC provider metadata. @@ -180,6 +229,7 @@ impl<'c> AsyncHttpClient<'c> for JwksAcceptClient<'_> { http::HeaderValue::from_static("application/json, application/jwk-set+json"), ); } + request.extensions_mut().insert(DiscoveryRequest); self.inner.call(request) } } @@ -246,7 +296,17 @@ impl ProviderRuntime { provider_id: &str, config: &OidcProviderConfig, ) -> Result { - let state = discover_provider(config, &self.http_client).await?; + self.recover_provider_state(provider_id, config) + .await + .map_err(|err| err.to_string()) + } + + pub(super) async fn recover_provider_state( + &self, + provider_id: &str, + config: &OidcProviderConfig, + ) -> Result { + let state = discover_provider_for_recovery(config, &self.http_client).await?; let mut map = self.states.write().map_err(|e| { let msg = e.to_string(); format!("provider state lock poisoned: {msg}") @@ -301,6 +361,15 @@ pub(super) async fn discover_provider( config: &OidcProviderConfig, http_client: &ReqwestHttpClient, ) -> Result { + discover_provider_for_recovery(config, http_client) + .await + .map_err(|err| err.to_string()) +} + +pub(super) async fn discover_provider_for_recovery( + config: &OidcProviderConfig, + http_client: &ReqwestHttpClient, +) -> Result { if let Some(issuer) = config.issuer.as_deref().filter(|issuer| !issuer.trim().is_empty()) { return discover_provider_from_config_url(config, issuer, http_client).await; } @@ -310,6 +379,8 @@ pub(super) async fn discover_provider( let base_issuer = normalize_config_url(&config.config_url)?; let candidates = issuer_candidates(&base_issuer); let mut last_errors = Vec::new(); + let mut retryable = false; + let mut diagnostics = ("validation", String::new()); for candidate_issuer in candidates.iter() { let issuer_url = IssuerUrl::new(candidate_issuer.clone()).map_err(|e| format!("invalid issuer URL: {e}"))?; @@ -332,26 +403,21 @@ pub(super) async fn discover_provider( metadata: DiscoveredProviderMetadata::Console(Box::new(metadata)), discovered_at: Instant::now(), }) - .map_err(|err| match err { - DiscoveryError::Request(OidcHttpError::ForbiddenOutbound(reason)) => { - format!("{OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY}: {reason}") - } - err => format!("discovery failed: {err}"), - }) + .map_err(|err| discovery_failure(err, "discovery failed", OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY)) }; match discovered { Ok(state) => return Ok(state), Err(error) - if error.starts_with(OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY) - || error.starts_with(OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY) => + if error.message.starts_with(OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY) + || error.message.starts_with(OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY) => { return Err(error); } Err(error) => { - let is_transient_transport = error.contains("Request failed"); + let is_transient_transport = error.message.contains("Request failed"); let should_retry = is_transient_transport && attempt + 1 < OIDC_DISCOVERY_TRANSPORT_RETRIES; if should_retry { - warn!( + debug!( event = EVENT_OIDC_DIAGNOSTICS, component = LOG_COMPONENT_IAM, subsystem = LOG_SUBSYSTEM_OIDC, @@ -368,8 +434,12 @@ pub(super) async fn discover_provider( continue; } + if error.retryable || !retryable { + diagnostics = error.diagnostics.clone(); + } + retryable |= error.retryable; last_errors.push(format!("issuer '{candidate_issuer}': {error}")); - warn!( + debug!( event = EVENT_OIDC_DIAGNOSTICS, component = LOG_COMPONENT_IAM, subsystem = LOG_SUBSYSTEM_OIDC, @@ -388,11 +458,11 @@ pub(super) async fn discover_provider( } } - Err(format!( - "discovery failed for all issuer variants {:?}: {}", - candidates, - last_errors.join("; ") - )) + Err(DiscoveryFailure { + message: format!("discovery failed for all issuer variants {:?}: {}", candidates, last_errors.join("; ")), + retryable, + diagnostics, + }) } pub(super) async fn validate_oidc_provider_config(config: &OidcProviderConfig) -> Result { @@ -429,7 +499,7 @@ async fn discover_provider_from_config_url( config: &OidcProviderConfig, issuer: &str, http_client: &ReqwestHttpClient, -) -> Result { +) -> Result { let issuer_url = IssuerUrl::new(issuer.trim().to_string()).map_err(|e| format!("invalid issuer URL: {e}"))?; let explicit_issuer = config.issuer.as_deref().is_some_and(|issuer| !issuer.trim().is_empty()); let discovery_url = if explicit_issuer { @@ -440,6 +510,7 @@ async fn discover_provider_from_config_url( .map_err(|err| err.to_string())? }; let request = http::Request::builder() + .extension(DiscoveryRequest) .uri(discovery_url.to_string()) .method(http::Method::GET) .header(http::header::ACCEPT, "application/json") @@ -449,12 +520,22 @@ async fn discover_provider_from_config_url( let response = match http_client.call(request).await { Ok(response) => response, Err(OidcHttpError::ForbiddenOutbound(reason)) => { - return Err(format!("{OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY}: {reason}")); + return Err(format!("{OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY}: {reason}").into()); + } + Err(err) => { + return Err(DiscoveryFailure { + retryable: retryable_discovery_transport(&err), + diagnostics: oidc_http_error_diagnostics(&err), + message: format!("discovery request failed: Request failed: {err}"), + }); } - Err(err) => return Err(format!("discovery request failed: Request failed: {err}")), }; if response.status() != http::StatusCode::OK { - return Err(format!("discovery failed: HTTP status code {} at {}", response.status(), discovery_url)); + return Err(DiscoveryFailure { + retryable: retryable_discovery_status(response.status()), + diagnostics: ("http_status", response.status().as_u16().to_string()), + message: format!("discovery failed: HTTP status code {} at {}", response.status(), discovery_url), + }); } if !explicit_issuer @@ -466,7 +547,9 @@ async fn discover_provider_from_config_url( .is_some_and(|essence| essence.eq_ignore_ascii_case("application/json")) }) { - return Err("Unexpected response Content-Type: expected application/json".into()); + return Err("Unexpected response Content-Type: expected application/json" + .to_string() + .into()); } let provider_metadata = DiscoveredProviderMetadata::parse(response.body(), config.hide_from_ui) @@ -476,7 +559,8 @@ async fn discover_provider_from_config_url( "unexpected issuer URI `{}` (expected `{}`)", provider_metadata.issuer().as_str(), issuer_url.as_str() - )); + ) + .into()); } let jwks_url = if explicit_issuer { @@ -494,10 +578,7 @@ async fn discover_provider_from_config_url( .await { Ok(jwks) => jwks, - Err(DiscoveryError::Request(OidcHttpError::ForbiddenOutbound(reason))) => { - return Err(format!("{OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY}: {reason}")); - } - Err(err) => return Err(format!("failed to fetch JWKS: {err}")), + Err(err) => return Err(discovery_failure(err, "failed to fetch JWKS", OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY)), }; Ok(ProviderState { diff --git a/crates/iam/src/oidc/provider_tests.rs b/crates/iam/src/oidc/provider_tests.rs index efa6c543b..849b0d4d5 100644 --- a/crates/iam/src/oidc/provider_tests.rs +++ b/crates/iam/src/oidc/provider_tests.rs @@ -299,6 +299,7 @@ async fn workload_discovery_verification_and_rotation() { configs: HashMap::from([(config.id.clone(), test_sourced_config(config.clone()))]), provider_runtime: ProviderRuntime::new(http_client, HashMap::from([(config.id.clone(), state)])), state_store: OidcStateStore::new(), + pending_discovery: Default::default(), }; assert!(sys.has_providers()); assert!(sys.provider_configs().all(|provider| provider.hide_from_ui)); diff --git a/crates/iam/src/oidc/recovery_tests.rs b/crates/iam/src/oidc/recovery_tests.rs new file mode 100644 index 000000000..f1b500b85 --- /dev/null +++ b/crates/iam/src/oidc/recovery_tests.rs @@ -0,0 +1,489 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::oidc::test_support::{build_mocked_oidc_provider_config, oidc_es256_key_and_jwk, test_sourced_config}; +use jsonwebtoken::{Algorithm, EncodingKey, Header}; +use rustfs_utils::egress::OutboundPolicy; +use std::sync::atomic::{AtomicU16, AtomicUsize, Ordering}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio_util::task::AbortOnDropHandle; + +struct RecoveryIdp { + base: String, + config: OidcProviderConfig, + key: EncodingKey, + discovery_status: Arc, + jwks_status: Arc, + requests: Arc, + cancelled_requests: Arc, + _server: AbortOnDropHandle<()>, +} + +impl RecoveryIdp { + async fn start(id: &str, workload: bool, explicit_issuer: bool) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind recovery IdP"); + let base = format!("http://{}", listener.local_addr().expect("recovery IdP address")); + let mut config = build_mocked_oidc_provider_config(id, &base); + config.hide_from_ui = workload; + config.claim_name = "groups".into(); + config.claim_prefix = "team/".into(); + config.issuer = explicit_issuer.then(|| format!("https://issuer.example.com/{id}")); + let (key, jwk) = oidc_es256_key_and_jwk(id); + let mut document = serde_json::json!({ + "issuer": config.issuer.as_deref().unwrap_or(&base), + "authorization_endpoint": format!("{base}/authorize"), + "token_endpoint": format!("{base}/token"), + "jwks_uri": format!("{base}/jwks"), + "response_types_supported": ["code"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["ES256"], + }); + if workload { + let fields = document.as_object_mut().expect("discovery document"); + fields.remove("authorization_endpoint"); + fields.remove("token_endpoint"); + } + let document = document.to_string(); + let jwks = serde_json::json!({"keys": [jwk]}).to_string(); + let discovery_status = Arc::new(AtomicU16::new(503)); + let jwks_status = Arc::new(AtomicU16::new(200)); + let requests = Arc::new(AtomicUsize::new(0)); + let cancelled_requests = Arc::new(AtomicUsize::new(0)); + let server_discovery_status = discovery_status.clone(); + let server_jwks_status = jwks_status.clone(); + let server_requests = requests.clone(); + let server_cancelled = cancelled_requests.clone(); + let server = tokio::spawn(async move { + loop { + let (mut stream, _) = listener.accept().await.expect("accept discovery request"); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + request.push(stream.read_u8().await.expect("read discovery request")); + assert!(request.len() < 8192); + } + let headers = String::from_utf8(request).expect("HTTP request headers"); + let path = headers + .lines() + .next() + .expect("request line") + .split_whitespace() + .nth(1) + .expect("path"); + let (status, body) = if path.ends_with("/.well-known/openid-configuration") { + (server_discovery_status.load(Ordering::SeqCst), document.as_str()) + } else { + assert_eq!(path, "/jwks"); + (server_jwks_status.load(Ordering::SeqCst), jwks.as_str()) + }; + server_requests.fetch_add(1, Ordering::SeqCst); + if status == 0 { + continue; // Simulate a transport failure before a response is available. + } + if status == 1 { + let mut byte = [0]; + assert_eq!(stream.read(&mut byte).await.expect("client closes cancelled discovery"), 0); + server_cancelled.fetch_add(1, Ordering::SeqCst); + continue; + } + let body = if status == 200 { body } else { "" }; + let response = format!( + "HTTP/1.1 {status} Test\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await.expect("write discovery response"); + } + }); + Self { + base, + config, + key, + discovery_status, + jwks_status, + requests, + cancelled_requests, + _server: AbortOnDropHandle::new(server), + } + } + + fn token(&self, key: &EncodingKey, audience: &str) -> String { + let mut header = Header::new(Algorithm::ES256); + header.kid = Some(self.config.id.clone()); + let now = time::OffsetDateTime::now_utc().unix_timestamp(); + jsonwebtoken::encode( + &header, + &serde_json::json!({ + "iss": self.config.issuer.as_deref().unwrap_or(&self.base), + "sub": "recovered-user", "aud": audience, "iat": now, "exp": now + 300, + "groups": ["readonly"], + }), + key, + ) + .expect("sign recovery test token") + } +} + +async fn initialize(idps: &[&RecoveryIdp]) -> Arc { + let origins = idps.iter().map(|idp| idp.base.as_str()).collect::>().join(","); + let client = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&origins).expect("allow test IdPs")); + let snapshot = OidcConfigSnapshot::new(idps.iter().map(|idp| test_sourced_config(idp.config.clone())).collect()); + Arc::new( + OidcSys::new_with_config_snapshot(client, snapshot) + .await + .expect("initialize OIDC with unavailable IdPs"), + ) +} + +async fn wait_until(mut condition: impl FnMut() -> bool) { + tokio::time::timeout(Duration::from_secs(5), async { + while !condition() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("recovery condition should be reached"); +} + +#[tokio::test] +async fn web_identity_recovers_transport_failure_and_preserves_verification() { + for explicit in [false, true] { + let idp = RecoveryIdp::start("workload", true, explicit).await; + idp.discovery_status.store(0, Ordering::SeqCst); + let sys = initialize(&[&idp]).await; + let adapter = StandardOidcAdapter::new(sys.clone()); + let mapper = CoreFederatedAuthorizationMapper::new(adapter.authorization_rules()); + assert!(sys.has_providers()); + assert!(sys.list_visible_providers().is_empty()); + let token = idp.token(&idp.key, &idp.config.client_id); + assert!(adapter.verify_identity(&token).await.is_err()); + + idp.discovery_status.store(200, Ordering::SeqCst); + tokio::time::timeout(Duration::from_secs(5), sys.recover_failed_providers()) + .await + .expect("recover workload discovery"); + let identity = adapter + .verify_identity(&token) + .await + .expect("verify recovered workload token"); + let authorization = mapper.map(identity); + assert_eq!(authorization.provider_id, "workload"); + assert_eq!(authorization.claims.sub, "recovered-user"); + assert_eq!(authorization.policies, vec!["team/readonly"]); + assert!( + sys.authorize_url("workload", "https://console.example.com/callback", None) + .await + .is_err() + ); + + let (wrong_key, _) = oidc_es256_key_and_jwk("wrong"); + assert!( + adapter + .verify_identity(&idp.token(&wrong_key, &idp.config.client_id)) + .await + .is_err() + ); + assert!(adapter.verify_identity(&idp.token(&idp.key, "wrong-audience")).await.is_err()); + } +} + +#[tokio::test] +async fn recovery_waits_for_jwks_and_claims_pending_providers_once() { + let idp = RecoveryIdp::start("console", false, true).await; + idp.discovery_status.store(200, Ordering::SeqCst); + idp.jwks_status.store(503, Ordering::SeqCst); + let sys = initialize(&[&idp]).await; + assert!(sys.provider_runtime.get_provider_state("console").is_err()); + let requests = idp.requests.load(Ordering::SeqCst); + idp.jwks_status.store(200, Ordering::SeqCst); + tokio::time::timeout( + Duration::from_secs(5), + futures::future::join_all((0..16).map(|_| sys.recover_failed_providers())), + ) + .await + .expect("concurrent recovery calls complete"); + assert!(sys.provider_runtime.get_provider_state("console").is_ok()); + assert_eq!( + idp.requests.load(Ordering::SeqCst), + requests + 2, + "only one discovery/JWKS round should run" + ); +} + +#[tokio::test] +async fn recovery_backoff_is_independent_of_login_traffic() { + let idp = RecoveryIdp::start("console", false, true).await; + let sys = initialize(&[&idp]).await; + let requests = idp.requests.load(Ordering::SeqCst); + let recovery_sys = sys.clone(); + let recovery = AbortOnDropHandle::new(tokio::spawn(async move { recovery_sys.recover_failed_providers().await })); + wait_until(|| idp.requests.load(Ordering::SeqCst) > requests).await; + assert_eq!(idp.requests.load(Ordering::SeqCst), requests + 1); + for _ in 0..32 { + assert!( + sys.authorize_url("console", "https://console.example.com/callback", None) + .await + .is_err() + ); + } + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + idp.requests.load(Ordering::SeqCst), + requests + 1, + "login attempts must not bypass discovery backoff" + ); + idp.discovery_status.store(200, Ordering::SeqCst); + tokio::time::timeout(Duration::from_secs(5), recovery) + .await + .expect("recovery deadline") + .expect("recovery task"); + assert!( + sys.authorize_url("console", "https://console.example.com/callback", None) + .await + .is_ok() + ); +} + +#[tokio::test] +async fn unavailable_provider_does_not_block_another_provider() { + let unavailable = RecoveryIdp::start("unavailable", false, true).await; + let available = RecoveryIdp::start("available", false, true).await; + let sys = initialize(&[&unavailable, &available]).await; + available.discovery_status.store(200, Ordering::SeqCst); + let recovery_sys = sys.clone(); + let recovery = AbortOnDropHandle::new(tokio::spawn(async move { recovery_sys.recover_failed_providers().await })); + wait_until(|| sys.provider_runtime.get_provider_state("available").is_ok()).await; + assert!(sys.provider_runtime.get_provider_state("unavailable").is_err()); + assert!( + sys.authorize_url("available", "https://console.example.com/callback", None) + .await + .is_ok() + ); + drop(recovery); +} + +#[tokio::test] +async fn recovery_preserves_ready_provider_and_inflight_sessions() { + let ready = RecoveryIdp::start("ready", false, true).await; + let recovering = RecoveryIdp::start("recovering", false, true).await; + ready.discovery_status.store(200, Ordering::SeqCst); + let sys = initialize(&[&ready, &recovering]).await; + let auth_url = sys + .authorize_url("ready", "https://console.example.com/callback", Some("/buckets".into())) + .await + .expect("begin login with ready provider"); + let auth: HashMap = url::Url::parse(&auth_url) + .expect("authorization URL") + .query_pairs() + .into_owned() + .collect(); + let logout = sys + .create_logout_token("ready", "verified-id-token") + .await + .expect("retain logout continuation"); + let discovered_at = sys + .provider_runtime + .get_provider_state("ready") + .expect("ready state") + .discovered_at; + let ready_requests = ready.requests.load(Ordering::SeqCst); + + recovering.discovery_status.store(200, Ordering::SeqCst); + tokio::time::timeout(Duration::from_secs(5), sys.recover_failed_providers()) + .await + .expect("recover second provider"); + assert_eq!(ready.requests.load(Ordering::SeqCst), ready_requests); + assert_eq!( + sys.provider_runtime + .get_provider_state("ready") + .expect("ready state remains") + .discovered_at, + discovered_at + ); + let session = sys + .state_store + .take(&auth["state"]) + .await + .expect("in-flight login remains available"); + assert_eq!(session.provider_id, "ready"); + assert_eq!(session.nonce, auth["nonce"]); + assert_eq!(session.redirect_after.as_deref(), Some("/buckets")); + assert_eq!( + PkceCodeChallenge::from_code_verifier_sha256(&PkceCodeVerifier::new(session.pkce_verifier)).as_str(), + auth["code_challenge"] + ); + assert_eq!( + sys.state_store + .take_logout(&logout) + .await + .expect("logout continuation remains available") + .provider_id, + "ready" + ); +} + +#[tokio::test] +async fn startup_schedules_only_recoverable_http_errors() { + for explicit in [false, true] { + let idp = RecoveryIdp::start("status", false, explicit).await; + for (status, retryable) in [ + (408, true), + (429, true), + (500, true), + (503, true), + (400, false), + (401, false), + (403, false), + (404, false), + ] { + idp.discovery_status.store(status, Ordering::SeqCst); + let sys = initialize(&[&idp]).await; + assert_eq!( + sys.pending_discovery.lock().await.len(), + usize::from(retryable), + "explicit={explicit}, status={status}" + ); + assert!(sys.provider_runtime.get_provider_state("status").is_err()); + } + } +} + +#[tokio::test] +async fn recovery_bounds_concurrency_and_cancels_inflight_requests() { + let mut idps = Vec::new(); + for index in 0..6 { + idps.push(RecoveryIdp::start(&format!("provider-{index}"), false, true).await); + } + let refs: Vec<_> = idps.iter().collect(); + let sys = initialize(&refs).await; + let initial_requests: usize = idps.iter().map(|idp| idp.requests.load(Ordering::SeqCst)).sum(); + for idp in &idps { + idp.discovery_status.store(1, Ordering::SeqCst); + } + let recovery_sys = sys.clone(); + let recovery = AbortOnDropHandle::new(tokio::spawn(async move { recovery_sys.recover_failed_providers().await })); + let requests = || idps.iter().map(|idp| idp.requests.load(Ordering::SeqCst)).sum::(); + wait_until(|| requests() >= initial_requests + 4).await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(requests(), initial_requests + 4, "at most four discovery rounds may run concurrently"); + drop(recovery); + wait_until(|| { + idps.iter() + .map(|idp| idp.cancelled_requests.load(Ordering::SeqCst)) + .sum::() + == 4 + }) + .await; + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(requests(), initial_requests + 4, "cancelled recovery must not start queued work"); + assert!( + idps.iter() + .all(|idp| sys.provider_runtime.get_provider_state(&idp.config.id).is_err()) + ); +} + +#[tokio::test] +async fn disabled_and_rejected_providers_do_not_schedule_recovery() { + let mut disabled = RecoveryIdp::start("disabled", false, true).await; + disabled.config.enabled = false; + let sys = initialize(&[&disabled]).await; + sys.recover_failed_providers().await; + assert!(!sys.has_providers()); + assert_eq!(disabled.requests.load(Ordering::SeqCst), 0); + + let blocked = RecoveryIdp::start("blocked", false, true).await; + let sys = OidcSys::new_with_config_snapshot( + ReqwestHttpClient::with_policy(OutboundPolicy::default()), + OidcConfigSnapshot::new(vec![test_sourced_config(blocked.config.clone())]), + ) + .await + .expect("policy rejection remains non-fatal for startup"); + assert!(sys.pending_discovery.lock().await.is_empty()); + sys.recover_failed_providers().await; + assert_eq!(blocked.requests.load(Ordering::SeqCst), 0); + assert!(sys.provider_runtime.get_provider_state("blocked").is_err()); +} + +#[tokio::test] +async fn recovery_stops_on_issuer_validation_failure() { + let mut idp = RecoveryIdp::start("console", false, true).await; + idp.config.issuer = Some("https://unexpected.example.com".into()); + let sys = initialize(&[&idp]).await; + idp.discovery_status.store(200, Ordering::SeqCst); + tokio::time::timeout(Duration::from_secs(5), sys.recover_failed_providers()) + .await + .expect("validation failure stops recovery"); + assert_eq!( + idp.requests.load(Ordering::SeqCst), + 2, + "issuer rejection must not fetch JWKS or schedule another attempt" + ); + assert!(sys.provider_runtime.get_provider_state("console").is_err()); + assert!( + sys.authorize_url("console", "https://console.example.com/callback", None) + .await + .is_err() + ); +} + +#[tokio::test] +async fn recovery_logs_failure_reasons_without_repeating_warnings() { + use std::io::{Read, Seek}; + + for (explicit, status, expected_kind) in [ + (false, 503, "http_status"), + (true, 503, "http_status"), + (true, 0, "request"), + (true, 1, "timeout"), + ] { + let idp = RecoveryIdp::start("logging", false, explicit).await; + let sys = initialize(&[&idp]).await; + let requests = idp.requests.load(Ordering::SeqCst); + idp.discovery_status.store(status, Ordering::SeqCst); + let mut output = tempfile::tempfile().expect("create captured log file"); + let subscriber = tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_max_level(tracing::Level::WARN) + .with_writer(output.try_clone().expect("clone captured log file")) + .finish(); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + tokio::time::timeout(Duration::from_secs(25), async { + tokio::join!(sys.recover_failed_providers(), async { + // Two failing rounds exercise suppression; a timeout is already bounded by the HTTP client. + while if status == 1 { + idp.cancelled_requests.load(Ordering::SeqCst) == 0 + } else { + idp.requests.load(Ordering::SeqCst) < requests + if explicit { 2 } else { 4 } + } { + tokio::time::sleep(Duration::from_millis(10)).await; + } + idp.discovery_status.store(200, Ordering::SeqCst); + }); + }) + .await + .expect("recover after recording failure diagnostics"); + output.rewind().expect("rewind captured log file"); + let mut logs = String::new(); + output.read_to_string(&mut logs).expect("read captured log file"); + assert_eq!(logs.lines().count(), 1, "explicit={explicit}, status={status}: {logs}"); + assert!(logs.contains("provider_discovery_recovery_pending"), "{logs}"); + assert!(logs.contains(&format!("error_kind=\"{expected_kind}\"")), "{logs}"); + if status == 503 { + assert!(logs.contains("error_detail=503"), "{logs}"); + } + assert!(logs.contains("provider_id=logging"), "{logs}"); + assert!(logs.contains("suppressed=0"), "{logs}"); + } +} diff --git a/crates/iam/src/oidc/runtime.rs b/crates/iam/src/oidc/runtime.rs index 3a5797fc1..7088d31d9 100644 --- a/crates/iam/src/oidc/runtime.rs +++ b/crates/iam/src/oidc/runtime.rs @@ -16,7 +16,7 @@ use super::{ OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderSummary, OidcSiteReplicationSnapshot, SourcedOidcProviderConfig, load_oidc_config_snapshot, }, - provider::{ProviderRuntime, discover_provider}, + provider::{ProviderRuntime, discover_provider_for_recovery}, state::{OidcAuthSession, OidcLogoutSession, OidcStateStore}, transport::{OidcExtraRootCaProvider, ReqwestHttpClient, inspect_token_response_body, oidc_http_error_diagnostics}, }; @@ -37,11 +37,19 @@ use openidconnect::{ PkceCodeVerifier, PostLogoutRedirectUrl, RedirectUrl, RequestTokenError, Scope, }; use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive}; +use rustfs_utils::logging::LogThrottle; use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::collections::HashMap; use std::sync::Arc; -use tracing::{debug, error, warn}; +use std::time::Duration; +use tokio::sync::{Mutex, Semaphore}; +use tracing::{debug, error, info, warn}; + +const OIDC_DISCOVERY_RECOVERY_CONCURRENCY: usize = 4; +const OIDC_DISCOVERY_RECOVERY_MAX_DELAY_MS: u64 = 30_000; +// A round can try two issuer variants three times, each fetching discovery and JWKS. +const OIDC_DISCOVERY_RECOVERY_TIMEOUT: Duration = Duration::from_secs(125); /// Claims extracted from an OIDC ID token. #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -60,6 +68,7 @@ pub struct OidcSys { pub(super) configs: HashMap, pub(super) provider_runtime: ProviderRuntime, pub(super) state_store: OidcStateStore, + pub(super) pending_discovery: Mutex>, } pub(super) fn trusted_aud(other_audiences: &[String], audience: &Audience) -> bool { @@ -81,8 +90,16 @@ impl OidcSys { async fn new_with_http_client(http_client: ReqwestHttpClient) -> Result { let server_config = crate::server_config::current_server_config(); let parsed_configs = load_oidc_config_snapshot(server_config.as_ref()); + Self::new_with_config_snapshot(http_client, parsed_configs).await + } + + async fn new_with_config_snapshot( + http_client: ReqwestHttpClient, + parsed_configs: OidcConfigSnapshot, + ) -> Result { let mut configs = HashMap::new(); let mut provider_states = HashMap::new(); + let mut pending_discovery = Vec::new(); for sourced_config in parsed_configs.into_providers() { let config = &sourced_config.config; @@ -91,13 +108,15 @@ impl OidcSys { continue; } - match discover_provider(config, &http_client).await { + match discover_provider_for_recovery(config, &http_client).await { Ok(state) => { debug!(provider = %config.id, "OIDC provider discovered"); provider_states.insert(config.id.clone(), state); - configs.insert(config.id.clone(), sourced_config); } Err(e) => { + if e.retryable { + pending_discovery.push(config.id.clone()); + } error!( event = EVENT_OIDC_DIAGNOSTICS, component = LOG_COMPONENT_IAM, @@ -114,21 +133,95 @@ impl OidcSys { ); } } + // Configuration also supplies redirect and authorization rules before recovery. + configs.insert(config.id.clone(), sourced_config); } Ok(Self { configs, provider_runtime: ProviderRuntime::new(http_client, provider_states), state_store: OidcStateStore::new(), + pending_discovery: Mutex::new(pending_discovery), }) } + /// Recover providers that failed startup discovery due to a transient error. + /// The server owns and cancels this future; dropping it cancels all outstanding requests. + /// Pending providers are claimed once so concurrent callers cannot duplicate recovery. + pub async fn recover_failed_providers(&self) { + let pending = std::mem::take(&mut *self.pending_discovery.lock().await); + let permits = Semaphore::new(OIDC_DISCOVERY_RECOVERY_CONCURRENCY); + futures::future::join_all(pending.iter().filter_map(|id| self.get_provider_config(id)).map(|config| { + let permits = &permits; + async move { + let mut delay_ms = 1_000_u64; + let failure_log = LogThrottle::new(60_000); + loop { + let sleep_ms = rand::random_range(delay_ms * 3 / 4..=delay_ms); + tokio::time::sleep(Duration::from_millis(sleep_ms)).await; + let Ok(permit) = permits.acquire().await else { + return; + }; + let result = tokio::time::timeout( + OIDC_DISCOVERY_RECOVERY_TIMEOUT, + self.provider_runtime.recover_provider_state(&config.id, config), + ) + .await; + drop(permit); + let (error_kind, error_detail) = match result { + Ok(Ok(_)) => { + info!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "provider_discovery_recovered", + provider_id = %config.id, + "oidc provider discovery recovered" + ); + return; + } + Ok(Err(error)) if !error.retryable => { + error!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "provider_discovery_recovery_stopped", + provider_id = %config.id, + error = %error, + "oidc provider discovery requires configuration correction" + ); + return; + } + Ok(Err(error)) => error.diagnostics, + Err(_) => ("attempt_timeout", OIDC_DISCOVERY_RECOVERY_TIMEOUT.as_secs().to_string()), + }; + if let Some(suppressed) = failure_log.claim() { + warn!( + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "provider_discovery_recovery_pending", + provider_id = %config.id, + error_kind, + error_detail = %error_detail, + suppressed, + "oidc provider discovery remains unavailable" + ); + } + delay_ms = delay_ms.saturating_mul(2).min(OIDC_DISCOVERY_RECOVERY_MAX_DELAY_MS); + } + } + })) + .await; + } + /// Create an OidcSys with no providers (useful for when OIDC is not configured). pub fn empty() -> Result { Ok(Self { configs: HashMap::new(), provider_runtime: ProviderRuntime::new(ReqwestHttpClient::new()?, HashMap::new()), state_store: OidcStateStore::new(), + pending_discovery: Mutex::default(), }) } @@ -895,3 +988,7 @@ impl FederatedProviderQuery for StandardOidcAdapter { #[cfg(test)] #[path = "runtime_tests.rs"] mod tests; + +#[cfg(test)] +#[path = "recovery_tests.rs"] +mod recovery_tests; diff --git a/crates/iam/src/oidc/runtime_tests.rs b/crates/iam/src/oidc/runtime_tests.rs index afca6f4c3..84dd95943 100644 --- a/crates/iam/src/oidc/runtime_tests.rs +++ b/crates/iam/src/oidc/runtime_tests.rs @@ -175,6 +175,7 @@ async fn web_identity_verification_refreshes_rotated_jwks() { configs: HashMap::from([(config.id.clone(), test_sourced_config(config.clone()))]), provider_runtime: ProviderRuntime::new(http_client, HashMap::from([(config.id.clone(), state)])), state_store: OidcStateStore::new(), + pending_discovery: Default::default(), }; let now = std::time::SystemTime::now() @@ -208,6 +209,15 @@ async fn web_identity_verification_refreshes_rotated_jwks() { } #[tokio::test] async fn complete_provider_console_login_preserves_hidden_and_issuer_modes() { + assert_complete_console_login(false).await; +} + +#[tokio::test] +async fn console_login_recovers_startup_discovery_with_authorization_rules() { + assert_complete_console_login(true).await; +} + +async fn assert_complete_console_login(initially_unavailable: bool) { use tokio::io::{AsyncReadExt, AsyncWriteExt}; for hidden in [false, true] { @@ -221,6 +231,8 @@ async fn complete_provider_console_login_preserves_hidden_and_issuer_modes() { }; let mut config = build_mocked_oidc_provider_config("console", &format!("{base}/.well-known/openid-configuration")); config.hide_from_ui = hidden; + config.claim_name = "groups".into(); + config.claim_prefix = "team/".into(); config.issuer = explicit.then(|| issuer.clone()); config.client_secret = Some(Nonce::new_random().secret().clone()); let server_config = config.clone(); @@ -230,6 +242,23 @@ async fn complete_provider_console_login_preserves_hidden_and_issuer_modes() { let (auth_tx, auth_rx) = tokio::sync::oneshot::channel::>(); let server = tokio::spawn(async move { let mut auth_rx = Some(auth_rx); + let failures = if initially_unavailable { + if explicit { 1 } else { 2 } + } else { + 0 + }; + for _ in 0..failures { + let (mut stream, _) = listener.accept().await.expect("startup discovery request"); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + request.push(stream.read_u8().await.expect("discovery request headers")); + assert!(request.len() < 8192); + } + stream + .write_all(b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await + .expect("startup discovery failure response"); + } for expected_path in ["/.well-known/openid-configuration", "/jwks", "/token"] { let (mut stream, _) = tokio::time::timeout(StdDuration::from_secs(10), listener.accept()) .await @@ -309,12 +338,21 @@ async fn complete_provider_console_login_preserves_hidden_and_issuer_modes() { } }); let http_client = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).unwrap()); - let discovered = discover_provider(&config, &http_client).await.unwrap(); - let sys = OidcSys { - configs: HashMap::from([(config.id.clone(), test_sourced_config(config))]), - provider_runtime: ProviderRuntime::new(http_client, HashMap::from([("console".into(), discovered)])), - state_store: OidcStateStore::new(), - }; + let sys = Arc::new( + OidcSys::new_with_config_snapshot(http_client, OidcConfigSnapshot::new(vec![test_sourced_config(config)])) + .await + .expect("OIDC initialization remains available during an IdP outage"), + ); + let adapter = StandardOidcAdapter::new(sys.clone()); + let mapper = CoreFederatedAuthorizationMapper::new(adapter.authorization_rules()); + assert_eq!(sys.list_visible_providers().len(), usize::from(!hidden)); + if initially_unavailable { + assert!(sys.provider_runtime.get_provider_state("console").is_err()); + assert!(sys.authorize_url("console", redirect, None).await.is_err()); + tokio::time::timeout(StdDuration::from_secs(10), sys.recover_failed_providers()) + .await + .expect("discovery should recover after the IdP becomes available"); + } let auth_url = sys.authorize_url("console", redirect, Some("/buckets".into())).await.unwrap(); let auth_url = Url::parse(&auth_url).unwrap(); assert_eq!(auth_url.as_str().split('?').next(), Some(format!("{base}/authorize").as_str())); @@ -328,15 +366,18 @@ async fn complete_provider_console_login_preserves_hidden_and_issuer_modes() { assert!(auth["scope"].split_whitespace().any(|scope| scope == "openid")); let state = auth["state"].clone(); auth_tx.send(auth).unwrap(); - let (claims, provider, session, _) = sys - .exchange_code(&state, "test-authorization-code", redirect) + let exchange = adapter + .exchange_identity(&state, "test-authorization-code", redirect) .await .unwrap_or_else(|err| panic!("hidden={hidden}, explicit={explicit}: {err}")); - assert_eq!(provider, "console"); - assert_eq!(claims.sub, "existing-user"); - assert_eq!(claims.email, "user@example.com"); - assert_eq!(claims.groups, vec!["readwrite"]); - assert_eq!(session.redirect_after.as_deref(), Some("/buckets")); + let (identity, redirect_after, _) = exchange.into_parts(); + let authorization = mapper.map(identity); + assert_eq!(authorization.provider_id, "console"); + assert_eq!(authorization.claims.sub, "existing-user"); + assert_eq!(authorization.claims.email, "user@example.com"); + assert_eq!(authorization.claims.groups, vec!["readwrite"]); + assert_eq!(authorization.policies, vec!["team/readwrite"]); + assert_eq!(redirect_after.as_deref(), Some("/buckets")); assert!(matches!(sys.exchange_code(&state, "test-authorization-code", redirect).await, Err(error) if error == "invalid or expired OIDC state")); server.await.unwrap(); @@ -470,6 +511,7 @@ async fn oidc_token_exchange_reports_forbidden_token_endpoint() { )]), ), state_store, + pending_discovery: Default::default(), }; let error = match sys diff --git a/crates/iam/src/oidc/test_support.rs b/crates/iam/src/oidc/test_support.rs index e81855537..51262f675 100644 --- a/crates/iam/src/oidc/test_support.rs +++ b/crates/iam/src/oidc/test_support.rs @@ -35,6 +35,7 @@ pub fn make_test_sys(configs: Vec) -> OidcSys { HashMap::new(), ), state_store: OidcStateStore::new(), + pending_discovery: Default::default(), } } diff --git a/crates/iam/src/oidc/transport.rs b/crates/iam/src/oidc/transport.rs index 1fd8706f0..18a708a4d 100644 --- a/crates/iam/src/oidc/transport.rs +++ b/crates/iam/src/oidc/transport.rs @@ -312,6 +312,10 @@ struct CachedOidcExtraRootCerts { certs: Vec, } +/// Discovery callers report failures with provider context and recovery throttling. +#[derive(Clone)] +pub(super) struct DiscoveryRequest; + /// HTTP client adapter bridging reqwest 0.13 to the `openidconnect` `AsyncHttpClient` trait. /// /// A fresh client is built for every request so the destination is re-validated and the @@ -555,6 +559,7 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient { Box::pin(async move { let started_at = Instant::now(); let (parts, body) = request.into_parts(); + let discovery_request = parts.extensions.get::().is_some(); let method = parts.method.clone(); let uri = parts.uri.to_string(); if tracing::enabled!(tracing::Level::DEBUG) { @@ -593,17 +598,19 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient { let response = response.map_err(|err| { let error = oidc_http_error_from_reqwest(&url, err); - error!( - event = EVENT_OIDC_HTTP, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "request_failed", - method = %method, - uri = %uri, - elapsed_ms, - error = %error, - "oidc outbound http" - ); + if !discovery_request { + error!( + event = EVENT_OIDC_HTTP, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "request_failed", + method = %method, + uri = %uri, + elapsed_ms, + error = %error, + "oidc outbound http" + ); + } error })?; @@ -612,18 +619,20 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient { let body_bytes = read_bounded_response_body(response, MAX_OIDC_RESPONSE_SIZE) .await .map_err(|err| { - error!( - event = EVENT_OIDC_HTTP, - component = LOG_COMPONENT_IAM, - subsystem = LOG_SUBSYSTEM_OIDC, - result = "response_body_failed", - method = %method, - uri = %uri, - status = status.as_u16(), - elapsed_ms, - error = %err, - "oidc outbound http" - ); + if !discovery_request { + error!( + event = EVENT_OIDC_HTTP, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + result = "response_body_failed", + method = %method, + uri = %uri, + status = status.as_u16(), + elapsed_ms, + error = %err, + "oidc outbound http" + ); + } err })?; if tracing::enabled!(tracing::Level::DEBUG) { diff --git a/rustfs/src/startup_auth.rs b/rustfs/src/startup_auth.rs index ec2cbe6f4..c704a9fbf 100644 --- a/rustfs/src/startup_auth.rs +++ b/rustfs/src/startup_auth.rs @@ -19,10 +19,12 @@ use rustfs_iam::{ }; use std::{ collections::hash_map::DefaultHasher, + future::Future, hash::{Hash, Hasher}, io::{Error, Result}, sync::Arc, }; +use tokio_util::{sync::CancellationToken, task::AbortOnDropHandle}; use tracing::{error, info, warn}; const LOG_COMPONENT_MAIN: &str = "main"; @@ -31,7 +33,7 @@ const EVENT_KEYSTONE_AUTH_INITIALIZED: &str = "keystone_auth_initialized"; const EVENT_KEYSTONE_AUTH_INITIALIZATION_FAILED: &str = "keystone_auth_initialization_failed"; const EVENT_OIDC_INITIALIZATION_FAILED: &str = "oidc_initialization_failed"; -pub(crate) async fn init_auth_integrations() -> Result<()> { +pub(crate) async fn init_auth_integrations(shutdown: CancellationToken) -> Result>> { let keystone_config = rustfs_keystone::KeystoneConfig::from_env().map_err(Error::other)?; if keystone_config.enable { match crate::auth_keystone::init_keystone_auth(keystone_config).await { @@ -55,8 +57,13 @@ pub(crate) async fn init_auth_integrations() -> Result<()> { match build_oidc_sys_with_extra_root_ca_provider(oidc_extra_root_ca_provider()).await { Ok(oidc) => { - let (service, oidc_config_query) = standard_oidc_runtime(oidc); - crate::runtime_sources::publish_federated_identity_runtime(service, oidc_config_query); + let (service, oidc_config_query) = standard_oidc_runtime(oidc.clone()); + if crate::runtime_sources::publish_federated_identity_runtime(service, oidc_config_query) { + return Ok(Some(spawn_oidc_discovery_recovery( + async move { oidc.recover_failed_providers().await }, + shutdown, + ))); + } } Err(e) => { warn!( @@ -69,7 +76,20 @@ pub(crate) async fn init_auth_integrations() -> Result<()> { } } - Ok(()) + Ok(None) +} + +fn spawn_oidc_discovery_recovery( + recovery: impl Future + Send + 'static, + shutdown: CancellationToken, +) -> AbortOnDropHandle<()> { + AbortOnDropHandle::new(tokio::spawn(async move { + tokio::select! { + biased; + () = shutdown.cancelled() => {} + () = recovery => {} + } + })) } fn standard_oidc_runtime(oidc: Arc) -> (Arc, Arc) { @@ -133,4 +153,48 @@ mod tests { drop(oidc_config_query); assert_eq!(Arc::strong_count(&retained), 1, "dropping the runtime should release its OIDC Arc"); } + + #[tokio::test] + async fn oidc_recovery_shutdown_releases_pending_work() { + let shutdown = CancellationToken::new(); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (released_tx, released_rx) = tokio::sync::oneshot::channel::<()>(); + let recovery = spawn_oidc_discovery_recovery( + async move { + let _release = released_tx; + let _ = started_tx.send(()); + std::future::pending::<()>().await; + }, + shutdown.child_token(), + ); + started_rx.await.expect("recovery started"); + shutdown.cancel(); + tokio::time::timeout(std::time::Duration::from_secs(5), recovery) + .await + .expect("recovery observes shutdown") + .expect("recovery joins cleanly"); + assert!(released_rx.await.is_err(), "shutdown drops in-flight recovery work"); + } + + #[tokio::test] + async fn oidc_recovery_owner_drop_releases_pending_work() { + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (released_tx, released_rx) = tokio::sync::oneshot::channel::<()>(); + let recovery = spawn_oidc_discovery_recovery( + async move { + let _release = released_tx; + let _ = started_tx.send(()); + std::future::pending::<()>().await; + }, + CancellationToken::new(), + ); + started_rx.await.expect("recovery started"); + drop(recovery); + assert!( + tokio::time::timeout(std::time::Duration::from_secs(5), released_rx) + .await + .expect("owner drop aborts recovery") + .is_err() + ); + } } diff --git a/rustfs/src/startup_lifecycle.rs b/rustfs/src/startup_lifecycle.rs index ed17225ed..d3360935e 100644 --- a/rustfs/src/startup_lifecycle.rs +++ b/rustfs/src/startup_lifecycle.rs @@ -130,6 +130,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec readiness, } = lifecycle; let StartupServiceRuntime { + oidc_recovery, optional_runtimes, heartbeat, inventory, @@ -166,6 +167,18 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec shutdown_token, ) .await; + if let Some(recovery) = oidc_recovery + && let Err(error) = recovery.await + { + tracing::warn!( + event = EVENT_SERVER_SHUTDOWN_STATE, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + result = "oidc_discovery_recovery_join_failed", + error = %error, + "OIDC discovery recovery task failed to join" + ); + } shutdown_connect_runtimes(heartbeat, inventory).await; if let Some(runtime) = local_trace_capture { runtime.shutdown().await; diff --git a/rustfs/src/startup_services.rs b/rustfs/src/startup_services.rs index 5dd0fe9a7..4a3b66a5e 100644 --- a/rustfs/src/startup_services.rs +++ b/rustfs/src/startup_services.rs @@ -36,13 +36,14 @@ use crate::{ }; use rustfs_common::GlobalReadiness; use std::{collections::BTreeSet, io::Result, sync::Arc}; -use tokio_util::sync::CancellationToken; +use tokio_util::{sync::CancellationToken, task::AbortOnDropHandle}; const EVENT_DRIVE_UNAVAILABLE: &str = "drive_unavailable"; const LOG_COMPONENT_CONNECT: &str = "connect"; const LOG_SUBSYSTEM_INVENTORY: &str = "inventory"; pub(crate) struct StartupServiceRuntime { + pub(crate) oidc_recovery: Option>, pub(crate) optional_runtimes: OptionalRuntimeServices, pub(crate) heartbeat: Option, pub(crate) inventory: Option, @@ -111,7 +112,7 @@ pub(crate) async fn init_startup_runtime_services( // self-pointing replication rules until the next restart. The task waits for IAM and // bucket metadata itself, and its first pass runs off this path. spawn_site_replication_reconcile_task(ctx.clone()); - init_auth_integrations().await?; + let oidc_recovery = init_auth_integrations(ctx.child_token()).await?; init_notification_runtime(endpoint_pools, buckets).await?; let enable_scanner = init_background_service_runtime(store.clone()).await?; init_observability_runtime(store.clone(), ctx.clone()).await; @@ -119,6 +120,7 @@ pub(crate) async fn init_startup_runtime_services( let inventory = start_inventory_runtime(heartbeat_config, heartbeat_nodes, inventory_drives, store, &ctx)?; Ok(StartupServiceRuntime { + oidc_recovery, optional_runtimes, heartbeat, inventory, From 0ef133bbb07f09ecdebec6b9d62cf49913d9a1a2 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 20:03:16 +0800 Subject: [PATCH 027/143] fix: refresh peer metadata without blocking bucket mutations (#8486) * fix: refresh peer bucket metadata after creation * fix(s3): release bucket mutation slots before peer reloads * test(ecstore): distinguish delete lock waits from disk latency --- crates/ecstore/src/api/mod.rs | 8 +- crates/ecstore/src/bucket/metadata_sys.rs | 138 +++++++ crates/ecstore/src/set_disk/ops/object.rs | 53 ++- docs/operations/bucket-operation-admission.md | 6 +- rustfs/src/app/bucket_usecase.rs | 345 +++++++++++++++--- rustfs/src/app/runtime_sources.rs | 3 +- rustfs/src/storage/rpc/node_service/bucket.rs | 16 +- rustfs/src/storage/storage_api.rs | 6 +- 8 files changed, 491 insertions(+), 84 deletions(-) diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index fc26177a9..ee36d0b76 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -199,10 +199,10 @@ pub mod bucket { get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config, - init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, - update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, - update_if_incarnation_at, update_quota_if_incarnation, update_quota_if_incarnation_at, update_under_transaction_lock, - update_under_transaction_lock_at, + init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, + remove_bucket_metadata_if_absent, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, + update_config_with, update_if_incarnation, update_if_incarnation_at, update_quota_if_incarnation, + update_quota_if_incarnation_at, update_under_transaction_lock, update_under_transaction_lock_at, }; #[cfg(feature = "test-util")] pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support}; diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index e8553f433..1dc3e43e1 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -314,6 +314,33 @@ pub async fn remove_bucket_metadata(bucket: &str) -> Result { Ok(lock.remove(bucket).await) } +/// Remove a peer's cached metadata only when the bucket is absent under the +/// namespace read fence. A delayed delete notification must not withdraw the +/// metadata of a same-name bucket that has already been recreated. +pub async fn remove_bucket_metadata_if_absent(api: Arc, bucket: &str) -> Result { + if is_meta_bucketname(bucket) { + return Ok(false); + } + // Lock order: bucket namespace read -> metadata system read (released) + // -> metadata publish, matching peer reloads. + let namespace_lock = api.new_ns_lock(bucket, bucket).await?; + let namespace_guard = namespace_lock + .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let sys = bucket_metadata_sys_of(&api.ctx)?; + let sys = sys.read().await.clone(); + if sys + .bucket_exists(bucket, &namespace_guard, "peer bucket metadata removal existence check") + .await? + { + return Ok(false); + } + await_bucket_namespace_operation(Some(&namespace_guard), bucket, "peer bucket metadata removal", async { + Ok(sys.remove(bucket).await) + }) + .await +} + fn start_refresh_buckets_metadata_loop(sys: Arc>) { let Some(cancel_token) = runtime_sources::background_services_cancel_token() else { warn!("bucket metadata refresh loop skipped because background cancellation token is not initialized"); @@ -4856,6 +4883,117 @@ mod tests { ); } + #[tokio::test] + #[serial] + async fn peer_reload_clears_negative_cache_after_remote_bucket_creation() { + let (dirs, node_a, node_b) = two_nodes_over_one_store().await; + let bucket = "cross-node-negative-cache-refresh"; + let node_b_sys = node_b.read().await.clone(); + + assert_eq!( + node_b_sys.get_bucket_incarnation_id(bucket).await.unwrap_err(), + Error::BucketNotFound(bucket.to_string()), + "the probe must populate node B's missing-bucket cache" + ); + + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist before creation metadata"); + } + let created = BucketMetadata::new(bucket); + let created_incarnation = created.bucket_incarnation_id; + node_a + .read() + .await + .persist_new_and_set(created) + .await + .expect("node A should publish the created bucket metadata"); + + node_b_sys + .reload_from_store(bucket) + .await + .expect("peer reload should clear the stale negative cache"); + assert_eq!( + node_b_sys.get_bucket_incarnation_id(bucket).await.unwrap(), + created_incarnation, + "node B should resolve the newly created bucket immediately" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[serial] + async fn peer_delete_clears_metadata_for_absent_bucket() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner"); + let bucket = "peer-delete-absent-bucket"; + + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("initial bucket creation"); + let cached = sys.read().await.get(bucket).await.expect("initial cached metadata"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("bucket deletion"); + + // Model a peer that still caches the metadata after the physical delete. + sys.read().await.set(bucket.to_string(), cached).await; + assert!( + remove_bucket_metadata_if_absent(store.clone(), bucket) + .await + .expect("conditional peer delete"), + "the peer must evict the deleted bucket's cached metadata" + ); + let sys = sys.read().await; + assert!(matches!(sys.get(bucket).await, Err(Error::ConfigNotFound))); + assert_eq!( + sys.get_bucket_incarnation_id(bucket) + .await + .expect_err("deleted bucket lookup"), + Error::BucketNotFound(bucket.to_string()), + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[serial] + async fn delayed_peer_delete_preserves_recreated_bucket_metadata() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner"); + let bucket = "delayed-peer-delete-recreated-bucket"; + + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("initial bucket creation"); + let old_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation"); + store + .delete_bucket(bucket, &DeleteBucketOptions::default()) + .await + .expect("bucket deletion"); + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket recreation"); + let new_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("new incarnation"); + assert_ne!(old_incarnation, new_incarnation); + + let removed = remove_bucket_metadata_if_absent(store.clone(), bucket) + .await + .expect("conditional peer delete"); + assert!(!removed, "a delayed delete must not remove metadata for a live recreated bucket"); + assert_eq!( + sys.read() + .await + .get(bucket) + .await + .expect("recreated metadata should remain cached") + .bucket_incarnation_id, + new_incarnation, + ); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn stale_config_request_cannot_mutate_a_recreated_bucket() { diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index c4e25a5da..fba95e548 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -22999,22 +22999,43 @@ mod delete_objects_lock_gating_tests { ..Default::default() }]; - let (deleted, errs) = tokio::time::timeout( - Duration::from_secs(1), - set_disks.delete_objects( - bucket, - objects, - ObjectOptions { - no_lock: true, - object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( - ObjectLockConfigState::ConfirmedAbsent, - ))), - ..Default::default() - }, - ), - ) - .await - .expect("no_lock batch delete path must not wait for the outer lock"); + let lock_manager = set_disks + .local_lock_manager + .as_fast_lock_manager() + .expect("isolated fixture should use the local lock manager"); + let lock_metrics = lock_manager.get_metrics().shard_metrics; + assert_eq!(lock_manager.current_lock_counts(), (1, 0)); + let mut delete = Box::pin(set_disks.delete_objects( + bucket, + objects, + ObjectOptions { + no_lock: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + )); + // Observe lock attempts on every poll: durable disk work can take more + // than a second on CI without waiting for the namespace lock at all. + let deletion = futures::future::poll_fn(|cx| { + let result = std::future::Future::poll(delete.as_mut(), cx); + assert_eq!( + lock_manager.current_lock_counts(), + (1, 0), + "no_lock batch delete must preserve the outer holder without adding lock waiters" + ); + let current = lock_manager.get_metrics().shard_metrics; + assert_eq!( + (current.total_acquisitions(), current.timeouts), + (lock_metrics.total_acquisitions(), lock_metrics.timeouts), + "no_lock batch delete must not acquire or time out on another namespace lock" + ); + result + }); + let (deleted, errs) = tokio::time::timeout(Duration::from_secs(30), deletion) + .await + .expect("no_lock batch delete disk work should complete"); assert!(errs[0].is_none(), "no_lock batch delete should not fail with a lock error: {:?}", errs[0]); assert!(deleted[0].found, "existing object must still be deleted"); diff --git a/docs/operations/bucket-operation-admission.md b/docs/operations/bucket-operation-admission.md index bab36f986..3e46f1bb7 100644 --- a/docs/operations/bucket-operation-admission.md +++ b/docs/operations/bucket-operation-admission.md @@ -1,10 +1,10 @@ # Bucket creation and deletion admission -`CreateBucket` and `DeleteBucket` share a process-wide execution budget of eight active transactions. Short bursts wait in a FIFO queue with space for 128 waiting requests. Each request can wait up to 30 seconds for an execution slot. These limits protect storage and namespace-lock resources while allowing parallel test workers to create isolated buckets. +`CreateBucket` and `DeleteBucket` share a process-wide execution budget of eight active storage mutations. Short bursts wait in a FIFO queue with space for 128 waiting requests. Each request can wait up to 30 seconds for an execution slot. A separate total budget bounds all unfinished operations to 136, including queued requests, mutations, and post-commit peer reloads. These limits protect storage, namespace-lock, and notification resources while allowing parallel test workers to create isolated buckets. -A request that finds the queue full or exceeds the queue wait returns HTTP 503 with the S3 error code `SlowDown` and `Retry-After: 1`. This is an overload response, not a bucket-name or filesystem error. The active and waiting limits are shared by creation and deletion and are currently fixed. +A request that finds the queue full, exceeds the queue wait, or reaches the total in-flight limit returns HTTP 503 with the S3 error code `SlowDown` and `Retry-After: 1`. Admission rejection happens before storage mutation begins. This is an overload response, not a bucket-name or filesystem error. The limits are shared by creation and deletion and are currently fixed. -Waiting requests do not start storage mutations. Disconnecting while waiting releases the queue slot. Once admitted, the transaction owns its execution slot until the mutation and post-commit hooks finish, including when its caller disconnects. +Waiting requests do not start storage mutations. Disconnecting while waiting releases both the queue slot and its total-operation slot. Once admitted, the detached task continues even when its caller disconnects. Creation releases its execution slot after the storage mutation, site-replication hook, and associated locks finish, then waits for peer metadata reloads before returning. Its total-operation slot remains held until that work finishes, so slow reloads cannot accumulate unbounded background tasks or occupy every storage execution slot. Deletion holds its execution slot through its usecase; its existing asynchronous peer cache invalidation is unchanged. For automated tests and applications: diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 074ec0c66..445978a09 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -69,7 +69,8 @@ use crate::app::object::shared::mark_on_demand_migration_list_local_only; use crate::app::object_data_cache::invalidate_object_data_cache_bucket_after_delete; use crate::app::runtime_sources::{ AppContext, current_app_context, current_encryption_service, current_notification_system, - current_notify_interface_for_context, current_object_data_cache_for_context, current_object_store_handle_for_context, + current_notification_system_for_context, current_notify_interface_for_context, current_object_data_cache_for_context, + current_object_store_handle_for_context, }; use crate::app::table_list_isolation; use crate::auth::get_condition_values_with_client_info; @@ -151,6 +152,7 @@ static BUCKET_OPERATION_ADMISSION: LazyLock> = Laz struct BucketOperationAdmission { active: Arc, waiting: Semaphore, + in_flight: Arc, } impl BucketOperationAdmission { @@ -158,12 +160,19 @@ impl BucketOperationAdmission { Self { active: Arc::new(Semaphore::new(concurrency)), waiting: Semaphore::new(queue_capacity), + in_flight: Arc::new(Semaphore::new(concurrency + queue_capacity)), } } - async fn acquire(&self, operation: &'static str) -> S3Result { + async fn acquire(&self, operation: &'static str) -> S3Result<(OwnedSemaphorePermit, OwnedSemaphorePermit)> { + // Reserve the entire operation before mutation, including post-commit + // work that can outlive both its execution slot and the request. + let in_flight = self.in_flight.clone().try_acquire_owned().map_err(|err| match err { + TryAcquireError::NoPermits => bucket_admission_slow_down(operation, "in-flight limit reached"), + TryAcquireError::Closed => bucket_admission_closed(operation), + })?; match self.active.clone().try_acquire_owned() { - Ok(permit) => return Ok(permit), + Ok(permit) => return Ok((permit, in_flight)), Err(TryAcquireError::Closed) => return Err(bucket_admission_closed(operation)), Err(TryAcquireError::NoPermits) => {} } @@ -174,10 +183,11 @@ impl BucketOperationAdmission { TryAcquireError::NoPermits => bucket_admission_slow_down(operation, "queue is full"), TryAcquireError::Closed => bucket_admission_closed(operation), })?; - tokio::time::timeout(BUCKET_OPERATION_QUEUE_TIMEOUT, self.active.clone().acquire_owned()) + let active = tokio::time::timeout(BUCKET_OPERATION_QUEUE_TIMEOUT, self.active.clone().acquire_owned()) .await .map_err(|_| bucket_admission_slow_down(operation, "queue wait timed out"))? - .map_err(|_| bucket_admission_closed(operation)) + .map_err(|_| bucket_admission_closed(operation))?; + Ok((active, in_flight)) } } @@ -615,12 +625,27 @@ async fn notify_bucket_metadata_reload( operation: &'static str, request_context: Option, scanner_maintenance_change: bool, +) { + notify_bucket_metadata_reload_with_context(bucket, operation, request_context, scanner_maintenance_change, None).await; +} + +async fn notify_bucket_metadata_reload_with_context( + bucket: String, + operation: &'static str, + request_context: Option, + scanner_maintenance_change: bool, + app_context: Option>, ) { record_local_scanner_maintenance_reload(&bucket, scanner_maintenance_change); // Keep reload detached across request cancellation, but wait before a healthy peer can serve the previous config. let (completed_tx, completed_rx) = tokio::sync::oneshot::channel(); spawn_background_with_context(request_context, async move { - if let Some(notification_sys) = current_notification_system() { + let notification_sys = if let Some(context) = app_context.as_deref() { + current_notification_system_for_context(Some(context)) + } else { + current_notification_system() + }; + if let Some(notification_sys) = notification_sys { let result = if scanner_maintenance_change { notification_sys.load_bucket_metadata_for_scanner_maintenance(&bucket).await } else { @@ -1357,28 +1382,30 @@ pub struct DefaultBucketUsecase { context: Option>, } -async fn await_bucket_usecase_on_fresh_task(operation: &'static str, future: F) -> S3Result +async fn await_bucket_usecase_on_fresh_task(operation: &'static str, make_future: MakeFuture) -> S3Result where T: Send + 'static, F: Future> + Send + 'static, + MakeFuture: FnOnce(OwnedSemaphorePermit) -> F + Send + 'static, { - await_bucket_usecase_on_fresh_task_with_admission(operation, Arc::clone(&BUCKET_OPERATION_ADMISSION), future).await + await_bucket_usecase_on_fresh_task_with_admission(operation, Arc::clone(&BUCKET_OPERATION_ADMISSION), make_future).await } -async fn await_bucket_usecase_on_fresh_task_with_admission( +async fn await_bucket_usecase_on_fresh_task_with_admission( operation: &'static str, admission: Arc, - future: F, + make_future: MakeFuture, ) -> S3Result where T: Send + 'static, F: Future> + Send + 'static, + MakeFuture: FnOnce(OwnedSemaphorePermit) -> F + Send + 'static, { - let permit = admission.acquire(operation).await?; + let (mutation_permit, in_flight) = admission.acquire(operation).await?; tokio::spawn( async move { - let _permit = permit; - future.await + let _in_flight = in_flight; + make_future(mutation_permit).await } .in_current_span(), ) @@ -1421,11 +1448,19 @@ impl DefaultBucketUsecase { )] pub async fn execute_create_bucket(&self, req: S3Request) -> S3Result> { let usecase = self.clone(); - await_bucket_usecase_on_fresh_task("bucket creation", async move { usecase.execute_create_bucket_inner(req).await }).await + await_bucket_usecase_on_fresh_task("bucket creation", move |mutation_permit| async move { + usecase.execute_create_bucket_inner(req, mutation_permit).await + }) + .await } - async fn execute_create_bucket_inner(&self, req: S3Request) -> S3Result> { + async fn execute_create_bucket_inner( + &self, + req: S3Request, + mutation_permit: OwnedSemaphorePermit, + ) -> S3Result> { let helper = OperationHelper::new(&req, EventName::BucketCreated, S3Operation::CreateBucket); + let request_context = req.extensions.get::().cloned(); let requester_is_owner = match req_info_ref(&req) { Ok(r) => r.is_owner, Err(_) => { @@ -1469,8 +1504,27 @@ impl DefaultBucketUsecase { }) .await?; + // The storage/site mutation and its lock fences have finished. Keep + // the operation's in-flight slot while peers reload, without holding + // an execution slot needed by other creates and deletes. + drop(mutation_permit); + match make_result { - Ok(()) => {} + Ok(()) => { + // A peer may have cached this bucket as missing before the + // create. Refresh peer metadata after the namespace and + // metadata commit fences are released so the peer reload can + // acquire its bucket namespace read lock without an ABBA + // interleaving with creation. + notify_bucket_metadata_reload_with_context( + bucket.clone(), + "create bucket", + request_context, + false, + self.context.clone(), + ) + .await; + } Err(StorageError::BucketExists(_)) => { // Per S3 spec: bucket namespace is global. Owner recreating returns 200 OK; // non-owner gets 409 BucketAlreadyExists. @@ -1495,7 +1549,11 @@ impl DefaultBucketUsecase { #[instrument(level = "debug", skip(self, req))] pub async fn execute_delete_bucket(&self, req: S3Request) -> S3Result> { let usecase = self.clone(); - await_bucket_usecase_on_fresh_task("bucket deletion", async move { usecase.execute_delete_bucket_inner(req).await }).await + await_bucket_usecase_on_fresh_task("bucket deletion", move |mutation_permit| async move { + let _mutation_permit = mutation_permit; + usecase.execute_delete_bucket_inner(req).await + }) + .await } async fn execute_delete_bucket_inner( @@ -3308,7 +3366,8 @@ mod tests { let parent = tokio::spawn(await_bucket_usecase_on_fresh_task_with_admission( "test bucket operation", admission.clone(), - async move { + move |mutation_permit| async move { + drop(mutation_permit); committed_for_task.notify_one(); release_hook_for_task.notified().await; hook_ran_for_task.store(true, Ordering::SeqCst); @@ -3320,6 +3379,8 @@ mod tests { parent.abort(); let _ = parent.await; + assert_eq!(admission.active.available_permits(), 1); + assert_eq!(admission.in_flight.available_permits(), 1, "cancelled post-commit work stays bounded"); release_hook.notify_waiters(); tokio::time::timeout(Duration::from_secs(1), finished_wait) .await @@ -3330,6 +3391,137 @@ mod tests { "the fresh task must own both the storage mutation and its post-commit hooks" ); assert_eq!(admission.active.available_permits(), 1); + assert_eq!(admission.in_flight.available_permits(), 2); + } + + #[tokio::test(start_paused = true)] + async fn bucket_usecase_admission_releases_execution_during_slow_post_commit_work() { + let admission = Arc::new(BucketOperationAdmission::new( + BUCKET_OPERATION_CONCURRENCY, + BUCKET_OPERATION_QUEUE_CAPACITY, + )); + let active_mutations = Arc::new(AtomicUsize::new(0)); + let peak_mutations = Arc::new(AtomicUsize::new(0)); + let (committed_tx, mut committed_rx) = tokio::sync::mpsc::unbounded_channel(); + let mut tasks = Vec::new(); + for _ in 0..100 { + let active = active_mutations.clone(); + let peak = peak_mutations.clone(); + let committed = committed_tx.clone(); + tasks.push(tokio::spawn(await_bucket_usecase_on_fresh_task_with_admission( + "bucket creation", + admission.clone(), + move |mutation_permit| async move { + let count = active.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(count, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(1)).await; + active.fetch_sub(1, Ordering::SeqCst); + drop(mutation_permit); + committed.send(()).expect("record commit"); + tokio::time::sleep(Duration::from_secs(4)).await; + Ok(()) + }, + ))); + } + tokio::time::timeout(Duration::from_secs(1), async { + for _ in 0..100 { + committed_rx.recv().await.expect("every mutation commits"); + } + }) + .await + .expect("peer reloads must not serialize subsequent mutations"); + assert_eq!(peak_mutations.load(Ordering::SeqCst), BUCKET_OPERATION_CONCURRENCY); + assert!( + tasks.iter().all(|task| !task.is_finished()), + "requests must still await their post-commit work" + ); + assert_eq!(admission.active.available_permits(), BUCKET_OPERATION_CONCURRENCY); + assert_eq!( + admission.in_flight.available_permits(), + BUCKET_OPERATION_CONCURRENCY + BUCKET_OPERATION_QUEUE_CAPACITY - 100 + ); + + await_bucket_usecase_on_fresh_task_with_admission("bucket deletion", admission.clone(), |mutation_permit| async move { + let _mutation_permit = mutation_permit; + Ok(()) + }) + .await + .expect("deletion can execute while creates await peer reloads"); + for task in tasks { + task.await.expect("request joins").expect("queued creation succeeds"); + } + assert_eq!( + admission.in_flight.available_permits(), + BUCKET_OPERATION_CONCURRENCY + BUCKET_OPERATION_QUEUE_CAPACITY + ); + } + + #[tokio::test] + async fn bucket_usecase_admission_bounds_post_commit_work_after_request_cancellation() { + let admission = Arc::new(BucketOperationAdmission::new(1, 1)); + let release = Arc::new(Semaphore::new(0)); + let (committed_tx, mut committed_rx) = tokio::sync::mpsc::unbounded_channel(); + let (finished_tx, mut finished_rx) = tokio::sync::mpsc::unbounded_channel(); + let mut parents = Vec::new(); + for _ in 0..2 { + let release = release.clone(); + let committed = committed_tx.clone(); + let finished = finished_tx.clone(); + parents.push(tokio::spawn(await_bucket_usecase_on_fresh_task_with_admission( + "bucket creation", + admission.clone(), + move |mutation_permit| async move { + drop(mutation_permit); + committed.send(()).expect("record commit"); + release.acquire().await.expect("release hook").forget(); + finished.send(()).expect("record completion"); + Ok(()) + }, + ))); + } + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..2 { + committed_rx.recv().await.expect("operation commits"); + } + }) + .await + .expect("both operations reach post-commit work"); + for parent in parents { + parent.abort(); + assert!(parent.await.expect_err("caller cancelled").is_cancelled()); + } + assert_eq!(admission.active.available_permits(), 1); + assert_eq!(admission.in_flight.available_permits(), 0); + let started = Arc::new(AtomicBool::new(false)); + let started_for_task = started.clone(); + let error = await_bucket_usecase_on_fresh_task_with_admission( + "bucket creation", + admission.clone(), + move |mutation_permit| async move { + let _mutation_permit = mutation_permit; + started_for_task.store(true, Ordering::SeqCst); + Ok(()) + }, + ) + .await + .expect_err("unfinished hooks consume the total operation budget"); + assert_eq!(error.code(), &S3ErrorCode::SlowDown); + assert_eq!(error.message(), Some("bucket creation in-flight limit reached; retry later")); + assert_eq!(error.headers().expect("retry headers")[http::header::RETRY_AFTER], "1"); + assert!(!started.load(Ordering::SeqCst)); + release.add_permits(2); + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..2 { + finished_rx.recv().await.expect("detached hook completes"); + } + while admission.in_flight.available_permits() != 2 { + tokio::task::yield_now().await; + } + }) + .await + .expect("capacity recovers after hooks complete"); + let permits = admission.acquire("bucket deletion").await.expect("new operation is admitted"); + drop(permits); } #[tokio::test(start_paused = true)] @@ -3338,16 +3530,22 @@ mod tests { let held = admission.active.clone().acquire_owned().await.expect("hold active slot"); let started = Arc::new(AtomicBool::new(false)); let started_for_task = started.clone(); - let result = await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission.clone(), async move { - started_for_task.store(true, Ordering::SeqCst); - Ok(()) - }) + let result = await_bucket_usecase_on_fresh_task_with_admission( + "test bucket operation", + admission.clone(), + move |mutation_permit| async move { + let _mutation_permit = mutation_permit; + started_for_task.store(true, Ordering::SeqCst); + Ok(()) + }, + ) .await; let error = result.expect_err("queued request must time out"); assert_eq!(error.code(), &S3ErrorCode::SlowDown); assert_eq!(error.headers().expect("retry headers")[http::header::RETRY_AFTER], "1"); assert!(!started.load(Ordering::SeqCst), "timed-out work must not start"); assert_eq!(admission.waiting.available_permits(), 1); + assert_eq!(admission.in_flight.available_permits(), 2); drop(held); assert_eq!(admission.active.available_permits(), 1); } @@ -3368,7 +3566,8 @@ mod tests { tasks.push(tokio::spawn(await_bucket_usecase_on_fresh_task_with_admission( "test bucket operation", admission_for_task, - async move { + move |mutation_permit| async move { + let _mutation_permit = mutation_permit; started_for_task.fetch_add(1, Ordering::SeqCst); release_for_task.acquire().await.expect("release gate").forget(); Ok(()) @@ -3403,7 +3602,8 @@ mod tests { let queued = tokio::spawn(await_bucket_usecase_on_fresh_task_with_admission( "test bucket operation", admission.clone(), - async move { + move |mutation_permit| async move { + let _mutation_permit = mutation_permit; started_for_task.store(true, Ordering::SeqCst); Ok(()) }, @@ -3430,35 +3630,58 @@ mod tests { queued.abort(); assert!(queued.await.expect_err("queued request was cancelled").is_cancelled()); assert_eq!(admission.waiting.available_permits(), 1); + assert_eq!(admission.in_flight.available_permits(), 2); drop(held); assert!(!started.load(Ordering::SeqCst), "cancelled waiter must not mutate storage"); - await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission, async { Ok(()) }) - .await - .expect("admission recovers after cancellation"); + await_bucket_usecase_on_fresh_task_with_admission("test bucket operation", admission, |mutation_permit| async move { + let _mutation_permit = mutation_permit; + Ok(()) + }) + .await + .expect("admission recovers after cancellation"); } #[tokio::test] async fn bucket_usecase_admission_maps_closed_semaphore_to_internal_error() { - let admission = BucketOperationAdmission::new(1, 1); - admission.active.close(); - let error = admission - .acquire("test bucket operation") - .await - .expect_err("closed admission must fail"); - assert_eq!(error.code(), &S3ErrorCode::InternalError); + for close_in_flight in [false, true] { + let admission = BucketOperationAdmission::new(1, 1); + if close_in_flight { + admission.in_flight.close(); + } else { + admission.active.close(); + } + let error = admission + .acquire("test bucket operation") + .await + .expect_err("closed admission must fail"); + assert_eq!(error.code(), &S3ErrorCode::InternalError); + assert_eq!(admission.in_flight.available_permits(), 2); + } } #[tokio::test] async fn bucket_usecase_task_maps_panics_to_internal_errors() { - async fn panicking_bucket_operation() -> S3Result<()> { - panic!("injected bucket task panic"); + for post_commit in [false, true] { + let admission = Arc::new(BucketOperationAdmission::new(1, 1)); + let result: S3Result<()> = await_bucket_usecase_on_fresh_task_with_admission( + "test bucket operation", + admission.clone(), + move |mutation_permit| async move { + let mut mutation_permit = Some(mutation_permit); + if post_commit { + drop(mutation_permit.take()); + } + panic!("injected bucket task panic"); + }, + ) + .await; + + let err = result.expect_err("a bucket task panic must be returned as an error"); + assert_eq!(err.code(), &S3ErrorCode::InternalError); + assert!(err.message().is_some_and(|message| message.contains("task failed"))); + assert_eq!(admission.active.available_permits(), 1); + assert_eq!(admission.in_flight.available_permits(), 2); } - - let result = await_bucket_usecase_on_fresh_task("test bucket operation", panicking_bucket_operation()).await; - - let err = result.expect_err("a bucket task panic must be returned as an error"); - assert_eq!(err.code(), &S3ErrorCode::InternalError); - assert!(err.message().is_some_and(|message| message.contains("task failed"))); } fn build_request(input: T, method: Method) -> S3Request { @@ -3482,30 +3705,46 @@ mod tests { } fn usecase_method_source<'a>(source: &'a str, method: &str) -> &'a str { - let start_marker = format!("pub async fn {method}"); - let start = source.find(&start_marker).expect("method should exist"); - let rest = &source[start + start_marker.len()..]; - let end = rest.find("\n pub async fn ").unwrap_or(rest.len()); + let public_start_marker = format!("pub async fn {method}"); + let private_start_marker = format!("async fn {method}"); + let (start, marker_len) = if let Some(start) = source.find(&public_start_marker) { + (start, public_start_marker.len()) + } else { + let start = source.find(&private_start_marker).expect("method should exist"); + (start, private_start_marker.len()) + }; + let rest = &source[start + marker_len..]; + let end = [rest.find("\n pub async fn "), rest.find("\n async fn ")] + .into_iter() + .flatten() + .min() + .unwrap_or(rest.len()); &rest[..end] } #[test] fn bucket_create_and_delete_use_admitted_fresh_tasks() { let source = include_str!("bucket_usecase.rs"); - for (method, operation, inner_method) in [ - ("execute_create_bucket", "bucket creation", "execute_create_bucket_inner"), - ("execute_delete_bucket", "bucket deletion", "execute_delete_bucket_inner"), + for (method, operation, inner_call) in [ + ( + "execute_create_bucket", + "bucket creation", + "execute_create_bucket_inner(req, mutation_permit).await", + ), + ("execute_delete_bucket", "bucket deletion", "execute_delete_bucket_inner(req).await"), ] { let body = usecase_method_source(source, method); assert!( body.contains(&format!("await_bucket_usecase_on_fresh_task(\"{operation}\"")), "{method} must use bounded detached-task admission" ); - assert!( - body.contains(&format!("{inner_method}(req).await")), - "{method} must run its mutation inside the admitted task" - ); + assert!(body.contains(inner_call), "{method} must run its mutation inside the admitted task"); } + let create_body = usecase_method_source(source, "execute_create_bucket_inner"); + assert!( + create_body.contains("notify_bucket_metadata_reload_with_context("), + "a successful bucket creation must refresh peer metadata caches" + ); } #[test] diff --git a/rustfs/src/app/runtime_sources.rs b/rustfs/src/app/runtime_sources.rs index ad7c7f949..389d97517 100644 --- a/rustfs/src/app/runtime_sources.rs +++ b/rustfs/src/app/runtime_sources.rs @@ -19,7 +19,8 @@ use crate::runtime_sources as root_runtime_sources; pub(crate) use crate::runtime_sources::ServerContextSlot; pub(crate) use crate::runtime_sources::{ AppContext, current_encryption_service, current_endpoints_handle, current_notification_system, - current_object_data_cache_handle_for_context, current_object_store_handle_for_context, + current_notification_system_for_context, current_object_data_cache_handle_for_context, + current_object_store_handle_for_context, }; use rustfs_s3select_api::{QueryResult, server::dbms::DatabaseManagerSystem}; use s3s::dto::SelectObjectContentInput; diff --git a/rustfs/src/storage/rpc/node_service/bucket.rs b/rustfs/src/storage/rpc/node_service/bucket.rs index e7f8f82ae..40317d898 100644 --- a/rustfs/src/storage/rpc/node_service/bucket.rs +++ b/rustfs/src/storage/rpc/node_service/bucket.rs @@ -17,7 +17,8 @@ use crate::storage::storage_api::rpc_consumer::node_service::contract::bucket::{ BucketOptions, DeleteBucketOptions, MakeBucketOptions, }; use crate::storage::storage_api::rpc_consumer::node_service::{ - DiskError, StoragePeerS3ClientExt as _, decode_heal_bucket_rpc_options, reload_bucket_metadata, remove_bucket_metadata, + DiskError, StoragePeerS3ClientExt as _, decode_heal_bucket_rpc_options, reload_bucket_metadata, + remove_bucket_metadata_if_absent, }; use rustfs_protos::proto_gen::node_service::*; use tonic::{Request, Response, Status}; @@ -38,9 +39,16 @@ impl NodeService { })); } - // Drop the bucket's cached metadata on this peer so it stops serving - // stale configuration after the bucket has been deleted. - match remove_bucket_metadata(&bucket).await { + // Drop the bucket's cached metadata only if the namespace is still + // absent. A delayed delete notification must not erase metadata for a + // same-name bucket that has already been recreated. + let Some(store) = self.resolve_object_store() else { + return Ok(Response::new(DeleteBucketMetadataResponse { + success: false, + error_info: Some("errServerNotInitialized".to_string()), + })); + }; + match remove_bucket_metadata_if_absent(store, &bucket).await { Ok(_) => Ok(Response::new(DeleteBucketMetadataResponse { success: true, error_info: None, diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 67c0e586a..c0a9608d5 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -275,7 +275,7 @@ pub(crate) mod rpc_consumer { SCANNER_PUBLICATION_LEASE_TTL_MS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt, StoragePeerS3ClientExt, TierDailyStatsWire, UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, get_global_transition_state, get_local_server_property, reload_bucket_metadata, - reload_transition_tier_config, remove_bucket_metadata, validate_batch_read_version_item_count, + reload_transition_tier_config, remove_bucket_metadata_if_absent, validate_batch_read_version_item_count, }; pub(crate) type StorageResult = super::super::Result; @@ -2010,8 +2010,8 @@ pub(crate) async fn reload_bucket_metadata(api: Arc, bucket: &str) -> R ecstore_bucket::metadata_sys::reload_bucket_metadata(api, bucket).await } -pub(crate) async fn remove_bucket_metadata(bucket: &str) -> Result { - ecstore_bucket::metadata_sys::remove_bucket_metadata(bucket).await +pub(crate) async fn remove_bucket_metadata_if_absent(api: Arc, bucket: &str) -> Result { + ecstore_bucket::metadata_sys::remove_bucket_metadata_if_absent(api, bucket).await } pub(crate) async fn update_bucket_metadata_config( From 0195732a1fcb1daf9d90c70de68344c6bec4f6d5 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 12 Oct 2026 06:02:03 -0700 Subject: [PATCH 028/143] test(ecstore): cover delete cleanup crashes and retries (#8471) fix(ecstore): preserve retry evidence during delete cleanup Bind normal cleanup to the coordinator transaction after quorum succeeds, and remove the rollback backup before marked data. Preserve data and markers if backup cleanup fails. Add deterministic LocalDisk crash and retry regressions plus real four-disk single and batch coordinator coverage. This prevents one post-quorum cleanup window; it does not resolve ambiguous inline or historical residue in #8464. Co-authored-by: Zhengchao An Co-authored-by: Hauser Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/ecstore/src/disk/local.rs | 320 ++++++++++++++++++++++ crates/ecstore/src/set_disk/ops/object.rs | 82 ++++++ 2 files changed, 402 insertions(+) diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 8b53add0b..66cc3e10b 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -19043,6 +19043,326 @@ mod test { }] } + // This is a real process interruption of LocalDisk, not a server or + // power-loss test. The child stops at the rollback-backup unlink. + #[cfg(unix)] + #[tokio::test] + async fn quorum_delete_cleanup_process_crash_preserves_retry_evidence() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + use std::os::unix::process::ExitStatusExt; + + const CHILD_ROOT: &str = "RUSTFS_TEST_QUORUM_CLEANUP_ROOT"; + const CHILD_TRANSACTION: &str = "RUSTFS_TEST_QUORUM_CLEANUP_TRANSACTION"; + const CHILD_READY: &str = "RUSTFS_TEST_QUORUM_CLEANUP_READY"; + if let Some(root) = std::env::var_os(CHILD_ROOT) { + let endpoint = + Endpoint::try_from(Path::new(&root).to_str().expect("UTF-8 child disk path")).expect("parse child endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("open child disk"); + let transaction = + Uuid::parse_str(&std::env::var(CHILD_TRANSACTION).expect("child transaction")).expect("parse child transaction"); + let ready = std::env::var_os(CHILD_READY).expect("child readiness path"); + let rollback_path = format!("object/{transaction}"); + let backup = disk + .io_get_object_path("bucket", &rollback_path) + .expect("child transaction path") + .join(STORAGE_FORMAT_FILE_BACKUP); + let _hook = hooks::install_at(hooks::Stage::Remove, &backup, move || { + std::fs::write(ready, b"before-backup-unlink").expect("publish reached boundary"); + loop { + std::thread::park(); + } + }); + disk.delete( + "bucket", + &rollback_path, + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(transaction), + ..Default::default() + }, + ) + .await + .expect("child cleanup should reach its boundary"); + panic!("child unexpectedly returned without being killed"); + } + + let dir = tempfile::tempdir().expect("create test disk"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("parse endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("create local disk"); + ensure_test_volume(&disk, "bucket").await; + let residue = write_committed_delete_residue(&disk, "bucket", "object").await; + let original_backup = fs::read(residue.backup()).await.expect("read original backup"); + let original_part = fs::read(residue.part()).await.expect("read original part"); + + for attempt in 0..2 { + let ready = dir.path().join(format!("cleanup-ready-{attempt}")); + let mut child = std::process::Command::new(std::env::current_exe().expect("current test binary")) + .args([ + "--exact", + "disk::local::test::quorum_delete_cleanup_process_crash_preserves_retry_evidence", + "--nocapture", + ]) + .env(CHILD_ROOT, dir.path()) + .env(CHILD_TRANSACTION, residue.transaction.to_string()) + .env(CHILD_READY, &ready) + .spawn() + .expect("spawn this test's cleanup process"); + let reached = tokio::time::timeout(Duration::from_secs(30), async { + while !ready.exists() { + if child.try_wait().expect("check child status").is_some() { + return false; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + true + }) + .await; + // Child is an owned process handle; no process-name matching or + // external PID is accepted by this test. + if child.try_wait().expect("check owned child").is_none() { + child.kill().expect("SIGKILL the owned cleanup child"); + } + let status = child.wait().expect("reap cleanup child"); + assert!(matches!(reached, Ok(true)), "child must reach the exact cleanup boundary"); + assert_eq!(status.signal(), Some(9), "child must have been interrupted by SIGKILL"); + println!( + "cleanup crash {attempt}: backup={} marker={} part={}", + residue.backup().exists(), + residue.marker().exists(), + residue.part().exists() + ); + assert_eq!(fs::read(residue.backup()).await.expect("backup at paused unlink"), original_backup); + assert_eq!(fs::read(residue.part()).await.expect("payload survives each crash"), original_part); + assert!(residue.marker().is_file(), "transaction marker survives each crash"); + assert!( + residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE) + .is_file(), + "quorum receipt survives each crash" + ); + } + + let reopened = LocalDisk::new(&endpoint, false) + .await + .expect("reopen disk after process interruption"); + reopened + .delete( + "bucket", + &format!("object/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await + .expect("retry the already committed coordinator transaction"); + assert_eq!(original_part, b"stale"); + let bucket_result = reopened.delete_volume("bucket", false).await; + println!("non-force bucket deletion after retry: {bucket_result:?}"); + assert!( + !residue.object_dir.exists(), + "repeated delete after two crashes must reclaim this known-successful transaction" + ); + bucket_result.expect("non-force bucket deletion after retry"); + } + + // Normal post-quorum cleanup must preserve its data-dir marker until the + // rollback backup is gone, including across an interrupted retry. + #[cfg(not(windows))] + #[tokio::test] + async fn quorum_delete_cleanup_removes_backup_before_data_and_can_retry() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + + let dir = tempfile::tempdir().expect("create test disk"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("parse endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("create local disk"); + let bucket = "bucket"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let data_path = residue.part(); + let rollback_path = format!("{object}/{}", residue.transaction); + let opts = DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }; + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel::<()>(); + let _hook = hooks::install_at(hooks::Stage::Remove, &data_path, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let mut cleanup = Box::pin(disk.delete(bucket, &rollback_path, opts.clone())); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + entered = entered_rx => entered.expect("cleanup must reach data removal"), + result = cleanup.as_mut() => panic!("cleanup ended before data removal: {result:?}"), + } + }) + .await + .expect("cleanup reaches physical payload unlink"); + + let backup_gone = !residue.backup().exists(); + let marker_present = residue.marker().exists(); + let original_part = fs::read(residue.part()).await.expect("paused data must remain readable"); + drop(cleanup); + + // The in-flight unlink retains metadata ownership after cancellation. + // Retry must wait for that physical operation before removing proof. + let mut retry = Box::pin(disk.delete(bucket, &rollback_path, opts)); + assert!( + tokio::time::timeout(Duration::from_millis(100), retry.as_mut()) + .await + .is_err(), + "retry cannot bypass the parked cleanup's namespace ownership" + ); + assert!(residue.marker().is_file()); + assert!( + residue + .object_dir + .join(residue.transaction.to_string()) + .join(DELETE_QUORUM_COMMITTED_FILE) + .is_file() + ); + drop(release); + tokio::time::timeout(Duration::from_secs(10), retry) + .await + .expect("retry completes after the abandoned unlink releases ownership") + .expect("retry cleanup"); + + assert!( + backup_gone, + "post-quorum cleanup must remove the backup before touching its data-dir evidence" + ); + assert!(marker_present, "data removal must still have its transaction marker"); + assert_eq!(original_part, b"stale"); + assert!(!residue.object_dir.exists(), "interrupted cleanup and retry must leave no object residue"); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn quorum_delete_cleanup_backup_error_preserves_data_and_marker() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + + let dir = tempfile::tempdir().expect("create test disk"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("parse endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("create local disk"); + let bucket = "bucket"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let backup_bytes = fs::read(residue.backup()).await.expect("read rollback evidence"); + let backup = residue.backup(); + let held_backup = residue.object_dir.join("held-backup"); + let blocked_backup = backup.clone(); + let retained_backup = held_backup.clone(); + // Replace only the classified backup inode with a directory so the + // real unlink fails, while retaining its exact bytes for retry. + let _hook = hooks::install_at(hooks::Stage::Remove, &backup, move || { + std::fs::rename(&blocked_backup, retained_backup).expect("retain rollback bytes"); + std::fs::create_dir(blocked_backup).expect("block the actual backup unlink"); + }); + let rollback_path = format!("{object}/{}", residue.transaction); + let opts = DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }; + disk.delete(bucket, &rollback_path, opts.clone()) + .await + .expect_err("backup removal failure must be returned"); + + assert!(backup.is_dir(), "the native backup unlink must have encountered the blocker"); + assert_eq!(fs::read(&held_backup).await.expect("backup bytes must survive"), backup_bytes); + assert!(residue.marker().exists(), "backup failure must retain the data-dir transaction marker"); + assert_eq!(fs::read(residue.part()).await.expect("payload must survive backup failure"), b"stale"); + + fs::remove_dir(&backup).await.expect("remove the one-shot unlink blocker"); + fs::rename(&held_backup, &backup) + .await + .expect("restore the unchanged backup inode"); + assert_eq!(fs::read(&backup).await.expect("canonical rollback backup"), backup_bytes); + disk.delete(bucket, &rollback_path, opts) + .await + .expect("retry cleanup after one-shot unlink failure"); + assert!(!residue.object_dir.exists(), "successful retry must reclaim the transaction"); + } + + #[tokio::test] + async fn quorum_delete_cleanup_preserves_other_generations_transactions_and_children() { + let dir = tempfile::tempdir().expect("create test disk"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("parse endpoint"); + let disk = LocalDisk::new(&endpoint, false).await.expect("create local disk"); + let bucket = "bucket"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + let residue = write_committed_delete_residue(&disk, bucket, object).await; + let newer_data_dir = Uuid::new_v4(); + let newer_data = residue.object_dir.join(newer_data_dir.to_string()); + fs::create_dir_all(&newer_data).await.expect("create newer data directory"); + fs::write(newer_data.join("part.1"), b"newer payload") + .await + .expect("write newer payload"); + let metadata = test_meta(test_file_info(object, Uuid::new_v4(), Some(newer_data_dir), None)); + fs::write(residue.object_dir.join(STORAGE_FORMAT_FILE), &metadata) + .await + .expect("publish newer metadata before old transaction cleanup"); + let unrelated = Uuid::new_v4(); + let unrelated_backup = residue + .object_dir + .join(unrelated.to_string()) + .join(STORAGE_FORMAT_FILE_BACKUP); + fs::create_dir_all(unrelated_backup.parent().expect("backup parent")) + .await + .expect("create unrelated transaction"); + fs::write(&unrelated_backup, b"corrupt unproven backup") + .await + .expect("write ambiguous backup"); + let mismatched_marker = newer_data.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{unrelated}")); + fs::write(&mismatched_marker, []).await.expect("write unrelated marker"); + let child = residue.object_dir.join("child").join(STORAGE_FORMAT_FILE); + fs::create_dir_all(child.parent().expect("child parent")) + .await + .expect("create descendant"); + fs::write(&child, b"child metadata").await.expect("write descendant metadata"); + + disk.delete( + bucket, + &format!("{object}/{}", residue.transaction), + DeleteOptions { + recursive: true, + immediate: true, + old_data_dir: Some(residue.transaction), + ..Default::default() + }, + ) + .await + .expect("clean only the successful transaction"); + + assert!(!residue.backup().exists() && !residue.part().exists() && !residue.marker().exists()); + assert_eq!( + fs::read(residue.object_dir.join(STORAGE_FORMAT_FILE)) + .await + .expect("read newer metadata"), + metadata + ); + assert_eq!(fs::read(newer_data.join("part.1")).await.expect("read newer payload"), b"newer payload"); + assert_eq!( + fs::read(&unrelated_backup).await.expect("read unproven backup"), + b"corrupt unproven backup" + ); + assert!(mismatched_marker.exists()); + assert_eq!(fs::read(child).await.expect("read descendant"), b"child metadata"); + } + // Internal maintenance retains its legacy cleanup behavior while user // buckets require the separate set-wide authorization boundary. #[tokio::test] diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index fba95e548..d825a892c 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -23651,6 +23651,88 @@ mod single_delete_namespace_owner_tests { .expect("seed a complete real object version"); } + #[cfg(not(windows))] + #[tokio::test] + async fn quorum_delete_cleanup_coordinator_binds_single_and_batch() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + + for batch in [false, true] { + let (_dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "quorum-cleanup-order"; + let object = "object"; + let version = Uuid::new_v4(); + for disk in &disks { + disk.make_volume(bucket).await.expect("create fixture bucket"); + } + seed_version(&set, bucket, object, version, &vec![0x51; 256 * 1024]).await; + let (observed, observations) = std::sync::mpsc::channel(); + let mut hooks = Vec::new(); + for disk in &disks { + let fi = disk + .read_version("", bucket, object, &version.to_string(), &ReadOptions::default()) + .await + .expect("read seeded version"); + assert!(!fi.inline_data(), "fixture must reach physical data cleanup"); + let crate::disk::Disk::Local(local) = disk.as_ref() else { + panic!("local fixture required"); + }; + let object_dir = local.get_disk().get_object_path_for_io(bucket, object).expect("object path"); + let data_dir = object_dir.join(fi.data_dir.expect("seeded data directory").to_string()); + let observed = observed.clone(); + hooks.push(hooks::install_at(hooks::Stage::Remove, &data_dir.join("part.1"), move || { + let backup_present = std::fs::read_dir(&object_dir) + .expect("object exists before payload unlink") + .any(|entry| { + entry + .expect("read transaction entry") + .path() + .join(crate::disk::STORAGE_FORMAT_FILE_BACKUP) + .exists() + }); + observed.send(backup_present).expect("record actual cleanup order"); + })); + } + if batch { + let (_, errors) = set + .delete_objects( + bucket, + vec![ObjectToDelete { + object_name: object.to_string(), + version_id: Some(version), + ..Default::default() + }], + ObjectOptions { + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await; + assert!(errors.iter().all(Option::is_none), "batch delete must commit: {errors:?}"); + } else { + set.delete_object_version( + bucket, + object, + &FileInfo { + name: object.to_string(), + version_id: Some(version), + ..Default::default() + }, + false, + ) + .await + .expect("single delete must commit"); + } + let observations = observations.try_iter().collect::>(); + assert_eq!(observations.len(), 4, "all four disks must reach cleanup: batch={batch}"); + assert!( + observations.iter().all(|backup_present| !backup_present), + "the coordinator must bind post-quorum cleanup before any payload unlink: batch={batch}" + ); + } + } + #[tokio::test] #[serial_test::serial] async fn internal_metadata_mutations_do_not_reenter_pool_metadata() { From a7188a16b4c4f1ea6f035784385f22e8bdd566ef Mon Sep 17 00:00:00 2001 From: Hauser Date: Mon, 12 Oct 2026 21:17:58 +0800 Subject: [PATCH 029/143] test(storage): make recovery and upgrade checks deterministic (#8491) * test(upgrade): separate rc.5 data and candidate query checks Verify historical fixture bytes before upgrading without requiring the pinned rc.5 reader to satisfy the candidate's cross-part range contract. Reuse the complete-body oracle and retain all upgraded range, partNumber, attributes and replication assertions. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(ecstore): replace timing races with recovery evidence Prove no_lock batch progress with the existing cleanup barrier while the caller retains its guard. Check namespace counters and every drive's metadata absence rather than using a one-second disk I/O deadline. Keep durable ILM's manual recovery as the sole tier journal consumer and verify exact migrated input before retaining the original recovery stats, receipts and remote-version assertions. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- .../src/upgrade_compatibility_test.rs | 14 +++++++---- crates/ecstore/src/set_disk/ops/object.rs | 11 ++++++++- crates/ecstore/src/store/init.rs | 24 +++++++++++++++++-- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/crates/e2e_test/src/upgrade_compatibility_test.rs b/crates/e2e_test/src/upgrade_compatibility_test.rs index 51ff93d6f..35e0c8eaf 100644 --- a/crates/e2e_test/src/upgrade_compatibility_test.rs +++ b/crates/e2e_test/src/upgrade_compatibility_test.rs @@ -2345,7 +2345,7 @@ fn layout_reported_parts(attributes: &aws_sdk_s3::operation::get_object_attribut attributes.object_parts().map(|parts| parts.parts().len()) } -async fn assert_layout_readable(client: &Client, case: &LayoutCase, context: &str) -> TestResult { +async fn assert_layout_full_body(client: &Client, case: &LayoutCase, context: &str) -> TestResult { let label = case.label(); let head = layout_head(client, case).await?; assert_eq!( @@ -2362,7 +2362,12 @@ async fn assert_layout_readable(client: &Client, case: &LayoutCase, context: &st let full = layout_get(client, case, None, None).await?.body.collect().await?.into_bytes(); assert_eq!(full.len(), case.body.len(), "{context}: {label}: full GET length"); assert!(full == case.body, "{context}: {label}: full GET body must equal the rc.5 upload"); + Ok(()) +} +async fn assert_layout_readable(client: &Client, case: &LayoutCase, context: &str) -> TestResult { + assert_layout_full_body(client, case, context).await?; + let label = case.label(); if case.is_multipart_layout() { let first = case.part_sizes[0]; let range = format!("bytes={}-{}", first - 32, first + 31); @@ -2641,10 +2646,11 @@ async fn direct_upgrade_from_rc5_preserves_multipart_layouts() -> TestResult { "rc.5 wrote a legacy layout" ); } - // The rc.5 writer must itself still read what it wrote, so a later - // failure is attributable to the upgrade rather than to the fixture. + // Prove the rc.5 writer persisted every original byte before upgrading. + // Its compressed cross-part range reader has a legacy limitation; the + // upgraded reader must still satisfy every range and partNumber assertion. for case in &cases { - assert_layout_readable(&old_client, case, "rc.5").await?; + assert_layout_full_body(&old_client, case, "rc.5").await?; } // Upgrade in place. diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index d825a892c..6f5ae956e 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -22986,7 +22986,7 @@ mod delete_objects_lock_gating_tests { } put_plain_object(&set_disks, bucket, "obj-a").await; - let _outer_guard = set_disks + let outer_guard = set_disks .new_ns_lock(bucket, "obj-a") .await .expect("namespace lock should be created") @@ -23039,6 +23039,15 @@ mod delete_objects_lock_gating_tests { assert!(errs[0].is_none(), "no_lock batch delete should not fail with a lock error: {:?}", errs[0]); assert!(deleted[0].found, "existing object must still be deleted"); + for disk in &disk_stores { + assert_eq!( + disk.read_version("", bucket, "obj-a", "", &crate::disk::ReadOptions::default()) + .await + .expect_err("every drive must confirm the deleted version is absent"), + DiskError::FileNotFound + ); + } + drop(outer_guard); } } diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 64656bcb4..86cf16be7 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -9594,8 +9594,18 @@ mod tests { #[cfg(feature = "test-util")] async fn decommission_migrates_and_verifies_registered_durable_ilm_records_scenario() { let temp_dir = tempfile::tempdir().expect("create temp store dir"); - let (ctx, store, _shutdown) = - without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-decommission", &[4, 4])).await; + // The explicit recovery below must be the only journal consumer, + // including when receipt validation outlasts the background interval. + let mut instance_ctx = crate::runtime::instance::InstanceContext::new(); + instance_ctx.suppress_tier_delete_journal_recovery_for_test(); + let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( + temp_dir.path(), + "durable-ilm-decommission", + &[(1, 4), (1, 4)], + CancellationToken::new(), + Some(Arc::new(instance_ctx)), + )) + .await; let tier_name = "DECOMMISSION-ILM"; let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await; @@ -10067,6 +10077,16 @@ mod tests { .await .expect("manual task receipt should restore after the corruption check"); + assert!(matches!( + com::read_config(store.pools[0].clone(), &tier_path).await, + Err(Error::ConfigNotFound) + )); + assert_eq!( + com::read_config(store.pools[1].clone(), &tier_path) + .await + .expect("manual recovery must still observe the exact migrated tier journal"), + tier_bytes + ); let tier_stats = recover_tier_delete_journal_entries(store.clone(), 100, None) .await .expect("tier journal recovery should consume the migrated record before completion"); From ac6cf8a50df6392269404ad7d2bace64a80c0439 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 12 Oct 2026 08:01:37 -0700 Subject: [PATCH 030/143] fix(test): discover data dirs by listing instead of info.data_dir (#8495) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The conditional_get_304_uses_metadata_without_decoding_damaged_data test constructed the data directory path from info.data_dir.to_string(), but the on-disk directory name may not match the UUID string form. Switch to iterating subdirectories of the object path (matching the delete_object_part_shards helper) so the test is robust regardless of the data_dir naming convention. Verified: cargo nextest run -p rustfs -- conditional_get_304 → PASS --- rustfs/src/app/object/get.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs index edbc2e03b..b26b72103 100644 --- a/rustfs/src/app/object/get.rs +++ b/rustfs/src/app/object/get.rs @@ -5893,8 +5893,18 @@ mod tests { *metadata.last_mut().expect("inline data bytes") ^= 1; tokio::fs::write(&meta_path, metadata).await.expect("damage inline payload"); } else { - let data_path = object_path.join(info.data_dir.expect("sharded object data directory").to_string()); - tokio::fs::remove_dir_all(data_path).await.expect("remove object data shards"); + // Discover version data dirs by listing the object directory + // (matching delete_object_part_shards) instead of constructing + // the path from info.data_dir, whose UUID string form may not + // match the on-disk directory name. + let mut entries = tokio::fs::read_dir(&object_path).await.expect("read object dir"); + while let Some(entry) = entries.next_entry().await.expect("read object dir entry") { + if entry.file_type().await.expect("entry file type").is_dir() { + tokio::fs::remove_dir_all(entry.path()) + .await + .expect("remove object data shards"); + } + } } } let resolved = store From d6fe5c30ce722d7f3773984b7d0f1912d63e30ae Mon Sep 17 00:00:00 2001 From: Hauser Date: Mon, 12 Oct 2026 23:02:02 +0800 Subject: [PATCH 031/143] test(e2e): borrow URLs in internode RPC helpers (#8497) Pass existing string slices directly to the client constructor to satisfy strict Clippy without temporary allocations. Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/e2e_test/src/internode_rpc_signature_e2e_test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs b/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs index a311aa76e..dcc911e77 100644 --- a/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs +++ b/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs @@ -264,7 +264,7 @@ async fn call_make_volume_response( request: MakeVolumeRequest, headers: HeaderMap, ) -> Result, Status> { - let mut client = node_service_time_out_client_no_auth(&url.to_string()) + let mut client = node_service_time_out_client_no_auth(url) .await .map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?; let mut rpc_request = Request::new(request); @@ -273,7 +273,7 @@ async fn call_make_volume_response( } async fn call_ping_response(url: &str, headers: HeaderMap) -> Result, Status> { - let mut client = node_service_time_out_client_no_auth(&url.to_string()) + let mut client = node_service_time_out_client_no_auth(url) .await .map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?; let mut rpc_request = Request::new(PingRequest { From 5d8e143505205d2f21f6f32a33f6aee9f29482b0 Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Mon, 12 Oct 2026 23:07:46 +0800 Subject: [PATCH 032/143] fix(admin): add OIDC policy log fields (#8496) --- rustfs/src/admin/service/federated_identity.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/rustfs/src/admin/service/federated_identity.rs b/rustfs/src/admin/service/federated_identity.rs index 7417a7322..cdc197633 100644 --- a/rustfs/src/admin/service/federated_identity.rs +++ b/rustfs/src/admin/service/federated_identity.rs @@ -30,6 +30,10 @@ use std::collections::{BTreeMap, HashMap}; use time::{Duration, OffsetDateTime}; use tracing::{debug, warn}; +const LOG_COMPONENT_ADMIN: &str = "admin"; +const LOG_SUBSYSTEM_OIDC: &str = "oidc"; +const EVENT_ADMIN_OIDC_POLICY: &str = "admin_oidc_policy"; + pub(crate) struct DefaultFederatedSessionBinding; // Legacy receivers reject an empty parsed mapping; current receivers ignore it when the virtual-parent marker is present. @@ -339,6 +343,9 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { for policy_name in &selected_policy_names { let exists = iam_store.policy_exists(policy_name).await.map_err(|err| { warn!( + event = EVENT_ADMIN_OIDC_POLICY, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, provider_id = %authorization.provider_id, parent_user = %parent_user, policy = %policy_name, @@ -367,6 +374,9 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { .map_err(|rejection| match rejection { OidcPolicyRejection::Unsupported(policy_names) => { warn!( + event = EVENT_ADMIN_OIDC_POLICY, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, provider_id = %authorization.provider_id, parent_user = %parent_user, unsupported_policies = ?policy_names, @@ -376,6 +386,9 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { } OidcPolicyRejection::Unresolved(policy_names) => { debug!( + event = EVENT_ADMIN_OIDC_POLICY, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, provider_id = %authorization.provider_id, parent_user = %parent_user, unresolved_policies = ?policy_names, @@ -389,6 +402,9 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { })?; if !selection.ignored.is_empty() { debug!( + event = EVENT_ADMIN_OIDC_POLICY, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, provider_id = %authorization.provider_id, parent_user = %parent_user, ignored_group_policies = ?selection.ignored, From a57a3ae51de8bab2b0e3e49e5ea27c77b4d87d64 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 00:50:59 +0800 Subject: [PATCH 033/143] fix(scanner): reuse observations and expose complete usage (#8498) * fix(scanner): reuse observations and expose complete usage Allow superseded retries to reach the existing validated cache planner while bootstrap remains pending. Serve complete observations in server info without changing quota authority or dirty acknowledgements. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(scanner): route fixture setup through storage boundary Keep the single-disk scanner fixture on the existing storage_api boundary instead of importing the ECStore API facade directly. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- .../src/diagnostics/admin_server_info.rs | 93 ++++++++++++- crates/scanner/src/scanner.rs | 27 ++-- crates/scanner/src/scanner/tests.rs | 125 +++++++++++++++--- crates/scanner/src/storage_api.rs | 1 + docs/operations/scanner-runtime-controls.md | 4 +- 5 files changed, 213 insertions(+), 37 deletions(-) diff --git a/crates/ecstore/src/diagnostics/admin_server_info.rs b/crates/ecstore/src/diagnostics/admin_server_info.rs index 3bbf7cf05..def38f38b 100644 --- a/crates/ecstore/src/diagnostics/admin_server_info.rs +++ b/crates/ecstore/src/diagnostics/admin_server_info.rs @@ -15,7 +15,9 @@ use crate::cluster::rpc::{ ScannerBucketListing, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, }; -use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend_cached}; +use crate::data_usage::{ + DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_admin_data_usage_from_backend_cached as load_server_info_usage_snapshot, +}; use crate::error::{Error, Result}; use crate::{ disk::endpoint::{Endpoint, EndpointType}, @@ -294,7 +296,7 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage { if let Some(store) = runtime_sources::object_store_handle() { mode = ITEM_ONLINE; apply_data_usage_result( - load_data_usage_from_backend_cached(store.clone()).await, + load_server_info_usage_snapshot(store.clone()).await, &mut buckets, &mut objects, &mut versions, @@ -967,6 +969,93 @@ mod tests { assert_eq!(usage.size, 6); } + #[tokio::test] + #[serial] + async fn server_info_reads_complete_observation_without_promoting_authority() { + use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks; + use crate::config::com::{read_config, save_config}; + use crate::data_usage::{ + DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH, invalidate_data_usage_snapshot_cache, + load_data_usage_from_backend_cached, + }; + use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo}; + use std::time::{Duration, SystemTime}; + + let (_dirs, store) = isolated_store_over_temp_disks().await; + let primary = DataUsageInfo { + last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)), + scanner_epoch: Some(1), + usage_snapshot_bootstrap_pending: true, + usage_snapshot_converged: Some(false), + ..Default::default() + }; + let observed = DataUsageInfo { + last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(2)), + scanner_epoch: Some(1), + scanner_cycle: Some(1), + buckets_count: 1, + objects_total_count: 3, + versions_total_count: 4, + delete_markers_total_count: 1, + objects_total_size: 6, + buckets_usage: [( + "bucket".to_string(), + BucketUsageInfo { + objects_count: 3, + versions_count: 4, + delete_markers_count: 1, + size: 6, + ..Default::default() + }, + )] + .into_iter() + .collect(), + usage_snapshot_complete: true, + usage_snapshot_converged: Some(false), + usage_snapshot_authoritative_baseline: Some(primary.snapshot_identity()), + ..Default::default() + }; + let primary_bytes = serde_json::to_vec(&primary).expect("bootstrap should encode"); + save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), primary_bytes.clone()) + .await + .expect("bootstrap should persist"); + save_config( + store.clone(), + DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), + serde_json::to_vec(&observed).expect("observation should encode"), + ) + .await + .expect("observation should persist"); + invalidate_data_usage_snapshot_cache().await; + + let displayed = super::load_server_info_usage_snapshot(store.clone()) + .await + .expect("server info should load the complete observation"); + assert_eq!(displayed.usage_snapshot_converged, Some(false)); + let mut buckets = rustfs_madmin::Buckets::default(); + let mut objects = rustfs_madmin::Objects::default(); + let mut versions = rustfs_madmin::Versions::default(); + let mut delete_markers = rustfs_madmin::DeleteMarkers::default(); + let mut usage = rustfs_madmin::Usage::default(); + apply_data_usage_result(Ok(displayed), &mut buckets, &mut objects, &mut versions, &mut delete_markers, &mut usage); + assert!(objects.error.is_none()); + assert_eq!( + (buckets.count, objects.count, versions.count, delete_markers.count, usage.size), + (1, 3, 4, 1, 6) + ); + let quota = load_data_usage_from_backend_cached(store.clone()) + .await + .expect("quota should retain its bootstrap state"); + assert!(!quota.is_complete_bucket_usage_snapshot()); + assert_eq!( + read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("primary should remain readable"), + primary_bytes + ); + invalidate_data_usage_snapshot_cache().await; + } + #[test] fn incomplete_data_usage_is_unavailable_in_server_info() { let mut buckets = rustfs_madmin::Buckets::default(); diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 3624be128..645485e30 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -1157,10 +1157,17 @@ impl ScannerMaintenanceFeatures { self.lifecycle || self.replication || self.inspection_failed } - fn requires_full_scan(self, observed_generation: Option, current_generation: u64, wake: ScannerCycleWakeReason) -> bool { + fn requires_full_scan( + self, + observed_generation: Option, + current_generation: u64, + wake: ScannerCycleWakeReason, + superseded_retry: bool, + ) -> bool { self.needs_regular_cycle() || observed_generation != Some(current_generation) - || !matches!(wake, ScannerCycleWakeReason::DirtyUsage | ScannerCycleWakeReason::ClusterActivity) + || !(matches!(wake, ScannerCycleWakeReason::DirtyUsage | ScannerCycleWakeReason::ClusterActivity) + || (wake == ScannerCycleWakeReason::Timer && superseded_retry)) } } @@ -1191,16 +1198,6 @@ impl ScannerUsageBootstrapRebuild { enabled && !self.pending } - fn requires_full_scan( - self, - maintenance_features: ScannerMaintenanceFeatures, - observed_generation: Option, - current_generation: u64, - wake: ScannerCycleWakeReason, - ) -> bool { - self.pending || maintenance_features.requires_full_scan(observed_generation, current_generation, wake) - } - fn record_cycle(&mut self, outcome: ScannerCycleOutcome) -> bool { if matches!( outcome, @@ -3362,11 +3359,13 @@ where leader_epoch, cycle_budget.clone(), ScannerCycleScheduling { - requires_full_scan: usage_bootstrap_rebuild.requires_full_scan( - maintenance_features, + // Cache reuse is admitted by the persisted coverage proof in ScannerIO. + // Bootstrap remains pending until authoritative publication succeeds. + requires_full_scan: maintenance_features.requires_full_scan( maintenance_generation_seen, scanner_maintenance_generation(), wake_reason, + superseded_retry_interval.is_some(), ), service_cohort: Some(service_cohort.clone()), }, diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 4c3c9879c..b43fc1eb2 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -138,6 +138,11 @@ pub(super) async fn setup_scanner_cycle_store_at_path_with_layout_and_disk_prein let endpoint_pools = EndpointServerPools::from(pools); let instance_ctx = Arc::new(InstanceContext::new()); instance_ctx.set_endpoints(endpoint_pools.clone()); + if pool_count == 1 && sets_per_pool == 1 && drives_per_set == 1 { + instance_ctx + .update_erasure_type(crate::storage_api::EcstoreSetupType::ErasureSD) + .await; + } if preinitialize_disks { init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) .await @@ -1462,8 +1467,19 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() { #[tokio::test] #[serial] async fn coordinator_persists_only_an_observation_during_pending_put() { + for drives_per_set in [1, 4] { + assert_bootstrap_observation_retry(drives_per_set).await; + } +} + +async fn assert_bootstrap_observation_retry(drives_per_set: usize) { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - let (_temp_dir, store) = setup_scanner_cycle_store().await; + let temp_dir = tempfile::tempdir().expect("bootstrap directory should be created"); + let store = setup_scanner_cycle_store_at_path_with_layout(temp_dir.path(), false, 1, 1, drives_per_set).await; + initialize_usage_baseline_bootstrap(store.clone()) + .await + .expect("cold startup should persist its bootstrap identity"); + let mut bootstrap = ScannerUsageBootstrapRebuild::from_startup(PersistedUsageFloorStartup::BootstrapPending); let mut pause_backlog = ScannerPauseBacklogController::claim(store.clone(), scanner_pause_backlog_now()) .await .expect("scanner pause backlog should be available"); @@ -1488,6 +1504,25 @@ async fn coordinator_persists_only_an_observation_during_pending_put() { .await .expect("fixture object should finish its rename fanout"); } + let cold_bucket = "scanner-coordinator-cold"; + store + .make_bucket(cold_bucket, &crate::storage_api::scan::MakeBucketOptions::default()) + .await + .expect("cold bucket should be created"); + let mut cold_reader = PutObjReader::from_vec(b"cold".to_vec()); + store.pools[0].disk_set[0] + .put_object( + cold_bucket, + "object", + &mut cold_reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("cold object should persist before scanning"); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); crate::scanner_io::record_dirty_usage_bucket(&bucket); let dirty_before = crate::scanner_io::dirty_usage_buckets_for_tests(); let baseline = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) @@ -1530,7 +1565,7 @@ async fn coordinator_persists_only_an_observation_during_pending_put() { ) .await .expect("the coordinator must finish its namespace walk while a PUT is pending"); - assert_eq!(budget.progress().0, 1, "the coordinator must reach actual object traversal"); + assert_eq!(budget.progress().0, 2, "the coordinator must reach actual object traversal"); assert!( matches!( outcome, @@ -1560,6 +1595,16 @@ async fn coordinator_persists_only_an_observation_during_pending_put() { .expect("the pending cycle should persist its non-authoritative observation"); let observed: DataUsageInfo = serde_json::from_slice(&observed).expect("observed usage should decode"); assert_eq!(observed.usage_snapshot_converged, Some(false)); + assert!(observed.is_complete_bucket_usage_snapshot()); + assert_eq!(observed.objects_total_count, 2); + assert!(!bootstrap.record_cycle(outcome)); + assert!(bootstrap.pending(), "an observation cannot discharge authority bootstrap"); + crate::storage_api::ecstore_invalidate_data_usage_snapshot_cache().await; + let displayed = crate::storage_api::scanner_io::load_admin_data_usage_from_backend_cached(store.clone()) + .await + .expect("admin should load the durable complete observation"); + assert_eq!(displayed.objects_total_count, 2); + assert_eq!(displayed.usage_snapshot_converged, Some(false)); let committed_body = b"committed-after-walk"; let mut reader = PutObjReader::from_vec(committed_body.to_vec()); @@ -1592,7 +1637,12 @@ async fn coordinator_persists_only_an_observation_during_pending_put() { 1, Arc::clone(&retry_budget), ScannerCycleScheduling { - requires_full_scan: true, + requires_full_scan: ScannerMaintenanceFeatures::default().requires_full_scan( + Some(scanner_maintenance_generation()), + scanner_maintenance_generation(), + ScannerCycleWakeReason::Timer, + true, + ), service_cohort: None, }, ), @@ -1601,13 +1651,15 @@ async fn coordinator_persists_only_an_observation_during_pending_put() { .expect("the same cycle must converge after the pending PUT drains"); assert_eq!( retry_budget.progress().0, - 1, - "the same-cycle retry must not reuse the pre-tail bucket cache" + if drives_per_set == 1 { 1 } else { 2 }, + "Normal retries reuse cold inventory; Deep retries still visit every bucket" ); assert!(matches!( outcome, ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance )); + assert!(bootstrap.record_cycle(outcome)); + assert!(!bootstrap.pending()); assert_eq!(cycle_info.next, retry_cycle + 1); assert!(!crate::scanner_io::dirty_usage_buckets_for_tests().contains_key(&bucket)); let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) @@ -1616,10 +1668,10 @@ async fn coordinator_persists_only_an_observation_during_pending_put() { let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("the persisted usage should decode"); assert_eq!(usage.usage_snapshot_converged, Some(true)); assert_eq!(usage.scanner_cycle, Some(retry_cycle)); - assert_eq!(usage.objects_total_count, 1); + assert_eq!(usage.objects_total_count, 2); assert_eq!( usage.objects_total_size, - u64::try_from(committed_body.len()).expect("fixture body length") + u64::try_from(committed_body.len() + 4).expect("fixture total body length") ); let bucket_usage = usage .buckets_usage @@ -8995,18 +9047,10 @@ fn usage_bootstrap_rebuild_bypasses_clean_idle_until_authoritative_cycle() { cycle_interval: Duration::from_secs(60), ..Default::default() }; - let features = ScannerMaintenanceFeatures::default(); - let generation = Some(scanner_maintenance_generation()); let mut clean_idle_backoff = ScannerCleanIdleBackoff { interval_multiplier: 8 }; let mut rebuild = ScannerUsageBootstrapRebuild::from_startup(PersistedUsageFloorStartup::BootstrapPending); assert!(!rebuild.clean_idle_backoff_enabled(true)); - assert!(rebuild.requires_full_scan( - features, - generation, - scanner_maintenance_generation(), - ScannerCycleWakeReason::DirtyUsage, - )); let plan = rebuild.wait_plan(scanner_cycle_wait_plan(&config, clean_idle_backoff, true, std::convert::identity), None); assert_eq!(plan.delay, Duration::ZERO); @@ -9040,6 +9084,40 @@ fn usage_bootstrap_rebuild_bypasses_clean_idle_until_authoritative_cycle() { ); } +#[test] +fn superseded_retry_preserves_full_maintenance_boundaries() { + let features = ScannerMaintenanceFeatures::default(); + for (wake, ordinary_full, retry_full) in [ + (ScannerCycleWakeReason::Timer, true, false), + (ScannerCycleWakeReason::DirtyUsage, false, false), + (ScannerCycleWakeReason::ClusterActivity, false, false), + (ScannerCycleWakeReason::ClusterMaintenance, true, true), + (ScannerCycleWakeReason::MovementGeneration, true, true), + (ScannerCycleWakeReason::Recovery, true, true), + ] { + assert_eq!(features.requires_full_scan(Some(7), 7, wake, false), ordinary_full, "{wake:?}"); + assert_eq!(features.requires_full_scan(Some(7), 7, wake, true), retry_full, "{wake:?} retry"); + assert!(features.requires_full_scan(None, 7, wake, true)); + assert!(features.requires_full_scan(Some(6), 7, wake, true)); + for unsafe_features in [ + ScannerMaintenanceFeatures { + lifecycle: true, + ..features + }, + ScannerMaintenanceFeatures { + replication: true, + ..features + }, + ScannerMaintenanceFeatures { + inspection_failed: true, + ..features + }, + ] { + assert!(unsafe_features.requires_full_scan(Some(7), 7, wake, true)); + } + } +} + #[tokio::test] #[serial] async fn scoped_scan_explicit_bitrot_keeps_dirty_planning_without_idle_backoff() { @@ -9068,12 +9146,12 @@ async fn scoped_scan_explicit_bitrot_keeps_dirty_planning_without_idle_backoff() (ScannerCycleWakeReason::ClusterMaintenance, true), ] { assert_eq!( - features.requires_full_scan(generation, scanner_maintenance_generation(), wake), + features.requires_full_scan(generation, scanner_maintenance_generation(), wake, false), full, "{wake:?}" ); } - assert!(features.requires_full_scan(None, scanner_maintenance_generation(), ScannerCycleWakeReason::DirtyUsage)); + assert!(features.requires_full_scan(None, scanner_maintenance_generation(), ScannerCycleWakeReason::DirtyUsage, false)); for unsafe_features in [ ScannerMaintenanceFeatures { lifecycle: true, @@ -9091,18 +9169,25 @@ async fn scoped_scan_explicit_bitrot_keeps_dirty_planning_without_idle_backoff() assert!(unsafe_features.requires_full_scan( generation, scanner_maintenance_generation(), - ScannerCycleWakeReason::DirtyUsage + ScannerCycleWakeReason::DirtyUsage, + false )); } crate::scanner_io::record_scanner_maintenance_change("maintenance-proof-change"); - assert!(features.requires_full_scan(generation, scanner_maintenance_generation(), ScannerCycleWakeReason::DirtyUsage)); + assert!(features.requires_full_scan( + generation, + scanner_maintenance_generation(), + ScannerCycleWakeReason::DirtyUsage, + false + )); let (refreshed, refreshed_generation) = detect_stable_scanner_maintenance_features(&ctx, &store) .await .expect("changed maintenance generation should be inspected"); assert!(!refreshed.requires_full_scan( Some(refreshed_generation), scanner_maintenance_generation(), - ScannerCycleWakeReason::DirtyUsage + ScannerCycleWakeReason::DirtyUsage, + false )); crate::scanner_io::clear_dirty_usage_buckets_for_tests(); }) diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index c0ff85a96..ad88e6ff0 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -101,6 +101,7 @@ pub(crate) use rustfs_ecstore::api::event::{EventArgs as EcstoreEventArgs, send_ #[cfg(test)] pub(crate) use rustfs_ecstore::api::layout::{ EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints, + SetupType as EcstoreSetupType, }; pub(crate) use rustfs_ecstore::api::notification::NotificationSys as EcstoreNotificationSys; pub(crate) use rustfs_ecstore::api::notification::scanner_peer_transport_error_message_is_retryable; diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index 33a182860..8ca95fafb 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -96,6 +96,8 @@ When a finite deadline expires, RustFS cancels cooperative scanner work and wait An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle cadence: dirty-usage notifications do not bypass that configured interval. The default adaptive policy continues to use dirty-usage notifications to wake the scanner between timer-driven cycles. +Under continuous writes, a structurally complete walk can finish as `superseded`. Its durable observation remains separate from quota authority and leaves dirty work pending. Superseded retries may reuse that complete inventory to refresh dirty buckets, even while authority bootstrap is pending; missing or invalid coverage still forces a full walk. Ordinary maintenance timers, Deep scans, active lifecycle/replication and changed maintenance configuration retain full scans. Server info and datausageinfo can display complete observations, while Last Completed Scan advances only after an authoritative completed cycle. A write-free window is therefore not required to display observed counts, but arbitrary continuous writes do not guarantee authoritative completion. Historically dirty buckets remain eligible for rewalk, so narrower retries do not guarantee shorter cycles. + ## Single-disk clean-idle scheduling An erasure single-disk deployment using the built-in cycle and bitrot defaults automatically backs off repeated clean idle scans instead of walking the same unchanged namespace every minute. Each successful timer-driven cycle that finds no dirty usage or unresolved maintenance work doubles the next interval. The status endpoint reports the effective interval and multiplier. @@ -227,7 +229,7 @@ When a running scanner leader already holds `leader.lock`, submit a durable asyn The async form returns HTTP 202 when a new or replayable intent is accepted. Reusing the same `idempotency_key` is safe for retries of the same operator action; a different payload for the same key is rejected as a conflict. Poll `GET /v3/scanner/usage-state/recovery-intents/{intent_id}` until the intent reaches a terminal state. -A successful reset leaves usage state in `bootstrap-pending` only as a fenced rebuild marker. With the scanner enabled, the leader must treat that marker as pending full-rebuild work: clean-idle backoff must not extend the wait, an empty pause-backlog catch-up ledger must not rate-limit the rebuild, and the next admitted scanner cycle must run as a full scan until an authoritative usage snapshot replaces the marker. +A successful reset leaves usage state in `bootstrap-pending` only as a fenced rebuild marker. With the scanner enabled, the leader must treat that marker as pending full-rebuild work: clean-idle backoff must not extend the wait, an empty pause-backlog catch-up ledger must not rate-limit the rebuild, and the next admitted scanner cycle must rebuild full inventory. After a complete observation is durably saved, superseded retries may refresh dirty buckets using its validated coverage; bootstrap remains pending until authoritative publication replaces the marker. ## Cleanup With The Scanner Disabled From 0e0f656ae1d606c451354578e5b98e243010efc5 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 01:59:46 +0800 Subject: [PATCH 034/143] test: isolate license and rebalance cancellation fixtures (#8500) Preserve canonical claim encoding for the signature tamper test and pause the admin cancellation fixture at its existing run fence instead of durable deletion. Co-authored-by: heihutu Co-authored-by: zhi22915 --- .../ecstore/src/services/rebalance/entry.rs | 54 +++++++++++-------- rustfs/tests/connect_license.rs | 4 +- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/crates/ecstore/src/services/rebalance/entry.rs b/crates/ecstore/src/services/rebalance/entry.rs index 6a9987707..05fbe5c06 100644 --- a/crates/ecstore/src/services/rebalance/entry.rs +++ b/crates/ecstore/src/services/rebalance/entry.rs @@ -50,7 +50,7 @@ fn ensure_rebalance_entry_active(cancel: &CancellationToken) -> Result<()> { Ok(()) } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] tokio::task_local! { static REBALANCE_ENTRY_RUN_FENCE_BARRIER: (Arc, Arc); } @@ -268,7 +268,7 @@ impl ECStore { self.rebalance_run_guard(rebalance_id.as_ref(), "rebalance entry") }) .await?; - #[cfg(test)] + #[cfg(any(test, feature = "test-util"))] if let Ok((arrived, release)) = REBALANCE_ENTRY_RUN_FENCE_BARRIER.try_with(Clone::clone) { arrived.notify_one(); release.notified().await; @@ -836,7 +836,8 @@ pub mod test_util { _temp_dirs: Vec, store: Arc, cancel: CancellationToken, - barrier: crate::data_movement::SourceCleanupDeleteBarrier, + entry_arrived: Arc, + entry_release: Arc, stop_probe: super::super::control::RebalanceStopWaitProbe, entry_task: Option>>, } @@ -897,31 +898,36 @@ pub mod test_util { cached: Some(file_meta), reusable: false, }; - let barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(bucket, object); + let entry_arrived = Arc::new(tokio::sync::Notify::new()); + let entry_release = Arc::new(tokio::sync::Notify::new()); let entry_store = Arc::clone(&store); let entry_set = Arc::clone(&set); let entry_cancel = cancel.clone(); - let entry_task = tokio::spawn(async move { - entry_store - .rebalance_entry( - RebalanceEntryTarget { - bucket: bucket.to_string(), - pool_index: 0, - }, - entry, - entry_set, - Arc::new(RebalanceBucketConfigs::default()), - Arc::from(rebalance_id), - entry_cancel, - ) - .await - }); + let entry_task = tokio::spawn(REBALANCE_ENTRY_RUN_FENCE_BARRIER.scope( + (Arc::clone(&entry_arrived), Arc::clone(&entry_release)), + async move { + entry_store + .rebalance_entry( + RebalanceEntryTarget { + bucket: bucket.to_string(), + pool_index: 0, + }, + entry, + entry_set, + Arc::new(RebalanceBucketConfigs::default()), + Arc::from(rebalance_id), + entry_cancel, + ) + .await + }, + )); Self { _temp_dirs, store, cancel, - barrier, + entry_arrived, + entry_release, stop_probe: super::super::control::RebalanceStopWaitProbe::install(rebalance_id), entry_task: Some(entry_task), } @@ -932,7 +938,9 @@ pub mod test_util { } pub async fn wait_until_entry_paused(&self) { - self.barrier.wait_until_paused().await; + tokio::time::timeout(std::time::Duration::from_secs(30), self.entry_arrived.notified()) + .await + .expect("the entry should acquire its run fence before pausing"); } pub async fn wait_until_admission_cancelled(&self) { @@ -946,7 +954,7 @@ pub mod test_util { } pub fn release_entry(&self) { - self.barrier.release(); + self.entry_release.notify_one(); } pub async fn assert_entry_cancelled(&mut self) { @@ -964,7 +972,7 @@ pub mod test_util { impl Drop for PausedRebalanceEntryTestFixture { fn drop(&mut self) { - self.barrier.release(); + self.entry_release.notify_one(); if let Some(task) = self.entry_task.take() { task.abort(); } diff --git a/rustfs/tests/connect_license.rs b/rustfs/tests/connect_license.rs index d5ad32bed..bbe55e583 100644 --- a/rustfs/tests/connect_license.rs +++ b/rustfs/tests/connect_license.rs @@ -126,8 +126,8 @@ fn tamper_foreign_scope_and_expiry_are_structured_and_preserve_installed_state() let payload = URL_SAFE_NO_PAD .decode_to_vec(envelope["payload"].as_str().expect("payload").as_bytes()) .expect("decode payload"); - let mut claims: Value = serde_json::from_slice(&payload).expect("decode claims"); - claims["deployment"] = Value::String(OTHER_DEPLOYMENT.to_owned()); + let mut claims: LicenseClaims = serde_json::from_slice(&payload).expect("decode claims"); + claims.deployment = OTHER_DEPLOYMENT.to_owned(); envelope["payload"] = Value::String(URL_SAFE_NO_PAD.encode_to_string(serde_json::to_vec(&claims).unwrap())); let tampered = write_artifact(temporary.path(), "tampered.json", &serde_json::to_string(&envelope).unwrap()); let error = apply_license_artifact(&tampered, &state, &valid).expect_err("tampered artifact must fail"); From 0a6aeda191886173efcbc6636bdb516903a88005 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 04:06:42 +0800 Subject: [PATCH 035/143] test(e2e): bound scanner witnesses by the reported schedule (#8501) * test(e2e): bound scanner witnesses by the reported schedule Require a fresh completed observed walk and account once for planned superseded waiting, preserving the active-cycle execution budget. Cover invalid schedules and early wakeup, and refresh exact platform selection digests without removing existing tests. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(e2e): refresh nightly scanner budget membership Record the three scanner budget regressions in both platform nightly selection digests, preserving every previously selected test. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- .config/e2e-full-selection.txt | 4 +- .config/e2e-nightly-selection.txt | 4 +- .../src/heal_erasure_disk_rebuild_test.rs | 79 ++++++++++++++++++- 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index 8669a2126..2040896ee 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=98dff0961c6eb21d6128bfd2d4a6100a624ddf1fb7db05dba360a5d3d0a310c2 -sha256-linux=890f5925b87e1bf3710c356253f3b2276d901e156df9ccc04ec9315374de6d22 +sha256-darwin=b8937d6de585895a837b1505c7a9f91aa45394fbdc47cd179287a35a6f1de8be +sha256-linux=e803ee5fbd29bd45b8542b4256b75d5a2196ccc7ccfd1ccac6e14553730e9047 diff --git a/.config/e2e-nightly-selection.txt b/.config/e2e-nightly-selection.txt index bf672a947..9a0869842 100644 --- a/.config/e2e-nightly-selection.txt +++ b/.config/e2e-nightly-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=5dbb9154b8feb94cedc8fb4aa6895d4b3259cc31b7df1f5c3b9928ef07f04535 -sha256-linux=d3a210be9eccf593d3723ae669b66680e9751dd01a6949f03df20614f913b707 +sha256-darwin=ce7b419faf1ff79840dfc6325d31ee8d1ca5eab85125b94d0bcd58cdf07f0f76 +sha256-linux=5e9e3d701ee873c13fb9094575f6889779c6078b1107624bb57663bef9b9d21f diff --git a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs index aaf45f5de..c5d2e20b7 100644 --- a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs +++ b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs @@ -1191,11 +1191,59 @@ mod tests { .into()) } + fn scanner_cycle_completion_budget( + cycle_end: u64, + interval: Option, + now: u64, + active: bool, + ) -> Result { + if cycle_end > now { + return Err("scanner cycle timestamp is in the future"); + } + if active { + return Ok(Duration::from_secs(60)); + } + let interval = interval + .filter(|interval| *interval != 0) + .ok_or("scanner schedule has no positive interval")?; + let next_cycle = cycle_end + .checked_add(interval) + .ok_or("scanner schedule timestamp overflows")?; + let budget = next_cycle + .saturating_sub(now) + .checked_add(60) + .ok_or("scanner completion budget overflows")?; + Ok(Duration::from_secs(budget)) + } + + #[test] + fn scanner_cycle_completion_budget_accounts_for_captured_retry_schedule() { + assert_eq!( + scanner_cycle_completion_budget(1_791_815_390, Some(256), 1_791_815_447, false), + Ok(Duration::from_secs(259)), + ); + assert_eq!(scanner_cycle_completion_budget(100, Some(1), 110, false), Ok(Duration::from_secs(60))); + } + + #[test] + fn scanner_cycle_completion_budget_does_not_grant_planned_sleep_to_active_cycle() { + assert_eq!(scanner_cycle_completion_budget(100, Some(256), 110, true), Ok(Duration::from_secs(60))); + assert_eq!(scanner_cycle_completion_budget(0, None, 110, true), Ok(Duration::from_secs(60))); + } + + #[test] + fn scanner_cycle_completion_budget_rejects_invalid_schedule() { + for (end, interval, now) in [(101, 1, 100), (100, 0, 100), (u64::MAX, 1, u64::MAX), (1, u64::MAX - 1, 1)] { + assert!(scanner_cycle_completion_budget(end, Some(interval), now, false).is_err()); + } + } + async fn wait_for_scanner_cycle_after( cluster: &RustFSTestClusterEnvironment, previous_cycle_end: u64, ) -> Result> { - let deadline = Instant::now() + Duration::from_secs(60); + let mut deadline = Instant::now() + Duration::from_secs(60); + let mut schedule_accounted = false; loop { let mut latest_cycle_end = 0; let mut versions_observed = false; @@ -1237,14 +1285,36 @@ mod tests { let versions_scanned = metrics["versions_scanned"] .as_u64() .ok_or("scanner status is missing its version-coverage counter")?; - latest_cycle_end = latest_cycle_end.max(cycle_end); + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)?.as_secs(); + if cycle_end > now { + return Err("scanner cycle timestamp is in the future".into()); + } + let schedule = &status["cycle_schedule"]; + if !schedule_accounted && schedule["execution_role"].as_str() == Some("leader") { + let active = metrics["current_cycle_active"] + .as_bool() + .ok_or("scanner status is missing its active-cycle flag")?; + if active || (cycle_end != 0 && schedule["effective_interval_available"].as_bool() == Some(true)) { + let interval = schedule["effective_interval_seconds"].as_u64(); + let budget = scanner_cycle_completion_budget(cycle_end, interval, now, active)?; + deadline = Instant::now().checked_add(budget).ok_or("scanner deadline overflows")?; + schedule_accounted = true; + } + } + // Superseded walks complete all sets while withholding quota authority. + // Deferred and partial attempts do not certify completed scanning. + if matches!(metrics["last_cycle_result"].as_str(), Some("success" | "superseded")) { + latest_cycle_end = latest_cycle_end.max(cycle_end); + } versions_observed |= versions_scanned > 0; observations.push(format!( - "node{node_index}: end={cycle_end}, versions={versions_scanned}, cycle={}, active={}, leader={}, result={}", + "node{node_index}: end={cycle_end}, versions={versions_scanned}, cycle={}, active={}, leader={}, result={}, partial_reason={}, schedule={}", metrics["current_cycle"], metrics["current_cycle_active"], metrics["leader_lock_state"], metrics["last_cycle_result"], + metrics["last_cycle_partial_reason"], + schedule, )); } // The coordinator records cycle completion, but remote workers @@ -1555,7 +1625,8 @@ mod tests { } if background_enabled { - wait_for_scanner_cycle_after(&cluster, 0).await?; + let baseline_cycle_floor = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)?.as_secs(); + wait_for_scanner_cycle_after(&cluster, baseline_cycle_floor).await?; } cluster.stop_node(1)?; From e7e6556f83489548e4a699e5dff59c813b52cdf2 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 07:04:27 +0800 Subject: [PATCH 036/143] fix(scanner): bound superseded waits for regular maintenance (#8502) Preserve ordinary lifecycle and replication cadence while retaining usage-only convergence backoff, explicit cycle floors and publication fencing. Cover repeated snapshots and the real convergence waiting path. Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/scanner/src/scanner.rs | 3 +- crates/scanner/src/scanner/activity.rs | 20 ++-- crates/scanner/src/scanner/tests.rs | 118 +++++++++++++++++++- docs/operations/scanner-runtime-controls.md | 2 + 4 files changed, 129 insertions(+), 14 deletions(-) diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 645485e30..58202a53d 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -3181,7 +3181,8 @@ where )); let mut wait_plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for); - let superseded_retry_interval = scanner_superseded_retry_interval(superseded_backoff, &runtime_config); + let superseded_retry_interval = + scanner_superseded_retry_interval(superseded_backoff, &runtime_config, maintenance_features); let deferred_retry_interval = deferred_backoff.retry_interval(runtime_config.cycle_interval); let mut convergence_retry_interval = superseded_retry_interval.or(deferred_retry_interval); if let Some(retry_interval) = convergence_retry_interval { diff --git a/crates/scanner/src/scanner/activity.rs b/crates/scanner/src/scanner/activity.rs index 2b63d115a..efa930a6e 100644 --- a/crates/scanner/src/scanner/activity.rs +++ b/crates/scanner/src/scanner/activity.rs @@ -152,17 +152,19 @@ impl ScannerRetryBackoff { pub(super) fn scanner_superseded_retry_interval( backoff: ScannerRetryBackoff, runtime_config: &ScannerRuntimeConfig, + maintenance_features: ScannerMaintenanceFeatures, ) -> Option { - let retry_interval = backoff.retry_interval(runtime_config.cycle_interval)?; - if runtime_config.cycle_interval_source == ScannerRuntimeConfigSource::Default { - return Some(retry_interval); + let mut retry_interval = backoff.retry_interval(runtime_config.cycle_interval)?; + // Preserve an operator-selected duty-cycle floor during convergence. + if runtime_config.cycle_interval_source != ScannerRuntimeConfigSource::Default { + retry_interval = retry_interval.max(runtime_config.cycle_interval); } - - // An explicit cycle is an operator-selected duty-cycle floor. A - // superseded snapshot keeps its dirty work pending, but retrying that work - // sooner than the configured cadence would turn continuous writes into a - // repeated full-walk loop despite the override. - Some(retry_interval.max(runtime_config.cycle_interval)) + // A complete but non-authoritative walk must not postpone lifecycle or + // replication evaluation beyond its ordinary maintenance cadence. + if maintenance_features.needs_regular_cycle() { + retry_interval = retry_interval.min(runtime_config.cycle_interval.max(Duration::from_secs(1))); + } + Some(retry_interval) } const SCANNER_PUBLICATION_PROOF_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30); diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index b43fc1eb2..8a4f5a7e0 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -8489,7 +8489,7 @@ fn superseded_retry_preserves_explicit_cycle_cadence() { ..Default::default() }; assert_eq!( - scanner_superseded_retry_interval(backoff, &default_config), + scanner_superseded_retry_interval(backoff, &default_config, ScannerMaintenanceFeatures::default()), Some(SCANNER_RETRY_BASE_INTERVAL), "the default adaptive cadence must retain fast convergence" ); @@ -8505,14 +8505,124 @@ fn superseded_retry_preserves_explicit_cycle_cadence() { ..Default::default() }; + for features in [ + ScannerMaintenanceFeatures::default(), + ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }, + ] { + assert_eq!( + scanner_superseded_retry_interval(backoff, &runtime_config, features), + Some(runtime_config.cycle_interval), + "{source:?} cycle cadence must not be shortened by convergence retries" + ); + } + } +} + +#[test] +fn superseded_retry_does_not_delay_required_maintenance() { + let features = ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }; + assert!(features.needs_regular_cycle()); + let config = ScannerRuntimeConfig { + cycle_interval: Duration::from_secs(2), + cycle_interval_source: ScannerRuntimeConfigSource::Env, + ..Default::default() + }; + let mut backoff = ScannerRetryBackoff::default(); + for consecutive_cycles in 1..=12 { + backoff.record_retryable_cycle(true); assert_eq!( - scanner_superseded_retry_interval(backoff, &runtime_config), - Some(runtime_config.cycle_interval), - "{source:?} cycle cadence must not be shortened by convergence retries" + scanner_superseded_retry_interval(backoff, &config, features), + Some(config.cycle_interval), + "superseded cycle {consecutive_cycles} must preserve regular maintenance" ); } } +#[test] +fn superseded_retry_only_caps_regular_maintenance() { + let config = ScannerRuntimeConfig { + cycle_interval: Duration::from_secs(60), + ..Default::default() + }; + let mut backoff = ScannerRetryBackoff::default(); + for _ in 0..6 { + backoff.record_retryable_cycle(true); + } + assert_eq!( + scanner_superseded_retry_interval(backoff, &config, ScannerMaintenanceFeatures::default()), + Some(Duration::from_secs(160)), + ); + for features in [ + ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }, + ScannerMaintenanceFeatures { + replication: true, + ..Default::default() + }, + ScannerMaintenanceFeatures { + inspection_failed: true, + ..Default::default() + }, + ] { + assert_eq!(scanner_superseded_retry_interval(backoff, &config, features), Some(config.cycle_interval)); + assert_eq!(scanner_superseded_retry_interval(ScannerRetryBackoff::default(), &config, features), None); + } +} + +#[tokio::test(start_paused = true)] +#[serial] +async fn superseded_maintenance_wait_keeps_convergence_fencing() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let config = ScannerRuntimeConfig { + cycle_interval: Duration::from_secs(60), + ..Default::default() + }; + let features = ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }; + let mut backoff = ScannerRetryBackoff::default(); + for _ in 0..12 { + backoff.record_retryable_cycle(true); + } + let interval = scanner_superseded_retry_interval(backoff, &config, features); + assert_eq!(interval, Some(config.cycle_interval)); + let generations = ScannerCycleObservedGenerations::for_wait( + &config, + interval, + crate::scanner_io::dirty_usage_generation(), + crate::runtime_config::scanner_runtime_config_generation(), + crate::scanner_io::scanner_maintenance_generation(), + ); + assert!(generations.dirty_usage.is_none()); + assert!(generations.defer_cluster_activity); + let ctx = CancellationToken::new(); + let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); + let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]); + let started = tokio::time::Instant::now(); + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + interval.expect("maintenance must retain its convergence interval"), + Some(Duration::from_secs(1)), + &mut seen, + generations, + || false, + || std::future::ready(Ok(changed.clone())), + ) + .await; + assert_eq!(reason, ScannerCycleWakeReason::Timer); + assert_eq!(started.elapsed(), config.cycle_interval); + assert_eq!(seen, Some(changed)); +} + #[test] fn publication_proof_retry_backoff_reaches_its_short_cap() { for (failures, expected) in [(1, 5), (2, 10), (3, 20), (4, 30), (20, 30)] { diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index 8ca95fafb..faf999409 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -98,6 +98,8 @@ An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle c Under continuous writes, a structurally complete walk can finish as `superseded`. Its durable observation remains separate from quota authority and leaves dirty work pending. Superseded retries may reuse that complete inventory to refresh dirty buckets, even while authority bootstrap is pending; missing or invalid coverage still forces a full walk. Ordinary maintenance timers, Deep scans, active lifecycle/replication and changed maintenance configuration retain full scans. Server info and datausageinfo can display complete observations, while Last Completed Scan advances only after an authoritative completed cycle. A write-free window is therefore not required to display observed counts, but arbitrary continuous writes do not guarantee authoritative completion. Historically dirty buckets remain eligible for rewalk, so narrower retries do not guarantee shorter cycles. +Superseded convergence retries retain exponential backoff for usage-only work. Active lifecycle or replication rules, and uncertain maintenance inspection, cap that retry interval at the ordinary configured cadence so complete observations cannot defer regular maintenance. An explicit cycle remains the duty-cycle floor. Convergence mode still suppresses dirty/activity shortcuts and does not grant quota authority; deferred backend/proof retries and pause catch-up limits retain their existing scheduling. Cycle runtime, pacing, and disk admission can still delay completed maintenance. Sustained supersedes with active maintenance may therefore run more full scans than usage-only convergence; use the existing pacing controls to manage that background I/O. + ## Single-disk clean-idle scheduling An erasure single-disk deployment using the built-in cycle and bitrot defaults automatically backs off repeated clean idle scans instead of walking the same unchanged namespace every minute. Each successful timer-driven cycle that finds no dirty usage or unresolved maintenance work doubles the next interval. The status endpoint reports the effective interval and multiplier. From 9d6f48c3251c12ac40ef21f4e18337f4cc0bce08 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 13 Oct 2026 10:25:12 +0800 Subject: [PATCH 037/143] fix(s3): match AWS duplicate bucket responses (#8503) --- rustfs/src/app/bucket_usecase.rs | 36 ++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 445978a09..69d06c455 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -1257,15 +1257,19 @@ fn build_list_objects_v2_metadata_output( } } -fn create_bucket_exists_response(is_owner: bool) -> S3Result> { - if is_owner { +fn create_bucket_exists_response(is_owner: bool, region: &str) -> S3Result> { + if is_owner && region == RUSTFS_REGION { + // AWS preserves the legacy idempotent response for buckets recreated in + // us-east-1 (the default RustFS region). return Ok(S3Response::new(CreateBucketOutput::default())); } - Err(s3_error!( - BucketAlreadyExists, - "The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again." - )) + let error_code = if is_owner { + S3ErrorCode::BucketAlreadyOwnedByYou + } else { + S3ErrorCode::BucketAlreadyExists + }; + Err(S3Error::with_message(error_code, "The requested bucket already exists.")) } fn resolve_notification_region(global_region: Option, request_region: Option) -> String { @@ -1526,9 +1530,13 @@ impl DefaultBucketUsecase { .await; } Err(StorageError::BucketExists(_)) => { - // Per S3 spec: bucket namespace is global. Owner recreating returns 200 OK; - // non-owner gets 409 BucketAlreadyExists. - let result = create_bucket_exists_response(requester_is_owner); + // Match AWS CreateBucket semantics: an owned bucket is idempotent in + // us-east-1 and returns BucketAlreadyOwnedByYou elsewhere. + let region = self + .global_region() + .map(|region| region.to_string()) + .unwrap_or_else(|| RUSTFS_REGION.to_string()); + let result = create_bucket_exists_response(requester_is_owner, ®ion); let _ = helper.complete(&result); return result; } @@ -4299,16 +4307,22 @@ mod tests { #[test] fn create_bucket_exists_response_returns_ok_for_owner() { - let response = create_bucket_exists_response(true).expect("owner recreate should succeed"); + let response = create_bucket_exists_response(true, RUSTFS_REGION).expect("owner recreate in us-east-1 should succeed"); assert_eq!(response.output.location, None); } #[test] fn create_bucket_exists_response_returns_bucket_already_exists_for_non_owner() { - let err = create_bucket_exists_response(false).expect_err("non-owner recreate should fail"); + let err = create_bucket_exists_response(false, RUSTFS_REGION).expect_err("non-owner recreate should fail"); assert_eq!(err.code(), &S3ErrorCode::BucketAlreadyExists); } + #[test] + fn create_bucket_exists_response_returns_owned_by_you_outside_us_east_1() { + let err = create_bucket_exists_response(true, "eu-west-1").expect_err("owner recreate outside us-east-1 should fail"); + assert_eq!(err.code(), &S3ErrorCode::BucketAlreadyOwnedByYou); + } + #[test] fn build_request_with_req_info_preserves_owner_state() { let input = CreateBucketInput::builder() From 9af2f7c04eea50b33a215aabf8ed6ec192e43d42 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 13 Oct 2026 11:05:21 +0800 Subject: [PATCH 038/143] fix(ilm): make missing lifecycle expiry targets idempotent (#8504) fix(ilm): treat missing expiry targets as success --- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 89 ++++++++++++++++--- 1 file changed, 76 insertions(+), 13 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 06919238f..a98e6e038 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -2417,6 +2417,16 @@ fn is_slow_down(err: &Error) -> bool { matches!(err, Error::SlowDown) } +/// Lifecycle expiry is idempotent when the exact object or version has already +/// disappeared between the scanner listing and the delete admission. +/// +/// Keep this predicate narrower than bucket/volume "not found" helpers: an +/// unavailable bucket or disk is an operational failure and must not be +/// reported as a completed expiry. +fn is_lifecycle_expiry_already_gone(err: &Error) -> bool { + is_err_object_not_found(err) || is_err_version_not_found(err) +} + /// Resolves the expiry worker count from the single documented knob, /// `RUSTFS_MAX_EXPIRY_WORKERS`: a set, parsable, non-zero value wins; /// anything else falls back to `min(cpus, 16)`. The historical @@ -4920,6 +4930,20 @@ async fn expire_transitioned_object_with_lock_lost_signal( //audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn); Ok(dobj) } + Err(err) if is_lifecycle_expiry_already_gone(&err) => { + debug!( + event = EVENT_LIFECYCLE_DELETE_COMPLETED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + bucket = %oi.bucket, + object = %oi.name, + operation = "delete_restored_object", + reason = "already_gone", + "Lifecycle delete target was already absent" + ); + crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await; + Ok(oi.clone()) + } Err(err) => Err(std::io::Error::other(err)), }; } @@ -4930,6 +4954,8 @@ async fn expire_transitioned_object_with_lock_lost_signal( // pointing to a remote tier version that has already been removed. If this // only creates a delete marker, remote cleanup must be driven by persisted // free-version recovery rather than the visible delete result. + // Keep remote transitioned not-found errors visible: local metadata + // absence alone does not prove that the tier object was cleaned up. let dobj = match api.delete_object(&oi.bucket, &oi.name, opts).await { Ok(obj) => obj, Err(e) => { @@ -5802,6 +5828,20 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal( .await { Ok(dobj) => dobj, + Err(err) if is_lifecycle_expiry_already_gone(&err) => { + debug!( + event = EVENT_LIFECYCLE_DELETE_COMPLETED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + bucket = %oi.bucket, + object = %oi.name, + operation = "delete_object", + reason = "already_gone", + "Lifecycle delete target was already absent" + ); + crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await; + return true; + } Err(e) => { error!( event = EVENT_LIFECYCLE_DELETE_FAILED, @@ -6041,18 +6081,19 @@ mod tests { cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at, enqueue_recovered_free_version_with_state, enqueue_transition_for_existing_objects_scoped, enqueue_transition_with_lifecycle, enqueue_transition_with_lifecycle_report, eval_action_from_lifecycle, - get_lock_acquire_timeout, jitter_tier_free_version_recovery_delay, lifecycle_action_blocked_by_replication, - lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object, lifecycle_replication_blocks_action, - lifecycle_rule_has_date_expiration, lifecycle_version_delete_target, manual_transition_duration_elapsed, - manual_transition_has_more_after_limit, manual_transition_recovery_progress_sink, manual_transition_version_marker, - manual_transition_worker_failure_reason, mark_delete_opts_skip_decommissioned_on_remote_success, - merge_stale_multipart_candidate, persist_manual_transition_job_progress_if_owned, - persist_manual_transition_page_checkpoint, recover_manual_transition_job, recover_manual_transition_jobs, - resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, - resolve_transition_worker_count, resolve_transition_workers_absolute_max, run_tier_free_version_recovery_coordinator, - run_tier_free_version_recovery_loop, select_restore_s3_location, set_lifecycle_observability_observer, - set_recovered_free_version_enqueue_observer, should_defer_date_expiry_for_recent_config_update, - transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery, + get_lock_acquire_timeout, is_lifecycle_expiry_already_gone, jitter_tier_free_version_recovery_delay, + lifecycle_action_blocked_by_replication, lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object, + lifecycle_replication_blocks_action, lifecycle_rule_has_date_expiration, lifecycle_version_delete_target, + manual_transition_duration_elapsed, manual_transition_has_more_after_limit, manual_transition_recovery_progress_sink, + manual_transition_version_marker, manual_transition_worker_failure_reason, + mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate, + persist_manual_transition_job_progress_if_owned, persist_manual_transition_page_checkpoint, + recover_manual_transition_job, recover_manual_transition_jobs, resolve_tier_free_version_recovery_enabled, + resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, resolve_transition_worker_count, + resolve_transition_workers_absolute_max, run_tier_free_version_recovery_coordinator, run_tier_free_version_recovery_loop, + select_restore_s3_location, set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer, + should_defer_date_expiry_for_recent_config_update, transitioned_cleanup_tuple, transitioned_object_delete_opts, + wait_for_tier_free_version_recovery, }; #[cfg(feature = "test-util")] use super::{delete_free_version_remote_object_then, encode_dir_object, get_transitioned_object_reader_with_tier_manager}; @@ -10511,6 +10552,26 @@ mod tests { ); } + #[test] + fn lifecycle_expiry_already_gone_only_accepts_object_or_version_absence() { + assert!(is_lifecycle_expiry_already_gone(&Error::FileNotFound)); + assert!(is_lifecycle_expiry_already_gone(&Error::ObjectNotFound( + "bucket".to_string(), + "object".to_string(), + ))); + assert!(is_lifecycle_expiry_already_gone(&Error::FileVersionNotFound)); + assert!(is_lifecycle_expiry_already_gone(&Error::VersionNotFound( + "bucket".to_string(), + "object".to_string(), + "version".to_string(), + ))); + + assert!(!is_lifecycle_expiry_already_gone(&Error::VolumeNotFound)); + assert!(!is_lifecycle_expiry_already_gone(&Error::DiskNotFound)); + assert!(!is_lifecycle_expiry_already_gone(&Error::ErasureReadQuorum)); + assert!(!is_lifecycle_expiry_already_gone(&Error::MethodNotAllowed)); + } + #[tokio::test] async fn manual_transition_counts_already_transitioned_object() { let lc = latest_transition_lifecycle(); @@ -13385,7 +13446,9 @@ mod tests { .bucket_incarnation_id_from_disk(&bucket) .await .expect("bucket incarnation"); - for (attempt_incarnation, expected) in [(Uuid::new_v4(), false), (incarnation, true), (incarnation, false)] { + // Replaying the same scanner snapshot after the delete is an + // already-gone success and must not be reported as a failure. + for (attempt_incarnation, expected) in [(Uuid::new_v4(), false), (incarnation, true), (incarnation, true)] { assert_eq!( super::apply_expiry_on_non_transitioned_objects( ecstore.clone(), From 34803e816501a17d68d5e681ad8c7323444698f5 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 13:07:50 +0800 Subject: [PATCH 039/143] perf(ecstore): isolate cleanup and preserve metadata authority (#8289) * chore: pin s3s to upstream git revision Use the upstream s3s git repository at 50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6 for both s3s and s3s-sigv4. Allow the official s3s git source in cargo-deny so source checks continue to pass. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * perf(ecstore): instrument put object hot path Co-Authored-By: heihutu Co-Authored-By: zhi22915 * perf(ecstore): split cleanup worker wait metrics Co-Authored-By: heihutu Co-Authored-By: zhi22915 * upgrade version * chore(metrics): add PUT post-store stage probes Add lightweight PUT post-store stage probes for issue 2615 validation. * fix cargo deny check warn * upgrade version * perf(ecstore): bound and isolate cleanup execution (#8151) * perf(ecstore): bound and isolate cleanup execution * fix(ecstore): preserve cleanup cancellation and lease release * feat(ecstore): default isolated cleanup to CPUs 1-2 * test(ecstore): decouple budget tests from CPU affinity * upgrade version * perf(ecstore): reuse pinned directory handles for object I/O * feat(observability): add sampled local GetObject stage summaries Add an opt-in GetObject stage aggregation path that works without the OTLP metrics export. When RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED is set, io-metrics keeps per-(path, stage, object class, size bucket) sampled counts and duration totals in a bounded in-process map, and a startup task flushes them periodically to /var/log/rustfs/get-stage-summary.log, tunable via the sample rate and interval env keys. * build(deps): bump workspace dependency releases Patch releases: async-compression 0.4.49, tokio-rustls 0.26.6, xxhash-rust 0.8.19, suppaftp 12.1.1, russh-sftp 3.0.1. Minor release: hotpath 0.27.0. * fix(ecstore): fence distributed metadata cache mutations Add authenticated cluster-wide mutation fences for the opt-in DistErasure GetObject metadata cache. Guard recursive prefix and batch deletes, fail closed on configuration mismatch or uncertain quorum, and cover replay, restart, peer loss, and mixed-configuration behavior. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * refactor(metrics): consolidate PUT observers on improve/base (#8039) * perf(object): instrument storage hot paths Co-Authored-By: heihutu Co-Authored-By: zhi22915 * refactor(metrics): consolidate PUT stage observation Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: zhi22915 Co-authored-by: heihutu * build(deps): bump workspace dependency releases * ci: allowlist rustix OFlags WRONLY in typos config * refactor(storage): route ecstore symbols through storage_api boundaries * perf(ecstore): cache quorum-selected early-stop metadata (#8308) Publish eligible latest-version GET metadata after a validated read quorum when early-stop cancels the remaining fanout. Keep metadata errors, selection quorum, online read quorum, and mutation-generation checks as publication gates. Add slow-peer repeat-hit and concurrent mutation-fence regression tests. Co-authored-by: heihutu Co-authored-by: zhi22915 * chore: merge latest main into improve/base (#8332) * refactor(storage): bind rename rollback to its commit context (#8324) * refactor(storage): separate metadata scheduling from quorum decisions (#8327) * test(storage): verify rejected rollback inspection on real disks (#8331) * test(storage): make commit fault checkpoints independent (#8326) * test(storage): make commit fault checkpoints independent * test(storage): simplify commit checkpoint lookup * fix(listing): count logical prefixes before pagination (#8328) * fix(listing): count logical prefixes before pagination * test(listing): normalize the fixed corpus seed --------- Co-authored-by: Chris Co-authored-by: zhi22915 * perf(ecstore): borrow metadata cache lookup key (#8333) Avoid request-scoped Arc allocations on eligible cache lookups while preserving full key identity. Co-authored-by: zhi22915 * chore: sync improve/base with latest main (#8334) * refactor(storage): bind rename rollback to its commit context (#8324) * refactor(storage): separate metadata scheduling from quorum decisions (#8327) * test(storage): verify rejected rollback inspection on real disks (#8331) * test(storage): make commit fault checkpoints independent (#8326) * test(storage): make commit fault checkpoints independent * test(storage): simplify commit checkpoint lookup * fix(listing): count logical prefixes before pagination (#8328) * fix(listing): count logical prefixes before pagination * test(listing): normalize the fixed corpus seed * fix(scanner): isolate and coalesce dirty usage journals (#8325) * fix(scanner): isolate and coalesce dirty usage journals Bind dirty replay records to a stable node owner so peers cannot overwrite or delete another node's pending journal. Replace the unbounded command queue with bounded coalesced state and revision-aware background saves. Keep legacy records isolated and enforce replay byte limits with conservative whole-bucket or unverified fallback. Add publication-stage diagnostics and storage regressions for ownership, clearing, retry, and restart behavior. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(scanner): collapse confirmed journal clear condition Use a let-chain to retain the generation fence before removing a confirmed bucket and updating its scope byte accounting. Satisfy collapsible_if without changing journal clearing behavior. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --------- Co-authored-by: Chris Co-authored-by: heihutu Co-authored-by: zhi22915 * fix(obs): distinguish metrics by process identity (#8343) Attach process.pid to the OpenTelemetry resource and promote it for Prometheus so counter series from restarted processes remain distinct. Preserve the stable RustFS node and service instance attributes. Co-authored-by: heihutu Co-authored-by: zhi22915 * merge: sync origin/main into improve/base (#8345) * refactor(storage): bind rename rollback to its commit context (#8324) * refactor(storage): separate metadata scheduling from quorum decisions (#8327) * test(storage): verify rejected rollback inspection on real disks (#8331) * test(storage): make commit fault checkpoints independent (#8326) * test(storage): make commit fault checkpoints independent * test(storage): simplify commit checkpoint lookup * fix(listing): count logical prefixes before pagination (#8328) * fix(listing): count logical prefixes before pagination * test(listing): normalize the fixed corpus seed * fix(scanner): isolate and coalesce dirty usage journals (#8325) * fix(scanner): isolate and coalesce dirty usage journals Bind dirty replay records to a stable node owner so peers cannot overwrite or delete another node's pending journal. Replace the unbounded command queue with bounded coalesced state and revision-aware background saves. Keep legacy records isolated and enforce replay byte limits with conservative whole-bucket or unverified fallback. Add publication-stage diagnostics and storage regressions for ownership, clearing, retry, and restart behavior. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(scanner): collapse confirmed journal clear condition Use a let-chain to retain the generation fence before removing a confirmed bucket and updating its scope byte accounting. Satisfy collapsible_if without changing journal clearing behavior. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 * fix(storage): retain typed owners across borrowed writes (#8329) * fix(storage): retain typed owners across borrowed writes * fix(storage): preserve Copy write completion policy * refactor(storage): remove redundant write owner references * fix(ecstore): hide rollback backup residue in prefix listings * test(connect): isolate short HTTP fixtures in nextest * fix(protocols): enforce S3 policies for file protocol operations (#8262) * test(ecstore): enforce main core regression gates (#8338) * test(ecstore): enforce main core regression gates * test(ecstore): pin external shards and completion regressions * fix(test): restore authenticated cluster fault scenarios (#8340) * fix(test): restore authenticated cluster fault scenarios * fix(test): use the E2E storage API boundary * ci: coerce the pool timeout input to a number timeout-minutes requires a numeric value, but inputs.pool_timeout_minutes is a string, so '${{ inputs.pool_timeout_minutes || 240 }}' evaluated to the string '240' and the runner rejected the template: 'An error occurred when attempting to determine the step timeout ... Unexpected value 240' (chain run 36505625402 job at 02:25:52, standalone run 36393453114 at 07:46:37). Both pool runs since #8155 merged have therefore executed without the intended step budget. Wrap the expression in fromJSON and update the policy test that pinned the broken literal. * ci: coerce the pool timeout input to a number timeout-minutes requires a numeric value, but inputs.pool_timeout_minutes is a string, so '${{ inputs.pool_timeout_minutes || 240 }}' evaluated to the string '240' and the runner rejected the template: 'An error occurred when attempting to determine the step timeout ... Unexpected value 240' (chain run 36505625402 job at 02:25:52, standalone run 36393453114 at 07:46:37). Both pool runs since #8155 merged have therefore executed without the intended step budget. Wrap the expression in fromJSON and update the policy test that pinned the broken literal. * fix(ecstore): own listing workers through cancellation (#8342) Consolidate the three page collectors, preserve completed worker failures, and abort nested merges when their owner is dropped. Co-authored-by: heihutu Co-authored-by: zhi22915 * fix(ecstore): route commit sync through guarded helper Use the Linux-capable guarded directory sync entry point for both commit sync sites. This keeps group commit and file-sync admission behavior while ensuring the already-open commit directory remains the sync target. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): expose rename directory guard in production The commit path now acquires the directory guard in production builds, not only tests, so destination metadata reads and durable sync share the guarded context. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: Chris Co-authored-by: heihutu Co-authored-by: zhi22915 Co-authored-by: GatewayJ <835269233@qq.com> Co-authored-by: hector <42570491+majinghe@users.noreply.github.com> * fix(ecstore): remove unsafe metadata cache admission paths Keep distributed GET metadata on quorum reads and quota admission on authoritative storage. Preserve local cache invalidation, remove the experimental RPC surface, and gate diagnostic work. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(ecstore): cover cold degraded mutation admission Check first PUT and DELETE with one peer absent and no SDK retries, then verify that losing write quorum still rejects writes. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): retain guarded Linux destination metadata reads Use the existing commit directory guard when reading old xl.meta on Linux. Cover symlinked destination metadata for inline and non-inline renames without changing the split commit structure. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * chore(test): remove retired cache RPC import Co-Authored-By: heihutu Co-Authored-By: zhi22915 * merge: sync latest main into improve/base (#8349) * test: allow cleanup fixture CI headroom (#8336) * fix(s3): encode multipart listing response keys (#8347) --------- Co-authored-by: cxymds Co-authored-by: Chris Co-authored-by: zhi22915 * merge: sync main auth and scanner updates (#8352) * test: allow cleanup fixture CI headroom (#8336) * fix(s3): encode multipart listing response keys (#8347) * perf(scanner): reduce scan and checkpoint peak allocations (#8346) * perf(scanner): reduce scan and checkpoint peak allocations * test(scanner): isolate partial commit recovery scenarios * fix(auth): return ExpiredToken for retained STS credentials (#8344) * fix(auth): return ExpiredToken for retained STS credentials Validate session identity independently of credential expiry in both IAM rejection and claims validation, and preserve typed S3 errors at the caller. Keep token binding, disabled-account precedence, and service-account behavior. Add exact expiry, IAM, and HTTP/XML regression coverage. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(ecstore): synchronize the metadata slowtail hedge regression Park the slow metadata task at a test-only fault checkpoint and release healthy reads after the fault is active. Assert the spare response and cancelled-task drain without imposing a 300 ms disk/scheduler deadline. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(ilm): synchronize copy compensation transition stages Observe automatic compensation reaching the mock tier PUT checkpoint before waiting for the final metadata commit. Assert forced enqueue failure, automatic backfill, one completed tier upload, and payload integrity. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --------- Co-authored-by: cxymds Co-authored-by: Chris Co-authored-by: heihutu Co-authored-by: zhi22915 * fix(ecstore): reject overlapping cleanup schedulers Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): reject overlapping cleanup schedulers Co-Authored-By: heihutu Co-Authored-By: zhi22915 * docs(ecstore): explain cleanup scheduler conflict Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(metrics): separate GET request and timeout outcomes (#8354) Co-authored-by: zhi22915 * docs: refresh security advisory lessons (#8350) * fix(heal): bound durable MRF rescheduling (#8351) Reuse stable live-owner proof anchors to avoid duplicate in-flight admission. Pace unverified completions and unavailable attempts with bounded jitter while retaining checkpoints, exact proofs, and generation isolation. Co-authored-by: heihutu Co-authored-by: zhi22915 * fix(rustfs): route recovery errors through S3 gateway Co-Authored-By: heihutu Co-Authored-By: zhi22915 * chore(deps): update workspace dependencies Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(io-metrics): use is_multiple_of for sampling Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): satisfy strict Clippy pagination checks Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(heal): avoid cloning Copy purge identity Co-Authored-By: heihutu Co-Authored-By: zhi22915 * add ignore .mimosa * fix(zip): adapt archive readers to zip 9 APIs Upgrade zip and its required dependencies alongside fallible archive-name handling in log ingestion, report bundles, bucket metadata import, and archive tests. Preserve existing skip and error propagation behavior. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): reuse quorum metadata for body GETs Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ci): align storage test stacks and peer reset classification Apply the existing bounded ECStore stack setup to the affected cross-crate storage fixtures. Validate conditional-only metadata publication separately from full body-layout cache reuse. Classify typed h2 shutdown resets as unreachable while retaining application, authentication, cancellation, and deadline distinctions. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * chore(deps): update workspace dependencies Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): preserve omitted-disk GET recovery Co-Authored-By: heihutu Co-Authored-By: zhi22915 * chore: integrate published workspace dependency updates Preserve the concurrent dependency refresh and retain the typed h2 reset classification and bounded storage-test stack setup. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(ecstore): preserve snapshot size budget after fallback move * fix(test): use one barrier for compensation COPY Let the forced-enqueue helper own the PUT barrier and compensation assertions. Waiting on an outer barrier overwritten by the helper always timed out after the upload had completed. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * chore(ecstore): apply current lint fixes Co-Authored-By: heihutu Co-Authored-By: zhi22915 * chore(deps): update sha2 to 0.11.1 Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: zhi22915 Co-authored-by: GatewayJ <835269233@qq.com> Co-authored-by: heihutu Co-authored-by: Chris Co-authored-by: hector <42570491+majinghe@users.noreply.github.com> Co-authored-by: cxymds --- .config/nextest.toml | 10 + .docker/observability/prometheus.yml | 1 + .gitignore | 1 + Cargo.lock | 393 ++--- Cargo.toml | 43 +- _typos.toml | 2 + crates/config/src/observability/mod.rs | 14 + .../src/namespace_lock_quorum_test.rs | 85 ++ crates/ecstore/Cargo.toml | 5 +- crates/ecstore/src/bucket/metadata_sys.rs | 77 + .../ecstore/src/bucket/quota/reservation.rs | 97 ++ crates/ecstore/src/cluster/rpc/client.rs | 38 +- .../ecstore/src/cluster/rpc/network_probe.rs | 47 + crates/ecstore/src/cluster/rpc/remote_disk.rs | 62 +- .../src/diagnostics/admin_server_info.rs | 25 +- crates/ecstore/src/disk/cleanup_runtime.rs | 1292 +++++++++++++++++ crates/ecstore/src/disk/disk_store.rs | 379 ++++- crates/ecstore/src/disk/fs.rs | 28 - crates/ecstore/src/disk/local.rs | 521 ++++--- crates/ecstore/src/disk/local/commit.rs | 92 +- .../ecstore/src/disk/local/delete_recovery.rs | 5 +- crates/ecstore/src/disk/mod.rs | 15 + crates/ecstore/src/disk/os.rs | 909 +++++++++++- .../src/set_disk/core/io_primitives.rs | 724 ++++++++- .../src/set_disk/core/metadata_read.rs | 1 + crates/ecstore/src/set_disk/mod.rs | 129 +- crates/ecstore/src/set_disk/ops/object.rs | 1193 +++++++++++++-- crates/ecstore/src/set_disk/read.rs | 298 +++- crates/ecstore/src/store/peer.rs | 28 + .../ecstore/tests/cleanup_isolation_test.rs | 174 +++ crates/ecstore/tests/fd_relative_io_test.rs | 283 ++++ .../tests/metadata_read_lock_reuse_test.rs | 95 ++ crates/ecstore/tests/storage_api.rs | 11 + crates/io-metrics/benches/metrics_pipeline.rs | 10 +- crates/io-metrics/src/lib.rs | 663 ++++++++- crates/log-analyzer/src/ingest/mod.rs | 8 +- crates/obs/src/telemetry/resource.rs | 17 +- deny.toml | 3 - docs/README.md | 3 + docs/operations/put-cleanup-isolation.md | 195 +++ rustfs/src/admin/handlers/bucket_meta.rs | 14 +- rustfs/src/app/object/get.rs | 68 +- rustfs/src/app/object/put.rs | 91 +- rustfs/src/connect/report_bundle.rs | 6 +- rustfs/src/server/hybrid.rs | 70 +- rustfs/src/server/layer.rs | 75 +- rustfs/src/startup_observability.rs | 104 +- rustfs/src/startup_runtime_sources.rs | 8 + rustfs/src/storage/rpc/node_service/disk.rs | 108 +- rustfs/src/storage/timeout_wrapper.rs | 8 +- rustfs/tests/connect_offline_bundle.rs | 2 +- rustfs/tests/connect_perf_drive.rs | 21 +- rustfs/tests/connect_perf_network.rs | 23 +- rustfs/tests/connect_top_net.rs | 9 +- 54 files changed, 7785 insertions(+), 798 deletions(-) create mode 100644 crates/ecstore/src/disk/cleanup_runtime.rs create mode 100644 crates/ecstore/tests/cleanup_isolation_test.rs create mode 100644 crates/ecstore/tests/fd_relative_io_test.rs create mode 100644 docs/operations/put-cleanup-isolation.md diff --git a/.config/nextest.toml b/.config/nextest.toml index 3881b9036..01a1141b5 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -88,6 +88,11 @@ threads-required = "num-test-threads" filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)' setup = 'ecstore-large-stack' +# These cross-crate real-storage fixtures need the same bounded stack as ECStore tests. +[[profile.default.scripts]] +filter = '(package(rustfs) & test(/^(admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))' +setup = 'ecstore-base-stack' + [[profile.default.scripts]] filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' setup = 'ecstore-base-stack' @@ -339,6 +344,11 @@ threads-required = "num-test-threads" filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)' setup = 'ecstore-large-stack' +# These cross-crate real-storage fixtures need the same bounded stack as ECStore tests. +[[profile.ci.scripts]] +filter = '(package(rustfs) & test(/^(admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))' +setup = 'ecstore-base-stack' + [[profile.ci.scripts]] filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' setup = 'ecstore-base-stack' diff --git a/.docker/observability/prometheus.yml b/.docker/observability/prometheus.yml index 1039e2972..4ac483e6e 100644 --- a/.docker/observability/prometheus.yml +++ b/.docker/observability/prometheus.yml @@ -61,6 +61,7 @@ scrape_configs: otlp: promote_resource_attributes: - service.instance.id + - process.pid - service.name - service.namespace - cloud.availability_zone diff --git a/.gitignore b/.gitignore index 8a26b9e94..1dad3b899 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,4 @@ worktrees/* # insta scratch files; the accepted .snap files ARE the assertions and are committed *.snap.new *.pending-snap +.mimosa \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index dc20abde7..3d4cf7f87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -791,7 +791,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -916,9 +916,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.18.1" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -927,9 +927,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" dependencies = [ "cc", "cmake", @@ -1020,7 +1020,7 @@ dependencies = [ "lru 0.18.5", "percent-encoding", "regex-lite", - "sha2 0.11.0", + "sha2 0.11.1", "tracing", "url", ] @@ -1120,7 +1120,7 @@ dependencies = [ "http 1.5.0", "p256 0.13.2", "percent-encoding", - "sha2 0.11.0", + "sha2 0.11.1", "subtle", "time", "tracing", @@ -1155,7 +1155,7 @@ dependencies = [ "md-5 0.11.0", "pin-project-lite", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "tracing", ] @@ -1518,7 +1518,7 @@ checksum = "144e573728da132683b9488acd528274c790e07fc06ff81ee29f9d8f8b1041e0" dependencies = [ "blowfish", "pbkdf2 0.13.0", - "sha2 0.11.0", + "sha2 0.11.1", ] [[package]] @@ -1561,19 +1561,13 @@ dependencies = [ [[package]] name = "bit-set" -version = "0.8.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" dependencies = [ - "bit-vec 0.8.0", + "bit-vec 0.10.1", ] -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - [[package]] name = "bit-vec" version = "0.9.1" @@ -1583,6 +1577,16 @@ dependencies = [ "serde", ] +[[package]] +name = "bit-vec" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" +dependencies = [ + "borsh", + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -1703,7 +1707,31 @@ dependencies = [ "prettyplease 0.3.0", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", +] + +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.9", ] [[package]] @@ -1950,9 +1978,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" dependencies = [ "find-msvc-tools", "jobserver", @@ -2107,7 +2135,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2199,8 +2227,8 @@ dependencies = [ "liblzma", "lz4", "memchr", - "zstd 0.14.0", - "zstd-safe 8.0.0", + "zstd 0.14.2", + "zstd-safe 8.1.0", ] [[package]] @@ -2304,9 +2332,12 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.4.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +checksum = "9eb6feb175df65d0a3ccd3e73484e940d0ac664e8034f0519fd49a93db3071d9" +dependencies = [ + "rustversion", +] [[package]] name = "convert_case" @@ -2764,7 +2795,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2808,7 +2839,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core 0.24.1", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -3310,7 +3341,7 @@ checksum = "14d6df9763c8fb968b2417d797aef4c55ac2e4a895aaef38c72c16a287aa4a40" dependencies = [ "datafusion-doc", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -3692,7 +3723,7 @@ checksum = "2e2b94854e8576378ccda7c8de8a66ed8b4e8acbd2c50ec3418ea6c8aaf4b567" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -3949,7 +3980,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3970,7 +4001,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -4053,7 +4084,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "suppaftp", "tempfile", "time", @@ -4066,7 +4097,7 @@ dependencies = [ "uuid", "walkdir", "zip", - "zstd 0.14.0", + "zstd 0.14.2", ] [[package]] @@ -4143,7 +4174,7 @@ dependencies = [ "ed25519 3.0.0", "rand_core 0.10.1", "serde", - "sha2 0.11.0", + "sha2 0.11.1", "signature 3.0.0", "subtle", "zeroize", @@ -4299,7 +4330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4581,7 +4612,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -4764,7 +4795,7 @@ dependencies = [ "rustls-pki-types", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "thiserror 2.0.21", "time", "tokio", @@ -4926,7 +4957,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "sha2 0.11.0", + "sha2 0.11.1", "thiserror 2.0.21", "tokio", "tokio-stream", @@ -5704,7 +5735,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" dependencies = [ "io-lifetimes 3.0.1", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -6021,9 +6052,9 @@ checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" [[package]] name = "lapin" -version = "4.12.0" +version = "4.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b86b2a46f42a1d565c7ba327e03725708c1ff212baa4c271cec291e265ae542" +checksum = "1f6674e18d68e810f66ea962f8080f1ded75f813c1cd3fae9ed53bab760dfcfd" dependencies = [ "amq-protocol", "async-rs", @@ -6057,7 +6088,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -6349,11 +6380,11 @@ dependencies = [ [[package]] name = "lzma-rust2" -version = "0.16.5" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca93e534d1142d1d0dcca6d25fe302508a5dfb40b302802904577725ea0b695b" +checksum = "fde178a3caf126c440fa15628147772d8ee590a39477d711353fbe2e58a73a5b" dependencies = [ - "sha2 0.11.0", + "sha2 0.11.1", ] [[package]] @@ -6536,7 +6567,7 @@ dependencies = [ "metrique-core", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -6599,7 +6630,7 @@ dependencies = [ "metrique-core", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure 0.14.0", ] @@ -6963,7 +6994,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7339,9 +7370,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" +checksum = "8c1f1a6ac9c875a4941e4c7c085b61e8f4daa326de03b4701768c3491a6ad9ee" dependencies = [ "futures-core", "futures-sink", @@ -7353,9 +7384,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b304e8d8f11326ba366a99ca5c7aaf7cd397b67268a332437175aeb42a40db" +checksum = "1f01f1c53c46ce80e5fd8cb7a646f13766fd8f315a8a5b75b9bad59a08d2c5c7" dependencies = [ "opentelemetry", "tracing", @@ -7366,9 +7397,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300" +checksum = "302249bd8ee804789faaa7b5b2cbf79bc61cb864714079c644e3828af5b7e811" dependencies = [ "async-trait", "bytes", @@ -7379,9 +7410,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344" +checksum = "0756af24841e367128ef31c831999937b8b5c5197d47914590d8f4e2e88c2e1a" dependencies = [ "flate2", "http 1.5.0", @@ -7397,9 +7428,9 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" +checksum = "d576f5dddc3299341bcd85a04be17b2564844c0c9563214f02d85f2524d1afae" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -7408,15 +7439,15 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d288b1e1bc3590052fbb8f0f635ef248dd9022cf1fbaeb6582b7ad6081a17b87" +checksum = "7a3cf639fefd3e852869114f9925b578da6dd8e99281a2089bb0e773ead46962" [[package]] name = "opentelemetry-stdout" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f71803f481d3f5ccbe477edb42a78f473613712b2327835911a9ea0ab069824" +checksum = "2afbc6661934787ce79bef1e35d0ffe9d62e945f07f664dc1283ce83292b908a" dependencies = [ "chrono", "opentelemetry", @@ -7425,9 +7456,9 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" +checksum = "372a912db2974d1990207006fd0e3aa0ef3fd4ad3be43c65cfaadf2206866061" dependencies = [ "futures-channel", "futures-executor", @@ -7546,7 +7577,7 @@ dependencies = [ "elliptic-curve 0.14.1", "primefield", "primeorder 0.14.0", - "sha2 0.11.0", + "sha2 0.11.1", ] [[package]] @@ -7572,7 +7603,7 @@ dependencies = [ "fiat-crypto 0.3.0", "primefield", "primeorder 0.14.0", - "sha2 0.11.0", + "sha2 0.11.1", ] [[package]] @@ -7586,7 +7617,7 @@ dependencies = [ "elliptic-curve 0.14.1", "primefield", "primeorder 0.14.0", - "sha2 0.11.0", + "sha2 0.11.1", ] [[package]] @@ -7612,7 +7643,7 @@ dependencies = [ "futures", "log", "rand 0.10.3", - "sha2 0.11.0", + "sha2 0.11.1", "thiserror 2.0.21", "tokio", "windows", @@ -8001,7 +8032,7 @@ dependencies = [ "pbkdf2 0.13.0", "rand_core 0.10.1", "scrypt 0.12.0", - "sha2 0.11.0", + "sha2 0.11.1", "spki 0.8.1", ] @@ -8119,7 +8150,7 @@ dependencies = [ "md-5 0.11.0", "memchr", "rand 0.10.3", - "sha2 0.11.0", + "sha2 0.11.1", "stringprep", ] @@ -8204,7 +8235,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" dependencies = [ "proc-macro2", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -8301,16 +8332,17 @@ dependencies = [ [[package]] name = "proptest" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" dependencies = [ "bit-set", - "bit-vec 0.8.0", + "bit-vec 0.10.1", "bitflags 2.13.2", + "chacha20", + "core_detect", "num-traits", - "rand 0.9.5", - "rand_chacha 0.9.0", + "rand 0.10.3", "rand_xorshift", "regex-syntax", "rusty-fork", @@ -8345,7 +8377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -8365,7 +8397,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "petgraph 0.8.3", @@ -8386,7 +8418,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -8399,7 +8431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -8726,7 +8758,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8848,11 +8880,11 @@ dependencies = [ [[package]] name = "rand_xorshift" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" dependencies = [ - "rand_core 0.9.5", + "rand_core 0.10.1", ] [[package]] @@ -9039,7 +9071,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -9257,7 +9289,7 @@ dependencies = [ "pkcs1 0.8.0-rc.4", "pkcs8 0.11.0", "rand_core 0.10.1", - "sha2 0.11.0", + "sha2 0.11.1", "signature 3.0.0", "spki 0.8.1", "zeroize", @@ -9372,7 +9404,7 @@ dependencies = [ "scrypt 0.12.0", "sec1 0.8.1", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "sha3 0.12.0", "signature 3.0.0", "spki 0.8.1", @@ -9462,7 +9494,7 @@ version = "8.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d84e8ba78bd384263e5922f084cbe1b081c3b7e69add59c8fb097b879ba968a" dependencies = [ - "sha2 0.11.0", + "sha2 0.11.1", "walkdir", ] @@ -9615,7 +9647,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "serial_test", - "sha2 0.11.0", + "sha2 0.11.1", "shadow-rs", "snap", "socket2", @@ -9643,7 +9675,7 @@ dependencies = [ "xxhash-rust", "zeroize", "zip", - "zstd 0.14.0", + "zstd 0.14.2", ] [[package]] @@ -9680,7 +9712,7 @@ dependencies = [ "md-5 0.11.0", "pretty_assertions", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "xxhash-rust", ] @@ -9730,7 +9762,7 @@ dependencies = [ "object 0.40.0", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "symbolic-demangle", ] @@ -9744,7 +9776,7 @@ dependencies = [ "rand 0.10.3", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "time", ] @@ -9763,7 +9795,7 @@ dependencies = [ "rsa 0.10.0-rc.18", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "test-case", "thiserror 2.0.21", "time", @@ -9804,6 +9836,7 @@ dependencies = [ "futures-util", "google-cloud-auth", "google-cloud-storage", + "h2", "hex-simd", "hmac 0.13.0", "hostname", @@ -9871,7 +9904,7 @@ dependencies = [ "serde", "serde_json", "serial_test", - "sha2 0.11.0", + "sha2 0.11.1", "shadow-rs", "smallvec", "temp-env", @@ -9940,7 +9973,7 @@ dependencies = [ "rustfs-utils", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tempfile", "thiserror 2.0.21", "time", @@ -9980,7 +10013,7 @@ dependencies = [ "serde", "serde_json", "serial_test", - "sha2 0.11.0", + "sha2 0.11.1", "temp-env", "tempfile", "thiserror 2.0.21", @@ -10039,7 +10072,7 @@ dependencies = [ "serde_with", "serial_test", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "subtle", "temp-env", "tempfile", @@ -10096,7 +10129,7 @@ dependencies = [ "pbkdf2 0.13.0", "rand 0.10.3", "rustls", - "sha2 0.11.0", + "sha2 0.11.1", "socket2", "thiserror 2.0.21", "tracing", @@ -10120,7 +10153,7 @@ dependencies = [ "rand 0.10.3", "rustfs-kafka", "rustls", - "sha2 0.11.0", + "sha2 0.11.1", "tokio", "tokio-rustls", "tracing", @@ -10190,7 +10223,7 @@ dependencies = [ "serde", "serde_ignored", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "subtle", "temp-env", "tempfile", @@ -10267,13 +10300,13 @@ dependencies = [ "regex", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tar", "tempfile", "thiserror 2.0.21", "walkdir", "zip", - "zstd 0.14.0", + "zstd 0.14.2", ] [[package]] @@ -10445,7 +10478,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "url", - "zstd 0.14.0", + "zstd 0.14.2", ] [[package]] @@ -10524,7 +10557,7 @@ dependencies = [ "serde", "serde_json", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "socket2", "subtle", "tempfile", @@ -10618,7 +10651,7 @@ dependencies = [ "serde", "serde_json", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "temp-env", "thiserror 2.0.21", "tokio", @@ -10646,7 +10679,7 @@ dependencies = [ "rustfs-utils", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tokio", "walkdir", ] @@ -10683,7 +10716,7 @@ dependencies = [ "s3s", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "thiserror 2.0.21", "time", "tokio", @@ -10803,7 +10836,7 @@ dependencies = [ "serde", "serde_json", "serial_test", - "sha2 0.11.0", + "sha2 0.11.1", "temp-env", "tempfile", "thiserror 2.0.21", @@ -10909,7 +10942,7 @@ dependencies = [ "s3s", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "snap", "sysinfo", "tempfile", @@ -10955,7 +10988,7 @@ dependencies = [ "rustls-pki-types", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tempfile", "thiserror 2.0.21", "tokio", @@ -11046,7 +11079,7 @@ dependencies = [ "rustix", "serde", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "siphasher", "snap", "temp-env", @@ -11056,7 +11089,7 @@ dependencies = [ "transform-stream", "url", "windows", - "zstd 0.14.0", + "zstd 0.14.2", ] [[package]] @@ -11071,7 +11104,7 @@ dependencies = [ "rustfs-tokio-tar", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tar-codec", "tar-framing", "thiserror 2.0.21", @@ -11131,7 +11164,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11214,7 +11247,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11308,7 +11341,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "smallvec", "std-next", "subtle", @@ -11372,7 +11405,7 @@ dependencies = [ "jiff", "nom 8.0.0", "serde", - "sha2 0.11.0", + "sha2 0.11.1", "smallvec", "std-next", "subtle", @@ -11437,9 +11470,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" dependencies = [ "dyn-clone", "ref-cast", @@ -11473,7 +11506,7 @@ dependencies = [ "cfg-if", "pbkdf2 0.13.0", "salsa20 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", ] [[package]] @@ -11596,7 +11629,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -11611,10 +11644,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ + "foldhash 0.2.0", "indexmap 2.14.2", "itoa", "memchr", @@ -11660,7 +11694,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -11689,7 +11723,7 @@ dependencies = [ "indexmap 2.14.2", "jiff", "schemars 0.9.0", - "schemars 1.2.2", + "schemars 1.2.3", "serde_core", "serde_json", "serde_with_macros", @@ -11705,7 +11739,7 @@ dependencies = [ "darling 0.24.1", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -11740,7 +11774,7 @@ checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -11784,9 +11818,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", "cpufeatures 0.3.1", @@ -11987,9 +12021,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.16.2" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" dependencies = [ "serde", ] @@ -12156,7 +12190,7 @@ dependencies = [ "sec1 0.8.1", "serde", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "signature 3.0.0", "ssh-cipher", "ssh-encoding", @@ -12259,9 +12293,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "suppaftp" -version = "12.1.1" +version = "12.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b161d71217da0cc91bd09c8f98ac5ad2a5e5b2735c6dfe1c6cb08b4471612f35" +checksum = "a560e9f42cc5bd4a92295eca3c7140d51eb685d688087f547f27e6426fbe97da" dependencies = [ "async-trait", "chrono", @@ -12327,9 +12361,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.6" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -12376,7 +12410,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -12485,10 +12519,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -12518,7 +12552,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -12529,7 +12563,7 @@ checksum = "f4ce35ece947bccac166e1ded639133827a953f69a48ba134d17f91108333082" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "test-case-core", ] @@ -12570,7 +12604,7 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -12707,7 +12741,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -12743,7 +12777,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c2ad44aa0ae96db89c4742212ed41645b2f597311ff6e1945542a4d9fadc2fb" dependencies = [ "rustls", - "sha2 0.11.0", + "sha2 0.11.1", "tokio", "tokio-postgres", "tokio-rustls", @@ -12785,9 +12819,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" dependencies = [ "bytes", "futures-core", @@ -12823,18 +12857,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.1.1+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.25.15+spec-1.1.0" +version = "0.25.17+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" dependencies = [ "indexmap 2.14.2", "toml_datetime", @@ -12844,9 +12878,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.3+spec-1.1.0" +version = "1.1.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" dependencies = [ "winnow", ] @@ -13223,9 +13257,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "unicode-normalization" @@ -13315,9 +13349,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.27.0" +version = "1.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -13468,7 +13502,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "wasm-bindgen-shared", ] @@ -13582,7 +13616,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -14043,24 +14077,24 @@ checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure 0.14.0", ] [[package]] name = "zerocopy" -version = "0.8.60" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6400fc4bb7426f6faac3fb07566122bb18622e3022bd2c36426a92ed489b1dbe" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.60" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99c27cc7525bad4df59a45e073c45b2c2aa820efb588234cc23a547ec69a34db" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" dependencies = [ "proc-macro2", "quote", @@ -14084,7 +14118,7 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure 0.14.0", ] @@ -14138,14 +14172,14 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "zip" -version = "8.6.0" +version = "9.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +checksum = "9e2fc56a6707efb066c40aead2409ebb1224e6dde24a074e9fbc78ced500507b" dependencies = [ "aes 0.9.3", "bzip2", @@ -14158,6 +14192,7 @@ dependencies = [ "indexmap 2.14.2", "lzma-rust2", "memchr", + "nix 0.31.3", "pbkdf2 0.13.0", "ppmd-rust", "sha1 0.11.0", @@ -14165,7 +14200,7 @@ dependencies = [ "typed-path", "zeroize", "zopfli", - "zstd 0.13.3", + "zstd 0.14.2", ] [[package]] @@ -14182,9 +14217,9 @@ checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zopfli" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +checksum = "aaf7fc5d30c28483d93805c4a5e12b05bbb52407fa67c5f8bd552374cd01fb11" dependencies = [ "bumpalo", "crc32fast", @@ -14203,11 +14238,11 @@ dependencies = [ [[package]] name = "zstd" -version = "0.14.0" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e" +checksum = "057cfd910cfac363a0ada849592624b4c9ff2e10bef504c3433810d78ed96f93" dependencies = [ - "zstd-safe 8.0.0", + "zstd-safe 8.1.0", ] [[package]] @@ -14221,18 +14256,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "8.0.0" +version = "8.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab" +checksum = "bdd44c6a7284e91f3717755b24315a302edd9153a01f753c3cba3d765e8eafac" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.1.0+zstd.1.5.7" +version = "2.1.1+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +checksum = "aeec9eaf2dffbbd09201e23bd0ffcbaa33bb8e9266a10734fd7ed90a85eca078" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 6a5791efb..a0b447ac7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -156,8 +156,9 @@ futures-lite = "2.6.1" futures-util = "0.3.34" pollster = "1.0.1" pulsar = { default-features = false, version = "6.9.0" } -lapin = { default-features = false, version = "4.12.0" } +lapin = { default-features = false, version = "4.12.1" } hyper = { version = "1.12.0" } +h2 = { version = "0.4.20" } hyper-rustls = { default-features = false, version = "0.27.10" } hyper-util = { version = "0.1.21" } http = "1.5.0" @@ -172,7 +173,7 @@ tokio = { version = "1.53.2" } tokio-rustls = { default-features = false, version = "0.26.6" } tokio-stream = { version = "0.1.19" } tokio-test = "0.4.6" -tokio-util = { version = "0.7.19" } +tokio-util = { version = "0.7.20" } tonic = { version = "0.14.6" } tonic-prost = { version = "0.14.6" } tonic-prost-build = { version = "0.14.6" } @@ -180,7 +181,7 @@ tower = { version = "0.5.3" } tower-http = { version = "0.7.1" } h3 = "0.0.8" h3-quinn = "0.0.10" -quinn = { version = "0.11.11", default-features = false, features = [ +quinn = { version = "0.11.12", default-features = false, features = [ "futures-io", "runtime-tokio", "rustls-aws-lc-rs", @@ -200,7 +201,7 @@ rmp-serde = { version = "1.3.1" } serde = { version = "1.0.229" } serde_ignored = { version = "0.1" } serde_with = { version = "3", default-features = false, features = ["macros", "std"] } -serde_json = { version = "1.0.151" } +serde_json = { version = "1.0.154" } serde_urlencoded = "0.7.1" # Cryptography and Security @@ -224,7 +225,7 @@ rustls-native-certs = "0.8" rustls-pki-types = "1.15.1" x509-parser = "0.18.1" sha1 = "0.11.0" -sha2 = "0.11.0" +sha2 = "0.11.1" subtle = "2.6" zeroize = { version = "1.9.1" } proptest = "1" @@ -268,7 +269,7 @@ crossbeam-queue = "0.3.14" crossbeam-channel = "0.5.17" crossbeam-deque = "0.8.8" crossbeam-utils = "0.8.23" -datafusion = { default-features = false, version = "55.1.0" } +datafusion = { default-features = false, version = "55.2.0" } derive_builder = "0.20.2" ed25519-dalek = "3.0.0" enumset = "1.1.14" @@ -319,14 +320,14 @@ rumqttc = { package = "rumqttc-next", version = "0.34.0" } redis = { version = "1.7.1" } rustify = { version = "0.7", default-features = false } rustix = { version = "1.1.5" } -rust-embed = { version = "8.12.0" } +rust-embed = { version = "8.13.0" } rustc-hash = { version = "2.1.3" } s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] } s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" } serial_test = "4.0.1" shadow-rs = { default-features = false, version = "2.0.0" } siphasher = "1.0.4" -smallvec = { version = "1.16.2" } +smallvec = { version = "1.16.3" } compact_str = "0.10.0" snap = "1.1.2" starshard = { version = "2.4.0" } @@ -340,12 +341,12 @@ tracing = { version = "0.1.44" } tracing-appender = "0.2.5" tracing-core = "0.1.36" tracing-error = "0.2.1" -tracing-opentelemetry = { version = "0.34" } +tracing-opentelemetry = { version = "0.34.0" } tracing-subscriber = { version = "0.3.23" } transform-stream = "0.3.1" url = "2.5.8" urlencoding = "2.1.3" -uuid = { version = "1.27.0" } +uuid = { version = "1.28.0" } vaultrs = { version = "0.8.0" } tar = "0.4.46" walkdir = "2.5.0" @@ -353,26 +354,26 @@ winapi-util = "0.1.11" windows = { version = "0.62.2" } windows-sys = "0.61.2" xxhash-rust = { version = "0.8.19" } -zip = "8.6.0" -zstd = "0.14.0" +zip = "9.0.2" +zstd = "0.14.2" # Observability and Metrics metrics = "0.24.6" metrics-util = "0.20" dial9 = { version = "0.5.3", default-features = false } -opentelemetry = { version = "0.33.0" } -opentelemetry-appender-tracing = { version = "0.33.0" } -opentelemetry-otlp = { version = "0.33.0" } -opentelemetry-proto = { version = "0.33.0", default-features = false, features = ["metrics", "gen-tonic-messages"] } -opentelemetry_sdk = { version = "0.33.0" } -opentelemetry-semantic-conventions = { version = "0.33.0" } -opentelemetry-stdout = { version = "0.33.0" } +opentelemetry = { version = "0.33.1" } +opentelemetry-appender-tracing = { version = "0.33.1" } +opentelemetry-otlp = { version = "0.33.1" } +opentelemetry-proto = { version = "0.33.1", default-features = false, features = ["metrics", "gen-tonic-messages"] } +opentelemetry_sdk = { version = "0.33.1" } +opentelemetry-semantic-conventions = { version = "0.33.1" } +opentelemetry-stdout = { version = "0.33.1" } pyroscope = { version = "2.1.1" } # FTP and SFTP libunftp = { version = "0.23.1" } unftp-core = "0.1.0" -suppaftp = { version = "12.1.1" } +suppaftp = { version = "12.2.0" } rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] } russh = { version = "0.64.1" } russh-sftp = "3.0.1" @@ -383,7 +384,7 @@ dav-server = "0.11.0" # Performance Analysis and Memory Profiling rustfs-mimalloc = { version = "0.5.6" } # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved. -hotpath = { version = "0.28.5", default-features = false } +hotpath = { version = "0.28.6", default-features = false } # Snapshot testing for output format regression detection insta = { version = "1.49" } diff --git a/_typos.toml b/_typos.toml index 7ff843137..e11c3c036 100644 --- a/_typos.toml +++ b/_typos.toml @@ -51,6 +51,8 @@ unparseable = "unparseable" # Disaster-recovery objectives: recovery time and recovery point. RTO = "RTO" rto = "rto" +# rustix OFlags variant (POSIX O_WRONLY equivalent). +WRONLY = "WRONLY" [files] extend-exclude = [] diff --git a/crates/config/src/observability/mod.rs b/crates/config/src/observability/mod.rs index 9a1acc6c4..ca001fc8b 100644 --- a/crates/config/src/observability/mod.rs +++ b/crates/config/src/observability/mod.rs @@ -51,6 +51,11 @@ pub const ENV_OBS_PROFILING_EXPORT_ENABLED: &str = "RUSTFS_OBS_PROFILING_EXPORT_ /// PUT records multiple timers and histograms when attribution is active. pub const ENV_OBS_PUT_STAGE_METRICS_ENABLED: &str = "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED"; +/// Enables sampled in-process GetObject stage summaries without OTLP histograms. +pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED"; +pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE"; +pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS"; + pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL"; pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED"; pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY"; @@ -159,6 +164,15 @@ mod tests { assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED"); assert_eq!(ENV_OBS_PROFILING_EXPORT_ENABLED, "RUSTFS_OBS_PROFILING_EXPORT_ENABLED"); assert_eq!(ENV_OBS_PUT_STAGE_METRICS_ENABLED, "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED"); + assert_eq!(ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED, "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED"); + assert_eq!( + ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE, + "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE" + ); + assert_eq!( + ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS, + "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS" + ); // Test log cleanup related env keys assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES"); assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES"); diff --git a/crates/e2e_test/src/namespace_lock_quorum_test.rs b/crates/e2e_test/src/namespace_lock_quorum_test.rs index 1edf47c2b..7f9e8c760 100644 --- a/crates/e2e_test/src/namespace_lock_quorum_test.rs +++ b/crates/e2e_test/src/namespace_lock_quorum_test.rs @@ -195,6 +195,91 @@ async fn wait_for_quorum_read_admission(clients: &[Client], bucket: &str) -> Tes } } +#[tokio::test] +async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> TestResult { + crate::common::init_logging(); + let mut cluster = RustFSTestClusterEnvironment::new(4).await?; + cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2"); + cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false"); + cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false"); + cluster.set_env("RUSTFS_GET_OBJECT_METADATA_CACHE_DISTRIBUTED_ENABLE", "false"); + cluster.start().await?; + let clients = cluster + .create_all_clients()? + .into_iter() + .map(|client| { + Client::from_conf( + client + .config() + .to_builder() + .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1)) + .build(), + ) + }) + .collect::>(); + let bucket = "cold-degraded-mutations"; + clients[0].create_bucket().bucket(bucket).send().await?; + // No object mutation may warm a process-wide configuration probe before + // the missing peer is introduced. Three shards still satisfy EC:2 writes. + cluster.stop_node(3)?; + let admission_deadline = Instant::now() + Duration::from_secs(30); + for client in &clients[..3] { + loop { + match client + .get_object() + .bucket(bucket) + .key("never-created-readiness-probe") + .send() + .await + { + Err(error) if error.as_service_error().and_then(|error| error.meta().code()) == Some("NoSuchKey") => break, + Err(error) + if error.raw_response().is_some_and(|response| response.status().as_u16() == 503) + && Instant::now() < admission_deadline => + { + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err(error) => return Err(error.into()), + Ok(_) => panic!("a cold readiness probe must not create an object"), + } + } + } + let body = b"first write while one peer is offline"; + clients[0] + .put_object() + .bucket(bucket) + .key("object") + .body(Bytes::from_static(body).into()) + .send() + .await?; + for client in &clients[..3] { + assert_quorum_object_body(client, bucket, "object", body).await?; + } + // This coordinator has not previously committed an object mutation either. + clients[1].delete_object().bucket(bucket).key("object").send().await?; + let error = clients[2] + .get_object() + .bucket(bucket) + .key("object") + .send() + .await + .expect_err("acknowledged deletion must be visible at another entrypoint"); + assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("NoSuchKey")); + + cluster.stop_node(2)?; + let error = clients[0] + .put_object() + .bucket(bucket) + .key("below-quorum") + .body(Bytes::from_static(body).into()) + .send() + .await + .expect_err("removing the cache gate must not weaken storage write quorum"); + assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable")); + cluster.stop(); + Ok(()) +} + #[tokio::test] async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult { crate::common::init_logging(); diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 5a41e01f4..2eee87cea 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -183,6 +183,7 @@ sha2 = { workspace = true } hex-simd = { workspace = true } tempfile.workspace = true hyper = { workspace = true, features = ["http2", "http1", "server"] } +h2 = { workspace = true } hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] } hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs"] } hostname.workspace = true @@ -200,8 +201,8 @@ md-5.workspace = true memmap2 = { workspace = true } libc.workspace = true # "process" adds getrlimit for the io_uring fd-cache RLIMIT_NOFILE gate -# (backlog#1178); "fs" comes from the workspace default. -rustix = { workspace = true, features = ["process", "fs"] } +# (backlog#1178); "thread" provides cleanup affinity; "fs" is a workspace default. +rustix = { workspace = true, features = ["process", "fs", "thread"] } rustfs-madmin.workspace = true reqwest = { workspace = true } aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 1dc3e43e1..187356d4f 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -178,6 +178,13 @@ pub(crate) fn object_lock_config_state_from_authoritative_metadata(bm: &BucketMe Ok(ObjectLockConfigState::ConfirmedAbsent) } +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct CachedQuotaMetadataSnapshot { + pub(crate) quota: Option, + pub(crate) bucket_incarnation: Uuid, + pub(crate) quota_revision: OffsetDateTime, +} + /// Convert the persisted serving-layer configuration into the storage-level /// [`DefaultRetention`](crate::bucket::object_lock::types::DefaultRetention) /// the WORM evaluation code consumes (rustfs/backlog#1842). A rule without a @@ -1450,6 +1457,9 @@ fn quota_config_and_incarnation_from_authority( BucketMetadataAuthority::Authoritative(metadata) if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() => { + if !metadata.quota_config_json.is_empty() && metadata.quota_config.is_none() { + return Err(Error::other("persisted bucket quota configuration is invalid")); + } Ok(( metadata.quota_config.clone(), metadata.bucket_incarnation_id, @@ -1464,6 +1474,15 @@ fn quota_config_and_incarnation_from_authority( } } +pub(crate) async fn get_cached_quota_config_and_incarnation_in( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, +) -> Result> { + let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?; + let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone(); + bucket_meta_sys.cached_quota_config_and_incarnation(bucket).await +} + pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> { let bucket_meta_sys_lock = get_bucket_metadata_sys()?; let bucket_meta_sys = bucket_meta_sys_lock.read().await; @@ -2979,6 +2998,27 @@ impl BucketMetadataSys { } } + async fn cached_quota_config_and_incarnation(&self, bucket: &str) -> Result> { + let metadata = { + let metadata_map = self.metadata_map.read().await; + metadata_map.get(bucket).cloned() + }; + let Some(metadata) = metadata else { + return Ok(None); + }; + if !metadata.bucket_incarnation_sidecar || metadata.bucket_incarnation_id.is_nil() { + return Ok(None); + } + if !metadata.quota_config_json.is_empty() && metadata.quota_config.is_none() { + return Err(Error::other("persisted bucket quota configuration is invalid")); + } + Ok(Some(CachedQuotaMetadataSnapshot { + quota: metadata.quota_config.clone(), + bucket_incarnation: metadata.bucket_incarnation_id, + quota_revision: metadata.quota_config_updated_at, + })) + } + pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> { let (bm, _) = self.get_config(bucket).await?; @@ -4883,6 +4923,43 @@ mod tests { ); } + #[tokio::test] + #[serial] + async fn quota_admission_rejects_stale_cached_no_quota() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let sys = bucket_metadata_sys_of(&store.ctx).expect("scoped metadata system"); + let bucket = "stale-no-quota-admission"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let (stale, _) = sys.read().await.get_config(bucket).await.expect("cache initial metadata"); + store + .update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec()) + .await + .expect("publish quota to authoritative storage"); + // Model a peer that has not received the newer configuration snapshot. + sys.read().await.set(bucket.to_owned(), stale).await; + assert!( + get_cached_quota_config_and_incarnation_in(&store.ctx, bucket) + .await + .expect("read stale cache") + .expect("cached bucket identity") + .quota + .is_none() + ); + + temp_env::async_with_vars([("RUSTFS_QUOTA_BEGIN_SAFE_NO_QUOTA_CACHE_FAST_PATH", Some("true"))], async { + let error = crate::bucket::quota::reservation::begin(&store.ctx, bucket, "object", &ObjectOptions::default(), 0, 0) + .await + .err() + .expect("persisted quota must reject admission without its snapshot"); + assert!(matches!(error, Error::PartMissingOrCorrupt), "{error:?}"); + }) + .await; + } + #[tokio::test] #[serial] async fn peer_reload_clears_negative_cache_after_remote_bucket_creation() { diff --git a/crates/ecstore/src/bucket/quota/reservation.rs b/crates/ecstore/src/bucket/quota/reservation.rs index 59c51c301..b269813f8 100644 --- a/crates/ecstore/src/bucket/quota/reservation.rs +++ b/crates/ecstore/src/bucket/quota/reservation.rs @@ -850,6 +850,11 @@ pub(crate) async fn begin( let snapshot_admission = opts.quota_admission; let data_movement = opts.data_movement; if crate::bucket::utils::is_meta_bucketname(bucket) { + let meta_bucket_fast_path_started = rustfs_io_metrics::put_stage_timer(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH, + meta_bucket_fast_path_started, + ); return Ok(QuotaContext { store: None, bucket: bucket.to_string(), @@ -906,9 +911,63 @@ pub(crate) async fn begin( }); } + let metadata_lock_started = rustfs_io_metrics::put_stage_timer(); let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK, + metadata_lock_started, + ); + let cache_lookup_started = rustfs_io_metrics::put_stage_timer(); + let cached_quota_snapshot = if rustfs_io_metrics::put_stage_metrics_enabled() { + match metadata_sys::get_cached_quota_config_and_incarnation_in(ctx, bucket).await { + Ok(Some(snapshot)) => { + rustfs_io_metrics::record_put_object_quota_cache_decision( + "candidate", + quota_cache_reason(snapshot.quota.as_ref()), + ); + Some(snapshot) + } + Ok(None) => { + rustfs_io_metrics::record_put_object_quota_cache_decision("miss", "no_authoritative_cache"); + None + } + Err(_) => { + rustfs_io_metrics::record_put_object_quota_cache_decision("miss", "invalid_cached_quota"); + None + } + } + } else { + None + }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP, + cache_lookup_started, + ); + let config_read_started = rustfs_io_metrics::put_stage_timer(); let (quota, bucket_incarnation, quota_revision) = metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(ctx, bucket, opts, &metadata_guard).await?; + if let Some(cached) = cached_quota_snapshot.as_ref() { + let reason = quota_cache_reason(quota.as_ref()); + if cached.quota == quota && cached.bucket_incarnation == bucket_incarnation && cached.quota_revision == quota_revision { + rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_match", reason); + } else { + rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch", reason); + if cached.quota != quota { + rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "quota"); + } + if cached.bucket_incarnation != bucket_incarnation { + rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "bucket_incarnation"); + } + if cached.quota_revision != quota_revision { + rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "quota_revision"); + } + } + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ, + config_read_started, + ); + let lock_lost_check_started = rustfs_io_metrics::put_stage_timer(); if metadata_guard.is_lock_lost() { return Err(StorageError::NamespaceLockQuorumUnavailable { mode: "quota_config", @@ -918,6 +977,11 @@ pub(crate) async fn begin( achieved: 0, }); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK, + lock_lost_check_started, + ); + let policy_evaluate_started = rustfs_io_metrics::put_stage_timer(); if quota .as_ref() .is_some_and(|quota| quota.has_unsupported_reservation_protocol()) @@ -926,6 +990,11 @@ pub(crate) async fn begin( return Err(StorageError::PartMissingOrCorrupt); } let durable_quota = quota.as_ref().filter(|quota| quota.uses_durable_reservations()); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE, + policy_evaluate_started, + ); + let capability_proof_started = rustfs_io_metrics::put_stage_timer(); let capability_proof = if durable_quota.is_some() { Some( crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof() @@ -934,7 +1003,12 @@ pub(crate) async fn begin( } else { None }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF, + capability_proof_started, + ); let durable_quota_limit = durable_quota.and_then(|quota| quota.quota); + let snapshot_admission_started = rustfs_io_metrics::put_stage_timer(); let reservation_protocol = durable_quota.and_then(|quota| quota.reservation_protocol); let snapshot_admission = match quota.as_ref().filter(|quota| !quota.uses_durable_reservations()) { Some(quota) => match (quota.quota, snapshot_admission) { @@ -954,6 +1028,10 @@ pub(crate) async fn begin( }, None => None, }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION, + snapshot_admission_started, + ); let legacy_data_movement = durable_quota_limit.is_none() && quota.as_ref().and_then(|quota| quota.quota).is_some() && snapshot_admission.is_none() @@ -965,11 +1043,21 @@ pub(crate) async fn begin( .then(|| quota.as_ref().and_then(|quota| quota.quota)) .flatten() }); + let object_store_lookup_started = rustfs_io_metrics::put_stage_timer(); let store = if durable_quota_limit.is_some() { Some(metadata_sys::object_store_in(ctx).await?) } else { None }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP, + object_store_lookup_started, + ); + let context_build_started = rustfs_io_metrics::put_stage_timer(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD, + context_build_started, + ); Ok(QuotaContext { store, bucket: bucket.to_string(), @@ -988,6 +1076,15 @@ pub(crate) async fn begin( }) } +fn quota_cache_reason(quota: Option<&crate::bucket::quota::BucketQuota>) -> &'static str { + match quota { + None => "no_quota", + Some(quota) if quota.uses_durable_reservations() => "durable_quota", + Some(quota) if quota.quota.is_some() => "snapshot_quota", + Some(_) => "quota_without_limit", + } +} + fn quota_capability_error(bucket: &str, object: &str) -> StorageError { StorageError::NamespaceLockQuorumUnavailable { mode: "quota_capability", diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index ca781d6be..c8170b613 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -353,6 +353,13 @@ fn apply_peer_replay_response( } } +fn rename_data_grpc_stage(path: &str) -> bool { + matches!( + path, + "/node_service.NodeService/RenameData" | "/node_service.NodeService/RenameDataAtIncarnation" + ) +} + impl Service> for ReplayScopeChannel where S: Service, Response = HttpResponse>, @@ -370,6 +377,12 @@ where } fn call(&mut self, mut request: HttpRequest) -> Self::Future { + let observe_rename_data = rustfs_io_metrics::put_stage_metrics_enabled() && rename_data_grpc_stage(request.uri().path()); + let request_scope_started = if observe_rename_data { + rustfs_io_metrics::put_stage_timer() + } else { + None + }; let authenticated = self.audience.as_ref().is_some_and(|_| { request .headers() @@ -404,11 +417,30 @@ where } } } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE, + request_scope_started, + ); let audience = self.audience.clone(); let future = self.inner.call(request); Box::pin(async move { - let response = future.await?; + let response = if observe_rename_data { + rustfs_io_metrics::observe_put_stage_future( + future, + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL, + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT, + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY, + ) + .await? + } else { + future.await? + }; + let replay_response_started = if observe_rename_data { + rustfs_io_metrics::put_stage_timer() + } else { + None + }; if let (Some(audience), Some(challenge)) = (audience, challenge) { let response_state = verify_tonic_peer_replay_capabilities_response(&audience, challenge, response.headers()); if let Err(error) = &response_state @@ -426,6 +458,10 @@ where } apply_peer_replay_response(audience, sent_state, response_state); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE, + replay_response_started, + ); Ok(response) }) } diff --git a/crates/ecstore/src/cluster/rpc/network_probe.rs b/crates/ecstore/src/cluster/rpc/network_probe.rs index 765cc7322..db2221cfb 100644 --- a/crates/ecstore/src/cluster/rpc/network_probe.rs +++ b/crates/ecstore/src/cluster/rpc/network_probe.rs @@ -325,6 +325,24 @@ fn map_status(status: tonic::Status) -> NetworkPeerProbeError { match status.code() { Code::Unavailable | Code::Unknown => NetworkPeerProbeError::Unreachable, Code::DeadlineExceeded => NetworkPeerProbeError::TimedOut, + Code::Cancelled | Code::Internal => { + let mut source = std::error::Error::source(&status); + while let Some(error) = source { + if error + .downcast_ref::() + .is_some_and(|error| matches!(error.reason(), Some(h2::Reason::CANCEL | h2::Reason::NO_ERROR))) + { + return NetworkPeerProbeError::Unreachable; + } + source = if let Some(error) = error.downcast_ref::() { + // io::Error::source skips its immediate custom error. + error.get_ref().map(|inner| inner as &(dyn std::error::Error + 'static)) + } else { + error.source() + }; + } + NetworkPeerProbeError::ProtocolFailure + } _ => NetworkPeerProbeError::ProtocolFailure, } } @@ -335,6 +353,35 @@ mod tests { use crate::layout::endpoint::Endpoint; use crate::layout::endpoints::{Endpoints, PoolEndpoints}; + #[test] + fn peer_transport_resets_are_unreachable_without_reclassifying_application_errors() { + for reason in [h2::Reason::CANCEL, h2::Reason::NO_ERROR] { + let status = tonic::Status::from_error(Box::new(h2::Error::from(reason))); + assert!(matches!(status.code(), Code::Cancelled | Code::Internal)); + let code = status.code(); + assert_eq!(map_status(status), NetworkPeerProbeError::Unreachable); + let mut wrapped = tonic::Status::new(code, "wrapped transport reset"); + wrapped.set_source(Arc::new(std::io::Error::other(h2::Error::from(reason)))); + assert_eq!(map_status(wrapped), NetworkPeerProbeError::Unreachable); + } + for reason in [h2::Reason::PROTOCOL_ERROR, h2::Reason::COMPRESSION_ERROR] { + let status = tonic::Status::from_error(Box::new(h2::Error::from(reason))); + assert_eq!(map_status(status), NetworkPeerProbeError::ProtocolFailure); + } + for status in [ + tonic::Status::cancelled("peer cancelled the application request"), + tonic::Status::internal("peer rejected the probe"), + tonic::Status::unauthenticated("authentication rejected"), + tonic::Status::permission_denied("authorization rejected"), + ] { + assert_eq!(map_status(status), NetworkPeerProbeError::ProtocolFailure); + } + assert_eq!( + map_status(tonic::Status::deadline_exceeded("probe deadline elapsed")), + NetworkPeerProbeError::TimedOut + ); + } + fn pacing(duration: Duration) -> Arc { NetworkPeerProbeClient::from_endpoint_pools(&topology()) .with_diagnostic_pacing(Instant::now(), duration) diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index e067609a0..b350fd497 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -2279,6 +2279,7 @@ impl RemoteDisk { self.execute_with_timeout_for_op( "rename_data", || async { + let prepare_started = rustfs_io_metrics::put_stage_timer(); let file_info = compat_json(fi)?; let file_info_bin = encode_file_info_msgpack(fi)?; let mut client = self.get_client().await?; @@ -2306,24 +2307,67 @@ impl RemoteDisk { } else { attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?; } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE, + prepare_started, + ); - let response = if incarnation_bound { - // Older peers return Unimplemented before mutation; never downgrade. - client.rename_data_at_incarnation(request).await? - } else { - client.rename_data(request).await? - } - .into_inner(); + let rpc_started = rustfs_io_metrics::put_stage_timer(); + let response = rustfs_io_metrics::observe_put_stage_future( + async { + if incarnation_bound { + // Older peers return Unimplemented before mutation; never downgrade. + client.rename_data_at_incarnation(request).await + } else { + client.rename_data(request).await + } + }, + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT, + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT, + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY, + ) + .await?; + let response_process_started = rustfs_io_metrics::put_stage_timer(); + let response_headers_started = rustfs_io_metrics::put_stage_timer(); + let _ = response.metadata().len(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS, + response_headers_started, + ); + let into_inner_started = rustfs_io_metrics::put_stage_timer(); + let response = response.into_inner(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER, + into_inner_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC, + rpc_started, + ); + let success_check_started = rustfs_io_metrics::put_stage_timer(); if !response.success { return Err(response.error.unwrap_or_default().into()); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK, + success_check_started, + ); + let decode_started = rustfs_io_metrics::put_stage_timer(); let rename_data_resp = decode_msgpack_or_json::( &response.rename_data_resp_bin, &response.rename_data_resp, "RenameDataResp", )?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE, + decode_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS, + response_process_started, + ); Ok(rename_data_resp) }, @@ -5116,6 +5160,8 @@ mod tests { cleanup_data_dir: Some(Uuid::new_v4()), sign: Some(vec![0x14, 0x35]), old_current_size: Some(crate::disk::OldCurrentSize::Present(64 * 1024)), + old_current_source_checked: false, + old_current_source: None, }; let json = serde_json::to_string(&response).expect("legacy rename_data JSON response should encode"); let decode_before = rustfs_io_metrics::internode_metrics::global_internode_metrics().msgpack_json_decode_total_for_test(); @@ -5518,6 +5564,8 @@ mod tests { cleanup_data_dir: Some(Uuid::new_v4()), sign: Some(vec![1_u8; 32]), old_current_size: Some(crate::disk::OldCurrentSize::Present(4096)), + old_current_source_checked: false, + old_current_source: None, }; let json = serde_json::to_vec(&response).expect("rename data response json should encode"); let named_msgpack = rmp_serde::encode::to_vec_named(&response).expect("rename data response named msgpack should encode"); diff --git a/crates/ecstore/src/diagnostics/admin_server_info.rs b/crates/ecstore/src/diagnostics/admin_server_info.rs index def38f38b..0ff94e93c 100644 --- a/crates/ecstore/src/diagnostics/admin_server_info.rs +++ b/crates/ecstore/src/diagnostics/admin_server_info.rs @@ -44,7 +44,11 @@ use std::{ use time::OffsetDateTime; use tokio::time::timeout; use tonic::Request; -use tracing::warn; +use tracing::{error, trace, warn}; + +const EVENT_SERVER_PING: &str = "server_ping"; +const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +const LOG_SUBSYSTEM_DIAGNOSTICS: &str = "diagnostics"; use shadow_rs::shadow; @@ -159,11 +163,22 @@ async fn is_server_resolvable(endpoint: &Endpoint) -> Result<()> { let response: PingResponse = client.ping(request).await?.into_inner(); - let ping_response_body = flatbuffers::root::(&response.body); - if let Err(e) = ping_response_body { - eprintln!("{e}"); + if flatbuffers::root::(&response.body).is_err() { + error!( + event = EVENT_SERVER_PING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DIAGNOSTICS, + result = "invalid_response", + "Server ping response decoding failed" + ); } else { - println!("ping_resp:body(flatbuffer): {ping_response_body:?}"); + trace!( + event = EVENT_SERVER_PING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DIAGNOSTICS, + result = "success", + "Server ping completed" + ); } Ok(()) diff --git a/crates/ecstore/src/disk/cleanup_runtime.rs b/crates/ecstore/src/disk/cleanup_runtime.rs new file mode 100644 index 000000000..8293e577a --- /dev/null +++ b/crates/ecstore/src/disk/cleanup_runtime.rs @@ -0,0 +1,1292 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Admission and execution isolation for PUT, local/RPC disk cleanup and GC. +//! PUT admission precedes staging; receiver and GC disk work share a separate +//! budget so coordinators waiting for peers cannot exhaust receiver capacity. + +use crate::disk::{LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_DISK}; +use std::collections::BTreeSet; +use std::future::Future; +use std::io; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::time::Instant; +use tokio::runtime::{Handle, Runtime}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +const ENABLE: &str = "RUSTFS_CLEANUP_ISOLATE_ENABLE"; +const BUDGET: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_MAX_PENDING"; +const BLOCKING_THREADS: &str = "RUSTFS_CLEANUP_BLOCKING_THREADS"; +const ASYNC_THREADS: &str = "RUSTFS_CLEANUP_ASYNC_THREADS"; +const CPUS: &str = "RUSTFS_CLEANUP_CPUS"; +const WORKERS: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS"; +const RENAME_TAIL_CLEANUP_LANE_ENABLE: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_ENABLE"; +const DISK_PENDING: &str = "RUSTFS_CLEANUP_DISK_MAX_PENDING"; +const DISK_WORKERS: &str = "RUSTFS_CLEANUP_DISK_WORKERS"; +const GC_WORKERS: &str = "RUSTFS_CLEANUP_GC_WORKERS"; +const EVENT_CLEANUP_RUNTIME: &str = "disk_cleanup_runtime"; +pub(crate) const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json"; + +#[derive(Clone, Copy)] +enum DiskAdmission { + Request, + Guarded, +} + +#[derive(Clone)] +pub(crate) struct Reservation { + _admission: Arc, + execution: Arc, + handle: Handle, +} + +struct Admission { + _permit: OwnedSemaphorePermit, + metrics_enabled: bool, + metric: &'static str, +} + +impl Admission { + fn new(permit: OwnedSemaphorePermit, metric: &'static str) -> Self { + let metrics_enabled = rustfs_io_metrics::put_stage_metrics_enabled(); + if metrics_enabled { + metrics::gauge!(metric).increment(1.0); + } + Self { + _permit: permit, + metrics_enabled, + metric, + } + } +} + +impl Drop for Admission { + fn drop(&mut self) { + if self.metrics_enabled { + metrics::gauge!(self.metric).decrement(1.0); + } + } +} + +/// A lease release cannot queue behind work that might need the releasing +/// reader's locks. It relinquishes the lease first, then tries this disk budget. +pub(crate) struct DiskExecution { + _active: Admission, + _pending: Admission, +} + +/// Created only inside an owned scan. Steps can borrow the scan's directory +/// iterator and paths: cancelling its waiter cannot cancel a step's syscalls. +#[cfg_attr(test, derive(Default))] +pub(crate) struct GcBudget { + gates: Option<(Arc, Arc)>, +} + +impl GcBudget { + pub(crate) async fn step(&self, future: F) -> super::error::Result + where + F: Future>, + { + let Some((pending, execution)) = &self.gates else { + return future.await; + }; + // Scan producers are bounded and hold no object locks here. Unlike + // requests, they can wait for admission without an unbounded backlog. + let started = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); + let pending = pending.clone().acquire_owned().await.map_err(io::Error::other)?; + execute_disk( + Admission::new(pending, "rustfs_cleanup_disk_pending"), + None, + execution.clone(), + started, + future, + ) + .await + } +} + +fn try_disk_permit(semaphore: &Arc) -> io::Result { + semaphore.clone().try_acquire_owned().map_err(|err| { + if rustfs_io_metrics::put_stage_metrics_enabled() { + metrics::counter!("rustfs_cleanup_disk_admission_rejected_total").increment(1); + } + io::Error::new(io::ErrorKind::WouldBlock, err) + }) +} + +async fn execute_disk( + pending: Admission, + active: Option, + execution: Arc, + started: Option, + future: F, +) -> super::error::Result +where + F: Future>, +{ + let _pending = pending; + let active = match active { + Some(active) => active, + None => execution.acquire_owned().await.map_err(super::error::DiskError::other)?, + }; + let _active = Admission::new(active, "rustfs_cleanup_disk_active"); + if let Some(started) = started { + metrics::histogram!("rustfs_cleanup_disk_execution_wait_seconds").record(started.elapsed().as_secs_f64()); + } + future.await +} + +struct Config { + pending: usize, + workers: usize, + async_threads: usize, + blocking_threads: usize, + cpus: Vec, + disk_pending: usize, + disk_workers: usize, + gc_workers: usize, +} + +fn positive_env(name: &str, default: usize) -> io::Result { + match std::env::var(name) { + Ok(value) => value + .parse::() + .ok() + .filter(|v| *v > 0 && *v <= Semaphore::MAX_PERMITS) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, format!("{name} must be a positive supported count"))), + Err(std::env::VarError::NotPresent) => Ok(default), + Err(err) => Err(io::Error::new(io::ErrorKind::InvalidInput, err)), + } +} + +// Keep the guard's accepted values aligned with the lane's existing parser. +fn legacy_rename_tail_cleanup_lane_enabled(value: Option<&str>) -> bool { + matches!(value, Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")) +} + +fn parse_cpus(value: &str) -> io::Result> { + let invalid = || { + io::Error::new( + io::ErrorKind::InvalidInput, + "cleanup CPUs must be comma-separated CPU IDs or ascending ranges", + ) + }; + if value.trim().is_empty() { + return Err(invalid()); + } + let mut cpus = BTreeSet::new(); + for part in value.split(',') { + let mut ends = part.trim().split('-'); + let first = ends.next().and_then(|s| s.parse::().ok()).ok_or_else(invalid)?; + let last = match ends.next() { + Some(s) => s.parse::().map_err(|_| invalid())?, + None => first, + }; + if ends.next().is_some() || first > last || last >= 1024 { + return Err(invalid()); + } + cpus.extend(first..=last); + } + Ok(cpus.into_iter().collect()) +} + +impl Config { + fn from_env() -> io::Result { + for name in [ + "RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP", + "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER", + "RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP", + ] { + if rustfs_utils::get_env_bool(name, false) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} cannot be combined with cleanup isolation"), + )); + } + } + if legacy_rename_tail_cleanup_lane_enabled(std::env::var(RENAME_TAIL_CLEANUP_LANE_ENABLE).ok().as_deref()) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{RENAME_TAIL_CLEANUP_LANE_ENABLE} cannot be combined with cleanup isolation"), + )); + } + let workers = positive_env(WORKERS, 2)?; + let pending = positive_env(BUDGET, 1024)?; + let blocking_threads = positive_env(BLOCKING_THREADS, 4)?; + let cpus = match std::env::var(CPUS) { + Ok(value) if value.trim().eq_ignore_ascii_case("none") => Vec::new(), + Ok(value) => parse_cpus(&value)?, + Err(std::env::VarError::NotPresent) if cfg!(target_os = "linux") => vec![1, 2], + Err(std::env::VarError::NotPresent) => Vec::new(), + Err(err) => return Err(io::Error::new(io::ErrorKind::InvalidInput, err)), + }; + let available = std::thread::available_parallelism().map_or(1, usize::from); + let async_threads = positive_env(ASYNC_THREADS, available.min(if cpus.is_empty() { 2 } else { cpus.len().min(2) }))?; + let disk_workers = positive_env(DISK_WORKERS, blocking_threads)?; + Ok(Self { + pending, + workers, + async_threads, + blocking_threads, + cpus, + // Bound queueing to sixteen service waves by default. Increasing + // async coordinator concurrency does not create more OS threads. + disk_pending: positive_env(DISK_PENDING, disk_workers.saturating_mul(16).min(Semaphore::MAX_PERMITS))?, + disk_workers, + gc_workers: positive_env(GC_WORKERS, 1)?, + }) + } + + fn validate(&self) -> io::Result<()> { + if self.pending == 0 + || self.pending > Semaphore::MAX_PERMITS + || self.workers == 0 + || self.workers > self.pending + || self.async_threads == 0 + || self.async_threads > 1024 + || self.blocking_threads == 0 + || self.blocking_threads > 1024 + || self.disk_pending == 0 + || self.disk_pending > Semaphore::MAX_PERMITS + || self.disk_workers == 0 + || self.disk_workers > self.disk_pending + || self.gc_workers == 0 + || self.gc_workers > self.disk_workers + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "cleanup counts must be positive: PUT workers <= PUT pending, GC workers <= disk workers <= disk pending, async/blocking threads <= 1024", + )); + } + validate_cpus(&self.cpus) + } +} + +#[cfg(target_os = "linux")] +fn validate_cpus(cpus: &[usize]) -> io::Result<()> { + if cpus.is_empty() { + return Ok(()); + } + let allowed = rustix::thread::sched_getaffinity(None)?; + if cpus + .iter() + .any(|cpu| *cpu >= rustix::thread::CpuSet::MAX_CPU || !allowed.is_set(*cpu)) + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "cleanup CPUs must be a subset of the caller's allowed CPU set", + )); + } + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +fn validate_cpus(cpus: &[usize]) -> io::Result<()> { + if cpus.is_empty() { + Ok(()) + } else { + Err(io::Error::new(io::ErrorKind::Unsupported, "cleanup CPU affinity requires Linux")) + } +} + +fn bind_thread(cpus: &[usize]) -> io::Result<()> { + if cpus.is_empty() { + return Ok(()); + } + #[cfg(target_os = "linux")] + { + let mut set = rustix::thread::CpuSet::new(); + for cpu in cpus { + set.set(*cpu); + } + rustix::thread::sched_setaffinity(None, &set)?; + if rustix::thread::sched_getaffinity(None)? != set { + return Err(io::Error::other("cleanup CPU affinity changed during thread startup")); + } + Ok(()) + } + #[cfg(not(target_os = "linux"))] + { + validate_cpus(cpus) + } +} + +struct CleanupRuntime { + runtime: Option, + admission: Arc, + execution: Arc, + affinity_failed: Arc, + disk_admission: Arc, + disk_execution: Arc, + gc_execution: Arc, +} + +impl CleanupRuntime { + fn new(config: Config) -> io::Result { + config.validate()?; + let affinity_failed = Arc::new(AtomicBool::new(false)); + let failed = affinity_failed.clone(); + let cpus = config.cpus; + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(config.async_threads) + .max_blocking_threads(config.blocking_threads) + .thread_name("rustfs-cleanup") + .enable_all() + .on_thread_start(move || { + // Explicit fsync dispatch must not escape into the shared + // fsync runtime. This callback also runs on blocking threads. + crate::disk::os::use_current_runtime_for_fsync(); + if let Err(err) = bind_thread(&cpus) + && !failed.swap(true, Ordering::AcqRel) + { + tracing::error!(event = EVENT_CLEANUP_RUNTIME, component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK, state = "affinity_failed", error = %err, + "Cleanup thread affinity failed"); + } + }) + .build()?; + Ok(Self { + runtime: Some(runtime), + admission: Arc::new(Semaphore::new(config.pending)), + execution: Arc::new(Semaphore::new(config.workers)), + affinity_failed, + disk_admission: Arc::new(Semaphore::new(config.disk_pending)), + disk_execution: Arc::new(Semaphore::new(config.disk_workers)), + gc_execution: Arc::new(Semaphore::new(config.gc_workers)), + }) + } + + fn reserve(&self) -> io::Result { + if self.affinity_failed.load(Ordering::Acquire) { + return Err(io::Error::other("cleanup thread affinity failed; refusing new cleanup admission")); + } + let metrics_enabled = rustfs_io_metrics::put_stage_metrics_enabled(); + // Callers may already own source-reader or data-movement locks. Do + // not wait here behind a PUT whose completion requires those locks. + let permit = self.admission.clone().try_acquire_owned().map_err(|err| { + if metrics_enabled { + metrics::counter!("rustfs_cleanup_admission_rejected_total").increment(1); + } + io::Error::new(io::ErrorKind::WouldBlock, err) + })?; + let runtime = self + .runtime + .as_ref() + .ok_or_else(|| io::Error::other("cleanup runtime stopped"))?; + Ok(Reservation { + _admission: Arc::new(Admission::new(permit, "rustfs_cleanup_admitted_puts")), + execution: self.execution.clone(), + handle: runtime.handle().clone(), + }) + } + + async fn disk_operation(&self, future: F, mode: DiskAdmission) -> super::error::Result + where + F: Future> + Send + 'static, + T: Send + 'static, + { + if self.affinity_failed.load(Ordering::Acquire) && !super::os::is_isolated_io_thread() { + return Err(io::Error::other("cleanup thread affinity failed; refusing new disk cleanup admission").into()); + } + let started = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); + let pending = try_disk_permit(&self.disk_admission)?; + let pending = Admission::new(pending, "rustfs_cleanup_disk_pending"); + let runtime = self + .runtime + .as_ref() + .ok_or_else(|| io::Error::other("cleanup runtime stopped"))?; + let execution = self.disk_execution.clone(); + // An externally guarded mutation may already hold a namespace lock + // needed by an active disk job. Never queue it behind that job. + let active = if matches!(mode, DiskAdmission::Guarded) { + try_disk_permit(&execution)? + } else { + // Keep queued work in the caller's existing future. A waiter + // timeout must discard an unstarted delete, not leave a task that + // can later delete a reused path after its caller released locks. + execution.clone().acquire_owned().await.map_err(io::Error::other)? + }; + // A permit does not mean the worker has polled this operation. Use + // the result receiver's lifetime to discard work cancelled during + // runtime scheduling, before its first poll can start filesystem I/O. + let (result_tx, result_rx) = tokio::sync::oneshot::channel(); + let task = runtime.spawn(async move { + if result_tx.is_closed() { + return; + } + // After the first poll, the owned job must retain both permits + // through I/O even if the waiter cancels and closes result_rx. + let result = execute_disk(pending, Some(active), execution, started, future).await; + let _ = result_tx.send(result); + }); + match result_rx.await { + Ok(result) => result, + Err(err) => { + // Preserve a panic's JoinError rather than erasing its cause + // into a closed-channel error. + task.await.map_err(super::error::DiskError::other)?; + Err(super::error::DiskError::other(err)) + } + } + } + + fn try_disk_execution(&self) -> io::Result { + let pending = try_disk_permit(&self.disk_admission)?; + let active = try_disk_permit(&self.disk_execution)?; + Ok(DiskExecution { + _pending: Admission::new(pending, "rustfs_cleanup_disk_pending"), + _active: Admission::new(active, "rustfs_cleanup_disk_active"), + }) + } + + async fn gc_operation(&self, scan: F) -> super::error::Result<()> + where + F: FnOnce(GcBudget) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + if self.affinity_failed.load(Ordering::Acquire) { + return Err(io::Error::other("cleanup thread affinity failed; refusing new GC scan").into()); + } + let handle = self + .runtime + .as_ref() + .ok_or_else(|| io::Error::other("cleanup runtime stopped"))? + .handle(); + let execution = self.gc_execution.clone(); + let budget = GcBudget { + gates: Some((self.disk_admission.clone(), self.disk_execution.clone())), + }; + handle + .spawn(async move { + let active = execution.acquire_owned().await.map_err(super::error::DiskError::other)?; + let _active = Admission::new(active, "rustfs_cleanup_gc_active"); + scan(budget).await + }) + .await + .map_err(super::error::DiskError::other)? + } +} + +impl Drop for CleanupRuntime { + fn drop(&mut self) { + if let Some(runtime) = self.runtime.take() { + runtime.shutdown_background(); + } + } +} + +fn instance() -> io::Result> { + static INSTANCE: OnceLock, String>> = OnceLock::new(); + match INSTANCE.get_or_init(|| { + if !rustfs_utils::get_env_bool(ENABLE, false) { + return Ok(None); + } + Config::from_env() + .and_then(CleanupRuntime::new) + .map(Some) + .map_err(|err| err.to_string()) + }) { + Ok(runtime) => Ok(runtime.as_ref()), + Err(err) => Err(io::Error::other(err.clone())), + } +} + +pub(crate) fn reserve() -> io::Result> { + match instance()? { + Some(runtime) => runtime.reserve().map(Some), + None => Ok(None), + } +} + +pub(crate) fn enabled() -> io::Result { + Ok(instance()?.is_some()) +} + +/// Local cleanup entry point, also used by remote RPC receivers. This budget +/// is distinct from coordinator execution: coordinators can wait for peers +/// without exhausting the peers' capacity to service disk cleanup. +pub(crate) async fn disk_operation(future: F) -> super::error::Result +where + F: Future> + Send + 'static, + T: Send + 'static, +{ + match instance()? { + Some(runtime) => runtime.disk_operation(future, DiskAdmission::Request).await, + None => future.await, + } +} + +pub(crate) async fn guarded_disk_operation(future: F) -> super::error::Result +where + F: Future> + Send + 'static, + T: Send + 'static, +{ + match instance()? { + Some(runtime) => runtime.disk_operation(future, DiskAdmission::Guarded).await, + None => future.await, + } +} + +/// Lease release must run even under cleanup saturation, or reader tokens +/// would leak. Only its optional deferred deletion consumes disk permits. +pub(crate) async fn release_lease(future: F) -> super::error::Result<()> +where + F: Future> + Send + 'static, +{ + match instance()? { + Some(runtime) => runtime + .runtime + .as_ref() + .ok_or_else(|| io::Error::other("cleanup runtime stopped"))? + .spawn(future) + .await + .map_err(super::error::DiskError::other)?, + None => future.await, + } +} + +pub(crate) fn try_disk_execution() -> io::Result> { + instance()?.map(CleanupRuntime::try_disk_execution).transpose() +} + +/// At most gc_workers scans can produce work. Each scan yields the shared +/// disk execution budget between entries without spawning a task per entry. +pub(crate) async fn run_gc(scan: F) -> super::error::Result<()> +where + F: FnOnce(GcBudget) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, +{ + match instance()? { + Some(runtime) => runtime.gc_operation(scan).await, + None => scan(GcBudget { gates: None }).await, + } +} + +/// Dispatch only already-admitted work. Keeping the runtime handle in the +/// reservation prevents configuration errors from dropping committed cleanup. +pub(crate) fn spawn(reservation: Option, future: F) -> tokio::task::JoinHandle +where + F: Future + Send + 'static, + F::Output: Send + 'static, +{ + let started = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); + match reservation { + Some(reservation) => reservation.handle.clone().spawn(async move { + // This semaphore is never closed. Both queued work and overflow + // use this gate, and cancellation of the waiter cannot release it. + let _permit = reservation.execution.clone().acquire_owned().await; + if let Some(started) = started { + metrics::histogram!("rustfs_cleanup_execution_wait_seconds").record(started.elapsed().as_secs_f64()); + } + let result = future.await; + drop(reservation); + result + }), + None => tokio::spawn(future), + } +} + +/// Queue workers, queue overflow and synchronous failure cleanup share a gate. +/// With isolation disabled, retain the original inline polling behavior. +pub(crate) async fn run(reservation: Option, future: F) +where + F: Future + Send + 'static, +{ + if reservation.is_none() { + future.await; + } else if let Err(err) = spawn(reservation, future).await { + tracing::error!(event = EVENT_CLEANUP_RUNTIME, component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK, state = "task_failed", error = %err, + "Cleanup task failed"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicUsize; + use std::time::Duration; + use tokio::sync::oneshot; + + fn config(pending: usize, workers: usize, blocking_threads: usize) -> Config { + Config { + pending, + workers, + async_threads: workers.min(2), + blocking_threads, + cpus: Vec::new(), + disk_pending: 4, + disk_workers: 2, + gc_workers: 1, + } + } + + async fn deadline(future: F) -> F::Output { + tokio::time::timeout(Duration::from_secs(20), future) + .await + .expect("cleanup test timed out") + } + + #[test] + fn cpu_list_rejects_invalid_or_unbounded_ranges() { + assert_eq!(parse_cpus("3, 1-3,6").unwrap(), vec![1, 2, 3, 6]); + for value in ["", " ", "1,", "-1", "2-1", "1-2-3", "no", "1024", "0-18446744073709551615"] { + assert_eq!(parse_cpus(value).unwrap_err().kind(), io::ErrorKind::InvalidInput, "{value}"); + } + } + + #[test] + fn cleanup_lane_guard_matches_legacy_boolean_values() { + for (value, expected) in [ + (Some("1"), true), + (Some("true"), true), + (Some("TRUE"), true), + (Some("yes"), true), + (Some("YES"), true), + (Some("on"), true), + (Some("ON"), true), + (Some("True"), false), + (Some("enabled"), false), + (None, false), + ] { + assert_eq!(legacy_rename_tail_cleanup_lane_enabled(value), expected, "{value:?}"); + } + } + + #[test] + fn invalid_environment_is_rejected_before_admission() { + const CHILD: &str = "RUSTFS_CLEANUP_CONFIG_TEST_CHILD"; + if std::env::var_os(CHILD).is_some() { + assert!(reserve().is_err()); + return; + } + // Each configuration needs a fresh process: production intentionally + // caches configuration, and mutating process env races parallel tests. + for (name, value) in [ + (WORKERS, "0"), + (WORKERS, "5"), + (BUDGET, "invalid"), + (BLOCKING_THREADS, "0"), + (BLOCKING_THREADS, "1025"), + (ASYNC_THREADS, "0"), + (ASYNC_THREADS, "1025"), + (DISK_PENDING, "0"), + (DISK_WORKERS, "3"), + (GC_WORKERS, "3"), + (CPUS, "1-0"), + ("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP", "true"), + ("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER", "true"), + ("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP", "true"), + (RENAME_TAIL_CLEANUP_LANE_ENABLE, "true"), + ] { + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "disk::cleanup_runtime::tests::invalid_environment_is_rejected_before_admission", + "--nocapture", + ]) + .env(CHILD, "1") + .env(ENABLE, "true") + .env(WORKERS, "1") + .env(BUDGET, "4") + .env(BLOCKING_THREADS, "1") + .env(ASYNC_THREADS, "1") + .env(DISK_PENDING, "2") + .env(DISK_WORKERS, "2") + .env(GC_WORKERS, "1") + .env_remove(CPUS) + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP") + .env_remove(RENAME_TAIL_CLEANUP_LANE_ENABLE) + .env(name, value) + .output() + .unwrap(); + assert!(result.status.success(), "{name}={value}: {}", String::from_utf8_lossy(&result.stderr)); + assert!( + String::from_utf8_lossy(&result.stdout).contains("1 passed"), + "child must execute the admission assertion" + ); + } + } + + #[test] + fn invalid_budgets_fail_before_building_threads() { + for config in [config(0, 1, 1), config(1, 0, 1), config(1, 2, 1), config(1, 1, 0)] { + assert!(matches!(CleanupRuntime::new(config), Err(err) if err.kind() == io::ErrorKind::InvalidInput)); + } + } + + #[tokio::test] + async fn disk_work_keeps_budget_after_rpc_waiter_cancellation() { + let mut settings = config(1, 1, 1); + settings.disk_pending = 1; + settings.disk_workers = 1; + let runtime = Arc::new(CleanupRuntime::new(settings).unwrap()); + let receiver = runtime.clone(); + let (entered, entry) = oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let rpc = tokio::spawn(async move { + receiver + .disk_operation( + async move { + tokio::task::spawn_blocking(move || { + assert_eq!(std::thread::current().name(), Some("rustfs-cleanup")); + entered.send(()).unwrap(); + released.recv_timeout(Duration::from_secs(20)).unwrap(); + }) + .await + .unwrap(); + Ok(()) + }, + DiskAdmission::Request, + ) + .await + }); + deadline(entry).await.unwrap(); + rpc.abort(); + assert!(rpc.await.unwrap_err().is_cancelled()); + assert_eq!(runtime.disk_admission.available_permits(), 0); + assert_eq!(runtime.disk_execution.available_permits(), 0); + let ran = Arc::new(AtomicBool::new(false)); + let invoked = ran.clone(); + let rejected = runtime + .disk_operation( + async move { + invoked.store(true, Ordering::SeqCst); + Ok(()) + }, + DiskAdmission::Request, + ) + .await; + assert!(matches!(rejected, Err(super::super::error::DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock)); + assert!(!ran.load(Ordering::SeqCst), "rejected work must not execute"); + release.send(()).unwrap(); + deadline(async { + while runtime.disk_admission.available_permits() == 0 { + tokio::task::yield_now().await; + } + }) + .await; + assert_eq!(runtime.disk_execution.available_permits(), 1); + } + + #[tokio::test] + async fn cancelling_queued_disk_work_discards_it_before_execution() { + let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime")); + let occupied = runtime + .disk_execution + .clone() + .acquire_many_owned(2) + .await + .expect("occupy all execution slots"); + let receiver = runtime.clone(); + let ran = Arc::new(AtomicBool::new(false)); + let invoked = ran.clone(); + let waiter = tokio::spawn(async move { + receiver + .disk_operation( + async move { + invoked.store(true, Ordering::SeqCst); + Ok(()) + }, + DiskAdmission::Request, + ) + .await + }); + deadline(async { + while runtime.disk_admission.available_permits() == 4 { + tokio::task::yield_now().await; + } + }) + .await; + waiter.abort(); + assert!(waiter.await.expect_err("cancel queued waiter").is_cancelled()); + assert_eq!( + runtime.disk_admission.available_permits(), + 4, + "unstarted work must release pending capacity immediately" + ); + drop(occupied); + deadline(runtime.disk_operation(async { Ok(()) }, DiskAdmission::Request)) + .await + .expect("later work must progress"); + assert!(!ran.load(Ordering::SeqCst), "cancelled queued deletion must never execute later"); + } + + #[tokio::test] + async fn cancelling_disk_work_before_worker_poll_preserves_reused_path() { + let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime")); + let (entered, entry) = oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let stalled = runtime.runtime.as_ref().expect("live runtime").spawn(async move { + entered.send(()).expect("worker entered"); + released + .recv_timeout(Duration::from_secs(30)) + .expect("release stalled worker"); + }); + deadline(entry).await.expect("worker must stall before dispatch"); + let dir = tempfile::tempdir().expect("temporary disk"); + let backup = dir.path().join("xl.meta.bkp"); + std::fs::write(&backup, b"previous transaction").expect("old backup"); + { + let path = backup.clone(); + let deletion = runtime.disk_operation( + async move { + tokio::fs::remove_file(path).await?; + Ok(()) + }, + DiskAdmission::Request, + ); + tokio::pin!(deletion); + assert!(futures::poll!(&mut deletion).is_pending()); + assert_eq!(runtime.disk_execution.available_permits(), 1, "execution was admitted but not polled"); + } + std::fs::write(&backup, b"next transaction").expect("reuse path after cancelled waiter"); + release.send(()).expect("resume worker"); + deadline(stalled).await.expect("worker resumed"); + deadline(async { + while runtime.disk_admission.available_permits() != 4 { + tokio::task::yield_now().await; + } + }) + .await; + assert_eq!( + std::fs::read(backup).expect("unstarted deletion must not execute after cancellation"), + b"next transaction" + ); + assert_eq!(runtime.disk_execution.available_permits(), 2); + } + + #[tokio::test] + async fn disk_task_panic_preserves_cause_and_releases_budget() { + async fn panicking_job() -> super::super::error::Result<()> { + panic!("injected cleanup panic"); + } + let runtime = CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime"); + let error = deadline(runtime.disk_operation(panicking_job(), DiskAdmission::Request)) + .await + .expect_err("worker panic must reach its caller"); + let super::super::error::DiskError::Io(error) = error else { + panic!("worker panic must retain its I/O error source"); + }; + let cause = error + .get_ref() + .expect("panic source") + .downcast_ref::() + .expect("preserved JoinError"); + assert!(cause.is_panic()); + assert_eq!(runtime.disk_admission.available_permits(), 4); + assert_eq!(runtime.disk_execution.available_permits(), 2); + } + + #[tokio::test] + async fn coordinator_waiting_for_disk_does_not_consume_receiver_capacity() { + let mut settings = config(1, 1, 1); + settings.disk_pending = 1; + settings.disk_workers = 1; + let runtime = Arc::new(CleanupRuntime::new(settings).unwrap()); + let reservation = runtime.reserve().unwrap(); + let receiver = runtime.clone(); + let result = deadline(spawn(Some(reservation), async move { + receiver + .disk_operation(async { Ok(42) }, DiskAdmission::Request) + .await + .unwrap() + })) + .await + .unwrap(); + assert_eq!(result, 42); + } + + #[tokio::test] + async fn gc_shares_disk_budget_and_guarded_requests_do_not_wait_under_locks() { + let mut settings = config(1, 1, 1); + settings.disk_pending = 1; + settings.disk_workers = 1; + let runtime = Arc::new(CleanupRuntime::new(settings).unwrap()); + let receiver = runtime.clone(); + let (entered, entry) = oneshot::channel(); + let (release, released) = oneshot::channel(); + let rpc = tokio::spawn(async move { + receiver + .disk_operation( + async move { + entered.send(()).unwrap(); + released.await.unwrap(); + Ok(()) + }, + DiskAdmission::Request, + ) + .await + }); + deadline(entry).await.unwrap(); + let guarded = runtime.disk_operation(async { Ok(()) }, DiskAdmission::Guarded).await; + assert!(matches!(guarded, Err(super::super::error::DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock)); + let budget = GcBudget { + gates: Some((runtime.disk_admission.clone(), runtime.disk_execution.clone())), + }; + let gc = budget.step(async { Ok(7) }); + tokio::pin!(gc); + assert!( + futures::poll!(&mut gc).is_pending(), + "GC must join the shared budget rather than bypass it" + ); + release.send(()).unwrap(); + deadline(rpc).await.unwrap().unwrap(); + assert_eq!(deadline(gc).await.unwrap(), 7); + + // Also reject a guarded mutation when pending capacity is free but + // execution is occupied. This prevents namespace -> budget inversion. + let execution = runtime.disk_execution.clone().acquire_owned().await.unwrap(); + let guarded = runtime.disk_operation(async { Ok(()) }, DiskAdmission::Guarded).await; + assert!(matches!(guarded, Err(super::super::error::DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock)); + assert_eq!(runtime.disk_admission.available_permits(), 1); + assert!(matches!(runtime.try_disk_execution(), Err(err) if err.kind() == io::ErrorKind::WouldBlock)); + assert_eq!(runtime.disk_admission.available_permits(), 1); + drop(execution); + let lease_cleanup = runtime.try_disk_execution().unwrap(); + assert_eq!(runtime.disk_admission.available_permits(), 0); + assert_eq!(runtime.disk_execution.available_permits(), 0); + drop(lease_cleanup); + assert_eq!(runtime.disk_execution.available_permits(), 1); + } + + #[tokio::test] + async fn gc_scan_limit_leaves_disk_slots_available_between_entries() { + let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).unwrap()); + let first_runtime = runtime.clone(); + let (entered, entry) = oneshot::channel(); + let (release, released) = oneshot::channel(); + let first = tokio::spawn(async move { + first_runtime + .gc_operation(move |_| async move { + entered.send(()).unwrap(); + released.await.unwrap(); + Ok(()) + }) + .await + }); + deadline(entry).await.unwrap(); + assert_eq!(runtime.gc_execution.available_permits(), 0); + assert_eq!( + deadline(runtime.disk_operation(async { Ok(42) }, DiskAdmission::Request)) + .await + .unwrap(), + 42 + ); + let second = runtime.gc_operation(|_| async { Ok(()) }); + tokio::pin!(second); + assert!(futures::poll!(&mut second).is_pending()); + release.send(()).unwrap(); + deadline(first).await.unwrap().unwrap(); + deadline(second).await.unwrap(); + } + + #[tokio::test] + async fn gc_steps_borrow_scan_state_and_keep_budget_after_waiter_cancellation() { + let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime")); + let receiver = runtime.clone(); + let (entered, entry) = oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let waiter = tokio::spawn(async move { + receiver + .gc_operation(move |budget| async move { + let scan_task = tokio::task::id(); + let mut count = 0; + for _ in 0..32 { + budget + .step(async { + assert_eq!(tokio::task::id(), scan_task, "each entry must stay on the owned scan task"); + count += 1; + Ok(()) + }) + .await?; + } + assert_eq!(count, 32); + budget + .step(async move { + tokio::task::spawn_blocking(move || { + entered.send(()).expect("notify started syscall"); + released.recv_timeout(Duration::from_secs(20)).expect("release syscall"); + }) + .await + .expect("blocking cleanup"); + Ok(()) + }) + .await + }) + .await + }); + deadline(entry).await.expect("GC must start"); + waiter.abort(); + assert!(waiter.await.expect_err("cancelled waiter").is_cancelled()); + assert_eq!(runtime.gc_execution.available_permits(), 0); + assert_eq!(runtime.disk_admission.available_permits(), 3); + assert_eq!(runtime.disk_execution.available_permits(), 1); + release.send(()).expect("release blocking cleanup"); + deadline(async { + while runtime.gc_execution.available_permits() == 0 { + tokio::task::yield_now().await; + } + }) + .await; + assert_eq!(runtime.disk_admission.available_permits(), 4); + assert_eq!(runtime.disk_execution.available_permits(), 2); + } + + #[test] + fn defaults_bound_queue_waves_and_separate_threads_from_jobs() { + const CHILD: &str = "RUSTFS_CLEANUP_DEFAULTS_TEST_CHILD"; + if std::env::var_os(CHILD).is_some() { + let mut config = Config::from_env().expect("default configuration"); + let expected_cpus = match std::env::var(CPUS).as_deref() { + Ok("none") => Vec::new(), + Ok("3-4") => vec![3, 4], + _ if cfg!(target_os = "linux") => vec![1, 2], + _ => Vec::new(), + }; + assert_eq!(config.cpus, expected_cpus); + assert_eq!(config.workers, 64); + assert!(config.async_threads <= 2); + assert_eq!(config.blocking_threads, 4); + assert_eq!(config.disk_workers, 4); + assert_eq!(config.disk_pending, 64); + // Queue/thread defaults are independent of the host's CPU mask. + // The integration test exercises actual default/custom affinity. + config.cpus.clear(); + config.validate().expect("defaults must be internally consistent"); + let threads = config.async_threads; + let runtime = CleanupRuntime::new(config).expect("runtime with many logical jobs"); + assert_eq!(runtime.runtime.as_ref().expect("live runtime").metrics().num_workers(), threads); + return; + } + for cpus in [None, Some("3-4"), Some("none")] { + let mut command = std::process::Command::new(std::env::current_exe().expect("test executable")); + command + .args([ + "--exact", + "disk::cleanup_runtime::tests::defaults_bound_queue_waves_and_separate_threads_from_jobs", + ]) + .env(CHILD, "1") + .env(WORKERS, "64") + .env(BUDGET, "1024") + .env(GC_WORKERS, "1") + .env_remove(ASYNC_THREADS) + .env_remove(BLOCKING_THREADS) + .env_remove(DISK_WORKERS) + .env_remove(DISK_PENDING) + .env_remove(CPUS) + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP"); + if let Some(cpus) = cpus { + command.env(CPUS, cpus); + } + let output = command.output().expect("run defaults test in a fresh process"); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + } + } + + #[tokio::test] + async fn affinity_failure_rejects_new_admission_without_dropping_admitted_work() { + let runtime = CleanupRuntime::new(config(2, 1, 1)).unwrap(); + let reservation = runtime.reserve().unwrap(); + runtime.affinity_failed.store(true, Ordering::Release); + assert!(runtime.reserve().is_err()); + assert_eq!(deadline(spawn(Some(reservation), async { 42 })).await.unwrap(), 42); + assert_eq!(runtime.admission.available_permits(), 2); + } + + #[tokio::test] + async fn admission_is_retained_by_every_commit_and_cleanup_owner() { + let runtime = CleanupRuntime::new(config(1, 1, 1)).unwrap(); + let request = runtime.reserve().unwrap(); + let commit = request.clone(); + let (release, released) = oneshot::channel(); + let cleanup = spawn(Some(commit.clone()), async move { + released.await.unwrap(); + }); + drop(request); + drop(commit); + assert!( + matches!(runtime.reserve(), Err(err) if err.kind() == io::ErrorKind::WouldBlock), + "queued and running cleanup must retain admission without waiting on external locks" + ); + release.send(()).unwrap(); + deadline(cleanup).await.unwrap(); + let next = runtime.reserve().unwrap(); + assert_eq!(runtime.admission.available_permits(), 0); + drop(next); + assert_eq!(runtime.admission.available_permits(), 1); + } + + #[tokio::test] + async fn cancelling_cleanup_waiter_does_not_cancel_work_or_release_budget() { + let runtime = CleanupRuntime::new(config(1, 1, 1)).unwrap(); + let reservation = runtime.reserve().unwrap(); + let (entered, entry) = oneshot::channel(); + let (release, released) = oneshot::channel(); + let waiter = tokio::spawn(run(Some(reservation), async move { + entered.send(()).unwrap(); + released.await.unwrap(); + })); + deadline(entry).await.unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + assert_eq!(runtime.admission.available_permits(), 0); + assert_eq!(runtime.execution.available_permits(), 0); + release.send(()).unwrap(); + deadline(async { + loop { + if let Ok(reservation) = runtime.reserve() { + drop(reservation); + break; + } + tokio::task::yield_now().await; + } + }) + .await; + assert_eq!(runtime.admission.available_permits(), 1); + } + + #[tokio::test] + async fn worker_and_overflow_dispatch_share_one_execution_budget() { + let runtime = CleanupRuntime::new(config(8, 2, 1)).unwrap(); + let active = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let gate = Arc::new(Semaphore::new(0)); + let (entered, mut entries) = tokio::sync::mpsc::unbounded_channel(); + let mut jobs = Vec::new(); + for index in 0..8 { + let reservation = runtime.reserve().unwrap(); + let active = active.clone(); + let peak = peak.clone(); + let gate = gate.clone(); + let entered = entered.clone(); + let work = async move { + let running = active.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(running, Ordering::SeqCst); + entered.send(()).unwrap(); + gate.acquire().await.unwrap().forget(); + active.fetch_sub(1, Ordering::SeqCst); + }; + // Both a dequeued job and a queue-full continuation enter `run`; + // no-tail temporary cleanup uses direct owned dispatch. + jobs.push(if index % 2 == 0 { + tokio::spawn(run(Some(reservation), work)) + } else { + spawn(Some(reservation), work) + }); + } + deadline(entries.recv()).await.unwrap(); + deadline(entries.recv()).await.unwrap(); + assert_eq!(runtime.execution.available_permits(), 0); + assert!(entries.try_recv().is_err()); + assert_eq!(runtime.admission.available_permits(), 0); + gate.add_permits(8); + for job in jobs { + deadline(job).await.unwrap(); + } + assert_eq!(peak.load(Ordering::SeqCst), 2); + assert_eq!(runtime.admission.available_permits(), 8); + } + + #[tokio::test] + async fn cleanup_children_and_blocking_work_use_dedicated_threads() { + let runtime = CleanupRuntime::new(config(1, 1, 1)).unwrap(); + let task = spawn(Some(runtime.reserve().unwrap()), async move { + assert_eq!(std::thread::current().name(), Some("rustfs-cleanup")); + tokio::spawn(async { + assert_eq!(std::thread::current().name(), Some("rustfs-cleanup")); + tokio::task::spawn_blocking(|| { + assert_eq!(std::thread::current().name(), Some("rustfs-cleanup")); + }) + .await + .unwrap(); + }) + .await + .unwrap(); + }); + deadline(task).await.unwrap(); + assert_ne!(std::thread::current().name(), Some("rustfs-cleanup")); + } + + #[tokio::test] + async fn blocking_pool_saturation_does_not_block_foreground_pool() { + let runtime = CleanupRuntime::new(config(2, 2, 1)).unwrap(); + let (entered, entry) = oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let first = spawn(Some(runtime.reserve().unwrap()), async move { + tokio::task::spawn_blocking(move || { + entered.send(()).unwrap(); + released.recv_timeout(Duration::from_secs(20)).unwrap(); + }) + .await + .unwrap(); + }); + deadline(entry).await.unwrap(); + let (submitted, submission) = oneshot::channel(); + let (second_entered, mut second_entry) = oneshot::channel(); + let second = spawn(Some(runtime.reserve().unwrap()), async move { + let task = tokio::task::spawn_blocking(move || { + second_entered.send(()).unwrap(); + }); + submitted.send(()).unwrap(); + task.await.unwrap(); + }); + deadline(submission).await.unwrap(); + assert!(matches!(second_entry.try_recv(), Err(oneshot::error::TryRecvError::Empty))); + assert_eq!(deadline(tokio::task::spawn_blocking(|| 42)).await.unwrap(), 42); + release.send(()).unwrap(); + deadline(first).await.unwrap(); + deadline(second).await.unwrap(); + deadline(second_entry).await.unwrap(); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn affinity_covers_async_and_blocking_threads_without_changing_caller() { + let caller = rustix::thread::sched_getaffinity(None).unwrap(); + let cpu = (0..rustix::thread::CpuSet::MAX_CPU).find(|cpu| caller.is_set(*cpu)).unwrap(); + let mut config = config(1, 1, 1); + config.cpus = vec![cpu]; + let runtime = CleanupRuntime::new(config).unwrap(); + deadline(spawn(Some(runtime.reserve().unwrap()), async move { + let expected = { + let mut set = rustix::thread::CpuSet::new(); + set.set(cpu); + set + }; + assert_eq!(rustix::thread::sched_getaffinity(None).unwrap(), expected); + let actual = tokio::task::spawn_blocking(|| rustix::thread::sched_getaffinity(None).unwrap()) + .await + .unwrap(); + assert_eq!(actual, expected); + })) + .await + .unwrap(); + assert_eq!(rustix::thread::sched_getaffinity(None).unwrap(), caller); + assert!(validate_cpus(&[rustix::thread::CpuSet::MAX_CPU]).is_err()); + } + + #[cfg(not(target_os = "linux"))] + #[test] + fn explicit_cpu_affinity_is_rejected_on_unsupported_platforms() { + assert_eq!(validate_cpus(&[0]).unwrap_err().kind(), io::ErrorKind::Unsupported); + validate_cpus(&[]).unwrap(); + } +} diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 01091325b..3996c9dfd 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::disk::cleanup_runtime; use crate::disk::io_admission::{ AdmissionReader, BoxedChunkReader, DiskIoAdmission, DiskIoClass, DiskIoContext, current_disk_io_context, with_disk_io_context, with_disk_io_permit, @@ -484,10 +485,10 @@ impl LocalDiskWrapper { opts: DeleteOptions, namespace_owner: Option>, ) -> Result<()> { - self.track_disk_health_mutation( + self.track_cleanup_mutation( "delete", DiskMetricMutation::Delete, - || async { Box::pin(self.disk.delete_with_namespace_owner(volume, path, opts, namespace_owner)).await }, + || async { Box::pin(self.cleanup_delete(volume, path, opts, namespace_owner)).await }, get_max_timeout_duration(), ) .await @@ -1475,6 +1476,28 @@ impl LocalDiskWrapper { ) } + async fn cleanup_delete( + &self, + volume: &str, + path: &str, + options: DeleteOptions, + owner: Option>, + ) -> Result<()> { + if !cleanup_runtime::enabled()? { + return self.disk.delete_with_namespace_owner(volume, path, options, owner).await; + } + let disk = self.disk.clone(); + let volume = volume.to_owned(); + let path = path.to_owned(); + let guarded = owner.is_some(); + let operation = async move { disk.delete_with_namespace_owner(&volume, &path, options, owner).await }; + if guarded { + cleanup_runtime::guarded_disk_operation(operation).await + } else { + cleanup_runtime::disk_operation(operation).await + } + } + /// Run a delete under an owned coordinator task when a publication guard /// is present. This keeps the guard alive if the RPC waiter is cancelled /// while the local namespace mutation is still in progress. @@ -1493,12 +1516,13 @@ impl LocalDiskWrapper { } else { get_max_timeout_duration() }; + let cleanup_owner = external_guard.clone(); run_owned_mutation(external_guard, move || async move { operation - .track_disk_health_mutation( + .track_cleanup_mutation( "delete", DiskMetricMutation::Delete, - || async { operation.disk.delete(&volume, &path, options).await }, + || async { operation.cleanup_delete(&volume, &path, options, cleanup_owner).await }, timeout_duration, ) .await @@ -2092,6 +2116,31 @@ impl LocalDiskWrapper { self.track_disk_health_with_op("unknown", operation, timeout_duration).await } + /// Cleanup latency includes waiting behind a node-wide execution budget, + /// so a waiter deadline alone is not evidence that this disk is faulty. + /// The active disk health probe retains its original timeout policy. + async fn track_cleanup_mutation( + &self, + op: &'static str, + mutation: DiskMetricMutation, + operation: F, + timeout_duration: Duration, + ) -> Result + where + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + // Runtime initialization must not prevent control-plane release of + // reader tokens. Disk I/O entries still propagate configuration errors. + let action = if cleanup_runtime::enabled().unwrap_or(false) { + TimeoutHealthAction::IgnoreFailure + } else { + TimeoutHealthAction::MarkFailure + }; + self.track_disk_health_with_op_timeout_action_and_mutation(op, operation, timeout_duration, action, mutation) + .await + } + async fn track_disk_health_mutation( &self, op: &'static str, @@ -2589,9 +2638,37 @@ impl DiskAPI for LocalDiskWrapper { } async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> { - self.track_disk_health_with_op( + // Quota fence release is control-plane work and may wait for in-flight + // mutations. Keep its original waiter lifecycle instead of creating + // detached cleanup tasks for repeated fence-release RPCs. + if volume == super::RUSTFS_META_BUCKET && super::is_quota_mutation_fence_path(path) { + return self + .track_disk_health_with_op( + "release_snapshot_lease", + || async { self.disk.release_snapshot_lease(volume, path, token).await }, + get_max_timeout_duration(), + ) + .await; + } + self.track_cleanup_mutation( "release_snapshot_lease", - || async { self.disk.release_snapshot_lease(volume, path, token).await }, + DiskMetricMutation::None, + || async { + if self.disk.release_snapshot_lease_without_cleanup(volume, path, token).await { + return Ok(()); + } + if !cleanup_runtime::enabled().unwrap_or(false) { + // The raw release removes the token before trying the + // reclaim budget. Initialization failure therefore keeps + // the cleanup intent and data, without leaking a reader + // or falling back to unbudgeted filesystem deletion. + return self.disk.release_snapshot_lease(volume, path, token).await; + } + let disk = self.disk.clone(); + let volume = volume.to_owned(); + let path = path.to_owned(); + cleanup_runtime::release_lease(async move { disk.release_snapshot_lease(&volume, &path, token).await }).await + }, get_max_timeout_duration(), ) .await @@ -2616,10 +2693,22 @@ impl DiskAPI for LocalDiskWrapper { } else { get_max_timeout_duration() }; - self.track_disk_health_mutation( + self.track_cleanup_mutation( "delete_data_dir", DiskMetricMutation::Delete, - || async { self.disk.delete_data_dir(volume, path, opts).await }, + || async { + if !cleanup_runtime::enabled()? { + return self.disk.delete_data_dir(volume, path, opts).await; + } + let disk = self.disk.clone(); + let volume = volume.to_owned(); + let path = path.to_owned(); + cleanup_runtime::disk_operation(async move { + let _scope = scope; + disk.delete_data_dir(&volume, &path, opts).await + }) + .await + }, timeout, ) .await @@ -2866,10 +2955,10 @@ impl DiskAPI for LocalDiskWrapper { } async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { - self.track_disk_health_mutation( + self.track_cleanup_mutation( "delete", DiskMetricMutation::Delete, - || async { self.disk.delete(volume, path, opt).await }, + || async { self.cleanup_delete(volume, path, opt, None).await }, get_max_timeout_duration(), ) .await @@ -2902,11 +2991,25 @@ impl DiskAPI for LocalDiskWrapper { } async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()> { - self.track_disk_health_mutation( + let cleanup = path.rsplit('/').next() == Some(cleanup_runtime::OLD_DATA_CLEANUP_RECEIPT_FILE); + self.track_disk_health_with_op_timeout_action_and_mutation( "write_all", - DiskMetricMutation::Write, - || async { self.disk.write_all(volume, path, data).await }, + || async { + if !cleanup || !cleanup_runtime::enabled()? { + return self.disk.write_all(volume, path, data).await; + } + let disk = self.disk.clone(); + let volume = volume.to_owned(); + let path = path.to_owned(); + cleanup_runtime::disk_operation(async move { disk.write_all(&volume, &path, data).await }).await + }, get_max_timeout_duration(), + if cleanup && cleanup_runtime::enabled()? { + TimeoutHealthAction::IgnoreFailure + } else { + TimeoutHealthAction::MarkFailure + }, + DiskMetricMutation::Write, ) .await } @@ -3299,6 +3402,256 @@ mod tests { let _ = walk.await; } + #[tokio::test] + async fn invalid_cleanup_config_does_not_leak_snapshot_readers() { + const CHILD: &str = "RUSTFS_CLEANUP_INVALID_LEASE_TEST_CHILD"; + if std::env::var_os(CHILD).is_none() { + let output = tokio::task::spawn_blocking(|| { + std::process::Command::new(std::env::current_exe().expect("test executable")) + .args([ + "--exact", + "disk::disk_store::tests::invalid_cleanup_config_does_not_leak_snapshot_readers", + "--nocapture", + ]) + .env(CHILD, "1") + .env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true") + .env("RUSTFS_CLEANUP_ASYNC_THREADS", "0") + .output() + .expect("run with invalid cleanup configuration") + }) + .await + .expect("child process task"); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + return; + } + let dir = tempfile::tempdir().expect("temporary disk"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk")); + let wrapper = LocalDiskWrapper::new(disk.clone(), false); + let volume = "invalid-cleanup-config"; + let path = "object/data"; + wrapper.make_volume(volume).await.expect("bucket"); + wrapper + .write_all(volume, "object/data/part.1", Bytes::from_static(b"live shard")) + .await + .expect("write shard"); + let token = wrapper.acquire_snapshot_lease(volume, path).await.expect("GET lease"); + wrapper + .release_snapshot_lease(volume, path, token) + .await + .expect("control-only release must not depend on cleanup initialization"); + assert!(matches!( + wrapper.renew_snapshot_lease(volume, path, token).await, + Err(DiskError::FileNotFound) + )); + + let token = wrapper + .acquire_snapshot_lease(volume, path) + .await + .expect("reader of old data"); + // Register reclaim intent directly; this only updates the lease + // registry while a reader is present and performs no filesystem I/O. + assert_eq!( + disk.delete_data_dir( + volume, + path, + DeleteOptions { + recursive: true, + ..Default::default() + } + ) + .await + .expect("defer reclaim"), + DataDirDeleteStatus::Deferred + ); + assert!( + wrapper.release_snapshot_lease(volume, path, token).await.is_err(), + "invalid configuration must still reject reclamation" + ); + assert!( + matches!(wrapper.renew_snapshot_lease(volume, path, token).await, Err(DiskError::FileNotFound)), + "failed reclamation must not strand a reader token" + ); + assert_eq!( + wrapper.read_all(volume, "object/data/part.1").await.expect("retained data"), + Bytes::from_static(b"live shard"), + "configuration failure must not fall back to unbudgeted deletion" + ); + } + + #[tokio::test] + async fn rpc_cleanup_entries_share_disk_budget() { + const CHILD: &str = "RUSTFS_CLEANUP_RPC_TEST_CHILD"; + if std::env::var_os(CHILD).is_none() { + let output = tokio::task::spawn_blocking(|| { + std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "disk::disk_store::tests::rpc_cleanup_entries_share_disk_budget", + "--nocapture", + ]) + .env(CHILD, "1") + .env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true") + .env("RUSTFS_CLEANUP_ASYNC_THREADS", "1") + .env("RUSTFS_CLEANUP_DISK_MAX_PENDING", "1") + .env("RUSTFS_CLEANUP_DISK_WORKERS", "1") + .env("RUSTFS_CLEANUP_GC_WORKERS", "1") + .env("RUSTFS_CLEANUP_CPUS", "none") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP") + .output() + .unwrap() + }) + .await + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + return; + } + let dir = tempfile::tempdir().unwrap(); + let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.unwrap()); + let wrapper = LocalDiskWrapper::new(disk, false); + let volume = "cleanup-receiver"; + let path = "object/old-data"; + let body_path = format!("{path}/part.1"); + let receipt_path = format!("{path}/{}", cleanup_runtime::OLD_DATA_CLEANUP_RECEIPT_FILE); + wrapper.make_volume(volume).await.unwrap(); + wrapper + .write_all(volume, &body_path, Bytes::from_static(b"old body")) + .await + .unwrap(); + let read_lease = wrapper + .acquire_snapshot_lease(volume, path) + .await + .expect("ordinary GET lease"); + let (entered, entry) = tokio::sync::oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let occupied = tokio::spawn(cleanup_runtime::disk_operation(async move { + entered.send(()).unwrap(); + // Deliberately stall the only cleanup scheduler to prove that + // ordinary GET completion does not dispatch to it. + released + .recv_timeout(Duration::from_secs(30)) + .expect("release stalled scheduler"); + Ok(()) + })); + tokio::time::timeout(Duration::from_secs(20), entry).await.unwrap().unwrap(); + tokio::time::timeout(Duration::from_secs(5), wrapper.release_snapshot_lease(volume, path, read_lease)) + .await + .expect("ordinary GET release must progress while the cleanup scheduler is stalled") + .expect("release without reclamation"); + let fence_path = super::super::quota_mutation_fence_path(volume, "object"); + let fence = wrapper + .acquire_snapshot_lease(super::super::RUSTFS_META_BUCKET, &fence_path) + .await + .unwrap(); + wrapper + .release_snapshot_lease(super::super::RUSTFS_META_BUCKET, &fence_path, fence) + .await + .unwrap(); + let timeout = wrapper + .track_cleanup_mutation( + "delete", + DiskMetricMutation::Delete, + std::future::pending::>, + Duration::from_millis(10), + ) + .await; + assert!(matches!(timeout, Err(DiskError::Timeout))); + assert!(!wrapper.health.is_faulty(), "cleanup queue delay must not mark a healthy disk faulty"); + let options = DeleteOptions { + recursive: true, + ..Default::default() + }; + let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + // This is the same entry point used after RPC authentication and + // scanner publication-lease validation by handle_delete. + let result = wrapper + .delete_with_publication_guard(volume, path, options.clone(), Some(Arc::new(DropProbe(drops.clone())))) + .await; + assert!(matches!(result, Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock)); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert!(matches!(wrapper.delete_data_dir(volume, path, options.clone()).await, + Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock)); + assert!(matches!(wrapper.write_all(volume, &receipt_path, Bytes::from_static(b"receipt")).await, + Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock)); + assert_eq!(wrapper.read_all(volume, &body_path).await.unwrap(), Bytes::from_static(b"old body")); + release.send(()).unwrap(); + occupied.await.unwrap().unwrap(); + wrapper + .write_all(volume, &receipt_path, Bytes::from_static(b"receipt")) + .await + .unwrap(); + wrapper + .delete_with_publication_guard(volume, path, options, None) + .await + .unwrap(); + assert!(matches!(wrapper.read_all(volume, &body_path).await, Err(DiskError::FileNotFound))); + // Snapshot readers defer old-data cleanup. Saturation must release + // their token while retaining reclaim intent, rather than leaking a + // live token or performing an unbudgeted delete on the RPC runtime. + wrapper + .write_all(volume, &body_path, Bytes::from_static(b"leased body")) + .await + .unwrap(); + let lease = wrapper.acquire_snapshot_lease(volume, path).await.unwrap(); + assert_eq!( + wrapper + .delete_data_dir( + volume, + path, + DeleteOptions { + recursive: true, + ..Default::default() + } + ) + .await + .unwrap(), + DataDirDeleteStatus::Deferred + ); + let (entered, entry) = tokio::sync::oneshot::channel(); + let (release, released) = tokio::sync::oneshot::channel(); + let occupied = tokio::spawn(cleanup_runtime::disk_operation(async move { + entered.send(()).unwrap(); + released.await.unwrap(); + Ok(()) + })); + tokio::time::timeout(Duration::from_secs(20), entry).await.unwrap().unwrap(); + assert!(matches!(wrapper.release_snapshot_lease(volume, path, lease).await, + Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock)); + assert_eq!(wrapper.read_all(volume, &body_path).await.unwrap(), Bytes::from_static(b"leased body")); + release.send(()).unwrap(); + occupied.await.unwrap().unwrap(); + assert_eq!( + wrapper + .delete_data_dir( + volume, + path, + DeleteOptions { + recursive: true, + ..Default::default() + } + ) + .await + .unwrap(), + DataDirDeleteStatus::Deleted + ); + assert!(matches!(wrapper.read_all(volume, &body_path).await, Err(DiskError::FileNotFound))); + } + #[tokio::test] async fn owned_mutation_keeps_publication_guard_after_waiter_cancellation() { let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0)); diff --git a/crates/ecstore/src/disk/fs.rs b/crates/ecstore/src/disk/fs.rs index 7dbbaf14d..15a52ef7b 100644 --- a/crates/ecstore/src/disk/fs.rs +++ b/crates/ecstore/src/disk/fs.rs @@ -223,12 +223,6 @@ pub fn rename_std(from: impl AsRef, to: impl AsRef) -> io::Result<() std::fs::rename(from, to) } -#[cfg(any(not(windows), test))] -#[tracing::instrument(level = "debug", skip_all)] -pub async fn read_file(path: impl AsRef) -> io::Result> { - fs::read(path.as_ref()).await -} - // Bucket existence cache - reduces statx syscalls for repeated bucket checks /// Cache for bucket directory existence checks. @@ -614,28 +608,6 @@ mod tests { assert_eq!(content, "test content"); } - #[tokio::test] - async fn test_read_file() { - let temp_dir = TempDir::new().unwrap(); - let file_path = temp_dir.path().join("test_read.txt"); - - let test_content = b"This is test content for reading"; - tokio::fs::write(&file_path, test_content).await.unwrap(); - - // Read file - let read_content = read_file(&file_path).await.unwrap(); - assert_eq!(read_content, test_content); - } - - #[tokio::test] - async fn test_read_file_nonexistent() { - let temp_dir = TempDir::new().unwrap(); - let file_path = temp_dir.path().join("nonexistent.txt"); - - // Should fail for non-existent file - assert!(read_file(&file_path).await.is_err()); - } - #[tokio::test] async fn test_same_file() { let temp_dir = TempDir::new().unwrap(); diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 66cc3e10b..ea3a39ff8 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -221,8 +221,10 @@ struct ListingMetadataRead { has_namespace_child_candidate: bool, } -fn read_all_data_std(path: &Path) -> core::result::Result<(Vec, Option), ReadAllError> { - let mut file = std::fs::File::open(path).map_err(ReadAllError::Open)?; +fn read_all_data_std( + opened: std::io::Result, +) -> core::result::Result<(Vec, Option), ReadAllError> { + let mut file = opened.map_err(ReadAllError::Open)?; let metadata = file.metadata().map_err(|err| ReadAllError::Disk(to_file_error(err).into()))?; if metadata.is_dir() { @@ -5353,7 +5355,7 @@ pub struct LocalDisk { /// pathname is later covered by another mount. io_root: PathBuf, #[cfg(target_os = "linux")] - mount_lease: std::fs::File, + mount_lease: Arc, #[cfg(target_os = "linux")] mount_lease_mount_id: Option, /// Public path for callers that need the configured disk layout. Internal @@ -5610,24 +5612,6 @@ fn mount_id_from_mountinfo_contents(mountinfo: &str, path: &Path) -> Option } impl LocalDisk { - #[cfg(target_os = "linux")] - fn open_mount_lease(root: &Path) -> Result<(std::fs::File, PathBuf, Option)> { - use rustix::fs::{Mode, OFlags, open}; - use std::os::fd::AsRawFd as _; - - let fd = open( - root, - OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, - Mode::empty(), - ) - .map_err(std::io::Error::from) - .map_err(DiskError::from)?; - let lease = std::fs::File::from(fd); - let io_root = PathBuf::from(format!("/proc/self/fd/{}/.", lease.as_raw_fd())); - let mount_id = mount_id_for_fd(&lease); - Ok((lease, io_root, mount_id)) - } - #[cfg(not(target_os = "linux"))] fn open_mount_lease(root: &Path) -> Result { Ok(root.to_path_buf()) @@ -5711,7 +5695,11 @@ impl LocalDisk { let root = publication_root.path().to_path_buf(); #[cfg(target_os = "linux")] - let (mount_lease, io_root, mount_lease_mount_id) = Self::open_mount_lease(&root)?; + let (mount_lease, io_root, mount_lease_mount_id) = ( + publication_root.directory().clone(), + publication_root.io_path().to_path_buf(), + mount_id_for_fd(publication_root.directory()), + ); #[cfg(not(target_os = "linux"))] let io_root = Self::open_mount_lease(&root)?; @@ -5759,9 +5747,11 @@ impl LocalDisk { state = "format_path_resolved", "Local disk format path resolved" ); - let (format_data, format_meta) = read_file_exists(&io_format_path).await.inspect_err(|err| { - log_startup_disk_error("read_format_json", &io_format_path, err); - })?; + let (format_data, format_meta) = read_file_exists(&io_format_path, &publication_root) + .await + .inspect_err(|err| { + log_startup_disk_error("read_format_json", &io_format_path, err); + })?; let mut id = None; // let mut format_legacy = false; @@ -5907,7 +5897,13 @@ impl LocalDisk { let io_root = disk.io_root.clone(); let publication_root = disk.publication_root.clone(); - tokio::spawn(Self::cleanup_deleted_objects_loop(io_root, publication_root, exit_rx)); + tokio::spawn(Self::cleanup_deleted_objects_loop( + io_root, + publication_root, + #[cfg(target_os = "linux")] + disk.mount_lease.clone(), + exit_rx, + )); debug!( event = EVENT_DISK_LOCAL_STARTUP_CLEANUP, component = LOG_COMPONENT_ECSTORE, @@ -5923,30 +5919,53 @@ impl LocalDisk { async fn cleanup_deleted_objects_loop( root: PathBuf, publication_root: os::PublicationRoot, + #[cfg(target_os = "linux")] mount_lease: Arc, mut exit_rx: tokio::sync::broadcast::Receiver<()>, ) { let start_at = Instant::now() + DELETED_OBJECTS_CLEANUP_INTERVAL; let mut interval = interval_at(start_at, DELETED_OBJECTS_CLEANUP_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { tokio::select! { _ = interval.tick() => { - if let Err(err) = Self::cleanup_deleted_objects(root.clone()).await { + let root = root.clone(); + let publication_root = publication_root.clone(); + #[cfg(target_os = "linux")] + let mount_lease = mount_lease.clone(); + if let Err(err) = super::cleanup_runtime::run_gc(move |budget| async move { + // Queued/detached scans must keep /proc/self/fd paths + // attached to the original mount after LocalDisk drops. + #[cfg(target_os = "linux")] + let _mount_lease = mount_lease; + if let Err(err) = Self::cleanup_deleted_objects(root.clone(), &budget).await { + error!( + event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + task = "deleted_objects", + state = "failed", + error = ?err, + "Disk local background cleanup failed" + ); + } + if let Err(err) = Self::cleanup_stale_tmp_objects(root.clone(), &publication_root, &budget).await { + error!( + event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + task = "stale_tmp_objects", + state = "failed", + error = ?err, + "Disk local background cleanup failed" + ); + } + Ok(()) + }).await { error!( event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - task = "deleted_objects", - state = "failed", - error = ?err, - "Disk local background cleanup failed" - ); - } - if let Err(err) = Self::cleanup_stale_tmp_objects(root.clone(), &publication_root).await { - error!( - event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - task = "stale_tmp_objects", + task = "cleanup_dispatch", state = "failed", error = ?err, "Disk local background cleanup failed" @@ -6037,95 +6056,123 @@ impl LocalDisk { } } - async fn cleanup_stale_tmp_objects(root: PathBuf, publication_root: &os::PublicationRoot) -> Result<()> { - Self::cleanup_stale_tmp_objects_with_expiry(root, publication_root, STALE_TMP_OBJECT_EXPIRY).await + async fn cleanup_stale_tmp_objects( + root: PathBuf, + publication_root: &os::PublicationRoot, + budget: &super::cleanup_runtime::GcBudget, + ) -> Result<()> { + Self::cleanup_stale_tmp_objects_with_expiry(root, publication_root, STALE_TMP_OBJECT_EXPIRY, budget).await } async fn cleanup_stale_tmp_objects_with_expiry( root: PathBuf, publication_root: &os::PublicationRoot, expiry: Duration, + budget: &super::cleanup_runtime::GcBudget, ) -> Result<()> { let tmp_path = Self::meta_path(&root, RUSTFS_META_TMP_BUCKET); - let mut entries = match fs::read_dir(&tmp_path).await { + let mut entries = match budget.step(async { Ok(fs::read_dir(&tmp_path).await?) }).await { Ok(entries) => entries, Err(e) => { - if e.kind() == ErrorKind::NotFound { + if matches!(&e, DiskError::Io(err) if err.kind() == ErrorKind::NotFound) { return Ok(()); } - return Err(e.into()); + return Err(e); } }; - while let Some(entry) = entries.next_entry().await? { - let name = entry.file_name().to_string_lossy().to_string(); - if name.is_empty() || name == "." || name == ".." || name == ".trash" { - continue; + loop { + let more = budget + .step(async { + let Some(entry) = entries.next_entry().await? else { + return Ok(false); + }; + Self::cleanup_stale_tmp_entry(entry, &root, publication_root, expiry).await?; + Ok(true) + }) + .await?; + if !more { + break; } - - let file_type = entry.file_type().await?; - if !file_type.is_dir() { - continue; - } - - let Some(age) = entry - .metadata() - .await? - .modified() - .ok() - .and_then(|modified| modified.elapsed().ok()) - else { - continue; - }; - if age <= expiry { - continue; - } - - let target_path = Self::meta_path(&root, RUSTFS_META_TMP_DELETED_BUCKET).join(Uuid::new_v4().to_string()); - rename_all(entry.path(), target_path, Self::meta_path(&root, RUSTFS_META_BUCKET), publication_root).await?; } Ok(()) } - async fn cleanup_deleted_objects(root: PathBuf) -> Result<()> { + async fn cleanup_stale_tmp_entry( + entry: fs::DirEntry, + root: &Path, + publication_root: &os::PublicationRoot, + expiry: Duration, + ) -> Result<()> { + let name = entry.file_name().to_string_lossy().to_string(); + if name.is_empty() || name == "." || name == ".." || name == ".trash" || !entry.file_type().await?.is_dir() { + return Ok(()); + } + let Some(age) = entry + .metadata() + .await? + .modified() + .ok() + .and_then(|modified| modified.elapsed().ok()) + else { + return Ok(()); + }; + if age <= expiry { + return Ok(()); + } + let target_path = Self::meta_path(root, RUSTFS_META_TMP_DELETED_BUCKET).join(Uuid::new_v4().to_string()); + rename_all(entry.path(), target_path, Self::meta_path(root, RUSTFS_META_BUCKET), publication_root).await?; + Ok(()) + } + + async fn cleanup_deleted_objects(root: PathBuf, budget: &super::cleanup_runtime::GcBudget) -> Result<()> { let trash = Self::meta_path(&root, RUSTFS_META_TMP_DELETED_BUCKET); - let mut entries = match fs::read_dir(&trash).await { + let mut entries = match budget.step(async { Ok(fs::read_dir(&trash).await?) }).await { Ok(entries) => entries, Err(e) => { - if e.kind() == ErrorKind::NotFound { + if matches!(&e, DiskError::Io(err) if err.kind() == ErrorKind::NotFound) { return Ok(()); } - return Err(e.into()); + return Err(e); } }; - while let Some(entry) = entries.next_entry().await? { - let name = entry.file_name().to_string_lossy().to_string(); - if name.is_empty() || name == "." || name == ".." { - continue; - } - - let file_type = entry.file_type().await?; - - let path = trash.join(name); - - if file_type.is_dir() { - if let Err(e) = tokio::fs::remove_dir_all(path).await - && e.kind() != ErrorKind::NotFound - { - return Err(e.into()); - } - } else if let Err(e) = tokio::fs::remove_file(path).await - && e.kind() != ErrorKind::NotFound - { - return Err(e.into()); + loop { + let more = budget + .step(async { + let Some(entry) = entries.next_entry().await? else { + return Ok(false); + }; + Self::cleanup_trash_entry(entry, &trash).await?; + Ok(true) + }) + .await?; + if !more { + break; } } Ok(()) } + async fn cleanup_trash_entry(entry: fs::DirEntry, trash: &Path) -> Result<()> { + let name = entry.file_name().to_string_lossy().to_string(); + if name.is_empty() || name == "." || name == ".." { + return Ok(()); + } + let path = trash.join(name); + let result = if entry.file_type().await?.is_dir() { + fs::remove_dir_all(path).await + } else { + fs::remove_file(path).await + }; + match result { + Err(err) if err.kind() != ErrorKind::NotFound => Err(err.into()), + _ => Ok(()), + } + } + fn is_valid_volname(volname: &str) -> bool { if volname.len() < 3 { return false; @@ -6252,6 +6299,19 @@ impl LocalDisk { self.local_disk_object_path(self.io_root(), bucket, key) } + /// Only for operations that open through PublicationRoot or a commit guard: + /// containment and symlink checks happen with the actual open, off the runtime. + fn io_get_object_open_path(&self, bucket: &str, key: &str) -> Result { + #[cfg(target_os = "linux")] + { + let (bucket_path, path) = build_local_disk_object_path(self.io_root(), bucket, key); + check_local_disk_object_path_components(self.io_root(), &bucket_path, &path)?; + Ok(path) + } + #[cfg(not(target_os = "linux"))] + self.io_get_object_path(bucket, key) + } + fn io_get_bucket_path(&self, bucket: &str) -> Result { self.local_disk_bucket_path(self.io_root(), bucket) } @@ -6410,7 +6470,7 @@ impl LocalDisk { let mut meta = FileMeta::new(); if !fi.fresh { - let (buf, _) = read_file_exists(&p).await?; + let (buf, _) = read_file_exists(&p, &self.publication_root).await?; if !buf.is_empty() { let _ = meta.unmarshal_msg(&buf).map_err(|_| { meta = FileMeta::new(); @@ -6436,6 +6496,36 @@ impl LocalDisk { Ok(()) } + /// Keep the common GET completion on its caller. Only a last-reader + /// release with pending reclamation needs an owned cleanup task. Leave + /// that token untouched so the slow path owns the complete transition. + pub(in crate::disk) async fn release_snapshot_lease_without_cleanup( + &self, + volume: &str, + path: &str, + token: SnapshotLeaseToken, + ) -> bool { + let key = SnapshotLeaseKey { + volume: volume.to_owned(), + path: path.to_owned(), + }; + let mut registry = self.snapshot_leases.lock(); + let Some(entry) = registry.entries.get_mut(&key) else { + return true; + }; + if !entry.deleting + && entry.pending_delete.is_some() + && (entry.tokens.is_empty() || (entry.tokens.len() == 1 && entry.tokens.contains(&token))) + { + return false; + } + entry.tokens.remove(&token); + if entry.tokens.is_empty() && !entry.deleting { + registry.entries.remove(&key); + } + true + } + async fn delete_data_dir_with_namespace_owner( &self, volume: &str, @@ -7489,10 +7579,11 @@ impl LocalDisk { // - metadata failure -> to_file_error // - parse failure -> propagated verbatim from read_xl_meta_no_data_sync (`?`) let path = file_path.as_ref().to_path_buf(); + let root = self.publication_root.clone(); let (data, modtime) = spawn_blocking_with_disk_io_permit(io_permit, move || -> Result<(Vec, Option)> { // Read-only open, equivalent to O_RDONLY (get_readonly_options only sets read(true)). - let mut f = std::fs::File::open(&path).map_err(to_file_error)?; + let mut f = root.open_readonly(&path).map_err(to_file_error)?; let meta = f.metadata().map_err(to_file_error)?; @@ -7546,6 +7637,7 @@ impl LocalDisk { let metadata_path = self.io_get_object_path(bucket, path.to_string_lossy().as_ref()); // A part's existence check, metadata read and decode share one dispatch. // Keep open errors unmapped for the existing missing-volume fallback. + let root = self.publication_root.clone(); let result = spawn_blocking_with_disk_io_permit(io_permit, move || -> Result<_> { let part_error = |error: String| ObjectPartInfo { number: num, @@ -7558,7 +7650,7 @@ impl LocalDisk { // Invalid metadata paths remain request errors, but missing data wins // first, as it does in the serial reader. let metadata_path = metadata_path?; - Ok(read_all_data_std(&metadata_path) + Ok(read_all_data_std(root.open_readonly(&metadata_path)) .map(|(data, _)| ObjectPartInfo::unmarshal(&data).unwrap_or_else(|err| part_error(err.to_string())))) }) .await; @@ -7578,8 +7670,9 @@ impl LocalDisk { let object_dir = self.io_get_object_path(volume, object_name)?; let metadata_path = object_dir.join(STORAGE_FORMAT_FILE); let volume_dir = self.io_get_bucket_path(volume)?; + let root = self.publication_root.clone(); let result = spawn_blocking_with_disk_io_permit(io_permit, move || { - let (bytes, _) = read_all_data_std(&metadata_path)?; + let (bytes, _) = read_all_data_std(root.open_readonly(&metadata_path))?; let file_meta = FileMeta::load(&bytes).ok(); let data_dirs: HashSet = file_meta .as_ref() @@ -7677,7 +7770,8 @@ impl LocalDisk { // gating the volume fallback on the open error alone is equivalent to // the original code, where the fallback lived solely in the open match arm. let path = file_path.as_ref().to_path_buf(); - let res = spawn_blocking_with_disk_io_permit(io_permit, move || read_all_data_std(&path)) + let root = self.publication_root.clone(); + let res = spawn_blocking_with_disk_io_permit(io_permit, move || read_all_data_std(root.open_readonly(&path))) .await .map_err(DiskError::from)?; @@ -8505,16 +8599,16 @@ impl LocalDisk { os::make_dir_all(parent, skip_parent).await?; } + let root = self.publication_root.clone(); tokio::task::spawn_blocking(move || { #[cfg(test)] run_owned_file_write_before_open(&path); - let mut file = std::fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(&path) - .map_err(to_file_error)?; + let (mut file, parent_handle) = if sync == SyncMode::FileAndDir { + root.create_truncate_with_parent(&path).map_err(to_file_error)? + } else { + (root.create_truncate(&path).map_err(to_file_error)?, None) + }; std::io::Write::write_all(&mut file, buf.as_ref()).map_err(to_file_error)?; if sync != SyncMode::None { file.sync_data().map_err(to_file_error)?; @@ -8523,7 +8617,7 @@ impl LocalDisk { if sync == SyncMode::FileAndDir && let Some(parent) = path.parent() { - os::fsync_dir_std(parent).map_err(to_file_error)?; + os::fsync_directory_handle_std(parent, parent_handle.as_ref()).map_err(to_file_error)?; } } Ok::<_, std::io::Error>(()) @@ -8548,7 +8642,18 @@ impl LocalDisk { os::make_dir_all(parent, skip_parent).await?; } - let f = super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?; + let f = if mode == O_CREATE | O_WRONLY | O_TRUNC { + let path = path.as_ref().to_path_buf(); + let root = self.publication_root.clone(); + File::from_std( + tokio::task::spawn_blocking(move || root.create_truncate(&path)) + .await + .map_err(DiskError::from)? + .map_err(to_file_error)?, + ) + } else { + super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)? + }; Ok(f) } @@ -9424,9 +9529,9 @@ fn file_meta_counts_toward_limit(meta: &FileMeta) -> bool { } // Filter std::io::ErrorKind::NotFound -async fn read_file_exists(path: impl AsRef) -> Result<(Bytes, Option)> { +async fn read_file_exists(path: impl AsRef, root: &os::PublicationRoot) -> Result<(Bytes, Option)> { let p = path.as_ref(); - let (data, meta) = match read_file_all(&p).await { + let (data, meta) = match read_file_all(&p, root).await { Ok((data, meta)) => (data, Some(meta)), Err(e) => { if e == Error::FileNotFound { @@ -9445,32 +9550,28 @@ async fn read_file_exists(path: impl AsRef) -> Result<(Bytes, Option) -> Result<(Bytes, Metadata)> { - let p = path.as_ref(); - let meta = read_file_metadata(&path).await?; - - let data = fs::read(&p) - .await - .inspect_err(|err| { - log_startup_disk_io_error("read_file_all", p, err); - }) - .map_err(to_file_error)?; - - Ok((data.into(), meta)) -} - -async fn read_file_metadata(p: impl AsRef) -> Result { - let path = p.as_ref(); - let meta = fs::metadata(path) - .await - .inspect_err(|err| { - if err.kind() != ErrorKind::NotFound { - log_startup_disk_io_error("read_file_metadata", path, err); - } - }) - .map_err(to_file_error)?; - - Ok(meta) +async fn read_file_all(path: impl AsRef, root: &os::PublicationRoot) -> Result<(Bytes, Metadata)> { + let path = path.as_ref().to_path_buf(); + let root = root.clone(); + tokio::task::spawn_blocking(move || { + let result = (|| { + let mut file = root.open_readonly(&path)?; + let metadata = file.metadata()?; + let mut data = Vec::new(); + std::io::Read::read_to_end(&mut file, &mut data)?; + Ok::<_, std::io::Error>((Bytes::from(data), metadata)) + })(); + result + .inspect_err(|err| { + if err.kind() != ErrorKind::NotFound { + log_startup_disk_io_error("read_file_all", &path, err); + } + }) + .map_err(to_file_error) + .map_err(DiskError::from) + }) + .await + .map_err(DiskError::from)? } fn skip_access_checks(p: impl AsRef) -> bool { @@ -9552,13 +9653,19 @@ fn check_local_disk_valid_path(root: &Path, path: impl AsRef) -> Result<() #[cfg(target_os = "linux")] fn check_local_disk_valid_object_path_at(root: &Path, root_fd: &std::fs::File, bucket_path: &Path, path: &Path) -> Result<()> { + check_local_disk_object_path_components(root, bucket_path, path)?; + reject_local_disk_symlink_components_at(root, root_fd, &normalize_path_components(path)) +} + +#[cfg(target_os = "linux")] +fn check_local_disk_object_path_components(root: &Path, bucket_path: &Path, path: &Path) -> Result<()> { let bucket_path = normalize_path_components(bucket_path); let path = normalize_path_components(path); if !bucket_path.starts_with(root) || !path.starts_with(&bucket_path) { return Err(DiskError::InvalidPath); } - reject_local_disk_symlink_components_at(root, root_fd, &path) + Ok(()) } #[cfg(target_os = "linux")] @@ -10407,9 +10514,9 @@ impl DiskAPI for LocalDisk { } } - let p = self.io_get_object_path(volume, path)?; + let p = self.io_get_object_open_path(volume, path)?; - let (data, _) = read_file_all(&p).await?; + let (data, _) = read_file_all(&p, &self.publication_root).await?; Ok(data) } @@ -11701,8 +11808,17 @@ impl DiskAPI for LocalDisk { entry.deleting = true; opts }; + // Release the reader token even under saturation. A rejected reclaim + // retains pending_delete (and its on-disk receipt) for a later retry. + // Do not wait for execution while a releasing reader may own locks. let claim = SnapshotLeaseDeleteClaim::new(&self.snapshot_leases, key); - let result = self.delete_unleased(volume, path, &opts).await; + let result = match super::cleanup_runtime::try_disk_execution() { + Ok(permit) => { + let _permit = permit; + self.delete_unleased(volume, path, &opts).await + } + Err(err) => Err(err.into()), + }; claim.finish(result.is_ok()); result } @@ -11976,7 +12092,7 @@ impl DiskAPI for LocalDisk { }; // if req.metadata_only {} - match read_file_all(&fpath).await { + match read_file_all(&fpath, &self.publication_root).await { Ok((data, meta)) => { found += 1; @@ -13572,10 +13688,15 @@ mod test { disk.wait_for_startup_cleanup().await; assert_eq!(disk.startup_cleanup_ready.load(Ordering::Acquire), 1); - LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().join("missing-root"), &publication_root, Duration::ZERO) - .await - .expect("missing tmp path should be a cleanup no-op"); - LocalDisk::cleanup_deleted_objects(dir.path().join("missing-root")) + LocalDisk::cleanup_stale_tmp_objects_with_expiry( + dir.path().join("missing-root"), + &publication_root, + Duration::ZERO, + &crate::disk::cleanup_runtime::GcBudget::default(), + ) + .await + .expect("missing tmp path should be a cleanup no-op"); + LocalDisk::cleanup_deleted_objects(dir.path().join("missing-root"), &crate::disk::cleanup_runtime::GcBudget::default()) .await .expect("missing trash path should be a cleanup no-op"); @@ -13588,9 +13709,14 @@ mod test { fs::create_dir_all(&trash_root).await.expect("trash dir should be created"); backdate_mtime(&stale_dir, Duration::from_secs(10)); - LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().to_path_buf(), &publication_root, Duration::ZERO) - .await - .expect("stale tmp directory should move to trash"); + LocalDisk::cleanup_stale_tmp_objects_with_expiry( + dir.path().to_path_buf(), + &publication_root, + Duration::ZERO, + &crate::disk::cleanup_runtime::GcBudget::default(), + ) + .await + .expect("stale tmp directory should move to trash"); assert!(!stale_dir.exists(), "stale tmp directory should be moved away"); assert!(live_file.exists(), "plain tmp files should be ignored by stale dir cleanup"); @@ -13600,7 +13726,7 @@ mod test { fs::create_dir_all(trash_root.join("trash-dir")) .await .expect("trash dir should be created"); - LocalDisk::cleanup_deleted_objects(dir.path().to_path_buf()) + LocalDisk::cleanup_deleted_objects(dir.path().to_path_buf(), &crate::disk::cleanup_runtime::GcBudget::default()) .await .expect("trash cleanup should remove files and directories"); assert!( @@ -20705,9 +20831,14 @@ mod test { // Backdate after the write above: creating stale/data refreshes the // scanned tmp/stale directory's mtime. backdate_mtime(&tmp.join("stale"), Duration::from_secs(10)); - LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().to_path_buf(), &publication_root, Duration::ZERO) - .await - .expect("operation should succeed"); + LocalDisk::cleanup_stale_tmp_objects_with_expiry( + dir.path().to_path_buf(), + &publication_root, + Duration::ZERO, + &crate::disk::cleanup_runtime::GcBudget::default(), + ) + .await + .expect("operation should succeed"); assert!(!tmp.join("stale").exists()); assert!(trash.exists()); @@ -20734,9 +20865,14 @@ mod test { fs::write(&fresh_dir, b"temporary").await.expect("operation should succeed"); fs::write(®ular_file, b"keep").await.expect("operation should succeed"); - LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().to_path_buf(), &publication_root, Duration::from_secs(60)) - .await - .expect("operation should succeed"); + LocalDisk::cleanup_stale_tmp_objects_with_expiry( + dir.path().to_path_buf(), + &publication_root, + Duration::from_secs(60), + &crate::disk::cleanup_runtime::GcBudget::default(), + ) + .await + .expect("operation should succeed"); assert!(tmp.join("fresh").exists()); assert!(regular_file.exists()); @@ -20745,6 +20881,44 @@ mod test { assert!(entries.next_entry().await.expect("operation should succeed").is_none()); } + #[cfg(target_os = "linux")] + #[tokio::test] + async fn background_cleanup_keeps_mount_lease_after_disk_drop() { + let dir = tempfile::tempdir().unwrap(); + let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(); + let disk = LocalDisk::new(&endpoint, false).await.unwrap(); + let root = disk.io_root.clone(); + let trash_entry = LocalDisk::meta_path(&root, RUSTFS_META_TMP_DELETED_BUCKET).join("detached-gc"); + fs::create_dir_all(&trash_entry).await.unwrap(); + fs::write(trash_entry.join("part.1"), b"garbage").await.unwrap(); + let live_file = root.join("keep"); + fs::write(&live_file, b"live").await.unwrap(); + let (exit, exited) = tokio::sync::broadcast::channel(1); + tokio::time::pause(); + let cleanup = tokio::spawn(LocalDisk::cleanup_deleted_objects_loop( + root.clone(), + disk.publication_root.clone(), + disk.mount_lease.clone(), + exited, + )); + tokio::task::yield_now().await; + drop(disk); + tokio::task::yield_now().await; + assert!(live_file.exists(), "the GC owner must retain the original /proc/self/fd root"); + tokio::time::advance(DELETED_OBJECTS_CLEANUP_INTERVAL + Duration::from_secs(1)).await; + tokio::time::resume(); + timeout(Duration::from_secs(20), async { + while trash_entry.exists() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("detached cleanup must finish against the pinned mount"); + assert_eq!(fs::read(&live_file).await.unwrap(), b"live"); + exit.send(()).unwrap(); + timeout(Duration::from_secs(20), cleanup).await.unwrap().unwrap(); + } + #[tokio::test(start_paused = true)] async fn cleanup_loop_interval_does_not_tick_immediately() { let start_at = Instant::now() + DELETED_OBJECTS_CLEANUP_INTERVAL; @@ -23887,10 +24061,12 @@ mod test { #[tokio::test] async fn test_read_file_exists() { - let test_file = "./test_read_exists.txt"; + let directory = tempfile::tempdir().expect("test directory"); + let root = os::PublicationRoot::new(directory.path()).expect("publication root"); + let test_file = &directory.path().join("read_exists"); // Test non-existent file - let (data, metadata) = read_file_exists(test_file).await.expect("operation should succeed"); + let (data, metadata) = read_file_exists(test_file, &root).await.expect("operation should succeed"); assert!(data.is_empty()); assert!(metadata.is_none()); @@ -23898,7 +24074,7 @@ mod test { fs::write(test_file, b"test content").await.expect("operation should succeed"); // Test existing file - let (data, metadata) = read_file_exists(test_file).await.expect("operation should succeed"); + let (data, metadata) = read_file_exists(test_file, &root).await.expect("operation should succeed"); assert_eq!(data.as_ref(), b"test content"); assert!(metadata.is_some()); @@ -23908,14 +24084,16 @@ mod test { #[tokio::test] async fn test_read_file_all() { - let test_file = "./test_read_all.txt"; + let directory = tempfile::tempdir().expect("test directory"); + let root = os::PublicationRoot::new(directory.path()).expect("publication root"); + let test_file = &directory.path().join("read_all"); let test_content = b"test content for read_all"; // Create test file fs::write(test_file, test_content).await.expect("operation should succeed"); // Test reading file - let (data, metadata) = read_file_all(test_file).await.expect("operation should succeed"); + let (data, metadata) = read_file_all(test_file, &root).await.expect("operation should succeed"); assert_eq!(data.as_ref(), test_content); assert!(metadata.is_file()); assert_eq!(metadata.len(), test_content.len() as u64); @@ -23924,22 +24102,6 @@ mod test { let _ = fs::remove_file(test_file).await; } - #[tokio::test] - async fn test_read_file_metadata() { - let test_file = "./test_metadata.txt"; - - // Create test file - fs::write(test_file, b"test").await.expect("operation should succeed"); - - // Test reading metadata - let metadata = read_file_metadata(test_file).await.expect("operation should succeed"); - assert!(metadata.is_file()); - assert_eq!(metadata.len(), 4); // "test" is 4 bytes - - // Clean up - let _ = fs::remove_file(test_file).await; - } - #[test] fn test_is_root_path() { // Unix root path @@ -24528,9 +24690,14 @@ mod test { Bytes::from_static(b"later") ); - disk.release_snapshot_lease(volume, &data_dir, renewed) - .await - .expect("renewed lease release should succeed"); + assert!( + disk.release_snapshot_lease_without_cleanup(volume, &data_dir, renewed).await, + "a non-final reader release must not dispatch cleanup" + ); + assert!( + !disk.release_snapshot_lease_without_cleanup(volume, &data_dir, second).await, + "the final reader must retain its token for owned deferred reclamation" + ); assert!( disk.read_all(volume, &first_part).await.is_ok(), "one remaining lease must keep the data directory" diff --git a/crates/ecstore/src/disk/local/commit.rs b/crates/ecstore/src/disk/local/commit.rs index 1fa6b8803..56ef71101 100644 --- a/crates/ecstore/src/disk/local/commit.rs +++ b/crates/ecstore/src/disk/local/commit.rs @@ -47,6 +47,33 @@ use tokio::fs; use tracing::{info, warn}; use uuid::Uuid; +fn transitioned_old_current_source( + dst_meta_existed: bool, + dst_meta_unparsable: bool, + xlmeta: &FileMeta, + volume: &str, + path: &str, + version_id: Uuid, +) -> (bool, Option) { + if dst_meta_unparsable { + return (false, None); + } + if !dst_meta_existed { + return (true, None); + } + let Ok(old_current) = xlmeta.into_fileinfo(volume, path, "", false, false, true) else { + return (false, None); + }; + if old_current.version_id.unwrap_or_default() != version_id { + return (true, None); + } + if old_current.transition_status == rustfs_filemeta::TRANSITION_COMPLETE { + (true, Some(old_current)) + } else { + (true, None) + } +} + /// Hold later repair publications after admitting one baseline object. The /// fixture arms this on one replacement disk before rejoining the cluster. #[cfg(feature = "e2e-test-hooks")] @@ -214,7 +241,7 @@ pub(super) async fn lock_rename_commit_directories( publication_root: &os::PublicationRoot, mutation_lease: Arc, ) -> Result { - #[cfg(windows)] + #[cfg(any(windows, target_os = "linux"))] let result = { let source_parent = source_parent.to_path_buf(); let destination_parent = destination_parent.to_path_buf(); @@ -222,7 +249,7 @@ pub(super) async fn lock_rename_commit_directories( let publication_root = publication_root.clone(); os::run_blocking_namespace_operation(mutation_lease, move || { let result = os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root); - #[cfg(test)] + #[cfg(all(test, windows))] if result.is_ok() { run_destination_commit_directory_preparation(&destination_parent); } @@ -230,7 +257,7 @@ pub(super) async fn lock_rename_commit_directories( }) .await }; - #[cfg(not(windows))] + #[cfg(not(any(windows, target_os = "linux")))] let result = { let _ = mutation_lease; os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root) @@ -244,12 +271,12 @@ pub(super) async fn lock_rename_commit_directories( result.map_err(to_file_error).map_err(DiskError::from) } -async fn read_rename_destination_metadata( +async fn prepare_rename_destination_metadata( file_path: &Path, rename_commit_guard: &os::RenameCommitGuard, mutation_lease: Arc, ) -> Result> { - #[cfg(windows)] + #[cfg(any(windows, target_os = "linux"))] let result = { let file_path = file_path.to_path_buf(); let rename_commit_guard = rename_commit_guard.clone(); @@ -258,10 +285,10 @@ async fn read_rename_destination_metadata( }) .await }; - #[cfg(not(windows))] + #[cfg(not(any(windows, target_os = "linux")))] let _ = (rename_commit_guard, mutation_lease); - #[cfg(not(windows))] - let result = match super::super::fs::read_file(file_path).await { + #[cfg(not(any(windows, target_os = "linux")))] + let result = match fs::read(file_path).await { Ok(data) => Ok(Some(data)), Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), Err(err) => Err(err), @@ -398,7 +425,7 @@ impl LocalDisk { // xl.meta path let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; - let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; + let dst_file_path = self.io_get_object_open_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; // data_dir path let has_data_dir_path = { @@ -453,8 +480,7 @@ impl LocalDisk { fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?; } // Acquire the common trees before reading destination metadata. On - // Windows this pins the object directory identity across metadata - // preparation, data publication, rollback backup, and final commit. + // Windows this pins the object directory identity across publication. let rename_commit_guard = lock_rename_commit_directories( src_file_parent, dst_file_parent, @@ -472,9 +498,12 @@ impl LocalDisk { source_volume_dir: &src_volume_dir, data_paths: has_data_dir_path.as_ref(), }; - let has_dst_buf = - read_rename_destination_metadata(commit.destination_metadata, &commit.directory_guard, commit.mutation_lease.clone()) - .await?; + let has_dst_buf = prepare_rename_destination_metadata( + commit.destination_metadata, + &commit.directory_guard, + commit.mutation_lease.clone(), + ) + .await?; if no_inline { // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta @@ -508,6 +537,14 @@ impl LocalDisk { } let version_id = fi.version_id.unwrap_or_default(); + let (old_current_source_checked, old_current_source) = transitioned_old_current_source( + has_dst_buf.is_some(), + dst_meta_unparsable, + &xlmeta, + dst_volume, + dst_path, + version_id, + ); let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); let rollback_data_dir = has_old_data_dir.or_else(|| { @@ -798,7 +835,9 @@ impl LocalDisk { && let Some(parent) = dst_file_path.parent() { let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dst_dir_group_commit(parent, Some(commit.mutation_lease.clone())).await { + if let Err(err) = + os::fsync_commit_directory(parent, &commit.directory_guard, commit.mutation_lease.clone(), None).await + { rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, fsync_started, @@ -897,6 +936,8 @@ impl LocalDisk { cleanup_data_dir: has_old_data_dir, sign: version_signature, old_current_size, + old_current_source_checked, + old_current_source, }) } else { // Inline metadata preparation is blocking. The transaction lease is @@ -910,6 +951,8 @@ impl LocalDisk { None }; let dst_path_for_failpoint = dst_path.to_string(); + let dst_volume_for_source = dst_volume.to_string(); + let dst_path_for_source = dst_path.to_string(); #[cfg(windows)] let source_parent = src_file_parent.to_path_buf(); let rename_commit_guard_for_preparation = commit.directory_guard.clone(); @@ -954,6 +997,14 @@ impl LocalDisk { }; let version_id = fi.version_id.unwrap_or_default(); + let (old_current_source_checked, old_current_source) = transitioned_old_current_source( + has_dst_buf.is_some(), + dst_meta_unparsable, + &xlmeta, + &dst_volume_for_source, + &dst_path_for_source, + version_id, + ); let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); let rollback_data_dir = old_data_dir.or_else(|| { @@ -1003,6 +1054,8 @@ impl LocalDisk { old_data_dir, version_signature, old_current_size, + old_current_source_checked, + old_current_source, staged_rollback_path, has_dst_buf.is_none(), prepared_metadata_source, @@ -1022,6 +1075,8 @@ impl LocalDisk { cleanup_data_dir, version_signature, old_current_size, + old_current_source_checked, + old_current_source, mut local_rollback_path, destination_was_absent, prepared_metadata_source, @@ -1147,10 +1202,11 @@ impl LocalDisk { && let Some(dst_parent) = dst_file_path.parent() { let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit( + if let Err(err) = os::fsync_commit_directory( dst_parent, + &commit.directory_guard, commit.mutation_lease.clone(), - admission, + Some(admission), ) .await { @@ -1262,6 +1318,8 @@ impl LocalDisk { cleanup_data_dir, sign: version_signature, old_current_size, + old_current_source_checked, + old_current_source, }) } } diff --git a/crates/ecstore/src/disk/local/delete_recovery.rs b/crates/ecstore/src/disk/local/delete_recovery.rs index f8f4e3485..00f268278 100644 --- a/crates/ecstore/src/disk/local/delete_recovery.rs +++ b/crates/ecstore/src/disk/local/delete_recovery.rs @@ -1953,10 +1953,7 @@ async fn barrier(volume: &str, object: &str, transaction: Uuid, stage: &str, dri .map_err(to_file_error)? == b"io_error" { - return Err(DiskError::Io(std::io::Error::new( - ErrorKind::Other, - "injected delete recovery barrier failure", - ))); + return Err(DiskError::Io(std::io::Error::other("injected delete recovery barrier failure"))); } Ok(()) } diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index c1166f405..0b90fe6a7 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -14,6 +14,7 @@ // #730: disk abstractions still carry staged health and direct-I/O migration paths. +pub(crate) mod cleanup_runtime; pub mod disk_store; pub mod endpoint; pub mod error; @@ -1776,6 +1777,14 @@ pub struct RenameDataResp { /// "cannot vote", never as `Absent`. #[serde(default)] pub old_current_size: Option, + /// Whether this disk decoded enough of the overwritten current version to + /// decide if a transitioned source has to be returned. + #[serde(default)] + pub old_current_source_checked: bool, + /// Transitioned old-current source used by PUT free-version cleanup receipt. + /// Empty when the old current is not transitioned or no source was known. + #[serde(default)] + pub old_current_source: Option, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -2517,6 +2526,8 @@ mod tests { cleanup_data_dir: Some(uuid), sign: Some(signature.clone()), old_current_size: Some(OldCurrentSize::Present(42)), + old_current_source_checked: false, + old_current_source: None, }; assert_eq!(resp.old_data_dir, Some(uuid)); @@ -2537,6 +2548,8 @@ mod tests { cleanup_data_dir: Some(Uuid::new_v4()), sign: Some(vec![0x01, 0x02, 0x03]), old_current_size, + old_current_source_checked: false, + old_current_source: None, }; let encoded = rmp_serde::encode::to_vec_named(&resp).expect("named msgpack should encode"); @@ -2574,6 +2587,8 @@ mod tests { assert_eq!(decoded.cleanup_data_dir, None); assert_eq!(decoded.sign, legacy.sign); assert_eq!(decoded.old_current_size, None); + assert!(!decoded.old_current_source_checked); + assert!(decoded.old_current_source.is_none()); } /// Test constants diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index 14a56b598..9c94610be 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -486,18 +486,25 @@ pub(crate) mod prepare_rename_test_hooks { /// Fsync a directory so recently created or renamed entries survive power loss. /// No-op on non-Unix platforms where directories cannot be opened for syncing. pub fn fsync_dir_std(dir: impl AsRef) -> io::Result<()> { + fsync_directory_handle_std(dir.as_ref(), None) +} + +pub(crate) fn fsync_directory_handle_std(dir: &Path, opened: Option<&std::fs::File>) -> io::Result<()> { #[cfg(test)] - fsync_dir_recorder::record(dir.as_ref()); + fsync_dir_recorder::record(dir); #[cfg(unix)] { #[cfg(test)] - if let Some(kind) = fsync_dir_recorder::take_failure(dir.as_ref()) { + if let Some(kind) = fsync_dir_recorder::take_failure(dir) { return Err(io::Error::from(kind)); } - std::fs::File::open(dir.as_ref())?.sync_all()?; + match opened { + Some(file) => file.sync_all()?, + None => std::fs::File::open(dir)?.sync_all()?, + } } #[cfg(not(unix))] - let _ = dir; + let _ = (dir, opened); Ok(()) } @@ -670,6 +677,8 @@ fn file_fdatasync_group_commit_wait() -> Duration { #[derive(Clone, Eq, Hash, PartialEq)] struct DstDirFsyncGroupKey { canonical_path: PathBuf, + // A batch must execute in the same resource class as its producers. + isolated: bool, #[cfg(unix)] dev: u64, #[cfg(unix)] @@ -686,13 +695,17 @@ impl DstDirFsyncGroupKey { use std::os::unix::fs::MetadataExt; Ok(Self { canonical_path, + isolated: FSYNC_ON_CURRENT_RUNTIME.get(), dev: metadata.dev(), ino: metadata.ino(), }) } #[cfg(not(unix))] { - Ok(Self { canonical_path }) + Ok(Self { + canonical_path, + isolated: FSYNC_ON_CURRENT_RUNTIME.get(), + }) } } } @@ -729,6 +742,17 @@ impl OpenedDstDirFsyncGroup { Ok(Self { key, dir }) } } + + #[cfg(target_os = "linux")] + fn from_file(dir: &Path, file: Arc) -> io::Result { + let key = DstDirFsyncGroupKey::from_metadata(dir.canonicalize()?, file.metadata()?)?; + Ok(Self { + #[cfg(test)] + dir: key.canonical_path.clone(), + key, + dir_file: file, + }) + } } struct DstDirFsyncWaiter { @@ -1002,19 +1026,45 @@ fn run_dst_dir_fsync_group_worker(group: Arc) -> impl std::fut } } +#[cfg(any(test, not(target_os = "linux")))] async fn fsync_dst_dir_group_commit_with_enabled( dir: impl AsRef, enabled: bool, namespace_owner: Option>, +) -> io::Result<()> { + fsync_dst_dir_group_commit_opened(dir.as_ref(), enabled, namespace_owner, None).await +} + +async fn fsync_dst_dir_group_commit_opened( + dir: &Path, + enabled: bool, + namespace_owner: Option>, + opened: Option>, ) -> io::Result<()> { if !enabled { - return fsync_dir_with_owner(dir.as_ref(), namespace_owner).await; + if let Some(file) = opened { + let dir = dir.to_path_buf(); + return fsync_spawn_blocking(move || { + let _owner = namespace_owner; + fsync_directory_handle_std(&dir, Some(&file)) + }) + .await?; + } + return fsync_dir_with_owner(dir, namespace_owner).await; } - let dir = dir.as_ref().to_path_buf(); - let opened = tokio::task::spawn_blocking(move || OpenedDstDirFsyncGroup::open(&dir)) - .await - .map_err(|err| io::Error::other(format!("blocking dst dir group open failed: {err}")))??; + let dir = dir.to_path_buf(); + let opened = tokio::task::spawn_blocking(move || { + #[cfg(target_os = "linux")] + if let Some(file) = opened { + return OpenedDstDirFsyncGroup::from_file(&dir, file); + } + #[cfg(not(target_os = "linux"))] + let _ = opened; + OpenedDstDirFsyncGroup::open(&dir) + }) + .await + .map_err(|err| io::Error::other(format!("blocking dst dir group open failed: {err}")))??; let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened, namespace_owner)?; if let Some(group) = worker { tokio::spawn(run_dst_dir_fsync_group_worker(group)); @@ -1027,6 +1077,49 @@ async fn fsync_dst_dir_group_commit_with_enabled( } } +/// Reuse the published directory without bypassing group commit or sync admission. +pub(crate) async fn fsync_commit_directory( + dir: &Path, + guard: &RenameCommitGuard, + lease: Arc, + admission: Option<&FileSyncAdmission>, +) -> io::Result<()> { + #[cfg(target_os = "linux")] + { + if dir != guard.directories.destination_path { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "sync directory does not match commit guard")); + } + let file = guard + .directories + .destination + .get() + .ok_or_else(|| io::Error::other("commit directory was not published"))? + .file + .clone(); + if !dst_dir_fsync_group_commit_enabled() + && let Some(admission) = admission + { + let dir = dir.to_path_buf(); + return run_blocking_namespace_file_sync_operation(lease, admission, move || { + #[cfg(test)] + fsync_dir_recorder::record_limited(&dir); + fsync_directory_handle_std(&dir, Some(&file)) + }) + .await; + } + fsync_dst_dir_group_commit_opened(dir, dst_dir_fsync_group_commit_enabled(), Some(lease), Some(file)).await + } + #[cfg(not(target_os = "linux"))] + { + let _ = guard; + match admission { + Some(admission) => fsync_dst_dir_group_commit_or_namespace_file_sync_limit(dir, lease, admission).await, + None => fsync_dst_dir_group_commit(dir, Some(lease)).await, + } + } +} + +#[cfg(not(target_os = "linux"))] pub(crate) async fn fsync_dst_dir_group_commit( dir: impl AsRef, namespace_owner: Option>, @@ -1034,6 +1127,7 @@ pub(crate) async fn fsync_dst_dir_group_commit( fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled(), namespace_owner).await } +#[cfg(not(target_os = "linux"))] pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit( dir: impl AsRef, lease: Arc, @@ -1061,7 +1155,7 @@ fn clear_dst_dir_fsync_group_commit_for_test() { DST_DIR_FSYNC_GROUP_COMMIT.clear_for_test(); } -type FileFdatasyncGroupKey = usize; +type FileFdatasyncGroupKey = (usize, bool); struct FileFdatasyncWaiter { files: Vec, @@ -1133,7 +1227,7 @@ impl FileFdatasyncGroupCommit { )); } let (result_tx, result_rx) = oneshot::channel(); - let key = Arc::as_ptr(&disk_permits) as FileFdatasyncGroupKey; + let key = (Arc::as_ptr(&disk_permits) as usize, FSYNC_ON_CURRENT_RUNTIME.get()); let mut registry = self.inner.lock(); registry.groups.retain(|_, group| group.disk_permits.strong_count() > 0); if registry.total_waiters >= MAX_FILE_FDATASYNC_WAITERS { @@ -1371,9 +1465,26 @@ static FSYNC_RUNTIME: LazyLock> = LazyLock::new( } }); +thread_local! { + static FSYNC_ON_CURRENT_RUNTIME: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Called only when starting threads belonging to an isolated I/O runtime. +/// Async and blocking threads use the same runtime's blocking budget for fsync. +pub(crate) fn use_current_runtime_for_fsync() { + FSYNC_ON_CURRENT_RUNTIME.set(true); +} + +pub(super) fn is_isolated_io_thread() -> bool { + FSYNC_ON_CURRENT_RUNTIME.get() +} + /// Spawn a blocking task on the fsync-dedicated runtime if configured, /// otherwise fall back to the main tokio blocking pool. fn fsync_spawn_blocking(f: impl FnOnce() -> T + Send + 'static) -> tokio::task::JoinHandle { + if FSYNC_ON_CURRENT_RUNTIME.get() { + return tokio::task::spawn_blocking(f); + } match FSYNC_RUNTIME.as_ref() { Some(rt) => rt.spawn_blocking(f), None => tokio::task::spawn_blocking(f), @@ -2268,6 +2379,20 @@ pub(crate) fn create_prepared_rename_source_with_commit_guard( #[cfg(not(windows))] { let _ = (dst_file_path, commit_guard); + #[cfg(target_os = "linux")] + let source = { + use rustix::fs::{Mode, OFlags}; + let directories = &commit_guard.directories; + let name = directories.file_name(src_file_path, true)?; + directories.file_name(dst_file_path, false)?; + open_linux_relative( + &directories.source.file, + Path::new(name), + OFlags::CREATE | OFlags::TRUNC | OFlags::WRONLY, + Mode::from_raw_mode(0o666), + )? + }; + #[cfg(not(target_os = "linux"))] let source = std::fs::OpenOptions::new() .create(true) .write(true) @@ -2288,18 +2413,39 @@ pub(crate) fn create_prepared_rename_source_with_commit_guard( } } -#[cfg(windows)] +#[cfg(any(windows, target_os = "linux"))] pub(crate) fn read_destination_file_with_commit_guard( file_path: &Path, commit_guard: &RenameCommitGuard, ) -> io::Result>> { - if file_path.parent() != Some(commit_guard.destination_parent.as_path()) { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "destination file parent does not match its commit guard", - )); + #[cfg(target_os = "linux")] + { + use rustix::fs::{Mode, OFlags}; + let directories = &commit_guard.directories; + let name = directories.file_name(file_path, false)?; + let Some(parent) = directories.destination.get() else { + return Ok(None); + }; + return match open_linux_relative(&parent.file, Path::new(name), OFlags::RDONLY, Mode::empty()) { + Ok(mut file) => { + let mut data = Vec::new(); + std::io::Read::read_to_end(&mut file, &mut data)?; + Ok(Some(data)) + } + Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(err), + }; + } + #[cfg(windows)] + { + if file_path.parent() != Some(commit_guard.destination_parent.as_path()) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "destination file parent does not match its commit guard", + )); + } + read_windows_relative_file(file_path, &commit_guard.destination_parent_guard) } - read_windows_relative_file(file_path, &commit_guard.destination_parent_guard) } #[cfg(windows)] @@ -2337,23 +2483,40 @@ pub(crate) async fn rename_all_with_prepared_source( dst_file_path: impl AsRef, base_dir: impl AsRef, publication_root: &PublicationRoot, - _commit_guard: &RenameCommitGuard, + commit_guard: &RenameCommitGuard, lease: Arc, ) -> Result<()> { let src_file_path = src_file_path.as_ref().to_path_buf(); let dst_file_path = dst_file_path.as_ref().to_path_buf(); let base_dir = base_dir.as_ref().to_path_buf(); + #[cfg(not(target_os = "linux"))] let publication_root = publication_root.clone(); + #[cfg(target_os = "linux")] + let _ = publication_root; + let commit_guard = commit_guard.clone(); let operation = { let src_file_path = src_file_path.clone(); let dst_file_path = dst_file_path.clone(); + #[cfg(not(target_os = "linux"))] let base_dir = base_dir.clone(); move || { - validate_prepared_rename_source(&prepared_source, &src_file_path)?; - let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?; - #[cfg(any(test, feature = "test-util"))] - prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path); - rename_prepared(&src_file_path, &dst_file_path, &preparation) + #[cfg(target_os = "linux")] + { + #[cfg(any(test, feature = "test-util"))] + prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path); + return commit_guard + .directories + .rename(&src_file_path, &dst_file_path, Some(&prepared_source)); + } + #[cfg(not(target_os = "linux"))] + { + let _ = commit_guard; + validate_prepared_rename_source(&prepared_source, &src_file_path)?; + let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?; + #[cfg(any(test, feature = "test-util"))] + prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path); + rename_prepared(&src_file_path, &dst_file_path, &preparation) + } } }; let result = run_blocking_namespace_operation(lease, operation).await; @@ -2370,10 +2533,35 @@ pub(crate) async fn rename_all_with_commit_guard( dst_file_path: impl AsRef, base_dir: impl AsRef, publication_root: &PublicationRoot, - _commit_guard: &RenameCommitGuard, + commit_guard: &RenameCommitGuard, lease: Arc, ) -> Result<()> { - rename_all_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await + #[cfg(target_os = "linux")] + { + let source = src_file_path.as_ref().to_path_buf(); + let destination = dst_file_path.as_ref().to_path_buf(); + let guard = commit_guard.clone(); + let _ = (base_dir, publication_root); + return run_blocking_namespace_operation(lease, move || { + guard.directories.prepare_destination()?; + #[cfg(test)] + prepared_publication_test_hooks::run_rename_destination(&source, &destination); + #[cfg(any(test, feature = "test-util"))] + { + prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &source); + prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &destination); + } + guard.directories.rename(&source, &destination, None) + }) + .await + .map_err(to_file_error) + .map_err(DiskError::from); + } + #[cfg(not(target_os = "linux"))] + { + let _ = commit_guard; + rename_all_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await + } } #[tracing::instrument(level = "debug", skip_all)] @@ -2501,7 +2689,7 @@ async fn reliable_rename_inner_with_lease( result } -#[cfg(not(windows))] +#[cfg(not(any(windows, target_os = "linux")))] fn validate_prepared_rename_source(prepared_source: &PreparedRenameSource, src_file_path: &Path) -> io::Result<()> { if prepared_source.path != src_file_path { return Err(io::Error::new( @@ -2701,6 +2889,8 @@ pub(crate) async fn fsync_dir_with_namespace_file_sync_limit( struct RenamePreparation { parent_guard: Option, + #[cfg(target_os = "linux")] + publication_root: PublicationRoot, #[cfg(windows)] _source_parent_guard: ExistingBaseDirectoryGuard, #[cfg(windows)] @@ -2709,15 +2899,32 @@ struct RenamePreparation { #[cfg(not(windows))] fn prepare_rename_with_retry( - src_file_path: &Path, + _src_file_path: &Path, dst_file_path: &Path, base_dir: &Path, publication_root: &PublicationRoot, ) -> io::Result { - let prune_budget = prepare_prune_budget(dst_file_path, base_dir); + let parent_guard = dst_file_path + .parent() + .map(|parent| prepare_destination_parent_with_retry(parent, base_dir, publication_root)) + .transpose()?; + Ok(RenamePreparation { + parent_guard, + #[cfg(target_os = "linux")] + publication_root: publication_root.clone(), + }) +} + +#[cfg(not(windows))] +fn prepare_destination_parent_with_retry( + parent: &Path, + base_dir: &Path, + publication_root: &PublicationRoot, +) -> io::Result { + let prune_budget = parent_prune_budget(parent, base_dir); let mut attempt = 0; loop { - match prepare_rename(src_file_path, dst_file_path, base_dir, publication_root) { + match mkdir_all_below_existing_base_std(parent, base_dir, publication_root) { Ok(preparation) => return Ok(preparation), Err(err) if should_retry_prepare(&err, attempt, prune_budget) => { attempt += 1; @@ -2807,20 +3014,6 @@ fn prepare_rename_with_retry( }) } -#[cfg(not(windows))] -fn prepare_rename( - _src_file_path: &Path, - dst_file_path: &Path, - base_dir: &Path, - publication_root: &PublicationRoot, -) -> io::Result { - let parent_guard = dst_file_path - .parent() - .map(|parent| mkdir_all_below_existing_base_std(parent, base_dir, publication_root)) - .transpose()?; - Ok(RenamePreparation { parent_guard }) -} - /// Publish a prepared rename. The retry budget starts fresh here: preparation /// keeps its own counter, so a chain rebuilt after a concurrent prune must not /// cost the rename its one retry. @@ -2836,9 +3029,23 @@ fn rename_prepared(_src_file_path: &Path, dst_file_path: &Path, preparation: &Re #[cfg(not(windows))] { + // A missing source parent is not a destination-prune race. Open it once + // outside both retry loops, retaining it through any rename retry. + #[cfg(target_os = "linux")] + let source_parent = preparation.publication_root.open_directory( + _src_file_path + .parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "missing rename source parent"))?, + )?; let mut attempt = 0; loop { - let rename_result = rename_into_existing_parent(_src_file_path, dst_file_path, preparation.parent_guard.as_ref()); + let rename_result = rename_into_existing_parent( + _src_file_path, + dst_file_path, + preparation.parent_guard.as_ref(), + #[cfg(target_os = "linux")] + &source_parent, + ); match rename_result { Ok(()) => return Ok(()), Err(err) if should_retry_rename(&err, attempt) => { @@ -2877,8 +3084,9 @@ fn rename_into_existing_parent( src_file_path: &Path, dst_file_path: &Path, parent_guard: Option<&ExistingBaseDirectoryGuard>, + #[cfg(target_os = "linux")] source_parent: &std::fs::File, ) -> io::Result<()> { - use rustix::fs::{Mode, OFlags, open, renameat}; + use rustix::fs::renameat; let Some(parent_guard) = parent_guard else { let rename_started = rustfs_io_metrics::put_stage_timer(); @@ -2889,6 +3097,7 @@ fn rename_into_existing_parent( ); return result; }; + #[cfg(not(target_os = "linux"))] let src_parent = src_file_path .parent() .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a parent directory"))?; @@ -2898,10 +3107,11 @@ fn rename_into_existing_parent( let dst_name = dst_file_path .file_name() .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a file name"))?; - let src_parent = open( + #[cfg(not(target_os = "linux"))] + let src_parent = rustix::fs::open( src_parent, - OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, - Mode::empty(), + rustix::fs::OFlags::RDONLY | rustix::fs::OFlags::DIRECTORY | rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::CLOEXEC, + rustix::fs::Mode::empty(), ) .map_err(io::Error::from)?; let dst_parent = parent_guard @@ -2909,6 +3119,9 @@ fn rename_into_existing_parent( .ok_or_else(|| io::Error::other("rename destination parent guard is empty"))?; let rename_started = rustfs_io_metrics::put_stage_timer(); + #[cfg(target_os = "linux")] + let result = renameat(source_parent, src_name, dst_parent, dst_name).map_err(io::Error::from); + #[cfg(not(target_os = "linux"))] let result = renameat(&src_parent, src_name, dst_parent, dst_name).map_err(io::Error::from); rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL, @@ -3091,16 +3304,18 @@ struct WindowsDirectoryHandle { /// Stable root for namespace-changing disk operations. /// -/// Windows opens the configured endpoint once and keeps that directory identity -/// pinned for the lifetime of the disk. Publication then resolves every source -/// and destination component relative to this handle instead of re-entering the -/// mutable pathname namespace. Other platforms retain the path so callers use a -/// uniform API while their existing `openat`/`renameat` guards remain unchanged. +/// Linux and Windows pin the configured directory for the lifetime of the disk. +/// Linux also exposes a procfs alias for operations not yet using relative IO; +/// both access forms share one root handle and therefore one mount identity. #[derive(Clone)] pub(crate) struct PublicationRoot { - path: PathBuf, + path: Arc, + #[cfg(target_os = "linux")] + directory: Arc, + #[cfg(target_os = "linux")] + io_path: Arc, #[cfg(windows)] - configured_path: PathBuf, + configured_path: Arc, #[cfg(windows)] directory: WindowsDirectoryHandle, } @@ -3114,15 +3329,34 @@ impl PublicationRoot { #[cfg(windows)] let (resolved_path, directory) = open_windows_publication_root(path)?; + #[cfg(target_os = "linux")] + let directory = { + use rustix::fs::{Mode, OFlags, open}; + Arc::new(std::fs::File::from(open( + path, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + )?)) + }; + #[cfg(target_os = "linux")] + let io_path = { + use std::os::fd::AsRawFd; + Arc::new(PathBuf::from(format!("/proc/self/fd/{}/.", directory.as_raw_fd()))) + }; + Ok(Self { #[cfg(not(windows))] - path: path.to_path_buf(), + path: Arc::new(path.to_path_buf()), #[cfg(windows)] - path: resolved_path, + path: Arc::new(resolved_path), #[cfg(windows)] - configured_path: path.to_path_buf(), + configured_path: Arc::new(path.to_path_buf()), #[cfg(windows)] directory, + #[cfg(target_os = "linux")] + directory, + #[cfg(target_os = "linux")] + io_path, }) } @@ -3130,15 +3364,170 @@ impl PublicationRoot { &self.path } + #[cfg(target_os = "linux")] + pub(crate) fn directory(&self) -> &Arc { + &self.directory + } + + #[cfg(target_os = "linux")] + pub(crate) fn io_path(&self) -> &Path { + &self.io_path + } + + pub(crate) fn open_readonly(&self, path: &Path) -> io::Result { + #[cfg(target_os = "linux")] + return self.open_relative(path, rustix::fs::OFlags::RDONLY, rustix::fs::Mode::empty()); + #[cfg(not(target_os = "linux"))] + std::fs::File::open(path) + } + + pub(crate) fn create_truncate(&self, path: &Path) -> io::Result { + #[cfg(target_os = "linux")] + { + use rustix::fs::{Mode, OFlags}; + self.open_relative(path, OFlags::CREATE | OFlags::WRONLY | OFlags::TRUNC, Mode::from_raw_mode(0o666)) + } + #[cfg(not(target_os = "linux"))] + std::fs::OpenOptions::new().create(true).write(true).truncate(true).open(path) + } + + pub(crate) fn create_truncate_with_parent(&self, path: &Path) -> io::Result<(std::fs::File, Option)> { + #[cfg(target_os = "linux")] + { + use rustix::fs::{Mode, OFlags}; + let parent = self.open_directory( + path.parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "missing file parent"))?, + )?; + let name = path + .file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "missing file name"))?; + let file = open_linux_relative( + &parent, + Path::new(name), + OFlags::CREATE | OFlags::WRONLY | OFlags::TRUNC, + Mode::from_raw_mode(0o666), + )?; + Ok((file, Some(parent))) + } + #[cfg(not(target_os = "linux"))] + Ok((self.create_truncate(path)?, None)) + } + + #[cfg(target_os = "linux")] + fn open_relative(&self, path: &Path, flags: rustix::fs::OFlags, mode: rustix::fs::Mode) -> io::Result { + let relative = path + .strip_prefix(self.io_path.as_path()) + .or_else(|_| path.strip_prefix(self.path.as_path())) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path must remain below its publication root"))?; + open_linux_relative(&self.directory, relative, flags, mode) + } + + #[cfg(target_os = "linux")] + fn open_directory(&self, path: &Path) -> io::Result { + self.open_relative( + path, + rustix::fs::OFlags::RDONLY | rustix::fs::OFlags::DIRECTORY, + rustix::fs::Mode::empty(), + ) + } + #[cfg(windows)] fn relative_path<'a>(&self, path: &'a Path) -> io::Result<&'a Path> { // The configured path only derives a suffix; traversal stays rooted at the pinned directory handle. - path.strip_prefix(&self.path) - .or_else(|_| path.strip_prefix(&self.configured_path)) + path.strip_prefix(self.path.as_path()) + .or_else(|_| path.strip_prefix(self.configured_path.as_path())) .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path must remain below its publication root")) } } +/// Open the actual IO handle with the same containment policy as path validation. +/// Older kernels retain fd-relative traversal; only ENOSYS enables the fallback. +#[cfg(target_os = "linux")] +fn open_linux_relative( + directory: &std::fs::File, + relative: &Path, + flags: rustix::fs::OFlags, + mode: rustix::fs::Mode, +) -> io::Result { + use rustix::fs::{OFlags, ResolveFlags, openat2}; + if relative + .components() + .any(|component| !matches!(component, Component::Normal(_) | Component::CurDir)) + { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "invalid relative disk path")); + } + let relative = if relative.as_os_str().is_empty() { + Path::new(".") + } else { + relative + }; + let flags = flags | OFlags::CLOEXEC | OFlags::NOFOLLOW; + match openat2(directory, relative, flags, mode, ResolveFlags::BENEATH | ResolveFlags::NO_SYMLINKS) { + Ok(fd) => Ok(fd.into()), + Err(rustix::io::Errno::NOSYS) => open_linux_relative_legacy(directory, relative, flags, mode), + Err(rustix::io::Errno::LOOP | rustix::io::Errno::XDEV) => Err(DiskError::InvalidPath.into()), + Err(err) => Err(err.into()), + } +} + +#[cfg(target_os = "linux")] +fn open_linux_relative_legacy( + directory: &std::fs::File, + relative: &Path, + flags: rustix::fs::OFlags, + mode: rustix::fs::Mode, +) -> io::Result { + use rustix::fs::OFlags; + + let mut parent = None; + let mut components = relative + .components() + .filter_map(|component| match component { + Component::Normal(name) => Some(name), + _ => None, + }) + .peekable(); + while let Some(name) = components.next() { + let current = parent.as_ref().unwrap_or(directory); + if components.peek().is_none() { + return open_linux_relative_component(current, name, flags, mode); + } + parent = Some(open_linux_relative_component( + current, + name, + OFlags::PATH | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + rustix::fs::Mode::empty(), + )?); + } + open_linux_relative_component(directory, std::ffi::OsStr::new("."), flags, mode) +} + +#[cfg(target_os = "linux")] +fn open_linux_relative_component( + directory: &std::fs::File, + name: &std::ffi::OsStr, + flags: rustix::fs::OFlags, + mode: rustix::fs::Mode, +) -> io::Result { + use rustix::fs::{AtFlags, FileType, openat, statat}; + use rustix::io::Errno; + match openat(directory, name, flags, mode) { + Ok(fd) => Ok(fd.into()), + Err(Errno::LOOP) => Err(DiskError::InvalidPath.into()), + // O_DIRECTORY | O_NOFOLLOW reports ENOTDIR for an intermediate link. + // Inspect only this failure so ordinary non-directory errors retain + // their classification without adding a stat to successful opens. + Err(Errno::NOTDIR) + if statat(directory, name, AtFlags::SYMLINK_NOFOLLOW) + .is_ok_and(|metadata| FileType::from_raw_mode(metadata.st_mode) == FileType::Symlink) => + { + Err(DiskError::InvalidPath.into()) + } + Err(err) => Err(err.into()), + } +} + #[cfg(windows)] #[derive(Clone)] pub(crate) struct ExistingBaseDirectoryGuard { @@ -3176,6 +3565,8 @@ pub(crate) type ExistingBaseDirectoryGuard = (); #[derive(Clone)] pub(crate) struct RenameCommitGuard { + #[cfg(target_os = "linux")] + directories: Arc, #[cfg(windows)] source_parent: PathBuf, #[cfg(windows)] @@ -3186,6 +3577,108 @@ pub(crate) struct RenameCommitGuard { destination_parent_guard: ExistingBaseDirectoryGuard, } +#[cfg(target_os = "linux")] +struct LinuxCommitDirectory { + file: Arc, + device: u64, + inode: u64, +} + +#[cfg(target_os = "linux")] +impl LinuxCommitDirectory { + fn new(file: std::fs::File) -> io::Result { + use std::os::unix::fs::MetadataExt; + let metadata = file.metadata()?; + Ok(Self { + file: Arc::new(file), + device: metadata.dev(), + inode: metadata.ino(), + }) + } +} + +#[cfg(target_os = "linux")] +struct LinuxCommitDirectories { + root: PublicationRoot, + source_path: PathBuf, + destination_path: PathBuf, + source: LinuxCommitDirectory, + destination: std::sync::OnceLock, + destination_base: PathBuf, +} + +#[cfg(target_os = "linux")] +impl LinuxCommitDirectories { + fn prepare_destination(&self) -> io::Result<&LinuxCommitDirectory> { + if let Some(directory) = self.destination.get() { + return Ok(directory); + } + let mut parents = prepare_destination_parent_with_retry(&self.destination_path, &self.destination_base, &self.root)?; + let directory = std::fs::File::from( + parents + .pop() + .ok_or_else(|| io::Error::other("missing destination directory guard"))?, + ); + let directory = LinuxCommitDirectory::new(directory)?; + Ok(self.destination.get_or_init(|| directory)) + } + + fn file_name<'a>(&self, path: &'a Path, source: bool) -> io::Result<&'a std::ffi::OsStr> { + let parent = if source { &self.source_path } else { &self.destination_path }; + if path.parent() != Some(parent) { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "file parent does not match its commit guard")); + } + path.file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "missing file name")) + } + + fn validate(&self) -> io::Result<()> { + use std::os::unix::fs::MetadataExt; + // A pinned directory may have been unlinked or replaced. Do not report a + // successful publication into a detached object tree. + for (path, directory) in [ + (&self.source_path, &self.source), + (&self.destination_path, self.prepare_destination()?), + ] { + let current = self.root.open_directory(path)?.metadata()?; + if (current.dev(), current.ino()) != (directory.device, directory.inode) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "commit directory identity changed")); + } + } + Ok(()) + } + + fn rename(&self, source: &Path, destination: &Path, prepared: Option<&PreparedRenameSource>) -> io::Result<()> { + let source_name = self.file_name(source, true)?; + let destination_name = self.file_name(destination, false)?; + let destination_parent = self.prepare_destination()?; + self.validate()?; + let mut attempt = 0; + loop { + if let Some(prepared) = prepared { + let metadata = rustix::fs::statat(&*self.source.file, source_name, rustix::fs::AtFlags::SYMLINK_NOFOLLOW)?; + if prepared.path != source || metadata.st_dev != prepared.device || metadata.st_ino != prepared.inode { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "rename source identity changed while publication was prepared", + )); + } + } + let started = rustfs_io_metrics::put_stage_timer(); + let result = rustix::fs::renameat(&*self.source.file, source_name, &*destination_parent.file, destination_name) + .map_err(io::Error::from); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL, + started, + ); + match result { + Err(err) if should_retry_rename(&err, attempt) => attempt += 1, + result => return result, + } + } + } +} + pub(crate) struct RenameDestinationPathGuard { #[cfg(windows)] directory: PathBuf, @@ -3385,6 +3878,29 @@ pub(crate) fn prepare_rename_commit_guard( destination_base: &Path, publication_root: &PublicationRoot, ) -> io::Result { + #[cfg(target_os = "linux")] + { + existing_base_directory_suffix(destination_parent, destination_base)?; + let source = LinuxCommitDirectory::new(publication_root.open_directory(source_parent)?)?; + let destination = std::sync::OnceLock::new(); + match publication_root.open_directory(destination_parent) { + Ok(directory) => { + let _ = destination.set(LinuxCommitDirectory::new(directory)?); + } + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => return Err(err), + } + return Ok(RenameCommitGuard { + directories: Arc::new(LinuxCommitDirectories { + root: publication_root.clone(), + source_path: source_parent.to_path_buf(), + destination_path: destination_parent.to_path_buf(), + source, + destination, + destination_base: destination_base.to_path_buf(), + }), + }); + } #[cfg(windows)] { // A same-directory rename must use one shared-write parent handle: @@ -3413,7 +3929,7 @@ pub(crate) fn prepare_rename_commit_guard( }) } - #[cfg(not(windows))] + #[cfg(not(any(windows, target_os = "linux")))] { let _ = (source_parent, destination_parent, destination_base, publication_root); Ok(RenameCommitGuard {}) @@ -4209,11 +4725,7 @@ fn windows_rename_source_is_allowed(attributes: u32, reparse_tag: u32) -> bool { attributes & FILE_ATTRIBUTE_REPARSE_POINT == 0 || reparse_tag == IO_REPARSE_TAG_DEDUP } -pub(crate) fn mkdir_all_below_existing_base_std( - dir_path: &Path, - base_dir: &Path, - publication_root: &PublicationRoot, -) -> io::Result { +fn existing_base_directory_suffix<'a>(dir_path: &'a Path, base_dir: &Path) -> io::Result<&'a Path> { let relative = dir_path .strip_prefix(base_dir) .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must remain below its base directory"))?; @@ -4226,15 +4738,31 @@ pub(crate) fn mkdir_all_below_existing_base_std( } } + Ok(relative) +} + +pub(crate) fn mkdir_all_below_existing_base_std( + dir_path: &Path, + base_dir: &Path, + publication_root: &PublicationRoot, +) -> io::Result { + let relative = existing_base_directory_suffix(dir_path, base_dir)?; + #[cfg(unix)] { - let _ = publication_root; - use rustix::fs::{Mode, OFlags, mkdirat, open, openat}; + use rustix::fs::{Mode, OFlags, mkdirat, openat}; use rustix::io::Errno; let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC; let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO; - let mut parents = vec![open(base_dir, flags, Mode::empty()).map_err(io::Error::from)?]; + #[cfg(target_os = "linux")] + let base = publication_root.open_directory(base_dir)?.into(); + #[cfg(not(target_os = "linux"))] + let base = { + let _ = publication_root; + rustix::fs::open(base_dir, flags, Mode::empty()).map_err(io::Error::from)? + }; + let mut parents = vec![base]; #[cfg(test)] let mut walked_path = base_dir.to_path_buf(); @@ -4354,10 +4882,18 @@ fn should_retry_rename(err: &io::Error, attempt: usize) -> bool { /// steal an attempt by making that same upward progress. A destination whose /// parent *is* the base gets a budget of zero, keeping `NotFound` immediately /// terminal for speculative cleanup renames. +#[cfg(any(windows, test))] fn prepare_prune_budget(dst_file_path: &Path, base_dir: &Path) -> usize { dst_file_path .parent() - .and_then(|parent| parent.strip_prefix(base_dir).ok()) + .map(|parent| parent_prune_budget(parent, base_dir)) + .unwrap_or(0) +} + +fn parent_prune_budget(parent: &Path, base_dir: &Path) -> usize { + parent + .strip_prefix(base_dir) + .ok() .map(|relative| relative.components().count()) .unwrap_or(0) } @@ -4484,6 +5020,60 @@ mod tests { use tempfile::tempdir; use tracing_subscriber::fmt::MakeWriter; + #[test] + fn fsync_groups_do_not_mix_cleanup_and_foreground_workers() { + let dir = tempdir().unwrap(); + let dst_groups = Arc::new(DstDirFsyncGroupCommit::default()); + let file_groups = Arc::new(FileFdatasyncGroupCommit::default()); + let disk_permits = Arc::new(Semaphore::new(1)); + let (_, dst_worker) = dst_groups.enqueue_for_test(dir.path()).unwrap(); + let (_, file_worker) = file_groups + .enqueue(disk_permits.clone(), vec![dir.path().join("one")]) + .unwrap(); + assert!(dst_worker.is_some()); + assert!(file_worker.is_some()); + let cleanup_dst_groups = dst_groups.clone(); + let cleanup_file_groups = file_groups.clone(); + std::thread::spawn(move || { + use_current_runtime_for_fsync(); + let (_, dst_worker) = cleanup_dst_groups.enqueue_for_test(dir.path()).unwrap(); + let (_, file_worker) = cleanup_file_groups + .enqueue(disk_permits, vec![dir.path().join("two")]) + .unwrap(); + assert!(dst_worker.is_some(), "cleanup must not join a foreground directory worker"); + assert!(file_worker.is_some(), "cleanup must not join a foreground file worker"); + }) + .join() + .unwrap(); + assert_eq!(dst_groups.inner.lock().groups.len(), 2); + assert_eq!(file_groups.inner.lock().groups.len(), 2); + } + + #[tokio::test] + async fn isolated_runtime_keeps_fsync_on_its_own_blocking_pool() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .max_blocking_threads(1) + .thread_name("isolated-fsync-test") + .on_thread_start(use_current_runtime_for_fsync) + .enable_all() + .build() + .unwrap(); + let result = runtime + .spawn(async { + fsync_spawn_blocking(|| { + assert_eq!(std::thread::current().name(), Some("isolated-fsync-test")); + }) + .await + .unwrap(); + }) + .await; + // A runtime cannot be dropped from another runtime's async context. + runtime.shutdown_background(); + result.unwrap(); + assert!(!FSYNC_ON_CURRENT_RUNTIME.get(), "the calling runtime must retain normal fsync routing"); + } + fn file_sync_limiter() -> Arc { Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS)) } @@ -4493,7 +5083,11 @@ mod tests { .first() .expect("test publication root requires at least one path") .to_path_buf(); - while !paths.iter().all(|path| path.starts_with(&common)) { + // The disk root must exist independently of a missing or symlinked + // rename base; those are inputs whose rejection the tests exercise. + while !paths.iter().all(|path| path.starts_with(&common)) + || !std::fs::symlink_metadata(&common).is_ok_and(|metadata| metadata.is_dir()) + { assert!(common.pop(), "test paths must share an absolute root"); } PublicationRoot::new(&common).expect("test publication root should open") @@ -7737,3 +8331,172 @@ mod tests { assert_eq!(err.kind(), io::ErrorKind::NotFound); } } + +#[cfg(all(test, target_os = "linux"))] +mod fd_relative_tests { + use super::*; + use std::io::{Read, Write}; + + #[tokio::test] + async fn guarded_first_publication_recovers_from_a_concurrent_parent_prune() { + let temp = tempfile::tempdir().expect("temporary disk"); + let root = PublicationRoot::new(temp.path()).expect("pinned root"); + let source = temp.path().join("tmp"); + let base = temp.path().join("bucket"); + let shared = base.join("prefix"); + let destination = shared.join("object"); + std::fs::create_dir(&source).expect("staging directory"); + std::fs::create_dir_all(&shared).expect("shared prefix"); + let guard = prepare_rename_commit_guard(&source, &destination, &base, &root).expect("commit directories"); + let src = source.join("xl.meta"); + let dst = destination.join("xl.meta"); + let mut staged = create_prepared_rename_source_with_commit_guard(&src, &dst, &guard).expect("staged metadata"); + staged.write_all(b"metadata", true).expect("write metadata"); + let pruned = shared.clone(); + prepare_rename_test_hooks::queue_after_component_opened(&shared, move || { + std::fs::remove_dir(pruned).expect("prune empty prefix"); + }); + let lease = acquire_namespace_mutation_lease(&dst).await; + rename_all_with_prepared_source(staged, &src, &dst, &base, &root, &guard, lease) + .await + .expect("publication must rebuild the pruned parent"); + assert_eq!(std::fs::read(dst).expect("published metadata"), b"metadata"); + assert!(!src.exists()); + } + + #[test] + fn legacy_relative_open_rejects_symlinks_and_opens_existing_components() { + use rustix::fs::{Mode, OFlags}; + let temp = tempfile::tempdir().expect("temporary disk"); + std::fs::create_dir(temp.path().join("parent")).expect("parent directory"); + std::fs::write(temp.path().join("parent/data"), b"data").expect("file data"); + std::os::unix::fs::symlink("parent", temp.path().join("link")).expect("directory link"); + std::os::unix::fs::symlink("data", temp.path().join("parent/link")).expect("file link"); + let root = PublicationRoot::new(temp.path()).expect("pinned root"); + let flags = OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC; + let mut bytes = Vec::new(); + open_linux_relative_legacy(root.directory(), Path::new("parent/data"), flags, Mode::empty()) + .expect("legacy open") + .read_to_end(&mut bytes) + .expect("legacy read"); + assert_eq!(bytes, b"data"); + for relative in ["link/data", "parent/link"] { + assert!(open_linux_relative_legacy(root.directory(), Path::new(relative), flags, Mode::empty()).is_err()); + } + } + + #[test] + fn root_relative_io_retains_mount_identity_after_path_replacement() { + let temp = tempfile::tempdir().expect("temporary disk"); + let disk = temp.path().join("disk"); + std::fs::create_dir(&disk).expect("disk root"); + std::fs::write(disk.join("data"), b"original").expect("original data"); + let root = PublicationRoot::new(&disk).expect("pinned root"); + let original = temp.path().join("original"); + std::fs::rename(&disk, &original).expect("move root"); + std::fs::create_dir(&disk).expect("replacement root"); + std::fs::write(disk.join("data"), b"replacement").expect("replacement data"); + for path in [disk.join("data"), root.io_path().join("data")] { + let mut actual = Vec::new(); + root.open_readonly(&path) + .expect("pinned read") + .read_to_end(&mut actual) + .expect("read bytes"); + assert_eq!(actual, b"original"); + } + let (mut file, parent) = root.create_truncate_with_parent(&disk.join("new")).expect("pinned creation"); + file.write_all(b"new").expect("write bytes"); + file.sync_data().expect("file sync"); + fsync_directory_handle_std(&disk, parent.as_ref()).expect("pinned parent sync"); + assert_eq!(std::fs::read(original.join("new")).expect("original disk file"), b"new"); + assert!(!disk.join("new").exists()); + } + + #[test] + fn root_relative_io_rejects_links_and_parent_escape_without_truncation() { + let temp = tempfile::tempdir().expect("temporary disk"); + let disk = temp.path().join("disk"); + std::fs::create_dir(&disk).expect("disk root"); + let outside = temp.path().join("outside"); + std::fs::write(&outside, b"preserved").expect("outside data"); + std::os::unix::fs::symlink(&outside, disk.join("link")).expect("file link"); + std::os::unix::fs::symlink(temp.path(), disk.join("parent")).expect("directory link"); + let root = PublicationRoot::new(&disk).expect("pinned root"); + for path in [disk.join("link"), disk.join("parent/outside"), disk.join("../outside")] { + assert!(root.open_readonly(&path).is_err(), "unsafe read: {path:?}"); + assert!(root.create_truncate(&path).is_err(), "unsafe write: {path:?}"); + } + assert_eq!(std::fs::read(&outside).expect("outside remains readable"), b"preserved"); + } + + #[tokio::test] + async fn guarded_publication_reuses_parents_and_syncs_without_changing_bytes() { + for grouped in [false, true] { + let _grouping = set_dst_dir_fsync_group_commit_for_test(grouped); + let temp = tempfile::tempdir().expect("temporary disk"); + let root = PublicationRoot::new(temp.path()).expect("pinned root"); + let source = temp.path().join("tmp"); + let destination = temp.path().join("bucket/object"); + let base = temp.path().join("bucket"); + std::fs::create_dir(&source).expect("staging directory"); + std::fs::create_dir(&base).expect("bucket directory"); + let guard = prepare_rename_commit_guard(&source, &destination, &base, &root).expect("commit directories"); + assert!(!destination.exists(), "preparation must not create the object directory"); + let src = source.join("xl.meta"); + let dst = destination.join("xl.meta"); + assert!( + read_destination_file_with_commit_guard(&dst, &guard) + .expect("absent read") + .is_none() + ); + let mut staged = create_prepared_rename_source_with_commit_guard(&src, &dst, &guard).expect("staged metadata"); + staged.write_all(b"metadata", true).expect("write metadata"); + let lease = acquire_namespace_mutation_lease(&dst).await; + rename_all_with_prepared_source(staged, &src, &dst, &base, &root, &guard, lease.clone()) + .await + .expect("publish"); + fsync_commit_directory(&destination, &guard, lease, None) + .await + .expect("directory sync"); + assert_eq!( + read_destination_file_with_commit_guard(&dst, &guard).expect("committed read"), + Some(b"metadata".to_vec()) + ); + assert!(!src.exists()); + assert_eq!(std::fs::read_dir(&destination).expect("object directory").count(), 1); + } + } + + #[tokio::test] + async fn guarded_publication_rejects_replaced_directory_and_source() { + for replace_directory in [false, true] { + let temp = tempfile::tempdir().expect("temporary disk"); + let root = PublicationRoot::new(temp.path()).expect("pinned root"); + let source = temp.path().join("tmp"); + let destination = temp.path().join("object"); + std::fs::create_dir(&source).expect("staging directory"); + std::fs::create_dir(&destination).expect("existing object directory"); + let guard = prepare_rename_commit_guard(&source, &destination, temp.path(), &root).expect("commit directories"); + let src = source.join("xl.meta"); + let dst = destination.join("xl.meta"); + std::fs::write(&dst, b"old").expect("old metadata"); + let mut staged = create_prepared_rename_source_with_commit_guard(&src, &dst, &guard).expect("staged metadata"); + staged.write_all(b"new", false).expect("new metadata"); + if replace_directory { + std::fs::rename(&destination, temp.path().join("detached")).expect("replace destination"); + std::fs::create_dir(&destination).expect("replacement directory"); + std::fs::write(&dst, b"old").expect("replacement metadata"); + } else { + std::fs::rename(&src, source.join("detached")).expect("replace staged file"); + std::fs::write(&src, b"unrelated").expect("replacement file"); + } + let lease = acquire_namespace_mutation_lease(&dst).await; + assert!( + rename_all_with_prepared_source(staged, &src, &dst, temp.path(), &root, &guard, lease) + .await + .is_err() + ); + assert_eq!(std::fs::read(&dst).expect("old metadata survives"), b"old"); + } + } +} diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 72af3fe60..2b99bd9c1 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -145,7 +145,7 @@ use std::{ time::{Duration, Instant}, }; use tokio::io::{AsyncRead, ReadBuf}; -use tokio::sync::RwLock; +use tokio::sync::{RwLock, oneshot}; use tokio::task::JoinSet; pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read"; @@ -2460,6 +2460,9 @@ pub(in crate::set_disk) struct RenameDataCommit { pub(in crate::set_disk) data_dir: Option, pub(in crate::set_disk) cleanup_disks: Vec>, pub(in crate::set_disk) old_current_size: Option, + pub(in crate::set_disk) old_current_source_checked: bool, + pub(in crate::set_disk) old_current_source_all_successful_checked: bool, + pub(in crate::set_disk) old_current_source: Option, pub(in crate::set_disk) committed_file_info: FileInfo, pub(in crate::set_disk) tail_drain: Option>>, } @@ -2475,6 +2478,500 @@ pub(in crate::set_disk) struct RenameTailOutcome { pub(in crate::set_disk) cleanup: Vec, } +const RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP"; +const RENAME_TAIL_CLEANUP_DEFER_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ENABLE"; +const RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY"; +const RENAME_TAIL_CLEANUP_DEFER_WORKERS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS"; +const RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY_MS"; +const RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW_MS"; +const RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_ENABLE_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_ENABLE"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_MIN_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_MIN"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_LOW_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_LOW"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_HIGH_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_HIGH"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS"; +const RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER"; +const RENAME_FOREGROUND_ACTIVE_LANES_ENV: &str = "RUSTFS_PUT_RENAME_FOREGROUND_ACTIVE_LANES"; +const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY: usize = 1024; +const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_WORKERS: usize = 2; +const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX: usize = 1; +const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW: usize = 128; +const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH: usize = 512; +const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS: u64 = 500; +const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS: u64 = 2_000; +const DEFAULT_RENAME_FOREGROUND_ACTIVE_LANES: usize = 1024; + +fn rename_tail_cleanup_counterfactual_skip_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP_ENV).ok().as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn rename_tail_cleanup_defer_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(RENAME_TAIL_CLEANUP_DEFER_ENV).ok().as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn rename_tail_cleanup_defer_queue_capacity() -> usize { + static CAPACITY: OnceLock = OnceLock::new(); + *CAPACITY.get_or_init(|| { + rustfs_utils::get_env_usize( + RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY_ENV, + DEFAULT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY, + ) + .max(1) + }) +} + +fn rename_tail_cleanup_defer_workers() -> usize { + static WORKERS: OnceLock = OnceLock::new(); + *WORKERS.get_or_init(|| { + rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_WORKERS_ENV, DEFAULT_RENAME_TAIL_CLEANUP_DEFER_WORKERS).max(1) + }) +} + +fn rename_tail_cleanup_defer_worker_delay() -> Duration { + static DELAY: OnceLock = OnceLock::new(); + *DELAY.get_or_init(|| Duration::from_millis(rustfs_utils::get_env_u64(RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY_MS_ENV, 0))) +} + +fn rename_tail_cleanup_defer_batch_window() -> Duration { + static WINDOW: OnceLock = OnceLock::new(); + *WINDOW.get_or_init(|| Duration::from_millis(rustfs_utils::get_env_u64(RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW_MS_ENV, 0))) +} + +fn rename_tail_cleanup_defer_batch_max() -> usize { + static BATCH_MAX: OnceLock = OnceLock::new(); + *BATCH_MAX.get_or_init(|| { + rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX_ENV, DEFAULT_RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX).max(1) + }) +} + +fn rename_tail_cleanup_defer_adaptive_batch_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_ENABLE_ENV) + .ok() + .as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn rename_tail_cleanup_defer_adaptive_batch_min() -> usize { + static BATCH_MIN: OnceLock = OnceLock::new(); + *BATCH_MIN.get_or_init(|| rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_MIN_ENV, 1).max(1)) +} + +fn rename_tail_cleanup_defer_adaptive_batch_low() -> usize { + static BATCH_LOW: OnceLock = OnceLock::new(); + *BATCH_LOW.get_or_init(|| { + rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_LOW_ENV, rename_tail_cleanup_defer_batch_max()) + .max(1) + }) +} + +fn rename_tail_cleanup_defer_adaptive_batch_high() -> usize { + static BATCH_HIGH: OnceLock = OnceLock::new(); + *BATCH_HIGH.get_or_init(|| { + rustfs_utils::get_env_usize( + RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_HIGH_ENV, + rename_tail_cleanup_defer_adaptive_batch_low().max(rename_tail_cleanup_defer_batch_max()), + ) + .max(1) + }) +} + +fn rename_tail_cleanup_defer_adaptive_queue_low() -> usize { + static QUEUE_LOW: OnceLock = OnceLock::new(); + *QUEUE_LOW.get_or_init(|| { + rustfs_utils::get_env_usize( + RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW_ENV, + DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW, + ) + }) +} + +fn rename_tail_cleanup_defer_adaptive_queue_high() -> usize { + static QUEUE_HIGH: OnceLock = OnceLock::new(); + *QUEUE_HIGH.get_or_init(|| { + rustfs_utils::get_env_usize( + RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH_ENV, + DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH, + ) + }) +} + +fn rename_tail_cleanup_defer_adaptive_age_low() -> Duration { + static AGE_LOW: OnceLock = OnceLock::new(); + *AGE_LOW.get_or_init(|| { + Duration::from_millis(rustfs_utils::get_env_u64( + RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS_ENV, + DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS, + )) + }) +} + +fn rename_tail_cleanup_defer_adaptive_age_high() -> Duration { + static AGE_HIGH: OnceLock = OnceLock::new(); + *AGE_HIGH.get_or_init(|| { + Duration::from_millis(rustfs_utils::get_env_u64( + RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS_ENV, + DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS, + )) + }) +} + +fn rename_tail_cleanup_defer_batch_target(queued_depth: usize, oldest_age: Option) -> usize { + let fixed = rename_tail_cleanup_defer_batch_max(); + if !rename_tail_cleanup_defer_adaptive_batch_enabled() { + return fixed; + } + + let min = rename_tail_cleanup_defer_adaptive_batch_min(); + let low = rename_tail_cleanup_defer_adaptive_batch_low().max(min); + let high = rename_tail_cleanup_defer_adaptive_batch_high().max(low); + let queue_low = rename_tail_cleanup_defer_adaptive_queue_low(); + let queue_high = rename_tail_cleanup_defer_adaptive_queue_high().max(queue_low); + let age_low = rename_tail_cleanup_defer_adaptive_age_low(); + let age_high = rename_tail_cleanup_defer_adaptive_age_high().max(age_low); + + let age = oldest_age.unwrap_or_default(); + if queued_depth >= queue_high || age >= age_high { + high + } else if queued_depth >= queue_low || age >= age_low { + low + } else { + min + } +} + +fn rename_tail_cleanup_defer_hold_worker_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER_ENV).ok().as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn rename_foreground_active_lanes() -> &'static Vec { + static LANES: OnceLock> = OnceLock::new(); + LANES.get_or_init(|| { + let lanes = + rustfs_utils::get_env_usize(RENAME_FOREGROUND_ACTIVE_LANES_ENV, DEFAULT_RENAME_FOREGROUND_ACTIVE_LANES).max(1); + (0..lanes).map(|_| AtomicUsize::new(0)).collect() + }) +} + +pub(in crate::set_disk) fn rename_foreground_active_count(disk_index: usize) -> usize { + let lanes = rename_foreground_active_lanes(); + lanes[disk_index % lanes.len()].load(Ordering::Relaxed) +} + +struct RenameForegroundActiveGuard { + disk_index: usize, +} + +impl Drop for RenameForegroundActiveGuard { + fn drop(&mut self) { + let lanes = rename_foreground_active_lanes(); + lanes[self.disk_index % lanes.len()].fetch_sub(1, Ordering::Relaxed); + } +} + +fn rename_foreground_active_guard(disk_index: usize) -> RenameForegroundActiveGuard { + let lanes = rename_foreground_active_lanes(); + let active = lanes[disk_index % lanes.len()].fetch_add(1, Ordering::Relaxed) + 1; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_FOREGROUND_ACTIVE, + active as f64, + ); + RenameForegroundActiveGuard { disk_index } +} + +type RenameTailCleanupFuture = Pin + Send + 'static>>; + +struct RenameTailCleanupJob { + enqueued_at: Instant, + targets: usize, + future: RenameTailCleanupFuture, +} + +struct RenameTailCleanupQueue { + sender: tokio::sync::mpsc::Sender, + queued: Arc, + oldest_enqueued_at: Arc>>, +} + +async fn run_rename_tail_cleanup_job(job: RenameTailCleanupJob) { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_START_DELAY, + job.enqueued_at.elapsed().as_secs_f64() * 1000.0, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_TARGETS, + job.targets as f64, + ); + let worker_delay = rename_tail_cleanup_defer_worker_delay(); + if !worker_delay.is_zero() { + let delay_started = rustfs_io_metrics::put_stage_timer(); + tokio::time::sleep(worker_delay).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY, + delay_started, + ); + } + let cleanup_started = rustfs_io_metrics::put_stage_timer(); + job.future.await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DURATION, + cleanup_started, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED, + 1.0, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED_LAG, + job.enqueued_at.elapsed().as_secs_f64() * 1000.0, + ); +} + +impl RenameTailCleanupQueue { + fn new() -> Self { + let (sender, receiver) = tokio::sync::mpsc::channel::(rename_tail_cleanup_defer_queue_capacity()); + let queued = Arc::new(AtomicUsize::new(0)); + let active_workers = Arc::new(AtomicUsize::new(0)); + let oldest_enqueued_at = Arc::new(tokio::sync::Mutex::new(None::)); + let receiver = Arc::new(tokio::sync::Mutex::new(receiver)); + if !rename_tail_cleanup_defer_hold_worker_enabled() { + for _ in 0..rename_tail_cleanup_defer_workers() { + let receiver = Arc::clone(&receiver); + let queued = Arc::clone(&queued); + let active_workers = Arc::clone(&active_workers); + let oldest_enqueued_at = Arc::clone(&oldest_enqueued_at); + tokio::spawn(async move { + loop { + let queued_before_poll = queued.load(Ordering::Relaxed); + let worker_poll_started = rustfs_io_metrics::put_stage_timer(); + let Some(job) = receiver.lock().await.recv().await else { + return; + }; + if queued_before_poll == 0 { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_IDLE_WAIT, + worker_poll_started, + ); + } else { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_POLL_WAIT, + worker_poll_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_JOB_WAIT, + worker_poll_started, + ); + } + let mut batch = vec![job]; + let batch_window = rename_tail_cleanup_defer_batch_window(); + let queued_snapshot = queued.load(Ordering::Relaxed); + let oldest_snapshot = *oldest_enqueued_at.lock().await; + let batch_max = rename_tail_cleanup_defer_batch_target( + queued_snapshot, + oldest_snapshot.map(|oldest| oldest.elapsed()), + ); + if batch_max > 1 && !batch_window.is_zero() { + let window_started = rustfs_io_metrics::put_stage_timer(); + tokio::time::sleep(batch_window).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW, + window_started, + ); + } + while batch.len() < batch_max { + let next = { + let mut locked = receiver.lock().await; + locked.try_recv().ok() + }; + let Some(next) = next else { + break; + }; + batch.push(next); + } + let queued_after_dequeue = queued.fetch_sub(batch.len(), Ordering::Relaxed).saturating_sub(batch.len()); + if queued_after_dequeue == 0 { + *oldest_enqueued_at.lock().await = None; + } + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH, + queued_after_dequeue as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE, + queued_after_dequeue as f64, + ); + if let Some(oldest) = *oldest_enqueued_at.lock().await { + let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE, + oldest_age_ms, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE, + oldest_age_ms, + ); + } + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_SIZE, + batch.len() as f64, + ); + if batch.len() > 1 { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BURST_BATCH_SIZE, + batch.len() as f64, + ); + } + let active = active_workers.fetch_add(1, Ordering::Relaxed) + 1; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_ACTIVE, + active as f64, + ); + for job in batch { + run_rename_tail_cleanup_job(job).await; + } + active_workers.fetch_sub(1, Ordering::Relaxed); + } + }); + } + } + Self { + sender, + queued, + oldest_enqueued_at, + } + } + + async fn try_enqueue(&self, job: RenameTailCleanupJob) -> std::result::Result<(), RenameTailCleanupJob> { + let queue_push_started = rustfs_io_metrics::put_stage_timer(); + { + let mut oldest = self.oldest_enqueued_at.lock().await; + if oldest.is_none() { + *oldest = Some(job.enqueued_at); + } + } + let queued_after_enqueue = self.queued.fetch_add(1, Ordering::Relaxed) + 1; + if rename_tail_cleanup_defer_hold_worker_enabled() { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED, + 1.0, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER, + 1.0, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH, + queued_after_enqueue as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE, + queued_after_enqueue as f64, + ); + if let Some(oldest) = *self.oldest_enqueued_at.lock().await { + let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE, + oldest_age_ms, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE, + oldest_age_ms, + ); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH, + queue_push_started, + ); + return Ok(()); + } + match self.sender.try_send(job) { + Ok(()) => { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED, + 1.0, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH, + queued_after_enqueue as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE, + queued_after_enqueue as f64, + ); + if let Some(oldest) = *self.oldest_enqueued_at.lock().await { + let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE, + oldest_age_ms, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE, + oldest_age_ms, + ); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH, + queue_push_started, + ); + Ok(()) + } + Err(err) => { + self.queued.fetch_sub(1, Ordering::Relaxed); + if self.queued.load(Ordering::Relaxed) == 0 { + *self.oldest_enqueued_at.lock().await = None; + } + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_FALLBACK_SYNC, + 1.0, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH, + self.queued.load(Ordering::Relaxed) as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE, + self.queued.load(Ordering::Relaxed) as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH, + queue_push_started, + ); + Err(err.into_inner()) + } + } + } +} + +fn rename_tail_cleanup_queue() -> &'static RenameTailCleanupQueue { + static QUEUE: OnceLock = OnceLock::new(); + QUEUE.get_or_init(RenameTailCleanupQueue::new) +} + const EVENT_SET_DISK_RENAME_ROLLBACK: &str = "set_disk_rename_rollback"; #[derive(Clone, Copy)] @@ -2569,10 +3066,9 @@ async fn inspect_incomplete_rename_rollback( request.recreate_missing = Some(false); request.update_parity = Some(false); request.recursive = Some(false); - match tokio::time::timeout(Duration::from_secs(1), submitter(request)).await { - Ok(result) => result, - Err(_) => ReadRepairAdmissionOutcome::Failed("rollback inspection admission timed out".to_string()), - } + tokio::time::timeout(Duration::from_secs(1), submitter(request)) + .await + .unwrap_or_else(|_| ReadRepairAdmissionOutcome::Failed("rollback inspection admission timed out".to_string())) } fn rename_rollback_task_outcome( @@ -2859,21 +3355,24 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal< SubmitFuture, >( tail_drain: tokio::task::JoinHandle>, - guard_release: tokio::sync::oneshot::Receiver, + guard_release: oneshot::Receiver, guards: Guards, request: rustfs_heal_contracts::heal_channel::HealChannelRequest, finalize: Finalize, cleanup: Cleanup, submit: Submit, ) where - Guards: Send, + Guards: Send + 'static, Finalize: FnOnce() -> FinalizeFuture + Send, FinalizeFuture: Future + Send, Cleanup: FnOnce(Guards, Vec) -> CleanupFuture + Send, CleanupFuture: Future + Send, + Cleanup: 'static, + CleanupFuture: 'static, Submit: FnOnce(rustfs_heal_contracts::heal_channel::HealChannelRequest) -> SubmitFuture + Send, SubmitFuture: Future + Send, { + let tail_async_started = rustfs_io_metrics::put_stage_timer(); let (needs_heal, tail_cleanup, tail_complete) = match tail_drain.await { Ok(Some(outcome)) => (outcome.convergence.needs_heal(), outcome.cleanup, true), Ok(None) => { @@ -2902,16 +3401,80 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal< (true, Vec::new(), false) } }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT, + tail_async_started, + ); // A dropped sender means the request continuation was cancelled. Cleanup // must still run; only the explicit `false` used by the hard-crash harness // suppresses all in-process continuation work. + let guard_release_wait_started = rustfs_io_metrics::put_stage_timer(); if matches!(guard_release.await, Ok(false)) { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT, + guard_release_wait_started, + ); drop(guards); return; } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT, + guard_release_wait_started, + ); + let finalize_started = rustfs_io_metrics::put_stage_timer(); finalize().await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_FINALIZE, + finalize_started, + ); if tail_complete { - cleanup(guards, tail_cleanup).await; + let cleanup_submit_started = rustfs_io_metrics::put_stage_timer(); + if rename_tail_cleanup_counterfactual_skip_enabled() { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP, + 1.0, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT, + cleanup_submit_started, + ); + drop(guards); + } else if rename_tail_cleanup_defer_enabled() { + let targets = tail_cleanup.len(); + let job = RenameTailCleanupJob { + enqueued_at: Instant::now(), + targets, + future: Box::pin(cleanup(guards, tail_cleanup)), + }; + if let Err(job) = rename_tail_cleanup_queue().try_enqueue(job).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT, + cleanup_submit_started, + ); + let cleanup_started = rustfs_io_metrics::put_stage_timer(); + job.future.await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP, + cleanup_started, + ); + } else { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT, + cleanup_submit_started, + ); + } + } else { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT, + cleanup_submit_started, + ); + let cleanup_started = rustfs_io_metrics::put_stage_timer(); + cleanup(guards, tail_cleanup).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP, + cleanup_started, + ); + } } else { // The outer coordinator may fail while a disk rename still owns the // staging source. Preserve it for later healing/GC instead of racing @@ -2919,7 +3482,12 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal< drop(guards); } if needs_heal { + let heal_submit_started = rustfs_io_metrics::put_stage_timer(); submit(request).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_HEAL_SUBMIT, + heal_submit_started, + ); } } @@ -2973,8 +3541,8 @@ impl SetDisks { bucket: &str, object: &str, write_quorum: usize, - ) -> crate::error::Result<(Vec>, Vec>)> { - let fence_path = crate::disk::quota_mutation_fence_path(bucket, object); + ) -> Result<(Vec>, Vec>)> { + let fence_path = disk::quota_mutation_fence_path(bucket, object); let results = join_all(disks.iter().map(|disk| { let disk = disk.clone(); let fence_path = fence_path.clone(); @@ -3016,8 +3584,8 @@ impl SetDisks { bucket: &str, object: &str, write_quorum: usize, - ) -> crate::error::Result<()> { - let fence_path = crate::disk::quota_mutation_fence_path(bucket, object); + ) -> Result<()> { + let fence_path = disk::quota_mutation_fence_path(bucket, object); let results = join_all(disks.iter().zip(tokens).filter_map(|(disk, token)| { let disk = disk.as_ref()?.clone(); let token = (*token)?; @@ -3046,11 +3614,25 @@ impl SetDisks { errs: &[Option], cleanup_data_dirs: &[Option], old_current_sizes: &[Option], + old_current_source_checked: &[bool], + old_current_sources: &[Option], write_quorum: usize, ) -> disk::error::Result { let data_dir = Self::reduce_common_data_dir(cleanup_data_dirs, write_quorum); let convergence = Self::classify_rename_convergence(disk_versions, errs); let old_current_size = Self::reduce_common_old_current_size(old_current_sizes, write_quorum); + let old_current_source_quorum_checked = + old_current_source_checked.iter().filter(|checked| **checked).count() >= write_quorum; + let old_current_source_all_successful_checked = errs + .iter() + .enumerate() + .filter(|(_, err)| err.is_none()) + .all(|(idx, _)| old_current_source_checked.get(idx).copied().unwrap_or(false)); + let old_current_source = if old_current_source_quorum_checked { + Self::reduce_common_old_current_source(old_current_sources, write_quorum) + } else { + None + }; let online_disks = Self::eval_disks(disks, errs); let committed_slot = online_disks.iter().position(Option::is_some).ok_or(DiskError::Unexpected)?; let committed_file_info = file_infos.get(committed_slot).cloned().ok_or(DiskError::Unexpected)?; @@ -3078,6 +3660,9 @@ impl SetDisks { data_dir, cleanup_disks, old_current_size, + old_current_source_checked: old_current_source_quorum_checked, + old_current_source_all_successful_checked, + old_current_source, committed_file_info, tail_drain: None, }) @@ -3195,9 +3780,10 @@ impl SetDisks { let src_object = Arc::new(src_object.to_string()); let dst_bucket = Arc::new(dst_bucket.to_string()); let dst_object = Arc::new(dst_object.to_string()); - let (commit_tx, commit_rx) = tokio::sync::oneshot::channel(); + let (commit_tx, commit_rx) = oneshot::channel(); let coordinator_failure_receipt = rollback_receipt.clone(); + let early_ack_spawn_to_quorum_wait_started = rustfs_io_metrics::put_stage_timer(); let tail_drain = tokio::spawn({ let fanout_src_bucket = src_bucket.clone(); let fanout_src_object = src_object.clone(); @@ -3230,6 +3816,11 @@ impl SetDisks { let Some(disk) = disk else { return Err(DiskError::DiskNotFound); }; + let disk_wait_location_stage = if disk.is_local() { + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL + } else { + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE + }; let is_delete_marker = file_info.is_canonical_delete_marker(); // Clone FileInfo and set erasure.index for this disk @@ -3249,6 +3840,7 @@ impl SetDisks { let disk_wait_started = rustfs_io_metrics::put_stage_timer(); dispatch_state = RenameDispatchState::MayHavePublished; + let _foreground_active_guard = rename_foreground_active_guard(i); let observed = disk .rename_data_borrowed_with_fence_observed( &src_bucket, @@ -3256,7 +3848,7 @@ impl SetDisks { &file_info, &dst_bucket, &dst_object, - crate::disk::RenameDataGuards { + disk::RenameDataGuards { scanner_publication_lease_token, namespace_owner: namespace_commit_guard .clone() @@ -3277,6 +3869,7 @@ impl SetDisks { rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT, duration_ms, ); + rustfs_io_metrics::record_put_object_stage_duration(disk_wait_location_stage, duration_ms); let position = if result.is_ok() { let rank = successful_rename_completion_rank .as_ref() @@ -3315,9 +3908,12 @@ impl SetDisks { let mut data_dirs = vec![None; disk_count]; let mut cleanup_data_dirs = vec![None; disk_count]; let mut old_current_sizes = vec![None; disk_count]; + let mut old_current_source_checked = vec![false; disk_count]; + let mut old_current_sources = vec![None; disk_count]; let mut capacity_scope_generation = None; let mut cleanup_at_snapshot = vec![false; disk_count]; + let mut tail_after_quorum_started = None; while let Some(joined) = tasks.join_next().await { results_seen += 1; match joined { @@ -3327,6 +3923,8 @@ impl SetDisks { cleanup_data_dirs[idx] = res.cleanup_data_dir; disk_versions[idx] = res.sign; old_current_sizes[idx] = res.old_current_size; + old_current_source_checked[idx] = res.old_current_source_checked; + old_current_sources[idx] = res.old_current_source; errs[idx] = None; success_count += 1; } @@ -3352,6 +3950,8 @@ impl SetDisks { &errs, &cleanup_data_dirs, &old_current_sizes, + &old_current_source_checked, + &old_current_sources, write_quorum, ); if let Ok(commit) = snapshot_commit.as_ref() { @@ -3362,8 +3962,13 @@ impl SetDisks { let _ = commit_tx.send(snapshot_commit); } sent_commit = true; + tail_after_quorum_started = rustfs_io_metrics::put_stage_timer(); } } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_AFTER_QUORUM_WAIT, + tail_after_quorum_started, + ); #[cfg(test)] rollback_fault_injection::after_fanout(&fanout_dst_object); @@ -3483,6 +4088,10 @@ impl SetDisks { } }); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT, + early_ack_spawn_to_quorum_wait_started, + ); let quorum_wait_started = rustfs_io_metrics::put_stage_timer(); let commit = match commit_rx.await { Ok(commit) => commit, @@ -3495,8 +4104,13 @@ impl SetDisks { rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT, quorum_wait_started, ); + let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer(); commit.map(|mut commit| { commit.tail_drain = Some(tail_drain); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER, + attach_tail_owner_started, + ); commit }) } @@ -3619,6 +4233,11 @@ impl SetDisks { let Some(disk) = disk else { return Err(DiskError::DiskNotFound); }; + let disk_wait_location_stage = if disk.is_local() { + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL + } else { + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE + }; let is_delete_marker = file_info.is_canonical_delete_marker(); let mut local_file_info; @@ -3652,6 +4271,7 @@ impl SetDisks { let disk_wait_started = rustfs_io_metrics::put_stage_timer(); dispatch_state = RenameDispatchState::MayHavePublished; + let _foreground_active_guard = rename_foreground_active_guard(i); let observed = disk .rename_data_borrowed_with_fence_observed( &src_bucket, @@ -3659,7 +4279,7 @@ impl SetDisks { file_info, &dst_bucket, &dst_object, - crate::disk::RenameDataGuards { + disk::RenameDataGuards { scanner_publication_lease_token, namespace_owner: namespace_commit_guard .clone() @@ -3680,6 +4300,7 @@ impl SetDisks { rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT, duration_ms, ); + rustfs_io_metrics::record_put_object_stage_duration(disk_wait_location_stage, duration_ms); let position = if result.is_ok() { let rank = successful_rename_completion_rank .as_ref() @@ -3715,9 +4336,16 @@ impl SetDisks { let mut data_dirs = vec![None; disk_count]; let mut cleanup_data_dirs = vec![None; disk_count]; let mut old_current_sizes = vec![None; disk_count]; + let mut old_current_source_checked = vec![false; disk_count]; + let mut old_current_sources = vec![None; disk_count]; let quorum_wait_started = rustfs_io_metrics::put_stage_timer(); + let sync_full_fanout_started = rustfs_io_metrics::put_stage_timer(); let fanout_result = fanout.await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_SYNC_FULL_FANOUT_WAIT, + sync_full_fanout_started, + ); rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT, quorum_wait_started, @@ -3758,6 +4386,8 @@ impl SetDisks { cleanup_data_dirs[idx] = res.cleanup_data_dir; disk_versions[idx].clone_from(&res.sign); old_current_sizes[idx] = res.old_current_size; + old_current_source_checked[idx] = res.old_current_source_checked; + old_current_sources[idx] = res.old_current_source.clone(); errs.push(None); } Ok(Err(e)) => { @@ -3894,6 +4524,18 @@ impl SetDisks { let data_dir = Self::reduce_common_data_dir(&cleanup_data_dirs, write_quorum); let convergence = Self::classify_rename_convergence(&disk_versions, &errs); let old_current_size = Self::reduce_common_old_current_size(&old_current_sizes, write_quorum); + let old_current_source_quorum_checked = + old_current_source_checked.iter().filter(|checked| **checked).count() >= write_quorum; + let old_current_source_all_successful_checked = errs + .iter() + .enumerate() + .filter(|(_, err)| err.is_none()) + .all(|(idx, _)| old_current_source_checked.get(idx).copied().unwrap_or(false)); + let old_current_source = if old_current_source_quorum_checked { + Self::reduce_common_old_current_source(&old_current_sources, write_quorum) + } else { + None + }; let online_disks = Self::eval_disks(disks, &errs); let committed_slot = online_disks.iter().position(Option::is_some).ok_or(DiskError::Unexpected)?; let committed_file_info = std::mem::take(&mut file_infos[committed_slot]); @@ -3921,11 +4563,34 @@ impl SetDisks { data_dir, cleanup_disks, old_current_size, + old_current_source_checked: old_current_source_quorum_checked, + old_current_source_all_successful_checked, + old_current_source, committed_file_info, tail_drain: None, }) } + pub(in crate::set_disk) fn reduce_common_old_current_source( + old_current_sources: &[Option], + write_quorum: usize, + ) -> Option { + let mut counts: HashMap<[u8; 32], (usize, FileInfo)> = HashMap::new(); + + for source in old_current_sources.iter().flatten() { + let entry = counts + .entry(Self::file_info_quorum_hash(source)) + .or_insert_with(|| (0, source.clone())); + entry.0 += 1; + } + + counts + .into_values() + .filter(|(count, _)| *count >= write_quorum) + .max_by_key(|(count, _)| *count) + .map(|(_, source)| source) + } + /// rustfs/backlog#1009: reduce the per-disk observations of the /// destination's previous current version to one set-level value, mirroring /// `reduce_common_data_dir`: the observation reported by at least @@ -4991,6 +5656,7 @@ impl SetDisks { fi.set_tier_free_version_id(&Uuid::new_v4().to_string()); let disks = self.get_disks_internal().await; + self.invalidate_get_object_metadata_cache(bucket, object).await; let mut futures = Vec::with_capacity(disks.len()); for (disk_index, disk_op) in disks.iter().enumerate() { @@ -5071,6 +5737,9 @@ impl SetDisks { } else { false }; + if absent { + self.invalidate_get_object_metadata_cache(bucket, object).await; + } Ok((m, absent)) } @@ -6566,9 +7235,9 @@ mod tests { ); scope.try_begin().expect("delete scope should enter flight"); let scope_guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone()); - let (started_tx, started_rx) = tokio::sync::oneshot::channel(); - let (release_tx, release_rx) = tokio::sync::oneshot::channel(); - let (finished_tx, finished_rx) = tokio::sync::oneshot::channel(); + let (started_tx, started_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + let (finished_tx, finished_rx) = oneshot::channel(); let waiter = tokio::spawn(run_scanner_publication_delete_owner(Some(scope.clone()), move || async move { started_tx.send(()).expect("delete owner should start"); @@ -9026,7 +9695,7 @@ mod tests { #[allow(unreachable_code)] None }); - let (release, released) = tokio::sync::oneshot::channel::(); + let (release, released) = oneshot::channel::(); drop(release); finish_rename_tail_heal( @@ -10314,7 +10983,7 @@ mod tests { info.size = 11; info.parts.clear(); info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None); - let crate::disk::Disk::Local(local) = disk.as_ref() else { + let disk::Disk::Local(local) = disk.as_ref() else { panic!("physical publication fixture requires local disks"); }; // Linux IO paths use a mount FD, which is also the namespace lock key. @@ -10322,7 +10991,7 @@ mod tests { .get_disk() .get_object_path_for_io(bucket, object) .expect("the publication path must resolve through the disk's mount lease"); - let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (entered_tx, entered_rx) = oneshot::channel(); let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); hooks.push(os::prepared_publication_test_hooks::install( &destination.join(STORAGE_FORMAT_FILE), @@ -10508,11 +11177,11 @@ mod tests { info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None); } let disk = disks[3].as_ref().expect("tail disk"); - let crate::disk::Disk::Local(local) = disk.as_ref() else { + let disk::Disk::Local(local) = disk.as_ref() else { panic!("local fixture"); }; let destination = local.get_disk().get_object_path_for_io(bucket, object).expect("tail IO path"); - let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (entered_tx, entered_rx) = oneshot::channel(); let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); let _hook = os::prepared_publication_test_hooks::install(&destination.join(STORAGE_FORMAT_FILE), move || { let _ = entered_tx.send(()); @@ -10524,7 +11193,7 @@ mod tests { let receipt = RenameRollbackReceipt::default(); let caller_receipt = receipt.clone(); let caller_disks = disks.clone(); - let (ack_tx, ack_rx) = tokio::sync::oneshot::channel(); + let (ack_tx, ack_rx) = oneshot::channel(); let caller = tokio::spawn(async move { let commit = SetDisks::rename_data_owned_with_fence( &caller_disks, @@ -12001,6 +12670,9 @@ mod tests { let bucket = "dangling-invalid-meta-bucket"; let object = "object"; let set = io_primitives_test_set(vec![None, None, None, None], 1).await; + let cached_generation = set + .get_object_metadata_cache_generation(bucket, object) + .expect("local cache generation should be enabled"); let mut data_errs_by_part = HashMap::new(); data_errs_by_part.insert( 1, @@ -12035,6 +12707,10 @@ mod tests { .expect("invalid metadata should be cleanable when part results prove the object is dangling"); assert!(!deleted.is_valid()); + assert!( + !set.is_get_object_metadata_cache_generation_current(cached_generation), + "dangling cleanup must retire locally cached metadata before deletion" + ); } #[tokio::test] diff --git a/crates/ecstore/src/set_disk/core/metadata_read.rs b/crates/ecstore/src/set_disk/core/metadata_read.rs index 37e727395..39e061c61 100644 --- a/crates/ecstore/src/set_disk/core/metadata_read.rs +++ b/crates/ecstore/src/set_disk/core/metadata_read.rs @@ -576,6 +576,7 @@ impl MetadataFanoutDiagnostics { pub(in crate::set_disk) fn record_quorum_candidate_latency(&self, path: &'static str, read_quorum: usize) { if let Some(latency) = self.quorum_candidate_latency(read_quorum) { + rustfs_io_metrics::record_get_object_metadata_quorum_result(path, "reached"); rustfs_io_metrics::record_get_object_quorum_reached_latency(path, latency.as_secs_f64()); } } diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 58d18b5a7..cd736e932 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -166,7 +166,7 @@ use sha2::Sha256; use std::hash::{BuildHasher, Hash, Hasher}; use std::mem::{self}; use std::pin::Pin; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, OnceLock}; use std::task::{Context, Poll}; use std::time::{Instant, SystemTime, UNIX_EPOCH}; @@ -410,6 +410,12 @@ impl Drop for ObjectLockDiagGuard { } let hold = self.acquired_at.elapsed(); + if self.op == "put_object_commit" && self.mode == "write" && rustfs_io_metrics::put_stage_metrics_enabled() { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD, + hold.as_secs_f64() * 1000.0, + ); + } record_object_lock_diag_hold_duration(self.op, self.mode, hold); let threshold = get_object_lock_diag_slow_hold_threshold(); if hold >= threshold { @@ -925,16 +931,21 @@ pub(in crate::set_disk) struct GetObjectFileInfo { owned: Option, shared: Option>, condition_stopped: bool, + late_metadata_fanout: Option, } struct OwnedGetObjectFileInfo { fi: FileInfo, parts_metadata: Vec, online_disks: Vec>, - late_metadata_fanout_disks: Option>>, } -type OwnedGetObjectFileInfoParts = (FileInfo, Vec, Vec>, Option>>); +enum LateMetadataFanout { + CapturedDisks(Vec>), + RefreshCurrentSet, +} + +type OwnedGetObjectFileInfoParts = (FileInfo, Vec, Vec>, Option); impl GetObjectFileInfo { fn owned(fi: FileInfo, parts_metadata: Vec, online_disks: Vec>) -> Self { @@ -943,10 +954,10 @@ impl GetObjectFileInfo { fi, parts_metadata, online_disks, - late_metadata_fanout_disks: None, }), shared: None, condition_stopped: false, + late_metadata_fanout: None, } } @@ -956,16 +967,9 @@ impl GetObjectFileInfo { online_disks: Vec>, late_metadata_fanout_disks: Vec>, ) -> Self { - Self { - owned: Some(OwnedGetObjectFileInfo { - fi, - parts_metadata, - online_disks, - late_metadata_fanout_disks: Some(late_metadata_fanout_disks), - }), - shared: None, - condition_stopped: false, - } + let mut snapshot = Self::owned(fi, parts_metadata, online_disks); + snapshot.late_metadata_fanout = Some(LateMetadataFanout::CapturedDisks(late_metadata_fanout_disks)); + snapshot } fn shared(entry: Arc) -> Self { @@ -974,9 +978,16 @@ impl GetObjectFileInfo { owned: None, shared: Some(entry), condition_stopped, + late_metadata_fanout: None, } } + fn shared_with_late_metadata_refresh(entry: Arc) -> Self { + let mut snapshot = Self::shared(entry); + snapshot.late_metadata_fanout = Some(LateMetadataFanout::RefreshCurrentSet); + snapshot + } + fn fi(&self) -> &FileInfo { match (&self.owned, &self.shared) { (Some(snapshot), None) => &snapshot.fi, @@ -1002,9 +1013,7 @@ impl GetObjectFileInfo { } fn has_late_metadata_fanout(&self) -> bool { - self.owned - .as_ref() - .is_some_and(|snapshot| snapshot.late_metadata_fanout_disks.is_some()) + self.late_metadata_fanout.is_some() } fn into_owned(self) -> (FileInfo, Vec, Vec>) { @@ -1013,16 +1022,22 @@ impl GetObjectFileInfo { } fn into_owned_with_late_metadata_fanout(self) -> OwnedGetObjectFileInfoParts { - match (self.owned, self.shared) { - (Some(snapshot), None) => ( - snapshot.fi, - snapshot.parts_metadata, - snapshot.online_disks, - snapshot.late_metadata_fanout_disks, - ), + let Self { + owned, + shared, + late_metadata_fanout, + .. + } = self; + match (owned, shared) { + (Some(snapshot), None) => (snapshot.fi, snapshot.parts_metadata, snapshot.online_disks, late_metadata_fanout), (None, Some(entry)) => match Arc::try_unwrap(entry) { - Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks, None), - Err(entry) => (entry.fi.clone(), entry.parts_metadata.clone(), entry.online_disks.clone(), None), + Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks, late_metadata_fanout), + Err(entry) => ( + entry.fi.clone(), + entry.parts_metadata.clone(), + entry.online_disks.clone(), + late_metadata_fanout, + ), }, _ => unreachable!("GET metadata snapshot representation must be exclusive"), } @@ -1413,11 +1428,11 @@ mod prepared_get_object_metadata_tests { let snapshot = GetObjectFileInfo::shared(Arc::clone(&cached)); assert!(snapshot.has_valid_representation()); - assert!(std::mem::size_of::() >= std::mem::size_of::()); - assert!( - std::mem::size_of::() - <= std::mem::size_of::() + 2 * std::mem::size_of::() - ); + // Keep the original payload budget after moving the captured disks out + // of OwnedGetObjectFileInfo. + let owned_payload_size = std::mem::size_of::<(OwnedGetObjectFileInfo, Option>>)>(); + assert!(std::mem::size_of::() >= owned_payload_size); + assert!(std::mem::size_of::() <= owned_payload_size + 2 * std::mem::size_of::()); assert_eq!(Arc::strong_count(&cached), 2, "a cache hit must add one snapshot reference"); assert_eq!(snapshot.fi().name, "object"); assert_eq!(snapshot.parts_metadata().len(), 1); @@ -4337,6 +4352,9 @@ pub struct SetDisks { /// writes skip the global registry mutex (backlog#1315). `Arc` so clones of /// a set share one generation marker. capacity_dirty_generation: Arc, + /// Per-set proof that successful rename_data peers return the old-current + /// source capability needed before PUT may skip its pre-rename lookup. + tier_free_version_rename_data_source_capable: Arc, /// Orphan prefixes whose last purge scan met data that can never be /// purged by listing (residue without a committed marker, an in-flight /// write), keyed by `bucket/prefix` with the time of that scan. Empty @@ -4592,6 +4610,14 @@ struct GetObjectMetadataCacheKey { hash: u64, } +#[derive(Clone, Copy, Debug)] +struct GetObjectMetadataCacheLookupKey<'a> { + bucket: &'a str, + object: &'a str, + generation: u64, + hash: u64, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct GetObjectMetadataCacheGeneration { index: usize, @@ -4680,6 +4706,33 @@ impl GetObjectMetadataCacheKey { } } +impl<'a> GetObjectMetadataCacheLookupKey<'a> { + fn new(bucket: &'a str, object: &'a str, generation: GetObjectMetadataCacheGeneration) -> Self { + Self { + bucket, + object, + generation: generation.value, + hash: generation.hash, + } + } +} + +impl Hash for GetObjectMetadataCacheLookupKey<'_> { + fn hash(&self, state: &mut H) { + self.hash.hash(state); + self.generation.hash(state); + } +} + +impl moka::Equivalent for GetObjectMetadataCacheLookupKey<'_> { + fn equivalent(&self, key: &GetObjectMetadataCacheKey) -> bool { + self.bucket == key.bucket.as_ref() + && self.object == key.object.as_ref() + && self.generation == key.generation + && self.hash == key.hash + } +} + impl Hash for GetObjectMetadataCacheKey { fn hash(&self, state: &mut H) { self.hash.hash(state); @@ -4690,7 +4743,9 @@ impl Hash for GetObjectMetadataCacheKey { #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum GetObjectMetadataCacheCapability { Full, - // Quorum-validated metadata without a complete body or write-target layout. + // A quorum-selected layout can serve body reads, but is not a mutation layout. + QuorumSelected, + // Quorum-validated metadata used only to evaluate a terminal read condition. ConditionOnly, } @@ -5233,6 +5288,7 @@ impl SetDisks { ctx, capacity_scope_cache: Arc::new(std::sync::RwLock::new(CapacityScopeCache::default())), capacity_dirty_generation: Arc::new(AtomicU64::new(u64::MAX)), + tier_free_version_rename_data_source_capable: Arc::new(AtomicBool::new(false)), orphan_purge_backoff: Arc::new(std::sync::Mutex::new(HashMap::new())), #[cfg(test)] storage_class_config_override: Arc::new(std::sync::RwLock::new(None)), @@ -5247,6 +5303,15 @@ impl SetDisks { &self.ctx } + pub(in crate::set_disk) fn tier_free_version_rename_data_source_capable(&self) -> bool { + self.tier_free_version_rename_data_source_capable.load(Ordering::Acquire) + } + + pub(in crate::set_disk) fn observe_tier_free_version_rename_data_source_capability(&self, capable: bool) { + self.tier_free_version_rename_data_source_capable + .store(capable, Ordering::Release); + } + #[cfg(test)] pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc) { self.local_lock_manager = ctx.lock_manager(); diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 6f5ae956e..0365d0aa2 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -20,6 +20,7 @@ //! SetDisks core (io_primitives) via inherent calls. use crate::core::pools::DecommissionCapacityAdmission; +use crate::disk::cleanup_runtime::{self, OLD_DATA_CLEANUP_RECEIPT_FILE}; use rustfs_common::mrf_channel::MrfDeleteMarkerPurge; use rustfs_filemeta::metadata_keys; @@ -36,9 +37,9 @@ use super::super::{ GET_STAGE_LOCK_ACQUIRE, GET_STAGE_METADATA, GET_STAGE_OBJECT_INFO, GET_STAGE_PATH_DECISION, GET_STAGE_READER_SETUP, GenericError, GetCodecStreamingDecision, GetCodecStreamingFallbackReason, GetDirectMemoryDecision, GetObjectReader, HTTPRangeSpec, HashAlgorithm, HashMap, HashReader, HashSet, HeaderMap, HealChannelPriority, InstanceContext, Instant, - LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, OBJECT_OP_IGNORED_ERRS, ObjectApiError, ObjectInfo, ObjectKey, - ObjectLockConfigSnapshot, ObjectLockConfigState, ObjectOptions, ObjectReader, ObjectToDelete, OffsetDateTime, Ordering, Pin, - PutObjReader, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, ReadPathPlan, ReaderImpl, ReplicateDecision, + LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, LateMetadataFanout, OBJECT_OP_IGNORED_ERRS, ObjectApiError, ObjectInfo, + ObjectKey, ObjectLockConfigSnapshot, ObjectLockConfigState, ObjectOptions, ObjectReader, ObjectToDelete, OffsetDateTime, + Ordering, Pin, PutObjReader, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, ReadPathPlan, ReaderImpl, ReplicateDecision, ReplicationObjectBridge, Result, SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS, SLASH_SEPARATOR, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks, SmallWritePath, StorageError, TRANSITION_COMPLETE, UpdateMetadataOpts, Uuid, WriteLayout, adaptive_duplex_buffer_size, @@ -93,6 +94,392 @@ impl<'a> DeleteCommitContext<'a> { } } +const TIER_FREE_VERSION_SOURCE_LOOKUP_COUNTERFACTUAL_SKIP_ENV: &str = + "RUSTFS_PUT_TIER_FREE_VERSION_SOURCE_LOOKUP_COUNTERFACTUAL_SKIP"; +const TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ENV: &str = "RUSTFS_PUT_TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA"; +const TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ASSUME_HOMOGENEOUS_ENV: &str = + "RUSTFS_PUT_TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ASSUME_HOMOGENEOUS"; +const RENAME_TAIL_CLEANUP_PER_DISK_LIMIT_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_PER_DISK_LIMIT"; +const RENAME_TAIL_CLEANUP_PER_DISK_LANES_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_PER_DISK_LANES"; +const RENAME_TAIL_CLEANUP_LANE_ENABLE_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_ENABLE"; +const RENAME_TAIL_CLEANUP_LANE_CAPACITY_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_CAPACITY"; +const RENAME_TAIL_CLEANUP_LANE_BATCH_MAX_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_BATCH_MAX"; +const RENAME_TAIL_CLEANUP_LANE_BATCH_WINDOW_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_BATCH_WINDOW_MS"; +const RENAME_TAIL_CLEANUP_LANE_ACTIVE_WINDOW_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_ACTIVE_WINDOW_MS"; +const RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP"; +const DEFAULT_RENAME_TAIL_CLEANUP_PER_DISK_LANES: usize = 1024; +const DEFAULT_RENAME_TAIL_CLEANUP_LANE_CAPACITY: usize = 1024; +const DEFAULT_RENAME_TAIL_CLEANUP_LANE_BATCH_MAX: usize = 1; + +fn tier_free_version_source_lookup_counterfactual_skip_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(TIER_FREE_VERSION_SOURCE_LOOKUP_COUNTERFACTUAL_SKIP_ENV) + .ok() + .as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn rename_tail_cleanup_zero_target_tmp_delete_skip_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP_ENV) + .ok() + .as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn tier_free_version_source_reuse_rename_data_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ENV).ok().as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn tier_free_version_source_reuse_rename_data_assume_homogeneous_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ASSUME_HOMOGENEOUS_ENV) + .ok() + .as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn rename_tail_cleanup_per_disk_limit() -> usize { + static LIMIT: std::sync::OnceLock = std::sync::OnceLock::new(); + *LIMIT.get_or_init(|| rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_PER_DISK_LIMIT_ENV, 0)) +} + +fn rename_tail_cleanup_per_disk_lanes() -> usize { + static LANES: std::sync::OnceLock = std::sync::OnceLock::new(); + *LANES.get_or_init(|| { + rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_PER_DISK_LANES_ENV, DEFAULT_RENAME_TAIL_CLEANUP_PER_DISK_LANES).max(1) + }) +} + +fn rename_tail_cleanup_foreground_yield() -> Duration { + static YIELD: std::sync::OnceLock = std::sync::OnceLock::new(); + *YIELD + .get_or_init(|| Duration::from_millis(rustfs_utils::get_env_u64("RUSTFS_PUT_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD_MS", 0))) +} + +fn rename_tail_cleanup_foreground_active_yield() -> Duration { + static YIELD: std::sync::OnceLock = std::sync::OnceLock::new(); + *YIELD.get_or_init(|| { + Duration::from_millis(rustfs_utils::get_env_u64("RUSTFS_PUT_RENAME_TAIL_CLEANUP_FOREGROUND_ACTIVE_YIELD_MS", 0)) + }) +} + +fn rename_tail_cleanup_lane_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(RENAME_TAIL_CLEANUP_LANE_ENABLE_ENV).ok().as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn rename_tail_cleanup_lane_capacity() -> usize { + static CAPACITY: std::sync::OnceLock = std::sync::OnceLock::new(); + *CAPACITY.get_or_init(|| { + rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_LANE_CAPACITY_ENV, DEFAULT_RENAME_TAIL_CLEANUP_LANE_CAPACITY).max(1) + }) +} + +fn rename_tail_cleanup_lane_batch_max() -> usize { + static BATCH_MAX: std::sync::OnceLock = std::sync::OnceLock::new(); + *BATCH_MAX.get_or_init(|| { + rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_LANE_BATCH_MAX_ENV, DEFAULT_RENAME_TAIL_CLEANUP_LANE_BATCH_MAX).max(1) + }) +} + +fn rename_tail_cleanup_lane_batch_window() -> Duration { + static WINDOW: std::sync::OnceLock = std::sync::OnceLock::new(); + *WINDOW.get_or_init(|| Duration::from_millis(rustfs_utils::get_env_u64(RENAME_TAIL_CLEANUP_LANE_BATCH_WINDOW_MS_ENV, 0))) +} + +fn rename_tail_cleanup_lane_active_window() -> Duration { + static WINDOW: std::sync::OnceLock = std::sync::OnceLock::new(); + *WINDOW.get_or_init(|| Duration::from_millis(rustfs_utils::get_env_u64(RENAME_TAIL_CLEANUP_LANE_ACTIVE_WINDOW_MS_ENV, 0))) +} + +async fn rename_tail_cleanup_per_disk_permit(disk_index: usize) -> Option { + let limit = rename_tail_cleanup_per_disk_limit(); + if limit == 0 { + return None; + } + static LANES: std::sync::OnceLock>> = std::sync::OnceLock::new(); + let lanes = LANES.get_or_init(|| { + (0..rename_tail_cleanup_per_disk_lanes()) + .map(|_| Arc::new(tokio::sync::Semaphore::new(limit))) + .collect() + }); + let semaphore = Arc::clone(&lanes[disk_index % lanes.len()]); + let wait_started = rustfs_io_metrics::put_stage_timer(); + let permit = semaphore.clone().acquire_owned().await.ok(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_PERMIT_WAIT, + wait_started, + ); + if let Some(semaphore_permit) = permit { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_INFLIGHT, + limit.saturating_sub(semaphore.available_permits()) as f64, + ); + Some(semaphore_permit) + } else { + None + } +} + +type RenameTailCleanupLaneFuture = Pin + Send + 'static>>; + +struct RenameTailCleanupLaneJob { + disk_index: usize, + future: RenameTailCleanupLaneFuture, + done: tokio::sync::oneshot::Sender<()>, +} + +struct RenameTailCleanupLane { + sender: tokio::sync::mpsc::Sender, + queued: Arc, + oldest_enqueued_at: Arc>>, +} + +struct RenameTailCleanupLaneQueue { + lanes: Vec, +} + +async fn run_rename_tail_cleanup_lane_job(job: RenameTailCleanupLaneJob) { + let service_started = rustfs_io_metrics::put_stage_timer(); + job.future.await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_SERVICE_TIME, + service_started, + ); + let _ = job.done.send(()); +} + +impl RenameTailCleanupLaneQueue { + fn new() -> Self { + let lanes = (0..rename_tail_cleanup_per_disk_lanes()) + .map(|_| { + let (sender, mut receiver) = + tokio::sync::mpsc::channel::(rename_tail_cleanup_lane_capacity()); + let queued = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let active_workers = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let oldest_enqueued_at = Arc::new(tokio::sync::Mutex::new(None::)); + let worker_queued = Arc::clone(&queued); + let worker_active = Arc::clone(&active_workers); + let worker_oldest_enqueued_at = Arc::clone(&oldest_enqueued_at); + tokio::spawn(async move { + loop { + let queued_before_poll = worker_queued.load(Ordering::Relaxed); + let worker_poll_started = rustfs_io_metrics::put_stage_timer(); + let Some(job) = receiver.recv().await else { + return; + }; + if queued_before_poll == 0 { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_IDLE_WAIT, + worker_poll_started, + ); + } else { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_POLL_WAIT, + worker_poll_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_JOB_WAIT, + worker_poll_started, + ); + } + let mut batch = vec![job]; + let batch_window = rename_tail_cleanup_lane_batch_window(); + let batch_max = rename_tail_cleanup_lane_batch_max(); + if batch_max > 1 && !batch_window.is_zero() { + tokio::time::sleep(batch_window).await; + } + while batch.len() < batch_max { + let Some(next) = receiver.try_recv().ok() else { + break; + }; + batch.push(next); + } + let queued_after_dequeue = worker_queued + .fetch_sub(batch.len(), Ordering::Relaxed) + .saturating_sub(batch.len()); + if queued_after_dequeue == 0 { + *worker_oldest_enqueued_at.lock().await = None; + } + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH, + queued_after_dequeue as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE, + queued_after_dequeue as f64, + ); + if let Some(oldest) = *worker_oldest_enqueued_at.lock().await { + let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE, + oldest_age_ms, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE_MAX_SAMPLE, + oldest_age_ms, + ); + } + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BATCH_SIZE, + batch.len() as f64, + ); + if batch.len() > 1 { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BURST_BATCH_SIZE, + batch.len() as f64, + ); + } + let active = worker_active.fetch_add(1, Ordering::Relaxed) + 1; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_ACTIVE, + active as f64, + ); + let active_window = rename_tail_cleanup_lane_active_window(); + if !active_window.is_zero() { + let mut deferred = Vec::new(); + let mut ready = Vec::new(); + for job in batch { + let active = rename_foreground_active_count(job.disk_index); + if active > 0 { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_FOREGROUND_ACTIVE_HIT, + active as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_REQUEUE, + 1.0, + ); + deferred.push(job); + } else { + ready.push(job); + } + } + for job in ready { + run_rename_tail_cleanup_lane_job(job).await; + } + if !deferred.is_empty() { + tokio::time::sleep(active_window).await; + for job in deferred { + run_rename_tail_cleanup_lane_job(job).await; + } + } + } else { + for job in batch { + run_rename_tail_cleanup_lane_job(job).await; + } + } + worker_active.fetch_sub(1, Ordering::Relaxed); + } + }); + RenameTailCleanupLane { + sender, + queued, + oldest_enqueued_at, + } + }) + .collect(); + Self { lanes } + } + + async fn enqueue( + &self, + disk_index: usize, + future: RenameTailCleanupLaneFuture, + ) -> Option> { + let lane = &self.lanes[disk_index % self.lanes.len()]; + let enqueued_at = Instant::now(); + let queue_push_started = rustfs_io_metrics::put_stage_timer(); + { + let mut oldest = lane.oldest_enqueued_at.lock().await; + if oldest.is_none() { + *oldest = Some(enqueued_at); + } + } + let queued_after_enqueue = lane.queued.fetch_add(1, Ordering::Relaxed) + 1; + let (done, done_rx) = tokio::sync::oneshot::channel(); + let job = RenameTailCleanupLaneJob { + disk_index, + future, + done, + }; + match lane.sender.try_send(job) { + Ok(()) => { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH, + queued_after_enqueue as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE, + queued_after_enqueue as f64, + ); + if let Some(oldest) = *lane.oldest_enqueued_at.lock().await { + let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE, + oldest_age_ms, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE_MAX_SAMPLE, + oldest_age_ms, + ); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_PUSH, + queue_push_started, + ); + Some(done_rx) + } + Err(err) => { + lane.queued.fetch_sub(1, Ordering::Relaxed); + if lane.queued.load(Ordering::Relaxed) == 0 { + *lane.oldest_enqueued_at.lock().await = None; + } + let job = err.into_inner(); + run_rename_tail_cleanup_lane_job(job).await; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE, + lane.queued.load(Ordering::Relaxed) as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_PUSH, + queue_push_started, + ); + None + } + } + } +} + +fn rename_tail_cleanup_lane_queue() -> &'static RenameTailCleanupLaneQueue { + static QUEUE: std::sync::OnceLock = std::sync::OnceLock::new(); + QUEUE.get_or_init(RenameTailCleanupLaneQueue::new) +} + fn delete_marker_purge_candidate( file_info: &FileInfo, expected_bucket_incarnation_id: Option, @@ -358,7 +745,9 @@ use crate::object_api::{ }; use crate::services::notification_sys::RemoteVersionStateFleetProofToken; use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata}; -use crate::set_disk::core::io_primitives::{RenameRollbackReceipt, RenameTailCleanup, finish_rename_tail_heal}; +use crate::set_disk::core::io_primitives::{ + RenameRollbackReceipt, RenameTailCleanup, finish_rename_tail_heal, rename_foreground_active_count, +}; #[cfg(test)] use crate::storage_api_contracts::namespace::NamespaceLocking; #[cfg(test)] @@ -528,7 +917,6 @@ async fn acquire_single_tier_delete_lease(opts: &ObjectOptions, source: &ObjectI Ok(Some(lease)) } -const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json"; const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024; const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256; @@ -2809,9 +3197,9 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { let object_size = usize::try_from(object_info.size) .map_err(|_| to_object_err(Error::other("two-phase GET object size is invalid"), vec![bucket, object]))?; let mut output = Vec::with_capacity(object_size); - let (fi, files, disks, late_metadata_fanout_disks) = snapshot.into_owned_with_late_metadata_fanout(); + let (fi, files, disks, late_metadata_fanout) = snapshot.into_owned_with_late_metadata_fanout(); let expected_identity = super::super::read::LateMetadataIdentity::from_file_info(&fi); - let late_metadata_fanout_disks = late_metadata_fanout_disks.ok_or_else(|| { + let late_metadata_fanout = late_metadata_fanout.ok_or_else(|| { to_object_err(Error::other("two-phase GET fallback context is missing"), vec![bucket, object]) })?; let initial_result = Self::get_object_with_fileinfo( @@ -2836,6 +3224,10 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { ) .await; if prepare_late_materialized_retry(&initial_result, &mut output, object_size) { + let late_metadata_fanout_disks = match late_metadata_fanout { + LateMetadataFanout::CapturedDisks(disks) => disks, + LateMetadataFanout::RefreshCurrentSet => self.get_disks_for_data_read().await, + }; #[cfg(test)] super::super::rename_fanout_barrier::checkpoint( object, @@ -3356,6 +3748,19 @@ pub(in crate::set_disk) fn merge_replication_metadata_lww( } impl SetDisks { + /// Failure cleanup stays awaited, but shares the same execution budget as + /// successful PUT continuations when isolation is enabled. + async fn delete_put_tmp_with_budget(&self, tmp_dir: &str, reservation: &Option) -> Result<()> { + if reservation.is_none() { + return self.delete_all(RUSTFS_META_TMP_BUCKET, tmp_dir).await; + } + let set = self.clone(); + let tmp_dir = tmp_dir.to_owned(); + cleanup_runtime::spawn(reservation.clone(), async move { set.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await }) + .await + .map_err(Error::other)? + } + async fn reclaim_missing_delete_residue( &self, bucket: &str, @@ -3434,26 +3839,107 @@ impl SetDisks { committed_data_dir: Option, epoch: Option, ) { + if rename_tail_cleanup_lane_enabled() { + let bucket = bucket.to_string(); + let object = object.to_string(); + let mut completions = Vec::with_capacity(targets.len()); + for target in targets { + let cleanup_set = self.clone(); + let cleanup_bucket = bucket.clone(); + let cleanup_object = object.clone(); + let disk_index = target.disk_index; + let future = Box::pin(async move { + cleanup_set + .cleanup_rename_tail_target(target, &cleanup_bucket, &cleanup_object, committed_data_dir, epoch) + .await; + }); + if let Some(done) = rename_tail_cleanup_lane_queue().enqueue(disk_index, future).await { + completions.push(done); + } + } + for done in completions { + let _ = done.await; + } + return; + } + for target in targets { - let mut disks = vec![None; self.set_drive_count]; - disks[target.disk_index] = Some(target.disk); - self.persist_old_data_cleanup_receipts(&disks, bucket, object, target.old_data_dir, committed_data_dir, epoch) - .await; - let cleanup = self - .commit_rename_data_dir_and_mark_capacity( - &disks, - bucket, - object, - &target.old_data_dir.to_string(), - &committed_data_dir.unwrap_or_default().to_string(), - 1, - ) - .await; - self.report_old_data_dir_cleanup(bucket, object, &target.old_data_dir.to_string(), &cleanup) + self.cleanup_rename_tail_target(target, bucket, object, committed_data_dir, epoch) .await; } } + async fn cleanup_rename_tail_target( + &self, + target: RenameTailCleanup, + bucket: &str, + object: &str, + committed_data_dir: Option, + epoch: Option, + ) { + let foreground_yield = rename_tail_cleanup_foreground_yield(); + if !foreground_yield.is_zero() { + let yield_started = rustfs_io_metrics::put_stage_timer(); + tokio::time::sleep(foreground_yield).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD, + yield_started, + ); + } + let active_yield = rename_tail_cleanup_foreground_active_yield(); + if !active_yield.is_zero() { + let active = rename_foreground_active_count(target.disk_index); + if active > 0 { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFERRED_FOREGROUND_ACTIVE, + active as f64, + ); + let yield_started = rustfs_io_metrics::put_stage_timer(); + tokio::time::sleep(active_yield).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD, + yield_started, + ); + } + } + let _cleanup_per_disk_permit = rename_tail_cleanup_per_disk_permit(target.disk_index).await; + let mut disks = vec![None; self.set_drive_count]; + disks[target.disk_index] = Some(target.disk); + let receipt_started = rustfs_io_metrics::put_stage_timer(); + self.persist_old_data_cleanup_receipts(&disks, bucket, object, target.old_data_dir, committed_data_dir, epoch) + .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_RECEIPT, + receipt_started, + ); + let commit_capacity_started = rustfs_io_metrics::put_stage_timer(); + let cleanup = self + .commit_rename_data_dir_and_mark_capacity( + &disks, + bucket, + object, + &target.old_data_dir.to_string(), + &committed_data_dir.unwrap_or_default().to_string(), + 1, + ) + .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COMMIT_CAPACITY, + commit_capacity_started, + ); + let report_started = rustfs_io_metrics::put_stage_timer(); + self.report_old_data_dir_cleanup(bucket, object, &target.old_data_dir.to_string(), &cleanup) + .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_REPORT, + report_started, + ); + } + + fn needs_old_data_cleanup_receipt(old_data_dir: Uuid, committed_data_dir: Option, epoch: Option) -> bool { + epoch.is_some() && committed_data_dir != Some(old_data_dir) + } + pub(in crate::set_disk) async fn persist_old_data_cleanup_receipts( &self, disks: &[Option], @@ -3463,10 +3949,10 @@ impl SetDisks { committed_data_dir: Option, epoch: Option, ) { - let Some(epoch) = epoch else { return }; - if committed_data_dir == Some(old_data_dir) { + if !Self::needs_old_data_cleanup_receipt(old_data_dir, committed_data_dir, epoch) { return; } + let Some(epoch) = epoch else { return }; let receipt = OldDataCleanupReceipt::new(epoch, old_data_dir, committed_data_dir); let Ok(encoded) = receipt.encode() else { return; @@ -3652,6 +4138,23 @@ impl SetDisks { .scanner_publication_commit_scope .clone() .map(ScannerPublicationCommitScopeGuard::new); + // Reserve before staging. Some callers already hold external locks, + // so admission must not wait for another PUT to release its budget. + // Commit/cleanup owners retain clones across cancellation and early ACK. + if opts + .put_object_cancellation + .as_ref() + .is_some_and(CancellationToken::is_cancelled) + { + return Err(StorageError::OperationCanceled); + } + let cleanup_reservation = cleanup_runtime::reserve().map_err(|err| { + if err.kind() == std::io::ErrorKind::WouldBlock { + StorageError::SlowDown + } else { + Error::other(err) + } + })?; let storage_class_config = self.storage_class_config_snapshot(); self.invalidate_get_object_metadata_cache(bucket, object).await; @@ -3771,7 +4274,13 @@ impl SetDisks { }; let mut tmp_cleanup_owned = false; let rollback_receipt = RenameRollbackReceipt::default(); + let operation_first_poll_delay_started = rustfs_io_metrics::put_stage_timer(); let operation = async { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY, + operation_first_poll_delay_started, + ); + let operation_body_to_commit_submit_started = rustfs_io_metrics::put_stage_timer(); let erasure = Arc::new(erasure_from_file_info(&fi, false)?); let put_object_size = known_put_object_storage_size(data.size()); @@ -4209,6 +4718,8 @@ impl SetDisks { } #[cfg(any(test, feature = "test-util"))] pause_put_object_commit(bucket, object, PutObjectCommitPause::AfterNamespace).await; + let namespace_lock_acquired_to_commit_ready_started = rustfs_io_metrics::put_stage_timer(); + let commit_ready_precondition_and_timestamp_started = rustfs_io_metrics::put_stage_timer(); if opts.http_preconditions.is_some() && let Some(err) = self.check_write_precondition(bucket, object, opts).await @@ -4229,8 +4740,13 @@ impl SetDisks { } } } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_PRECONDITION_AND_TIMESTAMP, + commit_ready_precondition_and_timestamp_started, + ); if let Some(expected) = opts.expected_current_version_id.as_deref() { + let expected_version_lookup_started = rustfs_io_metrics::put_stage_timer(); let current = self .get_object_info( bucket, @@ -4253,11 +4769,16 @@ impl SetDisks { if current.version_id.map(|version| version.to_string()).as_deref() != Some(expected) { return Err(StorageError::PreconditionFailed); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPECTED_VERSION_LOOKUP, + expected_version_lookup_started, + ); } if let Some(version_id) = opts.version_id.as_deref() && !is_meta_bucketname(bucket) { + let explicit_version_lookup_started = rustfs_io_metrics::put_stage_timer(); let current = self .get_object_info( bucket, @@ -4310,13 +4831,23 @@ impl SetDisks { Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {} Err(err) => return Err(err), } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPLICIT_VERSION_LOOKUP, + explicit_version_lookup_started, + ); } + let restore_verify_started = rustfs_io_metrics::put_stage_timer(); self.require_current_restore_operation_id(bucket, object, opts, expected_restore_operation_id, "put_object_commit") .await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_RESTORE_VERIFY, + restore_verify_started, + ); // Fence every commit-time read before entering rename_data. Once // rename_data returns Ok the write is durable and must not be aborted. + let fence_verify_started = rustfs_io_metrics::put_stage_timer(); if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) || publication_commit_guard .as_ref() @@ -4360,6 +4891,7 @@ impl SetDisks { }); } + let decommission_capacity_fence_started = rustfs_io_metrics::put_stage_timer(); if decommission_capacity_guard.is_none() && let Some(store) = opts.decommission_capacity_admission.as_ref() { @@ -4369,6 +4901,10 @@ impl SetDisks { .await?, ); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_DECOMMISSION_CAPACITY_FENCE, + decommission_capacity_fence_started, + ); // The object namespace is acquired above, after the input stream // has been fully staged. Only then admit the local publication @@ -4386,21 +4922,36 @@ impl SetDisks { achieved: 0, }); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_FENCE_VERIFY, + fence_verify_started, + ); let transaction_fencing_proof = object_transaction_fencing_fleet_proof(); if object_transaction_fencing_requested() && transaction_fencing_proof.is_none() { return Err(Error::other("object transaction fencing requires a live fleet capability proof")); } + let transaction_epoch_fence_read_started = rustfs_io_metrics::put_stage_timer(); let transaction_epoch_fence = if transaction_fencing_proof.is_some() { Some(read_object_transaction_epoch_fence(self, bucket, object).await?) } else { None }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_TRANSACTION_EPOCH_FENCE_READ, + transaction_epoch_fence_read_started, + ); + let quota_begin_started = rustfs_io_metrics::put_stage_timer(); let quota_context = reservation::begin(&self.ctx, bucket, object, opts, self.pool_index, self.set_index).await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_BEGIN, + quota_begin_started, + ); let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some(); let mut replication_quota_size = None; + let metadata_adjust_started = rustfs_io_metrics::put_stage_timer(); if opts.replication_request { if quota_context.is_enforced() && opts.preserve_ciphertext { return Err(Error::PartMissingOrCorrupt); @@ -4464,7 +5015,12 @@ impl SetDisks { } } } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_METADATA_ADJUST, + metadata_adjust_started, + ); + let quota_reserve_started = rustfs_io_metrics::put_stage_timer(); let (quota_old_size, quota_new_size) = if quota_context.is_enforced() { let new_size = match replication_quota_size { Some(size) => size, @@ -4482,8 +5038,13 @@ impl SetDisks { (0, 0) }; let quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_RESERVE, + quota_reserve_started, + ); let (commit_disks, quota_fence_tokens) = if quota_mutation_fence { - match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await { + let quota_mutation_fence_prepare_started = rustfs_io_metrics::put_stage_timer(); + let prepared = match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await { Ok((disks, tokens)) => { for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) { if let Some(token) = token { @@ -4500,10 +5061,16 @@ impl SetDisks { quota_reservation.abort().await; return Err(err); } - } + }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_MUTATION_FENCE_PREPARE, + quota_mutation_fence_prepare_started, + ); + prepared } else { (shuffle_disks.clone(), vec![None; shuffle_disks.len()]) }; + let commit_context_prepare_started = rustfs_io_metrics::put_stage_timer(); let transaction_epoch = transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&commit_disks, &mut parts_metadatas)); @@ -4550,10 +5117,26 @@ impl SetDisks { let commit_tier_free_version_receipt_sink = opts.tier_free_version_receipt_sink.clone(); let commit_skip_free_version = opts.skip_free_version; let request_cancellation = operation_cancellation.clone(); + let commit_cleanup_reservation = cleanup_reservation.clone(); tmp_cleanup_owned = true; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_CONTEXT_PREPARE, + commit_context_prepare_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY, + namespace_lock_acquired_to_commit_ready_started, + ); - let commit = move |cancellation: Option| async move { + let commit = move |commit_submit_to_closure_enter_started: Option, + cancellation: Option| async move { let write_context = write_context; + let commit_closure_total_started = rustfs_io_metrics::put_stage_timer(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER, + commit_submit_to_closure_enter_started, + ); + let commit_closure_enter_to_first_await_started = rustfs_io_metrics::put_stage_timer(); let mut _object_lock_guard = commit_object_lock_guard; let mut _decommission_object_lock_guard = commit_decommission_object_lock_guard; let mut _publication_guard = commit_publication_guard; @@ -4561,12 +5144,20 @@ impl SetDisks { let mut _decommission_capacity_guard = commit_decommission_capacity_guard; let mut quota_reservation = quota_reservation; let rename_stage_start = Instant::now(); + let commit_before_rename_started = rustfs_io_metrics::put_stage_timer(); + let lock_held_pre_rename_prepare_started = rustfs_io_metrics::put_stage_timer(); let pre_rename = async { #[cfg(any(test, feature = "test-util"))] pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterQuotaReservation).await; + let quota_mark_commit_started = rustfs_io_metrics::put_stage_timer(); quota_reservation.mark_commit_started().await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_QUOTA_MARK_COMMIT_STARTED, + quota_mark_commit_started, + ); #[cfg(any(test, feature = "test-util"))] pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::BeforeQuotaRename).await; + let first_fence_check_started = rustfs_io_metrics::put_stage_timer(); if quota_reservation.is_lock_lost() || !quota_reservation.capability_proof_matches() || write_context.is_lock_lost() @@ -4596,6 +5187,10 @@ impl SetDisks { achieved: 0, }); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_FIRST_FENCE_CHECK, + first_fence_check_started, + ); let restore_opts = ObjectOptions { version_id: commit_version_id.clone(), versioned: commit_versioned, @@ -4603,6 +5198,7 @@ impl SetDisks { no_lock: true, ..Default::default() }; + let restore_verify_started = rustfs_io_metrics::put_stage_timer(); commit_set .require_current_restore_operation_id( &commit_bucket, @@ -4612,6 +5208,10 @@ impl SetDisks { "put_object_quota_reservation", ) .await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_RESTORE_VERIFY, + restore_verify_started, + ); if let Some(proof) = transaction_fencing_proof.as_ref() && !object_transaction_fencing_fleet_proof_matches(proof) { @@ -4625,8 +5225,14 @@ impl SetDisks { PutObjectCommitPause::BeforeTransactionEpochVerify, ) .await; + let transaction_fence_verify_started = rustfs_io_metrics::put_stage_timer(); verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TRANSACTION_FENCE_VERIFY, + transaction_fence_verify_started, + ); } + let second_fence_check_started = rustfs_io_metrics::put_stage_timer(); if quota_reservation.is_lock_lost() || !quota_reservation.capability_proof_matches() || write_context.is_lock_lost() @@ -4656,11 +5262,20 @@ impl SetDisks { achieved: 0, }); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SECOND_FENCE_CHECK, + second_fence_check_started, + ); Ok(()) }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT, + commit_closure_enter_to_first_await_started, + ); + let pre_rename_await_total_started = rustfs_io_metrics::put_stage_timer(); let mut pre_rename_result = if cancellation.is_some() || request_cancellation.is_some() { tokio::select! { - biased; + biased; _ = wait_for_put_object_commit_cancellation(cancellation.as_ref(), request_cancellation.as_ref()) => { Err(StorageError::OperationCanceled) }, @@ -4669,6 +5284,12 @@ impl SetDisks { } else { pre_rename.await }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_AWAIT_TOTAL, + pre_rename_await_total_started, + ); + let pre_rename_post_await_to_rename_started = rustfs_io_metrics::put_stage_timer(); + let scanner_scope_check_started = rustfs_io_metrics::put_stage_timer(); if pre_rename_result.is_ok() && let Some(scope) = commit_scanner_publication_scope.as_ref() && let Err(err) = scope.try_begin() @@ -4683,6 +5304,10 @@ impl SetDisks { let _ = scope.mark_indeterminate(); pre_rename_result = Err(StorageError::OperationCanceled); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SCANNER_SCOPE_CHECK, + scanner_scope_check_started, + ); if let Err(err) = pre_rename_result { SetDisks::abort_quota_reservation_after_fence( quota_reservation, @@ -4694,7 +5319,10 @@ impl SetDisks { quota_mutation_fence, ) .await; - if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await { + if let Err(cleanup_err) = commit_set + .delete_put_tmp_with_budget(&commit_tmp_dir, &commit_cleanup_reservation) + .await + { warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { warn!( @@ -4707,43 +5335,72 @@ impl SetDisks { } return Err(err); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_PRE_RENAME_PREPARE, + lock_held_pre_rename_prepare_started, + ); - let put_tier_free_version_source = - if commit_put_tier_free_version_id.is_some() && commit_tier_free_version_receipt_sink.is_some() { - match commit_set - .get_object_info( - &commit_bucket, - &commit_object, - &ObjectOptions { - no_lock: true, - metadata_cache_safe: false, - versioned: commit_versioned, - version_suspended: commit_version_suspended, - ..Default::default() - }, - ) - .await - { - Ok(source) => Some(source), - Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => None, - Err(err) => { - debug!( - event = EVENT_LIFECYCLE_TRANSITIONED_DELETE_CLEANUP_OWNER, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_SET_DISK, - bucket = %commit_bucket, - object = %commit_object, - error = ?err, - "Skipped opportunistic tier free-version receipt source capture" - ); - None - } + let tier_free_version_source_lookup_started = rustfs_io_metrics::put_stage_timer(); + let reuse_rename_data_source_for_tier_free_version = tier_free_version_source_reuse_rename_data_enabled() + && (commit_set.tier_free_version_rename_data_source_capable() + || tier_free_version_source_reuse_rename_data_assume_homogeneous_enabled()); + let mut put_tier_free_version_source = if commit_put_tier_free_version_id.is_some() + && commit_tier_free_version_receipt_sink.is_some() + && !tier_free_version_source_lookup_counterfactual_skip_enabled() + && !reuse_rename_data_source_for_tier_free_version + { + match commit_set + .get_object_info( + &commit_bucket, + &commit_object, + &ObjectOptions { + no_lock: true, + metadata_cache_safe: false, + versioned: commit_versioned, + version_suspended: commit_version_suspended, + ..Default::default() + }, + ) + .await + { + Ok(source) => Some(source), + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => None, + Err(err) => { + debug!( + event = EVENT_LIFECYCLE_TRANSITIONED_DELETE_CLEANUP_OWNER, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket = %commit_bucket, + object = %commit_object, + error = ?err, + "Skipped opportunistic tier free-version receipt source capture" + ); + None } - } else { - None - }; + } + } else { + None + }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TIER_FREE_VERSION_SOURCE_LOOKUP, + tier_free_version_source_lookup_started, + ); + let assign_rename_indexes_started = rustfs_io_metrics::put_stage_timer(); Self::assign_rename_data_indexes(&mut parts_metadatas); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_ASSIGN_RENAME_INDEXES, + assign_rename_indexes_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_POST_AWAIT_TO_RENAME, + pre_rename_post_await_to_rename_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME, + commit_before_rename_started, + ); + let lock_held_rename_started = rustfs_io_metrics::put_stage_timer(); let mut rename_result = SetDisks::rename_data_owned_with_fence( &commit_disks, (RUSTFS_META_TMP_BUCKET, commit_tmp_dir.as_str()), @@ -4761,6 +5418,12 @@ impl SetDisks { ), ) .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME, + lock_held_rename_started, + ); + let commit_after_rename_to_return_started = rustfs_io_metrics::put_stage_timer(); + let lock_held_rename_result_to_tail_handoff_started = rustfs_io_metrics::put_stage_timer(); if let Some(scope) = commit_scanner_publication_scope.as_ref() { if rename_result.is_ok() { let _ = scope.mark_committed(); @@ -4775,7 +5438,15 @@ impl SetDisks { let mut rename_guard_release = None; let mut needs_immediate_heal = false; let mut tail_owns_tmp_cleanup = false; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME_RESULT_TO_TAIL_HANDOFF, + lock_held_rename_result_to_tail_handoff_started, + ); + let lock_held_tail_handoff_started = rustfs_io_metrics::put_stage_timer(); if let Ok(rename_commit) = rename_result.as_mut() { + commit_set.observe_tier_free_version_rename_data_source_capability( + rename_commit.tail_drain.is_none() && rename_commit.old_current_source_all_successful_checked, + ); commit_set.record_capacity_scope_if_needed(commit_capacity_scope_token, &rename_commit.capacity_disks); // Install the tail watcher before any post-commit await. The // latch keeps namespace guards through their prior handoff point. @@ -4810,6 +5481,7 @@ impl SetDisks { let fence_object = commit_object.clone(); let (guard_release_tx, guard_release_rx) = tokio::sync::oneshot::channel(); rename_guard_release = Some(guard_release_tx); + let cleanup_reservation = commit_cleanup_reservation.clone(); tokio::spawn(finish_rename_tail_heal( rename_tail_drain, guard_release_rx, @@ -4842,36 +5514,70 @@ impl SetDisks { decommission_object_lock_guard, decommission_capacity_guard, ), - targets| async move { + targets| { drop(object_lock_guard); drop(write_context); drop(publication_guard); drop(bucket_lifecycle_guard); - cleanup_set - .cleanup_rename_tail( - targets, - &cleanup_bucket, - &cleanup_object, - committed_data_dir, - transaction_epoch, - ) - .await; - if let Err(err) = cleanup_set.delete_all(RUSTFS_META_TMP_BUCKET, &cleanup_tmp_dir).await { - warn!(tmp_dir = %cleanup_tmp_dir, error = ?err, "failed to cleanup put_object temporary data"); - } else if issue3031_diag_enabled() { - warn!( - target: "rustfs_ecstore::set_disk", - tmp_dir = %cleanup_tmp_dir, - "issue3031_put_object_tmp_cleanup_done" + cleanup_runtime::run(cleanup_reservation, async move { + let zero_target_cleanup = targets.is_empty(); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COUNT, + targets.len() as f64, ); - } - drop(decommission_object_lock_guard); - drop(decommission_capacity_guard); + if zero_target_cleanup { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_ZERO_TARGET, + 1.0, + ); + } + cleanup_set + .cleanup_rename_tail( + targets, + &cleanup_bucket, + &cleanup_object, + committed_data_dir, + transaction_epoch, + ) + .await; + let tmp_delete_started = rustfs_io_metrics::put_stage_timer(); + if zero_target_cleanup && rename_tail_cleanup_zero_target_tmp_delete_skip_enabled() { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE_COUNTERFACTUAL_SKIP, + 1.0, + ); + } else if let Err(err) = + cleanup_set.delete_all(RUSTFS_META_TMP_BUCKET, &cleanup_tmp_dir).await + { + warn!(tmp_dir = %cleanup_tmp_dir, error = ?err, "failed to cleanup put_object temporary data"); + } else if issue3031_diag_enabled() { + warn!( + target: "rustfs_ecstore::set_disk", + tmp_dir = %cleanup_tmp_dir, + "issue3031_put_object_tmp_cleanup_done" + ); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE, + tmp_delete_started, + ); + let guard_drop_started = rustfs_io_metrics::put_stage_timer(); + drop(decommission_object_lock_guard); + drop(decommission_capacity_guard); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DECOMMISSION_GUARD_DROP, + guard_drop_started, + ); + }) }, |request| async move { heal_set.submit_rename_tail_heal(request).await }, )); } } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_TAIL_HANDOFF, + lock_held_tail_handoff_started, + ); if !tail_owns_tmp_cleanup { drop(_decommission_capacity_guard.take()); } @@ -4880,6 +5586,11 @@ impl SetDisks { pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameHandoff).await; } if quota_mutation_fence && !tail_owns_tmp_cleanup { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_TRIGGERED, + 1.0, + ); + let quota_fence_release_started = rustfs_io_metrics::put_stage_timer(); let _ = SetDisks::release_quota_mutation_fences( &commit_disks, "a_fence_tokens, @@ -4888,9 +5599,23 @@ impl SetDisks { write_quorum, ) .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE, + quota_fence_release_started, + ); + } else { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_SKIPPED, + 1.0, + ); } if rename_result.is_ok() { + let quota_commit_started = rustfs_io_metrics::put_stage_timer(); quota_reservation.commit().await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_COMMIT, + quota_commit_started, + ); } let rename_commit = match rename_result { Ok(commit) => commit, @@ -4900,7 +5625,10 @@ impl SetDisks { // rollback backup for recovery; cleanup is unsafe. return Err(err.into()); } - if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await { + if let Err(cleanup_err) = commit_set + .delete_put_tmp_with_budget(&commit_tmp_dir, &commit_cleanup_reservation) + .await + { warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { warn!( @@ -4918,7 +5646,17 @@ impl SetDisks { let op_old_dir = rename_commit.data_dir; let cleanup_disks = rename_commit.cleanup_disks; let old_current_size = rename_commit.old_current_size; + if put_tier_free_version_source.is_none() + && reuse_rename_data_source_for_tier_free_version + && rename_commit.old_current_source_checked + { + put_tier_free_version_source = rename_commit + .old_current_source + .as_ref() + .map(|source| build_get_object_info(source, &commit_bucket, &commit_object, commit_versioned)); + } let mut fi = rename_commit.committed_file_info; + let lock_held_post_rename_started = rustfs_io_metrics::put_stage_timer(); if let (Some(source), Some(free_version_id)) = (put_tier_free_version_source.as_ref(), commit_put_tier_free_version_id) && transitioned_delete_publishes_free_version(source, &fi, commit_skip_free_version) @@ -4934,6 +5672,11 @@ impl SetDisks { } if needs_immediate_heal { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_TRIGGERED, + 1.0, + ); + let heal_submit_started = rustfs_io_metrics::put_stage_timer(); let mut request = rustfs_heal_contracts::heal_channel::create_heal_request_with_options( commit_bucket.clone(), Some(commit_object.clone()), @@ -4947,28 +5690,83 @@ impl SetDisks { .map(|version_id| version_id.to_string()); request.expected_bucket_incarnation_id = source_bucket_incarnation_id; commit_set.submit_rename_tail_heal(request).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT, + heal_submit_started, + ); + } else { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_SKIPPED, + 1.0, + ); } let rename_stage_elapsed = rename_stage_start.elapsed(); let rename_stage_ms = rename_stage_elapsed.as_millis() as u64; - if let Some(old_dir) = op_old_dir { - commit_set - .persist_old_data_cleanup_receipts( - &cleanup_disks, - &commit_bucket, - &commit_object, - old_dir, - committed_data_dir, - transaction_epoch, - ) + if let Some(old_dir) = op_old_dir + && Self::needs_old_data_cleanup_receipt(old_dir, committed_data_dir, transaction_epoch) + { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_TRIGGERED, + 1.0, + ); + let cleanup_receipt_started = rustfs_io_metrics::put_stage_timer(); + if commit_cleanup_reservation.is_none() { + commit_set + .persist_old_data_cleanup_receipts( + &cleanup_disks, + &commit_bucket, + &commit_object, + old_dir, + committed_data_dir, + transaction_epoch, + ) + .await; + } else { + let cleanup_set = commit_set.clone(); + let receipt_disks = cleanup_disks.clone(); + let receipt_bucket = commit_bucket.clone(); + let receipt_object = commit_object.clone(); + cleanup_runtime::run(commit_cleanup_reservation.clone(), async move { + cleanup_set + .persist_old_data_cleanup_receipts( + &receipt_disks, + &receipt_bucket, + &receipt_object, + old_dir, + committed_data_dir, + transaction_epoch, + ) + .await; + }) .await; + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT, + cleanup_receipt_started, + ); + } else { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_SKIPPED, + 1.0, + ); } + let metadata_invalidate_started = rustfs_io_metrics::put_stage_timer(); commit_set .invalidate_get_object_metadata_cache(&commit_bucket, &commit_object) .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_METADATA_INVALIDATE, + metadata_invalidate_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_RENAME, + lock_held_post_rename_started, + ); + let lock_held_guard_release_started = rustfs_io_metrics::put_stage_timer(); if let Some(release) = rename_guard_release.take() { let _ = release.send(true); } @@ -4977,6 +5775,10 @@ impl SetDisks { drop(_object_lock_guard.take()); drop(_publication_guard.take()); drop(_bucket_lifecycle_guard.take()); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_GUARD_RELEASE, + lock_held_guard_release_started, + ); rustfs_io_metrics::record_put_object_stage_duration("set_disk_rename", duration_millis_f64(rename_stage_elapsed)); if (rename_stage_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS { @@ -5004,16 +5806,41 @@ impl SetDisks { // here must not negate an already-committed, durable write, so we // deliberately do NOT `?`-propagate it into a 503. On residue the // report path emits the leak metric and enqueues a heal. - let cleanup = commit_set - .commit_rename_data_dir_and_mark_capacity( - &cleanup_disks, - &commit_bucket, - &commit_object, - &old_dir.to_string(), - &committed_dir, - write_quorum, - ) + if commit_cleanup_reservation.is_none() { + let cleanup = commit_set + .commit_rename_data_dir_and_mark_capacity( + &cleanup_disks, + &commit_bucket, + &commit_object, + &old_dir.to_string(), + &committed_dir, + write_quorum, + ) + .await; + commit_set + .report_old_data_dir_cleanup(&commit_bucket, &commit_object, &old_dir.to_string(), &cleanup) + .await; + } else { + let cleanup_set = commit_set.clone(); + let cleanup_bucket = commit_bucket.clone(); + let cleanup_object = commit_object.clone(); + cleanup_runtime::run(commit_cleanup_reservation.clone(), async move { + let cleanup = cleanup_set + .commit_rename_data_dir_and_mark_capacity( + &cleanup_disks, + &cleanup_bucket, + &cleanup_object, + &old_dir.to_string(), + &committed_dir, + write_quorum, + ) + .await; + cleanup_set + .report_old_data_dir_cleanup(&cleanup_bucket, &cleanup_object, &old_dir.to_string(), &cleanup) + .await; + }) .await; + } let cleanup_elapsed = cleanup_stage_start.elapsed(); let cleanup_ms = cleanup_elapsed.as_millis() as u64; cleanup_stage_ms = Some(cleanup_ms); @@ -5021,9 +5848,6 @@ impl SetDisks { "set_disk_old_data_cleanup", duration_millis_f64(cleanup_elapsed), ); - commit_set - .report_old_data_dir_cleanup(&commit_bucket, &commit_object, &old_dir.to_string(), &cleanup) - .await; if (cleanup_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS { warn!( event = EVENT_SET_DISK_COMMIT_TAIL_SLOW, @@ -5103,7 +5927,7 @@ impl SetDisks { if !tail_owns_tmp_cleanup { let cleanup_set = commit_set.clone(); let cleanup_tmp_dir = commit_tmp_dir.clone(); - tokio::spawn(async move { + cleanup_runtime::spawn(commit_cleanup_reservation.clone(), async move { if let Err(err) = cleanup_set.delete_all(RUSTFS_META_TMP_BUCKET, &cleanup_tmp_dir).await { warn!(tmp_dir = %cleanup_tmp_dir, error = ?err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { @@ -5116,15 +5940,35 @@ impl SetDisks { }); } - Ok(( - ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned), - old_current_size, - )) + let object_info_build_started = rustfs_io_metrics::put_stage_timer(); + let object_info = ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_OBJECT_INFO_BUILD, + object_info_build_started, + ); + let post_object_info_to_return_started = rustfs_io_metrics::put_stage_timer(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_OBJECT_INFO_TO_RETURN, + post_object_info_to_return_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN, + commit_after_rename_to_return_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL, + commit_closure_total_started, + ); + Ok((object_info, old_current_size)) }; if let Some(handoff) = commit_cancellation_handoff_tx.take() { let _ = handoff.send(()); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT, + operation_body_to_commit_submit_started, + ); if detach_commit_owner { if let Some(scope_outcome_guard) = scope_outcome_guard.as_mut() { // The spawned commit closure owns the scope clone and is @@ -5133,15 +5977,40 @@ impl SetDisks { } let mut cancellation = PutObjectCommitCancellation::new(); let child_token = cancellation.child_token(); - let result = tokio::spawn(async move { Box::pin(commit(Some(child_token))).await }) + let commit_task_wait_started = rustfs_io_metrics::put_stage_timer(); + let commit_submit_to_closure_enter_started = rustfs_io_metrics::put_stage_timer(); + let result = + tokio::spawn( + async move { Box::pin(commit(commit_submit_to_closure_enter_started, Some(child_token))).await }, + ) .await .map_err(|err| Error::other(format!("put_object commit task failed: {err}")))?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT, + commit_task_wait_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL, + commit_task_wait_started, + ); cancellation.disarm(); result } else { - Box::pin(commit(None)).await + let commit_task_wait_started = rustfs_io_metrics::put_stage_timer(); + let commit_submit_to_closure_enter_started = rustfs_io_metrics::put_stage_timer(); + let result = Box::pin(commit(commit_submit_to_closure_enter_started, None)).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT, + commit_task_wait_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL, + commit_task_wait_started, + ); + result } }; + let operation_select_wait_started = rustfs_io_metrics::put_stage_timer(); let result: Result<(ObjectInfo, Option)> = if let Some((cancellation, mut handoff_rx)) = cancellation_wait { tokio::pin!(operation); @@ -5154,6 +6023,19 @@ impl SetDisks { } else { operation.await }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT, + operation_select_wait_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL, + operation_select_wait_started, + ); + let operation_result_return_started = rustfs_io_metrics::put_stage_timer(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN, + operation_result_return_started, + ); if issue3031_diag_enabled() && let Err(err) = &result @@ -5201,7 +6083,7 @@ impl SetDisks { // entry is reclaimed by cleanup_stale_tmp_objects (24h expiry, // 5-minute background loop). let set_disks = self.clone(); - tokio::spawn(async move { + cleanup_runtime::spawn(cleanup_reservation.clone(), async move { if let Err(err) = set_disks.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await { warn!(tmp_dir = %tmp_dir, error = ?err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { @@ -5216,7 +6098,7 @@ impl SetDisks { // Failure path (quorum loss / rollback): keep the cleanup inline so // a failed PUT never returns while its tmp shards are still on disk // (state-residue hardening tracked by backlog#864 / backlog#898). - if let Err(err) = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await { + if let Err(err) = self.delete_put_tmp_with_budget(&tmp_dir, &cleanup_reservation).await { warn!(tmp_dir = %tmp_dir, error = ?err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { warn!( @@ -19412,6 +20294,73 @@ mod put_object_tmp_cleanup_tests { ); } + #[tokio::test] + async fn put_cleanup_admission_rejects_before_staging_with_an_external_lock() { + const CHILD: &str = "RUSTFS_CLEANUP_ADMISSION_TEST_CHILD"; + if std::env::var_os(CHILD).is_none() { + let output = tokio::task::spawn_blocking(|| { + std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "set_disk::ops::object::put_object_tmp_cleanup_tests::put_cleanup_admission_rejects_before_staging_with_an_external_lock", + "--nocapture", + ]) + .env(CHILD, "1") + .env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true") + .env("RUSTFS_PUT_RENAME_TAIL_CLEANUP_MAX_PENDING", "1") + .env("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS", "1") + .env("RUSTFS_CLEANUP_CPUS", "none") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP") + .output() + .unwrap() + }) + .await + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + return; + } + let (dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "cleanup-admission-bucket"; + let object = "externally-locked"; + for disk in &disks { + disk.make_volume(bucket).await.unwrap(); + } + let occupied = cleanup_runtime::reserve().unwrap().expect("isolation enabled in child"); + let external_lock = set.acquire_write_lock_diag("admission_test", bucket, object).await.unwrap(); + let mut reader = PutObjReader::from_vec(vec![7u8; TEST_OBJECT_SIZE]); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + let result = tokio::time::timeout(Duration::from_secs(20), set.put_object(bucket, object, &mut reader, &opts)) + .await + .expect("admission must not wait while the caller owns locks"); + assert!(matches!(result, Err(StorageError::SlowDown))); + assert!(non_trash_tmp_entries(&dirs).await.is_empty(), "rejection must not stage shards"); + drop(external_lock); + drop(occupied); + let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]); + set.put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .unwrap(); + wait_for_tmp_workspace_to_drain(&dirs, "admitted PUT must eventually clean staging").await; + let mut actual = set + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .unwrap(); + let mut body = Vec::new(); + actual.stream.read_to_end(&mut body).await.unwrap(); + assert_eq!(body, vec![9u8; TEST_OBJECT_SIZE]); + } + #[tokio::test] async fn put_object_success_eventually_cleans_tmp_workspace() { let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; @@ -24239,6 +25188,22 @@ mod single_delete_namespace_owner_tests { } } +#[cfg(test)] +mod cleanup_receipt_admission_tests { + use super::*; + + #[test] + fn receipt_admission_skips_only_work_that_the_writer_would_skip() { + let old = Uuid::new_v4(); + let current = Uuid::new_v4(); + let epoch = Uuid::new_v4(); + assert!(!SetDisks::needs_old_data_cleanup_receipt(old, Some(current), None)); + assert!(!SetDisks::needs_old_data_cleanup_receipt(old, Some(old), Some(epoch))); + assert!(SetDisks::needs_old_data_cleanup_receipt(old, Some(current), Some(epoch))); + assert!(SetDisks::needs_old_data_cleanup_receipt(old, None, Some(epoch))); + } +} + #[cfg(test)] mod crash_delete_retry_tests { use super::*; diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 172e40140..b2d116309 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -15,9 +15,9 @@ use super::{ Arc, Bytes, DiskError, DiskOption, DiskStore, Endpoint, ErasureCache, Error, FileInfo, GetCodecStreamingFallbackReason, GetObjectFileInfo, GetObjectMetadataCacheCapability, GetObjectMetadataCacheEntry, GetObjectMetadataCacheGeneration, - GetObjectMetadataCacheKey, GetObjectReadPolicy, HashAlgorithm, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, - OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectOptions, RUSTFS_META_BUCKET, ReadOptions, Result, SetDisks, StorageError, - adaptive_duplex_buffer_size, build_get_codec_streaming_decode_engine, build_inline_bitrot_readers_from_refs, + GetObjectMetadataCacheKey, GetObjectMetadataCacheLookupKey, GetObjectReadPolicy, HashAlgorithm, LOG_COMPONENT_ECSTORE, + LOG_SUBSYSTEM_SET_DISK, OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectOptions, RUSTFS_META_BUCKET, ReadOptions, Result, SetDisks, + StorageError, adaptive_duplex_buffer_size, build_get_codec_streaming_decode_engine, build_inline_bitrot_readers_from_refs, collect_inline_data_shard_fileinfos_by_index, debug, error, get_codec_streaming_metrics_path, get_codec_streaming_multipart_max_parts, get_object_read_policy, is_codec_streaming_multipart_enabled, is_multipart_reader_setup_prefetch_enabled, object_fits_single_block, reduce_read_quorum_errs, to_object_err, @@ -208,6 +208,16 @@ use tokio::time::timeout; #[cfg(test)] use uuid::Uuid; +fn metadata_quorum_outcome(error: &StorageError) -> &'static str { + if error.is_not_found() { + "not_found" + } else if error.is_quorum_error() { + "not_reached" + } else { + "unavailable" + } +} + pub(super) struct GetObjectDownstreamWriter { inner: W, } @@ -400,7 +410,7 @@ impl SetDisks { let Some(generation) = self.get_object_metadata_cache_generation(bucket, object) else { return MetadataCacheLookup::Miss; }; - let key = GetObjectMetadataCacheKey::new(bucket, object, generation); + let key = GetObjectMetadataCacheLookupKey::new(bucket, object, generation); // moka handles TTL expiry automatically; no is_fresh() check needed let Some(entry) = self.get_object_metadata_cache.get(&key).await else { return MetadataCacheLookup::Miss; @@ -462,14 +472,14 @@ impl SetDisks { entry: Arc, after_insert: impl FnOnce(), ) { - // Every publisher shares Moka's per-key compute lock. A conditional - // snapshot cannot downgrade a full layout or extend its original TTL. + // Every publisher shares Moka's per-key compute lock. A partial snapshot + // cannot downgrade a full layout or extend its original TTL. self.get_object_metadata_cache .entry(key.clone()) .and_compute_with(|current| { std::future::ready( if !self.is_get_object_metadata_cache_generation_current(generation) - || (entry.capability == GetObjectMetadataCacheCapability::ConditionOnly + || (entry.capability != GetObjectMetadataCacheCapability::Full && current .as_ref() .is_some_and(|existing| existing.value().capability == GetObjectMetadataCacheCapability::Full)) @@ -602,6 +612,11 @@ impl SetDisks { purpose: MetadataReadPurpose, ) -> Result { let vid = opts.version_id.clone().unwrap_or_default(); + let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) { + GET_OBJECT_PATH_INTERNAL_META + } else { + GET_OBJECT_PATH_LEGACY_DUPLEX + }; let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); let metadata_cache_lookup_start = get_stage_timer_if_enabled(stage_metrics_enabled); @@ -617,12 +632,17 @@ impl SetDisks { match self.lookup_cached_get_object_fileinfo(bucket, object).await { MetadataCacheLookup::Hit(cached) if cached.capability == GetObjectMetadataCacheCapability::Full + || (cached.capability == GetObjectMetadataCacheCapability::QuorumSelected + && read_data + && allow_early_stop) || (purpose == MetadataReadPurpose::GetObjectReader + && cached.capability == GetObjectMetadataCacheCapability::ConditionOnly && allow_early_stop && crate::object_api::get_object_read_condition_is_terminal( crate::object_api::GetObjectReadMetadata::from_file_info(bucket, object, &cached.fi), )) => { + rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "reached"); rustfs_io_metrics::record_get_object_metadata_cache_decision( GET_OBJECT_PATH_SET_DISK, GET_METADATA_CACHE_DECISION_HIT, @@ -633,7 +653,15 @@ impl SetDisks { GET_STAGE_METADATA_CACHE_LOOKUP, metadata_cache_lookup_start, ); - return Ok(GetObjectFileInfo::shared(cached)); + let snapshot = if cached.capability == GetObjectMetadataCacheCapability::QuorumSelected + && non_inline_data_read_early_stop_allowed(read_data, bucket, object) + && late_materialization_candidate_is_safe(&cached.fi) + { + GetObjectFileInfo::shared_with_late_metadata_refresh(cached) + } else { + GetObjectFileInfo::shared(cached) + }; + return Ok(snapshot); } MetadataCacheLookup::Hit(_) => { // Keep the conditional entry for other requests; this read @@ -683,7 +711,7 @@ impl SetDisks { let allow_read_version_coalescing = purpose == MetadataReadPurpose::GetObjectReader && !crate::bucket::utils::is_meta_bucketname(bucket) && crate::runtime::global::get_metadata_read_version_coalescing_service_ready(); - let metadata_read = if allow_read_version_coalescing { + let metadata_read_result = if allow_read_version_coalescing { Self::read_metadata_for_get_object( &disks, "", @@ -695,7 +723,7 @@ impl SetDisks { allow_early_stop, self.default_parity_count, ) - .await? + .await } else { Self::read_metadata_observed( &disks, @@ -709,16 +737,19 @@ impl SetDisks { allow_early_stop, self.default_parity_count, ) - .await? + .await + }; + let metadata_read = match metadata_read_result { + Ok(metadata_read) => metadata_read, + Err(err) => { + let err = StorageError::from(err); + rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, metadata_quorum_outcome(&err)); + return Err(err); + } }; let metadata_fanout_complete = metadata_read.is_complete(); let condition_stopped = metadata_read.condition_stopped; let (mut parts_metadata, errs, metadata_fanout_diagnostics) = metadata_read.into_legacy(); - let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) { - GET_OBJECT_PATH_INTERNAL_META - } else { - GET_OBJECT_PATH_LEGACY_DUPLEX - }; metadata_fanout_diagnostics.record(metadata_metrics_path); // warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata); // warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs); @@ -731,6 +762,7 @@ impl SetDisks { { Ok(v) => v, Err(e) => { + rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, metadata_quorum_outcome(&e)); // error!("Self::object_quorum_from_meta: {:?}, bucket: {}, object: {}", &e, bucket, object); record_get_stage_duration_if_enabled( GET_OBJECT_PATH_SET_DISK, @@ -745,6 +777,7 @@ impl SetDisks { let write_quorum = usize::try_from(write_quorum) .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?; if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { + rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "not_reached"); error!("reduce_read_quorum_errs: {:?}, bucket: {}, object: {}", &err, bucket, object); record_get_stage_duration_if_enabled( GET_OBJECT_PATH_SET_DISK, @@ -755,7 +788,17 @@ impl SetDisks { } let (op_online_disks, mut fi, fileinfo_selection_quorum) = - Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum)?; + match Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum) { + Ok(selection) => selection, + Err(err) => { + let err = StorageError::from(err); + rustfs_io_metrics::record_get_object_metadata_quorum_result( + metadata_metrics_path, + metadata_quorum_outcome(&err), + ); + return Err(err); + } + }; let include_part_checksums = opts.include_part_checksums || opts.part_number.is_some() || opts.data_movement || opts.raw_data_movement_read; if include_part_checksums { @@ -778,7 +821,16 @@ impl SetDisks { "metadata_read_error", ) .await; - } else if use_metadata_cache && (metadata_fanout_complete || condition_stopped) { + } else if use_metadata_cache + && (metadata_fanout_complete + || condition_stopped + || (read_data + && allow_early_stop + && metadata_fanout_diagnostics.valid_responses() >= read_quorum + && errs.iter().all(Option::is_none) + && fileinfo_selection_quorum >= read_quorum + && op_online_disks.iter().filter(|disk| disk.is_some()).count() >= read_quorum)) + { #[cfg(test)] metadata_cache_tests::wait_before_metadata_cache_publish(bucket, object).await; self.cache_get_object_fileinfo( @@ -788,8 +840,10 @@ impl SetDisks { &fi, if metadata_fanout_complete { GetObjectMetadataCacheCapability::Full - } else { + } else if condition_stopped { GetObjectMetadataCacheCapability::ConditionOnly + } else { + GetObjectMetadataCacheCapability::QuorumSelected }, ), &parts_metadata, @@ -2650,6 +2704,7 @@ fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOpti #[cfg(test)] mod metadata_cache_tests { use super::*; + use crate::set_disk::core::io_primitives::disk_call_counters; use rustfs_heal_contracts::heal_channel::HealAdmissionDropReason; use serial_test::serial; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -4009,7 +4064,7 @@ mod metadata_cache_tests { } #[tokio::test] - #[serial(metadata_cache_publish_barrier)] + #[serial(metadata_cache_early_stop)] async fn metadata_cache_production_fanout_cannot_publish_after_invalidation() { // Isolated context: an ambient DistErasure window (another test's // SetupTypeGuard) would bypass metadata-cache publication entirely @@ -4063,6 +4118,191 @@ mod metadata_cache_tests { ); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial(metadata_cache_early_stop)] + async fn metadata_cache_publishes_quorum_selected_early_stop_result() { + use tokio::io::AsyncReadExt; + + let isolated_ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + isolated_ctx + .update_erasure_type(crate::layout::endpoints::SetupType::Erasure) + .await; + let (dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, isolated_ctx).await; + let bucket = "metadata-cache-early-stop"; + let object = "objects/repeated-key"; + let disks = set.disks.read().await.clone(); + for disk in disks.iter().flatten() { + disk.make_volume(bucket) + .await + .expect("test bucket should be created on each disk"); + } + let expected_body = vec![0x5a; 1024 * 1024]; + set.put_object( + bucket, + object, + &mut PutObjReader::from_vec(expected_body.clone()), + &ObjectOptions::default(), + ) + .await + .expect("test object should be written before enabling the slow-tail fault"); + + let calls = disk_call_counters::observe(object); + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("false")), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS", Some("500")), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS", Some("3")), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET", Some(bucket)), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX", Some("objects/")), + ], + async { + let first = tokio::time::timeout( + Duration::from_millis(400), + set.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true), + ) + .await + .expect("first metadata read should stop before the delayed peer") + .expect("read quorum should provide valid object metadata"); + assert!(first.owned.is_some(), "the first lookup should resolve through disk metadata"); + assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 4); + assert_eq!(calls.total(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT), 1); + + let second = set + .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true) + .await + .expect("second lookup should use the quorum-published metadata cache entry"); + assert!(second.shared_entry().is_some(), "the repeated GET should hit the published cache entry"); + assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 4); + + let cached = second.shared_entry().expect("second lookup should share cached metadata"); + assert_eq!(cached.fi.name, object); + let cached_online_count = cached.online_disks.iter().filter(|disk| disk.is_some()).count(); + assert_eq!(cached_online_count, 3, "the warm cache should contain three of four disks"); + assert_eq!(cached.capability, GetObjectMetadataCacheCapability::QuorumSelected); + + let data_dir = cached + .fi + .data_dir + .expect("quorum cache retains the object data directory") + .to_string(); + let selected_disk_indices = cached + .online_disks + .iter() + .enumerate() + .filter_map(|(index, disk)| disk.as_ref().map(|_| index)) + .take(2) + .collect::>(); + assert_eq!(selected_disk_indices.len(), 2, "the warmed cache should name enough selected shards"); + for index in selected_disk_indices { + tokio::fs::remove_file(dirs[index].path().join(bucket).join(object).join(&data_dir).join("part.1")) + .await + .expect("simulate loss of two cached selected shards"); + } + + let mut reader = set + .get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("a cached partial layout must recover through omitted disks"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("the GET should decode after refreshing the omitted disks"); + assert_eq!(body, expected_body, "recovered body must be byte-exact"); + }, + ) + .await; + + let full_layout = set + .get_object_fileinfo_gated(bucket, object, &ObjectOptions::default(), false, false) + .await + .expect("read-before-write consumers must fetch a complete layout"); + assert!( + full_layout.shared_entry().is_none(), + "a quorum-selected partial layout cannot satisfy mutation consumers" + ); + assert!( + calls.total(disk_call_counters::KIND_READ_VERSION) > 4, + "mutation consumers must independently fan out after a quorum-only cache hit" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial(metadata_cache_early_stop)] + async fn metadata_cache_early_stop_publication_is_fenced_by_invalidation() { + let isolated_ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + isolated_ctx + .update_erasure_type(crate::layout::endpoints::SetupType::Erasure) + .await; + let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, isolated_ctx).await; + let bucket = "metadata-cache-early-stop-fence"; + let object = "objects/repeated-key"; + let disks = set.disks.read().await.clone(); + for disk in disks.iter().flatten() { + disk.make_volume(bucket) + .await + .expect("test bucket should be created on each disk"); + } + set.put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![0x5a; 1024 * 1024]), + &ObjectOptions::default(), + ) + .await + .expect("test object should be written before enabling the slow-tail fault"); + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("false")), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS", Some("500")), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS", Some("3")), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET", Some(bucket)), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX", Some("objects/")), + ], + async { + let barrier = MetadataCachePublishBarrier::install(bucket, object); + let stale_generation = set + .get_object_metadata_cache_generation(bucket, object) + .expect("metadata cache generation should be active"); + let reader_set = Arc::clone(&set); + let read = tokio::spawn(async move { + reader_set + .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true) + .await + }); + barrier.wait_until_paused().await; + set.invalidate_get_object_metadata_cache(bucket, object).await; + barrier.release(); + + let snapshot = read + .await + .expect("early-stop metadata read should not panic") + .expect("read quorum should resolve the selected FileInfo"); + assert!(snapshot.owned.is_some()); + assert!( + set.get_object_metadata_cache + .get(&GetObjectMetadataCacheKey::new(bucket, object, stale_generation)) + .await + .is_none(), + "early-stop publication must not retain an entry under the retired generation" + ); + assert!( + set.cached_get_object_fileinfo(bucket, object).await.is_none(), + "invalidation must reject the quorum-selected early-stop publication" + ); + }, + ) + .await; + } + #[tokio::test] #[serial(metadata_cache_publish_barrier, body_cache_hook)] async fn conditional_inline_hedge_caches_only_terminal_metadata() { @@ -5082,12 +5322,15 @@ mod metadata_cache_tests { .get_object_metadata_cache_generation("bucket-a", "object-a") .expect("metadata cache generation should be active"); let first_key = GetObjectMetadataCacheKey::new("bucket-a", "object-a", generation); + let first_lookup = GetObjectMetadataCacheLookupKey::new("bucket-a", "object-a", generation); let second_key = GetObjectMetadataCacheKey { bucket: Arc::from("bucket-b"), object: Arc::from("object-b"), generation: generation.value, hash: generation.hash, }; + let second_lookup = GetObjectMetadataCacheLookupKey::new("bucket-b", "object-b", generation); + let unrelated_lookup = GetObjectMetadataCacheLookupKey::new("bucket-c", "object-c", generation); let first_fi = valid_test_fileinfo("object-a"); let second_fi = valid_test_fileinfo("object-b"); let entry = |fi: FileInfo| { @@ -5108,7 +5351,7 @@ mod metadata_cache_tests { assert_eq!( set.get_object_metadata_cache - .get(&first_key) + .get(&first_lookup) .await .expect("first colliding entry should remain addressable") .fi @@ -5117,13 +5360,14 @@ mod metadata_cache_tests { ); assert_eq!( set.get_object_metadata_cache - .get(&second_key) + .get(&second_lookup) .await .expect("second colliding entry should remain addressable") .fi .name, "object-b" ); + assert!(set.get_object_metadata_cache.get(&unrelated_lookup).await.is_none()); } #[tokio::test] @@ -5290,6 +5534,16 @@ mod tests { const CODEC_STREAMING_TEST_OBJECT: &str = "object"; static CAPTURED_READ_REPAIR_CALLS: AtomicUsize = AtomicUsize::new(0); + #[test] + fn metadata_quorum_outcome_uses_bounded_error_categories() { + assert_eq!(metadata_quorum_outcome(&StorageError::ErasureReadQuorum), "not_reached"); + assert_eq!(metadata_quorum_outcome(&StorageError::FileNotFound), "not_found"); + assert_eq!( + metadata_quorum_outcome(&StorageError::Io(std::io::Error::other("disk unavailable"))), + "unavailable" + ); + } + fn capture_read_repair_submitter( _request: rustfs_heal_contracts::heal_channel::HealChannelRequest, ) -> ReadRepairAdmissionFuture { diff --git a/crates/ecstore/src/store/peer.rs b/crates/ecstore/src/store/peer.rs index 655492c66..cc84c9cf7 100644 --- a/crates/ecstore/src/store/peer.rs +++ b/crates/ecstore/src/store/peer.rs @@ -18,6 +18,11 @@ use crate::disk::error::{DiskError, Result as DiskResult}; use crate::disk::{DeleteOptions, Disk, RenameDataGuards, RenameDataResp}; use crate::runtime::instance::{InstanceContext, NamespaceCommitGuard}; use crate::runtime::sources as runtime_sources; +use rustfs_io_metrics::{ + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME, + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD, + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL, +}; use tracing::{debug, error}; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; @@ -331,15 +336,38 @@ async fn rename_local_data_with_ctx( destination: (&str, &str), mut guards: RenameDataGuards, ) -> DiskResult { + let handler_started = rustfs_io_metrics::put_stage_timer(); + let lookup_started = rustfs_io_metrics::put_stage_timer(); let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?; + let lookup_ms = lookup_started.map(|started| started.elapsed().as_secs_f64() * 1000.0); + if let Some(duration_ms) = lookup_ms { + rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, duration_ms); + } let mutates_namespace = !is_meta_bucketname(source.0) || !is_meta_bucketname(destination.0); + let admit_started = rustfs_io_metrics::put_stage_timer(); let owner = admit_local_disk(ctx, &disk, disk_id, mutates_namespace).await?; + let admit_ms = admit_started.map(|started| started.elapsed().as_secs_f64() * 1000.0); + if let Some(duration_ms) = admit_ms { + rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, duration_ms); + } guards.namespace_owner = owner.as_ref().map(|owner| owner.clone() as Arc); + let local_rename_started = rustfs_io_metrics::put_stage_timer(); let result = disk .rename_data_borrowed_with_fence_observed(source.0, source.1, fi, destination.0, destination.1, guards) .await .result; + let local_rename_ms = local_rename_started.map(|started| started.elapsed().as_secs_f64() * 1000.0); + if let Some(duration_ms) = local_rename_ms { + rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME, duration_ms); + } drop(owner); + if let Some(handler_started) = handler_started { + let handler_ms = handler_started.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL, handler_ms); + let child_ms = lookup_ms.unwrap_or(0.0) + admit_ms.unwrap_or(0.0) + local_rename_ms.unwrap_or(0.0); + let overhead_ms = (handler_ms - child_ms).max(0.0); + rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD, overhead_ms); + } result } diff --git a/crates/ecstore/tests/cleanup_isolation_test.rs b/crates/ecstore/tests/cleanup_isolation_test.rs new file mode 100644 index 000000000..663681445 --- /dev/null +++ b/crates/ecstore/tests/cleanup_isolation_test.rs @@ -0,0 +1,174 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(target_os = "linux")] + +use bytes::Bytes; +mod storage_api; + +use std::time::Duration; +use storage_api::cleanup_isolation::{DeleteOptions, DiskAPI, DiskError, DiskOption, Endpoint, new_disk}; + +#[tokio::test] +async fn cleanup_disk_and_periodic_gc_use_affinity_without_changing_foreground() { + const CHILD: &str = "RUSTFS_CLEANUP_LINUX_TEST_CHILD"; + let foreground = rustix::thread::sched_getaffinity(None).unwrap(); + let cpu = (0..rustix::thread::CpuSet::MAX_CPU) + .find(|cpu| foreground.is_set(*cpu)) + .expect("the test process must have an allowed CPU"); + if std::env::var_os(CHILD).is_none() { + for mode in ["custom", "default", "none", "restricted"] { + if mode == "default" && (!foreground.is_set(1) || !foreground.is_set(2)) { + continue; + } + let output = tokio::task::spawn_blocking(move || { + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + "cleanup_disk_and_periodic_gc_use_affinity_without_changing_foreground", + "--nocapture", + ]) + .env(CHILD, mode) + .env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true") + .env("RUSTFS_PUT_RENAME_TAIL_CLEANUP_MAX_PENDING", "4") + .env("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS", "1") + .env("RUSTFS_CLEANUP_DISK_MAX_PENDING", "4") + .env("RUSTFS_CLEANUP_DISK_WORKERS", "1") + .env("RUSTFS_CLEANUP_GC_WORKERS", "1") + .env("RUSTFS_CLEANUP_BLOCKING_THREADS", "1") + .env("RUSTFS_CLEANUP_ASYNC_THREADS", "1") + .env_remove("RUSTFS_CLEANUP_CPUS") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP"); + match mode { + "custom" => { + command.env("RUSTFS_CLEANUP_CPUS", cpu.to_string()); + } + "none" => { + command.env("RUSTFS_CLEANUP_CPUS", "none"); + } + _ => {} + } + command.output().unwrap() + }) + .await + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + } + return; + } + + let mode = std::env::var(CHILD).unwrap(); + if mode == "restricted" { + let mut restricted = rustix::thread::CpuSet::new(); + restricted.set(cpu); + rustix::thread::sched_setaffinity(None, &restricted).expect("restrict initialization to one allowed CPU"); + } + let dir = tempfile::tempdir().unwrap(); + let disk = new_disk(&Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(), &DiskOption::default()) + .await + .unwrap(); + let bucket = "cleanup-affinity"; + disk.make_volume(bucket).await.unwrap(); + disk.write_all(bucket, "old/part.1", Bytes::from_static(b"garbage")) + .await + .unwrap(); + let receipt = disk + .write_all(bucket, "old/.rustfs-old-data-cleanup-receipt.json", Bytes::from_static(b"receipt")) + .await; + if mode == "restricted" { + assert!( + receipt + .expect_err("unavailable default CPUs must reject cleanup") + .to_string() + .contains("allowed CPU set") + ); + assert_eq!(disk.read_all(bucket, "old/part.1").await.unwrap(), Bytes::from_static(b"garbage")); + return; + } + receipt.unwrap(); + let expected_mask = match mode.as_str() { + "default" => "1-2".to_owned(), + "none" => std::fs::read_to_string("/proc/thread-self/status") + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("Cpus_allowed_list:")) + .unwrap() + .trim() + .to_owned(), + _ => cpu.to_string(), + }; + disk.write_all(bucket, "live/part.1", Bytes::from_static(b"keep")) + .await + .unwrap(); + disk.delete( + bucket, + "old", + DeleteOptions { + recursive: true, + ..Default::default() + }, + ) + .await + .unwrap(); + assert!(matches!(disk.read_all(bucket, "old/part.1").await, Err(DiskError::FileNotFound))); + + // One async worker and one lazily started blocking worker. Inspect the + // real process threads, including the threads executing filesystem work. + let mut cleanup_threads = 0; + for entry in std::fs::read_dir("/proc/self/task").unwrap() { + let task = entry.unwrap().path(); + let Ok(name) = std::fs::read_to_string(task.join("comm")) else { + continue; + }; + if name.trim() != "rustfs-cleanup" { + continue; + } + cleanup_threads += 1; + let status = std::fs::read_to_string(task.join("status")).unwrap(); + let mask = status + .lines() + .find_map(|line| line.strip_prefix("Cpus_allowed_list:")) + .unwrap() + .trim(); + assert_eq!(mask, expected_mask); + } + assert_eq!(cleanup_threads, 2, "async and blocking workers must both use the configured runtime"); + assert_eq!(rustix::thread::sched_getaffinity(None).unwrap(), foreground); + + let trash = dir.path().join(".rustfs.sys/tmp/.trash"); + assert!(std::fs::read_dir(&trash).unwrap().next().is_some()); + // Advance the foreground timer that dispatches the five-minute periodic + // scan; the dedicated runtime then performs the actual physical removal. + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(301)).await; + tokio::time::resume(); + tokio::time::timeout(Duration::from_secs(20), async { + while std::fs::read_dir(&trash).unwrap().next().is_some() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("periodic GC must physically remove trash"); + assert_eq!(disk.read_all(bucket, "live/part.1").await.unwrap(), Bytes::from_static(b"keep")); + assert_eq!(rustix::thread::sched_getaffinity(None).unwrap(), foreground); +} diff --git a/crates/ecstore/tests/fd_relative_io_test.rs b/crates/ecstore/tests/fd_relative_io_test.rs new file mode 100644 index 000000000..fa19deb52 --- /dev/null +++ b/crates/ecstore/tests/fd_relative_io_test.rs @@ -0,0 +1,283 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(all(target_os = "linux", feature = "test-util"))] + +mod storage_api; + +use bytes::Bytes; +use rustfs_filemeta::{FileInfo, ObjectPartInfo}; +use std::{path::Path, time::Duration}; +use storage_api::fd_relative::{ + DiskAPI, DiskError, DiskOption, DiskStore, Endpoint, LocalPublicationPause, LocalPublicationStage, ReadOptions, new_disk, +}; +use time::OffsetDateTime; +use uuid::Uuid; + +const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp"; + +async fn disk_at(path: &Path) -> DiskStore { + new_disk( + &Endpoint::try_from(path.to_str().expect("UTF-8 test path")).expect("endpoint"), + &DiskOption::default(), + ) + .await + .expect("local disk") +} + +fn metadata(data: Bytes, inline: bool, directory: Uuid) -> FileInfo { + let mut fi = FileInfo::new("object", 1, 0); + fi.erasure.index = 1; + fi.version_id = Some(Uuid::nil()); + fi.data_dir = Some(directory); + fi.size = i64::try_from(data.len()).expect("test data size"); + fi.parts = vec![ObjectPartInfo { + number: 1, + size: data.len(), + actual_size: fi.size, + ..Default::default() + }]; + fi.mod_time = Some(OffsetDateTime::now_utc()); + fi.data = inline.then_some(data); + if inline { + fi.set_inline_data(); + } + fi +} + +async fn put(disk: &DiskStore, staging: &str, data: Bytes, inline: bool) -> Uuid { + let directory = Uuid::new_v4(); + if !inline { + disk.write_all(RUSTFS_META_TMP_BUCKET, &format!("{staging}/{directory}/part.1"), data.clone()) + .await + .expect("stage shard"); + } + disk.rename_data(RUSTFS_META_TMP_BUCKET, staging, metadata(data, inline, directory), "bucket", "object") + .await + .expect("commit object"); + directory +} + +#[tokio::test] +async fn pinned_root_reads_and_writes_original_disk_after_path_replacement() { + let temp = tempfile::tempdir().expect("test directory"); + let root = temp.path().join("disk"); + std::fs::create_dir(&root).expect("disk root"); + let disk = disk_at(&root).await; + disk.make_volume("bucket").await.expect("bucket"); + disk.write_all("bucket", "metadata", Bytes::from_static(b"original")) + .await + .expect("initial write"); + let moved = temp.path().join("original"); + std::fs::rename(&root, &moved).expect("move mount path"); + std::fs::create_dir_all(root.join("bucket")).expect("replacement disk"); + std::fs::write(root.join("bucket/metadata"), b"replacement").expect("replacement sentinel"); + assert_eq!( + disk.read_all("bucket", "metadata").await.expect("pinned read"), + Bytes::from_static(b"original") + ); + disk.write_all("bucket", "metadata", Bytes::from_static(b"updated")) + .await + .expect("pinned write"); + assert_eq!(std::fs::read(moved.join("bucket/metadata")).expect("original disk"), b"updated"); + assert_eq!(std::fs::read(root.join("bucket/metadata")).expect("replacement disk"), b"replacement"); +} + +#[tokio::test] +async fn relative_reads_and_writes_reject_symlink_escape() { + let temp = tempfile::tempdir().expect("test directory"); + let disk = disk_at(temp.path()).await; + disk.make_volume("bucket").await.expect("bucket"); + let outside = tempfile::tempdir().expect("outside directory"); + std::fs::write(outside.path().join("sentinel"), b"preserved").expect("sentinel"); + std::os::unix::fs::symlink(outside.path(), temp.path().join("bucket/link")).expect("directory link"); + std::os::unix::fs::symlink(outside.path().join("sentinel"), temp.path().join("bucket/file-link")).expect("file link"); + for path in ["link/sentinel", "file-link"] { + assert!(matches!(disk.read_all("bucket", path).await, Err(DiskError::InvalidPath))); + assert!(matches!( + disk.write_all("bucket", path, Bytes::from_static(b"unsafe")).await, + Err(DiskError::InvalidPath) + )); + } + assert_eq!(std::fs::read(outside.path().join("sentinel")).expect("outside bytes"), b"preserved"); +} + +#[tokio::test] +async fn inline_and_non_inline_overwrites_remain_readable_after_reopen() { + for inline in [true, false] { + let temp = tempfile::tempdir().expect("test directory"); + let disk = disk_at(temp.path()).await; + disk.make_volume("bucket").await.expect("bucket"); + put(&disk, "first", Bytes::from_static(b"old bytes"), inline).await; + let directory = put(&disk, "second", Bytes::from_static(b"new bytes"), inline).await; + drop(disk); + let disk = disk_at(temp.path()).await; + let fi = disk + .read_version( + "bucket", + "bucket", + "object", + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("committed metadata after reopen"); + assert_eq!(fi.data_dir, Some(directory)); + if inline { + assert_eq!(fi.data, Some(Bytes::from_static(b"new bytes"))); + } else { + assert_eq!( + disk.read_all("bucket", &format!("object/{directory}/part.1")) + .await + .expect("shard bytes"), + Bytes::from_static(b"new bytes") + ); + } + } +} + +#[tokio::test] +async fn rename_rejects_symlinked_destination_metadata() { + for inline in [true, false] { + let temp = tempfile::tempdir().expect("disk directory"); + let outside = tempfile::tempdir().expect("outside directory"); + let disk = disk_at(temp.path()).await; + disk.make_volume("bucket").await.expect("bucket"); + put(&disk, "seed", Bytes::from_static(b"old object"), inline).await; + let destination = temp.path().join("bucket/object/xl.meta"); + let original = std::fs::read(&destination).expect("valid existing metadata"); + let external_metadata = outside.path().join("xl.meta"); + std::fs::write(&external_metadata, &original).expect("external valid metadata"); + std::fs::remove_file(&destination).expect("replace destination metadata"); + std::os::unix::fs::symlink(&external_metadata, &destination).expect("metadata escape link"); + + let directory = Uuid::new_v4(); + let data = Bytes::from_static(b"replacement"); + if !inline { + disk.write_all(RUSTFS_META_TMP_BUCKET, &format!("replacement/{directory}/part.1"), data.clone()) + .await + .expect("stage non-inline data"); + } + let result = disk + .rename_data( + RUSTFS_META_TMP_BUCKET, + "replacement", + metadata(data, inline, directory), + "bucket", + "object", + ) + .await; + assert!( + matches!(result, Err(DiskError::InvalidPath)), + "symlinked metadata must be rejected: {result:?}" + ); + assert_eq!(std::fs::read_link(&destination).expect("link must remain"), external_metadata); + assert_eq!(std::fs::read(&external_metadata).expect("external metadata must survive"), original); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn prepared_publication_rejects_source_or_directory_replacement() { + for replace_directory in [false, true] { + let temp = tempfile::tempdir().expect("test directory"); + let disk = disk_at(temp.path()).await; + disk.make_volume("bucket").await.expect("bucket"); + put(&disk, "first", Bytes::from_static(b"old bytes"), true).await; + let old = std::fs::read(temp.path().join("bucket/object/xl.meta")).expect("old metadata"); + let mut pause = LocalPublicationPause::install(&disk, "bucket", "object/xl.meta", LocalPublicationStage::PreparedRename) + .expect("publication pause"); + let writer = disk.clone(); + let task = tokio::spawn(async move { + writer + .rename_data( + RUSTFS_META_TMP_BUCKET, + "second", + metadata(Bytes::from_static(b"new bytes"), true, Uuid::new_v4()), + "bucket", + "object", + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), pause.entered()) + .await + .expect("publication reached pause") + .expect("pause entered"); + if replace_directory { + std::fs::rename(temp.path().join("bucket/object"), temp.path().join("detached")).expect("detach object"); + std::fs::create_dir(temp.path().join("bucket/object")).expect("replacement directory"); + std::fs::write(temp.path().join("bucket/object/xl.meta"), &old).expect("replacement metadata"); + } else { + let src = temp.path().join(RUSTFS_META_TMP_BUCKET).join("second/xl.meta"); + std::fs::rename(&src, src.with_extension("detached")).expect("detach staged metadata"); + std::fs::write(src, b"unrelated").expect("replacement staged file"); + } + drop(pause); + assert!(task.await.expect("writer task").is_err()); + assert_eq!( + std::fs::read(temp.path().join("bucket/object/xl.meta")).expect("old metadata survives"), + old + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancelled_waiter_keeps_publication_serialized_until_physical_completion() { + let temp = tempfile::tempdir().expect("test directory"); + let disk = disk_at(temp.path()).await; + disk.make_volume("bucket").await.expect("bucket"); + put(&disk, "first", Bytes::from_static(b"old bytes"), true).await; + let mut pause = LocalPublicationPause::install(&disk, "bucket", "object/xl.meta", LocalPublicationStage::PreparedRename) + .expect("publication pause"); + let writer = disk.clone(); + let pending = tokio::spawn(async move { put(&writer, "shared-stage", Bytes::from_static(b"cancelled waiter"), true).await }); + tokio::time::timeout(Duration::from_secs(30), pause.entered()) + .await + .expect("publication reached pause") + .expect("pause entered"); + pending.abort(); + assert!(pending.await.expect_err("waiter aborted").is_cancelled()); + let writer = disk.clone(); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let mut retry = tokio::spawn(async move { + let _ = started_tx.send(()); + put(&writer, "shared-stage", Bytes::from_static(b"latest bytes"), true).await + }); + started_rx.await.expect("retry started"); + assert!( + tokio::time::timeout(Duration::from_millis(200), &mut retry).await.is_err(), + "a retry must not publish while the cancelled writer still owns physical IO" + ); + drop(pause); + tokio::time::timeout(Duration::from_secs(30), retry) + .await + .expect("retry completed") + .expect("retry task"); + let fi = disk + .read_version( + "bucket", + "bucket", + "object", + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("latest metadata"); + assert_eq!(fi.data, Some(Bytes::from_static(b"latest bytes"))); +} diff --git a/crates/ecstore/tests/metadata_read_lock_reuse_test.rs b/crates/ecstore/tests/metadata_read_lock_reuse_test.rs index 6433bf0cd..c37949477 100644 --- a/crates/ecstore/tests/metadata_read_lock_reuse_test.rs +++ b/crates/ecstore/tests/metadata_read_lock_reuse_test.rs @@ -18,6 +18,7 @@ mod storage_api; use std::sync::Arc; use std::time::Duration; +use storage_api::contract_compat::ECStore; use storage_api::metadata_lock::{ BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys, @@ -170,6 +171,100 @@ async fn reused_metadata_lock_still_enforces_quota() { .expect("acquire writer"); } +#[tokio::test] +async fn authoritative_quota_read_does_not_ignore_later_quota() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "authoritative-quota-quota-set"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + + let mut first = PutObjReader::from_vec(b"cache no quota".to_vec()); + store + .put_object(bucket, "warm", &mut first, &ObjectOptions::default()) + .await + .expect("warm no-quota metadata cache"); + store + .update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec()) + .await + .expect("set quota after no-quota cache was observed"); + + let mut opts = quota_snapshot_options(&store, bucket).await; + assert!(opts.set_quota_admission(0, 1)); + let mut data = PutObjReader::from_vec(b"too large".to_vec()); + let error = store + .put_object(bucket, "object", &mut data, &opts) + .await + .expect_err("quota set after a no-quota cache hit must still reject growth"); + assert!(matches!(error, Error::QuotaExceeded { limit: 1, .. }), "{error:?}"); +} + +#[tokio::test] +async fn authoritative_quota_read_allows_write_after_quota_is_cleared() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "authoritative-quota-quota-cleared"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + store + .update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec()) + .await + .expect("set quota"); + store + .update_bucket_metadata_config(bucket, "quota.json", Vec::new()) + .await + .expect("clear quota"); + + let mut data = PutObjReader::from_vec(b"allowed after clear".to_vec()); + store + .put_object(bucket, "object", &mut data, &ObjectOptions::default()) + .await + .expect("authoritatively cleared quota should allow writes"); + + let mut reader = store + .get_object_reader(bucket, "object", None, Default::default(), &ObjectOptions::default()) + .await + .expect("read object written after quota clear"); + let mut bytes = Vec::new(); + reader.stream.read_to_end(&mut bytes).await.expect("read body"); + assert_eq!(bytes, b"allowed after clear"); +} + +#[tokio::test] +async fn authoritative_quota_read_fails_closed_on_invalid_quota_json() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "authoritative-quota-invalid-quota"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + store + .update_bucket_metadata_config(bucket, "quota.json", b"{not-json".to_vec()) + .await + .expect("persist invalid quota bytes for fail-closed read"); + + let mut data = PutObjReader::from_vec(b"must fail".to_vec()); + store + .put_object(bucket, "object", &mut data, &ObjectOptions::default()) + .await + .expect_err("invalid quota JSON must not degrade to no-quota"); +} + +async fn quota_snapshot_options(store: &Arc, bucket: &str) -> ObjectOptions { + ObjectOptions { + versioned: true, + version_id: Some(Uuid::new_v4().to_string()), + expected_bucket_incarnation_id: Some(store.bucket_incarnation_id(bucket).await.expect("load incarnation")), + object_lock_config_snapshot: Some(store.object_lock_config_snapshot(bucket).await.expect("capture snapshot")), + ..Default::default() + } +} + #[tokio::test] async fn cancelled_put_releases_shared_metadata_guard() { let (_dirs, store) = isolated_store_over_temp_disks().await; diff --git a/crates/ecstore/tests/storage_api.rs b/crates/ecstore/tests/storage_api.rs index acfd5c85e..d65582402 100644 --- a/crates/ecstore/tests/storage_api.rs +++ b/crates/ecstore/tests/storage_api.rs @@ -61,6 +61,17 @@ pub(crate) mod minio_generated_read { }; } +pub(crate) mod cleanup_isolation { + pub(crate) use super::{DiskAPI, DiskOption, Endpoint, new_disk}; + pub(crate) use rustfs_ecstore::api::disk::{DeleteOptions, DiskError}; +} + +#[cfg(all(feature = "test-util", target_os = "linux"))] +pub(crate) mod fd_relative { + pub(crate) use super::{DiskAPI, DiskOption, DiskStore, Endpoint, new_disk}; + pub(crate) use rustfs_ecstore::api::disk::{DiskError, LocalPublicationPause, LocalPublicationStage, ReadOptions}; +} + pub(crate) mod pinned_erasure_fixtures { pub(crate) use super::storage_contracts::{BucketOperations, HTTPRangeSpec, MakeBucketOptions, ObjectIO}; pub(crate) use super::{ECStore, Endpoint, Erasure, Error, ObjectOptions}; diff --git a/crates/io-metrics/benches/metrics_pipeline.rs b/crates/io-metrics/benches/metrics_pipeline.rs index fb1aa6857..3f213e586 100644 --- a/crates/io-metrics/benches/metrics_pipeline.rs +++ b/crates/io-metrics/benches/metrics_pipeline.rs @@ -1,13 +1,15 @@ use criterion::{Criterion, criterion_group, criterion_main}; -use rustfs_io_metrics::{MetricsCollector, PerformanceMetrics, record_get_object_request_started}; +use rustfs_io_metrics::{MetricsCollector, PerformanceMetrics, record_get_object_timeout_operation_start}; use rustfs_io_metrics::{record_s3_op, s3_http_metrics::S3HttpRequestGuard}; use rustfs_s3_ops::S3Operation; use std::hint::black_box; use std::sync::Arc; use std::time::Duration; -fn bench_record_get_object_request_started(c: &mut Criterion) { - c.bench_function("record_get_object_request_started", |b| b.iter(record_get_object_request_started)); +fn bench_record_get_object_timeout_operation_start(c: &mut Criterion) { + c.bench_function("record_get_object_timeout_operation_start", |b| { + b.iter(record_get_object_timeout_operation_start) + }); } fn bench_s3_http_outcomes(c: &mut Criterion) { @@ -49,7 +51,7 @@ fn bench_metrics_collector_record_io_operation(c: &mut Criterion) { criterion_group!( benches, - bench_record_get_object_request_started, + bench_record_get_object_timeout_operation_start, bench_s3_http_outcomes, bench_update_concurrent_requests, bench_metrics_collector_record_io_operation diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index f499c0f02..45e089bab 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -49,8 +49,9 @@ #[macro_use] extern crate metrics; +use std::collections::HashMap; use std::sync::{ - Mutex, + LazyLock, Mutex, atomic::{AtomicBool, AtomicU64, Ordering}, }; @@ -61,6 +62,25 @@ use std::sync::{ /// Enabled only through an explicit runtime opt-in. static PUT_STAGE_METRICS_ENABLED: AtomicBool = AtomicBool::new(false); static GET_STAGE_METRICS_ENABLED: AtomicBool = AtomicBool::new(false); +static GET_STAGE_LOCAL_SUMMARY_ENABLED: AtomicBool = AtomicBool::new(false); +static GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE: AtomicU64 = AtomicU64::new(64); +static GET_STAGE_LOCAL_SUMMARY_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +type GetStageLocalSummaryKey = (&'static str, &'static str, &'static str, &'static str); + +/// Sampled local aggregate for one bounded GetObject stage label set. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct GetStageLocalSummary { + pub path: &'static str, + pub stage: &'static str, + pub object_class: &'static str, + pub size_bucket: &'static str, + pub sampled_count: u64, + pub duration_nanoseconds: u64, +} + +static GET_STAGE_LOCAL_SUMMARY: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); /// Global switch for all remaining (non GET/PUT-stage) metric emission in this /// crate's free `record_*` functions — I/O scheduler, bytes-pool, zero-copy, @@ -87,6 +107,70 @@ pub fn set_get_stage_metrics_enabled(enabled: bool) { GET_STAGE_METRICS_ENABLED.store(enabled, Ordering::Relaxed); } +pub fn set_get_stage_local_summary_enabled(enabled: bool) { + GET_STAGE_LOCAL_SUMMARY_ENABLED.store(enabled, Ordering::Relaxed); +} + +pub fn set_get_stage_local_summary_sample_rate(sample_rate: u64) { + GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE.store(sample_rate.max(1), Ordering::Relaxed); +} + +pub fn get_stage_local_summary_enabled() -> bool { + GET_STAGE_LOCAL_SUMMARY_ENABLED.load(Ordering::Relaxed) +} + +pub fn get_stage_local_summary_sample_rate() -> u64 { + GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE.load(Ordering::Relaxed) +} + +pub fn take_get_stage_local_summary() -> Vec { + let Ok(mut summary) = GET_STAGE_LOCAL_SUMMARY.lock() else { + return Vec::new(); + }; + let mut rows = summary.drain().map(|(_, row)| row).collect::>(); + rows.sort_unstable_by_key(|row| (row.path, row.stage, row.object_class, row.size_bucket)); + rows +} + +fn record_get_stage_local_summary( + path: &'static str, + stage: &'static str, + object_class: &'static str, + size_bucket: &'static str, + duration_secs: f64, +) { + let sample_rate = get_stage_local_summary_sample_rate(); + if sample_rate > 1 + && !GET_STAGE_LOCAL_SUMMARY_SEQUENCE + .fetch_add(1, Ordering::Relaxed) + .is_multiple_of(sample_rate) + { + return; + } + + let Ok(duration) = std::time::Duration::try_from_secs_f64(duration_secs) else { + return; + }; + let Ok(duration_nanoseconds) = u64::try_from(duration.as_nanos()) else { + return; + }; + + let key = (path, stage, object_class, size_bucket); + let Ok(mut summary) = GET_STAGE_LOCAL_SUMMARY.lock() else { + return; + }; + let row = summary.entry(key).or_insert_with(|| GetStageLocalSummary { + path, + stage, + object_class, + size_bucket, + sampled_count: 0, + duration_nanoseconds: 0, + }); + row.sampled_count = row.sampled_count.saturating_add(1); + row.duration_nanoseconds = row.duration_nanoseconds.saturating_add(duration_nanoseconds); +} + /// Enable or disable general (non GET/PUT-stage) metric emission. /// /// Called once during startup, typically gated by `rustfs_obs::observability_metric_enabled()`. @@ -110,8 +194,210 @@ pub fn put_stage_timer() -> Option { } pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT: &str = "put_object_commit_namespace_lock_wait"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD: &str = "put_object_commit_namespace_lock_held"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME: &str = "put_object_commit_namespace_lock_held_rename"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_PRE_RENAME_PREPARE: &str = + "put_object_commit_namespace_lock_held_pre_rename_prepare"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME_RESULT_TO_TAIL_HANDOFF: &str = + "put_object_commit_namespace_lock_held_rename_result_to_tail_handoff"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_TAIL_HANDOFF: &str = + "put_object_commit_namespace_lock_held_tail_handoff"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_RENAME: &str = "put_object_commit_namespace_lock_held_post_rename"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_GUARD_RELEASE: &str = + "put_object_commit_namespace_lock_held_guard_release"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE: &str = + "put_object_commit_namespace_lock_held_quota_fence_release"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_COMMIT: &str = + "put_object_commit_namespace_lock_held_quota_commit"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT: &str = "put_object_commit_namespace_lock_held_heal_submit"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT: &str = + "put_object_commit_namespace_lock_held_cleanup_receipt"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_METADATA_INVALIDATE: &str = + "put_object_commit_namespace_lock_held_metadata_invalidate"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_OBJECT_INFO_BUILD: &str = + "put_object_commit_namespace_lock_held_object_info_build"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_OBJECT_INFO_TO_RETURN: &str = + "put_object_commit_namespace_lock_held_post_object_info_to_return"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT: &str = "put_object_commit_task_wait"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL: &str = "put_object_commit_future_total"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT: &str = "put_object_commit_operation_select_wait"; +pub const PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL: &str = "put_object_operation_await_total"; +pub const PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT: &str = "put_object_operation_body_to_commit_submit"; +pub const PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN: &str = "put_object_operation_result_return"; +pub const PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY: &str = "put_object_operation_first_poll_delay"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY: &str = + "put_object_commit_namespace_lock_acquired_to_commit_ready"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_PRECONDITION_AND_TIMESTAMP: &str = + "put_object_commit_ready_precondition_and_timestamp"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPECTED_VERSION_LOOKUP: &str = "put_object_commit_ready_expected_version_lookup"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPLICIT_VERSION_LOOKUP: &str = "put_object_commit_ready_explicit_version_lookup"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_RESTORE_VERIFY: &str = "put_object_commit_ready_restore_verify"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_FENCE_VERIFY: &str = "put_object_commit_ready_fence_verify"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_DECOMMISSION_CAPACITY_FENCE: &str = + "put_object_commit_ready_decommission_capacity_fence"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_TRANSACTION_EPOCH_FENCE_READ: &str = + "put_object_commit_ready_transaction_epoch_fence_read"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_BEGIN: &str = "put_object_commit_ready_quota_begin"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_METADATA_ADJUST: &str = "put_object_commit_ready_metadata_adjust"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_RESERVE: &str = "put_object_commit_ready_quota_reserve"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_MUTATION_FENCE_PREPARE: &str = + "put_object_commit_ready_quota_mutation_fence_prepare"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_CONTEXT_PREPARE: &str = "put_object_commit_ready_context_prepare"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH: &str = "put_object_quota_begin_meta_bucket_fast_path"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK: &str = "put_object_quota_begin_metadata_lock"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP: &str = "put_object_quota_begin_cache_lookup"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ: &str = "put_object_quota_begin_config_read"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK: &str = "put_object_quota_begin_lock_lost_check"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE: &str = "put_object_quota_begin_policy_evaluate"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF: &str = "put_object_quota_begin_capability_proof"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION: &str = "put_object_quota_begin_snapshot_admission"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP: &str = "put_object_quota_begin_object_store_lookup"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD: &str = "put_object_quota_begin_context_build"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER: &str = "put_object_commit_submit_to_closure_enter"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT: &str = "put_object_commit_closure_enter_to_first_await"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL: &str = "put_object_commit_closure_total"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME: &str = "put_object_commit_before_rename"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_AWAIT_TOTAL: &str = "put_object_commit_pre_rename_await_total"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_QUOTA_MARK_COMMIT_STARTED: &str = + "put_object_commit_pre_rename_quota_mark_commit_started"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_FIRST_FENCE_CHECK: &str = "put_object_commit_pre_rename_first_fence_check"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_RESTORE_VERIFY: &str = "put_object_commit_pre_rename_restore_verify"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TRANSACTION_FENCE_VERIFY: &str = + "put_object_commit_pre_rename_transaction_fence_verify"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SECOND_FENCE_CHECK: &str = "put_object_commit_pre_rename_second_fence_check"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SCANNER_SCOPE_CHECK: &str = "put_object_commit_pre_rename_scanner_scope_check"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TIER_FREE_VERSION_SOURCE_LOOKUP: &str = + "put_object_commit_pre_rename_tier_free_version_source_lookup"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_ASSIGN_RENAME_INDEXES: &str = + "put_object_commit_pre_rename_assign_rename_indexes"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_POST_AWAIT_TO_RENAME: &str = "put_object_commit_pre_rename_post_await_to_rename"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN: &str = "put_object_commit_after_rename_to_return"; pub const PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT: &str = "set_disk_rename_quorum_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_SYNC_FULL_FANOUT_WAIT: &str = "set_disk_rename_sync_full_fanout_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT: &str = "set_disk_rename_early_ack_spawn_to_quorum_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER: &str = "set_disk_rename_early_ack_attach_tail_owner"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_AFTER_QUORUM_WAIT: &str = "set_disk_rename_tail_after_quorum_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT: &str = "set_disk_rename_tail_async_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT: &str = "set_disk_rename_tail_guard_release_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_FINALIZE: &str = "set_disk_rename_tail_finalize"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP: &str = "set_disk_rename_tail_cleanup"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT: &str = "set_disk_rename_tail_cleanup_submit"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP: &str = "set_disk_rename_tail_cleanup_counterfactual_skip"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED: &str = "set_disk_rename_tail_cleanup_defer_enqueued"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_FALLBACK_SYNC: &str = "set_disk_rename_tail_cleanup_defer_fallback_sync"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH: &str = "set_disk_rename_tail_cleanup_defer_queue_push"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH: &str = "set_disk_rename_tail_cleanup_defer_queue_depth"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE: &str = + "set_disk_rename_tail_cleanup_defer_queue_depth_max_sample"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE: &str = "set_disk_rename_tail_cleanup_defer_oldest_age"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE: &str = + "set_disk_rename_tail_cleanup_defer_oldest_age_max_sample"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_TARGETS: &str = "set_disk_rename_tail_cleanup_defer_targets"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_START_DELAY: &str = "set_disk_rename_tail_cleanup_defer_start_delay"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED_LAG: &str = "set_disk_rename_tail_cleanup_defer_completed_lag"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER: &str = "set_disk_rename_tail_cleanup_defer_hold_worker"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY: &str = "set_disk_rename_tail_cleanup_defer_worker_delay"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_PERMIT_WAIT: &str = "set_disk_rename_tail_cleanup_per_disk_permit_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_INFLIGHT: &str = "set_disk_rename_tail_cleanup_per_disk_inflight"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD: &str = "set_disk_rename_tail_cleanup_foreground_yield"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFERRED_FOREGROUND_ACTIVE: &str = + "set_disk_rename_tail_cleanup_deferred_foreground_active"; +pub const PUT_STAGE_SET_DISK_RENAME_FOREGROUND_ACTIVE: &str = "set_disk_rename_foreground_active"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW: &str = "set_disk_rename_tail_cleanup_defer_batch_window"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_SIZE: &str = "set_disk_rename_tail_cleanup_defer_batch_size"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BURST_BATCH_SIZE: &str = + "set_disk_rename_tail_cleanup_defer_burst_batch_size"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_ACTIVE: &str = "set_disk_rename_tail_cleanup_defer_worker_active"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_POLL_WAIT: &str = + "set_disk_rename_tail_cleanup_defer_worker_poll_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_IDLE_WAIT: &str = + "set_disk_rename_tail_cleanup_defer_worker_idle_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_JOB_WAIT: &str = + "set_disk_rename_tail_cleanup_defer_worker_job_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED: &str = "set_disk_rename_tail_cleanup_defer_completed"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DURATION: &str = + "set_disk_rename_tail_cleanup_defer_worker_duration"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_RECEIPT: &str = "set_disk_rename_tail_cleanup_target_receipt"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COMMIT_CAPACITY: &str = + "set_disk_rename_tail_cleanup_target_commit_capacity"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_REPORT: &str = "set_disk_rename_tail_cleanup_target_report"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COUNT: &str = "set_disk_rename_tail_cleanup_target_count"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_ZERO_TARGET: &str = "set_disk_rename_tail_cleanup_zero_target"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE: &str = "set_disk_rename_tail_cleanup_tmp_delete"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE_COUNTERFACTUAL_SKIP: &str = + "set_disk_rename_tail_cleanup_tmp_delete_counterfactual_skip"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DECOMMISSION_GUARD_DROP: &str = + "set_disk_rename_tail_cleanup_decommission_guard_drop"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH: &str = "set_disk_rename_tail_cleanup_lane_queue_depth"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_PUSH: &str = "set_disk_rename_tail_cleanup_lane_queue_push"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE: &str = + "set_disk_rename_tail_cleanup_lane_queue_depth_max_sample"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE: &str = "set_disk_rename_tail_cleanup_lane_oldest_age"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE_MAX_SAMPLE: &str = + "set_disk_rename_tail_cleanup_lane_oldest_age_max_sample"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BATCH_SIZE: &str = "set_disk_rename_tail_cleanup_lane_batch_size"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BURST_BATCH_SIZE: &str = + "set_disk_rename_tail_cleanup_lane_burst_batch_size"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_ACTIVE: &str = "set_disk_rename_tail_cleanup_lane_worker_active"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_POLL_WAIT: &str = + "set_disk_rename_tail_cleanup_lane_worker_poll_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_IDLE_WAIT: &str = + "set_disk_rename_tail_cleanup_lane_worker_idle_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_JOB_WAIT: &str = "set_disk_rename_tail_cleanup_lane_worker_job_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_SERVICE_TIME: &str = "set_disk_rename_tail_cleanup_lane_service_time"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_REQUEUE: &str = "set_disk_rename_tail_cleanup_lane_requeue"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_FOREGROUND_ACTIVE_HIT: &str = + "set_disk_rename_tail_cleanup_lane_foreground_active_hit"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_HEAL_SUBMIT: &str = "set_disk_rename_tail_heal_submit"; pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT: &str = "set_disk_rename_disk_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL: &str = "set_disk_rename_disk_wait_local"; +pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE: &str = "set_disk_rename_disk_wait_remote"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE: &str = "set_disk_rename_remote_client_prepare"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC: &str = "set_disk_rename_remote_client_rpc"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT: &str = "set_disk_rename_remote_client_rpc_await"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT: &str = + "set_disk_rename_remote_client_rpc_await_poll_pending_count"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY: &str = + "set_disk_rename_remote_client_rpc_await_first_pending_to_ready"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE: &str = "set_disk_rename_remote_client_request_scope"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL: &str = "set_disk_rename_remote_client_transport_call"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT: &str = + "set_disk_rename_remote_client_transport_call_poll_pending_count"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY: &str = + "set_disk_rename_remote_client_transport_call_first_pending_to_ready"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE: &str = "set_disk_rename_remote_client_replay_response"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER: &str = "set_disk_rename_remote_client_into_inner"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK: &str = "set_disk_rename_remote_client_success_check"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE: &str = "set_disk_rename_remote_client_decode"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS: &str = "set_disk_rename_remote_client_response_process"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS: &str = "set_disk_rename_remote_client_response_headers"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE: &str = "set_disk_rename_remote_service_response_encode"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_TRIGGERED: &str = + "put_object_commit_namespace_lock_held_quota_fence_release_triggered"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_SKIPPED: &str = + "put_object_commit_namespace_lock_held_quota_fence_release_skipped"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_TRIGGERED: &str = + "put_object_commit_namespace_lock_held_heal_submit_triggered"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_SKIPPED: &str = + "put_object_commit_namespace_lock_held_heal_submit_skipped"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_TRIGGERED: &str = + "put_object_commit_namespace_lock_held_cleanup_receipt_triggered"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_SKIPPED: &str = + "put_object_commit_namespace_lock_held_cleanup_receipt_skipped"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL: &str = "set_disk_rename_remote_handler_total"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD: &str = "set_disk_rename_remote_handler_overhead"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP: &str = "set_disk_rename_remote_handler_lookup"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT: &str = "set_disk_rename_remote_handler_admit"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME: &str = "set_disk_rename_remote_handler_local_rename"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY: &str = "set_disk_rename_remote_service_entry"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER: &str = "set_disk_rename_remote_service_before_handler"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT: &str = "set_disk_rename_remote_service_handler_await"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER: &str = "set_disk_rename_remote_service_after_handler"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD: &str = "set_disk_rename_remote_service_response_build"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN: &str = "set_disk_rename_remote_service_response_return"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE: &str = "set_disk_rename_remote_transport_service_future"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME: &str = "set_disk_rename_remote_transport_first_frame"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE: &str = "set_disk_rename_remote_transport_body_complete"; pub const PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_file_sync_permit_wait"; pub const PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_global_file_sync_permit_wait"; pub const PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC: &str = "set_disk_rename_file_fdatasync"; @@ -460,7 +746,16 @@ pub fn record_get_object_request_start(concurrent_requests: usize) { gauge!("rustfs_io_get_object_concurrent_requests").set(concurrent_requests as f64); } -/// Record GetObject request start without concurrency context. +/// Record one timeout-wrapped GetObject storage operation start. +#[inline(always)] +pub fn record_get_object_timeout_operation_start() { + if !get_stage_metrics_enabled() { + return; + } + counter!("rustfs_io_get_object_timeout_operations_total").increment(1); +} + +/// Record a GetObject request start without concurrency context. #[inline(always)] pub fn record_get_object_request_started() { if !get_stage_metrics_enabled() { @@ -479,6 +774,16 @@ pub fn record_get_object_request_result(status: &str, duration_secs: f64) { histogram!("rustfs_io_get_object_request_duration_seconds", "status" => status.to_string()).record(duration_secs); } +/// Record the result of one timeout-wrapped GetObject storage operation. +#[inline(always)] +pub fn record_get_object_timeout_operation_result(status: &str, duration_secs: f64) { + if !get_stage_metrics_enabled() { + return; + } + counter!("rustfs_io_get_object_timeout_operation_results_total", "status" => status.to_string()).increment(1); + histogram!("rustfs_io_get_object_timeout_operation_duration_seconds", "status" => status.to_string()).record(duration_secs); +} + /// Record PutObject request start. #[inline(always)] pub fn record_put_object_request_start(concurrent_requests: usize) { @@ -760,6 +1065,10 @@ pub fn record_get_object_io_state( /// Record GetObject phase duration for the current read path. #[inline(always)] pub fn record_get_object_stage_duration(path: &'static str, stage: &'static str, duration_secs: f64) { + if get_stage_local_summary_enabled() { + record_get_stage_local_summary(path, stage, "", "", duration_secs); + return; + } if !get_stage_metrics_enabled() { return; } @@ -775,6 +1084,10 @@ pub fn record_get_object_stage_duration_by_size( size_bucket: &'static str, duration_secs: f64, ) { + if get_stage_local_summary_enabled() { + record_get_stage_local_summary(path, stage, object_class, size_bucket, duration_secs); + return; + } if !get_stage_metrics_enabled() { return; } @@ -827,6 +1140,15 @@ pub fn record_get_object_metadata_response(path: &'static str, outcome: &'static counter!("rustfs_io_get_object_metadata_response_total", "path" => path, "outcome" => outcome).increment(1); } +/// Record whether metadata resolution selected an object identity at read quorum. +#[inline(always)] +pub fn record_get_object_metadata_quorum_result(path: &'static str, outcome: &'static str) { + if !get_stage_metrics_enabled() { + return; + } + counter!("rustfs_io_get_object_metadata_quorum_results_total", "path" => path, "outcome" => outcome).increment(1); +} + /// Record one bounded metadata cache decision. #[inline(always)] pub fn record_get_object_metadata_cache_decision(path: &'static str, decision: &'static str, reason: &'static str) { @@ -1668,6 +1990,10 @@ pub fn record_get_object_metadata_phase_duration(duration_secs: f64) { /// Record metadata phase duration with early-stop state label. #[inline(always)] pub fn record_get_object_metadata_phase_duration_with_early_stop(duration_secs: f64, early_stop_active: &'static str) { + if get_stage_local_summary_enabled() { + record_get_stage_local_summary("legacy_duplex", "metadata", early_stop_active, "", duration_secs); + return; + } if !get_stage_metrics_enabled() { return; } @@ -2073,6 +2399,57 @@ pub fn record_put_object_stage_duration_from(stage: &'static str, started_at: Op } } +#[inline(always)] +pub fn record_put_object_stage_count(stage: &'static str, count: u64) { + if !put_stage_metrics_enabled() { + return; + } + counter!("rustfs_s3_put_object_stage_count", "stage" => stage).increment(count); +} + +/// Observe a future's latency and wakeup behavior only while detailed PUT +/// attribution is enabled. The disabled path awaits the original future +/// directly, so normal remote mutation RPCs do not gain a polling wrapper. +pub async fn observe_put_stage_future( + future: F, + duration_stage: &'static str, + pending_count_stage: &'static str, + first_pending_to_ready_stage: &'static str, +) -> F::Output +where + F: std::future::Future, +{ + if !put_stage_metrics_enabled() { + return future.await; + } + + let duration_started = put_stage_timer(); + let mut first_pending_at = None; + let mut pending_count = 0_u64; + let mut future = std::pin::pin!(future); + let output = std::future::poll_fn(|cx| match future.as_mut().poll(cx) { + std::task::Poll::Ready(output) => std::task::Poll::Ready(output), + std::task::Poll::Pending => { + pending_count = pending_count.saturating_add(1); + first_pending_at.get_or_insert_with(std::time::Instant::now); + std::task::Poll::Pending + } + }) + .await; + record_put_object_stage_duration_from(duration_stage, duration_started); + record_put_object_stage_count(pending_count_stage, pending_count); + record_put_object_stage_duration_from(first_pending_to_ready_stage, first_pending_at); + output +} + +#[inline(always)] +pub fn record_put_object_quota_cache_decision(decision: &'static str, reason: &'static str) { + if !put_stage_metrics_enabled() { + return; + } + counter!("rustfs_s3_put_object_quota_cache_total", "decision" => decision, "reason" => reason).increment(1); +} + #[inline(always)] pub fn record_put_object_commit_lock_admission(budget: &'static str, outcome: &'static str) { if !put_stage_metrics_enabled() { @@ -3219,8 +3596,162 @@ mod tests { assert_eq!(PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT, "put_object_commit_namespace_lock_wait"); let stages = [ PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_PRE_RENAME_PREPARE, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME_RESULT_TO_TAIL_HANDOFF, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_TAIL_HANDOFF, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_RENAME, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_GUARD_RELEASE, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_COMMIT, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_METADATA_INVALIDATE, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_OBJECT_INFO_BUILD, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_OBJECT_INFO_TO_RETURN, + PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT, + PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL, + PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT, + PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL, + PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT, + PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN, + PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_PRECONDITION_AND_TIMESTAMP, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPECTED_VERSION_LOOKUP, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPLICIT_VERSION_LOOKUP, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_RESTORE_VERIFY, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_FENCE_VERIFY, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_DECOMMISSION_CAPACITY_FENCE, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_TRANSACTION_EPOCH_FENCE_READ, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_BEGIN, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_METADATA_ADJUST, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_RESERVE, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_MUTATION_FENCE_PREPARE, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_CONTEXT_PREPARE, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD, + PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER, + PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT, + PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL, + PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_AWAIT_TOTAL, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_QUOTA_MARK_COMMIT_STARTED, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_FIRST_FENCE_CHECK, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_RESTORE_VERIFY, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TRANSACTION_FENCE_VERIFY, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SECOND_FENCE_CHECK, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SCANNER_SCOPE_CHECK, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TIER_FREE_VERSION_SOURCE_LOOKUP, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_ASSIGN_RENAME_INDEXES, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_POST_AWAIT_TO_RENAME, + PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN, PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT, + PUT_STAGE_SET_DISK_RENAME_SYNC_FULL_FANOUT_WAIT, + PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT, + PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER, + PUT_STAGE_SET_DISK_RENAME_TAIL_AFTER_QUORUM_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_FINALIZE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_FALLBACK_SYNC, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_TARGETS, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_START_DELAY, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED_LAG, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_PERMIT_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_INFLIGHT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFERRED_FOREGROUND_ACTIVE, + PUT_STAGE_SET_DISK_RENAME_FOREGROUND_ACTIVE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_SIZE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BURST_BATCH_SIZE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_ACTIVE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_POLL_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_IDLE_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_JOB_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DURATION, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_RECEIPT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COMMIT_CAPACITY, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_REPORT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COUNT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_ZERO_TARGET, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE_COUNTERFACTUAL_SKIP, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DECOMMISSION_GUARD_DROP, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_PUSH, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE_MAX_SAMPLE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BATCH_SIZE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BURST_BATCH_SIZE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_ACTIVE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_POLL_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_IDLE_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_JOB_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_SERVICE_TIME, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_REQUEUE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_FOREGROUND_ACTIVE_HIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_HEAL_SUBMIT, PUT_STAGE_SET_DISK_RENAME_DISK_WAIT, + PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL, + PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS, + PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_TRIGGERED, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_SKIPPED, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_TRIGGERED, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_SKIPPED, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_TRIGGERED, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_SKIPPED, + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL, + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD, + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME, + PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY, + PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER, + PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT, + PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER, + PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD, + PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN, + PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE, + PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME, + PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE, PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT, PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT, PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC, @@ -3233,11 +3764,36 @@ mod tests { let unique = stages.iter().copied().collect::>(); assert_eq!(unique.len(), stages.len()); assert!(stages.iter().all(|stage| { - (stage.starts_with("set_disk_rename_") || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT) + (stage.starts_with("set_disk_rename_") + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT + || *stage == PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL + || *stage == PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT + || *stage == PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN + || *stage == PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY + || stage.starts_with("put_object_commit_ready_") + || stage.starts_with("put_object_quota_begin_") + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME + || stage.starts_with("put_object_commit_pre_rename_") + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN + || stage.starts_with("put_object_commit_namespace_lock_held_")) && !stage.contains('/') && !stage.contains('{') })); + let count_stages = [ + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT, + ]; + assert_eq!(count_stages.iter().copied().collect::>().len(), count_stages.len()); + let recorder = DebuggingRecorder::new(); let snapshotter = recorder.snapshotter(); metrics::with_local_recorder(&recorder, || { @@ -3245,16 +3801,25 @@ mod tests { for stage in stages { record_put_object_stage_duration(stage, 1.0); } + for stage in count_stages { + record_put_object_stage_count(stage, 1); + } set_put_stage_metrics_enabled(true); for stage in stages { record_put_object_stage_duration(stage, 1.0); } + for stage in count_stages { + record_put_object_stage_count(stage, 1); + } set_put_stage_metrics_enabled(false); }); - let recorded = snapshotter - .snapshot() - .into_vec() + let rows = snapshotter.snapshot().into_vec(); + assert_eq!( + counter_total(&rows, "rustfs_s3_put_object_stage_count"), + Some(u64::try_from(count_stages.len()).expect("count stage length fits u64")), + ); + let recorded = rows .into_iter() .filter(|(composite, _, _, _)| { composite.kind() == MetricKind::Histogram && composite.key().name() == "rustfs_s3_put_object_stage_duration_ms" @@ -3349,6 +3914,63 @@ mod tests { ]))); } + #[test] + fn put_stage_future_observer_preserves_outputs_and_counts_pending() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .expect("build current-thread runtime for observer test"); + let duration_stage = PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT; + let count_stage = PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT; + let pending_stage = PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY; + + metrics::with_local_recorder(&recorder, || { + set_put_stage_metrics_enabled(false); + let output = runtime.block_on(observe_put_stage_future( + std::future::ready(42), + duration_stage, + count_stage, + pending_stage, + )); + assert_eq!(output, 42, "disabled observer must preserve the future output"); + }); + assert!(snapshotter.snapshot().into_vec().is_empty(), "disabled observer must emit no metrics"); + + metrics::with_local_recorder(&recorder, || { + set_put_stage_metrics_enabled(true); + let mut pending = 2; + let future = std::future::poll_fn(|cx| { + if pending > 0 { + pending -= 1; + cx.waker().wake_by_ref(); + std::task::Poll::Pending + } else { + std::task::Poll::Ready(Err::<(), _>("original error")) + } + }); + let output = runtime.block_on(observe_put_stage_future(future, duration_stage, count_stage, pending_stage)); + set_put_stage_metrics_enabled(false); + assert_eq!(output, Err("original error"), "enabled observer must preserve errors"); + }); + + let rows = snapshotter.snapshot().into_vec(); + assert_eq!(counter_total(&rows, "rustfs_s3_put_object_stage_count"), Some(2)); + let count_labels = rows + .iter() + .filter(|(composite, _, _, _)| composite.key().name() == "rustfs_s3_put_object_stage_count") + .flat_map(|(composite, _, _, _)| composite.key().labels().map(|label| label.value().to_string())) + .collect::>(); + assert_eq!(count_labels, vec![count_stage.to_string()]); + let durations = rows + .iter() + .filter(|(composite, _, _, _)| composite.key().name() == "rustfs_s3_put_object_stage_duration_ms") + .flat_map(|(composite, _, _, _)| composite.key().labels().map(|label| label.value().to_string())) + .collect::>(); + assert_eq!(durations, HashSet::from([duration_stage.to_string(), pending_stage.to_string()])); + } + #[test] fn put_rename_code_level_metrics_are_static_and_gated() { let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); @@ -3578,6 +4200,35 @@ mod tests { set_put_stage_metrics_enabled(false); } + #[test] + fn get_timeout_operation_and_metadata_quorum_metrics_have_separate_scopes() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + metrics::with_local_recorder(&recorder, || { + set_get_stage_metrics_enabled(true); + record_get_object_request_start(1); + record_get_object_request_result("ok", 0.01); + record_get_object_timeout_operation_start(); + record_get_object_timeout_operation_result("success", 0.005); + record_get_object_metadata_quorum_result("legacy_duplex", "reached"); + record_get_object_metadata_quorum_result("legacy_duplex", "not_reached"); + set_get_stage_metrics_enabled(false); + }); + + let rows = snapshotter.snapshot().into_vec(); + assert_eq!(counter_total(&rows, "rustfs_io_get_object_requests_total"), Some(1)); + assert_eq!(counter_total(&rows, "rustfs_io_get_object_request_results_total"), Some(1)); + assert_eq!(counter_total(&rows, "rustfs_io_get_object_timeout_operations_total"), Some(1)); + assert_eq!(counter_total(&rows, "rustfs_io_get_object_timeout_operation_results_total"), Some(1)); + assert_eq!(counter_total(&rows, "rustfs_io_get_object_metadata_quorum_results_total"), Some(2)); + assert_eq!( + histogram_samples(&rows, "rustfs_io_get_object_timeout_operation_duration_seconds"), + vec![0.005] + ); + } + #[test] fn test_record_get_object_path_and_stage() { let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); diff --git a/crates/log-analyzer/src/ingest/mod.rs b/crates/log-analyzer/src/ingest/mod.rs index 2ad960479..ba8b97b27 100644 --- a/crates/log-analyzer/src/ingest/mod.rs +++ b/crates/log-analyzer/src/ingest/mod.rs @@ -307,7 +307,13 @@ fn walk_zip_entries( if entry.is_dir() { continue; } - let entry_path = entry.name().to_string(); + let entry_path = match entry.name() { + Ok(name) => name.into_owned(), + Err(_) => { + ctx.skip(format!("{provenance}!"), SkipReason::Unreadable); + continue; + } + }; let child_provenance = format!("{provenance}!{entry_path}"); let child_node = node_from_entry_path(&entry_path).or_else(|| node.clone()); let child_name = leaf_name(&entry_path).to_string(); diff --git a/crates/obs/src/telemetry/resource.rs b/crates/obs/src/telemetry/resource.rs index 8a7d003ab..5f9c4aa33 100644 --- a/crates/obs/src/telemetry/resource.rs +++ b/crates/obs/src/telemetry/resource.rs @@ -16,9 +16,9 @@ //! //! A `Resource` describes the entity producing telemetry data. The resource //! built here includes the service name, service version, deployment -//! environment, the stable RustFS node identity, and the local machine IP -//! address so that data can be correlated across services in a distributed -//! system. +//! environment, the stable RustFS node identity, the process ID, and the local +//! machine IP address so telemetry series can be correlated across service +//! restarts and nodes. use crate::config::OtelConfig; use crate::node_identity::{RUSTFS_NODE_ATTRIBUTE, local_node_identity}; @@ -27,7 +27,7 @@ use opentelemetry_sdk::Resource; use opentelemetry_semantic_conventions::{ SCHEMA_URL, attribute::{ - DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_INSTANCE_ID as OTEL_SERVICE_INSTANCE_ID, + DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, PROCESS_PID, SERVICE_INSTANCE_ID as OTEL_SERVICE_INSTANCE_ID, SERVICE_VERSION as OTEL_SERVICE_VERSION, }, }; @@ -48,6 +48,8 @@ use std::borrow::Cow; /// - `network.local.address` — the primary local IP of the current host, /// useful as an operational fallback when the stable node name is not yet /// initialized. +/// - `process.pid` — distinguishes telemetry emitted by different processes +/// running on the same node. /// /// All attributes are attached to the resource using the semantic conventions /// schema URL to ensure compatibility with standard OTLP backends. @@ -70,6 +72,7 @@ pub(super) fn build_resource(config: &OtelConfig) -> Resource { KeyValue::new(RUSTFS_NODE_ATTRIBUTE, node_identity.clone()), KeyValue::new(OTEL_SERVICE_INSTANCE_ID, node_identity), KeyValue::new(NETWORK_LOCAL_ADDRESS, local_ip), + KeyValue::new(PROCESS_PID, i64::from(std::process::id())), ], SCHEMA_URL, ) @@ -117,6 +120,12 @@ mod tests { .map(|value| value.to_string()), Some("node1:9000".to_string()) ); + assert_eq!( + resource + .get(&Key::from_static_str(PROCESS_PID)) + .map(|value| value.to_string()), + Some(std::process::id().to_string()) + ); rustfs_common::set_global_local_node_name(&previous).await; } diff --git a/deny.toml b/deny.toml index b810318e7..1a84bdbbb 100644 --- a/deny.toml +++ b/deny.toml @@ -19,9 +19,6 @@ no-default-features = false version = 2 yanked = "deny" ignore = [ - # `paste 1.0.15` — unmaintained. No direct dependency. - # owner: rustfs-maintainers review: 2026-07 - { id = "RUSTSEC-2024-0436", reason = "paste unmaintained; transitive only; tracked for upgrade" }, # `rsa` Marvin timing sidechannel (RUSTSEC-2023-0071). Pulled in via # `openidconnect` (transitive) and historically used directly. No upstream # fix is available yet. Tracked separately for follow-up; remove this diff --git a/docs/README.md b/docs/README.md index d30e3758a..958dbda58 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,6 +8,9 @@ collection: ## Operations +For PUT cleanup admission, blocking-pool isolation, and optional CPU affinity, +see [PUT cleanup isolation](operations/put-cleanup-isolation.md). + - [External certificate issuance and TLS rotation](operations/tls-certificate-rotation.md) — publication, native reload, trust separation and failure handling. - [Architecture guard troubleshooting](operations/architecture-guard-troubleshooting.md) — diagnose repository guard failures. diff --git a/docs/operations/put-cleanup-isolation.md b/docs/operations/put-cleanup-isolation.md new file mode 100644 index 000000000..a9adaca5b --- /dev/null +++ b/docs/operations/put-cleanup-isolation.md @@ -0,0 +1,195 @@ +# PUT and disk cleanup isolation + +Cleanup can use process-wide admission budgets and a dedicated Tokio runtime. +This is opt-in. It preserves cleanup receipts, eligible-data moves to trash, +PUT temporary workspace cleanup, and periodic physical trash removal. It does +not disable healing or skip cleanup. + +## Configuration + +| Variable | Default | Meaning when isolation is enabled | +| --- | --- | --- | +| `RUSTFS_CLEANUP_ISOLATE_ENABLE` | `false` | Enable admission and execution isolation on this node. | +| `RUSTFS_PUT_RENAME_TAIL_CLEANUP_MAX_PENDING` | `1024` | Maximum admitted PUT ownership groups, including surviving commit and cleanup owners. | +| `RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS` | `2` | Maximum active PUT cleanup continuations. Also controls existing deferred queue workers; does not determine OS thread count. | +| `RUSTFS_CLEANUP_ASYNC_THREADS` | up to `2` | Dedicated async worker threads, limited by available parallelism and an explicit CPU mask by default. Independent of logical job concurrency. | +| `RUSTFS_CLEANUP_DISK_MAX_PENDING` | `16 × disk workers` (`64`) | Maximum admitted local disk cleanup operations, including queued and running operations. Shared by local requests, incoming RPC cleanup and periodic GC steps. | +| `RUSTFS_CLEANUP_DISK_WORKERS` | blocking threads (`4`) | Maximum active disk cleanup operations within that budget. | +| `RUSTFS_CLEANUP_GC_WORKERS` | `1` | Maximum periodic disk scans producing GC steps. | +| `RUSTFS_CLEANUP_BLOCKING_THREADS` | `4` | Maximum additional blocking threads on the cleanup runtime. | +| `RUSTFS_CLEANUP_CPUS` | `1-2` on Linux; unbound elsewhere | CPU affinity for cleanup async and blocking threads. Override with a CPU list such as `4-5,8`, or `none` to disable affinity while keeping isolation. | +| `RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ENABLE` | `false` | Use the existing bounded deferred queue before coordinator execution. Isolation also works without this queue. | +| `RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY` | `1024` | Capacity of that existing queue; this is not the total admission budget. | + +Counts must be positive. PUT workers must not exceed PUT pending capacity; +GC workers must not exceed disk workers, which must not exceed disk pending +capacity. Async and blocking thread counts must not exceed 1024. CPU IDs must be supported by the affinity implementation and belong +to the initiating thread's allowed CPU set. Non-Linux hosts reject explicit CPU +masks. Configuration is read once at first use, including disk cleanup or GC; +changes require a restart. Invalid configuration rejects new cleanup admission. +PUT checks configuration before staging data. + +Isolation remains disabled by default. When enabling it on Linux, the default +mask requires logical CPU IDs 1 and 2 to be available inside the process/container +CPU set. On smaller or restricted CPU sets, configure permitted CPU IDs or `none` +before enabling isolation. An unavailable default mask is an error; it does not +silently select different CPUs. Configure each node separately. + +Do not combine isolation with cleanup counterfactuals +`RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP`, +`RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER`, or +`RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP`. These combinations +are rejected because they deliberately suppress cleanup or stop queue progress. + +Do not enable `RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_ENABLE` together with +`RUSTFS_CLEANUP_ISOLATE_ENABLE`. The legacy per-disk lane adds a second queue +inside the isolated coordinator budget, so a slow lane can hold coordinator +capacity while unrelated cleanup waits. This combination is rejected before +PUT admission; either scheduler can still be enabled on its own. + +## Ownership and execution + +PUT attempts admission before staging. Full admission returns `SlowDown`, so +clients should retry with backoff. Admission does not wait: COPY readers and +data-movement callers can already hold locks needed by admitted PUTs. No body +is ingested or tmp data staged by a rejected PUT. + +The request, detached commit owner, and cleanup continuations share one +reservation. An early ACK or cancelled waiter cannot release it while an owner +still needs cleanup. After the existing rename-convergence and guard-release +handoff, deferred workers, queue overflow and direct continuations all acquire +the same coordinator execution semaphore. Queue overflow cannot bypass that +limit. Awaited failure cleanup and no-tail tmp cleanup use the same budget. +Full-tail receipt persistence remains awaited at its existing handoff point; +old-data reclamation remains after the existing guard release. Incomplete +rollback still preserves staging for recovery. + +Each local disk operation has separate pending and active permits. The wrapper +below RPC authentication uses this gate for low-level `delete`, `delete_data_dir` +and old-data cleanup receipt `write_all`. Local PUT fan-out and incoming peer +RPCs therefore share the same node-local disk budget. Receipt classification +uses the internal receipt filename; ordinary writes retain their current path. +RPC schemas and authentication are unchanged. Enable isolation on every node +to isolate peer execution; settings and CPU masks are not transmitted by RPC. + +Coordinator and receiver permits are deliberately distinct: a coordinator can +wait for peer RPCs without occupying the slots those peers need to service disk +cleanup. Full disk admission returns an I/O `WouldBlock` error through the +existing disk error path. A mutation carrying an external publication or +namespace owner also refuses a full active budget instead of waiting while it +holds a lock. Existing cleanup error handling and receipt recovery still apply. + +Ordinary snapshot-lease release stays on the caller and only updates the in-memory +registry. It does not allocate a cleanup task or wait for the cleanup scheduler. +Invalid cleanup configuration also must not strand a reader token: release still +updates the registry, while any deferred reclamation reports the initialization +error and keeps both its intent and data. It does not fall back to unbudgeted I/O. +If the last release triggers deferred old-data +reclamation, that reclamation tries the same pending and active disk permits. +Only this case dispatches an owned task to the cleanup runtime. +Saturation leaves the token released and the cleanup intent/receipt available +for retry; it does not strand a live reader token or bypass the budget. Quota +fence release keeps its original receiver lifecycle and does not acquire disk +execution permits, since it can wait for mutations that already own them. + +An admitted request waits for disk execution capacity in the caller's existing +future. Cancellation or timeout discards this unstarted operation and releases +pending capacity; it cannot later start deleting a reused path. After execution +capacity is acquired, the worker also checks whether its result receiver closed +before its first poll. A job cancelled while waiting for the runtime to schedule +it is discarded when the worker polls it, releasing its permits without starting +I/O. Once the worker starts the operation, the owned job retains its permits and +disk reference until completion. Cancelling or timing out its waiter does not +free disk capacity while that started job still runs. +This also retains the disk's mount ownership during detached execution. Cleanup +waiter timeouts include queue delay and do not by themselves mark a disk faulty; +the existing active disk-health probe retains its timeout policy. + +Periodic trash and stale-tmp scans have a separate scan limit. Each scan acquires +the shared disk budget to open its directory and then for each entry, releasing +it between entries. The steps borrow the scan's iterator and paths and run in +the same owned task, without a spawn/join or path clones per entry. A scan cannot occupy a disk execution slot while waiting +for another scan slot. GC waits for admission without holding object locks; +request admission is bounded without creating waiting producer tasks. A single +trash subtree is still removed as one operation, so a very large subtree can +hold a slot for its entire removal. Queued scans retain the original mount +handle even when the corresponding disk object is dropped. +Missed periodic ticks are skipped so a slow scan does not cause a burst of +catch-up scans. + +Cleanup runs on `rustfs-cleanup` threads. Nested Tokio tasks, Tokio filesystem +work and ordinary `spawn_blocking` use that runtime. Explicit fsync dispatch +also stays on its blocking pool. File/directory group-commit keys separate +cleanup and foreground producers, preventing batches from moving into the +other class's runtime. Fsync durability settings remain unchanged. + +The blocking limit bounds simultaneous blocking execution, not Tokio's internal +queue or kernel I/O. Disk admission bounds selected operations, not bytes, +syscall count or device IOPS. Existing synchronous path probes and disk-full +delete fallbacks can still run on the dedicated async workers, under the disk +operation budget. All classes still share the filesystem, journal, +page cache and devices with foreground work. + +If CPU binding fails at thread startup, new admissions fail and the failure is +logged; already-admitted cleanup can finish. Affinity does not reserve exclusive +cores, move foreground threads elsewhere, change cgroups, or control kernel +workers and interrupt affinity. + +## Scope and recovery + +The limits are per process, not cluster-wide quotas. This is not a durable work +queue. Existing cleanup receipts and tmp/trash recovery retain their role after +process exit. The option does not bound all storage mutations: metadata/version +DELETE transactions, heal/scanner coordination and +multipart orchestration retain their scheduling. Calls from those paths that +reach the selected disk wrapper entries do share the disk budget; internal +LocalDisk operations that bypass those entries do not. + +## Measurement and tuning + +On Linux, `cargo test -p rustfs-ecstore --test cleanup_isolation_test` exercises +the public disk facade, actual thread affinity, receipt/delete execution and +periodic physical trash removal with temporary disks. It does not benchmark +throughput. + +With PUT stage metrics enabled, observe: + +- `rustfs_cleanup_admitted_puts` and `rustfs_cleanup_admission_rejected_total`; +- `rustfs_cleanup_execution_wait_seconds`; +- `rustfs_cleanup_disk_pending`, `rustfs_cleanup_disk_active`, and + `rustfs_cleanup_disk_admission_rejected_total`; +- `rustfs_cleanup_disk_execution_wait_seconds` and `rustfs_cleanup_gc_active`; +- existing deferred queue depth, age, overflow, completion and service time. + +Compare isolation disabled, budget/pool isolation without CPU affinity, and +isolation with affinity. Keep durability, object sizes, overwrite ratio, +concurrency and cluster placement fixed. Measure PUT/GET throughput and p95/p99 +latency, rejections, queue age, receipt/trash growth, per-disk I/O latency, CPU +usage and context switches. Include a long steady-state interval covering +physical GC, and verify backlog drains afterward. + +Start with small worker and blocking-thread limits, then increase only when +cleanup cannot keep up without harming foreground latency. Too few workers or +CPUs can increase rejections and lower PUT throughput. Larger queues do not +increase sustainable device throughput. Affinity helps only when CPU scheduling +contention matters to the measured workload; no throughput gain is guaranteed. + +The defaults protect a node; they are not a claim of optimal production throughput. +The default disk queue permits sixteen service waves, rather than 256 waves with +1024 pending operations and four workers. For illustration, at a uniform 100 ms +per operation, these bounds correspond to roughly 1.6 s versus 25.6 s to drain a +full queue with no new arrivals. Real service times, stalls and disk fan-out vary; +there is no queue latency guarantee. The ordinary disk waiter timeout defaults to +30 s and does not cancel already-admitted work. + +Size pending capacity for measured arrival bursts and local/peer fan-out, not +only for PUT request concurrency. For example, two coordinator jobs each touching +16 local disks can submit 32 disk operations before incoming peer requests and +GC are counted. Sixty-four slots need not absorb every distributed burst. Increase +the explicit limit when measured rejection rates require it, while keeping queue +age comfortably below waiter deadlines. On many-disk nodes, measure whether four +active operations underutilize devices before increasing disk/blocking workers. +The 1024 PUT admission limit includes uploading requests and detached owners; +it is an object-count bound, not a memory or byte budget. The older deferred queue +remains optional and disabled by default; its overflow path cannot bypass the +new coordinator or disk limits. No unbounded foreground fallback is introduced. diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index 1a23d64dd..8641d2c05 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -525,10 +525,13 @@ impl Operation for ImportBucketMetadata { // First pass: read all file contents into memory let mut file_contents = Vec::new(); for i in 0..zip_reader.len() { - let mut file = zip_reader + let (mut file, file_path) = zip_reader .by_index(i) + .and_then(|file| { + let path = file.name()?.into_owned(); + Ok((file, path)) + }) .map_err(|e| s3_error!(InternalError, "failed to read archive entry: {e}"))?; - let file_path = file.name().to_string(); let mut content = Vec::new(); file.read_to_end(&mut content) @@ -1564,7 +1567,7 @@ mod backup_zip_compatibility_tests { let mut content = Vec::new(); file.read_to_end(&mut content).expect("read diagnostic config"); assert!(!content.windows(RAW_SECRET.len()).any(|window| window == RAW_SECRET)); - files.insert(file.name().to_string(), content); + files.insert(file.name().expect("diagnostic entry name").into_owned(), content); } assert!(!files.contains_key(&format!("_diagnostic/{BUCKET}/{config_file}"))); assert_eq!(files[&format!("_diagnostic/{BUCKET}/{BUCKET_VERSIONING_CONFIG}")], VERSIONING_XML); @@ -1701,8 +1704,9 @@ mod backup_zip_compatibility_tests { let mut content = Vec::new(); file.read_to_end(&mut content).expect("read diagnostic config"); assert!(!content.windows(SECRET.len()).any(|window| window == SECRET.as_bytes())); - assert!(file.name() == DIAGNOSTIC_EXPORT_MANIFEST || file.name().starts_with("_diagnostic/")); - files.insert(file.name().to_string(), content); + let file_path = file.name().expect("diagnostic entry name").into_owned(); + assert!(file_path == DIAGNOSTIC_EXPORT_MANIFEST || file_path.starts_with("_diagnostic/")); + files.insert(file_path, content); } assert_eq!(files[&format!("_diagnostic/{HEALTHY}/{BUCKET_VERSIONING_CONFIG}")], VERSIONING_XML); assert!(files.contains_key(&format!("_diagnostic/{UNREADABLE}/{BUCKET_VERSIONING_CONFIG}"))); diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs index b26b72103..eab0a72bf 100644 --- a/rustfs/src/app/object/get.rs +++ b/rustfs/src/app/object/get.rs @@ -277,6 +277,16 @@ const GET_OBJECT_STAGE_LIFECYCLE_EXPIRATION: &str = "lifecycle_expiration"; const GET_OBJECT_STAGE_METADATA_FILTER: &str = "metadata_filter"; +const GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT: &str = "body_cache_hook_hit"; +const GET_OBJECT_STAGE_BODY_CACHE_LOOKUP: &str = "body_cache_lookup"; +const GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF: &str = "body_cache_fill_handoff"; +const GET_OBJECT_STAGE_BODY_BUFFERED_HANDOFF: &str = "body_buffered_handoff"; +const GET_OBJECT_STAGE_OUTPUT_STRUCT_BUILD: &str = "output_struct_build"; +const GET_OBJECT_STAGE_PREPARE_READ_EXECUTION: &str = "prepare_read_execution"; +const GET_OBJECT_STAGE_RESPONSE_CORS_WRAP: &str = "response_cors_wrap"; +const GET_OBJECT_STAGE_RESPONSE_HEADER_INJECT: &str = "response_header_inject"; +const GET_OBJECT_STAGE_RESPONSE_EVENT_COMPLETE: &str = "response_event_complete"; + const GET_OBJECT_STREAM_WARN_THRESHOLD: Duration = Duration::from_secs(5); static GET_OBJECT_BUFFER_THRESHOLD_WARNED: AtomicBool = AtomicBool::new(false); @@ -555,10 +565,10 @@ fn get_object_stream_failure_reason(error_class: &'static str) -> &'static str { } } -/// A mid-stream reopen can recover only when another representation of the -/// committed object may exist. A single-disk, no-parity object has no alternate -/// shard after a bitrot/read-quorum failure, so retrying the same read only -/// repeats the failure and adds latency. +/// A mid-stream reopen can only recover when another representation of the +/// committed object may exist. Single-disk, no-parity objects have no alternate +/// shard to discover after a bitrot/read-quorum failure, so arming the normal +/// three-attempt resume loop only repeats the same failing read and adds delay. fn should_attach_get_object_resume(info: &ObjectInfo) -> bool { info.parity_blocks > 0 || info.data_blocks > 1 || !info.transitioned_object.tier.is_empty() } @@ -3470,12 +3480,15 @@ impl DefaultObjectUsecase { // already supplied this body, the request-level plan was built before // the authoritative lookup. Serve it without planning a second time. if cache_hook_served && let Some(bytes) = buffered_body.take() { - return Ok(Self::build_memory_bytes_blob( + let hook_hit_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); + let body = Self::build_memory_bytes_blob( bytes, response_content_length, GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE, lifecycle, - )); + ); + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT, hook_hit_start); + return Ok(body); } if !cache_fill_allowed { @@ -3516,20 +3529,27 @@ impl DefaultObjectUsecase { // authoritative because it ran after fresh metadata resolution, so the // app layer skips its own lookup and only uses the plan to fill. if !cache_hook_probed { - match lookup_get_object_body_cache_hit(cache_adapter, &cache_plan).await { + let cache_lookup_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); + let cache_lookup = lookup_get_object_body_cache_hit(cache_adapter, &cache_plan).await; + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_LOOKUP, cache_lookup_start); + match cache_lookup { GetObjectBodyCacheLookup::Hit(bytes) => { - return Ok(Self::build_memory_bytes_blob( + let cache_hit_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); + let body = Self::build_memory_bytes_blob( bytes, response_content_length, GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE, lifecycle, - )); + ); + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT, cache_hit_start); + return Ok(body); } GetObjectBodyCacheLookup::Disabled | GetObjectBodyCacheLookup::Skip | GetObjectBodyCacheLookup::Miss => {} } } if let Some(buffered_body) = buffered_body { + let cache_fill_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); // ODC-15: the body is already fully in hand, so keep the fill off the // response's critical path. For a cacheable plan, run the fill in a // detached task (Bytes is a cheap clone) and return immediately. For @@ -3553,13 +3573,17 @@ impl DefaultObjectUsecase { } let _ = fill_get_object_body_cache_from_buffered_body(cache_adapter, &cache_plan, &buffered_body).await; } + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF, cache_fill_handoff_start); - return Ok(Self::build_memory_bytes_blob( + let buffered_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); + let body = Self::build_memory_bytes_blob( buffered_body, response_content_length, GET_MEMORY_BODY_SOURCE_BUFFERED_BODY, lifecycle, - )); + ); + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_BUFFERED_HANDOFF, buffered_handoff_start); + return Ok(body); } let should_materialize_for_cache = cache_adapter.materialize_fill_enabled() @@ -3613,6 +3637,7 @@ impl DefaultObjectUsecase { { Ok(buf) => { let bytes = Bytes::from(buf); + let cache_fill_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); // ODC-15: fill off the response's critical path (see the // buffered-body branch above). let cache_adapter = cache_adapter.clone(); @@ -3625,6 +3650,10 @@ impl DefaultObjectUsecase { } let _ = fill_get_object_body_cache_from_materialized_body(&cache_adapter, &cache_plan, &fill_bytes).await; }); + record_get_object_s3_handler_stage_duration( + GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF, + cache_fill_handoff_start, + ); return Ok(Self::build_memory_bytes_blob( bytes, @@ -3763,13 +3792,19 @@ impl DefaultObjectUsecase { None => helper, }; let helper = helper.version_id(version_id_for_event); + let cors_wrap_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let mut response = wrap_response_with_cors(bucket, method, headers, output).await; + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_CORS_WRAP, cors_wrap_start); + let header_inject_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); inject_accept_ranges_header(&mut response.headers); // Emit XXHash3/64/128 and SHA-512 checksums that s3s GetObjectOutput cannot // carry (#1257). This is the download-side integrity path AWS SDKs verify. inject_additional_checksum_headers(&mut response.headers, &extra_checksum_headers); + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_HEADER_INJECT, header_inject_start); let result = Ok(response); + let event_complete_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let _ = helper.complete(&result); + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_EVENT_COMPLETE, event_complete_start); result } @@ -3879,9 +3914,8 @@ impl DefaultObjectUsecase { let metadata_filter_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let metadata = filter_object_metadata(&info.user_defined); record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_METADATA_FILTER, metadata_filter_start); - + let output_struct_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let parts_count = Self::get_object_parts_count(part_number, &info); - let output = GetObjectOutput { body: Some(body), content_length: Some(response_content_length), @@ -3915,6 +3949,7 @@ impl DefaultObjectUsecase { storage_class, ..Default::default() }; + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_OUTPUT_STRUCT_BUILD, output_struct_start); Ok(GetObjectOutputContext { output, @@ -4217,6 +4252,7 @@ impl DefaultObjectUsecase { let manager = get_concurrency_manager(); + let prepare_read_start = stage_metrics_enabled.then(std::time::Instant::now); let mut prepared_read = self .prepare_get_object_read_execution( &req, @@ -4232,6 +4268,7 @@ impl DefaultObjectUsecase { object_traffic_health.clone(), ) .await; + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_PREPARE_READ_EXECUTION, prepare_read_start); // An object missing locally (and only missing — other errors keep // their semantics) may still be served by a remote copy. if let Err(err) = &prepared_read @@ -4275,6 +4312,7 @@ impl DefaultObjectUsecase { return Self::complete_get_object_error(helper.version_id(version_id_for_event), err); } Some(OdmGetOutcome::RetryLocal) => { + let prepare_read_retry_start = stage_metrics_enabled.then(std::time::Instant::now); prepared_read = self .prepare_get_object_read_execution( &req, @@ -4290,6 +4328,10 @@ impl DefaultObjectUsecase { object_traffic_health, ) .await; + record_get_object_s3_handler_stage_duration( + GET_OBJECT_STAGE_PREPARE_READ_EXECUTION, + prepare_read_retry_start, + ); } } } diff --git a/rustfs/src/app/object/put.rs b/rustfs/src/app/object/put.rs index 9bff39cf2..d182ab1bf 100644 --- a/rustfs/src/app/object/put.rs +++ b/rustfs/src/app/object/put.rs @@ -1788,6 +1788,7 @@ impl DefaultObjectUsecase { )?; } + let checksum_input_stage_start = put_stage_metrics_enabled.then(Instant::now); let mut md5hex = match content_md5 { Some(PutObjectContentMd5::Base64(base64_md5)) => { let md5 = base64_simd::STANDARD @@ -1800,12 +1801,15 @@ impl DefaultObjectUsecase { }; let mut sha256hex = get_content_sha256_with_query(headers, query); + rustfs_io_metrics::record_put_object_stage_duration_from("app_checksum_input_prepare", checksum_input_stage_start); let mut write_plan = WritePlan::new(); // Additional-checksum (XXHash3/64/128, SHA-512) values to echo on the PutObject // response (#1256); captured at want_checksum set points before opts is moved. let mut put_extra_checksum_headers: Vec<(&'static str, String)> = Vec::new(); + let reader_prepare_stage_start = put_stage_metrics_enabled.then(Instant::now); let mut reader = if should_compress { + let compressed_reader_stage_start = put_stage_metrics_enabled.then(Instant::now); let body = tokio::io::BufReader::with_capacity( buffer_size, StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))), @@ -1828,51 +1832,98 @@ impl DefaultObjectUsecase { size = HashReader::SIZE_PRESERVE_LAYER; write_plan = write_plan.with_compression(algorithm); + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_reader_prepare_compressed", + compressed_reader_stage_start, + ); hrd } else { if use_zero_copy_eager_put_path { let zero_copy_start = std::time::Instant::now(); + let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now); let eager_body = read_zero_copy_put_body_exact(body, body_size as usize).await?; + rustfs_io_metrics::record_put_object_stage_duration_from("app_body_eager_read_zero_copy", eager_read_stage_start); rustfs_io_metrics::record_zero_copy_write(body_size as usize, zero_copy_start.elapsed().as_secs_f64() * 1000.0); - HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? + let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now); + let reader = + HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_hash_reader_setup_zero_copy", + hash_reader_stage_start, + ); + reader } else if use_empty_or_small_eager_put_path { if (body_size as usize) <= POOL_BYPASS_MAX_SIZE { // Bypass BytesPool for very small objects to avoid Small-tier // Mutex contention under high concurrency. Direct allocation // for ≤4KiB is negligible cost. + let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now); let eager_body = read_small_put_body_exact_direct( StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))), body_size as usize, ) .await?; - HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_body_eager_read_small_direct", + eager_read_stage_start, + ); + let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now); + let reader = HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false) + .map_err(ApiError::from)?; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_hash_reader_setup_small_direct", + hash_reader_stage_start, + ); + reader } else { let pool = get_concurrency_manager().bytes_pool(); + let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now); let eager_body = read_small_put_body_exact_pooled( StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))), body_size as usize, pool.as_ref(), ) .await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_body_eager_read_small_pooled", + eager_read_stage_start, + ); + let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now); let eager_reader = PooledBufferReader::new(eager_body, body_size as usize); - HashReader::from_stream(eager_reader, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? + let reader = HashReader::from_stream(eager_reader, size, actual_size, md5hex, sha256hex, false) + .map_err(ApiError::from)?; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_hash_reader_setup_small_pooled", + hash_reader_stage_start, + ); + reader } } else { + let streaming_reader_stage_start = put_stage_metrics_enabled.then(Instant::now); let body = tokio::io::BufReader::with_capacity( buffer_size, StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))), ); - HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? + let reader = + HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_reader_prepare_streaming", + streaming_reader_stage_start, + ); + reader } }; + rustfs_io_metrics::record_put_object_stage_duration_from("app_reader_prepare", reader_prepare_stage_start); if size >= 0 { + let checksum_setup_stage_start = put_stage_metrics_enabled.then(Instant::now); if let Err(err) = reader.add_checksum(headers, trailer_source(trailing_headers.clone()), false) { return Err(ApiError::from(err).into()); } opts.want_checksum = reader.checksum(); put_extra_checksum_headers = additional_checksum_echo_pairs(&opts.want_checksum); + rustfs_io_metrics::record_put_object_stage_duration_from("app_reader_checksum_setup", checksum_setup_stage_start); } rustfs_io_metrics::record_put_object_path(put_path); rustfs_io_metrics::record_put_object_stage_duration_from("ingress_prepare", ingress_stage_start); @@ -2113,17 +2164,36 @@ impl DefaultObjectUsecase { wait_for_put_post_store_test_hook(&bucket).await; let post_store_stage_start = put_stage_metrics_enabled.then(Instant::now); + let transition_enqueue_stage_start = put_stage_metrics_enabled.then(Instant::now); maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await; - let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_post_store_transition_enqueue", + transition_enqueue_stage_start, + ); + let cache_invalidate_stage_start = put_stage_metrics_enabled.then(Instant::now); + let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_post_store_data_cache_invalidate", + cache_invalidate_stage_start, + ); + + let versioning_lookup_stage_start = put_stage_metrics_enabled.then(Instant::now); let put_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_post_store_versioning_lookup", + versioning_lookup_stage_start, + ); // Fast in-memory update for immediate quota and admin usage consistency. // The previous current size comes from the prelookup when it ran, // otherwise from the rename_data backfill (rustfs/backlog#1009); the // backfill reproduces the lookup's observation bit for bit (latest // version's ObjectInfo.size — 0 for a delete-marker latest — or // not-found → None). + let quota_size_stage_start = put_stage_metrics_enabled.then(Instant::now); let committed_size = quota_accounting_object_size(&obj_info, quota_enabled)?; + rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_quota_size", quota_size_stage_start); + let usage_memory_stage_start = put_stage_metrics_enabled.then(Instant::now); match prelookup_previous_current_size.or_else(|| previous_current_size_from_backfill(backfilled_old_current_size)) { Some(previous_current_size) => { @@ -2149,16 +2219,27 @@ impl DefaultObjectUsecase { record_bucket_object_write_unknown_previous_memory(&bucket, committed_size, put_versioned).await; } } + rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_usage_memory", usage_memory_stage_start); if dsc.replicate_any() { + let replication_schedule_stage_start = put_stage_metrics_enabled.then(Instant::now); schedule_object_replication(obj_info.clone(), store, dsc).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_post_store_replication_schedule", + replication_schedule_stage_start, + ); } + let dirty_usage_stage_start = put_stage_metrics_enabled.then(Instant::now); rustfs_scanner::record_dirty_usage_object_from_producer( &bucket, &key, rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject, ); + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_post_store_dirty_usage_record", + dirty_usage_stage_start, + ); rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_bookkeeping", post_store_stage_start); let capacity_update_stage_start = put_stage_metrics_enabled.then(Instant::now); diff --git a/rustfs/src/connect/report_bundle.rs b/rustfs/src/connect/report_bundle.rs index 890629ac2..454626261 100644 --- a/rustfs/src/connect/report_bundle.rs +++ b/rustfs/src/connect/report_bundle.rs @@ -76,7 +76,11 @@ pub(crate) fn upload_source( Err(_) => return original_source(path, generated_bundle_uid), }; let names = (0..archive.len()) - .map(|index| archive.by_index(index).map(|entry| entry.name().to_owned())) + .map(|index| { + archive + .by_index(index) + .and_then(|entry| entry.name().map(|name| name.into_owned())) + }) .collect::, _>>()?; if names diff --git a/rustfs/src/server/hybrid.rs b/rustfs/src/server/hybrid.rs index c20d47215..79686946e 100644 --- a/rustfs/src/server/hybrid.rs +++ b/rustfs/src/server/hybrid.rs @@ -19,6 +19,7 @@ use hyper::{Request, Response}; use pin_project_lite::pin_project; use std::pin::Pin; use std::task::{Context, Poll}; +use std::time::Instant; use tower::Service; type BoxError = Box; @@ -35,6 +36,13 @@ pub(crate) fn is_grpc_request(req: &Request) -> bool { ) } +fn rename_data_grpc_stage(path: &str) -> bool { + matches!( + path, + "/node_service.NodeService/RenameData" | "/node_service.NodeService/RenameDataAtIncarnation" + ) +} + /// The service that can serve both gRPC and REST HTTP Requests #[derive(Clone)] pub struct HybridService { @@ -71,8 +79,11 @@ where /// as a REST request fn call(&mut self, req: Request) -> Self::Future { if is_grpc_request(&req) { + let observe_rename_data = rustfs_io_metrics::put_stage_metrics_enabled() && rename_data_grpc_stage(req.uri().path()); HybridFuture::Grpc { grpc_future: self.grpc.call(req), + observe_rename_data, + service_future_started: observe_rename_data.then(Instant::now), } } else { HybridFuture::Rest { @@ -93,7 +104,10 @@ pin_project! { }, Grpc { #[pin] - grpc_body: GrpcBody + grpc_body: GrpcBody, + first_frame_started: Option, + body_started: Option, + first_frame_recorded: bool, }, } } @@ -123,21 +137,44 @@ where fn is_end_stream(&self) -> bool { match self { Self::Rest { rest_body } => rest_body.is_end_stream(), - Self::Grpc { grpc_body } => grpc_body.is_end_stream(), + Self::Grpc { grpc_body, .. } => grpc_body.is_end_stream(), } } fn poll_frame(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll, Self::Error>>> { match self.project() { HybridBodyProj::Rest { rest_body } => rest_body.poll_frame(cx).map_err(Into::into), - HybridBodyProj::Grpc { grpc_body } => grpc_body.poll_frame(cx).map_err(Into::into), + HybridBodyProj::Grpc { + mut grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + } => { + let poll = grpc_body.as_mut().poll_frame(cx).map_err(Into::into); + if !*first_frame_recorded && let Poll::Ready(_) = &poll { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME, + first_frame_started.take(), + ); + *first_frame_recorded = true; + } + let body_complete = matches!(&poll, Poll::Ready(None)) + || (matches!(&poll, Poll::Ready(Some(Ok(_)))) && grpc_body.as_ref().get_ref().is_end_stream()); + if body_complete { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE, + body_started.take(), + ); + } + poll + } } } fn size_hint(&self) -> http_body::SizeHint { match self { Self::Rest { rest_body } => rest_body.size_hint(), - Self::Grpc { grpc_body } => grpc_body.size_hint(), + Self::Grpc { grpc_body, .. } => grpc_body.size_hint(), } } } @@ -154,6 +191,8 @@ pin_project! { Grpc { #[pin] grpc_future: GrpcFuture, + observe_rename_data: bool, + service_future_started: Option, }, } } @@ -174,8 +213,27 @@ where Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())), Poll::Pending => Poll::Pending, }, - HybridFutureProj::Grpc { grpc_future } => match grpc_future.poll(cx) { - Poll::Ready(Ok(res)) => Poll::Ready(Ok(res.map(|grpc_body| HybridBody::Grpc { grpc_body }))), + HybridFutureProj::Grpc { + grpc_future, + observe_rename_data, + service_future_started, + } => match grpc_future.poll(cx) { + Poll::Ready(Ok(res)) => { + if *observe_rename_data { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE, + service_future_started.take(), + ); + } + let first_frame_started = (*observe_rename_data).then(Instant::now); + let body_started = (*observe_rename_data).then(Instant::now); + Poll::Ready(Ok(res.map(|grpc_body| HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded: !*observe_rename_data, + }))) + } Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())), Poll::Pending => Poll::Pending, }, diff --git a/rustfs/src/server/layer.rs b/rustfs/src/server/layer.rs index cff0e72a8..1cbbeefbc 100644 --- a/rustfs/src/server/layer.rs +++ b/rustfs/src/server/layer.rs @@ -863,7 +863,20 @@ where } Response::from_parts(parts, HybridBody::Rest { rest_body }) } - HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }), + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + } => Response::from_parts( + parts, + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + }, + ), }; Ok(response) @@ -1012,7 +1025,20 @@ where } Response::from_parts(parts, HybridBody::Rest { rest_body }) } - HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }), + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + } => Response::from_parts( + parts, + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + }, + ), }; Ok(response) @@ -1163,7 +1189,20 @@ where Response::from_parts(parts, HybridBody::Rest { rest_body }) } - HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }), + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + } => Response::from_parts( + parts, + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + }, + ), }; Ok(response) @@ -1234,7 +1273,20 @@ where }, ) } - HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }), + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + } => Response::from_parts( + parts, + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + }, + ), }; Ok(response) @@ -1302,7 +1354,20 @@ where rest_body: RestBody::from(Bytes::new()), }, ), - HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }), + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + } => Response::from_parts( + parts, + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + }, + ), }; Ok(response) diff --git a/rustfs/src/startup_observability.rs b/rustfs/src/startup_observability.rs index 01ec050b8..114b1a808 100644 --- a/rustfs/src/startup_observability.rs +++ b/rustfs/src/startup_observability.rs @@ -13,12 +13,16 @@ // limitations under the License. use std::sync::Arc; +use std::time::Duration; use crate::init::{init_auto_tuner, init_update_check, print_server_info}; use crate::startup_runtime_sources; use crate::storage_api::startup::storage::{ECStore, init_compression_total_memory_from_backend}; +use tokio::io::AsyncWriteExt; use tokio_util::sync::CancellationToken; +const GET_STAGE_LOCAL_SUMMARY_FILE: &str = "/var/log/rustfs/get-stage-summary.log"; + pub(crate) async fn init_observability_runtime(store: Arc, ctx: CancellationToken) { print_server_info(); init_update_check(); @@ -27,6 +31,10 @@ pub(crate) async fn init_observability_runtime(store: Arc, ctx: Cancell let metrics_enabled = startup_runtime_sources::observability_metric_enabled(); configure_metric_gates(metrics_enabled); + if rustfs_io_metrics::get_stage_local_summary_enabled() { + init_get_stage_local_summary_runtime(ctx.clone()); + } + if metrics_enabled { // Load persisted compression stats into memory early, before any PUTs can occur. init_compression_total_memory_from_backend(store).await; @@ -37,21 +45,115 @@ pub(crate) async fn init_observability_runtime(store: Arc, ctx: Cancell } fn configure_metric_gates(metrics_enabled: bool) { + let local_summary_enabled = !metrics_enabled + && rustfs_utils::get_env_bool(rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED, false); + let sample_rate = + rustfs_utils::get_env_u64(rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE, 64).max(1); + startup_runtime_sources::set_get_stage_local_summary_enabled(local_summary_enabled); + startup_runtime_sources::set_get_stage_local_summary_sample_rate(sample_rate); let put_stage_metrics_enabled = metrics_enabled && rustfs_utils::get_env_bool( rustfs_config::observability::ENV_OBS_PUT_STAGE_METRICS_ENABLED, rustfs_config::DEFAULT_OBS_PUT_STAGE_METRICS_ENABLED, ); startup_runtime_sources::set_put_stage_metrics_enabled(put_stage_metrics_enabled); - startup_runtime_sources::set_get_stage_metrics_enabled(metrics_enabled); + startup_runtime_sources::set_get_stage_metrics_enabled(metrics_enabled || local_summary_enabled); startup_runtime_sources::set_metrics_enabled(metrics_enabled); } +fn init_get_stage_local_summary_runtime(ctx: CancellationToken) { + let interval_secs = + rustfs_utils::get_env_u64(rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS, 15).max(1); + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); + interval.tick().await; + loop { + tokio::select! { + _ = ctx.cancelled() => break, + _ = interval.tick() => { + let rows = rustfs_io_metrics::take_get_stage_local_summary(); + if rows.is_empty() { + continue; + } + let summary = rows.iter().map(|row| { + format!("{}|{}|{}|{}|{}|{}", row.path, row.stage, row.object_class, row.size_bucket, row.sampled_count, row.duration_nanoseconds) + }).collect::>().join(";"); + let timestamp_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + let line = format!( + "timestamp_ms={timestamp_ms}\tsample_rate={}\tstage_count={}\tstages={summary}\n", + rustfs_io_metrics::get_stage_local_summary_sample_rate(), + rows.len(), + ); + let Ok(mut file) = tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(GET_STAGE_LOCAL_SUMMARY_FILE) + .await + else { + break; + }; + if file.write_all(line.as_bytes()).await.is_err() { + break; + } + } + } + } + }); +} + #[cfg(test)] mod tests { use super::*; const PUT_STAGE_ENV: &str = rustfs_config::observability::ENV_OBS_PUT_STAGE_METRICS_ENABLED; + const LOCAL_SUMMARY_ENV: &str = rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED; + const LOCAL_SUMMARY_RATE_ENV: &str = rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE; + + #[test] + #[serial_test::serial] + fn get_stage_local_summary_can_run_without_metrics_export() { + let previous_metrics = rustfs_io_metrics::metrics_enabled(); + let previous_get_stages = rustfs_io_metrics::get_stage_metrics_enabled(); + let previous_local_summary = rustfs_io_metrics::get_stage_local_summary_enabled(); + let previous_sample_rate = rustfs_io_metrics::get_stage_local_summary_sample_rate(); + + temp_env::with_var(LOCAL_SUMMARY_ENV, Some("true"), || { + temp_env::with_var(LOCAL_SUMMARY_RATE_ENV, Some("1"), || { + configure_metric_gates(false); + assert!(!rustfs_io_metrics::metrics_enabled()); + assert!(rustfs_io_metrics::get_stage_metrics_enabled()); + assert!(rustfs_io_metrics::get_stage_local_summary_enabled()); + rustfs_io_metrics::take_get_stage_local_summary(); + + rustfs_io_metrics::record_get_object_stage_duration("legacy_duplex", "metadata", 0.001); + rustfs_io_metrics::record_get_object_stage_duration("legacy_duplex", "metadata", 0.003); + rustfs_io_metrics::record_get_object_stage_duration_by_size( + "legacy_duplex", + "inline_direct.decode", + "plain_single_part", + "le_4kib", + 0.002, + ); + + let rows = rustfs_io_metrics::take_get_stage_local_summary(); + let metadata = rows + .iter() + .find(|row| row.path == "legacy_duplex" && row.stage == "metadata") + .expect("local summary should retain sampled metadata observations"); + assert_eq!(metadata.sampled_count, 2); + assert_eq!(metadata.duration_nanoseconds, 4_000_000); + assert!(rows.iter().any(|row| row.size_bucket == "le_4kib")); + }); + }); + + startup_runtime_sources::set_metrics_enabled(previous_metrics); + startup_runtime_sources::set_get_stage_metrics_enabled(previous_get_stages); + startup_runtime_sources::set_get_stage_local_summary_enabled(previous_local_summary); + startup_runtime_sources::set_get_stage_local_summary_sample_rate(previous_sample_rate); + } #[test] #[serial_test::serial] diff --git a/rustfs/src/startup_runtime_sources.rs b/rustfs/src/startup_runtime_sources.rs index 73501a592..f69546688 100644 --- a/rustfs/src/startup_runtime_sources.rs +++ b/rustfs/src/startup_runtime_sources.rs @@ -99,6 +99,14 @@ pub(crate) fn set_get_stage_metrics_enabled(enabled: bool) { rustfs_io_metrics::set_get_stage_metrics_enabled(enabled); } +pub(crate) fn set_get_stage_local_summary_enabled(enabled: bool) { + rustfs_io_metrics::set_get_stage_local_summary_enabled(enabled); +} + +pub(crate) fn set_get_stage_local_summary_sample_rate(sample_rate: u64) { + rustfs_io_metrics::set_get_stage_local_summary_sample_rate(sample_rate); +} + pub(crate) fn set_metrics_enabled(enabled: bool) { rustfs_io_metrics::set_metrics_enabled(enabled); } diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index 22c1a88c1..c11c3d350 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -1365,6 +1365,8 @@ impl NodeService { &self, request: Request, ) -> Result, Status> { + let service_return_started = rustfs_io_metrics::put_stage_timer(); + let service_entry_started = rustfs_io_metrics::put_stage_timer(); if !request.get_ref().scanner_publication_lease_token.is_empty() || !request.get_ref().bucket_incarnation_id.is_empty() { let has_body_digest = request .metadata() @@ -1372,14 +1374,37 @@ impl NodeService { .and_then(|value| value.to_str().ok()) .is_some_and(|value| value != "UNSIGNED-PAYLOAD"); if !has_body_digest { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY, + service_entry_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN, + service_return_started, + ); return Err(Status::permission_denied("scanner publication lease rename requires a body-bound digest")); } } - verify_disk_mutation_digest( + if let Err(err) = verify_disk_mutation_digest( &request, rustfs_protos::canonical_rename_data_request_body(request.get_ref()), "rename_data", - )?; + ) { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY, + service_entry_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN, + service_return_started, + ); + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY, + service_entry_started, + ); + let before_handler_started = rustfs_io_metrics::put_stage_timer(); let request = request.into_inner(); let target = self.local_mutation_target(); let bucket_incarnation = if request.bucket_incarnation_id.is_empty() { @@ -1390,6 +1415,14 @@ impl NodeService { .filter(|id| !id.is_nil()) .ok_or_else(|| Status::invalid_argument("bucket incarnation must be a non-nil UUID"))?; if !request.scanner_publication_lease_token.is_empty() { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER, + before_handler_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN, + service_return_started, + ); return Err(Status::invalid_argument( "heal incarnation and scanner publication lease cannot be combined", )); @@ -1401,12 +1434,26 @@ impl NodeService { let decoded_file_info = match decode_rename_data_request_file_info(&request.file_info_bin, &request.file_info) { Ok(file_info) => file_info, Err(err) => { - return Ok(Response::new(RenameDataResponse { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER, + before_handler_started, + ); + let response_build_started = rustfs_io_metrics::put_stage_timer(); + let response = Response::new(RenameDataResponse { success: false, rename_data_resp: String::new(), rename_data_resp_bin: Vec::new().into(), error: Some(DiskError::other(format!("decode FileInfo failed: {err}")).into()), - })); + }); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD, + response_build_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN, + service_return_started, + ); + return Ok(response); } }; let scanner_publication_lease_token = if request.scanner_publication_lease_token.is_empty() { @@ -1415,6 +1462,14 @@ impl NodeService { let token = Uuid::from_slice(&request.scanner_publication_lease_token) .map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?; if token.is_nil() { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER, + before_handler_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN, + service_return_started, + ); return Err(Status::invalid_argument("scanner publication lease token must not be nil")); } Some(token) @@ -1422,6 +1477,11 @@ impl NodeService { let request_decoded_from_msgpack = decoded_file_info.from_msgpack; #[cfg(feature = "e2e-test-hooks")] let observation = startup_cas_rename_observation(&target, &request, &decoded_file_info.value); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER, + before_handler_started, + ); + let handler_await_started = rustfs_io_metrics::put_stage_timer(); let result = target .rename_local_data( &request.disk, @@ -1432,6 +1492,11 @@ impl NodeService { bucket_incarnation, ) .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT, + handler_await_started, + ); + let after_handler_started = rustfs_io_metrics::put_stage_timer(); #[cfg(feature = "e2e-test-hooks")] if let Some(mut observation) = observation { observation["ok"] = serde_json::json!(result.is_ok()); @@ -1439,28 +1504,47 @@ impl NodeService { let line = format!("RUSTFS_E2E_STARTUP_CAS {observation}\n"); let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes()); } - match result { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER, + after_handler_started, + ); + let response_build_started = rustfs_io_metrics::put_stage_timer(); + let response_encode_started = rustfs_io_metrics::put_stage_timer(); + let response = match result { Ok(rename_data_resp) => match encode_rename_data_response_payloads(&rename_data_resp, request_decoded_from_msgpack) { - Ok((rename_data_resp, rename_data_resp_bin)) => Ok(Response::new(RenameDataResponse { + Ok((rename_data_resp, rename_data_resp_bin)) => Response::new(RenameDataResponse { success: true, rename_data_resp, rename_data_resp_bin: rename_data_resp_bin.into(), error: None, - })), - Err(err) => Ok(Response::new(RenameDataResponse { + }), + Err(err) => Response::new(RenameDataResponse { success: false, rename_data_resp: String::new(), rename_data_resp_bin: Vec::new().into(), error: Some(err.into()), - })), + }), }, - Err(err) => Ok(Response::new(RenameDataResponse { + Err(err) => Response::new(RenameDataResponse { success: false, rename_data_resp: String::new(), rename_data_resp_bin: Vec::new().into(), error: Some(err.into()), - })), - } + }), + }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE, + response_encode_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD, + response_build_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN, + service_return_started, + ); + Ok(response) } pub(super) async fn handle_list_dir(&self, request: Request) -> Result, Status> { diff --git a/rustfs/src/storage/timeout_wrapper.rs b/rustfs/src/storage/timeout_wrapper.rs index f0ca0f882..94f0bed24 100644 --- a/rustfs/src/storage/timeout_wrapper.rs +++ b/rustfs/src/storage/timeout_wrapper.rs @@ -412,21 +412,21 @@ impl RequestTimeoutWrapper { debug!(timeout_secs = timeout_duration.as_secs(), "Starting timed operation"); - rustfs_io_metrics::record_get_object_request_started(); + rustfs_io_metrics::record_get_object_timeout_operation_start(); let cancel_token_for_op = self.cancel_token.clone(); match tokio::time::timeout(timeout_duration, operation(cancel_token_for_op)).await { Ok(Ok(result)) => { let elapsed = self.elapsed(); - rustfs_io_metrics::record_get_object_request_result("success", elapsed.as_secs_f64()); + rustfs_io_metrics::record_get_object_timeout_operation_result("success", elapsed.as_secs_f64()); debug!(elapsed_ms = elapsed.as_millis(), "Operation completed successfully"); TimedGetObjectResult::Success(result) } Ok(Err(e)) => { let elapsed = self.elapsed(); - rustfs_io_metrics::record_get_object_request_result("error", elapsed.as_secs_f64()); + rustfs_io_metrics::record_get_object_timeout_operation_result("error", elapsed.as_secs_f64()); debug!(elapsed_ms = elapsed.as_millis(), "Operation failed with error"); TimedGetObjectResult::Error(e) @@ -436,7 +436,7 @@ impl RequestTimeoutWrapper { self.cancel_token.cancel(); rustfs_io_metrics::record_get_object_timeout(None, Some(elapsed.as_secs_f64())); - rustfs_io_metrics::record_get_object_request_result("timeout", elapsed.as_secs_f64()); + rustfs_io_metrics::record_get_object_timeout_operation_result("timeout", elapsed.as_secs_f64()); warn!( timeout_secs = timeout_duration.as_secs(), diff --git a/rustfs/tests/connect_offline_bundle.rs b/rustfs/tests/connect_offline_bundle.rs index 9146e25b1..88cb394bf 100644 --- a/rustfs/tests/connect_offline_bundle.rs +++ b/rustfs/tests/connect_offline_bundle.rs @@ -155,7 +155,7 @@ fn connect_offline_bundle_is_deterministic_bounded_and_signed_over_exact_manifes assert_eq!(archive.len(), PATHS.len()); for (index, expected_path) in PATHS.into_iter().enumerate() { let file = archive.by_index(index).expect("archive member"); - assert_eq!(file.name(), expected_path); + assert_eq!(file.name().expect("archive entry name"), expected_path); assert_eq!(file.compression(), CompressionMethod::Stored); assert_eq!(file.last_modified(), Some(zip::DateTime::DEFAULT)); assert_eq!(file.unix_mode().map(|mode| mode & 0o777), Some(0o600)); diff --git a/rustfs/tests/connect_perf_drive.rs b/rustfs/tests/connect_perf_drive.rs index 43a3ba919..a3c88ec81 100644 --- a/rustfs/tests/connect_perf_drive.rs +++ b/rustfs/tests/connect_perf_drive.rs @@ -389,9 +389,15 @@ async fn successful_result_has_signed_bounded_private_offline_export() { let mut archive = zip::ZipArchive::new(Cursor::new(&export.archive_bytes)).expect("three-file archive"); assert_eq!(archive.len(), 3); - assert_eq!(archive.by_index(0).expect("envelope").name(), "envelope.json"); - assert_eq!(archive.by_index(1).expect("signature").name(), "envelope.sig"); - assert_eq!(archive.by_index(2).expect("result").name(), "result.json"); + assert_eq!( + archive.by_index(0).expect("envelope").name().expect("archive entry name"), + "envelope.json" + ); + assert_eq!( + archive.by_index(1).expect("signature").name().expect("archive entry name"), + "envelope.sig" + ); + assert_eq!(archive.by_index(2).expect("result").name().expect("archive entry name"), "result.json"); let envelope: serde_json::Value = serde_json::from_slice(&export.envelope_json).expect("envelope JSON"); assert_eq!(envelope["toolId"], "performance.drive"); @@ -535,7 +541,14 @@ fn production_cli_measures_and_signs_exact_binary_provenance() { let archive_bytes = fs::read(&output).expect("saved archive"); let mut archive = zip::ZipArchive::new(Cursor::new(archive_bytes.as_slice())).expect("signed archive"); let names = (0..archive.len()) - .map(|index| archive.by_index(index).expect("archive member").name().to_owned()) + .map(|index| { + archive + .by_index(index) + .expect("archive member") + .name() + .expect("archive entry name") + .into_owned() + }) .collect::>(); assert_eq!(names, ["envelope.json", "envelope.sig", "result.json"]); let envelope_bytes = read_archive_member(&mut archive, "envelope.json"); diff --git a/rustfs/tests/connect_perf_network.rs b/rustfs/tests/connect_perf_network.rs index 4136a9ca8..6ada066b8 100644 --- a/rustfs/tests/connect_perf_network.rs +++ b/rustfs/tests/connect_perf_network.rs @@ -669,9 +669,26 @@ async fn successful_result_has_signed_bounded_private_offline_export() { assert_eq!(export.archive_sha256, hex(&Sha256::digest(&export.archive_bytes))); let mut archive = zip::ZipArchive::new(Cursor::new(&export.archive_bytes)).expect("three-file archive"); assert_eq!(archive.len(), 3); - assert_eq!(archive.by_index(0).expect("envelope entry").name(), "envelope.json"); - assert_eq!(archive.by_index(1).expect("signature entry").name(), "envelope.sig"); - assert_eq!(archive.by_index(2).expect("result entry").name(), "result.json"); + assert_eq!( + archive + .by_index(0) + .expect("envelope entry") + .name() + .expect("archive entry name"), + "envelope.json" + ); + assert_eq!( + archive + .by_index(1) + .expect("signature entry") + .name() + .expect("archive entry name"), + "envelope.sig" + ); + assert_eq!( + archive.by_index(2).expect("result entry").name().expect("archive entry name"), + "result.json" + ); let envelope: serde_json::Value = serde_json::from_slice(&export.envelope_json).expect("envelope JSON"); assert_eq!(envelope["toolId"], "performance.network"); diff --git a/rustfs/tests/connect_top_net.rs b/rustfs/tests/connect_top_net.rs index 294cd3642..0da443a39 100644 --- a/rustfs/tests/connect_top_net.rs +++ b/rustfs/tests/connect_top_net.rs @@ -399,7 +399,14 @@ fn production_cli_exports_top_net_and_fails_closed_for_unavailable_and_invalid_r assert_eq!(fs::metadata(&net_output).expect("output metadata").permissions().mode() & 0o777, 0o600); let mut archive = ZipArchive::new(Cursor::new(bytes)).expect("top.net archive"); let names = (0..archive.len()) - .map(|index| archive.by_index(index).expect("archive member").name().to_owned()) + .map(|index| { + archive + .by_index(index) + .expect("archive member") + .name() + .expect("archive entry name") + .into_owned() + }) .collect::>(); assert_eq!(names, ["envelope.json", "envelope.sig", "result.json"]); From bd83bd3f6b1f1821e0549808fa722e3d9cc8c4fb Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Tue, 13 Oct 2026 17:12:00 +0800 Subject: [PATCH 040/143] fix(table-catalog): allow commits after object-backed rename (#8508) --- .../src/admin/handlers/table_catalog/tests.rs | 259 ++++++++++++++++-- rustfs/src/table_catalog/store/object.rs | 2 +- 2 files changed, 241 insertions(+), 20 deletions(-) diff --git a/rustfs/src/admin/handlers/table_catalog/tests.rs b/rustfs/src/admin/handlers/table_catalog/tests.rs index 9ce007d95..56b4f2a6d 100644 --- a/rustfs/src/admin/handlers/table_catalog/tests.rs +++ b/rustfs/src/admin/handlers/table_catalog/tests.rs @@ -4211,24 +4211,46 @@ fn table_metadata_file_name_scoping_is_bounded_and_identity_sensitive() { #[tokio::test] async fn renamed_and_recreated_tables_with_the_same_commit_id_use_disjoint_metadata_files() { + assert_renamed_and_recreated_table_commits(crate::table_catalog::TableCatalogBackingMode::DurableStrong, "curated").await; +} + +#[tokio::test] +async fn object_catalog_cross_namespace_rename_accepts_isolated_standard_commits() { + assert_renamed_and_recreated_table_commits(crate::table_catalog::TableCatalogBackingMode::ObjectBacked, "curated").await; +} + +#[tokio::test] +async fn object_catalog_same_namespace_rename_accepts_isolated_standard_commits() { + assert_renamed_and_recreated_table_commits(crate::table_catalog::TableCatalogBackingMode::ObjectBacked, "analytics").await; +} + +#[tokio::test] +async fn strong_catalog_same_namespace_rename_accepts_isolated_standard_commits() { + assert_renamed_and_recreated_table_commits(crate::table_catalog::TableCatalogBackingMode::DurableStrong, "analytics").await; +} + +async fn assert_renamed_and_recreated_table_commits(mode: crate::table_catalog::TableCatalogBackingMode, destination: &str) { let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); - let store = crate::table_catalog::StrongTableCatalogStore::new(metadata_backend.clone()); + let store = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(metadata_backend.clone(), mode); let source_namespace = crate::table_catalog::Namespace::parse("analytics").expect("source namespace should parse"); - let destination_namespace = crate::table_catalog::Namespace::parse("curated").expect("destination namespace should parse"); + let destination_namespace = crate::table_catalog::Namespace::parse(destination).expect("destination namespace should parse"); create_standard_events_table(&store, &metadata_backend, &source_namespace).await; - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["curated".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("destination namespace should be created"); + let original = store.load_table("warehouse", "analytics", "events").await.unwrap().unwrap(); + if source_namespace != destination_namespace { + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec![destination.to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("destination namespace should be created"); + } store - .rename_table("warehouse", "analytics", "events", "curated", "events_v2") + .rename_table("warehouse", "analytics", "events", destination, "events_v2") .await .expect("table should rename"); @@ -4249,7 +4271,7 @@ async fn renamed_and_recreated_tables_with_the_same_commit_id_use_disjoint_metad .expect("source identifier should be reusable"); let renamed = store - .load_table("warehouse", "curated", "events_v2") + .load_table("warehouse", destination, "events_v2") .await .expect("renamed table lookup should succeed") .expect("renamed table should exist"); @@ -4259,18 +4281,196 @@ async fn renamed_and_recreated_tables_with_the_same_commit_id_use_disjoint_metad .expect("recreated table lookup should succeed") .expect("recreated table should exist"); assert_ne!(renamed.table_id, recreated.table_id); + assert_ne!(renamed.table_uuid, recreated.table_uuid); + assert_ne!(renamed.warehouse_location, recreated.warehouse_location); + assert_eq!(renamed.table_id, original.table_id); + assert_eq!(renamed.table_uuid, original.table_uuid); + assert_eq!(renamed.warehouse_location, original.warehouse_location); + assert_eq!(renamed.metadata_location, original.metadata_location); + assert_eq!(renamed.version_token, original.version_token); + assert_eq!(renamed.generation, original.generation); let commit_id = "11111111-1111-4111-8111-111111111111"; + let renamed_request = || { + let mut request = standard_property_commit_request(commit_id, &renamed.table_uuid, "curated"); + request.idempotency_key = Some("renamed-request".to_string()); + request + }; let renamed_commit = standard_commit_table_response( &store, &trusted_table_commit_backend(&metadata_backend), "warehouse", &destination_namespace, "events_v2", - standard_property_commit_request(commit_id, &renamed.table_uuid, "curated"), + renamed_request(), ) .await .expect("renamed table commit should succeed"); + let committed = store + .load_table("warehouse", destination, "events_v2") + .await + .unwrap() + .unwrap(); + assert_eq!(committed.table_id, renamed.table_id); + assert_eq!(committed.table_uuid, renamed.table_uuid); + assert_eq!(committed.warehouse_location, renamed.warehouse_location); + assert_eq!(committed.generation, renamed.generation + 1); + assert_ne!(committed.version_token, renamed.version_token); + assert_ne!(committed.metadata_location, renamed.metadata_location); + assert_eq!(renamed_commit.metadata["table-uuid"], renamed.table_uuid); + assert_eq!(renamed_commit.metadata["location"], renamed.warehouse_location); + assert_eq!(renamed_commit.metadata["properties"]["owner"], "curated"); + let receipt = store + .get_commit_by_id("warehouse", &renamed.table_id, commit_id) + .await + .unwrap() + .unwrap(); + assert_eq!(receipt.status, crate::table_catalog::CommitLogStatus::Committed); + assert_eq!(receipt.new_version_token, committed.version_token); + assert_eq!(receipt.new_metadata_location, committed.metadata_location); + assert_eq!( + store + .get_commit_by_idempotency_key("warehouse", &renamed.table_id, receipt.idempotency_key.as_deref().unwrap()) + .await + .unwrap(), + Some(receipt.clone()) + ); + let replay = standard_commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &destination_namespace, + "events_v2", + renamed_request(), + ) + .await + .expect("same commit ID should replay after rename"); + assert_eq!(replay.metadata_location, renamed_commit.metadata_location); + assert_eq!(replay.metadata, renamed_commit.metadata); + assert_eq!( + store.load_table("warehouse", destination, "events_v2").await.unwrap(), + Some(committed.clone()) + ); + + let stale_request = crate::table_catalog::TableCommitRequest { + table_bucket: "warehouse".to_string(), + namespace: destination.to_string(), + table: "events_v2".to_string(), + commit_id: "stale-after-rename".to_string(), + idempotency_key: Some("stale-after-rename".to_string()), + operation: "append".to_string(), + expected_version_token: renamed.version_token.clone(), + expected_metadata_location: renamed.metadata_location.clone(), + new_metadata_location: committed.metadata_location.clone(), + requirements: Vec::new(), + writer: None, + }; + for metadata_only in [false, true] { + let mut request = stale_request.clone(); + if metadata_only { + request.expected_version_token = committed.version_token.clone(); + request.commit_id = "stale-metadata-after-rename".to_string(); + request.idempotency_key = Some(request.commit_id.clone()); + } + assert!(matches!( + store.commit_table(request.clone()).await, + Err(crate::table_catalog::TableCatalogStoreError::Conflict(_)) + )); + assert!( + store + .get_commit_by_id("warehouse", &renamed.table_id, &request.commit_id) + .await + .unwrap() + .is_none() + ); + assert!( + store + .get_commit_by_idempotency_key("warehouse", &renamed.table_id, request.idempotency_key.as_deref().unwrap()) + .await + .unwrap() + .is_none() + ); + assert_eq!( + store.load_table("warehouse", destination, "events_v2").await.unwrap(), + Some(committed.clone()) + ); + } + let current_namespace_path = crate::table_catalog::default_table_metadata_dir_path( + &destination_namespace, + &crate::table_catalog::IdentifierSegment::parse("events_v2").unwrap(), + ); + let foreign_namespace_path = crate::table_catalog::default_table_metadata_dir_path( + &source_namespace, + &crate::table_catalog::IdentifierSegment::parse("foreign").unwrap(), + ); + let retained_directory = committed.metadata_location.rsplit_once('/').unwrap().0; + let persisted_metadata = read_table_metadata_json(&metadata_backend, "warehouse", &committed.metadata_location) + .await + .unwrap(); + for (index, location) in [ + format!("{current_namespace_path}/00003.metadata.json"), + format!("{foreign_namespace_path}/00003.metadata.json"), + format!("s3://other-warehouse/{foreign_namespace_path}/00003.metadata.json"), + format!("{retained_directory}/../00003.metadata.json"), + format!("{retained_directory}/child/00003.metadata.json"), + format!("{retained_directory}/%2e%2e.json"), + ] + .into_iter() + .enumerate() + { + if index < 2 { + metadata_backend + .put_json("warehouse", &location, persisted_metadata.clone()) + .await; + } + let mut request = stale_request.clone(); + request.commit_id = format!("invalid-renamed-path-{index}"); + request.idempotency_key = Some(request.commit_id.clone()); + request.expected_version_token = committed.version_token.clone(); + request.expected_metadata_location = committed.metadata_location.clone(); + request.new_metadata_location = location; + assert!(matches!( + store.commit_table(request.clone()).await, + Err(crate::table_catalog::TableCatalogStoreError::Invalid(_)) + )); + assert!( + store + .get_commit_by_id("warehouse", &renamed.table_id, &request.commit_id) + .await + .unwrap() + .is_none() + ); + assert!( + store + .get_commit_by_idempotency_key("warehouse", &renamed.table_id, request.idempotency_key.as_deref().unwrap()) + .await + .unwrap() + .is_none() + ); + assert_eq!( + store.load_table("warehouse", destination, "events_v2").await.unwrap(), + Some(committed.clone()) + ); + assert_eq!( + store.load_table("warehouse", "analytics", "events").await.unwrap(), + Some(recreated.clone()) + ); + } + let wrong_identity = standard_commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &destination_namespace, + "events_v2", + standard_property_commit_request("22222222-2222-4222-8222-222222222222", &recreated.table_uuid, "wrong-identity"), + ) + .await + .expect_err("recreated source identity cannot commit to the renamed table"); + assert_eq!(wrong_identity.status_code(), Some(StatusCode::CONFLICT)); + assert_eq!( + store.load_table("warehouse", destination, "events_v2").await.unwrap(), + Some(committed.clone()) + ); let recreated_commit = standard_commit_table_response( &store, &trusted_table_commit_backend(&metadata_backend), @@ -4300,7 +4500,7 @@ async fn renamed_and_recreated_tables_with_the_same_commit_id_use_disjoint_metad assert!( !renamed_commit .metadata_location - .contains("/namespaces/curated/tables/events_v2/") + .contains(&format!("/namespaces/{destination}/tables/events_v2/")) ); assert!( recreated_commit @@ -4309,8 +4509,8 @@ async fn renamed_and_recreated_tables_with_the_same_commit_id_use_disjoint_metad ); assert_eq!(recreated_commit.metadata["table-uuid"], recreated.table_uuid); assert_eq!(recreated_commit.metadata["properties"]["owner"], "analytics"); - for metadata_location in [renamed_commit.metadata_location, recreated_commit.metadata_location] { - let object_key = test_snapshot_object_key("warehouse", &metadata_location); + for metadata_location in [&renamed_commit.metadata_location, &recreated_commit.metadata_location] { + let object_key = test_snapshot_object_key("warehouse", metadata_location); assert!( metadata_backend .object_exists("warehouse", &object_key) @@ -4318,6 +4518,27 @@ async fn renamed_and_recreated_tables_with_the_same_commit_id_use_disjoint_metad .expect("metadata lookup should succeed") ); } + let restarted = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(metadata_backend, mode); + assert_eq!( + restarted.load_table("warehouse", destination, "events_v2").await.unwrap(), + Some(committed) + ); + let recreated_after = restarted + .load_table("warehouse", "analytics", "events") + .await + .unwrap() + .unwrap(); + assert_eq!(recreated_after.table_id, recreated.table_id); + assert_eq!(recreated_after.table_uuid, recreated.table_uuid); + assert_eq!(recreated_after.warehouse_location, recreated.warehouse_location); + assert_eq!(recreated_after.generation, recreated.generation + 1); + assert_eq!( + restarted + .get_commit_by_id("warehouse", &renamed.table_id, commit_id) + .await + .unwrap(), + Some(receipt) + ); } #[tokio::test] diff --git a/rustfs/src/table_catalog/store/object.rs b/rustfs/src/table_catalog/store/object.rs index f62b2b93f..c4177251f 100644 --- a/rustfs/src/table_catalog/store/object.rs +++ b/rustfs/src/table_catalog/store/object.rs @@ -5875,7 +5875,7 @@ where )), ); } - if !is_valid_table_metadata_location(&namespace, &table, &request.new_metadata_location) { + if !is_valid_table_metadata_location_for_entry(¤t, &request.new_metadata_location) { return table_commit_result( &request.table_bucket, &request.namespace, From 9ca0e9b7179b8be42da448f9f119de1800f198ab Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 18:00:44 +0800 Subject: [PATCH 041/143] fix(obs): isolate bucket usage dashboards by cluster (#8510) Keep same-name buckets in independent clusters separate while deduplicating replicated usage observations. Document the existing scanner-backed exporter and cover its gauge contract and cluster-scoped queries. Co-authored-by: heihutu Co-authored-by: zhi22915 --- .../grafana/dashboards/rustfs.json | 4 +- .../observability/tests/bucket-usage.test.yml | 48 +++++++ crates/obs/README.md | 4 +- crates/obs/src/global.rs | 23 +++- crates/obs/src/metrics/storage_snapshot.rs | 63 +++++++++ docs/README.md | 3 + docs/operations/bucket-usage-metrics.md | 122 ++++++++++++++++++ 7 files changed, 261 insertions(+), 6 deletions(-) create mode 100644 .docker/observability/tests/bucket-usage.test.yml create mode 100644 docs/operations/bucket-usage-metrics.md diff --git a/.docker/observability/grafana/dashboards/rustfs.json b/.docker/observability/grafana/dashboards/rustfs.json index 15fbdc34b..1ca9a90a7 100644 --- a/.docker/observability/grafana/dashboards/rustfs.json +++ b/.docker/observability/grafana/dashboards/rustfs.json @@ -1748,7 +1748,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max by (job, bucket) (rustfs_cluster_usage_buckets_objects_count{job=~\"$job\", bucket=~\"$bucket\"})", + "expr": "max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id=\"$storage_cluster\", job=~\"$job\", bucket=~\"$bucket\"})", "legendFormat": "{{bucket}}", "range": true, "refId": "A" @@ -1848,7 +1848,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max by (job, bucket) (rustfs_cluster_usage_buckets_total_bytes{job=~\"$job\", bucket=~\"$bucket\"})", + "expr": "max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id=\"$storage_cluster\", job=~\"$job\", bucket=~\"$bucket\"})", "legendFormat": "{{bucket}}", "range": true, "refId": "A" diff --git a/.docker/observability/tests/bucket-usage.test.yml b/.docker/observability/tests/bucket-usage.test.yml new file mode 100644 index 000000000..d886a7dc9 --- /dev/null +++ b/.docker/observability/tests/bucket-usage.test.yml @@ -0,0 +1,48 @@ +evaluation_interval: 1m +tests: +- name: bucket usage keeps clusters separate and deduplicates reporting nodes + interval: 1m + input_series: + - series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a1"} + values: '4096 1024' + - series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a2"} + values: '4096 1024' + - series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="b",job="rustfs",bucket="shared",service_instance_id="b1"} + values: '65536 65536' + - series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job="rustfs",bucket="empty",service_instance_id="a1"} + values: '0 0' + - series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a1"} + values: '3 1' + - series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a2"} + values: '3 1' + - series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="b",job="rustfs",bucket="shared",service_instance_id="b1"} + values: '70 70' + - series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job="rustfs",bucket="empty",service_instance_id="a1"} + values: '0 0' + promql_expr_test: + - expr: max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job=~"rustfs",bucket=~".*"}) + eval_time: 1m + exp_samples: + - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="shared"}' + value: 1024 + - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="empty"}' + value: 0 + - expr: max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job=~"rustfs",bucket=~".*"}) + eval_time: 1m + exp_samples: + - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="shared"}' + value: 1 + - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="empty"}' + value: 0 + - expr: max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_total_bytes) + eval_time: 1m + exp_samples: + - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="shared"}' + value: 1024 + - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="empty"}' + value: 0 + - labels: '{rustfs_cluster_id="b",job="rustfs",bucket="shared"}' + value: 65536 + - expr: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",bucket="unknown"} + eval_time: 1m + exp_samples: [] diff --git a/crates/obs/README.md b/crates/obs/README.md index a66fc5dea..0a1a9caaf 100644 --- a/crates/obs/README.md +++ b/crates/obs/README.md @@ -127,7 +127,9 @@ All configuration is read from environment variables at startup. `RUSTFS_OBS_TLS_CA_FILE` is read only while OTLP exporters are initialized. Its PEM certificates are added to, not substituted for, the platform trust roots, and TLS chain, expiry, and hostname checks remain enabled. Restart RustFS after rotating the bundle. A missing, unreadable, non-absolute, empty, or invalid bundle fails OTLP initialization. -The export interval is separate from application metric collection. Without interval overrides, node/disk metrics refresh every 60 seconds (`RUSTFS_METRICS_NODE_INTERVAL_SEC`), cluster metrics every 60 seconds (`RUSTFS_METRICS_CLUSTER_INTERVAL_SEC`), and per-bucket metrics every 300 seconds (`RUSTFS_METRICS_BUCKET_INTERVAL_SEC`). See the [collection defaults](src/metrics/config.rs) and [interval configuration](src/metrics/scheduler.rs) for collector-specific and global overrides. +The export interval is separate from application metric collection. Without interval overrides, node/disk metrics refresh every 60 seconds (`RUSTFS_METRICS_NODE_INTERVAL_SEC`), cluster metrics including scanner bucket usage every 60 seconds (`RUSTFS_METRICS_CLUSTER_INTERVAL_SEC`), and the separate bucket collector every 300 seconds (`RUSTFS_METRICS_BUCKET_INTERVAL_SEC`). See the [collection defaults](src/metrics/config.rs) and [interval configuration](src/metrics/scheduler.rs) for collector-specific and global overrides. + +For bucket size and object-count metric names, all-bucket PromQL queries, accounting semantics, and scanner freshness, see [Bucket usage metrics](../../docs/operations/bucket-usage-metrics.md). With OTLP → Collector → Prometheus, the time from a failure to an alert also includes underlying health detection, collection and export work, transport and Collector batching, Prometheus `scrape_interval` and `evaluation_interval`, and any alert rule `for` duration. A shorter scrape interval cannot refresh a value that RustFS has not yet collected and exported. These periodic intervals do not guarantee a fixed detection or alert latency; see the [Prometheus alert rule documentation](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) for the role of `for`. diff --git a/crates/obs/src/global.rs b/crates/obs/src/global.rs index 08f1e83b0..a413a5d01 100644 --- a/crates/obs/src/global.rs +++ b/crates/obs/src/global.rs @@ -260,9 +260,26 @@ mod tests { } #[test] - fn test_grafana_dashboard_deduplicates_cluster_bucket_usage() { - assert!(GRAFANA_DASHBOARD.contains("max by (job, bucket) (rustfs_cluster_usage_buckets_objects_count")); - assert!(GRAFANA_DASHBOARD.contains("max by (job, bucket) (rustfs_cluster_usage_buckets_total_bytes")); + fn test_grafana_dashboard_scopes_and_deduplicates_cluster_bucket_usage() { + let dashboard: serde_json::Value = serde_json::from_str(GRAFANA_DASHBOARD).expect("dashboard must be valid JSON"); + for (title, metric) in [ + ("Bucket Objects", "rustfs_cluster_usage_buckets_objects_count"), + ("Bucket Size", "rustfs_cluster_usage_buckets_total_bytes"), + ] { + let panel = dashboard["panels"] + .as_array() + .expect("dashboard panels") + .iter() + .find(|panel| panel["title"] == title) + .expect("bucket usage panel"); + assert_eq!( + panel["targets"][0]["expr"], + format!( + "max by (rustfs_cluster_id, job, bucket) ({metric}{{rustfs_cluster_id=\"$storage_cluster\", job=~\"$job\", bucket=~\"$bucket\"}})" + ), + "{title} must select one cluster and deduplicate its reporting nodes" + ); + } assert!(!GRAFANA_DASHBOARD.contains("sum by (bucket) (rustfs_bucket_api_objects_total")); assert!(!GRAFANA_DASHBOARD.contains("sum by (bucket) (rustfs_bucket_api_usage_bytes")); } diff --git a/crates/obs/src/metrics/storage_snapshot.rs b/crates/obs/src/metrics/storage_snapshot.rs index 8977403e9..d3e1a74ee 100644 --- a/crates/obs/src/metrics/storage_snapshot.rs +++ b/crates/obs/src/metrics/storage_snapshot.rs @@ -408,6 +408,69 @@ mod tests { provider.shutdown().unwrap(); } + #[test] + fn actual_sdk_exports_bucket_size_and_object_count_as_current_gauges() { + use crate::metrics::collectors::{BucketUsageStats, collect_bucket_usage_metrics}; + + let (provider, exporter, mut snapshot) = setup_unscoped(); + let mut buckets = vec![ + BucketUsageStats { + bucket: "client-a".into(), + total_bytes: 4096, + objects_count: 3, + ..Default::default() + }, + BucketUsageStats { + bucket: "client-b".into(), + total_bytes: 8192, + objects_count: 7, + ..Default::default() + }, + BucketUsageStats { + bucket: "empty".into(), + ..Default::default() + }, + ]; + + for expected in [ + vec![("client-a", 4096.0, 3.0), ("client-b", 8192.0, 7.0), ("empty", 0.0, 0.0)], + vec![("client-a", 1024.0, 1.0), ("empty", 0.0, 0.0)], + ] { + let metrics = collect_bucket_usage_metrics(&buckets); + assert!( + metrics.iter().all(|metric| metric.metric_type == MetricType::Gauge), + "bucket inventory values must decrease without counter semantics" + ); + snapshot.replace(metrics).expect("publish current bucket usage"); + let points = flush(&provider, &exporter); + for (name, index) in [ + ("rustfs_cluster_usage_buckets_total_bytes", 0), + ("rustfs_cluster_usage_buckets_objects_count", 1), + ] { + let mut actual = points + .iter() + .filter(|point| point.name == name) + .map(|point| { + assert_eq!(point.scope, "service-test"); + assert_eq!(point.attributes.len(), 1, "usage is cluster-wide, not a per-node count"); + assert_eq!(point.attributes[0].key.as_str(), "bucket"); + (point.attributes[0].value.to_string(), point.value) + }) + .collect::>(); + actual.sort_by(|a, b| a.0.cmp(&b.0)); + let values = expected + .iter() + .map(|(bucket, bytes, objects)| (bucket.to_string(), if index == 0 { *bytes } else { *objects })) + .collect::>(); + assert_eq!(actual, values, "{name} must preserve each bucket's current value"); + } + buckets.remove(1); + buckets[0].total_bytes = 1024; + buckets[0].objects_count = 1; + } + provider.shutdown().expect("shut down bucket usage test provider"); + } + #[test] fn failed_or_stalled_collection_cannot_refresh_last_success_or_export_old_points() { let (provider, exporter, mut snapshot) = setup(); diff --git a/docs/README.md b/docs/README.md index 958dbda58..c32761e35 100644 --- a/docs/README.md +++ b/docs/README.md @@ -66,6 +66,9 @@ procedure is required. For storage dashboards, see [Storage metrics and observer selection](operations/storage-metrics.md): drive ownership, snapshot freshness, counter queries, and rolling upgrades. +For per-bucket capacity and object counts, see [Bucket usage metrics](operations/bucket-usage-metrics.md): +OTLP export, cluster deduplication, accounting semantics, and scanner freshness. + For optional shard commitments, see [Independent shard integrity rollout](operations/shard-integrity-rollout.md): activation, legacy repair results, multipart mode changes, and rollback limits. diff --git a/docs/operations/bucket-usage-metrics.md b/docs/operations/bucket-usage-metrics.md new file mode 100644 index 000000000..ed65d05a7 --- /dev/null +++ b/docs/operations/bucket-usage-metrics.md @@ -0,0 +1,122 @@ +# Bucket usage metrics + +Use these metrics to collect every bucket's logical usage through the existing +OpenTelemetry pipeline. RustFS reads the cached scanner usage snapshot once per +collection; exporting metrics does not enumerate objects or run an inline scan. + +## Export and query + +Configure the RustFS nodes with an OTLP/HTTP destination and a stable cluster ID: + +```bash +RUSTFS_OBS_ENDPOINT=http://otel-collector:4318 +RUSTFS_OBS_METRICS_EXPORT_ENABLED=true +OTEL_RESOURCE_ATTRIBUTES=rustfs.cluster.id=production-a +``` + +Append the cluster ID to any existing resource attributes. The shipped +[Collector configuration](../../.docker/observability/otel-collector-config.yaml) +receives OTLP and exposes Prometheus samples on port 8889; its +`resource_to_telemetry_conversion` promotes the cluster ID to +`rustfs_cluster_id`. Prometheus scrapes the Collector. See +[storage metrics](storage-metrics.md#configure-the-pipeline) for pipeline setup +and resource identity. + +The following existing gauges carry a `bucket` label: + +| Metric | Meaning | +|---|---| +| `rustfs_cluster_usage_buckets_total_bytes` | Logical scanner bytes across retained object versions, including noncurrent versions; excludes delete-marker bytes. | +| `rustfs_cluster_usage_buckets_objects_count` | Scanner object-entry/key count, separate from version and delete-marker counts. | + +Logical usage differs from physical drive usage, which includes erasure-coding +overhead and can differ because of compression or remote tiering. Object-entry +count can include keys with only noncurrent versions or delete markers, so it +need not equal the number of currently visible objects from `ListObjectsV2`. + +For monitoring, query all buckets in one cluster: + +```promql +max by (rustfs_cluster_id, job, bucket) ( + rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="production-a"} +) +``` + +```promql +max by (rustfs_cluster_id, job, bucket) ( + rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="production-a"} +) +``` + +Each reporting node exports the whole cluster's logical usage. Summing those +copies would multiply the result by the node count. Keep cluster identity when +grouping: independent clusters can share both a job and a bucket name. The +bundled Bucket Size and Bucket Objects panels use the selected **Storage +cluster** and deduplicate that cluster's reporting nodes. + +Nodes can briefly observe different snapshots. `max` is useful for dashboards +but can retain an older, larger value after deletes. For a consistent accounting +observation, select one reporting node for both bytes and object count: + +```promql +rustfs_cluster_usage_buckets_total_bytes{ + rustfs_cluster_id="production-a", service_instance_id="node1:9000" +} +``` + +Use the same selector with `rustfs_cluster_usage_buckets_objects_count`, and +check the promoted `service_instance_id` in your Collector output. These +floating-point, periodic gauges are usage observations, not an exact billing +ledger or a transactional view of writes and deletes. + +## Freshness and missing values + +Usage collection runs on `RUSTFS_METRICS_CLUSTER_INTERVAL_SEC` (60 seconds by +default), independently of the OTLP export interval +`RUSTFS_OBS_METER_INTERVAL` (30 seconds by default). The separate bucket +collector defaults to 300 seconds. Neither interval forces a scanner cycle. + +Check these gauges on the same reporting node as the bucket values: + +- `rustfs_cluster_usage_objects_since_last_update_seconds`: age of the + persisted usage snapshot when it was collected. A bucket cleanup can advance + the timestamp; it is not necessarily the age of a full object rescan. +- `rustfs_cluster_usage_objects_snapshot_converged`: `1` for a converged + snapshot, `0` when convergence is not confirmed, including observational or + unknown state. Complete observations can still supply usage during sustained writes. + +Before a complete snapshot covers the current bucket namespace, the usage +collector omits its values. A missing series is unknown; do not replace it with +zero for accounting. A bucket confirmed empty in a complete snapshot exports +zero bytes and zero objects. + +A temporary collection failure preserves the last valid export snapshot. +RustFS stops exporting that snapshot once collection stalls beyond three times +the larger collection/export interval. Deleted bucket series receive bounded +zero tombstones and then stop being exported after a valid updated snapshot. +The Collector can retain the last sample after export stops; the shipped +configuration uses a five-minute `metric_expiration`. Prometheus history also +retains past samples. + +If metrics are absent, first check OTLP connectivity and metrics export enablement, +then the scanner leader's `GET /rustfs/admin/v3/scanner/status` and the admin +usage snapshot. A shorter scrape interval cannot repair an incomplete scanner +snapshot. The scanner publication and cache path is shared with the Console's +usage display. + +## Regression checks + +The SDK test uses the production bucket collector and an actual OpenTelemetry +reader to verify two populated buckets, a confirmed-empty bucket, decreasing +usage, and removal from subsequent exports: + +```bash +RUSTUP_TOOLCHAIN=stable cargo test -p rustfs-obs actual_sdk_exports_bucket_size_and_object_count_as_current_gauges --lib +``` + +The PromQL fixture checks same-name buckets in separate clusters and multiple +reporting nodes. Run it from `.docker/observability/tests`: + +```bash +promtool test rules bucket-usage.test.yml +``` From 8a94c6493484b3b9fcda28e1aba4502aa5103578 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 18:56:44 +0800 Subject: [PATCH 042/143] fix(audit): preserve request identity and verified client address (#8513) Capture the authenticated request access key when building the audit entry and preserve it through completion instead of replacing it with root credentials. Resolve remotehost from validated client context or the socket peer, and ignore untrusted forwarding headers. Cover request identities, completion outcomes, client address precedence, and audit serialization without credential secrets. Co-authored-by: heihutu Co-authored-by: zhi22915 --- rustfs/src/storage/helper.rs | 176 ++++++++++++++++++++++++++++++----- 1 file changed, 153 insertions(+), 23 deletions(-) diff --git a/rustfs/src/storage/helper.rs b/rustfs/src/storage/helper.rs index b8eb3c15d..805b5a870 100644 --- a/rustfs/src/storage/helper.rs +++ b/rustfs/src/storage/helper.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::module_switches::{is_audit_module_enabled, is_notify_module_enabled}; -use crate::shared_types::convert_ecstore_object_info; +use crate::shared_types::{RemoteAddr, convert_ecstore_object_info}; use crate::storage::access::{ReqInfo, request_context_from_req}; use crate::storage::request_context::RequestContext; use crate::storage::sse::KmsRequestAuditScope; @@ -33,6 +33,7 @@ use rustfs_s3_types::EventName; use rustfs_targets::{ extract_params_header, extract_req_params, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent, }; +use rustfs_trusted_proxies::ClientInfo; use rustfs_utils::http::headers::AMZ_REQUEST_ID; use s3s::{S3Request, S3Response, S3Result}; use serde_json::Value; @@ -150,15 +151,6 @@ impl OperationHelper { .filter(|value| !value.is_empty()) .unwrap_or(path_object_key); - // Infer remote address - let remote_host = req - .headers - .get(rustfs_utils::http::X_FORWARDED_FOR) - .and_then(|v| v.to_str().ok()) - .or_else(|| req.headers.get(rustfs_utils::http::X_REAL_IP).and_then(|v| v.to_str().ok())) - .unwrap_or("") - .to_string(); - let trigger = op.as_str(); // Initialize audit builder @@ -179,15 +171,30 @@ impl OperationHelper { let request_id = request_context.request_id.clone(); let audit_builder = if audit_enabled { - Some( - AuditEntryBuilder::new("1.0", event, trigger, api_builder.clone().build()) - .remote_host(remote_host) - .user_agent(get_request_user_agent(&req.headers)) - .req_host(get_request_host(&req.headers)) - .req_path(req.uri.path().to_string()) - .req_query(extract_req_params(req)) - .request_id(&request_id), - ) + let remote_host = req + .extensions + .get::() + .map(|info| info.real_ip.to_string()) + .or_else(|| { + req.extensions + .get::>() + .and_then(|addr| addr.map(|addr| addr.0.ip().to_string())) + }) + .unwrap_or_default(); + let mut builder = AuditEntryBuilder::new("1.0", event, trigger, api_builder.clone().build()) + .remote_host(remote_host) + .user_agent(get_request_user_agent(&req.headers)) + .req_host(get_request_host(&req.headers)) + .req_path(req.uri.path().to_string()) + .req_query(extract_req_params(req)) + .request_id(&request_id); + if let Some(cred) = req_info + .and_then(|info| info.cred.as_ref()) + .filter(|cred| !cred.access_key.is_empty()) + { + builder = builder.access_key(&cred.access_key); + } + Some(builder) } else { None }; @@ -346,10 +353,6 @@ impl OperationHelper { final_builder = final_builder.error(err); } - if let Some(cred) = runtime_sources::current_action_credentials() { - final_builder = final_builder.access_key(&cred.access_key); - } - // Inject OpenTelemetry trace context into audit tags for distributed tracing correlation let mut tags = HashMap::new(); if let Some(ref tid) = state.request_context.trace_id { @@ -572,6 +575,133 @@ mod tests { ); } + #[test] + fn operation_helper_audits_the_authenticated_request_identity() { + with_vars( + [ + (rustfs_config::ENV_NOTIFY_ENABLE, Some("false")), + (rustfs_config::ENV_AUDIT_ENABLE, Some("true")), + ], + || { + refresh_notify_module_enabled(); + refresh_audit_module_enabled(); + + for (access_key, parent_user) in [ + (Some("testuser"), ""), + (Some("rustfsadmin"), ""), + (Some("service-account"), "testuser"), + (Some("sts-session"), "testuser"), + (None, ""), + (Some(""), ""), + ] { + let expected_access_key = access_key.filter(|key| !key.is_empty()); + for outcome in [None, Some(true), Some(false)] { + let input = GetObjectInput::builder() + .bucket("audit-bucket".to_string()) + .key("object.txt".to_string()) + .build() + .expect("get object input should build"); + let mut req = build_request(input, Method::GET, Uri::from_static("/audit-bucket/object.txt")); + req.extensions.insert(ReqInfo { + cred: access_key.map(|key| Credentials { + access_key: key.to_string(), + secret_key: "audit-secret-must-not-leak".to_string(), + session_token: "audit-token-must-not-leak".to_string(), + parent_user: parent_user.to_string(), + ..Default::default() + }), + ..Default::default() + }); + let mut helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject); + if let Some(success) = outcome { + let result = if success { + Ok(S3Response::new(GetObjectOutput::default())) + } else { + Err(S3Error::new(S3ErrorCode::NoSuchKey)) + }; + helper = helper.complete(&result); + } + let OperationHelper::Enabled(state) = &mut helper else { + panic!("helper should be enabled when the audit switch is on"); + }; + let entry = state.audit_builder.take().expect("audit builder should exist").build(); + assert_eq!(entry.access_key.as_deref(), expected_access_key); + let json = serde_json::to_value(&entry).expect("audit entry should serialize"); + assert_eq!(json.get("accessKey").and_then(|key| key.as_str()), expected_access_key); + let rendered = serde_json::to_string(&entry).expect("audit entry should serialize"); + assert!(!rendered.contains("audit-secret-must-not-leak")); + assert!(!rendered.contains("audit-token-must-not-leak")); + } + } + }, + ); + } + + #[test] + fn operation_helper_audits_verified_client_addresses() { + use crate::shared_types::RemoteAddr; + use rustfs_trusted_proxies::{ClientInfo, ValidationMode}; + use std::net::SocketAddr; + + with_vars( + [ + (rustfs_config::ENV_NOTIFY_ENABLE, Some("false")), + (rustfs_config::ENV_AUDIT_ENABLE, Some("true")), + ], + || { + refresh_notify_module_enabled(); + refresh_audit_module_enabled(); + + let peer: SocketAddr = "192.0.2.10:54321".parse().unwrap(); + let ipv6_peer: SocketAddr = "[2001:db8::10]:54321".parse().unwrap(); + let proxied = ClientInfo::from_trusted_proxy( + "198.51.100.20".parse().unwrap(), + None, + None, + peer.ip(), + 1, + ValidationMode::Strict, + Vec::new(), + ); + for (remote_addr, client_info, expected) in [ + (Some(RemoteAddr(peer)), None, "192.0.2.10"), + (Some(RemoteAddr(ipv6_peer)), None, "2001:db8::10"), + (Some(RemoteAddr(peer)), Some(proxied), "198.51.100.20"), + (Some(RemoteAddr(peer)), Some(ClientInfo::direct(peer)), "192.0.2.10"), + (None, None, ""), + ] { + for spoofed in [false, true] { + let input = GetObjectInput::builder() + .bucket("audit-bucket".to_string()) + .key("object.txt".to_string()) + .build() + .expect("get object input should build"); + let mut req = build_request(input, Method::GET, Uri::from_static("/audit-bucket/object.txt")); + req.extensions.insert(remote_addr); + if let Some(client_info) = client_info.as_ref() { + req.extensions.insert(client_info.clone()); + } + if spoofed { + req.headers + .insert("x-forwarded-for", HeaderValue::from_static("203.0.113.99")); + req.headers.insert("x-real-ip", HeaderValue::from_static("203.0.113.98")); + } + let result = Ok(S3Response::new(GetObjectOutput::default())); + let mut helper = + OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject).complete(&result); + let OperationHelper::Enabled(state) = &mut helper else { + panic!("helper should be enabled when the audit switch is on"); + }; + let entry = state.audit_builder.take().expect("audit builder should exist").build(); + assert_eq!(entry.remote_host.as_deref(), Some(expected)); + let rendered = serde_json::to_value(&entry).expect("audit entry should serialize"); + assert_eq!(rendered["remotehost"], expected); + } + } + }, + ); + } + #[test] fn operation_helper_adds_objects_to_audit_details() { with_vars( From fb5c002a30a57cdadad46b49021dbc70fbce6000 Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Tue, 13 Oct 2026 18:59:29 +0800 Subject: [PATCH 043/143] refactor(iam): encapsulate federated authorization models (#8514) --- crates/iam/src/federation/mapper.rs | 245 +++++++++++++++++- crates/iam/src/federation/mod.rs | 10 +- crates/iam/src/federation/model.rs | 153 +---------- crates/iam/src/federation/test_support.rs | 54 ++++ crates/iam/src/federation/transaction.rs | 51 ++-- crates/iam/src/oidc/recovery_tests.rs | 6 +- crates/iam/src/oidc/runtime_tests.rs | 10 +- docs/architecture/crate-boundaries.md | 2 + rustfs/src/admin/handlers/oidc.rs | 10 +- rustfs/src/admin/handlers/sts.rs | 2 +- .../src/admin/service/federated_identity.rs | 157 +++++------ 11 files changed, 440 insertions(+), 260 deletions(-) create mode 100644 crates/iam/src/federation/test_support.rs diff --git a/crates/iam/src/federation/mapper.rs b/crates/iam/src/federation/mapper.rs index 1dca51e5c..6680a2471 100644 --- a/crates/iam/src/federation/mapper.rs +++ b/crates/iam/src/federation/mapper.rs @@ -12,12 +12,123 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{FederatedAuthorization, VerifiedFederatedIdentity}; +use super::{FederatedClaims, VerifiedFederatedIdentity}; use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC}; use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive}; +use rustfs_utils::HashAlgorithm; use std::{collections::HashMap, sync::Arc}; use tracing::debug; +#[cfg(any(test, feature = "test-util"))] +#[path = "test_support.rs"] +pub(super) mod test_support; + +/// Read-only authorization result produced by core federation mapping. +/// +/// The claims representation stays inside IAM: +/// +/// ```compile_fail +/// use rustfs_iam::federation::FederatedClaims; +/// ``` +/// +/// Callers cannot mutate mapped fields: +/// +/// ```compile_fail +/// use rustfs_iam::federation::FederatedAuthorization; +/// +/// fn mutate(authorization: &mut FederatedAuthorization) { +/// authorization.groups.clear(); +/// } +/// ``` +/// +/// Callers cannot construct authorization results: +/// +/// ```compile_fail +/// use rustfs_iam::federation::FederatedAuthorization; +/// +/// fn construct(authorization: FederatedAuthorization) -> FederatedAuthorization { +/// FederatedAuthorization { ..authorization } +/// } +/// ``` +#[derive(Debug, Clone)] +pub struct FederatedAuthorization { + provider_id: String, + claims: FederatedClaims, + policies: Vec, + group_claim_policies: Vec, + groups: Vec, + roles_claim_key: Option, + roles: Vec, +} + +impl FederatedAuthorization { + pub fn provider_id(&self) -> &str { + &self.provider_id + } + + pub fn subject(&self) -> &str { + &self.claims.sub + } + + pub fn email(&self) -> &str { + &self.claims.email + } + + pub fn username(&self) -> &str { + &self.claims.username + } + + pub fn session_identity(&self) -> String { + self.claims.session_identity() + } + + pub fn policies(&self) -> &[String] { + &self.policies + } + + /// Subset of `policies` derived only from the groups claim (including merged roles claim + /// values); these may be ignored when no policy of that name exists. Everything else in + /// `policies` must resolve. + pub fn group_claim_policies(&self) -> &[String] { + &self.group_claim_policies + } + + pub fn mapped_iam_groups(&self) -> &[String] { + &self.groups + } + + pub fn roles_claim_key(&self) -> Option<&str> { + self.roles_claim_key.as_deref() + } + + pub fn roles(&self) -> &[String] { + &self.roles + } + + pub fn has_authorization_context(&self) -> bool { + !self.policies.is_empty() || !self.groups.is_empty() + } + + pub fn oidc_virtual_parent(&self) -> Option { + let issuer = self.claims.raw.get("iss")?.as_str()?; + let subject = self.claims.sub.as_str(); + if issuer.is_empty() || subject.is_empty() { + return None; + } + + let subject_len = u64::try_from(subject.len()).ok()?; + let issuer_len = u64::try_from(issuer.len()).ok()?; + let mut source = Vec::with_capacity(23 + subject.len() + issuer.len()); + source.extend_from_slice(b"openid:"); + source.extend_from_slice(&subject_len.to_be_bytes()); + source.extend_from_slice(subject.as_bytes()); + source.extend_from_slice(&issuer_len.to_be_bytes()); + source.extend_from_slice(issuer.as_bytes()); + let digest = HashAlgorithm::SHA256.hash_encode(&source); + Some(base64_simd::URL_SAFE_NO_PAD.encode_to_string(digest.as_ref())) + } +} + #[derive(Clone)] pub(crate) struct FederatedAuthorizationRule { provider_id: String, @@ -308,8 +419,136 @@ fn claim_value_type(value: &serde_json::Value) -> &'static str { #[cfg(test)] mod tests { use super::*; - use crate::federation::{FederatedClaims, FederatedProviderRef, VerifiedFederatedIdentity}; - use serde_json::json; + use crate::federation::{FederatedAuthorizationFixture, FederatedClaims, FederatedProviderRef, VerifiedFederatedIdentity}; + use serde_json::{Value, json}; + + fn authorization(policies: Vec, groups: Vec) -> FederatedAuthorization { + FederatedAuthorizationFixture { + provider_id: "standard_oidc".to_string(), + sub: "subject".to_string(), + attributes: HashMap::from([("iss".to_string(), Value::String("https://idp.example.test".to_string()))]), + policies, + mapped_iam_groups: groups, + ..Default::default() + } + .into_authorization() + } + + #[test] + fn authorization_context_accepts_policy_or_group() { + assert!(!authorization(Vec::new(), Vec::new()).has_authorization_context()); + assert!(authorization(vec!["consoleAdmin".to_string()], Vec::new()).has_authorization_context()); + assert!(authorization(Vec::new(), vec!["RustFS.ConsoleAdmin".to_string()]).has_authorization_context()); + } + + #[test] + fn oidc_virtual_parent_is_issuer_scoped() { + let first = authorization(Vec::new(), Vec::new()); + let mut second = first.clone(); + second + .claims + .raw + .insert("iss".to_string(), Value::String("https://other-idp.example.test".to_string())); + + assert_eq!( + first.oidc_virtual_parent().as_deref(), + Some("HwDfWftzOy4jiuS3WjKytC_Sg_A2hKhrRAFtBDhoBr0") + ); + assert!(!rustfs_policy::auth::contains_reserved_chars( + first.oidc_virtual_parent().as_deref().expect("virtual parent") + )); + assert_ne!(first.oidc_virtual_parent(), second.oidc_virtual_parent()); + } + + #[test] + fn oidc_virtual_parent_length_delimits_identity_parts() { + let mut first = authorization(Vec::new(), Vec::new()); + first.claims.sub = "subject".to_string(); + first.claims.raw.insert( + "iss".to_string(), + Value::String("https://issuer.example/path:https://other.example".to_string()), + ); + let mut second = authorization(Vec::new(), Vec::new()); + second.claims.sub = "subject:https://issuer.example/path".to_string(); + second + .claims + .raw + .insert("iss".to_string(), Value::String("https://other.example".to_string())); + + assert_ne!(first.oidc_virtual_parent(), second.oidc_virtual_parent()); + } + + #[test] + fn oidc_virtual_parent_requires_verified_identity_parts() { + let mut missing_issuer = authorization(Vec::new(), Vec::new()); + missing_issuer.claims.raw.clear(); + let mut missing_subject = authorization(Vec::new(), Vec::new()); + missing_subject.claims.sub.clear(); + + assert!(missing_issuer.oidc_virtual_parent().is_none()); + assert!(missing_subject.oidc_virtual_parent().is_none()); + } + + #[test] + fn oidc_virtual_parent_preserves_exact_subject() { + let plain = authorization(Vec::new(), Vec::new()); + let mut padded = plain.clone(); + padded.claims.sub = format!(" {} ", plain.claims.sub); + + assert_ne!(plain.oidc_virtual_parent(), padded.oidc_virtual_parent()); + } + + #[test] + fn authorization_accessors_preserve_exact_values_and_distinct_groups() { + let attributes = HashMap::from([("custom".to_string(), json!(["value", null, 7]))]); + let authorization = FederatedAuthorizationFixture { + provider_id: " Corp Provider ".to_string(), + sub: " subject ".to_string(), + email: " email@example.test ".to_string(), + username: " username ".to_string(), + source_groups: vec!["different-source-group".to_string()], + attributes: attributes.clone(), + policies: vec![" policy-b ".to_string(), "policy-a".to_string(), " policy-b ".to_string()], + group_claim_policies: vec![" policy-b ".to_string(), " policy-b ".to_string()], + mapped_iam_groups: vec!["mapped-a".to_string(), "mapped-b".to_string()], + roles_claim_key: Some(" Roles ".to_string()), + roles: vec!["role-b".to_string(), " role-a ".to_string(), "role-b".to_string()], + } + .into_authorization(); + + assert_eq!(authorization.provider_id(), " Corp Provider "); + assert_eq!(authorization.subject(), " subject "); + assert_eq!(authorization.email(), " email@example.test "); + assert_eq!(authorization.username(), " username "); + assert_eq!(authorization.session_identity(), " username "); + assert_eq!(authorization.policies(), [" policy-b ", "policy-a", " policy-b "]); + assert_eq!(authorization.group_claim_policies(), [" policy-b ", " policy-b "]); + assert_eq!(authorization.claims.groups, ["different-source-group"]); + assert_eq!(authorization.claims.raw, attributes); + assert_eq!(authorization.mapped_iam_groups(), ["mapped-a", "mapped-b"]); + assert_eq!(authorization.roles_claim_key(), Some(" Roles ")); + assert_eq!(authorization.roles(), ["role-b", " role-a ", "role-b"]); + } + + #[test] + fn default_authorization_fixture_has_no_identity_or_mapping() { + let authorization = FederatedAuthorizationFixture::default().into_authorization(); + + assert_eq!(authorization.provider_id(), ""); + assert_eq!(authorization.subject(), ""); + assert_eq!(authorization.email(), ""); + assert_eq!(authorization.username(), ""); + assert_eq!(authorization.session_identity(), "oidc-user-unknown"); + assert!(authorization.claims.groups.is_empty()); + assert!(authorization.claims.raw.is_empty()); + assert!(authorization.policies().is_empty()); + assert!(authorization.group_claim_policies().is_empty()); + assert!(authorization.mapped_iam_groups().is_empty()); + assert!(authorization.roles_claim_key().is_none()); + assert!(authorization.roles().is_empty()); + assert!(!authorization.has_authorization_context()); + assert!(authorization.oidc_virtual_parent().is_none()); + } fn mapper(rule: FederatedAuthorizationRule) -> CoreFederatedAuthorizationMapper { CoreFederatedAuthorizationMapper::new(FederatedAuthorizationRules::new([rule])) diff --git a/crates/iam/src/federation/mod.rs b/crates/iam/src/federation/mod.rs index b7498cf0a..6b54f62e4 100644 --- a/crates/iam/src/federation/mod.rs +++ b/crates/iam/src/federation/mod.rs @@ -24,11 +24,13 @@ pub use error::{FederatedSessionBindingError, FederationError, Result}; pub(crate) use mapper::FederatedAuthorizationRule; #[cfg(test)] pub(crate) use mapper::FederatedAuthorizationRuleRef; -pub use mapper::{CoreFederatedAuthorizationMapper, FederatedAuthorizationRules}; -pub(crate) use model::OpaqueLogoutContinuation; +#[cfg(any(test, feature = "test-util"))] +pub use mapper::test_support::FederatedAuthorizationFixture; +pub use mapper::{CoreFederatedAuthorizationMapper, FederatedAuthorization, FederatedAuthorizationRules}; +pub(crate) use model::{FederatedClaims, OpaqueLogoutContinuation}; pub use model::{ - FederatedAuthorization, FederatedClaims, FederatedLoginSession, FederatedProviderRef, FederatedSession, - FederatedSessionTransaction, OIDC_VIRTUAL_PARENT_CLAIM, VerifiedFederatedIdentity, + FederatedLoginSession, FederatedProviderRef, FederatedSession, FederatedSessionTransaction, OIDC_VIRTUAL_PARENT_CLAIM, + VerifiedFederatedIdentity, }; pub(crate) use provider::{FederatedProviderQuery, StandardOidcAuthentication, VerifiedFederatedCodeExchange}; pub use provider::{FederatedProviderView, FederatedRedirectPolicy}; diff --git a/crates/iam/src/federation/model.rs b/crates/iam/src/federation/model.rs index 76b8a9816..54b6725b9 100644 --- a/crates/iam/src/federation/model.rs +++ b/crates/iam/src/federation/model.rs @@ -13,23 +13,24 @@ // limitations under the License. use rustfs_credentials::Credentials; -use rustfs_utils::HashAlgorithm; use serde_json::Value; use std::{collections::HashMap, fmt}; +use super::FederatedAuthorization; + pub const OIDC_VIRTUAL_PARENT_CLAIM: &str = "x-rustfs-internal-oidc-parent"; #[derive(Debug, Clone)] -pub struct FederatedClaims { - pub sub: String, - pub email: String, - pub username: String, - pub groups: Vec, - pub raw: HashMap, +pub(crate) struct FederatedClaims { + pub(crate) sub: String, + pub(crate) email: String, + pub(crate) username: String, + pub(crate) groups: Vec, + pub(crate) raw: HashMap, } impl FederatedClaims { - pub fn session_identity(&self) -> String { + pub(crate) fn session_identity(&self) -> String { if !self.username.is_empty() { self.username.clone() } else if !self.email.is_empty() { @@ -125,45 +126,6 @@ impl VerifiedFederatedIdentity { } } -#[derive(Debug, Clone)] -pub struct FederatedAuthorization { - pub provider_id: String, - pub claims: FederatedClaims, - pub policies: Vec, - /// Subset of `policies` derived only from the groups claim (including merged roles claim - /// values); these may be ignored when no policy of that name exists. Everything else in - /// `policies` must resolve. - pub group_claim_policies: Vec, - pub groups: Vec, - pub roles_claim_key: Option, - pub roles: Vec, -} - -impl FederatedAuthorization { - pub fn has_authorization_context(&self) -> bool { - !self.policies.is_empty() || !self.groups.is_empty() - } - - pub fn oidc_virtual_parent(&self) -> Option { - let issuer = self.claims.raw.get("iss")?.as_str()?; - let subject = self.claims.sub.as_str(); - if issuer.is_empty() || subject.is_empty() { - return None; - } - - let subject_len = u64::try_from(subject.len()).ok()?; - let issuer_len = u64::try_from(issuer.len()).ok()?; - let mut source = Vec::with_capacity(23 + subject.len() + issuer.len()); - source.extend_from_slice(b"openid:"); - source.extend_from_slice(&subject_len.to_be_bytes()); - source.extend_from_slice(subject.as_bytes()); - source.extend_from_slice(&issuer_len.to_be_bytes()); - source.extend_from_slice(issuer.as_bytes()); - let digest = HashAlgorithm::SHA256.hash_encode(&source); - Some(base64_simd::URL_SAFE_NO_PAD.encode_to_string(digest.as_ref())) - } -} - /// Carries the provider-bound ID token between code exchange and logout token creation. pub(crate) struct OpaqueLogoutContinuation { provider: FederatedProviderRef, @@ -224,21 +186,6 @@ mod tests { } } - fn authorization(policies: Vec, groups: Vec) -> FederatedAuthorization { - FederatedAuthorization { - provider_id: "standard_oidc".to_string(), - claims: FederatedClaims { - raw: HashMap::from([("iss".to_string(), Value::String("https://idp.example.test".to_string()))]), - ..claims("", "", "subject") - }, - policies, - group_claim_policies: Vec::new(), - groups, - roles_claim_key: None, - roles: Vec::new(), - } - } - #[test] fn session_identity_preserves_existing_fallback_order() { assert_eq!(claims("john", "john@example.com", "sub-1").session_identity(), "john"); @@ -247,70 +194,6 @@ mod tests { assert_eq!(claims("", "", "").session_identity(), "oidc-user-unknown"); } - #[test] - fn authorization_context_accepts_policy_or_group() { - assert!(!authorization(Vec::new(), Vec::new()).has_authorization_context()); - assert!(authorization(vec!["consoleAdmin".to_string()], Vec::new()).has_authorization_context()); - assert!(authorization(Vec::new(), vec!["RustFS.ConsoleAdmin".to_string()]).has_authorization_context()); - } - - #[test] - fn oidc_virtual_parent_is_issuer_scoped() { - let first = authorization(Vec::new(), Vec::new()); - let mut second = first.clone(); - second - .claims - .raw - .insert("iss".to_string(), Value::String("https://other-idp.example.test".to_string())); - - assert_eq!( - first.oidc_virtual_parent().as_deref(), - Some("HwDfWftzOy4jiuS3WjKytC_Sg_A2hKhrRAFtBDhoBr0") - ); - assert!(!rustfs_policy::auth::contains_reserved_chars( - first.oidc_virtual_parent().as_deref().expect("virtual parent") - )); - assert_ne!(first.oidc_virtual_parent(), second.oidc_virtual_parent()); - } - - #[test] - fn oidc_virtual_parent_length_delimits_identity_parts() { - let mut first = authorization(Vec::new(), Vec::new()); - first.claims.sub = "subject".to_string(); - first.claims.raw.insert( - "iss".to_string(), - Value::String("https://issuer.example/path:https://other.example".to_string()), - ); - let mut second = authorization(Vec::new(), Vec::new()); - second.claims.sub = "subject:https://issuer.example/path".to_string(); - second - .claims - .raw - .insert("iss".to_string(), Value::String("https://other.example".to_string())); - - assert_ne!(first.oidc_virtual_parent(), second.oidc_virtual_parent()); - } - - #[test] - fn oidc_virtual_parent_requires_verified_identity_parts() { - let mut missing_issuer = authorization(Vec::new(), Vec::new()); - missing_issuer.claims.raw.clear(); - let mut missing_subject = authorization(Vec::new(), Vec::new()); - missing_subject.claims.sub.clear(); - - assert!(missing_issuer.oidc_virtual_parent().is_none()); - assert!(missing_subject.oidc_virtual_parent().is_none()); - } - - #[test] - fn oidc_virtual_parent_preserves_exact_subject() { - let plain = authorization(Vec::new(), Vec::new()); - let mut padded = plain.clone(); - padded.claims.sub = format!(" {} ", plain.claims.sub); - - assert_ne!(plain.oidc_virtual_parent(), padded.oidc_virtual_parent()); - } - #[test] fn verified_identity_preserves_exact_values_and_distinct_groups() { let attributes = HashMap::from([ @@ -341,24 +224,6 @@ mod tests { assert!(!debug.contains(" username ")); assert!(!debug.contains(" source-b ")); assert!(!debug.contains("identity-attribute-poison")); - - let authorization = FederatedAuthorization { - provider_id: " Corp Provider ".to_string(), - claims: FederatedClaims { - sub: "subject".to_string(), - email: String::new(), - username: String::new(), - groups: vec!["different-source-group".to_string()], - raw: HashMap::new(), - }, - policies: Vec::new(), - group_claim_policies: Vec::new(), - groups: vec!["mapped-a".to_string(), "mapped-b".to_string()], - roles_claim_key: None, - roles: Vec::new(), - }; - assert_eq!(authorization.claims.groups, ["different-source-group"]); - assert_eq!(authorization.groups, ["mapped-a", "mapped-b"]); } #[test] diff --git a/crates/iam/src/federation/test_support.rs b/crates/iam/src/federation/test_support.rs new file mode 100644 index 000000000..3933ae910 --- /dev/null +++ b/crates/iam/src/federation/test_support.rs @@ -0,0 +1,54 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{FederatedAuthorization, FederatedClaims}; +use serde_json::Value; +use std::collections::HashMap; + +/// Exact authorization inputs for tests that exercise the session binding boundary. +#[derive(Default)] +pub struct FederatedAuthorizationFixture { + pub provider_id: String, + pub sub: String, + pub email: String, + pub username: String, + pub source_groups: Vec, + pub attributes: HashMap, + pub policies: Vec, + pub group_claim_policies: Vec, + pub mapped_iam_groups: Vec, + pub roles_claim_key: Option, + pub roles: Vec, +} + +impl FederatedAuthorizationFixture { + /// Preserves supplied values without authentication, validation, or policy mapping. + pub fn into_authorization(self) -> FederatedAuthorization { + FederatedAuthorization { + provider_id: self.provider_id, + claims: FederatedClaims { + sub: self.sub, + email: self.email, + username: self.username, + groups: self.source_groups, + raw: self.attributes, + }, + policies: self.policies, + group_claim_policies: self.group_claim_policies, + groups: self.mapped_iam_groups, + roles_claim_key: self.roles_claim_key, + roles: self.roles, + } + } +} diff --git a/crates/iam/src/federation/transaction.rs b/crates/iam/src/federation/transaction.rs index 3f7ec15dc..84f4c2969 100644 --- a/crates/iam/src/federation/transaction.rs +++ b/crates/iam/src/federation/transaction.rs @@ -17,6 +17,7 @@ use super::{ FederatedProviderView, FederatedRedirectPolicy, FederatedSession, FederatedSessionBinding, FederatedSessionTransaction, FederationError, OpaqueLogoutContinuation, Result, StandardOidcAuthentication, }; +use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC}; use std::sync::Arc; const DEFAULT_OIDC_PROVIDER_ID: &str = "default"; @@ -133,22 +134,28 @@ impl FederatedIdentityService { let authorization = self.verify_identity(jwt).await?; if !authorization.has_authorization_context() { tracing::warn!( - provider_id = %authorization.provider_id, - username = %authorization.claims.username, - sub = %authorization.claims.sub, - policy_count = authorization.policies.len(), - group_count = authorization.groups.len(), + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + provider_id = %authorization.provider_id(), + username = %authorization.username(), + sub = %authorization.subject(), + policy_count = authorization.policies().len(), + group_count = authorization.mapped_iam_groups().len(), "AssumeRoleWithWebIdentity has no mapped policies or groups" ); return Err(FederationError::NoAuthorizationContext); } tracing::debug!( - provider_id = %authorization.provider_id, - username = %authorization.claims.username, - policy_count = authorization.policies.len(), - group_count = authorization.groups.len(), - policies = ?authorization.policies, - groups = ?authorization.groups, + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + provider_id = %authorization.provider_id(), + username = %authorization.username(), + policy_count = authorization.policies().len(), + group_count = authorization.mapped_iam_groups().len(), + policies = ?authorization.policies(), + groups = ?authorization.mapped_iam_groups(), "AssumeRoleWithWebIdentity mapped OIDC policies and groups" ); @@ -379,19 +386,19 @@ mod tests { ) -> core::result::Result { self.events.lock().expect("event log should not be poisoned").push("bind"); self.transactions.lock().expect("transactions should not be poisoned").push(( - transaction.authorization.provider_id.clone(), + transaction.authorization.provider_id().to_string(), transaction.duration_seconds, transaction.session_policy.clone(), )); self.mapped_policies .lock() .expect("mapped policies should not be poisoned") - .push(transaction.authorization.policies.clone()); + .push(transaction.authorization.policies().to_vec()); if self.fail { return Err(FederatedSessionBindingError::Internal("binding failed".to_string())); } Ok(Credentials { - access_key: transaction.authorization.claims.session_identity(), + access_key: transaction.authorization.session_identity(), ..Default::default() }) } @@ -487,7 +494,7 @@ mod tests { .await .expect("callback flow should complete"); assert_eq!(login.session.credentials.access_key, "user"); - assert_eq!(login.session.authorization.provider_id, "corp"); + assert_eq!(login.session.authorization.provider_id(), "corp"); assert_eq!(login.redirect_after.as_deref(), Some("/browser")); assert_eq!(login.logout_token, "logout-token"); assert_eq!( @@ -501,7 +508,7 @@ mod tests { .await .expect("web identity flow should complete"); assert_eq!(web_identity.credentials.access_key, "user"); - assert_eq!(web_identity.authorization.provider_id, "partner"); + assert_eq!(web_identity.authorization.provider_id(), "partner"); assert_eq!(events.lock().expect("event log should not be poisoned").as_slice(), ["verify", "bind"]); assert_eq!( binding @@ -530,8 +537,8 @@ mod tests { .complete_authorization_code("state", "code", "https://console.example/callback", 3600, binding.as_ref()) .await .expect("standard callback flow should complete"); - assert_eq!(login.session.authorization.provider_id, "corp"); - assert_eq!(login.session.authorization.policies, ["readwrite"]); + assert_eq!(login.session.authorization.provider_id(), "corp"); + assert_eq!(login.session.authorization.policies(), ["readwrite"]); assert_eq!( events.lock().expect("event log should not be poisoned").as_slice(), ["exchange", "bind", "logout"] @@ -542,8 +549,8 @@ mod tests { .assume_role_with_web_identity("jwt", 7200, Some("session-policy".to_string()), binding.as_ref()) .await .expect("standard web identity flow should complete"); - assert_eq!(web_identity.authorization.provider_id, "partner"); - assert_eq!(web_identity.authorization.policies, ["readwrite"]); + assert_eq!(web_identity.authorization.provider_id(), "partner"); + assert_eq!(web_identity.authorization.policies(), ["readwrite"]); assert_eq!(events.lock().expect("event log should not be poisoned").as_slice(), ["verify", "bind"]); assert_eq!( binding @@ -586,8 +593,8 @@ mod tests { .await .expect("a mapped group is an authorization context"); - assert_eq!(session.authorization.policies, ["developers"]); - assert_eq!(session.authorization.groups, ["developers"]); + assert_eq!(session.authorization.policies(), ["developers"]); + assert_eq!(session.authorization.mapped_iam_groups(), ["developers"]); assert_eq!(events.lock().expect("event log should not be poisoned").as_slice(), ["verify", "bind"]); } diff --git a/crates/iam/src/oidc/recovery_tests.rs b/crates/iam/src/oidc/recovery_tests.rs index f1b500b85..96fece5bc 100644 --- a/crates/iam/src/oidc/recovery_tests.rs +++ b/crates/iam/src/oidc/recovery_tests.rs @@ -177,9 +177,9 @@ async fn web_identity_recovers_transport_failure_and_preserves_verification() { .await .expect("verify recovered workload token"); let authorization = mapper.map(identity); - assert_eq!(authorization.provider_id, "workload"); - assert_eq!(authorization.claims.sub, "recovered-user"); - assert_eq!(authorization.policies, vec!["team/readonly"]); + assert_eq!(authorization.provider_id(), "workload"); + assert_eq!(authorization.subject(), "recovered-user"); + assert_eq!(authorization.policies(), ["team/readonly"]); assert!( sys.authorize_url("workload", "https://console.example.com/callback", None) .await diff --git a/crates/iam/src/oidc/runtime_tests.rs b/crates/iam/src/oidc/runtime_tests.rs index 84dd95943..52181e530 100644 --- a/crates/iam/src/oidc/runtime_tests.rs +++ b/crates/iam/src/oidc/runtime_tests.rs @@ -371,12 +371,12 @@ async fn assert_complete_console_login(initially_unavailable: bool) { .await .unwrap_or_else(|err| panic!("hidden={hidden}, explicit={explicit}: {err}")); let (identity, redirect_after, _) = exchange.into_parts(); + assert_eq!(identity.source_groups(), ["readwrite"]); let authorization = mapper.map(identity); - assert_eq!(authorization.provider_id, "console"); - assert_eq!(authorization.claims.sub, "existing-user"); - assert_eq!(authorization.claims.email, "user@example.com"); - assert_eq!(authorization.claims.groups, vec!["readwrite"]); - assert_eq!(authorization.policies, vec!["team/readwrite"]); + assert_eq!(authorization.provider_id(), "console"); + assert_eq!(authorization.subject(), "existing-user"); + assert_eq!(authorization.email(), "user@example.com"); + assert_eq!(authorization.policies(), ["team/readwrite"]); assert_eq!(redirect_after.as_deref(), Some("/buckets")); assert!(matches!(sys.exchange_code(&state, "test-authorization-code", redirect).await, Err(error) if error == "invalid or expired OIDC state")); diff --git a/docs/architecture/crate-boundaries.md b/docs/architecture/crate-boundaries.md index 7a37c707f..9d940e37b 100644 --- a/docs/architecture/crate-boundaries.md +++ b/docs/architecture/crate-boundaries.md @@ -85,6 +85,8 @@ The server-config model (`Config`, `KV`, `KVS`) and the global server-config sna `crates/iam/src/oidc/` owns provider configuration, discovery, HTTP transport, login state, and the OIDC runtime. `crates/iam/src/federation/` owns verified identity transactions and authorization mapping; its production code does not import concrete OIDC runtime or provider configuration types. Production `FederatedAuthorization` values are constructed by `CoreFederatedAuthorizationMapper` in `crates/iam/src/federation/mapper.rs`. +`FederatedClaims` is crate-internal. `FederatedAuthorization` has private fields owned by the mapper module and exposes read-only accessors to session binding consumers, including `mapped_iam_groups()` for the IAM groups selected by authorization mapping. Test fixtures are exposed only through the `test-util` feature or unit-test configuration. + `rustfs/src/startup_auth.rs` constructs `StandardOidcAdapter` and publishes the federation service with its OIDC query as one runtime pair. `AppContext` stores and publishes that pair through `FederatedIdentityRuntimeSnapshot`; consumers receive interfaces rather than constructing a second OIDC runtime. `scripts/check_oidc_architecture_boundaries.sh` enforces these boundaries through the existing architecture guard, including the Keycloak workflow path for `crates/iam/src/oidc/`. `crates/iam/src/oidc/config.rs` owns OIDC configuration input, validation, persisted KVS conversion, and `OidcSiteReplicationSnapshot`. `rustfs/src/admin/service/oidc_config.rs` owns stored configuration reads, compare-and-save updates, restart comparison, and provider discovery validation; the OIDC Admin handler translates HTTP requests and errors. `OidcConfigQuery::site_replication_snapshot()` returns only the provider ID, claim name, role policy, client ID, and hashed client secret needed by site replication. `rustfs/src/admin/handlers/site_replication.rs` maps this narrow view to its existing response format without reading raw OIDC credentials. The Keycloak workflow tracks the Admin configuration service path along with IAM OIDC code. diff --git a/rustfs/src/admin/handlers/oidc.rs b/rustfs/src/admin/handlers/oidc.rs index 8a095d2e1..383ae9d72 100644 --- a/rustfs/src/admin/handlers/oidc.rs +++ b/rustfs/src/admin/handlers/oidc.rs @@ -477,7 +477,7 @@ impl Operation for OidcCallbackHandler { callback_federation_error(error) })?; let authorization = &login.session.authorization; - let actual_provider_id = &authorization.provider_id; + let actual_provider_id = authorization.provider_id(); debug!( event = EVENT_ADMIN_OIDC_STATE, @@ -493,10 +493,10 @@ impl Operation for OidcCallbackHandler { component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_OIDC, provider_id = %actual_provider_id, - policy_count = authorization.policies.len(), - group_count = authorization.groups.len(), - policies = ?authorization.policies, - groups = ?authorization.groups, + policy_count = authorization.policies().len(), + group_count = authorization.mapped_iam_groups().len(), + policies = ?authorization.policies(), + groups = ?authorization.mapped_iam_groups(), state = "claims_mapped", "admin oidc state" ); diff --git a/rustfs/src/admin/handlers/sts.rs b/rustfs/src/admin/handlers/sts.rs index 054a1d11c..75211130c 100644 --- a/rustfs/src/admin/handlers/sts.rs +++ b/rustfs/src/admin/handlers/sts.rs @@ -432,7 +432,7 @@ async fn handle_assume_role_with_web_identity(body: AssumeRoleRequest) -> S3Resu })?; let authorization = &session.authorization; let new_cred = &session.credentials; - let subject = authorization.claims.session_identity(); + let subject = authorization.session_identity(); let xml = assume_role_with_web_identity_response_xml(new_cred, &subject); diff --git a/rustfs/src/admin/service/federated_identity.rs b/rustfs/src/admin/service/federated_identity.rs index cdc197633..679539079 100644 --- a/rustfs/src/admin/service/federated_identity.rs +++ b/rustfs/src/admin/service/federated_identity.rs @@ -108,34 +108,33 @@ fn select_oidc_policies( fn build_oidc_token_claims(transaction: &FederatedSessionTransaction) -> HashMap { let authorization = &transaction.authorization; - let claims = &authorization.claims; let mut token_claims = HashMap::new(); - token_claims.insert("sub".to_string(), Value::String(claims.sub.clone())); + token_claims.insert("sub".to_string(), Value::String(authorization.subject().to_string())); token_claims.insert("iss".to_string(), Value::String("rustfs-oidc".to_string())); - token_claims.insert("oidc_provider".to_string(), Value::String(authorization.provider_id.clone())); + token_claims.insert("oidc_provider".to_string(), Value::String(authorization.provider_id().to_string())); - if !claims.email.is_empty() { - token_claims.insert("email".to_string(), Value::String(claims.email.clone())); + if !authorization.email().is_empty() { + token_claims.insert("email".to_string(), Value::String(authorization.email().to_string())); } - if !claims.username.is_empty() { - token_claims.insert("preferred_username".to_string(), Value::String(claims.username.clone())); + if !authorization.username().is_empty() { + token_claims.insert("preferred_username".to_string(), Value::String(authorization.username().to_string())); } - if !authorization.groups.is_empty() { + if !authorization.mapped_iam_groups().is_empty() { token_claims.insert( "groups".to_string(), Value::Array( authorization - .groups + .mapped_iam_groups() .iter() .map(|group| Value::String(group.clone())) .collect(), ), ); } - if !authorization.roles.is_empty() { + if !authorization.roles().is_empty() { token_claims.insert( "roles".to_string(), - Value::Array(authorization.roles.iter().cloned().map(Value::String).collect()), + Value::Array(authorization.roles().iter().cloned().map(Value::String).collect()), ); } @@ -246,7 +245,6 @@ fn issue_credentials( secret: Option<&str>, ) -> Result { let authorization = &transaction.authorization; - let claims = &authorization.claims; let parent_user = authorization.oidc_virtual_parent().ok_or_else(|| { FederatedSessionBindingError::InvalidRequest("verified OIDC identity is missing issuer or subject".to_string()) })?; @@ -266,16 +264,19 @@ fn issue_credentials( )); } debug!( - provider_id = %authorization.provider_id, + event = EVENT_ADMIN_OIDC_POLICY, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, + provider_id = %authorization.provider_id(), parent_user = %parent_user, - email = %claims.email, - username = %claims.username, - sub = %claims.sub, - policy_count = authorization.policies.len(), - group_count = authorization.groups.len(), - policies = ?authorization.policies, - groups = ?authorization.groups, - roles_claim_key = ?authorization.roles_claim_key, + email = %authorization.email(), + username = %authorization.username(), + sub = %authorization.subject(), + policy_count = authorization.policies().len(), + group_count = authorization.mapped_iam_groups().len(), + policies = ?authorization.policies(), + groups = ?authorization.mapped_iam_groups(), + roles_claim_key = ?authorization.roles_claim_key(), has_session_policy = transaction.session_policy.is_some(), "OIDC STS credential claims prepared" ); @@ -295,7 +296,7 @@ fn issue_credentials( let mut credentials = get_new_credentials_with_metadata(&token_claims, secret) .map_err(|error| FederatedSessionBindingError::Internal(format!("credential generation failed: {error}")))?; credentials.parent_user = parent_user; - credentials.groups = Some(authorization.groups.clone()); + credentials.groups = Some(authorization.mapped_iam_groups().to_vec()); Ok(credentials) } @@ -329,12 +330,12 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { FederatedSessionBindingError::InvalidRequest("verified OIDC identity is missing issuer or subject".to_string()) })?; let (selected_policy_names, ignorable_policy_names) = match iam_store - .sts_policy_db_get(&parent_user, &Some(authorization.groups.clone())) + .sts_policy_db_get(&parent_user, &Some(authorization.mapped_iam_groups().to_vec())) .await .map_err(|_| FederatedSessionBindingError::Internal("failed to resolve OIDC policy mapping".to_string()))? { mapped_policy_names if !mapped_policy_names.is_empty() => (mapped_policy_names, &[][..]), - _ => (authorization.policies.clone(), authorization.group_claim_policies.as_slice()), + _ => (authorization.policies().to_vec(), authorization.group_claim_policies()), }; // A name may be skipped only after storage confirms the policy is absent. A failed lookup // must not be mistaken for absence, or an unreadable Deny policy would be dropped from the @@ -346,7 +347,7 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { event = EVENT_ADMIN_OIDC_POLICY, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_OIDC, - provider_id = %authorization.provider_id, + provider_id = %authorization.provider_id(), parent_user = %parent_user, policy = %policy_name, error = %err, @@ -377,7 +378,7 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { event = EVENT_ADMIN_OIDC_POLICY, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_OIDC, - provider_id = %authorization.provider_id, + provider_id = %authorization.provider_id(), parent_user = %parent_user, unsupported_policies = ?policy_names, "OIDC STS rejected existing policies whose names are not allowed in session claims" @@ -389,7 +390,7 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { event = EVENT_ADMIN_OIDC_POLICY, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_OIDC, - provider_id = %authorization.provider_id, + provider_id = %authorization.provider_id(), parent_user = %parent_user, unresolved_policies = ?policy_names, "OIDC STS rejected required policies without a current policy" @@ -405,7 +406,7 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { event = EVENT_ADMIN_OIDC_POLICY, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_OIDC, - provider_id = %authorization.provider_id, + provider_id = %authorization.provider_id(), parent_user = %parent_user, ignored_group_policies = ?selection.ignored, "OIDC STS ignoring group claim values without a matching policy" @@ -418,10 +419,10 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { if tracing::enabled!(tracing::Level::DEBUG) { log_oidc_policy_diagnostics( &iam_store, - &authorization.provider_id, + authorization.provider_id(), &credentials.parent_user, &selected_policy_names, - &authorization.groups, + authorization.mapped_iam_groups(), ) .await; } @@ -444,7 +445,7 @@ mod tests { use super::*; use crate::admin::runtime_sources::{AppContext, current_object_store_handle, publish_test_app_context}; use hmac::{Hmac, KeyInit, Mac}; - use rustfs_iam::federation::{FederatedAuthorization, FederatedClaims}; + use rustfs_iam::federation::FederatedAuthorizationFixture; use rustfs_iam::store::{ Store, UserType, object::{IAM_CONFIG_POLICIES_PREFIX, IAM_CONFIG_PREFIX}, @@ -459,31 +460,37 @@ mod tests { use sha2::Sha512; use std::sync::Arc; - fn transaction() -> FederatedSessionTransaction { + fn authorization_fixture() -> FederatedAuthorizationFixture { + FederatedAuthorizationFixture { + provider_id: "default".to_string(), + sub: "subject".to_string(), + email: "user@example.com".to_string(), + username: "user".to_string(), + source_groups: vec!["source-group".to_string()], + attributes: HashMap::from([ + ("iss".to_string(), serde_json::json!("https://idp.example.test")), + ("raw_poison".to_string(), serde_json::json!("must-not-leak")), + ]), + policies: vec!["readwrite".to_string()], + group_claim_policies: Vec::new(), + mapped_iam_groups: vec!["devs".to_string()], + roles_claim_key: Some("roles".to_string()), + roles: vec!["admin".to_string(), "reader".to_string()], + } + } + + fn transaction_from_fixture(authorization: FederatedAuthorizationFixture) -> FederatedSessionTransaction { FederatedSessionTransaction { - authorization: FederatedAuthorization { - provider_id: "default".to_string(), - claims: FederatedClaims { - sub: "subject".to_string(), - email: "user@example.com".to_string(), - username: "user".to_string(), - groups: vec!["source-group".to_string()], - raw: HashMap::from([ - ("iss".to_string(), serde_json::json!("https://idp.example.test")), - ("raw_poison".to_string(), serde_json::json!("must-not-leak")), - ]), - }, - policies: vec!["readwrite".to_string()], - group_claim_policies: Vec::new(), - groups: vec!["devs".to_string()], - roles_claim_key: Some("roles".to_string()), - roles: vec!["admin".to_string(), "reader".to_string()], - }, + authorization: authorization.into_authorization(), duration_seconds: 3600, session_policy: None, } } + fn transaction() -> FederatedSessionTransaction { + transaction_from_fixture(authorization_fixture()) + } + async fn ready_test_iam() -> Arc> { let _ = rustfs_credentials::init_global_action_credentials( Some("TESTROOTACCESSKEY".to_string()), @@ -547,11 +554,12 @@ mod tests { #[test] fn token_claims_omit_empty_optional_oidc_fields() { - let mut transaction = transaction(); - transaction.authorization.claims.email.clear(); - transaction.authorization.claims.username.clear(); - transaction.authorization.groups.clear(); - transaction.authorization.roles.clear(); + let mut authorization = authorization_fixture(); + authorization.email.clear(); + authorization.username.clear(); + authorization.mapped_iam_groups.clear(); + authorization.roles.clear(); + let transaction = transaction_from_fixture(authorization); assert_eq!( build_oidc_token_claims(&transaction), @@ -650,9 +658,10 @@ mod tests { #[serial] async fn binding_uses_sts_policy_when_regular_mapping_collides() { let iam = ready_test_iam().await; - let mut transaction = transaction(); - transaction.authorization.claims.sub = "binding-sts-policy-subject".to_string(); - transaction.authorization.policies = vec!["readwrite".to_string()]; + let mut authorization = authorization_fixture(); + authorization.sub = "binding-sts-policy-subject".to_string(); + authorization.policies = vec!["readwrite".to_string()]; + let transaction = transaction_from_fixture(authorization); let parent = transaction .authorization .oidc_virtual_parent() @@ -766,7 +775,7 @@ mod tests { let mut transaction = transaction(); transaction.session_policy = Some("not-json".to_string()); - let error = issue_credentials(&transaction, &transaction.authorization.policies, None) + let error = issue_credentials(&transaction, transaction.authorization.policies(), None) .expect_err("invalid policy should fail first"); assert!(matches!(error, FederatedSessionBindingError::InvalidRequest(_))); } @@ -798,10 +807,11 @@ mod tests { #[test] fn missing_oidc_issuer_fails_closed_before_credential_issuance() { - let mut transaction = transaction(); - transaction.authorization.claims.raw.clear(); + let mut authorization = authorization_fixture(); + authorization.attributes.clear(); + let transaction = transaction_from_fixture(authorization); - let error = issue_credentials(&transaction, &transaction.authorization.policies, Some("signing-secret")) + let error = issue_credentials(&transaction, transaction.authorization.policies(), Some("signing-secret")) .expect_err("credential issuance should reject a missing issuer"); assert!(matches!(error, FederatedSessionBindingError::InvalidRequest(_))); } @@ -881,12 +891,12 @@ mod tests { } fn group_claim_transaction(subject: &str, groups: &[&str]) -> FederatedSessionTransaction { - let mut transaction = transaction(); - transaction.authorization.claims.sub = subject.to_string(); - transaction.authorization.groups = names(groups); - transaction.authorization.policies = names(groups); - transaction.authorization.group_claim_policies = names(groups); - transaction + let mut authorization = authorization_fixture(); + authorization.sub = subject.to_string(); + authorization.mapped_iam_groups = names(groups); + authorization.policies = names(groups); + authorization.group_claim_policies = names(groups); + transaction_from_fixture(authorization) } fn assert_binding_rejected(error: FederatedSessionBindingError, expected_message: &str) { @@ -914,7 +924,7 @@ mod tests { .expect("issued session token should verify"); assert_eq!(claims.get("policy"), Some(&serde_json::json!("readonly"))); - assert_eq!(credentials.groups.as_deref(), Some(transaction.authorization.groups.as_slice())); + assert_eq!(credentials.groups.as_deref(), Some(transaction.authorization.mapped_iam_groups())); let groups = credentials.groups.clone(); let conditions = HashMap::new(); @@ -1003,10 +1013,11 @@ mod tests { #[serial] async fn binding_keeps_role_policy_strict() { ready_test_iam().await; - let mut transaction = transaction(); - transaction.authorization.claims.sub = "binding-strict-role-policy-subject".to_string(); - transaction.authorization.policies = names(&["readonly", "missing-role-policy"]); - transaction.authorization.group_claim_policies = Vec::new(); + let mut authorization = authorization_fixture(); + authorization.sub = "binding-strict-role-policy-subject".to_string(); + authorization.policies = names(&["readonly", "missing-role-policy"]); + authorization.group_claim_policies = Vec::new(); + let transaction = transaction_from_fixture(authorization); let error = DefaultFederatedSessionBinding .bind(&transaction) From adf17bbfbaed0fb0f6c16a592ecd80be4642e70d Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 13 Oct 2026 19:12:28 +0800 Subject: [PATCH 044/143] fix(heal): reject incomplete administrator bucket discovery (#8512) --- .config/e2e-full-selection.txt | 4 +- .config/e2e-nightly-selection.txt | 4 +- .../src/heal_erasure_disk_rebuild_test.rs | 152 ++++++++++++++++++ crates/ecstore/src/store/bucket.rs | 91 ++++++++++- crates/heal/src/error.rs | 5 +- crates/heal/src/heal/channel.rs | 7 + crates/heal/src/heal/erasure_healer.rs | 13 ++ crates/heal/src/heal/erasure_healer/admin.rs | 3 +- .../resume_loop_tests/admin_outcome.rs | 47 ++++++ crates/heal/src/heal/manager/tests.rs | 29 ++++ .../heal/manager/tests/running_mainline.rs | 7 + crates/heal/src/heal/storage.rs | 12 ++ crates/heal/src/heal/task/heal_bucket.rs | 7 +- crates/heal/src/heal/task/tests.rs | 64 ++++++++ crates/heal/src/lib.rs | 7 + crates/heal/tests/heal_bug_fixes_test.rs | 14 ++ .../heal/tests/heal_concurrent_delete_test.rs | 7 + crates/heal/tests/storage_api.rs | 2 +- .../src/scanner_folder/tests/mrf_ownership.rs | 7 + crates/scanner/src/storage_api.rs | 2 + docs/operations/bucket-heal-recovery.md | 23 +++ rustfs/src/storage/rpc/node_service.rs | 7 + rustfs/src/storage/storage_api.rs | 2 + 23 files changed, 504 insertions(+), 12 deletions(-) diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index 2040896ee..8b2d39b00 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=b8937d6de585895a837b1505c7a9f91aa45394fbdc47cd179287a35a6f1de8be -sha256-linux=e803ee5fbd29bd45b8542b4256b75d5a2196ccc7ccfd1ccac6e14553730e9047 +sha256-darwin=a58501e7f6f1407201170c300f5fe91032a43addd4cb215306cf321ab59a0805 +sha256-linux=57791d182063049c5f0ca91ae0a6bf64006af6f991a5094e0cb7c8d8491187ed diff --git a/.config/e2e-nightly-selection.txt b/.config/e2e-nightly-selection.txt index 9a0869842..10bd6dfeb 100644 --- a/.config/e2e-nightly-selection.txt +++ b/.config/e2e-nightly-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=ce7b419faf1ff79840dfc6325d31ee8d1ca5eab85125b94d0bcd58cdf07f0f76 -sha256-linux=5e9e3d701ee873c13fb9094575f6889779c6078b1107624bb57663bef9b9d21f +sha256-darwin=8ef80b7353d1e0c4f863a2aca7e996d23858b8c32b9edf8fd272076fa63a83d8 +sha256-linux=99fe1cd128ade2df69b9e3ecf94b91021185a9769293420b1ba5d24760f2ea99 diff --git a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs index c5d2e20b7..b02fc304f 100644 --- a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs +++ b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs @@ -2633,4 +2633,156 @@ mod tests { }; assert!(!outage_candidate_targets_replacement(&insufficient_peer_shards, 12, replacement_set_slot)); } + async fn two_node_heal_status( + cluster: &RustFSTestClusterEnvironment, + bucket: Option<&str>, + recreate: bool, + scan_mode: u8, + ) -> Result> { + let url = format!("{}/rustfs/admin/v3/heal/{}", cluster.nodes[0].url, bucket.unwrap_or_default()); + let options = serde_json::json!({ + "recursive": true, "dryRun": false, "remove": false, + "recreate": recreate, "scanMode": scan_mode, "updateParity": false, "nolock": false, + }); + let started = + start_root_heal_when_control_ready(&url, &options.to_string(), &cluster.access_key, &cluster.secret_key).await?; + let started: serde_json::Value = serde_json::from_str(&started)?; + let token = started["clientToken"] + .as_str() + .ok_or_else(|| format!("missing heal token: {started}"))?; + let status_url = format!("{url}?clientToken={token}"); + let deadline = Instant::now() + Duration::from_secs(240); + loop { + let body = timeout( + Duration::from_secs(20), + signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key), + ) + .await??; + let status: serde_json::Value = serde_json::from_str(&body)?; + match status["summary"].as_str() { + Some("finished" | "stopped") => return Ok(status), + Some("running") if Instant::now() < deadline => sleep(Duration::from_millis(250)).await, + _ => return Err(format!("heal did not reach a terminal state: {status}").into()), + } + } + } + + async fn verify_two_node_directory_recovery(remove_bucket: bool, scan_mode: u8) -> Result<(), Box> { + init_logging(); + let mut cluster = RustFSTestClusterEnvironment::new(2).await?; + // Isolate explicit administrator recovery from scanner, MRF, and disk healing. + for key in [ + "RUSTFS_HEAL_AUTO_HEAL_ENABLE", + "RUSTFS_HEAL_MRF_ENABLE", + "RUSTFS_SCANNER_ENABLED", + "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", + ] { + cluster.set_env(key, "false"); + } + cluster.start().await?; + let source = cluster.create_s3_client(0)?; + let target = cluster.create_s3_client(1)?; + let bucket = "two-node-directory-recovery"; + source.create_bucket().bucket(bucket).send().await?; + let objects = [ + ("empty", Vec::new()), + ("nested/inline", vec![0x3c; 1024]), + ("nested/external", (0_u8..=255).cycle().take(1024 * 1024).collect::>()), + ]; + for (key, bytes) in &objects { + source + .put_object() + .bucket(bucket) + .key(*key) + .body(ByteStream::from(bytes.clone())) + .send() + .await?; + } + let target_bucket = Path::new(&cluster.nodes[1].data_dir).join(bucket); + let original = objects + .iter() + .map(|(key, _)| census_object_version_on_disk(Path::new(&cluster.nodes[1].data_dir), bucket, key, None)) + .collect::, _>>()?; + for census in &original { + assert!(census.has_xl_meta, "fixture must have a target metadata copy: {census:?}"); + } + assert!( + !original[2].present_part_fingerprints.is_empty(), + "large fixture must exercise external parts" + ); + let removed = Path::new(&cluster.temp_dir).join("removed"); + tokio::fs::create_dir(&removed).await?; + if remove_bucket { + tokio::fs::rename(&target_bucket, removed.join(bucket)).await?; + let status = two_node_heal_status(&cluster, None, false, scan_mode).await?; + assert_eq!( + status["summary"], "stopped", + "missing bucket must not look like an empty successful scan: {status}" + ); + assert_eq!(status["outcome"]["execution"]["state"], "aborted"); + assert_eq!(status["outcome"]["coverage"], "partial"); + assert_eq!(status["outcome"]["counters"]["processed"], 0); + assert!(!target_bucket.exists(), "default discovery must not silently recreate a bucket"); + let scoped_default = two_node_heal_status(&cluster, Some(bucket), false, scan_mode).await?; + assert_eq!( + scoped_default["summary"], "stopped", + "default namespace quorum protection must remain: {scoped_default}" + ); + } else { + for (index, (key, _)) in objects.iter().enumerate() { + tokio::fs::rename(target_bucket.join(key), removed.join(index.to_string())).await?; + } + } + let status = two_node_heal_status(&cluster, remove_bucket.then_some(bucket), remove_bucket, scan_mode).await?; + assert_eq!(status["summary"], "finished", "explicit recovery should complete: {status}"); + assert_eq!(status["outcome"]["execution"]["state"], "completed"); + assert_eq!(status["outcome"]["coverage"], "complete"); + for ((key, _), before) in objects.iter().zip(&original) { + let after = census_object_version_on_disk(Path::new(&cluster.nodes[1].data_dir), bucket, key, None)?; + assert!(after.has_xl_meta, "metadata must be restored before GET: {key}"); + assert_eq!( + after.present_part_fingerprints, before.present_part_fingerprints, + "external shard bytes must match: {key}" + ); + assert_eq!( + after.inline_data_fingerprint, before.inline_data_fingerprint, + "inline shard bytes must match: {key}" + ); + } + cluster.stop_node(0)?; + for (key, expected) in &objects { + let actual = timeout(Duration::from_secs(30), async { + target + .get_object() + .bucket(bucket) + .key(*key) + .send() + .await? + .body + .collect() + .await + .map(|body| body.into_bytes()) + .map_err(Box::::from) + }) + .await??; + assert_eq!(actual.len(), expected.len(), "target-only read length: {key}"); + assert_eq!(sha256_hex(&actual), sha256_hex(expected), "target-only content: {key}"); + } + Ok(()) + } + + #[tokio::test] + async fn test_two_node_bucket_discovery_and_explicit_recreate() -> Result<(), Box> { + verify_two_node_directory_recovery(true, 2).await + } + + #[tokio::test] + async fn test_two_node_object_directory_recovery_normal() -> Result<(), Box> { + verify_two_node_directory_recovery(false, 1).await + } + + #[tokio::test] + async fn test_two_node_object_directory_recovery_deep() -> Result<(), Box> { + verify_two_node_directory_recovery(false, 2).await + } } diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 57ccd3581..1d02140d8 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -20,6 +20,7 @@ use crate::bucket::{ use crate::error::is_err_bucket_not_found; use crate::runtime::sources as runtime_sources; use crate::set_disk::{BucketInfoQuorum, get_lock_acquire_timeout}; +use crate::storage_api_contracts::admin::DiskSetSelector; use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp}; use crate::storage_api_contracts::namespace::NamespaceLocking as _; use futures::stream::{self, StreamExt}; @@ -1232,13 +1233,13 @@ impl ECStore { pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result> { // TODO(backlog): support cached bucket listing via opts.cached Ok(self - .list_bucket_from_sets(opts, usize::MAX, BucketInfoQuorum::Write) + .list_bucket_from_sets(opts, usize::MAX, BucketInfoQuorum::Write, None) .await? .buckets) } pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result { - self.list_bucket_from_sets(opts, SCANNER_BUCKET_LIST_SET_CONCURRENCY, BucketInfoQuorum::Write) + self.list_bucket_from_sets(opts, SCANNER_BUCKET_LIST_SET_CONCURRENCY, BucketInfoQuorum::Write, None) .await } @@ -1254,20 +1255,44 @@ impl ECStore { }, SCANNER_BUCKET_LIST_SET_CONCURRENCY, BucketInfoQuorum::Read, + None, ) .await? .buckets) } + /// Preserve namespace completeness for administrator traversal. A scoped + /// request must not depend on unrelated sets being available. + pub async fn list_buckets_for_admin_heal( + &self, + selector: Option, + ) -> Result { + self.list_bucket_from_sets( + &BucketOptions { + no_metadata: true, + ..Default::default() + }, + SCANNER_BUCKET_LIST_SET_CONCURRENCY, + BucketInfoQuorum::Write, + selector, + ) + .await + } + async fn list_bucket_from_sets( &self, opts: &BucketOptions, max_concurrency: usize, quorum: BucketInfoQuorum, + selector: Option, ) -> Result { - let set_count = self.pools.iter().map(|pool| pool.disk_set.len()).sum(); + let sets = self + .bucket_sets() + .filter(|(pool, set, _)| selector.is_none_or(|scope| scope.pool_idx == *pool && scope.set_idx == *set)) + .collect::>(); + let set_count = sets.len(); let concurrency = bucket_list_set_concurrency(set_count, max_concurrency); - let mut set_listings = stream::iter(self.bucket_sets().map(move |(pool_index, set_index, set)| async move { + let mut set_listings = stream::iter(sets.into_iter().map(move |(pool_index, set_index, set)| async move { set.list_bucket_with_quorum(quorum) .await .map(|(buckets, complete)| (pool_index, set_index, buckets, complete)) @@ -2236,6 +2261,64 @@ mod tests { } } + #[tokio::test] + async fn admin_bucket_discovery_preserves_completeness_and_scope() { + use crate::storage_api_contracts::admin::DiskSetSelector; + let (temp, store) = setup_bucket_quorum_test_env(&[2, 2], None).await; + let empty = store + .list_buckets_for_admin_heal(None) + .await + .expect("empty namespace enumeration"); + assert!(empty.buckets.is_empty()); + assert!(empty.topology_complete); + for disk in 0..2 { + tokio::fs::create_dir(temp.path().join(format!("pool0-disk{disk}/healthy-bucket"))) + .await + .expect("healthy bucket"); + } + tokio::fs::create_dir(temp.path().join("pool1-disk0/partial-bucket")) + .await + .expect("single surviving bucket directory"); + let global = store + .list_buckets_for_admin_heal(None) + .await + .expect("online disk enumeration"); + assert!(!global.topology_complete, "a filtered sub-quorum bucket must invalidate coverage"); + assert_eq!(global.buckets.iter().map(|b| b.name.as_str()).collect::>(), ["healthy-bucket"]); + let healthy = store + .list_buckets_for_admin_heal(Some(DiskSetSelector::new(0, 0))) + .await + .expect("healthy scope"); + assert!(healthy.topology_complete); + assert_eq!(healthy.buckets.len(), 1); + assert_eq!(healthy.set_buckets.len(), 1); + let partial = store + .list_buckets_for_admin_heal(Some(DiskSetSelector::new(1, 0))) + .await + .expect("partial scope"); + assert!(!partial.topology_complete); + assert!( + partial.buckets.is_empty(), + "missing buckets cannot be mistaken for a complete empty namespace" + ); + let invalid = store + .list_buckets_for_admin_heal(Some(DiskSetSelector::new(0, 1))) + .await + .expect("unconfigured scope"); + assert!(!invalid.topology_complete); + assert!(invalid.buckets.is_empty()); + let offline = take_set_disks_offline(&store, &store.pools[1].disk_set[0], &[0]).await; + assert!(store.list_buckets_for_admin_heal(None).await.is_err(), "quorum errors must remain errors"); + assert!( + store + .list_buckets_for_admin_heal(Some(DiskSetSelector::new(0, 0))) + .await + .expect("unrelated offline pool") + .topology_complete + ); + restore_set_disks(&store, &store.pools[1].disk_set[0], offline).await; + } + #[test] fn should_not_override_when_metadata_created_is_unix_epoch() { assert!(!should_override_created_from_metadata(OffsetDateTime::UNIX_EPOCH)); diff --git a/crates/heal/src/error.rs b/crates/heal/src/error.rs index d1db7292a..ac901e921 100644 --- a/crates/heal/src/error.rs +++ b/crates/heal/src/error.rs @@ -90,6 +90,9 @@ pub enum Error { source: Box, }, + #[error("Bucket discovery incomplete for the requested heal scope; not all buckets could be traversed")] + BucketDiscoveryIncomplete, + #[error("Invalid heal type: {heal_type}")] InvalidHealType { heal_type: String }, @@ -127,7 +130,7 @@ impl Error { match self { Error::TaskCancelled | Error::TaskTimeout | Error::StaleBucketIncarnation { .. } => false, Error::ReplacementTargetNotReady(_) | Error::DanglingDeleteDeferred { .. } => true, - Error::TransientSkip { .. } => true, + Error::TransientSkip { .. } | Error::BucketDiscoveryIncomplete => true, // Lock failures classify by LockError's own taxonomy: only the // fatal variants (ResourceNotFound / PermissionDenied / // Configuration) are terminal - retrying cannot fix them - while diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index 20b37eff2..5e78e43d8 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -853,6 +853,13 @@ mod tests { async fn get_bucket_info(&self, _bucket: &str) -> crate::Result> { Ok(None) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> crate::Result> { + self.list_buckets().await + } + async fn list_buckets(&self) -> crate::Result> { Ok(vec![]) } diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index 5a02978ba..fdc45fcf4 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -2390,6 +2390,8 @@ mod resume_loop_tests { #[derive(Default)] struct FakeStorage { admin_disk: Mutex>, + bucket_discovery_incomplete: AtomicBool, + bucket_discovery_scopes: Mutex>>, bucket_pages: Mutex>, incarnations: Mutex>, receipts: Mutex>>, @@ -2469,6 +2471,17 @@ mod resume_loop_tests { ..Default::default() })) } + async fn list_buckets_for_admin_heal( + &self, + selector: Option, + ) -> Result> { + self.bucket_discovery_scopes.lock().expect("discovery scopes").push(selector); + if self.bucket_discovery_incomplete.load(Ordering::SeqCst) { + return Err(Error::BucketDiscoveryIncomplete); + } + self.list_buckets().await + } + async fn list_buckets(&self) -> Result> { Ok(self .bucket_pages diff --git a/crates/heal/src/heal/erasure_healer/admin.rs b/crates/heal/src/heal/erasure_healer/admin.rs index 5c9a686cd..bb88cdded 100644 --- a/crates/heal/src/heal/erasure_healer/admin.rs +++ b/crates/heal/src/heal/erasure_healer/admin.rs @@ -51,9 +51,10 @@ impl ErasureSetHealer { } let resolved; let buckets = if buckets.is_empty() { + let (pool_idx, set_idx) = crate::heal::utils::parse_set_disk_id(set_disk_id)?; resolved = self .storage - .list_buckets() + .list_buckets_for_admin_heal(Some(crate::heal::storage_api::storage::DiskSetSelector::new(pool_idx, set_idx))) .await? .into_iter() .map(|bucket| bucket.name) diff --git a/crates/heal/src/heal/erasure_healer/resume_loop_tests/admin_outcome.rs b/crates/heal/src/heal/erasure_healer/resume_loop_tests/admin_outcome.rs index e893f8eca..30a51fbe7 100644 --- a/crates/heal/src/heal/erasure_healer/resume_loop_tests/admin_outcome.rs +++ b/crates/heal/src/heal/erasure_healer/resume_loop_tests/admin_outcome.rs @@ -483,3 +483,50 @@ async fn admin_erasure_checkpoint_publication_cannot_undo_acknowledged_cancel() assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); assert_eq!((outcome.counters.processed, outcome.counters.healed), (1, 1)); } + +#[tokio::test] +async fn admin_bucket_discovery_rejects_incomplete_checkpoint_and_freezes_complete_empty_scope() { + let env = make_env().await; + env.storage.bucket_discovery_incomplete.store(true, Ordering::SeqCst); + let result = env + .healer + .prepare_admin_checkpoint(&env.checkpoint, &[], "pool_0_set_0") + .await; + assert!(matches!(result, Err(Error::BucketDiscoveryIncomplete))); + assert!(env.checkpoint.get_checkpoint().await.admin.is_none()); + let reloaded = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id) + .await + .expect("reload checkpoint"); + assert!(reloaded.get_checkpoint().await.admin.is_none(), "incomplete scope must not be persisted"); + assert_eq!( + env.storage.bucket_discovery_scopes.lock().expect("scopes").as_slice(), + &[Some(crate::heal::storage_api::storage::DiskSetSelector::new(0, 0))] + ); + + env.storage.bucket_discovery_incomplete.store(false, Ordering::SeqCst); + assert!( + env.healer + .prepare_admin_checkpoint(&env.checkpoint, &[], "pool_0_set_0") + .await + .expect("complete empty scope") + .is_empty() + ); + env.storage.bucket_discovery_incomplete.store(true, Ordering::SeqCst); + objects(&env.storage, "created-after-admission", &["object"]); + let reloaded = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id) + .await + .expect("reload frozen scope"); + assert!( + env.healer + .prepare_admin_checkpoint(&reloaded, &[], "pool_0_set_0") + .await + .expect("resume frozen scope") + .is_empty() + ); + assert_eq!( + env.storage.bucket_discovery_scopes.lock().expect("scopes").len(), + 2, + "resume must not rediscover or expand scope" + ); + assert!(env.storage.calls().is_empty()); +} diff --git a/crates/heal/src/heal/manager/tests.rs b/crates/heal/src/heal/manager/tests.rs index 997c85e64..4829b7c0b 100644 --- a/crates/heal/src/heal/manager/tests.rs +++ b/crates/heal/src/heal/manager/tests.rs @@ -601,6 +601,13 @@ impl HealStorageAPI for MockStorage { Ok(None) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> Result> { + self.list_buckets().await + } + async fn list_buckets(&self) -> Result> { if let Some(hook) = manager_recovery_test_hook() { *hook.listed.lock().expect("manager recovery listed lock should not poison") = true; @@ -5902,3 +5909,25 @@ fn test_heal_config_respects_feature_flags() { }, ); } + +#[test] +fn admin_bucket_discovery_retries_keep_the_existing_budget() { + let storage: Arc = Arc::new(MockStorage); + let mut request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal); + request.source = HealRequestSource::Admin; + let id = request.id.clone(); + let error = Err(Error::BucketDiscoveryIncomplete); + for attempt in 1..=MAX_RECOVERABLE_HEAL_RETRIES { + let task = HealTask::from_request(request, storage.clone()); + let (retry, delay, _) = retry_request_for_result(&task, &error).expect("discovery can recover within budget"); + assert_eq!(retry.id, id); + assert_eq!(retry.retry_attempts, attempt); + assert!(delay > Duration::ZERO); + request = retry; + } + let task = HealTask::from_request(request, storage); + assert!( + retry_request_for_result(&task, &error).is_none(), + "persistent incompleteness must terminate" + ); +} diff --git a/crates/heal/src/heal/manager/tests/running_mainline.rs b/crates/heal/src/heal/manager/tests/running_mainline.rs index d7b827ce6..4cc153d28 100644 --- a/crates/heal/src/heal/manager/tests/running_mainline.rs +++ b/crates/heal/src/heal/manager/tests/running_mainline.rs @@ -80,6 +80,13 @@ impl HealStorageAPI for RunningStorage { ..Default::default() })) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> Result> { + self.list_buckets().await + } + async fn list_buckets(&self) -> Result> { Ok(Vec::new()) } diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index 35facd16b..d67519ff2 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -491,6 +491,10 @@ pub trait HealStorageAPI: Send + Sync { /// Get all buckets async fn list_buckets(&self) -> Result>; + /// Return only a complete bucket inventory for an administrator task. + /// Incomplete discovery must fail before traversal or scope checkpointing. + async fn list_buckets_for_admin_heal(&self, selector: Option) -> Result>; + /// Check object exists async fn object_exists(&self, bucket: &str, object: &str) -> Result; @@ -1335,6 +1339,14 @@ impl HealStorageAPI for ECStoreHealStorage { } } + async fn list_buckets_for_admin_heal(&self, selector: Option) -> Result> { + let listing = self.ecstore.list_buckets_for_admin_heal(selector).await?; + if !listing.topology_complete { + return Err(Error::BucketDiscoveryIncomplete); + } + Ok(listing.buckets) + } + async fn list_buckets(&self) -> Result> { debug!( target: "rustfs::heal::storage", diff --git a/crates/heal/src/heal/task/heal_bucket.rs b/crates/heal/src/heal/task/heal_bucket.rs index cf6b4786f..9c95c8c03 100644 --- a/crates/heal/src/heal/task/heal_bucket.rs +++ b/crates/heal/src/heal/task/heal_bucket.rs @@ -308,7 +308,12 @@ impl HealTask { "Heal cluster started" ); - let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?; + let bucket_infos = if self.source == HealRequestSource::Admin { + self.await_with_control(self.storage.list_buckets_for_admin_heal(None)) + .await? + } else { + self.await_with_control(self.storage.list_buckets()).await? + }; let mut failed = 0_u64; let mut retryable = 0_u64; let mut permanent = 0_u64; diff --git a/crates/heal/src/heal/task/tests.rs b/crates/heal/src/heal/task/tests.rs index 9919fb522..4058335bc 100644 --- a/crates/heal/src/heal/task/tests.rs +++ b/crates/heal/src/heal/task/tests.rs @@ -24,6 +24,59 @@ mod canonical_outcome { use super::*; use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage}; + #[tokio::test] + async fn admin_bucket_discovery_incomplete_cannot_finish_an_empty_cluster() { + for dry_run in [false, true] { + let storage = Arc::new(MockStorage { + listed_buckets: Mutex::new(Some(Vec::new())), + bucket_discovery_incomplete: true, + ..Default::default() + }); + let mut request = HealRequest::new( + HealType::Cluster, + HealOptions { + dry_run, + recursive: true, + ..Default::default() + }, + HealPriority::Normal, + ); + request.source = HealRequestSource::Admin; + let task = HealTask::from_request(request, storage.clone()); + assert!(matches!(task.execute().await, Err(Error::BucketDiscoveryIncomplete))); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable)); + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + assert_eq!(outcome.counters.processed, 0); + assert_eq!(outcome.counters.failed, 0); + assert!(storage.bucket_heal_calls.lock().expect("bucket calls").is_empty()); + assert!(storage.heal_object_calls.lock().expect("object calls").is_empty()); + let wire = serde_json::to_value(&outcome).expect("outcome serializes"); + assert_eq!( + crate::heal::outcome::legacy_wire_status("finished", &wire, false) + .expect("legacy status") + .0, + "stopped" + ); + } + } + + #[tokio::test] + async fn admin_bucket_discovery_complete_empty_cluster_finishes() { + let storage = Arc::new(MockStorage { + listed_buckets: Mutex::new(Some(Vec::new())), + ..Default::default() + }); + let mut request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal); + request.source = HealRequestSource::Admin; + let task = HealTask::from_request(request, storage); + task.execute().await.expect("complete empty namespace is valid"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Completed); + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + assert_eq!(outcome.counters.processed, 0); + } + fn bucket_task(storage: Arc) -> HealTask { HealTask::from_request( HealRequest::new( @@ -1481,6 +1534,7 @@ struct MockStorage { truncate_without_token: Mutex, include_object_dir_candidate: Mutex, listed_buckets: Mutex>>, + bucket_discovery_incomplete: bool, bucket_heal_errors: Mutex>>, bucket_heal_calls: Mutex>, block_heal_object: Mutex, @@ -1980,6 +2034,16 @@ impl HealStorageAPI for MockStorage { Ok(*self.usage_baseline.lock().unwrap()) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> Result> { + if self.bucket_discovery_incomplete { + return Err(Error::BucketDiscoveryIncomplete); + } + self.list_buckets().await + } + async fn list_buckets(&self) -> Result> { let buckets = self .listed_buckets diff --git a/crates/heal/src/lib.rs b/crates/heal/src/lib.rs index 8a4fdf8b5..d792aa877 100644 --- a/crates/heal/src/lib.rs +++ b/crates/heal/src/lib.rs @@ -398,6 +398,13 @@ mod tests { Ok(None) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> Result, Error> { + self.list_buckets().await + } + async fn list_buckets(&self) -> Result, Error> { Ok(Vec::new()) } diff --git a/crates/heal/tests/heal_bug_fixes_test.rs b/crates/heal/tests/heal_bug_fixes_test.rs index d2690b6be..832c71cf7 100644 --- a/crates/heal/tests/heal_bug_fixes_test.rs +++ b/crates/heal/tests/heal_bug_fixes_test.rs @@ -110,6 +110,13 @@ fn test_heal_task_status_atomic_update() { async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result> { Ok(None) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> rustfs_heal::Result> { + self.list_buckets().await + } + async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(vec![]) } @@ -205,6 +212,13 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { Ok(None) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> rustfs_heal::Result> { + self.list_buckets().await + } + async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(Vec::new()) } diff --git a/crates/heal/tests/heal_concurrent_delete_test.rs b/crates/heal/tests/heal_concurrent_delete_test.rs index 2770295dc..4a8630a4f 100644 --- a/crates/heal/tests/heal_concurrent_delete_test.rs +++ b/crates/heal/tests/heal_concurrent_delete_test.rs @@ -79,6 +79,13 @@ impl HealStorageAPI for PausedTraversal { self.inner.get_bucket_info(bucket).await } + async fn list_buckets_for_admin_heal( + &self, + selector: Option, + ) -> Result> { + self.inner.list_buckets_for_admin_heal(selector).await + } + async fn list_buckets(&self) -> Result> { self.inner.list_buckets().await } diff --git a/crates/heal/tests/storage_api.rs b/crates/heal/tests/storage_api.rs index b47737203..f9bae2473 100644 --- a/crates/heal/tests/storage_api.rs +++ b/crates/heal/tests/storage_api.rs @@ -2,7 +2,7 @@ pub(crate) mod bug_fixes { pub(crate) use rustfs_ecstore::api::disk::DiskStore; pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint; - pub(crate) use rustfs_storage_api::BucketInfo; + pub(crate) use rustfs_storage_api::{BucketInfo, DiskSetSelector}; } #[allow(unused_imports)] diff --git a/crates/scanner/src/scanner_folder/tests/mrf_ownership.rs b/crates/scanner/src/scanner_folder/tests/mrf_ownership.rs index 3e7f9f570..a166a20ef 100644 --- a/crates/scanner/src/scanner_folder/tests/mrf_ownership.rs +++ b/crates/scanner/src/scanner_folder/tests/mrf_ownership.rs @@ -243,6 +243,13 @@ impl HealStorageAPI for NoticeStorage { async fn bucket_incarnation_id(&self, _: &str) -> rustfs_heal::Result> { Ok(Some(self.bucket_incarnation_id)) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> rustfs_heal::Result> { + self.list_buckets().await + } + async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(Vec::new()) } diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index ad88e6ff0..4523fa02c 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -145,6 +145,8 @@ use rustfs_storage_api as storage_contracts; pub(crate) type EcstoreHealResultItem = ::HealResultItem; pub(crate) mod owner { + #[cfg(test)] + pub(crate) use super::storage_contracts::DiskSetSelector; pub(crate) use super::{ MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementError, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner, diff --git a/docs/operations/bucket-heal-recovery.md b/docs/operations/bucket-heal-recovery.md index 60d10b3ca..ed6453644 100644 --- a/docs/operations/bucket-heal-recovery.md +++ b/docs/operations/bucket-heal-recovery.md @@ -1,10 +1,33 @@ # Bucket heal recovery and bucket recreation +**Use this when:** diagnosing incomplete administrator heal coverage, restoring a missing bucket directory, or replaying a bucket heal after restart. + Explicit administrator bucket heals bind the bucket name to its authoritative, non-nil incarnation UUID at admission. The same identity follows the task through queueing, retries, shutdown persistence, and startup replay. Deleting a bucket and creating another with the same name does not transfer the old task to the new bucket. +## Incomplete bucket discovery + +An administrator heal of the cluster or a selected pool/set must enumerate a complete bucket inventory before traversal. A bucket directory present on fewer disks than the namespace write quorum can be omitted from the ordinary bucket list even when every disk answers. Administrator discovery preserves that incomplete result instead of reporting an empty successful traversal. A pool/set task checks only its selected set; unavailable unrelated pools do not invalidate that scope. + +Incomplete discovery uses the existing bounded task retry budget. Once exhausted, the canonical outcome is `execution.state=aborted`, `execution.reason=untraversable`, and `coverage=partial`; the legacy summary is `stopped`. Object counters can remain zero because traversal never started. A genuinely empty, completely enumerated namespace still finishes successfully. Dry runs use the same coverage checks. + +A pool/set task persists its resolved bucket scope only after complete discovery. Resumed checkpoints keep their original frozen scope, including a valid empty scope. This change does not reinterpret historical checkpoints or terminal reports; start a fresh task to verify coverage after upgrading. + +## Restoring a missing bucket directory + +For a bucket that is still intended to exist but has lost its directory on one drive: + +1. Check that the peer drives and authoritative bucket metadata remain available. Identify the affected bucket explicitly. +2. Start an authenticated administrator heal for that bucket with `recursive=true` and `recreate=true`. A deep scan (`scanMode=2`) additionally checks data integrity. For the v3 API, the start endpoint is `POST /rustfs/admin/v3/heal/`; retain the returned `clientToken` and poll the same endpoint with that query parameter. +3. Inspect both execution and coverage in the terminal outcome. A readable object on a healthy peer alone does not establish that the missing copy was restored. +4. Verify recovered object contents and lengths against known hashes. In an isolated recovery test, stop the original healthy source and read through the repaired target to demonstrate independent recovery. Do not take a production source offline merely to inspect a heal result. + +Default bucket heal still enforces its existing namespace quorum checks. `recreate` is an explicit recovery option, not permission to revive a bucket deleted through S3. Bucket incarnation checks continue to reject stale tasks after deletion and same-name recreation. A cluster-wide `recreate` request also requires complete discovery; when discovery is incomplete, use the explicit affected-bucket path after resolving its authority. + +Legacy objects without independent storage commitments can be physically repaired while their canonical dispositions remain `unknown`. Do not infer a verified repair count from progress counters; see [heal terminal reports](../architecture/heal-terminal-reports.md). + ## Recovery records and stale tokens Pending records use schema 3 in `.rustfs.sys/root-heal-.json` on the existing diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 255bf1044..588bc26f6 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -3209,6 +3209,13 @@ mod tests { Ok(None) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> rustfs_heal::Result> { + self.list_buckets().await + } + async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(Vec::new()) } diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index c0a9608d5..ce971c878 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -59,6 +59,8 @@ fn bucket_targets_metadata_lock_shard(bucket: &str) -> usize { pub(crate) mod contract { pub(crate) mod admin { + #[cfg(test)] + pub(crate) use super::super::storage_contracts::DiskSetSelector; pub(crate) use super::super::storage_contracts::StorageAdminApi; } From ab512768d5ac9de30cfd3608316fb44858aeb405 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 20:05:36 +0800 Subject: [PATCH 045/143] fix(ecstore): group rename data commit observations (#8516) Group disk observations into a borrowed struct to keep the commit helper within Clippy's argument limit without changing quorum snapshot logic. Co-authored-by: heihutu Co-authored-by: zhi22915 --- .../src/set_disk/core/io_primitives.rs | 48 ++++++++++++------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 2b99bd9c1..6ade60cf4 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -2451,6 +2451,17 @@ impl RenameConvergence { } } +struct RenameDataObservations<'a> { + disks: &'a [Option], + file_infos: &'a [FileInfo], + disk_versions: &'a [Option>], + errs: &'a [Option], + cleanup_data_dirs: &'a [Option], + old_current_sizes: &'a [Option], + old_current_source_checked: &'a [bool], + old_current_sources: &'a [Option], +} + pub(in crate::set_disk) struct RenameDataCommit { pub(in crate::set_disk) online_disks: Vec>, /// Disks whose capacity must be marked for this commit. Under early ACK, @@ -3608,16 +3619,19 @@ impl SetDisks { } fn rename_data_commit_from_observations( - disks: &[Option], - file_infos: &[FileInfo], - disk_versions: &[Option>], - errs: &[Option], - cleanup_data_dirs: &[Option], - old_current_sizes: &[Option], - old_current_source_checked: &[bool], - old_current_sources: &[Option], + observations: RenameDataObservations<'_>, write_quorum: usize, ) -> disk::error::Result { + let RenameDataObservations { + disks, + file_infos, + disk_versions, + errs, + cleanup_data_dirs, + old_current_sizes, + old_current_source_checked, + old_current_sources, + } = observations; let data_dir = Self::reduce_common_data_dir(cleanup_data_dirs, write_quorum); let convergence = Self::classify_rename_convergence(disk_versions, errs); let old_current_size = Self::reduce_common_old_current_size(old_current_sizes, write_quorum); @@ -3944,14 +3958,16 @@ impl SetDisks { if !sent_commit && success_count >= write_quorum { let snapshot_commit = Self::rename_data_commit_from_observations( - &coordinator_disks, - &file_infos, - &disk_versions, - &errs, - &cleanup_data_dirs, - &old_current_sizes, - &old_current_source_checked, - &old_current_sources, + RenameDataObservations { + disks: &coordinator_disks, + file_infos: &file_infos, + disk_versions: &disk_versions, + errs: &errs, + cleanup_data_dirs: &cleanup_data_dirs, + old_current_sizes: &old_current_sizes, + old_current_source_checked: &old_current_source_checked, + old_current_sources: &old_current_sources, + }, write_quorum, ); if let Ok(commit) = snapshot_commit.as_ref() { From a79c6072f1b32d55994906158e2278fe23d58d0b Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 13 Oct 2026 05:31:23 -0700 Subject: [PATCH 046/143] fix: suppress clippy too_many_arguments on rename_data_commit_from_observations (#8515) fix(ecstore): suppress clippy too_many_arguments on rename_data_commit_from_observations The function takes 9 slice parameters for the renamedata commit phase. Annotate with #[allow(clippy::too_many_arguments)] to match the existing codebase pattern for internal multi-slice commit helpers. --- crates/ecstore/src/set_disk/core/io_primitives.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 6ade60cf4..de172a855 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -3618,6 +3618,7 @@ impl SetDisks { } } + #[allow(clippy::too_many_arguments)] fn rename_data_commit_from_observations( observations: RenameDataObservations<'_>, write_quorum: usize, From da52303f2a05ef1bcff1295715dddcfc12e901ee Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 20:32:27 +0800 Subject: [PATCH 047/143] fix(server): restore socket peer fallback for client rate limits (#8519) Read the optional socket peer extension supplied by external ingress so API and Console client limits remain effective without trusted proxies. Keep validated client precedence and the existing missing-peer policy. Exercise the full external stack across HTTP request versions, direct IPv4/IPv6 peers, forged headers, and distinct trusted proxy clients. Align API and Console fixtures with the production extension contract. Co-authored-by: heihutu Co-authored-by: zhi22915 --- rustfs/src/admin/console.rs | 10 +- rustfs/src/server/http.rs | 157 ++++++++++++++++++++++++++++++++ rustfs/src/server/rate_limit.rs | 18 ++-- 3 files changed, 175 insertions(+), 10 deletions(-) diff --git a/rustfs/src/admin/console.rs b/rustfs/src/admin/console.rs index 4019c9876..f03a3f3ee 100644 --- a/rustfs/src/admin/console.rs +++ b/rustfs/src/admin/console.rs @@ -1051,10 +1051,12 @@ mod tests { .uri(format!("{CONSOLE_PREFIX}/index.html")) .body(Body::empty()) .expect("failed to build request"); - request.extensions_mut().insert(crate::server::RemoteAddr(SocketAddr::new( - IpAddr::from([198, 51, 100, last_octet]), - 50000, - ))); + request + .extensions_mut() + .insert(Some(crate::server::RemoteAddr(SocketAddr::new( + IpAddr::from([198, 51, 100, last_octet]), + 50000, + )))); request }; diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index d5a90e6bc..e0615835a 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -2818,6 +2818,163 @@ mod tests { use tokio::sync::{Notify, mpsc}; use tower::{Layer, Service, ServiceBuilder}; + type RateLimitTestCore = tower::util::BoxCloneService< + HttpRequest, + Response>>, + Box, + >; + type RateLimitTestStack = tower::util::BoxCloneService< + HttpRequest, + Response>>, + Box, + >; + + fn rate_limited_external_stack( + peer: SocketAddr, + proxy_layer: Option, + inner: RateLimitTestCore, + ) -> RateLimitTestStack { + use crate::server::rate_limit::RateLimitQuota; + + let readiness = Arc::new(GlobalReadiness::new()); + readiness.mark_stage(rustfs_common::SystemStage::FullReady); + RateLimitTestStack::new(external_service_stack!( + inner, + Some(RemoteAddr(peer)), + proxy_layer, + false, + Some(RateLimitLayer::new( + Some(RateLimitQuota { + requests_per_minute: 1, + burst: 1 + }), + None, + vec![] + )), + false, + readiness, + None::>, + HttpCompressionConfig::default(), + ServerContextSlot::new(), + false + )) + } + + #[tokio::test] + async fn external_stack_limits_socket_peers_without_trusted_proxies() { + use crate::server::hybrid::HybridBody; + use tower::ServiceExt; + + for peer in ["192.0.2.10:51000", "[2001:db8::10]:51000"] { + let peer: SocketAddr = peer.parse().expect("socket peer"); + for version in [Version::HTTP_11, Version::HTTP_2, Version::HTTP_3] { + let calls = Arc::new(AtomicUsize::new(0)); + let inner_calls = calls.clone(); + let inner = tower::service_fn(move |request: HttpRequest| { + assert_eq!( + request + .extensions() + .get::>() + .copied() + .flatten() + .map(|addr| addr.0), + Some(peer) + ); + inner_calls.fetch_add(1, Ordering::SeqCst); + async { + Ok::<_, Box>(Response::new( + HybridBody::>::Rest { + rest_body: s3s::Body::empty(), + }, + )) + } + }); + let mut service = rate_limited_external_stack(peer, None, RateLimitTestCore::new(inner)); + + for (forwarded, expected) in [ + ("203.0.113.1", StatusCode::OK), + ("203.0.113.2", StatusCode::TOO_MANY_REQUESTS), + ] { + let request = HttpRequest::builder() + .uri("/bucket/object") + .version(version) + .header("x-forwarded-for", forwarded) + .header("x-real-ip", forwarded) + .body(s3s::Body::empty()) + .expect("request"); + let response = service + .ready() + .await + .expect("service ready") + .call(request) + .await + .expect("response"); + assert_eq!(response.status(), expected, "peer={peer}, version={version:?}"); + if expected == StatusCode::TOO_MANY_REQUESTS { + assert!(response.headers().contains_key(http::header::RETRY_AFTER)); + let body = response.into_body().collect().await.expect("body").to_bytes(); + assert!(String::from_utf8_lossy(&body).contains("TooManyRequests")); + } + } + assert_eq!(calls.load(Ordering::SeqCst), 1, "throttled requests must not reach the inner service"); + } + } + } + + #[tokio::test] + async fn external_stack_limits_validated_proxy_clients_independently() { + use crate::server::hybrid::HybridBody; + use rustfs_trusted_proxies::{ + LegacyTrustedProxyLayer, TrustedProxy, TrustedProxyConfig, TrustedProxyLayer, ValidationMode, + }; + use tower::ServiceExt; + + let peer: SocketAddr = "192.0.2.100:51000".parse().expect("proxy peer"); + for version in [Version::HTTP_11, Version::HTTP_2, Version::HTTP_3] { + let proxy_config = + TrustedProxyConfig::new(vec![TrustedProxy::Single(peer.ip())], ValidationMode::HopByHop, false, 3, false, vec![]); + let proxy_layer = TrustedProxyLayer::legacy(LegacyTrustedProxyLayer::new(proxy_config, None, true)); + let calls = Arc::new(AtomicUsize::new(0)); + let inner_calls = calls.clone(); + let inner = tower::service_fn(move |request: HttpRequest| { + let info = request.extensions().get::().expect("validated client context"); + assert!(info.is_from_trusted_proxy); + assert_ne!(info.real_ip, peer.ip()); + inner_calls.fetch_add(1, Ordering::SeqCst); + async { + Ok::<_, Box>(Response::new( + HybridBody::>::Rest { + rest_body: s3s::Body::empty(), + }, + )) + } + }); + let mut service = rate_limited_external_stack(peer, Some(proxy_layer), RateLimitTestCore::new(inner)); + + for (client, expected) in [ + ("203.0.113.1", StatusCode::OK), + ("203.0.113.2", StatusCode::OK), + ("203.0.113.1", StatusCode::TOO_MANY_REQUESTS), + ] { + let request = HttpRequest::builder() + .uri("/bucket/object") + .version(version) + .header("x-forwarded-for", client) + .body(s3s::Body::empty()) + .expect("request"); + let response = service + .ready() + .await + .expect("service ready") + .call(request) + .await + .expect("response"); + assert_eq!(response.status(), expected, "client={client}, version={version:?}"); + } + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + } + #[test] fn peer_transport_close_kind_uses_io_error_sources() { #[derive(Debug)] diff --git a/rustfs/src/server/rate_limit.rs b/rustfs/src/server/rate_limit.rs index 483b805a0..ca39b8d28 100644 --- a/rustfs/src/server/rate_limit.rs +++ b/rustfs/src/server/rate_limit.rs @@ -301,14 +301,16 @@ impl RateLimiter { /// Resolve the client identity for rate limiting. /// /// Only trusted sources are consulted: `ClientInfo` is inserted by the -/// trusted-proxy layer after validating the proxy chain, and `RemoteAddr` is -/// the raw socket peer. Spoofable headers (`X-Forwarded-For`, `X-Real-IP`) -/// are deliberately never read here. +/// trusted-proxy layer after validating the proxy chain, and `Option` +/// carries the socket peer injected by the external HTTP stack. Spoofable +/// headers (`X-Forwarded-For`, `X-Real-IP`) are deliberately never read here. pub(crate) fn client_ip(req: &Request) -> Option { if let Some(info) = req.extensions().get::() { return Some(info.real_ip); } - req.extensions().get::().map(|remote| remote.0.ip()) + req.extensions() + .get::>() + .and_then(|remote| remote.map(|remote| remote.0.ip())) } /// Resolve the host a request addresses, for virtual-hosted-style bucket @@ -929,7 +931,7 @@ mod tests { fn request_from(peer: IpAddr, path: &str) -> Request<()> { let mut req = Request::builder().uri(path).body(()).expect("request"); - req.extensions_mut().insert(RemoteAddr(SocketAddr::new(peer, 51000))); + req.extensions_mut().insert(Some(RemoteAddr(SocketAddr::new(peer, 51000)))); req } @@ -1047,7 +1049,11 @@ mod tests { async fn missing_client_identity_fails_open() { let mut service = service_with_quota(60, 1); for _ in 0..3 { - let req = Request::builder().uri("/bucket/object").body(()).expect("request"); + let mut req = Request::builder().uri("/bucket/object").body(()).expect("request"); + req.extensions_mut().insert(None::); + req.headers_mut() + .insert("x-forwarded-for", HeaderValue::from_static("203.0.113.1")); + req.headers_mut().insert("x-real-ip", HeaderValue::from_static("203.0.113.2")); assert_eq!(service.call(req).await.expect("ok").status(), StatusCode::OK); } } From 3b428bffeffd23a58f4c77f7e207ef7e05c945f1 Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:18:58 +0800 Subject: [PATCH 048/143] fix(heal): merge disjoint replacement generations in cluster status (#8521) --- rustfs/src/admin/handlers/heal.rs | 96 ++++++++++++++++++++++++++----- 1 file changed, 82 insertions(+), 14 deletions(-) diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index f6c02aa0c..147bf275e 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -732,6 +732,21 @@ fn merge_replacement_recovery_records( records.into_values().collect() } +/// Distinct replacement generations must not claim the same target slot: two +/// completed histories for one replaced disk cannot be reconciled into a +/// definitive cluster view. Identical copies of one task are deduplicated by +/// the merge, and disagreeing copies already surface as `Unknown` records. +fn replacement_generations_claim_overlapping_targets(records: &[rustfs_heal::ReplacementRecoveryRecord]) -> bool { + records.iter().any(|left| { + !left.target_slots.is_empty() + && records.iter().any(|right| { + right.task_id != left.task_id + && !right.target_slots.is_empty() + && right.target_slots.iter().any(|slot| left.target_slots.contains(slot)) + }) + }) +} + fn aggregate_replacement_recovery_cluster_status( mut snapshots: Vec, peer_statuses: Vec, String>>, @@ -762,23 +777,18 @@ fn aggregate_replacement_recovery_cluster_status( reason.get_or_insert("peer_replacement_recovery_status_not_definitive"); } - // Replacement recovery state is anchored on one surviving disk. Nodes - // without that disk legitimately report a definitive empty snapshot, so - // only non-empty snapshots must agree with each other. - let mut non_empty_records = snapshots - .iter() - .filter(|snapshot| !snapshot.records.is_empty()) - .map(|snapshot| canonical_replacement_records(&snapshot.records)); - if let Some(first_records) = non_empty_records.next() - && non_empty_records.any(|records| records != first_records) - { - reason.get_or_insert("peer_replacement_recovery_status_conflict"); - } - + // Replacement completion proofs are anchored per generation on one + // surviving disk, so sequential automatic generations legitimately leave + // different completed tasks on different peers, and peers without an + // anchor report definitive empty snapshots. Peers conflict only when they + // disagree about one task (the merge marks those copies `Unknown`) or + // when distinct generations claim the same target slot; disjoint + // generations merge into one definitive cluster history. let records = merge_replacement_recovery_records(&snapshots); if records .iter() .any(|record| matches!(record.state, rustfs_heal::ReplacementRecoveryState::Unknown)) + || replacement_generations_claim_overlapping_targets(&records) { reason.get_or_insert("peer_replacement_recovery_status_conflict"); } @@ -2369,7 +2379,10 @@ mod tests { } #[test] - fn replacement_recovery_cluster_requires_matching_peer_records() { + fn replacement_recovery_cluster_rejects_overlapping_targets_across_generations() { + // Both helper records claim the same target slot, so two completed + // generations over one replaced disk stay non-definitive even though + // the task ids differ. let local = replacement_snapshot("11111111-1111-4111-8111-111111111111"); let peer = replacement_snapshot("22222222-2222-4222-8222-222222222222"); let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); @@ -2379,6 +2392,61 @@ mod tests { assert_eq!(cluster.records.len(), 2); } + #[test] + fn replacement_recovery_cluster_merges_disjoint_generations_across_anchors() { + // rustfs/rustfs#8366: sequential automatic generations anchor their + // completion proofs on different surviving disks, so peers hold + // disjoint completed tasks and the remaining peers hold none at all. + let mut rebuilt_record = replacement_record("11111111-1111-4111-8111-111111111111"); + rebuilt_record.target_slots = vec!["http://node-a:9000/mnt/disk2".to_string()]; + let mut anchor_record = replacement_record("22222222-2222-4222-8222-222222222222"); + anchor_record.target_slots = vec!["http://node-a:9000/mnt/disk1".to_string()]; + let rebuilt = rustfs_heal::ReplacementRecoverySnapshot { + records: vec![rebuilt_record], + definitive: true, + reason: None, + }; + let anchor = rustfs_heal::ReplacementRecoverySnapshot { + records: vec![anchor_record], + definitive: true, + reason: None, + }; + let empty = rustfs_heal::ReplacementRecoverySnapshot { + records: Vec::new(), + definitive: true, + reason: None, + }; + let cluster = + aggregate_replacement_recovery_cluster_status(vec![rebuilt], vec![Ok(Some(anchor)), Ok(Some(empty))], 3, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records.len(), 2); + assert!( + cluster + .records + .iter() + .all(|record| matches!(record.state, rustfs_heal::ReplacementRecoveryState::Completed)) + ); + } + + #[test] + fn replacement_recovery_cluster_rejects_peer_disagreement_about_one_task() { + let local = replacement_snapshot("11111111-1111-4111-8111-111111111111"); + let mut disagreeing = replacement_record("11111111-1111-4111-8111-111111111111"); + disagreeing.set_disk_id = Some("pool_0_set_1".to_string()); + let peer = rustfs_heal::ReplacementRecoverySnapshot { + records: vec![disagreeing], + definitive: true, + reason: None, + }; + let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); + + assert!(!cluster.definitive); + assert_eq!(cluster.reason.as_deref(), Some("peer_replacement_recovery_status_conflict")); + assert_eq!(cluster.records[0].state, rustfs_heal::ReplacementRecoveryState::Unknown); + } + #[test] fn replacement_recovery_cluster_accepts_definitive_empty_peers() { let local = rustfs_heal::ReplacementRecoverySnapshot { From 025ed4c6d16bd42b31a9507674a239eb94514f83 Mon Sep 17 00:00:00 2001 From: Hauser Date: Wed, 14 Oct 2026 00:36:33 +0800 Subject: [PATCH 049/143] fix(heal): preserve retained replacement histories (#8525) Keep definitive local histories while rejecting ambiguous overlapping peer records. Co-authored-by: heihutu Co-authored-by: zhi22915 --- docs/operations/scanner-runtime-controls.md | 2 +- rustfs/src/admin/handlers/heal.rs | 140 ++++++++++++++++++-- 2 files changed, 129 insertions(+), 13 deletions(-) diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index faf999409..eb0add3bd 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -493,7 +493,7 @@ Stale-upload cleanup differs in one crash-recovery detail. RustFS's stale multip Treat replacement recovery as verified only after the repair task has completed for the exact replacement instance and an operator has confirmed the target disk contains the expected `xl.meta` and data parts for every relevant object version. A replacement that is not mounted, is unsafe to format, loses its marker, or returns a partial target outcome must be treated as deferred or incomplete. Do not automate destructive replacement actions from an `idle` observation alone. A new node must not infer replacement completion from an old or unavailable peer; regard that information as unknown or degraded until every required peer can report the same replacement instance and verified completion. -`GET /rustfs/admin/v4/heal/replacement-recovery` reports durable automatic replacement records from survivor disks. `local.records[]` entries distinguish `waiting_for_replacement`, `running`, `incomplete`, `unrecoverable`, `cleanup_pending`, `completed`, and `unknown`; `local.definitive=false` or any `unknown` record means the node could not prove a local replacement state. The `cluster` section queries the replacement-recovery peer RPC and sets `cluster.definitive=true` only when the expected peer topology is complete, every peer supports the RPC, every peer snapshot is locally definitive, and all peers report the same replacement records. Old peers, unavailable peers, malformed peer payloads, topology gaps, and generation disagreements are reported as degraded or unknown rather than complete. +`GET /rustfs/admin/v4/heal/replacement-recovery` reports durable automatic replacement records from survivor disks. `local.records[]` entries distinguish `waiting_for_replacement`, `running`, `incomplete`, `unrecoverable`, `cleanup_pending`, `completed`, and `unknown`; `local.definitive=false` or any `unknown` record means the node could not prove a local replacement state. The `cluster` section queries the replacement-recovery peer RPC and sets `cluster.definitive=true` only when the expected peer topology is complete, every peer supports the RPC, every peer snapshot is locally definitive, and copies of each task agree. Different anchors can report different tasks, and a definitive node may report an empty list or a subset of another node's history. Disjoint target generations merge directly. Generations sharing a target slot must have been reported together in at least one definitive local snapshot; this preserves retained completion proofs and committed handoff histories while keeping overlapping records found only on separate peers non-definitive. A definitive history can include retired or unfinished generations, so consumers must inspect each record's state rather than treating `cluster.definitive` as proof that every generation completed. Old peers, unavailable peers, malformed peer payloads, topology gaps, and task disagreements are reported as degraded or unknown rather than complete. Replacement resume and checkpoint files use an independent on-disk schema. A newer reader rejects a future schema rather than continuing with data it cannot interpret, while an older binary cannot safely enforce the new generation fence. Do not roll a cluster back after a replacement generation has started; complete that recovery with the current-or-newer release, and if it cannot complete, keep that version for diagnosis rather than deleting its durable records or continuing with an older binary. diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 147bf275e..13449b763 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -732,17 +732,35 @@ fn merge_replacement_recovery_records( records.into_values().collect() } -/// Distinct replacement generations must not claim the same target slot: two -/// completed histories for one replaced disk cannot be reconciled into a -/// definitive cluster view. Identical copies of one task are deduplicated by -/// the merge, and disagreeing copies already surface as `Unknown` records. -fn replacement_generations_claim_overlapping_targets(records: &[rustfs_heal::ReplacementRecoveryRecord]) -> bool { +/// A definitive local snapshot can retain multiple generations for one slot, +/// including completed replacements and committed handoff predecessors. Only +/// overlapping generations never observed together in a definitive snapshot +/// leave the merged peer histories ambiguous. Disagreeing copies of one task +/// are already rejected by the merge. +fn replacement_generations_claim_conflicting_targets( + records: &[rustfs_heal::ReplacementRecoveryRecord], + snapshots: &[rustfs_heal::ReplacementRecoverySnapshot], +) -> bool { + let local_histories = snapshots + .iter() + .filter(|snapshot| snapshot.definitive) + .map(|snapshot| { + snapshot + .records + .iter() + .map(|record| record.task_id.as_str()) + .collect::>() + }) + .collect::>(); records.iter().any(|left| { !left.target_slots.is_empty() && records.iter().any(|right| { right.task_id != left.task_id && !right.target_slots.is_empty() && right.target_slots.iter().any(|slot| left.target_slots.contains(slot)) + && !local_histories + .iter() + .any(|history| history.contains(left.task_id.as_str()) && history.contains(right.task_id.as_str())) }) }) } @@ -782,13 +800,13 @@ fn aggregate_replacement_recovery_cluster_status( // different completed tasks on different peers, and peers without an // anchor report definitive empty snapshots. Peers conflict only when they // disagree about one task (the merge marks those copies `Unknown`) or - // when distinct generations claim the same target slot; disjoint - // generations merge into one definitive cluster history. + // claim overlapping targets without a shared definitive local history. + // Retained histories and disjoint generations can be merged. let records = merge_replacement_recovery_records(&snapshots); if records .iter() .any(|record| matches!(record.state, rustfs_heal::ReplacementRecoveryState::Unknown)) - || replacement_generations_claim_overlapping_targets(&records) + || replacement_generations_claim_conflicting_targets(&records, &snapshots) { reason.get_or_insert("peer_replacement_recovery_status_conflict"); } @@ -2205,6 +2223,14 @@ mod tests { } } + fn replacement_snapshot_with_retained_history() -> rustfs_heal::ReplacementRecoverySnapshot { + let mut snapshot = replacement_snapshot("11111111-1111-4111-8111-111111111111"); + snapshot + .records + .push(replacement_record("22222222-2222-4222-8222-222222222222")); + snapshot + } + #[tokio::test] async fn heal_selector_preflight_rejects_before_probe_and_token_creation() { use crate::admin::storage_api::runtime::PoolEndpoints; @@ -2379,10 +2405,9 @@ mod tests { } #[test] - fn replacement_recovery_cluster_rejects_overlapping_targets_across_generations() { - // Both helper records claim the same target slot, so two completed - // generations over one replaced disk stay non-definitive even though - // the task ids differ. + fn replacement_recovery_cluster_rejects_overlapping_targets_without_shared_history() { + // Separate anchors cannot establish that these overlapping records + // describe compatible historical replacement instances. let local = replacement_snapshot("11111111-1111-4111-8111-111111111111"); let peer = replacement_snapshot("22222222-2222-4222-8222-222222222222"); let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); @@ -2447,6 +2472,97 @@ mod tests { assert_eq!(cluster.records[0].state, rustfs_heal::ReplacementRecoveryState::Unknown); } + #[test] + fn replacement_recovery_cluster_accepts_retained_history_on_one_anchor() { + let local = replacement_snapshot_with_retained_history(); + let empty = rustfs_heal::ReplacementRecoverySnapshot { + records: Vec::new(), + definitive: true, + reason: None, + }; + let cluster = aggregate_replacement_recovery_cluster_status(vec![local.clone()], vec![Ok(Some(empty))], 2, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records, local.records); + } + + #[test] + fn replacement_recovery_cluster_accepts_matching_retained_histories() { + let local = replacement_snapshot_with_retained_history(); + let cluster = aggregate_replacement_recovery_cluster_status(vec![local.clone()], vec![Ok(Some(local.clone()))], 2, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records, local.records); + } + + #[test] + fn replacement_recovery_cluster_accepts_committed_handoff_history() { + let mut local = replacement_snapshot_with_retained_history(); + let successor = local.records[1].task_id.clone(); + let predecessor = &mut local.records[0]; + predecessor.state = rustfs_heal::ReplacementRecoveryState::Unrecoverable; + predecessor.reason = Some(format!("replacement responsibility transferred to {successor}")); + predecessor.verified_at = None; + let cluster = aggregate_replacement_recovery_cluster_status(vec![local.clone()], Vec::new(), 1, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records, local.records); + } + + #[test] + fn replacement_recovery_cluster_merges_retained_history_with_disjoint_peer_generation() { + let local = replacement_snapshot_with_retained_history(); + let mut peer = replacement_snapshot("33333333-3333-4333-8333-333333333333"); + peer.records[0].target_slots = vec!["http://node-a:9000/mnt/disk2".to_string()]; + let mut expected = local.records.clone(); + expected.extend(peer.records.clone()); + let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records, expected); + } + + #[test] + fn replacement_recovery_cluster_accepts_retained_history_reported_only_by_peer() { + let local = replacement_snapshot("11111111-1111-4111-8111-111111111111"); + let peer = replacement_snapshot_with_retained_history(); + let expected = peer.records.clone(); + let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records, expected); + } + + #[test] + fn replacement_recovery_cluster_rejects_overlap_outside_retained_history() { + let local = replacement_snapshot_with_retained_history(); + let peer = replacement_snapshot("33333333-3333-4333-8333-333333333333"); + let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); + + assert!(!cluster.definitive); + assert_eq!(cluster.reason.as_deref(), Some("peer_replacement_recovery_status_conflict")); + assert_eq!(cluster.records.len(), 3); + } + + #[test] + fn replacement_recovery_cluster_rejects_disagreeing_copies_in_retained_history() { + let local = replacement_snapshot_with_retained_history(); + let mut peer = local.clone(); + peer.records[0].verified_at = Some(43); + let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); + + assert!(!cluster.definitive); + assert_eq!(cluster.reason.as_deref(), Some("peer_replacement_recovery_status_conflict")); + assert_eq!(cluster.records.len(), 2); + assert_eq!(cluster.records[0].state, rustfs_heal::ReplacementRecoveryState::Unknown); + assert_eq!(cluster.records[1].state, rustfs_heal::ReplacementRecoveryState::Completed); + } + #[test] fn replacement_recovery_cluster_accepts_definitive_empty_peers() { let local = rustfs_heal::ReplacementRecoverySnapshot { From b80590f1a73819012b9674d46c45154c9d1b775e Mon Sep 17 00:00:00 2001 From: Hauser Date: Wed, 14 Oct 2026 02:06:21 +0800 Subject: [PATCH 050/143] perf(heal): index replacement history reconciliation (#8527) refactor(heal): index replacement generation reconciliation Reconcile owned records and their definitive sources in one pass. Group claims by target slot and use a shared-anchor fast path while retaining pairwise ambiguity checks. Co-authored-by: heihutu Co-authored-by: zhi22915 --- docs/operations/scanner-runtime-controls.md | 2 +- rustfs/src/admin/handlers/heal.rs | 250 +++++++++++++------- 2 files changed, 165 insertions(+), 87 deletions(-) diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index eb0add3bd..372cce791 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -493,7 +493,7 @@ Stale-upload cleanup differs in one crash-recovery detail. RustFS's stale multip Treat replacement recovery as verified only after the repair task has completed for the exact replacement instance and an operator has confirmed the target disk contains the expected `xl.meta` and data parts for every relevant object version. A replacement that is not mounted, is unsafe to format, loses its marker, or returns a partial target outcome must be treated as deferred or incomplete. Do not automate destructive replacement actions from an `idle` observation alone. A new node must not infer replacement completion from an old or unavailable peer; regard that information as unknown or degraded until every required peer can report the same replacement instance and verified completion. -`GET /rustfs/admin/v4/heal/replacement-recovery` reports durable automatic replacement records from survivor disks. `local.records[]` entries distinguish `waiting_for_replacement`, `running`, `incomplete`, `unrecoverable`, `cleanup_pending`, `completed`, and `unknown`; `local.definitive=false` or any `unknown` record means the node could not prove a local replacement state. The `cluster` section queries the replacement-recovery peer RPC and sets `cluster.definitive=true` only when the expected peer topology is complete, every peer supports the RPC, every peer snapshot is locally definitive, and copies of each task agree. Different anchors can report different tasks, and a definitive node may report an empty list or a subset of another node's history. Disjoint target generations merge directly. Generations sharing a target slot must have been reported together in at least one definitive local snapshot; this preserves retained completion proofs and committed handoff histories while keeping overlapping records found only on separate peers non-definitive. A definitive history can include retired or unfinished generations, so consumers must inspect each record's state rather than treating `cluster.definitive` as proof that every generation completed. Old peers, unavailable peers, malformed peer payloads, topology gaps, and task disagreements are reported as degraded or unknown rather than complete. +`GET /rustfs/admin/v4/heal/replacement-recovery` reports durable automatic replacement records from survivor disks. `local.records[]` entries distinguish `waiting_for_replacement`, `running`, `incomplete`, `unrecoverable`, `cleanup_pending`, `completed`, and `unknown`; `local.definitive=false` or any `unknown` record means the node could not prove a local replacement state. The `cluster` section queries the replacement-recovery peer RPC and sets `cluster.definitive=true` only when the expected peer topology is complete, every peer supports the RPC, every peer snapshot is locally definitive, and copies of each task agree. Different anchors can report different tasks, and a definitive node may report an empty list or a subset of another node's history. Disjoint target generations merge directly. Every pair of generations sharing a target slot must appear together in at least one definitive local snapshot; this preserves retained completion proofs and committed handoff histories while keeping overlapping records found only on separate peers non-definitive. Shared history is not transitive: observing A with B and B with C does not establish that A and C are compatible. A definitive history can include retired or unfinished generations, so consumers must inspect each record's state rather than treating `cluster.definitive` as proof that every generation completed. Old peers, unavailable peers, malformed peer payloads, topology gaps, and task disagreements are reported as degraded or unknown rather than complete. Replacement resume and checkpoint files use an independent on-disk schema. A newer reader rejects a future schema rather than continuing with data it cannot interpret, while an older binary cannot safely enforce the new generation fence. Do not roll a cluster back after a replacement generation has started; complete that recovery with the current-or-newer release, and if it cannot complete, keep that version for diagnosis rather than deleting its durable records or continuing with an older binary. diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 13449b763..48f647d77 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -673,96 +673,80 @@ where } } -fn canonical_replacement_records( - records: &[rustfs_heal::ReplacementRecoveryRecord], -) -> Vec { - let mut records = records.to_vec(); - for record in &mut records { - record.target_slots.sort(); +fn reconcile_replacement_recovery_records( + snapshots: Vec, +) -> (Vec, bool) { + struct Observation { + record: rustfs_heal::ReplacementRecoveryRecord, + sources: Vec, } - records.sort_by(|left, right| { - ( - &left.task_id, - &left.generation, - &left.set_disk_id, - left.state, - &left.target_slots, - &left.verified_at, - &left.reason, - ) - .cmp(&( - &right.task_id, - &right.generation, - &right.set_disk_id, - right.state, - &right.target_slots, - &right.verified_at, - &right.reason, - )) - }); - records -} -fn merge_replacement_recovery_records( - snapshots: &[rustfs_heal::ReplacementRecoverySnapshot], -) -> Vec { - let mut records = BTreeMap::::new(); - for snapshot in snapshots { - for record in canonical_replacement_records(&snapshot.records) { - match records.entry(record.task_id.clone()) { - std::collections::btree_map::Entry::Vacant(entry) => { - entry.insert(record); - } - std::collections::btree_map::Entry::Occupied(entry) if entry.get() == &record => {} + let mut observations = BTreeMap::::new(); + for (source, snapshot) in snapshots.into_iter().enumerate() { + for mut record in snapshot.records { + record.target_slots.sort(); + let observation = match observations.entry(record.task_id.clone()) { + std::collections::btree_map::Entry::Vacant(entry) => entry.insert(Observation { + record, + sources: Vec::new(), + }), std::collections::btree_map::Entry::Occupied(mut entry) => { - let task_id = entry.key().clone(); - entry.insert(rustfs_heal::ReplacementRecoveryRecord { - task_id, - state: rustfs_heal::ReplacementRecoveryState::Unknown, - generation: None, - set_disk_id: None, - target_slots: Vec::new(), - reason: Some("peer replacement recovery records disagree".to_string()), - verified_at: None, - }); + if entry.get().record != record { + entry.get_mut().record = rustfs_heal::ReplacementRecoveryRecord { + task_id: entry.key().clone(), + state: rustfs_heal::ReplacementRecoveryState::Unknown, + generation: None, + set_disk_id: None, + target_slots: Vec::new(), + reason: Some("peer replacement recovery records disagree".to_string()), + verified_at: None, + }; + } + entry.into_mut() } + }; + // Snapshot order keeps sources sorted; repeated copies within one + // snapshot must not duplicate a generation's provenance. + if snapshot.definitive && observation.sources.last() != Some(&source) { + observation.sources.push(source); } } } - records.into_values().collect() -} -/// A definitive local snapshot can retain multiple generations for one slot, -/// including completed replacements and committed handoff predecessors. Only -/// overlapping generations never observed together in a definitive snapshot -/// leave the merged peer histories ambiguous. Disagreeing copies of one task -/// are already rejected by the merge. -fn replacement_generations_claim_conflicting_targets( - records: &[rustfs_heal::ReplacementRecoveryRecord], - snapshots: &[rustfs_heal::ReplacementRecoverySnapshot], -) -> bool { - let local_histories = snapshots - .iter() - .filter(|snapshot| snapshot.definitive) - .map(|snapshot| { - snapshot - .records + let conflict = if observations + .values() + .any(|observation| observation.record.state == rustfs_heal::ReplacementRecoveryState::Unknown) + { + true + } else { + let mut target_sources = BTreeMap::<&str, Vec<&[usize]>>::new(); + for observation in observations.values() { + for target in &observation.record.target_slots { + target_sources.entry(target).or_default().push(&observation.sources); + } + } + target_sources.into_values().any(|sources| { + if sources.len() < 2 { + return false; + } + // A complete retained history normally has one common anchor. + // Checking that witness avoids comparing every historical pair. + if sources[0] .iter() - .map(|record| record.task_id.as_str()) - .collect::>() - }) - .collect::>(); - records.iter().any(|left| { - !left.target_slots.is_empty() - && records.iter().any(|right| { - right.task_id != left.task_id - && !right.target_slots.is_empty() - && right.target_slots.iter().any(|slot| left.target_slots.contains(slot)) - && !local_histories - .iter() - .any(|history| history.contains(left.task_id.as_str()) && history.contains(right.task_id.as_str())) + .any(|source| sources[1..].iter().all(|other| other.binary_search(source).is_ok())) + { + return false; + } + // Pairwise shared history is sufficient even without one anchor + // containing the whole group. A transitive chain alone is not. + sources.iter().enumerate().any(|(index, left)| { + sources[index + 1..] + .iter() + .any(|right| !left.iter().any(|source| right.binary_search(source).is_ok())) }) - }) + }) + }; + (observations.into_values().map(|observation| observation.record).collect(), conflict) } fn aggregate_replacement_recovery_cluster_status( @@ -802,12 +786,8 @@ fn aggregate_replacement_recovery_cluster_status( // disagree about one task (the merge marks those copies `Unknown`) or // claim overlapping targets without a shared definitive local history. // Retained histories and disjoint generations can be merged. - let records = merge_replacement_recovery_records(&snapshots); - if records - .iter() - .any(|record| matches!(record.state, rustfs_heal::ReplacementRecoveryState::Unknown)) - || replacement_generations_claim_conflicting_targets(&records, &snapshots) - { + let (records, conflict) = reconcile_replacement_recovery_records(snapshots); + if conflict { reason.get_or_insert("peer_replacement_recovery_status_conflict"); } @@ -2563,6 +2543,104 @@ mod tests { assert_eq!(cluster.records[1].state, rustfs_heal::ReplacementRecoveryState::Completed); } + #[test] + fn replacement_recovery_cluster_reconciles_target_and_source_matrix() { + let task_ids = [ + "11111111-1111-4111-8111-111111111111", + "22222222-2222-4222-8222-222222222222", + "33333333-3333-4333-8333-333333333333", + ]; + for targets in 0u8..8 { + let mut records = task_ids.map(replacement_record); + for (task, record) in records.iter_mut().enumerate() { + record.target_slots = vec![format!("http://node-a:9000/mnt/disk{}", (targets >> task) & 1)]; + } + // All non-empty source subsets cover pairwise histories without + // one common anchor, transitive-only histories, and empty peers. + for a in 1u8..8 { + for b in 1u8..8 { + for c in 1u8..8 { + let masks = [a, b, c]; + let snapshots = (0..3) + .map(|source| rustfs_heal::ReplacementRecoverySnapshot { + records: records + .iter() + .enumerate() + .filter(|(task, _)| masks[*task] & (1 << source) != 0) + .map(|(_, record)| record.clone()) + .collect(), + definitive: true, + reason: None, + }) + .collect::>(); + let expected = (0..3).all(|left| { + (left + 1..3).all(|right| { + records[left].target_slots != records[right].target_slots || masks[left] & masks[right] != 0 + }) + }); + let mut snapshots = snapshots.into_iter(); + let local = snapshots.next().expect("three source snapshots"); + let cluster = aggregate_replacement_recovery_cluster_status( + vec![local], + snapshots.map(|snapshot| Ok(Some(snapshot))).collect(), + 3, + true, + ); + + assert_eq!(cluster.definitive, expected, "targets={targets}, sources={masks:?}"); + assert_eq!(cluster.reason.is_none(), expected); + assert_eq!(cluster.records, records); + } + } + } + } + } + + #[test] + fn replacement_recovery_cluster_rejects_overlap_on_a_secondary_target() { + let mut local = replacement_snapshot_with_retained_history(); + local.records[0].target_slots.push("http://node-a:9000/mnt/disk2".to_string()); + let mut peer = replacement_snapshot("33333333-3333-4333-8333-333333333333"); + peer.records[0].target_slots = vec!["http://node-a:9000/mnt/disk2".to_string()]; + let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); + + assert!(!cluster.definitive); + assert_eq!(cluster.reason.as_deref(), Some("peer_replacement_recovery_status_conflict")); + assert_eq!(cluster.records.len(), 3); + } + + #[test] + fn replacement_recovery_cluster_deduplicates_reordered_targets() { + let mut local = replacement_snapshot("11111111-1111-4111-8111-111111111111"); + local.records[0].target_slots.push("http://node-a:9000/mnt/disk2".to_string()); + let mut peer = local.clone(); + peer.records[0].target_slots.reverse(); + let expected = local.records.clone(); + let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records, expected); + } + + #[test] + fn replacement_recovery_cluster_accepts_history_from_a_high_index_peer() { + let empty = rustfs_heal::ReplacementRecoverySnapshot { + records: Vec::new(), + definitive: true, + reason: None, + }; + let mut peers = vec![Ok(Some(empty.clone())); 80]; + let history = replacement_snapshot_with_retained_history(); + let expected = history.records.clone(); + peers.push(Ok(Some(history))); + let cluster = aggregate_replacement_recovery_cluster_status(vec![empty], peers, 82, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records, expected); + } + #[test] fn replacement_recovery_cluster_accepts_definitive_empty_peers() { let local = rustfs_heal::ReplacementRecoverySnapshot { From 93d51d172f51711799d91b1932abce02513d40e3 Mon Sep 17 00:00:00 2001 From: Hauser Date: Wed, 14 Oct 2026 02:19:22 +0800 Subject: [PATCH 051/143] fix(server): cancel request watchdogs with response futures (#8528) Tie watchdog cancellation to response future ownership before the first poll, and prefer cancellation when the watchdog deadline is also ready. Preserve completion logging and genuine slow-request warnings. Add paused-clock regression coverage for dropped, completed, panicking, and still-pending request futures. Co-authored-by: heihutu Co-authored-by: zhi22915 --- rustfs/src/server/layer.rs | 155 ++++++++++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 3 deletions(-) diff --git a/rustfs/src/server/layer.rs b/rustfs/src/server/layer.rs index 1cbbeefbc..bcc21fb37 100644 --- a/rustfs/src/server/layer.rs +++ b/rustfs/src/server/layer.rs @@ -587,6 +587,7 @@ where let watchdog_context = context.clone(); async move { tokio::select! { + biased; _ = watchdog.cancelled() => {} _ = tokio::time::sleep(HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD) => { watchdog_context.log_slow_inflight(); @@ -596,11 +597,11 @@ where }); } + // An unpolled response future must also cancel its watchdog on drop. + let watchdog_guard = watchdog.map(CancellationToken::drop_guard); Box::pin(async move { let result = inner.call(req).await; - if let Some(watchdog) = watchdog { - watchdog.cancel(); - } + drop(watchdog_guard); match &result { Ok(response) => context.log_response(response), Err(error) => context.log_failure(error), @@ -5747,6 +5748,154 @@ mod tests { assert_eq!(redact_sensitive_uri_query(&uri), "/rustfs/admin/v3/users?token=not-a-download-token"); } + #[tokio::test(start_paused = true)] + async fn request_logging_watchdog_cancels_dropped_futures() { + for poll_request in [false, true] { + for deadline_ready in [false, true] { + let attempts = if deadline_ready { 32 } else { 1 }; + for attempt in 0..attempts { + let writer = SharedWriter::default(); + let captured = writer.buffer.clone(); + let _subscriber = tracing::subscriber::set_default( + tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_writer(writer) + .finish(), + ); + assert!(tracing::enabled!(target: HTTP_SERVER_LOG_TARGET, Level::WARN)); + let inner = + tower::service_fn(|_: HttpRequest<()>| std::future::pending::, Infallible>>()); + let mut service = RequestLoggingLayer.layer(inner); + let mut response = service.call(HttpRequest::new(())); + if poll_request { + assert!(futures::poll!(&mut response).is_pending()); + } + tokio::task::yield_now().await; + if deadline_ready { + tokio::time::advance(HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD).await; + } + drop(response); + tokio::time::advance(HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD + Duration::from_millis(1)).await; + tokio::task::yield_now().await; + + let output = String::from_utf8(captured.lock().expect("logs").clone()).expect("UTF-8 logs"); + assert!( + !output.contains(HTTP_REQUEST_INFLIGHT_SLOW_EVENT), + "polled={poll_request}, deadline_ready={deadline_ready}, attempt={attempt}: {output}" + ); + } + } + } + } + + #[tokio::test(start_paused = true)] + async fn request_logging_watchdog_cancels_panicking_future() { + let writer = SharedWriter::default(); + let captured = writer.buffer.clone(); + let _subscriber = tracing::subscriber::set_default( + tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_writer(writer) + .finish(), + ); + assert!(tracing::enabled!(target: HTTP_SERVER_LOG_TARGET, Level::WARN)); + let inner = tower::service_fn(|_: HttpRequest<()>| { + std::future::poll_fn(|_| -> Poll, Infallible>> { panic!("test service panic") }) + }); + let mut service = RequestLoggingLayer.layer(inner); + let mut response = service.call(HttpRequest::new(())); + tokio::task::yield_now().await; + let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + response.as_mut().poll(&mut Context::from_waker(std::task::Waker::noop())) + })); + assert!(panic.is_err()); + tokio::time::advance(HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD + Duration::from_millis(1)).await; + tokio::task::yield_now().await; + + let output = String::from_utf8(captured.lock().expect("logs").clone()).expect("UTF-8 logs"); + assert!(!output.contains(HTTP_REQUEST_INFLIGHT_SLOW_EVENT), "{output}"); + drop(response); + } + + #[tokio::test(start_paused = true)] + async fn request_logging_watchdog_cancels_completed_futures() { + for fail_request in [false, true] { + let writer = SharedWriter::default(); + let captured = writer.buffer.clone(); + let _subscriber = tracing::subscriber::set_default( + tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_writer(writer) + .finish(), + ); + assert!(tracing::enabled!(target: HTTP_SERVER_LOG_TARGET, Level::WARN)); + let inner = tower::service_fn(move |_: HttpRequest<()>| { + std::future::ready(if fail_request { + Err(io::Error::other("test service failure")) + } else { + Ok(Response::new(())) + }) + }); + let mut service = RequestLoggingLayer.layer(inner); + let mut response = service.call(HttpRequest::new(())); + tokio::task::yield_now().await; + let Poll::Ready(result) = futures::poll!(&mut response) else { + panic!("service should complete immediately"); + }; + assert_eq!(result.is_err(), fail_request); + // Retaining a completed future must not retain an active watchdog. + tokio::time::advance(HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD + Duration::from_millis(1)).await; + tokio::task::yield_now().await; + + let output = String::from_utf8(captured.lock().expect("logs").clone()).expect("UTF-8 logs"); + assert!(!output.contains(HTTP_REQUEST_INFLIGHT_SLOW_EVENT), "{output}"); + let expected_event = if fail_request { + HTTP_REQUEST_FAILED_EVENT + } else { + HTTP_REQUEST_COMPLETED_EVENT + }; + assert_eq!(output.matches(expected_event).count(), 1, "{output}"); + drop(response); + } + } + + #[tokio::test(start_paused = true)] + async fn request_logging_watchdog_warns_once_for_pending_request() { + let writer = SharedWriter::default(); + let captured = writer.buffer.clone(); + let _subscriber = tracing::subscriber::set_default( + tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_writer(writer) + .finish(), + ); + assert!(tracing::enabled!(target: HTTP_SERVER_LOG_TARGET, Level::WARN)); + let inner = tower::service_fn(|_: HttpRequest<()>| std::future::pending::, Infallible>>()); + let mut service = RequestLoggingLayer.layer(inner); + let mut response = service.call(HttpRequest::new(())); + assert!(futures::poll!(&mut response).is_pending()); + tokio::task::yield_now().await; + tokio::time::advance(HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD - Duration::from_millis(1)).await; + tokio::task::yield_now().await; + assert!(captured.lock().expect("logs").is_empty()); + + tokio::time::advance(Duration::from_millis(1)).await; + tokio::task::yield_now().await; + let output = String::from_utf8(captured.lock().expect("logs").clone()).expect("UTF-8 logs"); + assert_eq!(output.matches(HTTP_REQUEST_INFLIGHT_SLOW_EVENT).count(), 1, "{output}"); + assert!(output.contains("response_pending"), "{output}"); + + tokio::time::advance(HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD).await; + tokio::task::yield_now().await; + let output = String::from_utf8(captured.lock().expect("logs").clone()).expect("UTF-8 logs"); + assert_eq!(output.matches(HTTP_REQUEST_INFLIGHT_SLOW_EVENT).count(), 1, "{output}"); + drop(response); + } + #[tokio::test] async fn request_logging_bounds_s3_failure_bursts_without_losing_counts_or_leaking_queries() { use rustfs_io_metrics::s3_http_metrics::s3_http_metrics_snapshot; From 4502a1f9b0b6ba44553470fa71381e4076cd110e Mon Sep 17 00:00:00 2001 From: Hauser Date: Wed, 14 Oct 2026 02:59:59 +0800 Subject: [PATCH 052/143] fix(ci): repair profiling and compact transition test gates (#8526) * fix(deps): patch profiling dependency for RUSTSEC-2026-0344 Vendor the published Pyroscope package with an object 0.40 dependency and lock ruzstd 0.9.1. Preserve runtime sources and profiling features, add bounded decoder regressions, and run the patched package in audit CI. Co-Authored-By: heihutu Co-Authored-By: zhi22915 Signed-off-by: Hauser * fix(deps): correct vendored pyroscope documentation typos Correct copied comments and changelog spelling for the repository typo check, and describe these comment-only changes in the vendor provenance. Co-Authored-By: heihutu Co-Authored-By: zhi22915 Signed-off-by: Hauser * fix(ecstore): refresh compact transition proof per test case Install the short-lived homogeneous fleet proof immediately before each kill-point transaction. Earlier cancellation and recovery work must not age the next case's writer capability fixture. Report an early transition result instead of waiting for an unreachable kill point. Preserve cancellation, remote-owner and payload assertions. Co-Authored-By: heihutu Co-Authored-By: zhi22915 Signed-off-by: Hauser --------- Signed-off-by: Hauser Co-authored-by: heihutu Co-authored-by: zhi22915 --- .github/workflows/audit.yml | 9 + Cargo.lock | 12 +- Cargo.toml | 6 + crates/ecstore/src/store/init.rs | 14 +- third-party/pyroscope/.gitignore | 2 + third-party/pyroscope/CHANGELOG.md | 265 +++++++ third-party/pyroscope/Cargo.toml | 233 ++++++ third-party/pyroscope/Cargo.toml.orig | 119 +++ third-party/pyroscope/LICENSE | 202 +++++ third-party/pyroscope/README.md | 22 + third-party/pyroscope/RUSTFS.md | 9 + third-party/pyroscope/build.rs | 19 + third-party/pyroscope/examples/jemalloc.rs | 67 ++ third-party/pyroscope/rustfmt.toml | 0 third-party/pyroscope/src/backend/backend.rs | 134 ++++ third-party/pyroscope/src/backend/jemalloc.rs | 87 +++ third-party/pyroscope/src/backend/mod.rs | 71 ++ third-party/pyroscope/src/backend/pprof.rs | 240 ++++++ .../src/backend/pprofrs/addr_validate.rs | 140 ++++ .../pprofrs/backtrace/framehop_unwinder.rs | 189 +++++ .../backtrace/framehop_unwinder/shlib.rs | 78 ++ .../src/backend/pprofrs/backtrace/mod.rs | 51 ++ .../src/backend/pprofrs/collector.rs | 558 ++++++++++++++ .../pyroscope/src/backend/pprofrs/error.rs | 17 + .../pyroscope/src/backend/pprofrs/frames.rs | 236 ++++++ .../pyroscope/src/backend/pprofrs/mod.rs | 32 + .../pyroscope/src/backend/pprofrs/profiler.rs | 540 +++++++++++++ .../pyroscope/src/backend/pprofrs/report.rs | 103 +++ .../pyroscope/src/backend/pprofrs/timer.rs | 70 ++ third-party/pyroscope/src/backend/ruleset.rs | 82 ++ third-party/pyroscope/src/backend/tests.rs | 378 +++++++++ third-party/pyroscope/src/backend/types.rs | 303 ++++++++ .../pyroscope/src/encode/gen/google.rs | 233 ++++++ third-party/pyroscope/src/encode/gen/mod.rs | 3 + third-party/pyroscope/src/encode/gen/push.rs | 31 + third-party/pyroscope/src/encode/gen/types.rs | 11 + third-party/pyroscope/src/encode/mod.rs | 2 + third-party/pyroscope/src/encode/pprof.rs | 159 ++++ third-party/pyroscope/src/error.rs | 67 ++ third-party/pyroscope/src/ffikit.rs | 74 ++ third-party/pyroscope/src/lib.rs | 67 ++ third-party/pyroscope/src/pyroscope.rs | 723 ++++++++++++++++++ third-party/pyroscope/src/session.rs | 270 +++++++ third-party/pyroscope/src/timer/epoll.rs | 272 +++++++ third-party/pyroscope/src/timer/kqueue.rs | 195 +++++ third-party/pyroscope/src/timer/mod.rs | 38 + third-party/pyroscope/src/timer/sleep.rs | 118 +++ third-party/pyroscope/src/utils.rs | 167 ++++ third-party/pyroscope/tests/agent.rs | 41 + third-party/pyroscope/tests/pprof_backend.rs | 49 ++ third-party/pyroscope/tests/rustfs_zstd.rs | 41 + third-party/pyroscope/tests/session.rs | 61 ++ third-party/pyroscope/tests/timer-epoll.rs | 86 +++ third-party/pyroscope/tests/timer.rs | 29 + 54 files changed, 7015 insertions(+), 10 deletions(-) create mode 100644 third-party/pyroscope/.gitignore create mode 100644 third-party/pyroscope/CHANGELOG.md create mode 100644 third-party/pyroscope/Cargo.toml create mode 100644 third-party/pyroscope/Cargo.toml.orig create mode 100644 third-party/pyroscope/LICENSE create mode 100644 third-party/pyroscope/README.md create mode 100644 third-party/pyroscope/RUSTFS.md create mode 100644 third-party/pyroscope/build.rs create mode 100644 third-party/pyroscope/examples/jemalloc.rs create mode 100644 third-party/pyroscope/rustfmt.toml create mode 100644 third-party/pyroscope/src/backend/backend.rs create mode 100644 third-party/pyroscope/src/backend/jemalloc.rs create mode 100644 third-party/pyroscope/src/backend/mod.rs create mode 100644 third-party/pyroscope/src/backend/pprof.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/addr_validate.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder/shlib.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/backtrace/mod.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/collector.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/error.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/frames.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/mod.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/profiler.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/report.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/timer.rs create mode 100644 third-party/pyroscope/src/backend/ruleset.rs create mode 100644 third-party/pyroscope/src/backend/tests.rs create mode 100644 third-party/pyroscope/src/backend/types.rs create mode 100644 third-party/pyroscope/src/encode/gen/google.rs create mode 100644 third-party/pyroscope/src/encode/gen/mod.rs create mode 100644 third-party/pyroscope/src/encode/gen/push.rs create mode 100644 third-party/pyroscope/src/encode/gen/types.rs create mode 100644 third-party/pyroscope/src/encode/mod.rs create mode 100644 third-party/pyroscope/src/encode/pprof.rs create mode 100644 third-party/pyroscope/src/error.rs create mode 100644 third-party/pyroscope/src/ffikit.rs create mode 100644 third-party/pyroscope/src/lib.rs create mode 100644 third-party/pyroscope/src/pyroscope.rs create mode 100644 third-party/pyroscope/src/session.rs create mode 100644 third-party/pyroscope/src/timer/epoll.rs create mode 100644 third-party/pyroscope/src/timer/kqueue.rs create mode 100644 third-party/pyroscope/src/timer/mod.rs create mode 100644 third-party/pyroscope/src/timer/sleep.rs create mode 100644 third-party/pyroscope/src/utils.rs create mode 100644 third-party/pyroscope/tests/agent.rs create mode 100644 third-party/pyroscope/tests/pprof_backend.rs create mode 100644 third-party/pyroscope/tests/rustfs_zstd.rs create mode 100644 third-party/pyroscope/tests/session.rs create mode 100644 third-party/pyroscope/tests/timer-epoll.rs create mode 100644 third-party/pyroscope/tests/timer.rs diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index c1406bec2..ea0e3efdf 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -20,6 +20,7 @@ on: paths: - '**/Cargo.toml' - '**/Cargo.lock' + - 'third-party/pyroscope/**' - 'deny.toml' - '.github/actions/**' - '.github/workflows/**' @@ -39,6 +40,7 @@ on: paths: - '**/Cargo.toml' - '**/Cargo.lock' + - 'third-party/pyroscope/**' - 'deny.toml' - '.github/actions/**' - '.github/workflows/**' @@ -129,6 +131,13 @@ jobs: - name: Run cargo-deny run: cargo deny check --hide-inclusion-graph advisories sources bans licenses + - name: Test patched profiling dependency + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target/pyroscope-dependency + run: >- + cargo test --manifest-path third-party/pyroscope/Cargo.toml + --features backend-pprof-rs --lib --test rustfs_zstd + workflow-pin-report: name: Workflow Pin Report if: github.repository == 'rustfs/rustfs' && (github.event_name != 'pull_request' || github.event.action != 'closed') diff --git a/Cargo.lock b/Cargo.lock index 3d4cf7f87..afba0c499 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7257,9 +7257,7 @@ version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" dependencies = [ - "flate2", "memchr", - "ruzstd", ] [[package]] @@ -7269,9 +7267,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd229a0361b9d0d4396176e02d65897f487eebeab7caa6d443855ee152ca0b9c" dependencies = [ "crc32fast", + "flate2", "hashbrown 0.17.1", "indexmap 2.14.2", "memchr", + "ruzstd", ] [[package]] @@ -8530,8 +8530,6 @@ dependencies = [ [[package]] name = "pyroscope" version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b1989a61e856eba4ef903eadad8313f8376e88e7c9cc5d5db8217d832a64d9" dependencies = [ "aligned-vec", "backtrace", @@ -8542,7 +8540,7 @@ dependencies = [ "log", "memmap2", "nix 0.31.3", - "object 0.39.1", + "object 0.40.0", "once_cell", "prost 0.14.4", "reqwest", @@ -11288,9 +11286,9 @@ dependencies = [ [[package]] name = "ruzstd" -version = "0.8.3" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7c1c839d570d835527c9a5e4db7cb2198683a988cb9d7293fc8674e6bd58fc8" +checksum = "69e4ce4a039f42bf25920b95667e4d789e98749ab11bbb8d2c2936f788421bb5" dependencies = [ "twox-hash", ] diff --git a/Cargo.toml b/Cargo.toml index a0b447ac7..fd410f859 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ members = [ "crates/io-core", # Buffered Bytes, mmap-then-copy, and aligned pread I/O helpers "crates/zip", # ZIP file handling and compression ] +exclude = ["third-party/pyroscope"] resolver = "3" [workspace.package] @@ -421,3 +422,8 @@ inherits = "release" inherits = "release" debug = true strip = "none" + +# Remove this patch when Pyroscope releases support for object 0.40 or later. +# object 0.39 requires ruzstd 0.8, affected by RUSTSEC-2026-0344. +[patch.crates-io] +pyroscope = { path = "third-party/pyroscope" } diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 86cf16be7..6086f80e6 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -22143,7 +22143,6 @@ mod tests { .make_bucket(bucket, &MakeBucketOptions::default()) .await .expect("kill-point bucket should be created"); - let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test"); for (index, point, expected_state, expected_revision, expected_committed, expected_recovered) in [ ( @@ -22187,10 +22186,14 @@ mod tests { .await .expect("kill-point source should be written"); let remote_before = backend.object_count().await; + // Earlier cancellation/recovery cases must not consume this + // transaction's 30-second homogeneous-fleet proof window. + let _compaction_proof = + install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test"); let barrier = TransitionTransactionKillPointBarrier::install(bucket, &object, point); let transition_store = store.clone(); let transition_object = object.clone(); - let transition = tokio::spawn(async move { + let mut transition = tokio::spawn(async move { transition_store .transition_object( bucket, @@ -22209,7 +22212,12 @@ mod tests { ) .await }); - barrier.wait_until_paused().await; + tokio::select! { + _ = barrier.wait_until_paused() => {} + result = &mut transition => { + panic!("transition exited before reaching {point:?}: {result:?}"); + } + } transition.abort(); assert!( transition diff --git a/third-party/pyroscope/.gitignore b/third-party/pyroscope/.gitignore new file mode 100644 index 000000000..042776aad --- /dev/null +++ b/third-party/pyroscope/.gitignore @@ -0,0 +1,2 @@ +/Cargo.lock +/target/ diff --git a/third-party/pyroscope/CHANGELOG.md b/third-party/pyroscope/CHANGELOG.md new file mode 100644 index 000000000..eca61685b --- /dev/null +++ b/third-party/pyroscope/CHANGELOG.md @@ -0,0 +1,265 @@ +# Changelog + +## [2.1.1](https://github.com/grafana/pyroscope-rs/compare/lib-2.1.0...lib-2.1.1) (2026-07-21) + + +### Bug Fixes + +* avoid slice construction in address validation ([#569](https://github.com/grafana/pyroscope-rs/issues/569)) ([d6f45fe](https://github.com/grafana/pyroscope-rs/commit/d6f45fef1443d26c64a40721013ecfa7b1d3eccd)) + + +### Miscellaneous Chores + +* **deps:** lock file maintenance ([#570](https://github.com/grafana/pyroscope-rs/issues/570)) ([c37f52e](https://github.com/grafana/pyroscope-rs/commit/c37f52e71719da9ddb4a32ac8386a4c94534c805)) + +## [2.1.0](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.6...lib-2.1.0) (2026-07-07) + + +### Features + +* add a method to set the timer's ticker in PyroscopeAgent ([#557](https://github.com/grafana/pyroscope-rs/issues/557)) ([cc13d38](https://github.com/grafana/pyroscope-rs/commit/cc13d38cb5e84017a4f5e229648457b790bd6a1a)) +* add otel.scope.name, otel.scope.version, process.runtime.name, process.runtime.version labels ([#562](https://github.com/grafana/pyroscope-rs/issues/562)) ([32f076e](https://github.com/grafana/pyroscope-rs/commit/32f076efd45aa2f3f203eb0bf8b03f222cb40648)) + + +### Bug Fixes + +* **security/unknown/:** update security memmap2 to v0.9.11 [security] ([#558](https://github.com/grafana/pyroscope-rs/issues/558)) ([d4d83f8](https://github.com/grafana/pyroscope-rs/commit/d4d83f8d350d927739c79a53bcfd7cd0115c77a4)) + + +### Miscellaneous Chores + +* bump nix ([#566](https://github.com/grafana/pyroscope-rs/issues/566)) ([1c02f30](https://github.com/grafana/pyroscope-rs/commit/1c02f3058135d1db06262ca57ead33fdc1593e6f)) +* **deps:** lock file maintenance ([#555](https://github.com/grafana/pyroscope-rs/issues/555)) ([3917b99](https://github.com/grafana/pyroscope-rs/commit/3917b99d6de4f4d6cec078aba2ec7b6f4e7044c7)) + +## [2.0.6](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.5...lib-2.0.6) (2026-05-28) + + +### Miscellaneous Chores + +* **deps:** vendor pprofrs ([#552](https://github.com/grafana/pyroscope-rs/issues/552)) ([6684026](https://github.com/grafana/pyroscope-rs/commit/6684026ba0ea4fc997e4869703d671f23ee377a7)) + + +### Continuous Integration + +* remove pr-title-check workflow ([#549](https://github.com/grafana/pyroscope-rs/issues/549)) ([e4739e4](https://github.com/grafana/pyroscope-rs/commit/e4739e4bfa280e771fbd932f39d53faa599718ba)) + +## [2.0.5](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.4...lib-2.0.5) (2026-05-27) + + +### Miscellaneous Chores + +* **deps:** Remove unnecessary framehop dep ([#550](https://github.com/grafana/pyroscope-rs/issues/550)) ([d9c0e02](https://github.com/grafana/pyroscope-rs/commit/d9c0e02c7386b3a55e0270083c1135cac0e3ce0e)) + +## [2.0.4](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.3...lib-2.0.4) (2026-05-20) + + +### Bug Fixes + +* do not recreate profiling guard, just reset the data ([#518](https://github.com/grafana/pyroscope-rs/issues/518)) ([b1fe114](https://github.com/grafana/pyroscope-rs/commit/b1fe1141ee0d86caa59db383298c818763f3fa95)) +* remove kindasafe CI leftovers ([#521](https://github.com/grafana/pyroscope-rs/issues/521)) ([f8bc0f3](https://github.com/grafana/pyroscope-rs/commit/f8bc0f31c3b775b9d0e97fb412af76059f0819ec)) + + +### Miscellaneous Chores + +* **deps:** bump openssl from 0.10.79 to 0.10.80 ([#548](https://github.com/grafana/pyroscope-rs/issues/548)) ([f3ed911](https://github.com/grafana/pyroscope-rs/commit/f3ed911e8b179d6fd63eacc2a793e884f126dfa0)) +* **deps:** lock file maintenance ([#535](https://github.com/grafana/pyroscope-rs/issues/535)) ([9554564](https://github.com/grafana/pyroscope-rs/commit/955456438ea768adfe0a89cd1ef7da22e7974bcd)) +* **deps:** lock file maintenance ([#541](https://github.com/grafana/pyroscope-rs/issues/541)) ([3abc7b6](https://github.com/grafana/pyroscope-rs/commit/3abc7b6d0c01f420a16cd2d08f44f10424c3ef3c)) +* **deps:** lock file maintenance ([#547](https://github.com/grafana/pyroscope-rs/issues/547)) ([cd30ebf](https://github.com/grafana/pyroscope-rs/commit/cd30ebfb07bfbf243125df53f051a2f2d1dd5d63)) +* **deps:** pin rust docker tag to a9cfb75 ([#544](https://github.com/grafana/pyroscope-rs/issues/544)) ([ea14091](https://github.com/grafana/pyroscope-rs/commit/ea140911e4b8bbabc8c42fbb2c7d43e0384ec4fe)) +* **deps:** remove names crate dependency ([#527](https://github.com/grafana/pyroscope-rs/issues/527)) ([5893aac](https://github.com/grafana/pyroscope-rs/commit/5893aacb5bc3660dcd269b2fee33847f1fe1ab82)) +* **deps:** update rust crate libc to v0.2.186 ([#533](https://github.com/grafana/pyroscope-rs/issues/533)) ([a7468c1](https://github.com/grafana/pyroscope-rs/commit/a7468c1db5d7955dfa0eb702a8b0371775ff2350)) +* **deps:** update rust crate reqwest to v0.13.3 ([#539](https://github.com/grafana/pyroscope-rs/issues/539)) ([b8c6fbb](https://github.com/grafana/pyroscope-rs/commit/b8c6fbb97cc5f3b22185be35c9e19e72237b912e)) +* **deps:** update rust:trixie docker digest to a9cfb75 ([#538](https://github.com/grafana/pyroscope-rs/issues/538)) ([bff1d5d](https://github.com/grafana/pyroscope-rs/commit/bff1d5d98f2a81d7a291c257bfb8291b41ffc8e5)) +* **renovate:** vulnerabilities-only with profiling deps allowlist ([#540](https://github.com/grafana/pyroscope-rs/issues/540)) ([4aa8e56](https://github.com/grafana/pyroscope-rs/commit/4aa8e5656472a08ddea21cb2128b85203920243b)) + + +### Continuous Integration + +* add ARM cross-compilation test job ([#536](https://github.com/grafana/pyroscope-rs/issues/536)) ([54bb03f](https://github.com/grafana/pyroscope-rs/commit/54bb03f6996a74ad98cc75bdd08d790bce61efa3)) +* auto-publish to crates.io after release-please PR merge ([#529](https://github.com/grafana/pyroscope-rs/issues/529)) ([a4f76b6](https://github.com/grafana/pyroscope-rs/commit/a4f76b6f9de6b6c35e4d947aba4dbfe49d8c019a)) +* Potential fix for code scanning alert no. 14: Workflow does not contain permissions ([#542](https://github.com/grafana/pyroscope-rs/issues/542)) ([51ac4b7](https://github.com/grafana/pyroscope-rs/commit/51ac4b78e4a8cc0c8130984b52181fce12e3589f)) + +## [2.0.3](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.2...lib-2.0.3) (2026-04-26) + + +### Bug Fixes + +* **epoll:** use libc::c_long for itimerspec fields ([#532](https://github.com/grafana/pyroscope-rs/issues/532)) ([093406e](https://github.com/grafana/pyroscope-rs/commit/093406e667175e5bc28ca7a1c5efe62be6b9b4a0)) + + +### Miscellaneous Chores + +* **deps:** update googleapis/release-please-action action to v5 ([#530](https://github.com/grafana/pyroscope-rs/issues/530)) ([6c97b9d](https://github.com/grafana/pyroscope-rs/commit/6c97b9d414c7a1e26b07334a1cf0922e1c2cd2c9)) + +## [2.0.2](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.1...lib-2.0.2) (2026-04-24) + + +### Miscellaneous Chores + +* **deps:** bump rustls-webpki from 0.103.12 to 0.103.13 ([#524](https://github.com/grafana/pyroscope-rs/issues/524)) ([935384d](https://github.com/grafana/pyroscope-rs/commit/935384dc4d3479a1a12df2502898a8813b775884)) +* **deps:** lock file maintenance ([#522](https://github.com/grafana/pyroscope-rs/issues/522)) ([4bd95bb](https://github.com/grafana/pyroscope-rs/commit/4bd95bb47f84f3aa6db6aa22173a7d74f774ab45)) +* **deps:** update dependency ruby to v4.0.3 ([#523](https://github.com/grafana/pyroscope-rs/issues/523)) ([e126aa2](https://github.com/grafana/pyroscope-rs/commit/e126aa24517ba6efea62302e465ea736e4cef54c)) +* **deps:** update rust:trixie docker digest to e4f09e8 ([#515](https://github.com/grafana/pyroscope-rs/issues/515)) ([f450327](https://github.com/grafana/pyroscope-rs/commit/f450327b1901f43d97d96eee5e0d73ccb64aa768)) + +## [2.0.1](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.0...lib-2.0.1) (2026-04-19) + + +### Bug Fixes + +* revert 0976d999e4a50c14459b8a0b39e72434be6d4bfc ([#483](https://github.com/grafana/pyroscope-rs/issues/483)) ([ad3d032](https://github.com/grafana/pyroscope-rs/commit/ad3d03217a14a18868b8ba21f3dbf4ed844d759a)) + + +### Miscellaneous Chores + +* **deps:** lock file maintenance ([#467](https://github.com/grafana/pyroscope-rs/issues/467)) ([09411da](https://github.com/grafana/pyroscope-rs/commit/09411daca91120c4863983273863c55b02b4dd89)) +* **deps:** lock file maintenance ([#478](https://github.com/grafana/pyroscope-rs/issues/478)) ([447d8ee](https://github.com/grafana/pyroscope-rs/commit/447d8eef3621be5b9b09b0b9eca3968bca081c2a)) +* **deps:** lock file maintenance ([#498](https://github.com/grafana/pyroscope-rs/issues/498)) ([4072a04](https://github.com/grafana/pyroscope-rs/commit/4072a04626abd98aa18bc6cda37fb0ccc0803f59)) +* **deps:** lock file maintenance ([#502](https://github.com/grafana/pyroscope-rs/issues/502)) ([bef3bc3](https://github.com/grafana/pyroscope-rs/commit/bef3bc3ec08b57b74d5d177204118492cdaa50b2)) +* **deps:** pin dependencies ([#427](https://github.com/grafana/pyroscope-rs/issues/427)) ([a46830a](https://github.com/grafana/pyroscope-rs/commit/a46830ac9a75a7634d9cc1ecc514a21b5c0f0603)) +* **deps:** pin rust docker tag to e8e2bb5 ([#501](https://github.com/grafana/pyroscope-rs/issues/501)) ([c177d5b](https://github.com/grafana/pyroscope-rs/commit/c177d5b8f28041d3956851d93e61be3aebc6d975)) +* **deps:** update dtolnay/rust-toolchain digest to 3c5f7ea ([#480](https://github.com/grafana/pyroscope-rs/issues/480)) ([c4f2e7d](https://github.com/grafana/pyroscope-rs/commit/c4f2e7dd1b1234f8870ce0f858f92542aef37c2e)) +* **deps:** update dtolnay/rust-toolchain digest to 3c5f7ea ([#500](https://github.com/grafana/pyroscope-rs/issues/500)) ([fc21bda](https://github.com/grafana/pyroscope-rs/commit/fc21bda959bdf5d5d5a44f559a2b9de8072d2fba)) +* **deps:** update googleapis/release-please-action digest to 5c625bf ([#509](https://github.com/grafana/pyroscope-rs/issues/509)) ([c347e17](https://github.com/grafana/pyroscope-rs/commit/c347e1775487456b996b87ce4672a20d9b9f4541)) +* **deps:** update rust crate env_logger to v0.11.10 ([#490](https://github.com/grafana/pyroscope-rs/issues/490)) ([24fb12d](https://github.com/grafana/pyroscope-rs/commit/24fb12d4c26c9cc167585f3fd93172015864a0e4)) +* **deps:** update rust crate libc to v0.2.183 ([#489](https://github.com/grafana/pyroscope-rs/issues/489)) ([b44bac8](https://github.com/grafana/pyroscope-rs/commit/b44bac8f4274882d793e9e9f92ab0ad9fc4692f0)) +* **deps:** update rust crate libc to v0.2.185 ([#514](https://github.com/grafana/pyroscope-rs/issues/514)) ([e72f5c6](https://github.com/grafana/pyroscope-rs/commit/e72f5c6e4944e7b2ac949d6dbd3201dc426704d5)) +* **deps:** update rust crate libflate to v2.3.0 ([#516](https://github.com/grafana/pyroscope-rs/issues/516)) ([da2835f](https://github.com/grafana/pyroscope-rs/commit/da2835f536771e089635e9fd8b97a860b09b9479)) +* **deps:** update rust crate rbspy to 0.43 ([#471](https://github.com/grafana/pyroscope-rs/issues/471)) ([1e524d3](https://github.com/grafana/pyroscope-rs/commit/1e524d38d84dfedc799515cb70ee434ef91a28e6)) +* **deps:** update rust crate rbspy to 0.44 ([#477](https://github.com/grafana/pyroscope-rs/issues/477)) ([c7cfe39](https://github.com/grafana/pyroscope-rs/commit/c7cfe39fecd8a0624df5c5d22b709e73747ddb4d)) +* **deps:** update rust crate uuid to v1.23.0 ([#481](https://github.com/grafana/pyroscope-rs/issues/481)) ([9c4b53a](https://github.com/grafana/pyroscope-rs/commit/9c4b53aef55b13dc6da99396119e33ec9dc02aa0)) +* **deps:** update rust crate uuid to v1.23.1 ([#517](https://github.com/grafana/pyroscope-rs/issues/517)) ([6a89238](https://github.com/grafana/pyroscope-rs/commit/6a89238e7127cb5f3fa0f480ba7447369a21fc12)) +* **deps:** update rust-lang/crates-io-auth-action action to v1.0.4 ([#479](https://github.com/grafana/pyroscope-rs/issues/479)) ([f4e08f3](https://github.com/grafana/pyroscope-rs/commit/f4e08f34aa2b14a1e417c005d37f75981940d675)) +* **deps:** update rust:trixie docker digest to 652612f ([#510](https://github.com/grafana/pyroscope-rs/issues/510)) ([7f76541](https://github.com/grafana/pyroscope-rs/commit/7f76541afc2a03cd783b0a0627b7a876f18c4918)) +* **deps:** update softprops/action-gh-release action to v2.5.2 ([#473](https://github.com/grafana/pyroscope-rs/issues/473)) ([1700be3](https://github.com/grafana/pyroscope-rs/commit/1700be3e34e7351a2f5f030187a2fac1ce1082e9)) +* **deps:** update softprops/action-gh-release action to v2.5.3 ([#474](https://github.com/grafana/pyroscope-rs/issues/474)) ([12e34c1](https://github.com/grafana/pyroscope-rs/commit/12e34c109bd0e408dc0401430f4da4b5245a09da)) +* **deps:** update softprops/action-gh-release action to v2.6.1 ([#475](https://github.com/grafana/pyroscope-rs/issues/475)) ([209c4c0](https://github.com/grafana/pyroscope-rs/commit/209c4c03ce08fe411046316dc333654fb64ab75b)) +* remove FFI code and kindasafe (moved to separate repos) ([#488](https://github.com/grafana/pyroscope-rs/issues/488)) ([35a4c6a](https://github.com/grafana/pyroscope-rs/commit/35a4c6aa0808f0392e39e752d95a91536068ab74)) + + +### Documentation + +* fix broken Rust documentation and add doc-test CI ([#484](https://github.com/grafana/pyroscope-rs/issues/484)) ([98ae8a5](https://github.com/grafana/pyroscope-rs/commit/98ae8a595f18946f2edf27544ea6e2bb29724cbf)) + + +### Continuous Integration + +* add musl/Alpine to Rust test matrix ([#495](https://github.com/grafana/pyroscope-rs/issues/495)) ([a4b7ad4](https://github.com/grafana/pyroscope-rs/commit/a4b7ad4c9e530576adbbf278b104d65c970d43b0)) +* add release-please configuration ([#491](https://github.com/grafana/pyroscope-rs/issues/491)) ([fbb2d8e](https://github.com/grafana/pyroscope-rs/commit/fbb2d8edc7cf400343650e77580464d32369d93d)) + +## v2.0.0 + +## Breaking Changes +- `Backend::report()` now returns `ReportBatch` instead of `Vec` ([#437](https://github.com/grafana/pyroscope-rs/pull/437), [#447](https://github.com/grafana/pyroscope-rs/pull/447)) +- New `ReportData` enum replaces the plain `Vec` in `ReportBatch`, supporting both structured reports (`ReportData::Reports`) and pre-encoded pprof bytes (`ReportData::RawPprof`) ([#447](https://github.com/grafana/pyroscope-rs/pull/447)) +- Minimum supported Rust version bumped from 1.64 to 1.66 + +## New Features +- **Jemalloc memory profiling backend** — new `backend-jemalloc` feature flag enables heap profiling via `jemalloc_pprof`. Use `pyroscope::backend::jemalloc::jemalloc_backend()` to get started ([#378](https://github.com/grafana/pyroscope-rs/pull/378)) +- **`ReportBatch` type** — backends now return a `ReportBatch` with a `profile_type` field (e.g. `"process_cpu"`, `"memory"`), enabling multi-profile support ([#437](https://github.com/grafana/pyroscope-rs/pull/437)) + +## Dependencies +- Updated `pprof` (pprof-pyroscope-fork) to v0.1500.3 ([#407](https://github.com/grafana/pyroscope-rs/pull/407)) +- Updated `object` crate to 0.38 ([#430](https://github.com/grafana/pyroscope-rs/pull/430)) +- Disabled py-spy default features to exclude CLI dependencies ([#418](https://github.com/grafana/pyroscope-rs/pull/418)) +- Added `jemalloc_pprof` 0.8 and `tokio` 1 as workspace dependencies + +## v1.0.0 +## Breaking Changes +- Removed `auth_token` from Python and Ruby FFI bindings and related code +- Removed `detect_subprocesses` from Python and Ruby configs +- Config constructor now requires app/spy identity and sample rate inputs +- Removed support for collapsed format +- Removed global tags from ruleset + +## New Features +- Integrated pprof-rs backend into main crate behind optional `backend-pprof-rs` feature +- Switched to push API (from `/ingest` to `/push`) +- Generated push API protos +- Added `ThreadId` type +- Added `rustls-no-provider` TLS feature + +## Bug Fixes / Improvements +- Unified signal logic +- Report cleanup functions +- Optimized ruleset +- Removed obscure thread id hash check +- Ruby: inline thread_id crate; remove detect_subprocess +- Dependency updates (reqwest 0.13, prost 0.14, thiserror 2.0, serde_json 1.0.115, uuid 1.20, libflate 2.1) + +## v0.5.4 +## New Features +- Add report transformation function which allows changing the report before + sending it to the Pyroscope Server. +- Add Gzip support. + +## Bug Fixes +- Use URL Builder. ([786c89b](https://github.com/pyroscope-io/pyroscope-rs/commit/786c89bb99839c45778410012a6da60267d395df)) + +## v0.5.3 +## New Features +- Add BackendConfig to make reporting of pid, thread_id and thread_name + optional. +- Backends can add a suffix to the "application_name" + +## Bug Fixes +- **main**: fixed an obsecure bug when counting stacktraces ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/bdecaa13aeae3ce7d4c3d97f88bdd104ec35e7c5)) + +## v0.5.2 +## New features +- Authentication Token support + +## API Changes +- use rust-tls instead of openssl + +## v0.5.1 +## API Changes +- Backend shutdown bug +- Docs update + +## v0.5.0 +## API Changes +- New API for building, starting and stopping the profiling agent. +- Backend supports reporting multiple threads. +- Tagging within local thread-scope + +## v0.4.0 +## API Changes +- Backend now support passing a configuration struct. +- TimerSignal enum +- pprof-rs backend is split into a different package. It has to be imported manually. + +## What's Changed +* fix: avoid binding two unrelated vars to the same type by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/18 +* avoid almost all unwraps by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/14 +* use more features of `thiserror` by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/11 +* introduce LOG_TAGs, avoid repetitive prefixes by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/10 +* allow configurable accumulation_cycle by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/21 +* Add CI Targets by @omarabid in https://github.com/pyroscope-io/pyroscope-rs/pull/22 +* 0.4.0 release by @omarabid in https://github.com/pyroscope-io/pyroscope-rs/pull/23 + +## New Contributors +* @drahnr made their first contribution in https://github.com/pyroscope-io/pyroscope-rs/pull/18 + +**Full Changelog**: https://github.com/pyroscope-io/pyroscope-rs/compare/0.3.1...lib-0.4.0 + +## v0.3.1 +Minor release with bug fixes. + +## Bug Fixes +- **session**: avoid breaking SessionManager if request fails ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/1254bcc9a3b0d76b7adbeb82ba21f33b875c0b39)) +- **typo**: variable name typo ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/0d8caffbe7855bec8158333dba2923cd07286a5f)) +- **pprof**: fix #12 ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/936d3d99a2cc82812bc8251fd2fbf8152a87d4cb)) +- **copyright**: fix #13 ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/b8eaf13137810df932e1b70e33b3ad3e25b65ece)) + +## Code Refactoring +- **option**: replace unwrap for various Options ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/3fd4e794d74523855c66f65c0b7fc8ff9dfe4604)) +- **time**: add get_time_range ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/a6d4dbcef494b2bfe8016a817201499937cf3528)) + +## v0.3.0 +First stable release + +## v0.0.2-alpha +Second beta release + +## v0.0.1-alpha +Initial beta release diff --git a/third-party/pyroscope/Cargo.toml b/third-party/pyroscope/Cargo.toml new file mode 100644 index 000000000..d4deaa8d5 --- /dev/null +++ b/third-party/pyroscope/Cargo.toml @@ -0,0 +1,233 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.66" +name = "pyroscope" +version = "2.1.1" +authors = ["Abid Omar "] +build = "build.rs" +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = """ +Pyroscope Profiler Agent for continuous profiling of Rust, Python and Ruby applications. +""" +homepage = "https://grafana.com/docs/pyroscope/latest/configure-client/language-sdks/rust/" +documentation = "https://docs.rs/pyroscope" +readme = "README.md" +keywords = [ + "pyroscope", + "profiler", + "profiling", + "pprof", +] +license = "Apache-2.0" +repository = "https://github.com/grafana/pyroscope-rs" + +[features] +backend-jemalloc = ["dep:jemalloc_pprof"] +backend-pprof-rs = [ + "dep:backtrace", + "dep:backtrace", + "dep:once_cell", + "dep:nix", + "dep:spin", + "dep:tempfile", + "dep:findshlibs", + "dep:smallvec", + "dep:framehop", + "dep:memmap2", + "dep:object", + "dep:aligned-vec", + "dep:symbolic-demangle", +] +default = ["rustls-tls"] +native-tls = ["reqwest/native-tls"] +native-tls-vendored = ["reqwest/native-tls-vendored"] +rustls-no-provider = ["reqwest/rustls-no-provider"] +rustls-tls = ["reqwest/rustls"] + +[lib] +name = "pyroscope" +path = "src/lib.rs" + +[[example]] +name = "jemalloc" +path = "examples/jemalloc.rs" +required-features = ["backend-jemalloc"] + +[[test]] +name = "agent" +path = "tests/agent.rs" + +[[test]] +name = "pprof_backend" +path = "tests/pprof_backend.rs" + +[[test]] +name = "session" +path = "tests/session.rs" + +[[test]] +name = "timer" +path = "tests/timer.rs" + +[[test]] +name = "timer-epoll" +path = "tests/timer-epoll.rs" + +[dependencies.aligned-vec] +version = "0.6" +optional = true + +[dependencies.backtrace] +version = "0.3" +optional = true + +[dependencies.findshlibs] +version = "0.10" +optional = true + +[dependencies.framehop] +version = "0.16" +optional = true + +[dependencies.jemalloc_pprof] +version = "0.9" +features = ["symbolize"] +optional = true + +[dependencies.libc] +version = "^0.2.124" + +[dependencies.libflate] +version = "2.1.0" + +[dependencies.log] +version = "0.4" + +[dependencies.memmap2] +version = "0.9" +optional = true + +[dependencies.nix] +version = "0.31" +features = [ + "signal", + "fs", +] +optional = true +default-features = false + +[dependencies.object] +version = "0.40" +optional = true + +[dependencies.once_cell] +version = "1.9" +optional = true + +[dependencies.prost] +version = "0.14" + +[dependencies.reqwest] +version = "0.13" +features = [ + "blocking", + "query", +] +default-features = false + +[dependencies.serde_json] +version = "1.0.115" + +[dependencies.smallvec] +version = "1.7" +optional = true + +[dependencies.spin] +version = "0.12" +optional = true + +[dependencies.symbolic-demangle] +version = "13.1.0" +features = ["rust"] +optional = true +default-features = false + +[dependencies.tempfile] +version = "3.1" +optional = true + +[dependencies.thiserror] +version = "2.0" + +[dependencies.url] +version = "2.2.2" + +[dependencies.uuid] +version = "1.20.0" +features = ["v4"] + +[dev-dependencies.assert_matches] +version = "=1.5.0" + +[dev-dependencies.claims] +version = "=0.8.0" + +[dev-dependencies.env_logger] +version = "=0.11.10" + +[dev-dependencies.tikv-jemallocator] +version = "0.7" +features = ["profiling"] + +[profile.bench] +opt-level = 3 +lto = true +codegen-units = 1 +debug = 0 +debug-assertions = false +rpath = false + +[profile.dev] +opt-level = 0 +lto = false +codegen-units = 4 +debug = 2 +debug-assertions = true +rpath = false + +[profile.release] +opt-level = 3 +lto = true +codegen-units = 1 +debug = 0 +debug-assertions = false +rpath = false +strip = "symbols" + +[profile.test] +opt-level = 1 +lto = false +codegen-units = 4 +debug = 2 +debug-assertions = true +rpath = false + +[[test]] +name = "rustfs_zstd" +path = "tests/rustfs_zstd.rs" +required-features = ["backend-pprof-rs"] diff --git a/third-party/pyroscope/Cargo.toml.orig b/third-party/pyroscope/Cargo.toml.orig new file mode 100644 index 000000000..cfeb84fc8 --- /dev/null +++ b/third-party/pyroscope/Cargo.toml.orig @@ -0,0 +1,119 @@ +[package] +name = "pyroscope" +description = """ +Pyroscope Profiler Agent for continuous profiling of Rust, Python and Ruby applications. +""" +keywords = ["pyroscope", "profiler", "profiling", "pprof"] +authors = ["Abid Omar "] +version = "2.1.1" +edition = "2021" +license = "Apache-2.0" +homepage = "https://grafana.com/docs/pyroscope/latest/configure-client/language-sdks/rust/" +documentation = "https://docs.rs/pyroscope" +repository = "https://github.com/grafana/pyroscope-rs" +readme = "README.md" +autobins = false +autoexamples = true +autotests = true +autobenches = true +rust-version = "1.66" + +[dependencies] +thiserror = "2.0" +log = "0.4" +reqwest = { version = "0.13", features = ["blocking", "query"], default-features = false } +uuid = { version = "1.20.0", features = ["v4"] } +url = "2.2.2" +libflate = "2.1.0" +libc = "^0.2.124" +prost = "0.14" +serde_json = "1.0.115" +jemalloc_pprof = { version = "0.9", features = ["symbolize"], optional = true } + + +# pprofrs dependencies +backtrace = { version = "0.3", optional = true } +once_cell = { version = "1.9", optional = true } +nix = { version = "0.31", default-features = false, features = ["signal", "fs"], optional = true } +spin = { version = "0.12", optional = true } +tempfile = { version = "3.1", optional = true } +findshlibs = { version = "0.10", optional = true } +smallvec = { version = "1.7", optional = true } +framehop = { version = "0.16", optional = true } +memmap2 = { version = "0.9", optional = true } +object = { version = "0.40", optional = true } +aligned-vec = { version = "0.6", optional = true } +symbolic-demangle = { version = "13.1.0", default-features = false, features = ["rust"], optional = true } + + + +[dev-dependencies] +assert_matches = "=1.5.0" +claims = "=0.8.0" +tikv-jemallocator = { version = "0.7", features = ["profiling"] } +env_logger = "=0.11.10" + +[[example]] +name = "jemalloc" +required-features = ["backend-jemalloc"] + +[features] +default = ["rustls-tls"] +rustls-tls = ["reqwest/rustls"] +native-tls = ["reqwest/native-tls"] +native-tls-vendored = ["reqwest/native-tls-vendored"] +rustls-no-provider = ["reqwest/rustls-no-provider"] +backend-pprof-rs = [ + "dep:backtrace", + "dep:backtrace", + "dep:once_cell", + "dep:nix", + "dep:spin", + "dep:tempfile", + "dep:findshlibs", + "dep:smallvec", + "dep:framehop", + "dep:memmap2", + "dep:object", + "dep:aligned-vec", + "dep:symbolic-demangle", +] +backend-jemalloc = ["dep:jemalloc_pprof"] + +[profile.dev] +opt-level = 0 +debug = true +rpath = false +lto = false +debug-assertions = true +codegen-units = 4 + +[profile.release] +opt-level = 3 +debug = false +strip = "symbols" +rpath = false +lto = true +debug-assertions = false +codegen-units = 1 + +[profile.test] +opt-level = 1 +debug = true +rpath = false +lto = false +debug-assertions = true +codegen-units = 4 + +[profile.bench] +opt-level = 3 +debug = false +rpath = false +lto = true +debug-assertions = false +codegen-units = 1 + +[[test]] +name = "rustfs_zstd" +path = "tests/rustfs_zstd.rs" +required-features = ["backend-pprof-rs"] diff --git a/third-party/pyroscope/LICENSE b/third-party/pyroscope/LICENSE new file mode 100644 index 000000000..d403710af --- /dev/null +++ b/third-party/pyroscope/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2022 Pyroscope + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third-party/pyroscope/README.md b/third-party/pyroscope/README.md new file mode 100644 index 000000000..6775f68b7 --- /dev/null +++ b/third-party/pyroscope/README.md @@ -0,0 +1,22 @@ +## Pyroscope Profiler + +**Pyroscope Profiler for Rust. Profile your Rust applications.** + +[![license](https://img.shields.io/badge/license-Apache2.0-blue.svg)](LICENSE) +[![Crate](https://img.shields.io/crates/v/pyroscope.svg)](https://crates.io/crates/pyroscope) + + +### Major Contributors + +We'd like to give a big thank you to the following contributors who have made significant contributions to this project: + +* [Abid Omar](https://github.com/omarabid) +* [Anatoly Korniltsev](https://github.com/korniltsev) +* [Bernhard Schuster](https://github.com/drahnr) + + +### License + +Pyroscope is distributed under the Apache License (Version 2.0). + +See [LICENSE](LICENSE) for details. diff --git a/third-party/pyroscope/RUSTFS.md b/third-party/pyroscope/RUSTFS.md new file mode 100644 index 000000000..d65c20cf3 --- /dev/null +++ b/third-party/pyroscope/RUSTFS.md @@ -0,0 +1,9 @@ +# Temporary Pyroscope dependency patch + +This is the published [pyroscope 2.1.1](https://crates.io/crates/pyroscope/2.1.1) source, from upstream commit `bc958d9e9d934ab97bc3c6887ed62c5880467d6b`, with its original license retained. Repository automation is omitted and documentation trailing whitespace is normalized. Copied comments and documentation also have spelling corrections required by the repository typo check. + +The only runtime dependency change is the `object` dependency requirement, from `0.39` to `0.40`, in both manifests. Runtime behavior, features, build script, examples, and original tests are unchanged; source edits beyond the dependency requirement only correct comments. `tests/rustfs_zstd.rs` adds bounded valid and oversized-block regression coverage through `object`'s decompression API, registered in both manifests. `object 0.39` requires vulnerable `ruzstd 0.8`; `object 0.40` permits the patched `ruzstd 0.9.1` required by [RUSTSEC-2026-0344](https://rustsec.org/advisories/RUSTSEC-2026-0344). Keep the patched version in the workspace lockfile and retain the existing Cargo Deny gate. + +Remove this directory, the workspace exclusion, and the `[patch.crates-io]` entry once an upstream Pyroscope release accepts `object 0.40` or later. Track validation and removal in [rustfs/backlog#2825](https://github.com/rustfs/backlog/issues/2825). + +Run the retained upstream unit tests and the RustFS regressions with `cargo test --manifest-path third-party/pyroscope/Cargo.toml --features backend-pprof-rs --lib --test rustfs_zstd`. diff --git a/third-party/pyroscope/build.rs b/third-party/pyroscope/build.rs new file mode 100644 index 000000000..c178ad379 --- /dev/null +++ b/third-party/pyroscope/build.rs @@ -0,0 +1,19 @@ +use std::env; +use std::process::Command; + +fn main() { + let rustc = env::var("RUSTC").unwrap(); + + let output = Command::new(rustc) + .arg("--version") + .output() + .expect("Failed to run rustc"); + + let version_string = + String::from_utf8(output.stdout).expect("rustc --version stdout is not utf8"); + + println!( + "cargo:rustc-env=PYROSCOPE__RUSTC_VERSION={}", + version_string.trim() + ); +} diff --git a/third-party/pyroscope/examples/jemalloc.rs b/third-party/pyroscope/examples/jemalloc.rs new file mode 100644 index 000000000..af10ab894 --- /dev/null +++ b/third-party/pyroscope/examples/jemalloc.rs @@ -0,0 +1,67 @@ +//! Jemalloc memory profiling example. +//! +//! This example demonstrates how to use the jemalloc backend for memory profiling. +//! It requires jemalloc to be configured as the global allocator with profiling enabled. +//! +//! # Requirements +//! +//! Add these dependencies to your Cargo.toml: +//! +//! ```toml +//! [dependencies] +//! pyroscope = { version = "2.0.0", features = ["backend-jemalloc"] } +//! tikv-jemallocator = { version = "0.6", features = ["profiling"] } +//! ``` +//! +//! # Running +//! +//! ```sh +//! # Enable jemalloc profiling via environment variable +//! _RJEM_MALLOC_CONF=prof:true,prof_active:true,lg_prof_sample:19 \ +//! cargo run --example jemalloc --features backend-jemalloc +//! ``` + +use pyroscope::backend::jemalloc::jemalloc_backend; +use pyroscope::pyroscope::PyroscopeAgentBuilder; +use std::time::Duration; + +// Configure jemalloc as the global allocator. +// Profiling must also be enabled at runtime via MALLOC_CONF or _RJEM_MALLOC_CONF. +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +fn main() -> Result<(), Box> { + env_logger::init(); + + let agent = PyroscopeAgentBuilder::new( + "http://localhost:4040", + "example.jemalloc", + 100, + "pyroscope-rs", + env!("CARGO_PKG_VERSION"), + jemalloc_backend(), + ) + .tags(vec![("env", "dev")]) + .build()?; + + let agent_running = agent.start()?; + + // Simulate heavy allocations for 30 seconds. + let start = std::time::Instant::now(); + let mut iteration = 0u64; + while start.elapsed() < Duration::from_secs(30) { + // Allocate vectors of varying sizes (1KB to 1MB). + for i in 0..100 { + let size = 1024 * (1 + (iteration as usize + i) % 1024); + let v: Vec = vec![0u8; size]; + std::hint::black_box(&v); + } + iteration += 1; + } + eprintln!("Completed {iteration} iterations"); + + let agent_ready = agent_running.stop()?; + agent_ready.shutdown(); + + Ok(()) +} diff --git a/third-party/pyroscope/rustfmt.toml b/third-party/pyroscope/rustfmt.toml new file mode 100644 index 000000000..e69de29bb diff --git a/third-party/pyroscope/src/backend/backend.rs b/third-party/pyroscope/src/backend/backend.rs new file mode 100644 index 000000000..4274a9f54 --- /dev/null +++ b/third-party/pyroscope/src/backend/backend.rs @@ -0,0 +1,134 @@ +#![allow(clippy::module_inception)] + +use crate::error::{PyroscopeError, Result}; +use std::{ + fmt::Debug, + sync::{Arc, Mutex}, +}; + +use super::{ReportBatch, ThreadTag}; + +/// Backend Config +#[derive(Debug, Copy, Clone, Default)] +pub struct BackendConfig { + pub report_thread_id: bool, + pub report_thread_name: bool, + pub report_pid: bool, +} + +/// Backend Trait +pub trait Backend: Send { + /// Initialize the backend. + fn initialize(&mut self) -> Result<()>; + /// Drop the backend. + fn shutdown(self: Box) -> Result<()>; + /// Generate profiling report + fn report(&mut self) -> Result; + fn add_tag(&self, tag: ThreadTag) -> Result<()>; + fn remove_tag(&self, tag: ThreadTag) -> Result<()>; +} + +/// Marker struct for Empty BackendImpl +#[derive(Debug)] +pub struct BackendBare; + +/// Marker struct for Uninitialized Backend +#[derive(Debug)] +pub struct BackendUninitialized; + +/// Marker struct for Initialized Backend +#[derive(Debug)] +pub struct BackendReady; + +/// Backend State Trait +pub trait BackendState {} +impl BackendState for BackendBare {} +impl BackendState for BackendUninitialized {} +impl BackendState for BackendReady {} + +/// Backend Accessibility Trait +pub trait BackendAccessible: BackendState {} +impl BackendAccessible for BackendUninitialized {} +impl BackendAccessible for BackendReady {} + +/// Precursor Backend Implementation +/// This struct is used to implement the Backend trait. It serves two purposes: +/// 1. It enforces state transitions using the Type System. +/// 2. It manages the lifetime of the backend through an Arc>. +pub struct BackendImpl { + /// Backend + pub backend: Arc>>>, + + /// Backend State + _state: std::marker::PhantomData, +} + +impl BackendImpl { + /// Create a new BackendImpl instance + pub fn new(backend_box: Box) -> BackendImpl { + BackendImpl { + backend: Arc::new(Mutex::new(Some(backend_box))), + _state: std::marker::PhantomData, + } + } +} + +impl BackendImpl { + /// Initialize the backend + pub fn initialize(self) -> Result> { + let backend = self.backend.clone(); + + // Initialize the backend + backend + .lock()? + .as_mut() + .ok_or(PyroscopeError::BackendImpl)? + .initialize()?; + + // Transition to BackendReady + Ok(BackendImpl { + backend, + _state: std::marker::PhantomData, + }) + } +} + +impl BackendImpl { + pub fn add_tag(&self, tag: ThreadTag) -> Result<()> { + self.backend + .lock()? + .as_ref() + .ok_or(PyroscopeError::BackendImpl)? + .add_tag(tag) + } + + pub fn remove_tag(&self, rule: ThreadTag) -> Result<()> { + self.backend + .lock()? + .as_ref() + .ok_or(PyroscopeError::BackendImpl)? + .remove_tag(rule) + } +} + +impl BackendImpl { + /// Shutdown the backend and destroy BackendImpl + pub fn shutdown(self) -> Result<()> { + self.backend + .lock()? + .take() + .ok_or(PyroscopeError::BackendImpl)? + .shutdown()?; + + Ok(()) + } + + /// Generate profiling report + pub fn report(&mut self) -> Result { + self.backend + .lock()? + .as_mut() + .ok_or(PyroscopeError::BackendImpl)? + .report() + } +} diff --git a/third-party/pyroscope/src/backend/jemalloc.rs b/third-party/pyroscope/src/backend/jemalloc.rs new file mode 100644 index 000000000..b4b3de042 --- /dev/null +++ b/third-party/pyroscope/src/backend/jemalloc.rs @@ -0,0 +1,87 @@ +use crate::backend::{ + Backend, BackendImpl, BackendUninitialized, ReportBatch, ReportData, ThreadTag, +}; +use crate::error::{PyroscopeError, Result}; + +const LOG_TAG: &str = "Pyroscope::Jemalloc"; + +/// Create a jemalloc memory profiling backend. +/// +/// # Example +/// +/// ```no_run +/// use pyroscope::pyroscope::PyroscopeAgentBuilder; +/// use pyroscope::backend::jemalloc::jemalloc_backend; +/// +/// # fn main() -> Result<(), Box> { +/// let agent = PyroscopeAgentBuilder::new( +/// "http://localhost:4040", "my-app", 100, +/// "pyroscope-rs", env!("CARGO_PKG_VERSION"), +/// jemalloc_backend(), +/// ) +/// .build()?; +/// agent.start()?; +/// # Ok(()) +/// # } +/// ``` +pub fn jemalloc_backend() -> BackendImpl { + BackendImpl::new(Box::new(Jemalloc)) +} + +#[derive(Debug)] +struct Jemalloc; + +impl Backend for Jemalloc { + fn initialize(&mut self) -> Result<()> { + let prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().ok_or_else(|| { + PyroscopeError::new( + "jemalloc: PROF_CTL not available. Ensure jemalloc is configured with prof:true", + ) + })?; + let guard = prof_ctl.try_lock().map_err(|_| { + PyroscopeError::new( + "jemalloc: failed to acquire PROF_CTL lock during initialization. \ + This is unexpected at startup; ensure no other code holds the lock.", + ) + })?; + if !guard.activated() { + return Err(PyroscopeError::new( + "jemalloc: profiling is not activated. Ensure malloc_conf includes prof:true,prof_active:true", + )); + } + + log::info!(target: LOG_TAG, "Jemalloc profiling backend initialized"); + + Ok(()) + } + + fn shutdown(self: Box) -> Result<()> { + log::trace!(target: LOG_TAG, "Shutting down jemalloc backend"); + Ok(()) + } + + fn report(&mut self) -> Result { + let prof_ctl = jemalloc_pprof::PROF_CTL + .as_ref() + .ok_or_else(|| PyroscopeError::new("jemalloc: PROF_CTL not available"))?; + let mut guard = prof_ctl.try_lock().map_err(|_| { + PyroscopeError::new("jemalloc: failed to acquire PROF_CTL lock for report") + })?; + let pprof_data = guard + .dump_pprof() + .map_err(|e| PyroscopeError::new(&format!("jemalloc: dump_pprof failed: {e}")))?; + + Ok(ReportBatch { + profile_type: "memory".into(), + data: ReportData::RawPprof(pprof_data), + }) + } + + fn add_tag(&self, _tag: ThreadTag) -> Result<()> { + Ok(()) + } + + fn remove_tag(&self, _tag: ThreadTag) -> Result<()> { + Ok(()) + } +} diff --git a/third-party/pyroscope/src/backend/mod.rs b/third-party/pyroscope/src/backend/mod.rs new file mode 100644 index 000000000..8dae70c9b --- /dev/null +++ b/third-party/pyroscope/src/backend/mod.rs @@ -0,0 +1,71 @@ +pub mod backend; +#[cfg(feature = "backend-jemalloc")] +pub mod jemalloc; +#[cfg(all( + feature = "backend-pprof-rs", + any( + all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") + ), + all( + target_os = "macos", + any(target_arch = "x86_64", target_arch = "aarch64") + ) + ) +))] +pub mod pprof; +#[cfg(all( + feature = "backend-pprof-rs", + any( + all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") + ), + all( + target_os = "macos", + any(target_arch = "x86_64", target_arch = "aarch64") + ) + ) +))] +mod pprofrs; + +#[cfg(all( + feature = "backend-pprof-rs", + any( + all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") + ), + all( + target_os = "macos", + any(target_arch = "x86_64", target_arch = "aarch64") + ) + ) +))] +pub use pprof::*; + +#[cfg(all( + feature = "backend-pprof-rs", + not(any( + all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") + ), + all( + target_os = "macos", + any(target_arch = "x86_64", target_arch = "aarch64") + ) + )) +))] +compile_error!("feature `backend-pprof-rs` is only supported on Linux/macOS x86_64/aarch64"); + +pub mod ruleset; +pub mod tests; +pub mod types; + +pub use backend::*; +#[cfg(feature = "backend-jemalloc")] +pub use jemalloc::*; +pub use ruleset::*; +pub use types::*; diff --git a/third-party/pyroscope/src/backend/pprof.rs b/third-party/pyroscope/src/backend/pprof.rs new file mode 100644 index 000000000..12d85c958 --- /dev/null +++ b/third-party/pyroscope/src/backend/pprof.rs @@ -0,0 +1,240 @@ +use crate::backend::pprofrs::{ProfilerGuard, ProfilerGuardBuilder}; +use crate::backend::{ + Backend, BackendConfig, BackendImpl, BackendUninitialized, Report, ReportBatch, ReportData, + StackBuffer, StackFrame, StackTrace, ThreadTag, ThreadTagsSet, +}; +use crate::error::{PyroscopeError, Result}; +use std::{ + collections::HashMap, + ffi::OsStr, + ops::Deref, + sync::{Arc, Mutex}, +}; + +const LOG_TAG: &str = "Pyroscope::Pprofrs"; + +pub fn pprof_backend( + config: PprofConfig, + backend_config: BackendConfig, +) -> BackendImpl { + BackendImpl::new(Box::new(Pprof::new(config, backend_config))) +} + +#[derive(Debug)] +pub struct PprofConfig { + pub sample_rate: u32, +} + +impl Default for PprofConfig { + fn default() -> Self { + PprofConfig { sample_rate: 100 } + } +} + +#[derive(Default)] +pub struct Pprof<'a> { + buffer: Arc>, + config: PprofConfig, + backend_config: BackendConfig, + guard: Arc>>>, + ruleset: ThreadTagsSet, +} + +impl std::fmt::Debug for Pprof<'_> { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { + write!(fmt, "Pprof Backend") + } +} + +impl<'a> Pprof<'a> { + pub fn new(config: PprofConfig, backend_config: BackendConfig) -> Self { + Pprof { + buffer: Arc::new(Mutex::new(StackBuffer::default())), + config, + backend_config, + guard: Arc::new(Mutex::new(None)), + ruleset: ThreadTagsSet::default(), + } + } +} + +impl Backend for Pprof<'_> { + fn shutdown(self: Box) -> Result<()> { + log::trace!(target: LOG_TAG, "Shutting down sampler thread"); + Ok(()) + } + + fn initialize(&mut self) -> Result<()> { + let profiler = ProfilerGuardBuilder::default() + .frequency(self.config.sample_rate as i32) + .build() + .map_err(|e| PyroscopeError::new(e.to_string().as_str()))?; + + *self.guard.lock()? = Some(profiler); + + Ok(()) + } + + fn report(&mut self) -> Result { + self.dump_report()?; + + let buffer = self.buffer.clone(); + + let report: StackBuffer = buffer.lock()?.deref().to_owned(); + + let reports: Vec = report.into(); + + buffer.lock()?.clear(); + + Ok(ReportBatch { + profile_type: "process_cpu".into(), + data: ReportData::Reports(reports), + }) + } + + fn add_tag(&self, tag: ThreadTag) -> Result<()> { + if self.guard.lock()?.as_ref().is_some() { + self.dump_report()?; + } + + self.ruleset.add(tag)?; + + Ok(()) + } + + fn remove_tag(&self, tag: ThreadTag) -> Result<()> { + if self.guard.lock()?.as_ref().is_some() { + self.dump_report()?; + } + + self.ruleset.remove(tag)?; + + Ok(()) + } +} + +impl Pprof<'_> { + /// Workaround for pprof-rs to interrupt the profiler. + pub fn dump_report(&self) -> Result<()> { + let report = self + .guard + .lock()? + .as_ref() + .ok_or_else(|| PyroscopeError::new("pprof-rs: ProfilerGuard report error"))? + .report() + .build_and_clear(true) + .map_err(|e| PyroscopeError::new(e.to_string().as_str()))?; + + let stack_buffer = Into::::into(Into::::into(( + report, + &self.backend_config, + ))); + + let data: HashMap = stack_buffer + .data + .iter() + .map(|(stacktrace, ss)| { + let stacktrace = stacktrace.to_owned().add_tag_rules(&self.ruleset); + (stacktrace, ss.to_owned()) + }) + .collect(); + + let buffer = self.buffer.clone(); + + for (stacktrace, count) in data { + buffer.lock()?.record_with_count(stacktrace, count)?; + } + + Ok(()) + } +} + +struct StackBufferWrapper(StackBuffer); + +impl From for StackBuffer { + fn from(stackbuffer: StackBufferWrapper) -> Self { + stackbuffer.0 + } +} + +impl From<(crate::backend::pprofrs::Report, &BackendConfig)> for StackBufferWrapper { + fn from(arg: (crate::backend::pprofrs::Report, &BackendConfig)) -> Self { + let (report, config) = arg; + let buffer_data: HashMap = report + .data + .iter() + .map(|(key, value)| { + ( + Into::::into((key.to_owned(), config)).into(), + value.to_owned() as usize, + ) + }) + .collect(); + StackBufferWrapper(StackBuffer::new(buffer_data)) + } +} + +struct StackTraceWrapper(StackTrace); + +impl From for StackTrace { + fn from(stack_trace: StackTraceWrapper) -> Self { + stack_trace.0 + } +} + +impl From<(crate::backend::pprofrs::Frames, &BackendConfig)> for StackTraceWrapper { + fn from(arg: (crate::backend::pprofrs::Frames, &BackendConfig)) -> Self { + let (frames, config) = arg; + let thread_id = frames.thread_id as libc::pthread_t; + StackTraceWrapper(StackTrace::new( + config, + None, + Some(thread_id.into()), + Some(frames.thread_name), + frames + .frames + .concat() + .iter() + .map(|frame| Into::::into(frame.to_owned()).into()) + .collect(), + )) + } +} + +struct StackFrameWrapper(StackFrame); + +impl From for StackFrame { + fn from(stack_frame: StackFrameWrapper) -> Self { + stack_frame.0 + } +} + +impl From for StackFrameWrapper { + fn from(symbol: crate::backend::pprofrs::Symbol) -> Self { + StackFrameWrapper(StackFrame::new( + None, + Some(symbol.name()), + Some( + symbol + .filename + .clone() + .unwrap_or_default() + .file_name() + .unwrap_or_else(|| OsStr::new("")) + .to_str() + .unwrap_or("") + .to_string(), + ), + Some( + symbol + .filename + .unwrap_or_default() + .to_str() + .unwrap_or("") + .to_string(), + ), + None, + symbol.lineno, + )) + } +} diff --git a/third-party/pyroscope/src/backend/pprofrs/addr_validate.rs b/third-party/pyroscope/src/backend/pprofrs/addr_validate.rs new file mode 100644 index 000000000..c7dd803be --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/addr_validate.rs @@ -0,0 +1,140 @@ +use nix::{errno::Errno, unistd::read}; +use std::mem::size_of; +use std::os::fd::{AsRawFd, OwnedFd}; + +#[derive(Default)] +pub(crate) struct Pipes { + read_fd: Option, + write_fd: Option, +} +#[inline] +#[cfg(any(target_os = "android", target_os = "linux"))] +fn create_pipe() -> nix::Result<(OwnedFd, OwnedFd)> { + use nix::fcntl::OFlag; + use nix::unistd::pipe2; + + pipe2(OFlag::O_CLOEXEC | OFlag::O_NONBLOCK) +} + +#[inline] +#[cfg(any(target_os = "macos", target_os = "freebsd"))] +fn create_pipe() -> nix::Result<(OwnedFd, OwnedFd)> { + use nix::fcntl::{fcntl, FcntlArg, FdFlag, OFlag}; + use nix::unistd::pipe; + + fn set_flags(fd: &OwnedFd) -> nix::Result<()> { + let mut flags = FdFlag::from_bits(fcntl(fd, FcntlArg::F_GETFD)?).unwrap(); + flags |= FdFlag::FD_CLOEXEC; + fcntl(fd, FcntlArg::F_SETFD(flags))?; + let mut flags = OFlag::from_bits(fcntl(fd, FcntlArg::F_GETFL)?).unwrap(); + flags |= OFlag::O_NONBLOCK; + fcntl(fd, FcntlArg::F_SETFL(flags))?; + Ok(()) + } + + let (read_fd, write_fd) = pipe()?; + set_flags(&read_fd)?; + set_flags(&write_fd)?; + Ok((read_fd, write_fd)) +} + +fn open_pipe(pipes: &mut Pipes) -> nix::Result<()> { + pipes.read_fd = None; + pipes.write_fd = None; + + let (read_fd, write_fd) = create_pipe()?; + + pipes.read_fd = Some(read_fd); + pipes.write_fd = Some(write_fd); + + Ok(()) +} + +// validate whether the address `addr` is readable through `write()` to a pipe +// +// if the second argument of `write(fd, buf, count)` is not a valid address, the +// `write()` will return an error the error number should be `EFAULT` in most +// cases, but we regard all errors (except EINTR) as a failure of validation +pub fn validate(pipes: &mut Pipes, addr: *const libc::c_void) -> bool { + // Short-circuit null pointers without asking the kernel to validate them. + if addr.is_null() { + return false; + } + + const CHECK_LENGTH: usize = 2 * size_of::<*const libc::c_void>() / size_of::(); + + // read data in the pipe + let valid_read = loop { + match &pipes.read_fd { + None => break false, + Some(read_fd) => { + let mut buf = [0u8; CHECK_LENGTH]; + + match read(read_fd, &mut buf) { + Ok(bytes) => break bytes > 0, + Err(_err @ Errno::EINTR) => continue, + Err(_err @ Errno::EAGAIN) => break true, + Err(_) => break false, + } + } + } + }; + + if !valid_read && open_pipe(pipes).is_err() { + return false; + } + + let Some(write_fd) = &pipes.write_fd else { + return false; // impossible + }; + loop { + let res = unsafe { libc::write(write_fd.as_raw_fd(), addr, CHECK_LENGTH) }; + let res = if res == -1 { + Err(Errno::last()) + } else { + Ok(res) + }; + match res { + Ok(bytes) => break bytes > 0, + Err(_err @ Errno::EINTR) => continue, + Err(_) => break false, + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use claims::assert_some; + + #[test] + fn validate_stack() { + let mut p = Pipes::default(); + let i = 0; + + assert!(validate(&mut p, &i as *const _ as *const libc::c_void)); + assert_some!(p.read_fd); + assert_some!(p.write_fd); + } + + #[test] + fn validate_heap() { + let mut p = Pipes::default(); + let vec = vec![0; 1000]; + + for i in vec.iter() { + assert!(validate(&mut p, i as *const _ as *const libc::c_void)); + } + assert_some!(p.read_fd); + assert_some!(p.write_fd); + } + + #[test] + fn failed_validate() { + let mut p = Pipes::default(); + assert!(!validate(&mut p, std::ptr::null::())); + assert!(!validate(&mut p, -1_i32 as usize as *const libc::c_void)); + assert_some!(p.read_fd); + assert_some!(p.write_fd); + } +} diff --git a/third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder.rs b/third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder.rs new file mode 100644 index 000000000..f9797a75b --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder.rs @@ -0,0 +1,189 @@ +use crate::backend::pprofrs::addr_validate::Pipes; +use framehop::{ + CacheNative, MustNotAllocateDuringUnwind, UnwindRegsNative, Unwinder, UnwinderNative, +}; +use libc::{c_void, ucontext_t}; +use once_cell::sync::Lazy; +use spin::RwLock; + +mod shlib; + +#[cfg(all(target_arch = "aarch64", target_os = "macos"))] +fn get_regs_from_context(ucontext: *mut c_void) -> Option<(UnwindRegsNative, u64)> { + let ucontext: *mut ucontext_t = ucontext as *mut ucontext_t; + if ucontext.is_null() { + return None; + } + + let thread_state = unsafe { + let mcontext = (*ucontext).uc_mcontext; + if mcontext.is_null() { + return None; + } else { + (*mcontext).__ss + } + }; + + Some(( + UnwindRegsNative::new(thread_state.__lr, thread_state.__sp, thread_state.__fp), + thread_state.__pc, + )) +} + +#[cfg(all(target_arch = "x86_64", target_os = "macos"))] +fn get_regs_from_context(ucontext: *mut c_void) -> Option<(UnwindRegsNative, u64)> { + let ucontext: *mut ucontext_t = ucontext as *mut ucontext_t; + if ucontext.is_null() { + return None; + } + + let thread_state = unsafe { + let mcontext = (*ucontext).uc_mcontext; + if mcontext.is_null() { + return None; + } else { + (*mcontext).__ss + } + }; + + Some(( + UnwindRegsNative::new(thread_state.__rip, thread_state.__rsp, thread_state.__rbp), + thread_state.__rip, + )) +} + +#[cfg(all(target_arch = "aarch64", target_os = "linux"))] +fn get_regs_from_context(ucontext: *mut c_void) -> Option<(UnwindRegsNative, u64)> { + let ucontext: *mut ucontext_t = ucontext as *mut ucontext_t; + if ucontext.is_null() { + return None; + } + + let regs = unsafe { &(*ucontext).uc_mcontext.regs }; + let sp = unsafe { (*ucontext).uc_mcontext.sp }; + Some((UnwindRegsNative::new(regs[30], sp, regs[29]), regs[30])) +} + +#[cfg(all(target_arch = "x86_64", target_os = "linux"))] +fn get_regs_from_context(ucontext: *mut c_void) -> Option<(UnwindRegsNative, u64)> { + let ucontext: *mut ucontext_t = ucontext as *mut ucontext_t; + if ucontext.is_null() { + return None; + } + let regs = unsafe { &(*ucontext).uc_mcontext.gregs }; + + Some(( + UnwindRegsNative::new( + regs[libc::REG_RIP as usize] as u64, + regs[libc::REG_RSP as usize] as u64, + regs[libc::REG_RBP as usize] as u64, + ), + regs[libc::REG_RIP as usize] as u64, + )) +} + +struct FramehopUnwinder { + unwinder: UnwinderNative, MustNotAllocateDuringUnwind>, + cache: CacheNative, + pipes: Pipes, +} + +impl FramehopUnwinder { + pub fn new() -> Self { + let mut unwinder = UnwinderNative::new(); + for obj in shlib::get_objects() { + unwinder.add_module(obj.clone()); + } + let cache = CacheNative::default(); + let pipes = Pipes::default(); + FramehopUnwinder { + unwinder, + cache, + pipes, + } + } + + pub fn iter_frames bool>(&mut self, ctx: *mut c_void, mut cb: F) { + let (regs, pc) = match get_regs_from_context(ctx) { + Some(fp) => fp, + None => return, + }; + + let mut closure = |addr| read_stack(&mut self.pipes, addr); + let mut iter = self + .unwinder + .iter_frames(pc, regs, &mut self.cache, &mut closure); + while let Ok(Some(frame)) = iter.next() { + if !cb(&Frame { + ip: frame.address() as usize, + }) { + break; + } + } + } +} + +fn read_stack(pipes: &mut Pipes, addr: u64) -> Result { + let aligned_addr = addr & !0b111; + if crate::backend::pprofrs::addr_validate::validate(pipes, aligned_addr as _) { + Ok(unsafe { (aligned_addr as *const u64).read() }) + } else { + Err(()) + } +} + +static UNWINDER: Lazy> = + Lazy::new(|| RwLock::new(FramehopUnwinder::new())); +#[derive(Clone, Debug)] +pub struct Frame { + pub ip: usize, +} + +extern "C" { + fn _Unwind_FindEnclosingFunction(pc: *mut c_void) -> *mut c_void; + +} + +impl super::Frame for Frame { + type S = backtrace::Symbol; + fn ip(&self) -> usize { + self.ip + } + + fn symbol_address(&self) -> *mut c_void { + if cfg!(target_os = "macos") || cfg!(target_os = "ios") { + self.ip as *mut c_void + } else { + unsafe { _Unwind_FindEnclosingFunction(self.ip as *mut c_void) } + } + } + + fn resolve_symbol(&self, cb: F) { + backtrace::resolve(self.ip as *mut c_void, cb); + } +} + +pub struct Trace; + +impl super::Trace for Trace { + type Frame = Frame; + + fn init() { + let _ = UNWINDER.read(); + } + + fn trace bool>(ctx: *mut c_void, cb: F) + where + Self: Sized, + { + // For Linux, this `try_write` should always succeed, because `SIGPROF` will never be delivered to + // another thread while the signal handler is running. However, I'm not sure about other OSes, so + // we use `try_write` to be safe instead of using `static mut` and `unsafe` directly. + match UNWINDER.try_write() { + None => (), + Some(mut unwinder) => { + unwinder.iter_frames(ctx, cb); + } + } + } +} diff --git a/third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder/shlib.rs b/third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder/shlib.rs new file mode 100644 index 000000000..0ad59089e --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder/shlib.rs @@ -0,0 +1,78 @@ +use std::path::PathBuf; + +use findshlibs::{SharedLibrary, TargetSharedLibrary}; +use framehop::{Module, ModuleSectionInfo}; +use memmap2::Mmap; +use object::{Object, ObjectSection}; +use once_cell::sync::Lazy; + +static OBJECTS: Lazy>>> = Lazy::new(find_objects); + +pub fn get_objects() -> &'static [Module>] { + &OBJECTS +} + +pub struct ObjectInfo<'f>(object::File<'f>); + +impl<'file, D> ModuleSectionInfo for ObjectInfo<'file> +where + D: From<&'file [u8]>, +{ + fn base_svma(&self) -> u64 { + if let Some(section) = self.0.section_by_name("__TEXT") { + // in mach-o addresses are relative to __TEXT + section.address() + } else { + self.0.relative_address_base() + } + } + + fn section_svma_range(&mut self, name: &[u8]) -> Option> { + if let Some(section) = self.0.section_by_name_bytes(name) { + let start = section.address(); + let end = start + section.size(); + Some(start..end) + } else { + None + } + } + + fn section_data(&mut self, name: &[u8]) -> Option { + if let Some(section) = self.0.section_by_name_bytes(name) { + let data = section.data().ok()?; + Some(D::from(data)) + } else { + None + } + } +} + +fn open_mmap(path: &PathBuf) -> Option { + let file = std::fs::File::open(path).ok()?; + let mmap = unsafe { Mmap::map(&file) }.ok()?; + Some(mmap) +} + +fn find_objects() -> Vec>> { + let mut objects = Vec::new(); + // objects + TargetSharedLibrary::each(|shlib| { + let path = PathBuf::from(shlib.name()); + let base_avma = shlib.actual_load_addr().0 as u64; + let avma_range = base_avma..base_avma + shlib.len() as u64; + if let Some(mmap) = open_mmap(&path) { + if let Ok(obj) = object::File::parse(&*mmap) { + let section_info = ObjectInfo(obj); + + objects.push(Module::new( + path.to_string_lossy().to_string(), + avma_range, + base_avma, + section_info, + )); + } + } + }); + + objects +} diff --git a/third-party/pyroscope/src/backend/pprofrs/backtrace/mod.rs b/third-party/pyroscope/src/backend/pprofrs/backtrace/mod.rs new file mode 100644 index 000000000..5d5843359 --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/backtrace/mod.rs @@ -0,0 +1,51 @@ +// Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. + +use libc::c_void; +use std::path::PathBuf; + +pub trait Symbol: Sized { + fn name(&self) -> Option>; + fn lineno(&self) -> Option; + fn filename(&self) -> Option; +} + +impl Symbol for backtrace::Symbol { + fn name(&self) -> Option> { + self.name().map(|name| name.as_bytes().to_vec()) + } + + fn lineno(&self) -> Option { + self.lineno() + } + + fn filename(&self) -> Option { + self.filename().map(|filename| filename.to_owned()) + } +} + +pub trait Frame: Sized + Clone { + type S: Symbol; + + fn resolve_symbol(&self, cb: F); + + #[allow(dead_code)] + fn symbol_address(&self) -> *mut c_void; + + #[allow(dead_code)] + fn ip(&self) -> usize; +} + +pub trait Trace { + type Frame; + + // init will be called before running the first trace in signal handler + fn init() {} + + fn trace bool>(_: *mut libc::c_void, cb: F) + where + Self: Sized; +} + +pub mod framehop_unwinder; + +pub use framehop_unwinder::Trace as TraceImpl; diff --git a/third-party/pyroscope/src/backend/pprofrs/collector.rs b/third-party/pyroscope/src/backend/pprofrs/collector.rs new file mode 100644 index 000000000..57cfd9db2 --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/collector.rs @@ -0,0 +1,558 @@ +// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. + +use std::collections::hash_map::DefaultHasher; +use std::convert::TryInto; +use std::fmt::Debug; +use std::hash::{Hash, Hasher}; +use std::io::{Read, Seek, SeekFrom, Write}; + +use crate::backend::pprofrs::frames::UnresolvedFrames; + +use aligned_vec::AVec; +use tempfile::NamedTempFile; + +pub const BUCKETS: usize = 1 << 12; +pub const BUCKETS_ASSOCIATIVITY: usize = 4; +pub const BUFFER_LENGTH: usize = (1 << 18) / std::mem::size_of::>(); + +#[derive(Debug)] +pub struct Entry { + pub item: T, + pub count: isize, +} + +impl Default for Entry { + fn default() -> Self { + Entry { + item: Default::default(), + count: 0, + } + } +} + +#[derive(Debug)] +pub struct Bucket { + pub length: usize, + entries: Box<[Entry; BUCKETS_ASSOCIATIVITY]>, +} + +impl Default for Bucket { + fn default() -> Bucket { + let entries = Box::default(); + + Self { length: 0, entries } + } +} + +impl Bucket { + pub fn clear(&mut self) { + self.length = 0; + } + + pub fn add(&mut self, key: T, count: isize) -> Option> { + let mut done = false; + self.entries[0..self.length].iter_mut().for_each(|ele| { + if ele.item == key { + ele.count += count; + done = true; + } + }); + + if done { + None + } else if self.length < BUCKETS_ASSOCIATIVITY { + let ele = &mut self.entries[self.length]; + ele.item = key; + ele.count = count; + + self.length += 1; + None + } else { + let mut min_index = 0; + let mut min_count = self.entries[0].count; + for index in 0..self.length { + let count = self.entries[index].count; + if count < min_count { + min_index = index; + min_count = count; + } + } + + let mut new_entry = Entry { item: key, count }; + std::mem::swap(&mut self.entries[min_index], &mut new_entry); + Some(new_entry) + } + } + + pub fn iter(&self) -> BucketIterator<'_, T> { + BucketIterator:: { + related_bucket: self, + index: 0, + } + } +} + +pub struct BucketIterator<'a, T: 'static> { + related_bucket: &'a Bucket, + index: usize, +} + +impl<'a, T> Iterator for BucketIterator<'a, T> { + type Item = &'a Entry; + + fn next(&mut self) -> Option { + if self.index < self.related_bucket.length { + self.index += 1; + Some(&self.related_bucket.entries[self.index - 1]) + } else { + None + } + } +} + +pub struct HashCounter { + buckets: Box<[Bucket; BUCKETS]>, +} + +impl Default for HashCounter { + fn default() -> Self { + let mut v: Vec> = Vec::with_capacity(BUCKETS); + v.resize_with(BUCKETS, Default::default); + let buckets = v.into_boxed_slice().try_into().unwrap(); + + Self { buckets } + } +} + +impl HashCounter { + pub fn clear(&mut self) { + for bucket in self.buckets.iter_mut() { + bucket.clear(); + } + } + + fn hash(key: &T) -> u64 { + let mut s = DefaultHasher::new(); + key.hash(&mut s); + s.finish() + } + + pub fn add(&mut self, key: T, count: isize) -> Option> { + let hash_value = Self::hash(&key); + let bucket = &mut self.buckets[(hash_value % BUCKETS as u64) as usize]; + + bucket.add(key, count) + } + + pub fn iter(&self) -> impl Iterator> { + self.buckets.iter().flat_map(|bucket| bucket.iter()) + } +} + +pub struct TempFdArray { + file: NamedTempFile, + buffer: Box<[T; BUFFER_LENGTH]>, + buffer_index: usize, + flush_n: usize, +} + +impl TempFdArray { + fn new() -> std::io::Result> { + let file = NamedTempFile::new()?; + + let mut v: Vec = Vec::with_capacity(BUFFER_LENGTH); + v.resize_with(BUFFER_LENGTH, Default::default); + let buffer = v.into_boxed_slice().try_into().unwrap(); + + Ok(Self { + file, + buffer, + buffer_index: 0, + flush_n: 0, + }) + } +} + +impl TempFdArray { + fn clear(&mut self) -> std::io::Result<()> { + self.buffer_index = 0; + self.flush_n = 0; + self.file.as_file().set_len(0)?; + self.file.seek(SeekFrom::Start(0))?; + Ok(()) + } + + fn flush_buffer(&mut self) -> std::io::Result<()> { + self.buffer_index = 0; + let buf = unsafe { + std::slice::from_raw_parts( + self.buffer.as_ptr() as *const u8, + BUFFER_LENGTH * std::mem::size_of::(), + ) + }; + self.flush_n += 1; + self.file.write_all(buf)?; + + Ok(()) + } + + fn push(&mut self, entry: T) -> std::io::Result<()> { + if self.buffer_index >= BUFFER_LENGTH { + self.flush_buffer()?; + } + + self.buffer[self.buffer_index] = entry; + self.buffer_index += 1; + + Ok(()) + } + + fn try_iter(&self) -> std::io::Result> { + let size = BUFFER_LENGTH * self.flush_n * std::mem::size_of::(); + + let mut file_vec = AVec::with_capacity(std::mem::align_of::(), size); + let mut file = self.file.reopen()?; + + unsafe { + // it's safe as the capacity is initialized to `size`, and it'll be filled with `size` bytes + file_vec.set_len(size); + } + file.read_exact(&mut file_vec[0..size])?; + file.seek(SeekFrom::End(0))?; + + Ok(TempFdArrayIterator { + buffer: &self.buffer[0..self.buffer_index], + file_vec, + index: 0, + }) + } +} + +pub struct TempFdArrayIterator<'a, T> { + pub buffer: &'a [T], + pub file_vec: AVec, + pub index: usize, +} + +impl<'a, T> Iterator for TempFdArrayIterator<'a, T> { + type Item = &'a T; + + fn next(&mut self) -> Option { + if self.index < self.buffer.len() { + self.index += 1; + Some(&self.buffer[self.index - 1]) + } else { + let length = self.file_vec.len() / std::mem::size_of::(); + let ts = + unsafe { std::slice::from_raw_parts(self.file_vec.as_ptr() as *const T, length) }; + if self.index - self.buffer.len() < ts.len() { + self.index += 1; + Some(&ts[self.index - self.buffer.len() - 1]) + } else { + None + } + } + } +} + +pub struct Collector { + map: HashCounter, + temp_array: TempFdArray>, +} + +impl Collector { + pub fn new() -> std::io::Result { + Ok(Self { + map: HashCounter::::default(), + temp_array: TempFdArray::>::new()?, + }) + } +} + +impl Collector { + pub fn clear(&mut self) -> std::io::Result<()> { + self.map.clear(); + self.temp_array.clear()?; + Ok(()) + } + + pub fn add(&mut self, key: T, count: isize) -> std::io::Result<()> { + if let Some(evict) = self.map.add(key, count) { + self.temp_array.push(evict)?; + } + + Ok(()) + } + + pub fn try_iter(&self) -> std::io::Result>> { + Ok(self.map.iter().chain(self.temp_array.try_iter()?)) + } + + #[cfg(test)] + pub fn flushed_to_disk(&self) -> usize { + self.temp_array.flush_n + } +} + +#[cfg(test)] +mod test_utils { + use super::*; + use std::collections::BTreeMap; + + pub fn add_map(hashmap: &mut BTreeMap, entry: &Entry) { + match hashmap.get_mut(&entry.item) { + None => { + hashmap.insert(entry.item, entry.count); + } + Some(count) => *count += entry.count, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeMap; + + #[test] + fn stack_hash_counter() { + let mut stack_hash_counter = HashCounter::::default(); + stack_hash_counter.add(0, 1); + stack_hash_counter.add(1, 1); + stack_hash_counter.add(1, 1); + + stack_hash_counter.iter().for_each(|item| { + if item.item == 0 { + assert_eq!(item.count, 1); + } else if item.item == 1 { + assert_eq!(item.count, 2); + } else { + unreachable!(); + } + }); + } + + #[test] + fn evict_test() { + let mut stack_hash_counter = HashCounter::::default(); + let mut real_map = BTreeMap::new(); + + for item in 0..(1 << 10) * 4 { + for _ in 0..(item % 4) { + match stack_hash_counter.add(item, 1) { + None => {} + Some(evict) => { + test_utils::add_map(&mut real_map, &evict); + } + } + } + } + + stack_hash_counter.iter().for_each(|entry| { + test_utils::add_map(&mut real_map, entry); + }); + + for item in 0..(1 << 10) * 4 { + let count = (item % 4) as isize; + match real_map.get(&item) { + Some(item) => { + assert_eq!(*item, count); + } + None => { + assert_eq!(count, 0); + } + } + } + } + + #[test] + fn collector_test() { + let mut collector = Collector::new().unwrap(); + let mut real_map = BTreeMap::new(); + + for item in 0..(1 << 12) * 4 { + for _ in 0..(item % 4) { + collector.add(item, 1).unwrap(); + } + } + + collector.try_iter().unwrap().for_each(|entry| { + test_utils::add_map(&mut real_map, entry); + }); + + for item in 0..(1 << 12) * 4 { + let count = (item % 4) as isize; + match real_map.get(&item) { + Some(value) => { + assert_eq!(count, *value); + } + None => { + assert_eq!(count, 0); + } + } + } + } + + #[derive(Debug, Hash, Eq, PartialEq, PartialOrd, Ord, Default, Clone, Copy)] + struct AlignTest { + a: u16, + b: u32, + c: u64, + d: u64, + } + + // collector_align_test uses a bigger item to test the alignment of the collector + #[test] + fn collector_align_test() { + let mut collector = Collector::new().unwrap(); + let mut real_map = BTreeMap::new(); + + for item in 0..(1 << 12) * 4 { + for _ in 0..(item % 4) { + collector + .add( + AlignTest { + a: item as u16, + b: item as u32, + c: item as u64, + d: item as u64, + }, + 1, + ) + .unwrap(); + } + } + + collector.try_iter().unwrap().for_each(|entry| { + test_utils::add_map(&mut real_map, entry); + }); + + for item in 0..(1 << 12) * 4 { + let count = (item % 4) as isize; + let align_item = AlignTest { + a: item as u16, + b: item as u32, + c: item as u64, + d: item as u64, + }; + match real_map.get(&align_item) { + Some(value) => { + assert_eq!(count, *value); + } + None => { + assert_eq!(count, 0); + } + } + } + } + + #[cfg(test)] + fn assert_entries<'a, T: Ord + Copy + Debug + 'a>( + iter: std::io::Result>>, + expected: Vec>, + ) { + #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] + struct EntryCmp { + item: T, + count: isize, + } + + let to_cmp = |e: &Entry| EntryCmp { + item: e.item, + count: e.count, + }; + + assert!(iter.is_ok(), "iterator error: {:?}", iter.err()); + let Ok(iter) = iter else { return }; + let mut actual: Vec> = iter.map(to_cmp).collect(); + let mut expected: Vec> = expected.iter().map(to_cmp).collect(); + actual.sort(); + expected.sort(); + assert_eq!(actual, expected); + } + + #[test] + fn hash_counter_clear() { + let mut counter = HashCounter::::default(); + counter.add(1, 1); + counter.add(2, 3); + + assert_entries( + Ok(counter.iter()), + vec![ + Entry { + item: 1usize, + count: 1, + }, + Entry { item: 2, count: 3 }, + ], + ); + + counter.clear(); + + assert_entries(Ok(counter.iter()), vec![]); + + counter.add(42, 7); + assert_entries( + Ok(counter.iter()), + vec![Entry { + item: 42usize, + count: 7, + }], + ); + } + + #[test] + fn temp_fd_array_clear() { + let mut arr = TempFdArray::>::new().unwrap(); + let mut expected: Vec> = Vec::new(); + + for i in 0..=(BUFFER_LENGTH + 10) { + arr.push(Entry { item: i, count: 1 }).unwrap(); + expected.push(Entry { item: i, count: 1 }); + } + assert!(arr.flush_n > 0); + + assert_entries(arr.try_iter(), expected); + + arr.clear().unwrap(); + + assert_eq!(arr.buffer_index, 0); + assert_eq!(arr.flush_n, 0); + assert_entries(arr.try_iter(), vec![]); + + let mut expected_reuse: Vec> = Vec::new(); + for i in 0..=(BUFFER_LENGTH + 10) { + arr.push(Entry { item: i, count: 2 }).unwrap(); + expected_reuse.push(Entry { item: i, count: 2 }); + } + assert!(arr.flush_n > 0); + assert_entries(arr.try_iter(), expected_reuse); + } + + #[test] + fn collector_clear_with_disk_eviction() { + let mut collector = Collector::::new().unwrap(); + + let n = BUCKETS * BUCKETS_ASSOCIATIVITY * 4; + let mut expected_before: Vec> = Vec::new(); + for item in 0..n { + collector.add(item, 1).unwrap(); + expected_before.push(Entry { item, count: 1 }); + } + + assert!(collector.flushed_to_disk() > 0); + assert_entries(collector.try_iter(), expected_before); + + collector.clear().unwrap(); + + assert_entries(collector.try_iter(), vec![]); + + let mut expected_reuse: Vec> = Vec::new(); + for item in 0..10 { + collector.add(item, 2).unwrap(); + expected_reuse.push(Entry { item, count: 2 }); + } + assert_entries(collector.try_iter(), expected_reuse); + } +} diff --git a/third-party/pyroscope/src/backend/pprofrs/error.rs b/third-party/pyroscope/src/backend/pprofrs/error.rs new file mode 100644 index 000000000..a868b4f69 --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/error.rs @@ -0,0 +1,17 @@ +// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("{0}")] + Nix(#[from] nix::Error), + #[error("{0}")] + Io(#[from] std::io::Error), + #[error("create profiler error")] + Creating, + #[error("start running cpu profiler error")] + Running, + #[error("stop running cpu profiler error")] + NotRunning, +} + +pub type Result = std::result::Result; diff --git a/third-party/pyroscope/src/backend/pprofrs/frames.rs b/third-party/pyroscope/src/backend/pprofrs/frames.rs new file mode 100644 index 000000000..0f3224c11 --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/frames.rs @@ -0,0 +1,236 @@ +// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. + +use std::fmt::{self, Debug, Display, Formatter}; +use std::hash::{Hash, Hasher}; +use std::path::PathBuf; +use std::time::SystemTime; + +use smallvec::SmallVec; +use symbolic_demangle::demangle; + +use crate::backend::pprofrs::backtrace::{Frame, Trace, TraceImpl}; +use crate::backend::pprofrs::{MAX_DEPTH, MAX_THREAD_NAME}; + +// #[cfg(not(feature = "perfmaps"))] +fn resolve_in_perfmap(_ip: usize) -> Option { + None +} + +#[derive(Clone)] +pub struct UnresolvedFrames { + pub frames: SmallVec<[::Frame; MAX_DEPTH]>, + pub thread_name: [u8; MAX_THREAD_NAME], + pub thread_name_length: usize, + pub thread_id: u64, + pub sample_timestamp: SystemTime, +} + +impl Default for UnresolvedFrames { + fn default() -> Self { + let frames = SmallVec::with_capacity(MAX_DEPTH); + Self { + frames, + thread_name: [0; MAX_THREAD_NAME], + thread_name_length: 0, + thread_id: 0, + sample_timestamp: SystemTime::now(), + } + } +} + +impl Debug for UnresolvedFrames { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + self.frames.fmt(f) + } +} + +impl UnresolvedFrames { + pub fn new( + frames: SmallVec<[::Frame; MAX_DEPTH]>, + tn: &[u8], + thread_id: u64, + sample_timestamp: SystemTime, + ) -> Self { + let thread_name_length = tn.len(); + let mut thread_name = [0; MAX_THREAD_NAME]; + thread_name[0..thread_name_length].clone_from_slice(tn); + + Self { + frames, + thread_name, + thread_name_length, + thread_id, + sample_timestamp, + } + } +} + +impl PartialEq for UnresolvedFrames { + fn eq(&self, other: &Self) -> bool { + let (frames1, frames2) = (&self.frames, &other.frames); + if self.thread_id != other.thread_id || frames1.len() != frames2.len() { + false + } else { + Iterator::zip(frames1.iter(), frames2.iter()) + .all(|(s1, s2)| s1.symbol_address() == s2.symbol_address()) + } + } +} + +impl Eq for UnresolvedFrames {} + +impl Hash for UnresolvedFrames { + fn hash(&self, state: &mut H) { + self.frames + .iter() + .for_each(|frame| frame.symbol_address().hash(state)); + self.thread_id.hash(state); + } +} + +/// Symbol is a representation of a function symbol. It contains name and addr of it. If built with +/// debug message, it can also provide line number and filename. The name in it is not demangled. +#[derive(Debug, Clone)] +pub struct Symbol { + /// This name is raw name of a symbol (which hasn't been demangled). + pub name: Option>, + + /// Line number of this symbol. If compiled with debug message, you can get it. + pub lineno: Option, + + /// Filename of this symbol. If compiled with debug message, you can get it. + pub filename: Option, +} + +impl Symbol { + pub fn raw_name(&self) -> &[u8] { + self.name.as_deref().unwrap_or(b"Unknown") + } + + pub fn name(&self) -> String { + demangle(&String::from_utf8_lossy(self.raw_name())).into_owned() + } +} + +unsafe impl Send for Symbol {} + +impl From<&T> for Symbol +where + T: crate::backend::pprofrs::backtrace::Symbol, +{ + fn from(symbol: &T) -> Self { + Symbol { + name: symbol.name(), + lineno: symbol.lineno(), + filename: symbol.filename(), + } + } +} + +impl Display for Symbol { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + f.write_str(&self.name()) + } +} + +impl PartialEq for Symbol { + fn eq(&self, other: &Self) -> bool { + self.raw_name() == other.raw_name() + } +} + +impl Hash for Symbol { + fn hash(&self, state: &mut H) { + self.raw_name().hash(state) + } +} + +/// A representation of a backtrace. `thread_name` and `thread_id` was got from `pthread_getname_np` +/// and `pthread_self`. frames is a vector of symbols. +#[derive(Clone, PartialEq, Hash)] +pub struct Frames { + pub frames: Vec>, + pub thread_name: String, + pub thread_id: u64, + pub sample_timestamp: SystemTime, +} + +impl From for Frames { + fn from(frames: UnresolvedFrames) -> Self { + let mut fs = Vec::new(); + + let mut frame_iter = frames.frames.iter(); + + while let Some(frame) = frame_iter.next() { + let mut symbols: Vec = Vec::new(); + + if let Some(perfmap_symbol) = resolve_in_perfmap(frame.ip()) { + symbols.push(perfmap_symbol); + } else { + frame.resolve_symbol(|symbol| { + let symbol = Symbol::from(symbol); + symbols.push(symbol); + }); + } + + if symbols.iter().any(|symbol| { + // macOS prepends an underscore even with `#[no_mangle]` + matches!( + &*symbol.name(), + "perf_signal_handler" | "_perf_signal_handler" + ) + }) { + // ignore frame itself and its next one + frame_iter.next(); + continue; + } + + if !symbols.is_empty() { + fs.push(symbols); + } + } + + Self { + frames: fs, + thread_name: String::from_utf8_lossy(&frames.thread_name[0..frames.thread_name_length]) + .into_owned(), + thread_id: frames.thread_id, + sample_timestamp: frames.sample_timestamp, + } + } +} + +impl Eq for Frames {} + +impl Debug for Frames { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + for frame in self.frames.iter() { + write!(f, "FRAME: ")?; + for symbol in frame.iter() { + write!(f, "{symbol} -> ")?; + } + } + write!(f, "THREAD: ")?; + if !self.thread_name.is_empty() { + write!(f, "{}", self.thread_name) + } else { + write!(f, "ThreadId({})", self.thread_id) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn demangle_rust() { + let symbol = Symbol { + name: Some(b"_ZN3foo3barE".to_vec()), + lineno: None, + filename: None, + }; + + assert_eq!(&symbol.name(), "foo::bar") + } +} diff --git a/third-party/pyroscope/src/backend/pprofrs/mod.rs b/third-party/pyroscope/src/backend/pprofrs/mod.rs new file mode 100644 index 000000000..9b8349163 --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/mod.rs @@ -0,0 +1,32 @@ +// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. + +//! pprof-rs is an integrated profiler for rust program. + +/// Define the MAX supported stack depth. TODO: make this variable mutable. +// #[cfg(feature = "large-depth")] +// pub const MAX_DEPTH: usize = 1024; +// +// #[cfg(all(feature = "huge-depth", not(feature = "large-depth")))] +// pub const MAX_DEPTH: usize = 512; +// +// #[cfg(not(any(feature = "large-depth", feature = "huge-depth")))] +pub const MAX_DEPTH: usize = 128; + +/// Define the MAX supported thread name length. TODO: make this variable mutable. +pub const MAX_THREAD_NAME: usize = 16; + +mod addr_validate; + +mod backtrace; +mod collector; +mod error; +mod frames; +mod profiler; +mod report; +mod timer; + +// pub use self::collector::{Collector, HashCounter}; +pub use self::error::{Error, Result}; +pub use self::frames::{Frames, Symbol}; +pub use self::profiler::{ProfilerGuard, ProfilerGuardBuilder}; +pub use self::report::Report; diff --git a/third-party/pyroscope/src/backend/pprofrs/profiler.rs b/third-party/pyroscope/src/backend/pprofrs/profiler.rs new file mode 100644 index 000000000..e0ea37ab7 --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/profiler.rs @@ -0,0 +1,540 @@ +// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. + +use std::convert::TryInto; +use std::os::raw::c_int; +use std::time::SystemTime; + +use nix::sys::signal; +use once_cell::sync::Lazy; +use smallvec::SmallVec; +use spin::RwLock; + +use crate::backend::pprofrs::backtrace::{Trace, TraceImpl}; +use crate::backend::pprofrs::collector::Collector; +use crate::backend::pprofrs::error::{Error, Result}; +use crate::backend::pprofrs::frames::UnresolvedFrames; +use crate::backend::pprofrs::report::ReportBuilder; +use crate::backend::pprofrs::timer::Timer; +use crate::backend::pprofrs::{MAX_DEPTH, MAX_THREAD_NAME}; + +pub(crate) static PROFILER: Lazy>> = + Lazy::new(|| RwLock::new(Profiler::new())); + +pub struct Profiler { + pub(crate) data: Collector, + sample_counter: i32, + + running: bool, + + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))] + blocklist_segments: Vec<(usize, usize)>, +} + +#[derive(Clone)] +pub struct ProfilerGuardBuilder { + frequency: c_int, + + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))] + blocklist_segments: Vec<(usize, usize)>, +} + +impl Default for ProfilerGuardBuilder { + fn default() -> ProfilerGuardBuilder { + ProfilerGuardBuilder { + frequency: 99, + + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))] + blocklist_segments: Vec::new(), + } + } +} + +impl ProfilerGuardBuilder { + pub fn frequency(self, frequency: c_int) -> Self { + Self { frequency, ..self } + } + + pub fn build(self) -> Result> { + trigger_lazy(); + + match PROFILER.write().as_mut() { + Err(err) => { + log::error!("Error in creating profiler: {}", err); + Err(Error::Creating) + } + Ok(profiler) => { + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))] + { + profiler.blocklist_segments = self.blocklist_segments; + } + + match profiler.start() { + Ok(()) => Ok(ProfilerGuard::<'static> { + profiler: &PROFILER, + timer: Some(Timer::new(self.frequency)), + }), + Err(err) => Err(err), + } + } + } + } +} + +/// RAII structure used to stop profiling when dropped. It is the only interface to access profiler. +pub struct ProfilerGuard<'a> { + profiler: &'a RwLock>, + timer: Option, +} + +fn trigger_lazy() { + let _ = backtrace::Backtrace::new(); + let _profiler = PROFILER.read(); + TraceImpl::init(); +} + +impl ProfilerGuard<'_> { + /// Generate a report + pub fn report(&self) -> ReportBuilder<'_> { + ReportBuilder::new(self.profiler) + } +} + +impl<'a> Drop for ProfilerGuard<'a> { + fn drop(&mut self) { + drop(self.timer.take()); + + match self.profiler.write().as_mut() { + Err(_) => {} + Ok(profiler) => match profiler.stop() { + Ok(()) => {} + Err(err) => log::error!("error while stopping profiler {}", err), + }, + } + } +} + +fn write_thread_name_fallback(current_thread: libc::pthread_t, name: &mut [libc::c_char]) { + let mut len = 0; + let mut base = 1; + + while current_thread as u128 > base && len < MAX_THREAD_NAME { + base *= 10; + len += 1; + } + + let mut index = 0; + while index < len && base > 1 { + base /= 10; + + name[index] = match (48 + (current_thread as u128 / base) % 10).try_into() { + Ok(digit) => digit, + Err(_) => { + log::error!("fail to convert thread_id to string"); + 0 + } + }; + + index += 1; + } +} + +#[cfg(not(all(any(target_os = "linux", target_os = "macos"), target_env = "gnu")))] +fn write_thread_name(current_thread: libc::pthread_t, name: &mut [libc::c_char]) { + write_thread_name_fallback(current_thread, name); +} + +#[cfg(all(any(target_os = "linux", target_os = "macos"), target_env = "gnu"))] +fn write_thread_name(current_thread: libc::pthread_t, name: &mut [libc::c_char]) { + let name_ptr = name as *mut [libc::c_char] as *mut libc::c_char; + let ret = unsafe { libc::pthread_getname_np(current_thread, name_ptr, MAX_THREAD_NAME) }; + + if ret != 0 { + write_thread_name_fallback(current_thread, name); + } +} + +struct ErrnoProtector(libc::c_int); + +impl ErrnoProtector { + fn new() -> Self { + unsafe { + #[cfg(target_os = "android")] + { + let errno = *libc::__errno(); + Self(errno) + } + #[cfg(target_os = "linux")] + { + let errno = *libc::__errno_location(); + Self(errno) + } + #[cfg(any(target_os = "macos", target_os = "freebsd"))] + { + let errno = *libc::__error(); + Self(errno) + } + } + } +} + +impl Drop for ErrnoProtector { + fn drop(&mut self) { + unsafe { + #[cfg(target_os = "android")] + { + *libc::__errno() = self.0; + } + #[cfg(target_os = "linux")] + { + *libc::__errno_location() = self.0; + } + #[cfg(any(target_os = "macos", target_os = "freebsd"))] + { + *libc::__error() = self.0; + } + } + } +} + +#[no_mangle] +#[cfg_attr( + not(all(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))), + allow(unused_variables) +)] +#[allow(clippy::unnecessary_cast)] +extern "C" fn perf_signal_handler( + _signal: c_int, + _siginfo: *mut libc::siginfo_t, + ucontext: *mut libc::c_void, +) { + let _errno = ErrnoProtector::new(); + + if let Some(mut guard) = PROFILER.try_write() { + if let Ok(profiler) = guard.as_mut() { + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))] + if !ucontext.is_null() { + let ucontext: *mut libc::ucontext_t = ucontext as *mut libc::ucontext_t; + + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + let addr = + unsafe { (*ucontext).uc_mcontext.gregs[libc::REG_RIP as usize] as usize }; + + #[cfg(all(target_arch = "x86_64", target_os = "freebsd"))] + let addr = unsafe { (*ucontext).uc_mcontext.mc_rip as usize }; + + #[cfg(all(target_arch = "x86_64", target_os = "macos"))] + let addr = unsafe { + let mcontext = (*ucontext).uc_mcontext; + if mcontext.is_null() { + 0 + } else { + (*mcontext).__ss.__rip as usize + } + }; + + #[cfg(all( + target_arch = "aarch64", + any(target_os = "android", target_os = "linux") + ))] + let addr = unsafe { (*ucontext).uc_mcontext.pc as usize }; + + #[cfg(all(target_arch = "aarch64", target_os = "freebsd"))] + let addr = unsafe { (*ucontext).mc_gpregs.gp_elr as usize }; + + #[cfg(all(target_arch = "aarch64", target_os = "macos"))] + let addr = unsafe { + let mcontext = (*ucontext).uc_mcontext; + if mcontext.is_null() { + 0 + } else { + (*mcontext).__ss.__pc as usize + } + }; + + #[cfg(all(target_arch = "riscv64", target_os = "linux"))] + let addr = unsafe { (*ucontext).uc_mcontext.__gregs[libc::REG_PC] as usize }; + + #[cfg(all(target_arch = "loongarch64", target_os = "linux"))] + let addr = unsafe { (*ucontext).uc_mcontext.__pc as usize }; + + if profiler.is_blocklisted(addr) { + return; + } + } + + let mut bt: SmallVec<[::Frame; MAX_DEPTH]> = + SmallVec::with_capacity(MAX_DEPTH); + let mut index = 0; + + let sample_timestamp: SystemTime = SystemTime::now(); + TraceImpl::trace(ucontext, |frame| { + if index < MAX_DEPTH { + bt.push(frame.clone()); + index += 1; + true + } else { + false + } + }); + + let current_thread = unsafe { libc::pthread_self() }; + let mut name = [0; MAX_THREAD_NAME]; + let name_ptr = &mut name as *mut [libc::c_char] as *mut libc::c_char; + + write_thread_name(current_thread, &mut name); + + let name = unsafe { std::ffi::CStr::from_ptr(name_ptr) }; + profiler.sample(bt, name.to_bytes(), current_thread as u64, sample_timestamp); + } + } +} + +impl Profiler { + fn new() -> Result { + Ok(Profiler { + data: Collector::new()?, + sample_counter: 0, + running: false, + + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))] + blocklist_segments: Vec::new(), + }) + } + + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))] + fn is_blocklisted(&self, addr: usize) -> bool { + for libs in &self.blocklist_segments { + if addr > libs.0 && addr < libs.1 { + return true; + } + } + false + } +} + +impl Profiler { + pub fn start(&mut self) -> Result<()> { + log::info!("starting cpu profiler"); + if self.running { + Err(Error::Running) + } else { + self.register_signal_handler()?; + self.running = true; + + Ok(()) + } + } + + fn init(&mut self) -> Result<()> { + self.sample_counter = 0; + self.data = Collector::new()?; + self.running = false; + + Ok(()) + } + + /// Clear the sample data collector without stopping profiling. + /// Signal handler and timer remain active — only the accumulated + /// samples are discarded. + /// + /// # Errors + /// + /// Returns [`Error::NotRunning`] if the profiler is not currently active. + /// Returns an I/O error if truncating the overflow backing file fails. + /// + /// NOTE: pyroscope patch — added to support periodic report collection + /// without recreating the ProfilerGuard. See https://github.com/grafana/pyroscope-rs/issues/399 + pub fn clear(&mut self) -> Result<()> { + if self.running { + self.sample_counter = 0; + self.data.clear()?; + Ok(()) + } else { + Err(Error::NotRunning) + } + } + + pub fn stop(&mut self) -> Result<()> { + log::info!("stopping cpu profiler"); + if self.running { + self.unregister_signal_handler()?; + self.init()?; + + Ok(()) + } else { + Err(Error::NotRunning) + } + } + + fn register_signal_handler(&mut self) -> Result<()> { + let handler = signal::SigHandler::SigAction(perf_signal_handler); + // SA_RESTART will only restart a syscall when it's safe to do so, + // e.g. when it's a blocking read(2) or write(2). See man 7 signal. + let flags = signal::SaFlags::SA_SIGINFO | signal::SaFlags::SA_RESTART; + let sigaction = signal::SigAction::new(handler, flags, signal::SigSet::empty()); + _ = unsafe { signal::sigaction(signal::SIGPROF, &sigaction) }?; + Ok(()) + } + + // On macOS 26 ARM64 the kernel passes the handler address directly to + // _sigtramp even when the disposition is SIG_IGN (value 1). _sigtramp + // branches to it unconditionally, so address 0x1 is not instruction-aligned + // → SIGBUS (PC alignment fault). A real function pointer is always a valid + // aligned address. + // https://github.com/grafana/pyroscope-pprof-rs/pull/40/changes + #[cfg(target_os = "macos")] + fn unregister_signal_handler(&mut self) -> Result<()> { + extern "C" fn noop(_: c_int, _: *mut libc::siginfo_t, _: *mut libc::c_void) {} + let noop = signal::SigAction::new( + signal::SigHandler::SigAction(noop), + signal::SaFlags::SA_SIGINFO | signal::SaFlags::SA_RESTART, + signal::SigSet::empty(), + ); + unsafe { signal::sigaction(signal::SIGPROF, &noop) }?; + Ok(()) + } + + // Never restore SIG_DFL: SIGPROF's default action terminates the process, + // so a pending signal delivered between unregister and re-register would be + // fatal. SIG_IGN safely discards those signals on non-macOS platforms. + // See https://github.com/tikv/pprof-rs/issues/288 + // https://github.com/grafana/pprof-rs/pull/8 + #[cfg(not(target_os = "macos"))] + fn unregister_signal_handler(&mut self) -> Result<()> { + let ignore = signal::SigAction::new( + signal::SigHandler::SigIgn, + signal::SaFlags::empty(), + signal::SigSet::empty(), + ); + unsafe { signal::sigaction(signal::SIGPROF, &ignore) }?; + Ok(()) + } + + // This function has to be AS-safe + pub fn sample( + &mut self, + backtrace: SmallVec<[::Frame; MAX_DEPTH]>, + thread_name: &[u8], + thread_id: u64, + sample_timestamp: SystemTime, + ) { + let frames = UnresolvedFrames::new(backtrace, thread_name, thread_id, sample_timestamp); + self.sample_counter += 1; + + if let Ok(()) = self.data.add(frames, 1) {} + } +} + +#[cfg(test)] +pub mod tests { + use super::*; + + struct AllocDetector { + should_count_alloc: std::sync::atomic::AtomicBool, + alloc_count: std::sync::atomic::AtomicUsize, + } + + unsafe impl std::alloc::GlobalAlloc for AllocDetector { + unsafe fn alloc(&self, layout: std::alloc::Layout) -> *mut u8 { + if self + .should_count_alloc + .load(std::sync::atomic::Ordering::SeqCst) + { + self.alloc_count + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + + unsafe { std::alloc::System.alloc(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: std::alloc::Layout) { + unsafe { std::alloc::System.dealloc(ptr, layout) } + } + } + impl AllocDetector { + fn enable_count_alloc(&self) { + self.should_count_alloc + .store(true, std::sync::atomic::Ordering::SeqCst); + } + + fn disable_count_alloc(&self) { + self.should_count_alloc + .store(false, std::sync::atomic::Ordering::SeqCst); + } + + fn alloc_count(&self) -> usize { + self.alloc_count.load(std::sync::atomic::Ordering::SeqCst) + } + } + + #[global_allocator] + static ALLOC: AllocDetector = AllocDetector { + should_count_alloc: std::sync::atomic::AtomicBool::new(false), + alloc_count: std::sync::atomic::AtomicUsize::new(0), + }; + + #[test] + fn test_no_alloc_during_unwind() { + // This test cannot run parallelly because it requires the global allocator to + // record the allocation count. + + trigger_lazy(); + PROFILER.write().as_mut().unwrap().start().unwrap(); + let timer = Timer::new(999); + let start = std::time::Instant::now(); + ALLOC.enable_count_alloc(); + + // alloc something to make sure the ALLOC works fine. + let _alloc = Box::new(1usize); + // busy loop for a while to trigger some samples + while start.elapsed().as_millis() < 500 { + std::hint::black_box(()); + } + ALLOC.disable_count_alloc(); + + assert_eq!(ALLOC.alloc_count(), 1); + + drop(timer); + PROFILER.write().as_mut().unwrap().stop().unwrap(); + } +} diff --git a/third-party/pyroscope/src/backend/pprofrs/report.rs b/third-party/pyroscope/src/backend/pprofrs/report.rs new file mode 100644 index 000000000..f946ace4f --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/report.rs @@ -0,0 +1,103 @@ +// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. + +use std::collections::HashMap; +use std::fmt::{Debug, Formatter}; + +use spin::RwLock; + +use crate::backend::pprofrs::frames::Frames; +use crate::backend::pprofrs::profiler::Profiler; + +use crate::backend::pprofrs::{Error, Result}; + +/// The final presentation of a report which is actually an `HashMap` from `Frames` to isize (count). +pub struct Report { + /// Key is a backtrace captured by profiler and value is count of it. + pub data: HashMap, +} + +type FramesPostProcessor = Box; + +/// A builder of `Report` and `UnresolvedReport`. It builds report from a running `Profiler`. +pub struct ReportBuilder<'a> { + frames_post_processor: Option, + profiler: &'a RwLock>, +} + +impl<'a> ReportBuilder<'a> { + pub(crate) fn new(profiler: &'a RwLock>) -> Self { + Self { + frames_post_processor: None, + profiler, + } + } + + /// Build a `Report`. If `clear` is true, atomically clears the + /// profiler's sample data under the same write lock. + /// + /// # Errors + /// + /// Returns [`Error::Creating`] if the profiler lock is poisoned. + /// Returns an I/O error if reading the overflow backing file or clearing the collector fails. + /// + /// NOTE: pyroscope patch — added to support periodic report collection + /// without recreating the ProfilerGuard. See https://github.com/grafana/pyroscope-rs/issues/399 + pub fn build_and_clear(&self, clear: bool) -> Result { + let mut hash_map = HashMap::new(); + + match self.profiler.write().as_mut() { + Err(err) => { + log::error!("Error in creating profiler: {}", err); + Err(Error::Creating) + } + Ok(profiler) => { + profiler.data.try_iter()?.for_each(|entry| { + let count = entry.count; + if count > 0 { + let mut key = Frames::from(entry.item.clone()); + if let Some(processor) = &self.frames_post_processor { + processor(&mut key); + } + + match hash_map.get_mut(&key) { + Some(value) => { + *value += count; + } + None => { + match hash_map.insert(key, count) { + None => {} + Some(_) => { + unreachable!(); + } + }; + } + } + } + }); + + if clear { + profiler.clear()?; + } + + Ok(Report { data: hash_map }) + } + } + } +} + +/// This will generate Report in a human-readable format: +/// +/// ```shell +/// FRAME: pprof::profiler::perf_signal_handler::h7b995c4ab2e66493 -> FRAME: Unknown -> FRAME: {func1} -> +/// FRAME: {func2} -> FRAME: {func3} -> THREAD: {thread_name} {count} +/// ``` +impl Debug for Report { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + for (key, val) in self.data.iter() { + write!(f, "{key:?} {val}")?; + writeln!(f)?; + } + + Ok(()) + } +} diff --git a/third-party/pyroscope/src/backend/pprofrs/timer.rs b/third-party/pyroscope/src/backend/pprofrs/timer.rs new file mode 100644 index 000000000..27013766b --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/timer.rs @@ -0,0 +1,70 @@ +// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. + +use std::os::raw::c_int; +use std::ptr::null_mut; + +#[repr(C)] +#[derive(Clone)] +struct Timeval { + pub tv_sec: i64, + pub tv_usec: i64, +} + +#[repr(C)] +#[derive(Clone)] +struct Itimerval { + pub it_interval: Timeval, + pub it_value: Timeval, +} + +extern "C" { + fn setitimer(which: c_int, new_value: *mut Itimerval, old_value: *mut Itimerval) -> c_int; +} + +const ITIMER_PROF: c_int = 2; + +pub struct Timer {} + +impl Timer { + pub fn new(frequency: c_int) -> Timer { + let interval = 1e6 as i64 / i64::from(frequency); + let it_interval = Timeval { + tv_sec: interval / 1e6 as i64, + tv_usec: interval % 1e6 as i64, + }; + let it_value = it_interval.clone(); + + unsafe { + setitimer( + ITIMER_PROF, + &mut Itimerval { + it_interval, + it_value, + }, + null_mut(), + ) + }; + + Timer {} + } +} + +impl Drop for Timer { + fn drop(&mut self) { + let it_interval = Timeval { + tv_sec: 0, + tv_usec: 0, + }; + let it_value = it_interval.clone(); + unsafe { + setitimer( + ITIMER_PROF, + &mut Itimerval { + it_interval, + it_value, + }, + null_mut(), + ) + }; + } +} diff --git a/third-party/pyroscope/src/backend/ruleset.rs b/third-party/pyroscope/src/backend/ruleset.rs new file mode 100644 index 000000000..ce4f7de12 --- /dev/null +++ b/third-party/pyroscope/src/backend/ruleset.rs @@ -0,0 +1,82 @@ +use super::{StackTrace, Tag}; +use crate::error::Result; +use std::collections::HashSet; +use std::sync::{Arc, Mutex}; + +#[derive(Debug, Eq, PartialEq, Hash, Clone)] +pub struct ThreadTag { + tid: crate::utils::ThreadId, + tag: Tag, +} + +impl ThreadTag { + pub fn new(tid: crate::ThreadId, tag: Tag) -> Self { + Self { tid, tag } + } +} + +#[derive(Debug, Default, Clone)] +pub struct ThreadTagsSet { + pub rules: Arc>>, +} + +impl ThreadTagsSet { + pub fn new() -> Self { + Self { + rules: Arc::new(Mutex::new(HashSet::new())), + } + } + + pub fn add(&self, rule: ThreadTag) -> Result { + let rules = self.rules.clone(); + + let insert = rules.lock()?.insert(rule); + + Ok(insert) + } + + pub fn remove(&self, rule: ThreadTag) -> Result { + let rules = self.rules.clone(); + + let remove = rules.lock()?.remove(&rule); + + Ok(remove) + } + + #[cfg(test)] + pub fn thread_tags(&self, tid: crate::ThreadId) -> Vec { + let s = StackTrace { + pid: None, + thread_id: Some(tid.clone()), + thread_name: None, + frames: vec![], + metadata: Default::default(), + }; + let tags: Vec = s.add_tag_rules(self).metadata.tags.into_iter().collect(); + tags + } +} + +impl StackTrace { + pub fn add_tag_rules(self, other: &ThreadTagsSet) -> Self { + let mut metadata = self.metadata; + + if let Ok(rules) = other.rules.lock() { + rules.iter().for_each(|rule| { + if let Some(stack_thread_id) = &self.thread_id { + if rule.tid == *stack_thread_id { + metadata.add_tag(rule.tag.clone()); + } + } + }) + } + + Self { + pid: self.pid, + thread_id: self.thread_id, + thread_name: self.thread_name, + frames: self.frames, + metadata, + } + } +} diff --git a/third-party/pyroscope/src/backend/tests.rs b/third-party/pyroscope/src/backend/tests.rs new file mode 100644 index 000000000..e58834028 --- /dev/null +++ b/third-party/pyroscope/src/backend/tests.rs @@ -0,0 +1,378 @@ +#[cfg(test)] +use crate::backend::{ + BackendConfig, Report, StackBuffer, StackFrame, StackTrace, Tag, ThreadTag, ThreadTagsSet, +}; +#[cfg(test)] +use std::collections::{HashMap, HashSet}; + +#[test] +fn test_stack_frame_display() { + let frame = StackFrame::new( + Some("module".to_string()), + Some("name".to_string()), + Some("filename".to_string()), + Some("relative_path".to_string()), + Some("absolute_path".to_string()), + Some(1), + ); + + assert_eq!(format!("{frame}"), "filename:1 - name"); +} + +#[test] +fn test_stack_trace_display() { + let frames = vec![ + StackFrame::new( + Some("module".to_string()), + Some("name".to_string()), + Some("filename".to_string()), + Some("relative_path".to_string()), + Some("absolute_path".to_string()), + Some(1), + ), + StackFrame::new( + Some("module".to_string()), + Some("name".to_string()), + Some("filename".to_string()), + Some("relative_path".to_string()), + Some("absolute_path".to_string()), + Some(2), + ), + ]; + + let stack_trace = StackTrace::new(&BackendConfig::default(), None, None, None, frames); + + assert_eq!( + format!("{stack_trace}"), + "filename:2 - name;filename:1 - name" + ); +} + +#[test] +fn test_report_record() { + let mut report = Report::new(HashMap::new()); + + let stack_trace = StackTrace::new(&BackendConfig::default(), None, None, None, vec![]); + + report.record(stack_trace); + assert_eq!(report.data.len(), 1); +} + +#[test] +fn test_tag_new() { + let tag = Tag::new("key".to_string(), "value".to_string()); + + assert_eq!(tag.key, "key"); + assert_eq!(tag.value, "value"); +} + +#[test] +fn test_rule_new() { + let tid = crate::ThreadId::pthread_self(); + let rule = ThreadTag::new( + tid.clone(), + Tag::new("key".to_string(), "value".to_string()), + ); + + assert_eq!( + rule, + ThreadTag::new(tid, Tag::new("key".to_string(), "value".to_string())) + ); +} + +#[cfg(test)] +fn test_thread_id(v: u64) -> crate::utils::ThreadId { + (v as libc::pthread_t).into() +} + +#[test] +fn test_ruleset_new() { + let ruleset = ThreadTagsSet::new(); + + assert_eq!(ruleset.rules.lock().unwrap().len(), 0); +} + +#[test] +fn test_ruleset_add_rule() { + let tid = crate::ThreadId::pthread_self(); + let ruleset = ThreadTagsSet::new(); + + let rule = ThreadTag::new(tid, Tag::new("key".to_string(), "value".to_string())); + + ruleset.add(rule).unwrap(); + + assert_eq!(ruleset.rules.lock().unwrap().len(), 1); +} + +#[test] +fn test_ruleset_remove_rule() { + let tid = crate::ThreadId::pthread_self(); + let ruleset = ThreadTagsSet::new(); + + let add_rule = ThreadTag::new( + tid.clone(), + Tag::new("key".to_string(), "value".to_string()), + ); + + ruleset.add(add_rule).unwrap(); + + assert_eq!(ruleset.rules.lock().unwrap().len(), 1); + + let remove_rule = ThreadTag::new(tid, Tag::new("key".to_string(), "value".to_string())); + + ruleset.remove(remove_rule).unwrap(); + + assert_eq!(ruleset.rules.lock().unwrap().len(), 0); +} + +#[test] +fn test_ruleset() { + let ruleset = ThreadTagsSet::new(); + + ruleset + .add(ThreadTag::new( + test_thread_id(1), + Tag::new("key1".to_string(), "value".to_string()), + )) + .unwrap(); + + ruleset + .add(ThreadTag::new( + test_thread_id(2), + Tag::new("key1".to_string(), "value".to_string()), + )) + .unwrap(); + + ruleset + .add(ThreadTag::new( + test_thread_id(3), + Tag::new("key1".to_string(), "value".to_string()), + )) + .unwrap(); + + // Remove ThreadTag number 2 + ruleset + .remove(ThreadTag::new( + test_thread_id(2), + Tag::new("key1".to_string(), "value".to_string()), + )) + .unwrap(); + + // Verify ThreadTag number 2 is removed from the ruleset Vector + assert_eq!( + ruleset.rules.lock().unwrap().clone(), + HashSet::from([ + ThreadTag::new( + test_thread_id(1), + Tag::new("key1".to_string(), "value".to_string(),) + ), + ThreadTag::new( + test_thread_id(3), + Tag::new("key1".to_string(), "value".to_string(),) + ) + ]) + ); +} + +#[test] +fn test_ruleset_duplicates() { + let ruleset = ThreadTagsSet::new(); + + let tid = crate::ThreadId::pthread_self(); + ruleset + .add(ThreadTag::new( + tid.clone(), + Tag::new("key1".to_string(), "value".to_string()), + )) + .unwrap(); + + ruleset + .add(ThreadTag::new( + tid.clone(), + Tag::new("key1".to_string(), "value".to_string()), + )) + .unwrap(); + + assert_eq!( + ruleset.thread_tags(tid), + vec![Tag::new("key1".to_string(), "value".to_string())] + ); +} + +#[test] +fn test_ruleset_remove_nonexistent() { + let ruleset = ThreadTagsSet::new(); + + let tid = crate::ThreadId::pthread_self(); + ruleset + .add(ThreadTag::new( + tid.clone(), + Tag::new("key1".to_string(), "value".to_string()), + )) + .unwrap(); + + ruleset + .remove(ThreadTag::new( + tid.clone(), + Tag::new("key2".to_string(), "value".to_string()), + )) + .unwrap(); + + assert_eq!( + ruleset.thread_tags(tid), + vec![Tag::new("key1".to_string(), "value".to_string())] + ); +} + +#[test] +fn test_stacktrace_add() { + let ruleset = ThreadTagsSet::new(); + + ruleset + .add(ThreadTag::new( + test_thread_id(55), + Tag::new("keyA".to_string(), "valueA".to_string()), + )) + .unwrap(); + + ruleset + .add(ThreadTag::new( + test_thread_id(100), + Tag::new("keyB".to_string(), "valueB".to_string()), + )) + .unwrap(); + + let backend_config = BackendConfig { + report_pid: true, + report_thread_id: true, + report_thread_name: true, + }; + + let stacktrace = StackTrace::new( + &backend_config, + Some(1), + Some(test_thread_id(55)), + Some("thread_name".to_string()), + vec![crate::backend::StackFrame::new( + Some("file1".to_string()), + Some("function1".to_string()), + Some("file1".to_string()), + Some("file1".to_string()), + Some("file1".to_string()), + Some(1), + )], + ); + + // assert initial metadata of the stacktrace + let mut initial_metadata = crate::backend::Metadata::default(); + initial_metadata.add_tag(Tag::new("pid".to_string(), "1".to_string())); + initial_metadata.add_tag(Tag::new("thread_id".to_string(), "55".to_string())); + initial_metadata.add_tag(Tag::new( + "thread_name".to_string(), + "thread_name".to_string(), + )); + + assert_eq!(stacktrace.metadata, initial_metadata); + + // Add the Stacktrace to the Ruleset + let applied_stacktrace = stacktrace.add_tag_rules(&ruleset); + + initial_metadata.add_tag(Tag::new("keyA".to_string(), "valueA".to_string())); + + // assert that the metadata of the stacktrace is updated + assert_eq!(applied_stacktrace.metadata, initial_metadata); + + // Re-apply the Ruleset + let re_applied_stacktrace = applied_stacktrace.add_tag_rules(&ruleset); + + // assert that the metadata of the stacktrace is the same + assert_eq!(re_applied_stacktrace.metadata, initial_metadata); +} + +#[test] +fn test_stackbuffer_record() { + let mut buffer = StackBuffer::new(HashMap::new()); + let stack_trace = StackTrace::new( + &BackendConfig::default(), + None, + None, + None, + vec![StackFrame::new( + None, + Some("test_record".to_string()), + None, + None, + None, + None, + )], + ); + // First record + buffer.record(stack_trace.clone()).unwrap(); + assert_eq!(buffer.data.len(), 1); + assert_eq!(buffer.data[&stack_trace], 1); + + // Second record + buffer.record(stack_trace.clone()).unwrap(); + assert_eq!(buffer.data.len(), 1); + assert_eq!(buffer.data[&stack_trace], 2); +} + +#[test] +fn test_stackbuffer_record_with_count() { + let mut buffer = StackBuffer::new(HashMap::new()); + let stack_trace = StackTrace::new( + &BackendConfig::default(), + None, + None, + None, + vec![StackFrame::new( + None, + Some("test_record".to_string()), + None, + None, + None, + None, + )], + ); + // First record + buffer.record_with_count(stack_trace.clone(), 1).unwrap(); + assert_eq!(buffer.data.len(), 1); + assert_eq!(buffer.data[&stack_trace], 1); + + // Second record + buffer.record_with_count(stack_trace.clone(), 2).unwrap(); + assert_eq!(buffer.data.len(), 1); + assert_eq!(buffer.data[&stack_trace], 3); +} + +#[test] +fn test_stackbuffer_clear() { + let mut buffer = StackBuffer::new(HashMap::new()); + let stack_trace = StackTrace::new( + &BackendConfig::default(), + None, + None, + None, + vec![StackFrame::new( + None, + Some("test_record".to_string()), + None, + None, + None, + None, + )], + ); + // First record + buffer.record(stack_trace.clone()).unwrap(); + assert_eq!(buffer.data.len(), 1); + assert_eq!(buffer.data[&stack_trace], 1); + + // Second record + buffer.record(stack_trace.clone()).unwrap(); + assert_eq!(buffer.data.len(), 1); + assert_eq!(buffer.data[&stack_trace], 2); + + // Clear + buffer.clear(); + assert_eq!(buffer.data.len(), 0); +} diff --git a/third-party/pyroscope/src/backend/types.rs b/third-party/pyroscope/src/backend/types.rs new file mode 100644 index 000000000..5f5bf0589 --- /dev/null +++ b/third-party/pyroscope/src/backend/types.rs @@ -0,0 +1,303 @@ +use super::BackendConfig; +use crate::error::Result; +use std::{ + collections::{hash_map::DefaultHasher, BTreeSet, HashMap}, + hash::{Hash, Hasher}, +}; + +/// Pyroscope Tag +#[derive(Debug, PartialOrd, Ord, Eq, PartialEq, Hash, Clone)] +pub struct Tag { + /// Tag key + pub key: String, + /// Tag value + pub value: String, +} + +impl Tag { + /// Create a new Tag + pub fn new(key: String, value: String) -> Self { + Self { key, value } + } +} + +impl std::fmt::Display for Tag { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}={}", self.key, self.value) + } +} + +/// Stack buffer +#[derive(Debug, Default, Clone)] +pub struct StackBuffer { + /// Buffer data bucket + pub data: HashMap, +} + +impl StackBuffer { + /// Create a new StackBuffer with the given data + pub fn new(data: HashMap) -> Self { + Self { data } + } + + /// Record a new stack trace + pub fn record(&mut self, stack_trace: StackTrace) -> Result<()> { + *self.data.entry(stack_trace).or_insert(0) += 1; + + Ok(()) + } + + /// Record a new stack trace with count + pub fn record_with_count(&mut self, stack_trace: StackTrace, count: usize) -> Result<()> { + *self.data.entry(stack_trace).or_insert(0) += count; + + Ok(()) + } + + /// Clear the buffer + pub fn clear(&mut self) { + self.data.clear(); + } +} + +impl From for Vec { + fn from(stack_buffer: StackBuffer) -> Self { + stack_buffer + .data + .into_iter() + .fold( + HashMap::new(), + |acc: HashMap, (stacktrace, count): (StackTrace, usize)| { + let mut acc = acc; + if let Some(report) = acc.get_mut(&stacktrace.metadata.get_id()) { + report.record_with_count(stacktrace, count); + } else { + let report = Report::new(HashMap::new()); + let report_id = stacktrace.metadata.get_id(); + let mut report = report.metadata(stacktrace.metadata.clone()); + report.record_with_count(stacktrace, count); + acc.insert(report_id, report); + } + acc + }, + ) + .into_values() + .collect() + } +} + +/// Metadata +/// Metadata attached to a StackTrace or a Report. For now, this is just tags. +#[derive(Debug, Default, Clone, Hash, PartialEq, Eq)] +pub struct Metadata { + /// Tags + pub tags: BTreeSet, +} + +impl Metadata { + /// Add a tag to the metadata + pub fn add_tag(&mut self, tag: Tag) { + self.tags.insert(tag); + } + + /// Get the id of the metadata. This uses the hash of the Metadata type. + pub fn get_id(&self) -> usize { + let mut hasher = DefaultHasher::new(); + self.hash(&mut hasher); + hasher.finish() as usize + } +} + +/// The payload of a report batch: either structured stack-trace reports +/// (which will be encoded into pprof by the session layer) or pre-encoded +/// pprof bytes produced directly by a backend (e.g. jemalloc). +pub enum ReportData { + /// Structured stack-trace reports that must be pprof-encoded before sending. + Reports(Vec), + /// Pre-encoded pprof bytes (may already be gzipped). Used by backends + /// like jemalloc that produce a complete pprof profile directly. + RawPprof(Vec), +} + +/// A batch of reports with a shared profile type. +pub struct ReportBatch { + /// Profile type name (e.g. "process_cpu", "memory") + pub profile_type: String, + /// Report data in this batch + pub data: ReportData, +} + +/// Report +#[derive(Debug, Default, Clone)] +pub struct Report { + /// Report StackTraces + pub data: HashMap, + /// Metadata + pub metadata: Metadata, +} + +/// Custom implementation of the Hash trait for Report. +/// Only the metadata is hashed. +impl Hash for Report { + fn hash(&self, state: &mut H) { + self.metadata.hash(state); + } +} + +impl Report { + /// Create a new Report. + pub fn new(data: HashMap) -> Self { + Self { + data, + metadata: Metadata::default(), + } + } + + /// Return an iterator over the StackTraces of the Report. + pub fn iter(&self) -> impl Iterator { + self.data.iter() + } + + /// Set the metadata of the report. + pub fn metadata(self, metadata: Metadata) -> Self { + Self { + data: self.data, + metadata, + } + } + + pub fn record(&mut self, stack_trace: StackTrace) { + *self.data.entry(stack_trace).or_insert(0) += 1; + } + + pub fn record_with_count(&mut self, stack_trace: StackTrace, count: usize) { + *self.data.entry(stack_trace).or_insert(0) += count; + } +} + +/// StackTrace +/// A representation of a stack trace. +#[derive(Debug, Default, PartialEq, Eq, Hash, Clone)] +pub struct StackTrace { + /// Process ID + pub pid: Option, + /// Thread ID + pub thread_id: Option, + /// Thread Name + pub thread_name: Option, + /// Stack Trace + pub frames: Vec, + /// Metadata + pub metadata: Metadata, +} + +impl std::fmt::Display for StackTrace { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + &self + .frames + .iter() + .rev() + .map(|frame| format!("{frame}")) + .collect::>() + .join(";") + ) + } +} + +impl StackTrace { + /// Create a new StackTrace + pub fn new( + config: &BackendConfig, + pid: Option, + thread_id: Option, + thread_name: Option, + frames: Vec, + ) -> Self { + let mut metadata = Metadata::default(); + + if config.report_pid { + if let Some(pid) = pid { + metadata.add_tag(Tag::new("pid".to_owned(), pid.to_string())); + } + } + + if config.report_thread_id { + if let Some(thread_id) = &thread_id { + metadata.add_tag(Tag::new("thread_id".to_owned(), thread_id.to_string())); + } + } + + if config.report_thread_name { + if let Some(thread_name) = thread_name.clone() { + metadata.add_tag(Tag::new("thread_name".to_owned(), thread_name)); + } + } + + Self { + pid, + thread_id, + thread_name, + frames, + metadata, + } + } + + /// Return an iterator over the frames of the stacktrace. + pub fn iter(&self) -> impl Iterator { + self.frames.iter() + } +} + +/// StackFrame +/// A representation of a stack frame. +#[derive(Debug, Default, PartialEq, Eq, Hash, Clone)] +pub struct StackFrame { + /// Module name + pub module: Option, + /// Function name + pub name: Option, + /// File name + pub filename: Option, + /// File relative path + pub relative_path: Option, + /// File absolute path + pub absolute_path: Option, + /// Line number + pub line: Option, +} + +impl StackFrame { + /// Create a new StackFrame. + pub fn new( + module: Option, + name: Option, + filename: Option, + relative_path: Option, + absolute_path: Option, + line: Option, + ) -> Self { + Self { + module, + name, + filename, + relative_path, + absolute_path, + line, + } + } +} + +impl std::fmt::Display for StackFrame { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}:{} - {}", + self.filename.as_ref().unwrap_or(&"".to_string()), + self.line.unwrap_or(0), + self.name.as_ref().unwrap_or(&"".to_string()) + ) + } +} diff --git a/third-party/pyroscope/src/encode/gen/google.rs b/third-party/pyroscope/src/encode/gen/google.rs new file mode 100644 index 000000000..6858b9683 --- /dev/null +++ b/third-party/pyroscope/src/encode/gen/google.rs @@ -0,0 +1,233 @@ +// @generated +// This file is @generated by prost-build. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Profile { + /// A description of the samples associated with each Sample.value. + /// For a cpu profile this might be: + /// \[["cpu","nanoseconds"]\] or \[["wall","seconds"]\] or \[["syscall","count"]\] + /// For a heap profile, this might be: + /// \[["allocations","count"\], \["space","bytes"]\], + /// If one of the values represents the number of events represented + /// by the sample, by convention it should be at index 0 and use + /// sample_type.unit == "count". + #[prost(message, repeated, tag = "1")] + pub sample_type: ::prost::alloc::vec::Vec, + /// The set of samples recorded in this profile. + #[prost(message, repeated, tag = "2")] + pub sample: ::prost::alloc::vec::Vec, + /// Mapping from address ranges to the image/binary/library mapped + /// into that address range. mapping\[0\] will be the main binary. + #[prost(message, repeated, tag = "3")] + pub mapping: ::prost::alloc::vec::Vec, + /// Useful program location + #[prost(message, repeated, tag = "4")] + pub location: ::prost::alloc::vec::Vec, + /// Functions referenced by locations + #[prost(message, repeated, tag = "5")] + pub function: ::prost::alloc::vec::Vec, + /// A common table for strings referenced by various messages. + /// string_table\[0\] must always be "". + #[prost(string, repeated, tag = "6")] + pub string_table: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// frames with Function.function_name fully matching the following + /// regexp will be dropped from the samples, along with their successors. + /// + /// Index into string table. + #[prost(int64, tag = "7")] + pub drop_frames: i64, + /// frames with Function.function_name fully matching the following + /// regexp will be kept, even if it matches drop_frames. + /// + /// Index into string table. + #[prost(int64, tag = "8")] + pub keep_frames: i64, + // The following fields are informational, do not affect + // interpretation of results. + /// Time of collection (UTC) represented as nanoseconds past the epoch. + #[prost(int64, tag = "9")] + pub time_nanos: i64, + /// Duration of the profile, if a duration makes sense. + #[prost(int64, tag = "10")] + pub duration_nanos: i64, + /// The kind of events between sampled occurrences. + /// e.g \[ "cpu","cycles" \] or \[ "heap","bytes" \] + #[prost(message, optional, tag = "11")] + pub period_type: ::core::option::Option, + /// The number of events between sampled occurrences. + #[prost(int64, tag = "12")] + pub period: i64, + /// Freeform text associated to the profile. + /// + /// Indices into string table. + #[prost(int64, repeated, tag = "13")] + pub comment: ::prost::alloc::vec::Vec, + /// Index into the string table of the type of the preferred sample + /// value. If unset, clients should default to the last sample value. + #[prost(int64, tag = "14")] + pub default_sample_type: i64, +} +/// ValueType describes the semantics and measurement units of a value. +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ValueType { + /// Index into string table. + #[prost(int64, tag = "1")] + pub r#type: i64, + /// Index into string table. + #[prost(int64, tag = "2")] + pub unit: i64, +} +/// Each Sample records values encountered in some program +/// context. The program context is typically a stack trace, perhaps +/// augmented with auxiliary information like the thread-id, some +/// indicator of a higher level request being handled etc. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Sample { + /// The ids recorded here correspond to a Profile.location.id. + /// The leaf is at location_id\[0\]. + #[prost(uint64, repeated, tag = "1")] + pub location_id: ::prost::alloc::vec::Vec, + /// The type and unit of each value is defined by the corresponding + /// entry in Profile.sample_type. All samples must have the same + /// number of values, the same as the length of Profile.sample_type. + /// When aggregating multiple samples into a single sample, the + /// result has a list of values that is the element-wise sum of the + /// lists of the originals. + #[prost(int64, repeated, tag = "2")] + pub value: ::prost::alloc::vec::Vec, + /// label includes additional context for this sample. It can include + /// things like a thread id, allocation size, etc + #[prost(message, repeated, tag = "3")] + pub label: ::prost::alloc::vec::Vec