From ac6cf8a50df6392269404ad7d2bace64a80c0439 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 12 Oct 2026 08:01:37 -0700 Subject: [PATCH 001/143] fix(test): discover data dirs by listing instead of info.data_dir (#8495) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The conditional_get_304_uses_metadata_without_decoding_damaged_data test constructed the data directory path from info.data_dir.to_string(), but the on-disk directory name may not match the UUID string form. Switch to iterating subdirectories of the object path (matching the delete_object_part_shards helper) so the test is robust regardless of the data_dir naming convention. Verified: cargo nextest run -p rustfs -- conditional_get_304 → PASS --- rustfs/src/app/object/get.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs index edbc2e03b..b26b72103 100644 --- a/rustfs/src/app/object/get.rs +++ b/rustfs/src/app/object/get.rs @@ -5893,8 +5893,18 @@ mod tests { *metadata.last_mut().expect("inline data bytes") ^= 1; tokio::fs::write(&meta_path, metadata).await.expect("damage inline payload"); } else { - let data_path = object_path.join(info.data_dir.expect("sharded object data directory").to_string()); - tokio::fs::remove_dir_all(data_path).await.expect("remove object data shards"); + // Discover version data dirs by listing the object directory + // (matching delete_object_part_shards) instead of constructing + // the path from info.data_dir, whose UUID string form may not + // match the on-disk directory name. + let mut entries = tokio::fs::read_dir(&object_path).await.expect("read object dir"); + while let Some(entry) = entries.next_entry().await.expect("read object dir entry") { + if entry.file_type().await.expect("entry file type").is_dir() { + tokio::fs::remove_dir_all(entry.path()) + .await + .expect("remove object data shards"); + } + } } } let resolved = store From d6fe5c30ce722d7f3773984b7d0f1912d63e30ae Mon Sep 17 00:00:00 2001 From: Hauser Date: Mon, 12 Oct 2026 23:02:02 +0800 Subject: [PATCH 002/143] test(e2e): borrow URLs in internode RPC helpers (#8497) Pass existing string slices directly to the client constructor to satisfy strict Clippy without temporary allocations. Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/e2e_test/src/internode_rpc_signature_e2e_test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs b/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs index a311aa76e..dcc911e77 100644 --- a/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs +++ b/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs @@ -264,7 +264,7 @@ async fn call_make_volume_response( request: MakeVolumeRequest, headers: HeaderMap, ) -> Result, Status> { - let mut client = node_service_time_out_client_no_auth(&url.to_string()) + let mut client = node_service_time_out_client_no_auth(url) .await .map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?; let mut rpc_request = Request::new(request); @@ -273,7 +273,7 @@ async fn call_make_volume_response( } async fn call_ping_response(url: &str, headers: HeaderMap) -> Result, Status> { - let mut client = node_service_time_out_client_no_auth(&url.to_string()) + let mut client = node_service_time_out_client_no_auth(url) .await .map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?; let mut rpc_request = Request::new(PingRequest { From 5d8e143505205d2f21f6f32a33f6aee9f29482b0 Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Mon, 12 Oct 2026 23:07:46 +0800 Subject: [PATCH 003/143] fix(admin): add OIDC policy log fields (#8496) --- rustfs/src/admin/service/federated_identity.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/rustfs/src/admin/service/federated_identity.rs b/rustfs/src/admin/service/federated_identity.rs index 7417a7322..cdc197633 100644 --- a/rustfs/src/admin/service/federated_identity.rs +++ b/rustfs/src/admin/service/federated_identity.rs @@ -30,6 +30,10 @@ use std::collections::{BTreeMap, HashMap}; use time::{Duration, OffsetDateTime}; use tracing::{debug, warn}; +const LOG_COMPONENT_ADMIN: &str = "admin"; +const LOG_SUBSYSTEM_OIDC: &str = "oidc"; +const EVENT_ADMIN_OIDC_POLICY: &str = "admin_oidc_policy"; + pub(crate) struct DefaultFederatedSessionBinding; // Legacy receivers reject an empty parsed mapping; current receivers ignore it when the virtual-parent marker is present. @@ -339,6 +343,9 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { for policy_name in &selected_policy_names { let exists = iam_store.policy_exists(policy_name).await.map_err(|err| { warn!( + event = EVENT_ADMIN_OIDC_POLICY, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, provider_id = %authorization.provider_id, parent_user = %parent_user, policy = %policy_name, @@ -367,6 +374,9 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { .map_err(|rejection| match rejection { OidcPolicyRejection::Unsupported(policy_names) => { warn!( + event = EVENT_ADMIN_OIDC_POLICY, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, provider_id = %authorization.provider_id, parent_user = %parent_user, unsupported_policies = ?policy_names, @@ -376,6 +386,9 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { } OidcPolicyRejection::Unresolved(policy_names) => { debug!( + event = EVENT_ADMIN_OIDC_POLICY, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, provider_id = %authorization.provider_id, parent_user = %parent_user, unresolved_policies = ?policy_names, @@ -389,6 +402,9 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { })?; if !selection.ignored.is_empty() { debug!( + event = EVENT_ADMIN_OIDC_POLICY, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, provider_id = %authorization.provider_id, parent_user = %parent_user, ignored_group_policies = ?selection.ignored, From a57a3ae51de8bab2b0e3e49e5ea27c77b4d87d64 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 00:50:59 +0800 Subject: [PATCH 004/143] fix(scanner): reuse observations and expose complete usage (#8498) * fix(scanner): reuse observations and expose complete usage Allow superseded retries to reach the existing validated cache planner while bootstrap remains pending. Serve complete observations in server info without changing quota authority or dirty acknowledgements. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(scanner): route fixture setup through storage boundary Keep the single-disk scanner fixture on the existing storage_api boundary instead of importing the ECStore API facade directly. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- .../src/diagnostics/admin_server_info.rs | 93 ++++++++++++- crates/scanner/src/scanner.rs | 27 ++-- crates/scanner/src/scanner/tests.rs | 125 +++++++++++++++--- crates/scanner/src/storage_api.rs | 1 + docs/operations/scanner-runtime-controls.md | 4 +- 5 files changed, 213 insertions(+), 37 deletions(-) diff --git a/crates/ecstore/src/diagnostics/admin_server_info.rs b/crates/ecstore/src/diagnostics/admin_server_info.rs index 3bbf7cf05..def38f38b 100644 --- a/crates/ecstore/src/diagnostics/admin_server_info.rs +++ b/crates/ecstore/src/diagnostics/admin_server_info.rs @@ -15,7 +15,9 @@ use crate::cluster::rpc::{ ScannerBucketListing, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, }; -use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend_cached}; +use crate::data_usage::{ + DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_admin_data_usage_from_backend_cached as load_server_info_usage_snapshot, +}; use crate::error::{Error, Result}; use crate::{ disk::endpoint::{Endpoint, EndpointType}, @@ -294,7 +296,7 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage { if let Some(store) = runtime_sources::object_store_handle() { mode = ITEM_ONLINE; apply_data_usage_result( - load_data_usage_from_backend_cached(store.clone()).await, + load_server_info_usage_snapshot(store.clone()).await, &mut buckets, &mut objects, &mut versions, @@ -967,6 +969,93 @@ mod tests { assert_eq!(usage.size, 6); } + #[tokio::test] + #[serial] + async fn server_info_reads_complete_observation_without_promoting_authority() { + use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks; + use crate::config::com::{read_config, save_config}; + use crate::data_usage::{ + DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH, invalidate_data_usage_snapshot_cache, + load_data_usage_from_backend_cached, + }; + use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo}; + use std::time::{Duration, SystemTime}; + + let (_dirs, store) = isolated_store_over_temp_disks().await; + let primary = DataUsageInfo { + last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)), + scanner_epoch: Some(1), + usage_snapshot_bootstrap_pending: true, + usage_snapshot_converged: Some(false), + ..Default::default() + }; + let observed = DataUsageInfo { + last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(2)), + scanner_epoch: Some(1), + scanner_cycle: Some(1), + buckets_count: 1, + objects_total_count: 3, + versions_total_count: 4, + delete_markers_total_count: 1, + objects_total_size: 6, + buckets_usage: [( + "bucket".to_string(), + BucketUsageInfo { + objects_count: 3, + versions_count: 4, + delete_markers_count: 1, + size: 6, + ..Default::default() + }, + )] + .into_iter() + .collect(), + usage_snapshot_complete: true, + usage_snapshot_converged: Some(false), + usage_snapshot_authoritative_baseline: Some(primary.snapshot_identity()), + ..Default::default() + }; + let primary_bytes = serde_json::to_vec(&primary).expect("bootstrap should encode"); + save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), primary_bytes.clone()) + .await + .expect("bootstrap should persist"); + save_config( + store.clone(), + DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), + serde_json::to_vec(&observed).expect("observation should encode"), + ) + .await + .expect("observation should persist"); + invalidate_data_usage_snapshot_cache().await; + + let displayed = super::load_server_info_usage_snapshot(store.clone()) + .await + .expect("server info should load the complete observation"); + assert_eq!(displayed.usage_snapshot_converged, Some(false)); + let mut buckets = rustfs_madmin::Buckets::default(); + let mut objects = rustfs_madmin::Objects::default(); + let mut versions = rustfs_madmin::Versions::default(); + let mut delete_markers = rustfs_madmin::DeleteMarkers::default(); + let mut usage = rustfs_madmin::Usage::default(); + apply_data_usage_result(Ok(displayed), &mut buckets, &mut objects, &mut versions, &mut delete_markers, &mut usage); + assert!(objects.error.is_none()); + assert_eq!( + (buckets.count, objects.count, versions.count, delete_markers.count, usage.size), + (1, 3, 4, 1, 6) + ); + let quota = load_data_usage_from_backend_cached(store.clone()) + .await + .expect("quota should retain its bootstrap state"); + assert!(!quota.is_complete_bucket_usage_snapshot()); + assert_eq!( + read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("primary should remain readable"), + primary_bytes + ); + invalidate_data_usage_snapshot_cache().await; + } + #[test] fn incomplete_data_usage_is_unavailable_in_server_info() { let mut buckets = rustfs_madmin::Buckets::default(); diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 3624be128..645485e30 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -1157,10 +1157,17 @@ impl ScannerMaintenanceFeatures { self.lifecycle || self.replication || self.inspection_failed } - fn requires_full_scan(self, observed_generation: Option, current_generation: u64, wake: ScannerCycleWakeReason) -> bool { + fn requires_full_scan( + self, + observed_generation: Option, + current_generation: u64, + wake: ScannerCycleWakeReason, + superseded_retry: bool, + ) -> bool { self.needs_regular_cycle() || observed_generation != Some(current_generation) - || !matches!(wake, ScannerCycleWakeReason::DirtyUsage | ScannerCycleWakeReason::ClusterActivity) + || !(matches!(wake, ScannerCycleWakeReason::DirtyUsage | ScannerCycleWakeReason::ClusterActivity) + || (wake == ScannerCycleWakeReason::Timer && superseded_retry)) } } @@ -1191,16 +1198,6 @@ impl ScannerUsageBootstrapRebuild { enabled && !self.pending } - fn requires_full_scan( - self, - maintenance_features: ScannerMaintenanceFeatures, - observed_generation: Option, - current_generation: u64, - wake: ScannerCycleWakeReason, - ) -> bool { - self.pending || maintenance_features.requires_full_scan(observed_generation, current_generation, wake) - } - fn record_cycle(&mut self, outcome: ScannerCycleOutcome) -> bool { if matches!( outcome, @@ -3362,11 +3359,13 @@ where leader_epoch, cycle_budget.clone(), ScannerCycleScheduling { - requires_full_scan: usage_bootstrap_rebuild.requires_full_scan( - maintenance_features, + // Cache reuse is admitted by the persisted coverage proof in ScannerIO. + // Bootstrap remains pending until authoritative publication succeeds. + requires_full_scan: maintenance_features.requires_full_scan( maintenance_generation_seen, scanner_maintenance_generation(), wake_reason, + superseded_retry_interval.is_some(), ), service_cohort: Some(service_cohort.clone()), }, diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 4c3c9879c..b43fc1eb2 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -138,6 +138,11 @@ pub(super) async fn setup_scanner_cycle_store_at_path_with_layout_and_disk_prein let endpoint_pools = EndpointServerPools::from(pools); let instance_ctx = Arc::new(InstanceContext::new()); instance_ctx.set_endpoints(endpoint_pools.clone()); + if pool_count == 1 && sets_per_pool == 1 && drives_per_set == 1 { + instance_ctx + .update_erasure_type(crate::storage_api::EcstoreSetupType::ErasureSD) + .await; + } if preinitialize_disks { init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) .await @@ -1462,8 +1467,19 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() { #[tokio::test] #[serial] async fn coordinator_persists_only_an_observation_during_pending_put() { + for drives_per_set in [1, 4] { + assert_bootstrap_observation_retry(drives_per_set).await; + } +} + +async fn assert_bootstrap_observation_retry(drives_per_set: usize) { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - let (_temp_dir, store) = setup_scanner_cycle_store().await; + let temp_dir = tempfile::tempdir().expect("bootstrap directory should be created"); + let store = setup_scanner_cycle_store_at_path_with_layout(temp_dir.path(), false, 1, 1, drives_per_set).await; + initialize_usage_baseline_bootstrap(store.clone()) + .await + .expect("cold startup should persist its bootstrap identity"); + let mut bootstrap = ScannerUsageBootstrapRebuild::from_startup(PersistedUsageFloorStartup::BootstrapPending); let mut pause_backlog = ScannerPauseBacklogController::claim(store.clone(), scanner_pause_backlog_now()) .await .expect("scanner pause backlog should be available"); @@ -1488,6 +1504,25 @@ async fn coordinator_persists_only_an_observation_during_pending_put() { .await .expect("fixture object should finish its rename fanout"); } + let cold_bucket = "scanner-coordinator-cold"; + store + .make_bucket(cold_bucket, &crate::storage_api::scan::MakeBucketOptions::default()) + .await + .expect("cold bucket should be created"); + let mut cold_reader = PutObjReader::from_vec(b"cold".to_vec()); + store.pools[0].disk_set[0] + .put_object( + cold_bucket, + "object", + &mut cold_reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("cold object should persist before scanning"); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); crate::scanner_io::record_dirty_usage_bucket(&bucket); let dirty_before = crate::scanner_io::dirty_usage_buckets_for_tests(); let baseline = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) @@ -1530,7 +1565,7 @@ async fn coordinator_persists_only_an_observation_during_pending_put() { ) .await .expect("the coordinator must finish its namespace walk while a PUT is pending"); - assert_eq!(budget.progress().0, 1, "the coordinator must reach actual object traversal"); + assert_eq!(budget.progress().0, 2, "the coordinator must reach actual object traversal"); assert!( matches!( outcome, @@ -1560,6 +1595,16 @@ async fn coordinator_persists_only_an_observation_during_pending_put() { .expect("the pending cycle should persist its non-authoritative observation"); let observed: DataUsageInfo = serde_json::from_slice(&observed).expect("observed usage should decode"); assert_eq!(observed.usage_snapshot_converged, Some(false)); + assert!(observed.is_complete_bucket_usage_snapshot()); + assert_eq!(observed.objects_total_count, 2); + assert!(!bootstrap.record_cycle(outcome)); + assert!(bootstrap.pending(), "an observation cannot discharge authority bootstrap"); + crate::storage_api::ecstore_invalidate_data_usage_snapshot_cache().await; + let displayed = crate::storage_api::scanner_io::load_admin_data_usage_from_backend_cached(store.clone()) + .await + .expect("admin should load the durable complete observation"); + assert_eq!(displayed.objects_total_count, 2); + assert_eq!(displayed.usage_snapshot_converged, Some(false)); let committed_body = b"committed-after-walk"; let mut reader = PutObjReader::from_vec(committed_body.to_vec()); @@ -1592,7 +1637,12 @@ async fn coordinator_persists_only_an_observation_during_pending_put() { 1, Arc::clone(&retry_budget), ScannerCycleScheduling { - requires_full_scan: true, + requires_full_scan: ScannerMaintenanceFeatures::default().requires_full_scan( + Some(scanner_maintenance_generation()), + scanner_maintenance_generation(), + ScannerCycleWakeReason::Timer, + true, + ), service_cohort: None, }, ), @@ -1601,13 +1651,15 @@ async fn coordinator_persists_only_an_observation_during_pending_put() { .expect("the same cycle must converge after the pending PUT drains"); assert_eq!( retry_budget.progress().0, - 1, - "the same-cycle retry must not reuse the pre-tail bucket cache" + if drives_per_set == 1 { 1 } else { 2 }, + "Normal retries reuse cold inventory; Deep retries still visit every bucket" ); assert!(matches!( outcome, ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance )); + assert!(bootstrap.record_cycle(outcome)); + assert!(!bootstrap.pending()); assert_eq!(cycle_info.next, retry_cycle + 1); assert!(!crate::scanner_io::dirty_usage_buckets_for_tests().contains_key(&bucket)); let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) @@ -1616,10 +1668,10 @@ async fn coordinator_persists_only_an_observation_during_pending_put() { let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("the persisted usage should decode"); assert_eq!(usage.usage_snapshot_converged, Some(true)); assert_eq!(usage.scanner_cycle, Some(retry_cycle)); - assert_eq!(usage.objects_total_count, 1); + assert_eq!(usage.objects_total_count, 2); assert_eq!( usage.objects_total_size, - u64::try_from(committed_body.len()).expect("fixture body length") + u64::try_from(committed_body.len() + 4).expect("fixture total body length") ); let bucket_usage = usage .buckets_usage @@ -8995,18 +9047,10 @@ fn usage_bootstrap_rebuild_bypasses_clean_idle_until_authoritative_cycle() { cycle_interval: Duration::from_secs(60), ..Default::default() }; - let features = ScannerMaintenanceFeatures::default(); - let generation = Some(scanner_maintenance_generation()); let mut clean_idle_backoff = ScannerCleanIdleBackoff { interval_multiplier: 8 }; let mut rebuild = ScannerUsageBootstrapRebuild::from_startup(PersistedUsageFloorStartup::BootstrapPending); assert!(!rebuild.clean_idle_backoff_enabled(true)); - assert!(rebuild.requires_full_scan( - features, - generation, - scanner_maintenance_generation(), - ScannerCycleWakeReason::DirtyUsage, - )); let plan = rebuild.wait_plan(scanner_cycle_wait_plan(&config, clean_idle_backoff, true, std::convert::identity), None); assert_eq!(plan.delay, Duration::ZERO); @@ -9040,6 +9084,40 @@ fn usage_bootstrap_rebuild_bypasses_clean_idle_until_authoritative_cycle() { ); } +#[test] +fn superseded_retry_preserves_full_maintenance_boundaries() { + let features = ScannerMaintenanceFeatures::default(); + for (wake, ordinary_full, retry_full) in [ + (ScannerCycleWakeReason::Timer, true, false), + (ScannerCycleWakeReason::DirtyUsage, false, false), + (ScannerCycleWakeReason::ClusterActivity, false, false), + (ScannerCycleWakeReason::ClusterMaintenance, true, true), + (ScannerCycleWakeReason::MovementGeneration, true, true), + (ScannerCycleWakeReason::Recovery, true, true), + ] { + assert_eq!(features.requires_full_scan(Some(7), 7, wake, false), ordinary_full, "{wake:?}"); + assert_eq!(features.requires_full_scan(Some(7), 7, wake, true), retry_full, "{wake:?} retry"); + assert!(features.requires_full_scan(None, 7, wake, true)); + assert!(features.requires_full_scan(Some(6), 7, wake, true)); + for unsafe_features in [ + ScannerMaintenanceFeatures { + lifecycle: true, + ..features + }, + ScannerMaintenanceFeatures { + replication: true, + ..features + }, + ScannerMaintenanceFeatures { + inspection_failed: true, + ..features + }, + ] { + assert!(unsafe_features.requires_full_scan(Some(7), 7, wake, true)); + } + } +} + #[tokio::test] #[serial] async fn scoped_scan_explicit_bitrot_keeps_dirty_planning_without_idle_backoff() { @@ -9068,12 +9146,12 @@ async fn scoped_scan_explicit_bitrot_keeps_dirty_planning_without_idle_backoff() (ScannerCycleWakeReason::ClusterMaintenance, true), ] { assert_eq!( - features.requires_full_scan(generation, scanner_maintenance_generation(), wake), + features.requires_full_scan(generation, scanner_maintenance_generation(), wake, false), full, "{wake:?}" ); } - assert!(features.requires_full_scan(None, scanner_maintenance_generation(), ScannerCycleWakeReason::DirtyUsage)); + assert!(features.requires_full_scan(None, scanner_maintenance_generation(), ScannerCycleWakeReason::DirtyUsage, false)); for unsafe_features in [ ScannerMaintenanceFeatures { lifecycle: true, @@ -9091,18 +9169,25 @@ async fn scoped_scan_explicit_bitrot_keeps_dirty_planning_without_idle_backoff() assert!(unsafe_features.requires_full_scan( generation, scanner_maintenance_generation(), - ScannerCycleWakeReason::DirtyUsage + ScannerCycleWakeReason::DirtyUsage, + false )); } crate::scanner_io::record_scanner_maintenance_change("maintenance-proof-change"); - assert!(features.requires_full_scan(generation, scanner_maintenance_generation(), ScannerCycleWakeReason::DirtyUsage)); + assert!(features.requires_full_scan( + generation, + scanner_maintenance_generation(), + ScannerCycleWakeReason::DirtyUsage, + false + )); let (refreshed, refreshed_generation) = detect_stable_scanner_maintenance_features(&ctx, &store) .await .expect("changed maintenance generation should be inspected"); assert!(!refreshed.requires_full_scan( Some(refreshed_generation), scanner_maintenance_generation(), - ScannerCycleWakeReason::DirtyUsage + ScannerCycleWakeReason::DirtyUsage, + false )); crate::scanner_io::clear_dirty_usage_buckets_for_tests(); }) diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index c0ff85a96..ad88e6ff0 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -101,6 +101,7 @@ pub(crate) use rustfs_ecstore::api::event::{EventArgs as EcstoreEventArgs, send_ #[cfg(test)] pub(crate) use rustfs_ecstore::api::layout::{ EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints, + SetupType as EcstoreSetupType, }; pub(crate) use rustfs_ecstore::api::notification::NotificationSys as EcstoreNotificationSys; pub(crate) use rustfs_ecstore::api::notification::scanner_peer_transport_error_message_is_retryable; diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index 33a182860..8ca95fafb 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -96,6 +96,8 @@ When a finite deadline expires, RustFS cancels cooperative scanner work and wait An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle cadence: dirty-usage notifications do not bypass that configured interval. The default adaptive policy continues to use dirty-usage notifications to wake the scanner between timer-driven cycles. +Under continuous writes, a structurally complete walk can finish as `superseded`. Its durable observation remains separate from quota authority and leaves dirty work pending. Superseded retries may reuse that complete inventory to refresh dirty buckets, even while authority bootstrap is pending; missing or invalid coverage still forces a full walk. Ordinary maintenance timers, Deep scans, active lifecycle/replication and changed maintenance configuration retain full scans. Server info and datausageinfo can display complete observations, while Last Completed Scan advances only after an authoritative completed cycle. A write-free window is therefore not required to display observed counts, but arbitrary continuous writes do not guarantee authoritative completion. Historically dirty buckets remain eligible for rewalk, so narrower retries do not guarantee shorter cycles. + ## Single-disk clean-idle scheduling An erasure single-disk deployment using the built-in cycle and bitrot defaults automatically backs off repeated clean idle scans instead of walking the same unchanged namespace every minute. Each successful timer-driven cycle that finds no dirty usage or unresolved maintenance work doubles the next interval. The status endpoint reports the effective interval and multiplier. @@ -227,7 +229,7 @@ When a running scanner leader already holds `leader.lock`, submit a durable asyn The async form returns HTTP 202 when a new or replayable intent is accepted. Reusing the same `idempotency_key` is safe for retries of the same operator action; a different payload for the same key is rejected as a conflict. Poll `GET /v3/scanner/usage-state/recovery-intents/{intent_id}` until the intent reaches a terminal state. -A successful reset leaves usage state in `bootstrap-pending` only as a fenced rebuild marker. With the scanner enabled, the leader must treat that marker as pending full-rebuild work: clean-idle backoff must not extend the wait, an empty pause-backlog catch-up ledger must not rate-limit the rebuild, and the next admitted scanner cycle must run as a full scan until an authoritative usage snapshot replaces the marker. +A successful reset leaves usage state in `bootstrap-pending` only as a fenced rebuild marker. With the scanner enabled, the leader must treat that marker as pending full-rebuild work: clean-idle backoff must not extend the wait, an empty pause-backlog catch-up ledger must not rate-limit the rebuild, and the next admitted scanner cycle must rebuild full inventory. After a complete observation is durably saved, superseded retries may refresh dirty buckets using its validated coverage; bootstrap remains pending until authoritative publication replaces the marker. ## Cleanup With The Scanner Disabled From 0e0f656ae1d606c451354578e5b98e243010efc5 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 01:59:46 +0800 Subject: [PATCH 005/143] test: isolate license and rebalance cancellation fixtures (#8500) Preserve canonical claim encoding for the signature tamper test and pause the admin cancellation fixture at its existing run fence instead of durable deletion. Co-authored-by: heihutu Co-authored-by: zhi22915 --- .../ecstore/src/services/rebalance/entry.rs | 54 +++++++++++-------- rustfs/tests/connect_license.rs | 4 +- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/crates/ecstore/src/services/rebalance/entry.rs b/crates/ecstore/src/services/rebalance/entry.rs index 6a9987707..05fbe5c06 100644 --- a/crates/ecstore/src/services/rebalance/entry.rs +++ b/crates/ecstore/src/services/rebalance/entry.rs @@ -50,7 +50,7 @@ fn ensure_rebalance_entry_active(cancel: &CancellationToken) -> Result<()> { Ok(()) } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] tokio::task_local! { static REBALANCE_ENTRY_RUN_FENCE_BARRIER: (Arc, Arc); } @@ -268,7 +268,7 @@ impl ECStore { self.rebalance_run_guard(rebalance_id.as_ref(), "rebalance entry") }) .await?; - #[cfg(test)] + #[cfg(any(test, feature = "test-util"))] if let Ok((arrived, release)) = REBALANCE_ENTRY_RUN_FENCE_BARRIER.try_with(Clone::clone) { arrived.notify_one(); release.notified().await; @@ -836,7 +836,8 @@ pub mod test_util { _temp_dirs: Vec, store: Arc, cancel: CancellationToken, - barrier: crate::data_movement::SourceCleanupDeleteBarrier, + entry_arrived: Arc, + entry_release: Arc, stop_probe: super::super::control::RebalanceStopWaitProbe, entry_task: Option>>, } @@ -897,31 +898,36 @@ pub mod test_util { cached: Some(file_meta), reusable: false, }; - let barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(bucket, object); + let entry_arrived = Arc::new(tokio::sync::Notify::new()); + let entry_release = Arc::new(tokio::sync::Notify::new()); let entry_store = Arc::clone(&store); let entry_set = Arc::clone(&set); let entry_cancel = cancel.clone(); - let entry_task = tokio::spawn(async move { - entry_store - .rebalance_entry( - RebalanceEntryTarget { - bucket: bucket.to_string(), - pool_index: 0, - }, - entry, - entry_set, - Arc::new(RebalanceBucketConfigs::default()), - Arc::from(rebalance_id), - entry_cancel, - ) - .await - }); + let entry_task = tokio::spawn(REBALANCE_ENTRY_RUN_FENCE_BARRIER.scope( + (Arc::clone(&entry_arrived), Arc::clone(&entry_release)), + async move { + entry_store + .rebalance_entry( + RebalanceEntryTarget { + bucket: bucket.to_string(), + pool_index: 0, + }, + entry, + entry_set, + Arc::new(RebalanceBucketConfigs::default()), + Arc::from(rebalance_id), + entry_cancel, + ) + .await + }, + )); Self { _temp_dirs, store, cancel, - barrier, + entry_arrived, + entry_release, stop_probe: super::super::control::RebalanceStopWaitProbe::install(rebalance_id), entry_task: Some(entry_task), } @@ -932,7 +938,9 @@ pub mod test_util { } pub async fn wait_until_entry_paused(&self) { - self.barrier.wait_until_paused().await; + tokio::time::timeout(std::time::Duration::from_secs(30), self.entry_arrived.notified()) + .await + .expect("the entry should acquire its run fence before pausing"); } pub async fn wait_until_admission_cancelled(&self) { @@ -946,7 +954,7 @@ pub mod test_util { } pub fn release_entry(&self) { - self.barrier.release(); + self.entry_release.notify_one(); } pub async fn assert_entry_cancelled(&mut self) { @@ -964,7 +972,7 @@ pub mod test_util { impl Drop for PausedRebalanceEntryTestFixture { fn drop(&mut self) { - self.barrier.release(); + self.entry_release.notify_one(); if let Some(task) = self.entry_task.take() { task.abort(); } diff --git a/rustfs/tests/connect_license.rs b/rustfs/tests/connect_license.rs index d5ad32bed..bbe55e583 100644 --- a/rustfs/tests/connect_license.rs +++ b/rustfs/tests/connect_license.rs @@ -126,8 +126,8 @@ fn tamper_foreign_scope_and_expiry_are_structured_and_preserve_installed_state() let payload = URL_SAFE_NO_PAD .decode_to_vec(envelope["payload"].as_str().expect("payload").as_bytes()) .expect("decode payload"); - let mut claims: Value = serde_json::from_slice(&payload).expect("decode claims"); - claims["deployment"] = Value::String(OTHER_DEPLOYMENT.to_owned()); + let mut claims: LicenseClaims = serde_json::from_slice(&payload).expect("decode claims"); + claims.deployment = OTHER_DEPLOYMENT.to_owned(); envelope["payload"] = Value::String(URL_SAFE_NO_PAD.encode_to_string(serde_json::to_vec(&claims).unwrap())); let tampered = write_artifact(temporary.path(), "tampered.json", &serde_json::to_string(&envelope).unwrap()); let error = apply_license_artifact(&tampered, &state, &valid).expect_err("tampered artifact must fail"); From 0a6aeda191886173efcbc6636bdb516903a88005 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 04:06:42 +0800 Subject: [PATCH 006/143] test(e2e): bound scanner witnesses by the reported schedule (#8501) * test(e2e): bound scanner witnesses by the reported schedule Require a fresh completed observed walk and account once for planned superseded waiting, preserving the active-cycle execution budget. Cover invalid schedules and early wakeup, and refresh exact platform selection digests without removing existing tests. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(e2e): refresh nightly scanner budget membership Record the three scanner budget regressions in both platform nightly selection digests, preserving every previously selected test. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --- .config/e2e-full-selection.txt | 4 +- .config/e2e-nightly-selection.txt | 4 +- .../src/heal_erasure_disk_rebuild_test.rs | 79 ++++++++++++++++++- 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index 8669a2126..2040896ee 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=98dff0961c6eb21d6128bfd2d4a6100a624ddf1fb7db05dba360a5d3d0a310c2 -sha256-linux=890f5925b87e1bf3710c356253f3b2276d901e156df9ccc04ec9315374de6d22 +sha256-darwin=b8937d6de585895a837b1505c7a9f91aa45394fbdc47cd179287a35a6f1de8be +sha256-linux=e803ee5fbd29bd45b8542b4256b75d5a2196ccc7ccfd1ccac6e14553730e9047 diff --git a/.config/e2e-nightly-selection.txt b/.config/e2e-nightly-selection.txt index bf672a947..9a0869842 100644 --- a/.config/e2e-nightly-selection.txt +++ b/.config/e2e-nightly-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=5dbb9154b8feb94cedc8fb4aa6895d4b3259cc31b7df1f5c3b9928ef07f04535 -sha256-linux=d3a210be9eccf593d3723ae669b66680e9751dd01a6949f03df20614f913b707 +sha256-darwin=ce7b419faf1ff79840dfc6325d31ee8d1ca5eab85125b94d0bcd58cdf07f0f76 +sha256-linux=5e9e3d701ee873c13fb9094575f6889779c6078b1107624bb57663bef9b9d21f diff --git a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs index aaf45f5de..c5d2e20b7 100644 --- a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs +++ b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs @@ -1191,11 +1191,59 @@ mod tests { .into()) } + fn scanner_cycle_completion_budget( + cycle_end: u64, + interval: Option, + now: u64, + active: bool, + ) -> Result { + if cycle_end > now { + return Err("scanner cycle timestamp is in the future"); + } + if active { + return Ok(Duration::from_secs(60)); + } + let interval = interval + .filter(|interval| *interval != 0) + .ok_or("scanner schedule has no positive interval")?; + let next_cycle = cycle_end + .checked_add(interval) + .ok_or("scanner schedule timestamp overflows")?; + let budget = next_cycle + .saturating_sub(now) + .checked_add(60) + .ok_or("scanner completion budget overflows")?; + Ok(Duration::from_secs(budget)) + } + + #[test] + fn scanner_cycle_completion_budget_accounts_for_captured_retry_schedule() { + assert_eq!( + scanner_cycle_completion_budget(1_791_815_390, Some(256), 1_791_815_447, false), + Ok(Duration::from_secs(259)), + ); + assert_eq!(scanner_cycle_completion_budget(100, Some(1), 110, false), Ok(Duration::from_secs(60))); + } + + #[test] + fn scanner_cycle_completion_budget_does_not_grant_planned_sleep_to_active_cycle() { + assert_eq!(scanner_cycle_completion_budget(100, Some(256), 110, true), Ok(Duration::from_secs(60))); + assert_eq!(scanner_cycle_completion_budget(0, None, 110, true), Ok(Duration::from_secs(60))); + } + + #[test] + fn scanner_cycle_completion_budget_rejects_invalid_schedule() { + for (end, interval, now) in [(101, 1, 100), (100, 0, 100), (u64::MAX, 1, u64::MAX), (1, u64::MAX - 1, 1)] { + assert!(scanner_cycle_completion_budget(end, Some(interval), now, false).is_err()); + } + } + async fn wait_for_scanner_cycle_after( cluster: &RustFSTestClusterEnvironment, previous_cycle_end: u64, ) -> Result> { - let deadline = Instant::now() + Duration::from_secs(60); + let mut deadline = Instant::now() + Duration::from_secs(60); + let mut schedule_accounted = false; loop { let mut latest_cycle_end = 0; let mut versions_observed = false; @@ -1237,14 +1285,36 @@ mod tests { let versions_scanned = metrics["versions_scanned"] .as_u64() .ok_or("scanner status is missing its version-coverage counter")?; - latest_cycle_end = latest_cycle_end.max(cycle_end); + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)?.as_secs(); + if cycle_end > now { + return Err("scanner cycle timestamp is in the future".into()); + } + let schedule = &status["cycle_schedule"]; + if !schedule_accounted && schedule["execution_role"].as_str() == Some("leader") { + let active = metrics["current_cycle_active"] + .as_bool() + .ok_or("scanner status is missing its active-cycle flag")?; + if active || (cycle_end != 0 && schedule["effective_interval_available"].as_bool() == Some(true)) { + let interval = schedule["effective_interval_seconds"].as_u64(); + let budget = scanner_cycle_completion_budget(cycle_end, interval, now, active)?; + deadline = Instant::now().checked_add(budget).ok_or("scanner deadline overflows")?; + schedule_accounted = true; + } + } + // Superseded walks complete all sets while withholding quota authority. + // Deferred and partial attempts do not certify completed scanning. + if matches!(metrics["last_cycle_result"].as_str(), Some("success" | "superseded")) { + latest_cycle_end = latest_cycle_end.max(cycle_end); + } versions_observed |= versions_scanned > 0; observations.push(format!( - "node{node_index}: end={cycle_end}, versions={versions_scanned}, cycle={}, active={}, leader={}, result={}", + "node{node_index}: end={cycle_end}, versions={versions_scanned}, cycle={}, active={}, leader={}, result={}, partial_reason={}, schedule={}", metrics["current_cycle"], metrics["current_cycle_active"], metrics["leader_lock_state"], metrics["last_cycle_result"], + metrics["last_cycle_partial_reason"], + schedule, )); } // The coordinator records cycle completion, but remote workers @@ -1555,7 +1625,8 @@ mod tests { } if background_enabled { - wait_for_scanner_cycle_after(&cluster, 0).await?; + let baseline_cycle_floor = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)?.as_secs(); + wait_for_scanner_cycle_after(&cluster, baseline_cycle_floor).await?; } cluster.stop_node(1)?; From e7e6556f83489548e4a699e5dff59c813b52cdf2 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 07:04:27 +0800 Subject: [PATCH 007/143] fix(scanner): bound superseded waits for regular maintenance (#8502) Preserve ordinary lifecycle and replication cadence while retaining usage-only convergence backoff, explicit cycle floors and publication fencing. Cover repeated snapshots and the real convergence waiting path. Co-authored-by: heihutu Co-authored-by: zhi22915 --- crates/scanner/src/scanner.rs | 3 +- crates/scanner/src/scanner/activity.rs | 20 ++-- crates/scanner/src/scanner/tests.rs | 118 +++++++++++++++++++- docs/operations/scanner-runtime-controls.md | 2 + 4 files changed, 129 insertions(+), 14 deletions(-) diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 645485e30..58202a53d 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -3181,7 +3181,8 @@ where )); let mut wait_plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for); - let superseded_retry_interval = scanner_superseded_retry_interval(superseded_backoff, &runtime_config); + let superseded_retry_interval = + scanner_superseded_retry_interval(superseded_backoff, &runtime_config, maintenance_features); let deferred_retry_interval = deferred_backoff.retry_interval(runtime_config.cycle_interval); let mut convergence_retry_interval = superseded_retry_interval.or(deferred_retry_interval); if let Some(retry_interval) = convergence_retry_interval { diff --git a/crates/scanner/src/scanner/activity.rs b/crates/scanner/src/scanner/activity.rs index 2b63d115a..efa930a6e 100644 --- a/crates/scanner/src/scanner/activity.rs +++ b/crates/scanner/src/scanner/activity.rs @@ -152,17 +152,19 @@ impl ScannerRetryBackoff { pub(super) fn scanner_superseded_retry_interval( backoff: ScannerRetryBackoff, runtime_config: &ScannerRuntimeConfig, + maintenance_features: ScannerMaintenanceFeatures, ) -> Option { - let retry_interval = backoff.retry_interval(runtime_config.cycle_interval)?; - if runtime_config.cycle_interval_source == ScannerRuntimeConfigSource::Default { - return Some(retry_interval); + let mut retry_interval = backoff.retry_interval(runtime_config.cycle_interval)?; + // Preserve an operator-selected duty-cycle floor during convergence. + if runtime_config.cycle_interval_source != ScannerRuntimeConfigSource::Default { + retry_interval = retry_interval.max(runtime_config.cycle_interval); } - - // An explicit cycle is an operator-selected duty-cycle floor. A - // superseded snapshot keeps its dirty work pending, but retrying that work - // sooner than the configured cadence would turn continuous writes into a - // repeated full-walk loop despite the override. - Some(retry_interval.max(runtime_config.cycle_interval)) + // A complete but non-authoritative walk must not postpone lifecycle or + // replication evaluation beyond its ordinary maintenance cadence. + if maintenance_features.needs_regular_cycle() { + retry_interval = retry_interval.min(runtime_config.cycle_interval.max(Duration::from_secs(1))); + } + Some(retry_interval) } const SCANNER_PUBLICATION_PROOF_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30); diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index b43fc1eb2..8a4f5a7e0 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -8489,7 +8489,7 @@ fn superseded_retry_preserves_explicit_cycle_cadence() { ..Default::default() }; assert_eq!( - scanner_superseded_retry_interval(backoff, &default_config), + scanner_superseded_retry_interval(backoff, &default_config, ScannerMaintenanceFeatures::default()), Some(SCANNER_RETRY_BASE_INTERVAL), "the default adaptive cadence must retain fast convergence" ); @@ -8505,14 +8505,124 @@ fn superseded_retry_preserves_explicit_cycle_cadence() { ..Default::default() }; + for features in [ + ScannerMaintenanceFeatures::default(), + ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }, + ] { + assert_eq!( + scanner_superseded_retry_interval(backoff, &runtime_config, features), + Some(runtime_config.cycle_interval), + "{source:?} cycle cadence must not be shortened by convergence retries" + ); + } + } +} + +#[test] +fn superseded_retry_does_not_delay_required_maintenance() { + let features = ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }; + assert!(features.needs_regular_cycle()); + let config = ScannerRuntimeConfig { + cycle_interval: Duration::from_secs(2), + cycle_interval_source: ScannerRuntimeConfigSource::Env, + ..Default::default() + }; + let mut backoff = ScannerRetryBackoff::default(); + for consecutive_cycles in 1..=12 { + backoff.record_retryable_cycle(true); assert_eq!( - scanner_superseded_retry_interval(backoff, &runtime_config), - Some(runtime_config.cycle_interval), - "{source:?} cycle cadence must not be shortened by convergence retries" + scanner_superseded_retry_interval(backoff, &config, features), + Some(config.cycle_interval), + "superseded cycle {consecutive_cycles} must preserve regular maintenance" ); } } +#[test] +fn superseded_retry_only_caps_regular_maintenance() { + let config = ScannerRuntimeConfig { + cycle_interval: Duration::from_secs(60), + ..Default::default() + }; + let mut backoff = ScannerRetryBackoff::default(); + for _ in 0..6 { + backoff.record_retryable_cycle(true); + } + assert_eq!( + scanner_superseded_retry_interval(backoff, &config, ScannerMaintenanceFeatures::default()), + Some(Duration::from_secs(160)), + ); + for features in [ + ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }, + ScannerMaintenanceFeatures { + replication: true, + ..Default::default() + }, + ScannerMaintenanceFeatures { + inspection_failed: true, + ..Default::default() + }, + ] { + assert_eq!(scanner_superseded_retry_interval(backoff, &config, features), Some(config.cycle_interval)); + assert_eq!(scanner_superseded_retry_interval(ScannerRetryBackoff::default(), &config, features), None); + } +} + +#[tokio::test(start_paused = true)] +#[serial] +async fn superseded_maintenance_wait_keeps_convergence_fencing() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let config = ScannerRuntimeConfig { + cycle_interval: Duration::from_secs(60), + ..Default::default() + }; + let features = ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }; + let mut backoff = ScannerRetryBackoff::default(); + for _ in 0..12 { + backoff.record_retryable_cycle(true); + } + let interval = scanner_superseded_retry_interval(backoff, &config, features); + assert_eq!(interval, Some(config.cycle_interval)); + let generations = ScannerCycleObservedGenerations::for_wait( + &config, + interval, + crate::scanner_io::dirty_usage_generation(), + crate::runtime_config::scanner_runtime_config_generation(), + crate::scanner_io::scanner_maintenance_generation(), + ); + assert!(generations.dirty_usage.is_none()); + assert!(generations.defer_cluster_activity); + let ctx = CancellationToken::new(); + let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); + let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]); + let started = tokio::time::Instant::now(); + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + interval.expect("maintenance must retain its convergence interval"), + Some(Duration::from_secs(1)), + &mut seen, + generations, + || false, + || std::future::ready(Ok(changed.clone())), + ) + .await; + assert_eq!(reason, ScannerCycleWakeReason::Timer); + assert_eq!(started.elapsed(), config.cycle_interval); + assert_eq!(seen, Some(changed)); +} + #[test] fn publication_proof_retry_backoff_reaches_its_short_cap() { for (failures, expected) in [(1, 5), (2, 10), (3, 20), (4, 30), (20, 30)] { diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index 8ca95fafb..faf999409 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -98,6 +98,8 @@ An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle c Under continuous writes, a structurally complete walk can finish as `superseded`. Its durable observation remains separate from quota authority and leaves dirty work pending. Superseded retries may reuse that complete inventory to refresh dirty buckets, even while authority bootstrap is pending; missing or invalid coverage still forces a full walk. Ordinary maintenance timers, Deep scans, active lifecycle/replication and changed maintenance configuration retain full scans. Server info and datausageinfo can display complete observations, while Last Completed Scan advances only after an authoritative completed cycle. A write-free window is therefore not required to display observed counts, but arbitrary continuous writes do not guarantee authoritative completion. Historically dirty buckets remain eligible for rewalk, so narrower retries do not guarantee shorter cycles. +Superseded convergence retries retain exponential backoff for usage-only work. Active lifecycle or replication rules, and uncertain maintenance inspection, cap that retry interval at the ordinary configured cadence so complete observations cannot defer regular maintenance. An explicit cycle remains the duty-cycle floor. Convergence mode still suppresses dirty/activity shortcuts and does not grant quota authority; deferred backend/proof retries and pause catch-up limits retain their existing scheduling. Cycle runtime, pacing, and disk admission can still delay completed maintenance. Sustained supersedes with active maintenance may therefore run more full scans than usage-only convergence; use the existing pacing controls to manage that background I/O. + ## Single-disk clean-idle scheduling An erasure single-disk deployment using the built-in cycle and bitrot defaults automatically backs off repeated clean idle scans instead of walking the same unchanged namespace every minute. Each successful timer-driven cycle that finds no dirty usage or unresolved maintenance work doubles the next interval. The status endpoint reports the effective interval and multiplier. From 9d6f48c3251c12ac40ef21f4e18337f4cc0bce08 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 13 Oct 2026 10:25:12 +0800 Subject: [PATCH 008/143] fix(s3): match AWS duplicate bucket responses (#8503) --- rustfs/src/app/bucket_usecase.rs | 36 ++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 445978a09..69d06c455 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -1257,15 +1257,19 @@ fn build_list_objects_v2_metadata_output( } } -fn create_bucket_exists_response(is_owner: bool) -> S3Result> { - if is_owner { +fn create_bucket_exists_response(is_owner: bool, region: &str) -> S3Result> { + if is_owner && region == RUSTFS_REGION { + // AWS preserves the legacy idempotent response for buckets recreated in + // us-east-1 (the default RustFS region). return Ok(S3Response::new(CreateBucketOutput::default())); } - Err(s3_error!( - BucketAlreadyExists, - "The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again." - )) + let error_code = if is_owner { + S3ErrorCode::BucketAlreadyOwnedByYou + } else { + S3ErrorCode::BucketAlreadyExists + }; + Err(S3Error::with_message(error_code, "The requested bucket already exists.")) } fn resolve_notification_region(global_region: Option, request_region: Option) -> String { @@ -1526,9 +1530,13 @@ impl DefaultBucketUsecase { .await; } Err(StorageError::BucketExists(_)) => { - // Per S3 spec: bucket namespace is global. Owner recreating returns 200 OK; - // non-owner gets 409 BucketAlreadyExists. - let result = create_bucket_exists_response(requester_is_owner); + // Match AWS CreateBucket semantics: an owned bucket is idempotent in + // us-east-1 and returns BucketAlreadyOwnedByYou elsewhere. + let region = self + .global_region() + .map(|region| region.to_string()) + .unwrap_or_else(|| RUSTFS_REGION.to_string()); + let result = create_bucket_exists_response(requester_is_owner, ®ion); let _ = helper.complete(&result); return result; } @@ -4299,16 +4307,22 @@ mod tests { #[test] fn create_bucket_exists_response_returns_ok_for_owner() { - let response = create_bucket_exists_response(true).expect("owner recreate should succeed"); + let response = create_bucket_exists_response(true, RUSTFS_REGION).expect("owner recreate in us-east-1 should succeed"); assert_eq!(response.output.location, None); } #[test] fn create_bucket_exists_response_returns_bucket_already_exists_for_non_owner() { - let err = create_bucket_exists_response(false).expect_err("non-owner recreate should fail"); + let err = create_bucket_exists_response(false, RUSTFS_REGION).expect_err("non-owner recreate should fail"); assert_eq!(err.code(), &S3ErrorCode::BucketAlreadyExists); } + #[test] + fn create_bucket_exists_response_returns_owned_by_you_outside_us_east_1() { + let err = create_bucket_exists_response(true, "eu-west-1").expect_err("owner recreate outside us-east-1 should fail"); + assert_eq!(err.code(), &S3ErrorCode::BucketAlreadyOwnedByYou); + } + #[test] fn build_request_with_req_info_preserves_owner_state() { let input = CreateBucketInput::builder() From 9af2f7c04eea50b33a215aabf8ed6ec192e43d42 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 13 Oct 2026 11:05:21 +0800 Subject: [PATCH 009/143] fix(ilm): make missing lifecycle expiry targets idempotent (#8504) fix(ilm): treat missing expiry targets as success --- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 89 ++++++++++++++++--- 1 file changed, 76 insertions(+), 13 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 06919238f..a98e6e038 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -2417,6 +2417,16 @@ fn is_slow_down(err: &Error) -> bool { matches!(err, Error::SlowDown) } +/// Lifecycle expiry is idempotent when the exact object or version has already +/// disappeared between the scanner listing and the delete admission. +/// +/// Keep this predicate narrower than bucket/volume "not found" helpers: an +/// unavailable bucket or disk is an operational failure and must not be +/// reported as a completed expiry. +fn is_lifecycle_expiry_already_gone(err: &Error) -> bool { + is_err_object_not_found(err) || is_err_version_not_found(err) +} + /// Resolves the expiry worker count from the single documented knob, /// `RUSTFS_MAX_EXPIRY_WORKERS`: a set, parsable, non-zero value wins; /// anything else falls back to `min(cpus, 16)`. The historical @@ -4920,6 +4930,20 @@ async fn expire_transitioned_object_with_lock_lost_signal( //audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn); Ok(dobj) } + Err(err) if is_lifecycle_expiry_already_gone(&err) => { + debug!( + event = EVENT_LIFECYCLE_DELETE_COMPLETED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + bucket = %oi.bucket, + object = %oi.name, + operation = "delete_restored_object", + reason = "already_gone", + "Lifecycle delete target was already absent" + ); + crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await; + Ok(oi.clone()) + } Err(err) => Err(std::io::Error::other(err)), }; } @@ -4930,6 +4954,8 @@ async fn expire_transitioned_object_with_lock_lost_signal( // pointing to a remote tier version that has already been removed. If this // only creates a delete marker, remote cleanup must be driven by persisted // free-version recovery rather than the visible delete result. + // Keep remote transitioned not-found errors visible: local metadata + // absence alone does not prove that the tier object was cleaned up. let dobj = match api.delete_object(&oi.bucket, &oi.name, opts).await { Ok(obj) => obj, Err(e) => { @@ -5802,6 +5828,20 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal( .await { Ok(dobj) => dobj, + Err(err) if is_lifecycle_expiry_already_gone(&err) => { + debug!( + event = EVENT_LIFECYCLE_DELETE_COMPLETED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + bucket = %oi.bucket, + object = %oi.name, + operation = "delete_object", + reason = "already_gone", + "Lifecycle delete target was already absent" + ); + crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await; + return true; + } Err(e) => { error!( event = EVENT_LIFECYCLE_DELETE_FAILED, @@ -6041,18 +6081,19 @@ mod tests { cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at, enqueue_recovered_free_version_with_state, enqueue_transition_for_existing_objects_scoped, enqueue_transition_with_lifecycle, enqueue_transition_with_lifecycle_report, eval_action_from_lifecycle, - get_lock_acquire_timeout, jitter_tier_free_version_recovery_delay, lifecycle_action_blocked_by_replication, - lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object, lifecycle_replication_blocks_action, - lifecycle_rule_has_date_expiration, lifecycle_version_delete_target, manual_transition_duration_elapsed, - manual_transition_has_more_after_limit, manual_transition_recovery_progress_sink, manual_transition_version_marker, - manual_transition_worker_failure_reason, mark_delete_opts_skip_decommissioned_on_remote_success, - merge_stale_multipart_candidate, persist_manual_transition_job_progress_if_owned, - persist_manual_transition_page_checkpoint, recover_manual_transition_job, recover_manual_transition_jobs, - resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, - resolve_transition_worker_count, resolve_transition_workers_absolute_max, run_tier_free_version_recovery_coordinator, - run_tier_free_version_recovery_loop, select_restore_s3_location, set_lifecycle_observability_observer, - set_recovered_free_version_enqueue_observer, should_defer_date_expiry_for_recent_config_update, - transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery, + get_lock_acquire_timeout, is_lifecycle_expiry_already_gone, jitter_tier_free_version_recovery_delay, + lifecycle_action_blocked_by_replication, lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object, + lifecycle_replication_blocks_action, lifecycle_rule_has_date_expiration, lifecycle_version_delete_target, + manual_transition_duration_elapsed, manual_transition_has_more_after_limit, manual_transition_recovery_progress_sink, + manual_transition_version_marker, manual_transition_worker_failure_reason, + mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate, + persist_manual_transition_job_progress_if_owned, persist_manual_transition_page_checkpoint, + recover_manual_transition_job, recover_manual_transition_jobs, resolve_tier_free_version_recovery_enabled, + resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, resolve_transition_worker_count, + resolve_transition_workers_absolute_max, run_tier_free_version_recovery_coordinator, run_tier_free_version_recovery_loop, + select_restore_s3_location, set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer, + should_defer_date_expiry_for_recent_config_update, transitioned_cleanup_tuple, transitioned_object_delete_opts, + wait_for_tier_free_version_recovery, }; #[cfg(feature = "test-util")] use super::{delete_free_version_remote_object_then, encode_dir_object, get_transitioned_object_reader_with_tier_manager}; @@ -10511,6 +10552,26 @@ mod tests { ); } + #[test] + fn lifecycle_expiry_already_gone_only_accepts_object_or_version_absence() { + assert!(is_lifecycle_expiry_already_gone(&Error::FileNotFound)); + assert!(is_lifecycle_expiry_already_gone(&Error::ObjectNotFound( + "bucket".to_string(), + "object".to_string(), + ))); + assert!(is_lifecycle_expiry_already_gone(&Error::FileVersionNotFound)); + assert!(is_lifecycle_expiry_already_gone(&Error::VersionNotFound( + "bucket".to_string(), + "object".to_string(), + "version".to_string(), + ))); + + assert!(!is_lifecycle_expiry_already_gone(&Error::VolumeNotFound)); + assert!(!is_lifecycle_expiry_already_gone(&Error::DiskNotFound)); + assert!(!is_lifecycle_expiry_already_gone(&Error::ErasureReadQuorum)); + assert!(!is_lifecycle_expiry_already_gone(&Error::MethodNotAllowed)); + } + #[tokio::test] async fn manual_transition_counts_already_transitioned_object() { let lc = latest_transition_lifecycle(); @@ -13385,7 +13446,9 @@ mod tests { .bucket_incarnation_id_from_disk(&bucket) .await .expect("bucket incarnation"); - for (attempt_incarnation, expected) in [(Uuid::new_v4(), false), (incarnation, true), (incarnation, false)] { + // Replaying the same scanner snapshot after the delete is an + // already-gone success and must not be reported as a failure. + for (attempt_incarnation, expected) in [(Uuid::new_v4(), false), (incarnation, true), (incarnation, true)] { assert_eq!( super::apply_expiry_on_non_transitioned_objects( ecstore.clone(), From 34803e816501a17d68d5e681ad8c7323444698f5 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 13:07:50 +0800 Subject: [PATCH 010/143] perf(ecstore): isolate cleanup and preserve metadata authority (#8289) * chore: pin s3s to upstream git revision Use the upstream s3s git repository at 50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6 for both s3s and s3s-sigv4. Allow the official s3s git source in cargo-deny so source checks continue to pass. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * perf(ecstore): instrument put object hot path Co-Authored-By: heihutu Co-Authored-By: zhi22915 * perf(ecstore): split cleanup worker wait metrics Co-Authored-By: heihutu Co-Authored-By: zhi22915 * upgrade version * chore(metrics): add PUT post-store stage probes Add lightweight PUT post-store stage probes for issue 2615 validation. * fix cargo deny check warn * upgrade version * perf(ecstore): bound and isolate cleanup execution (#8151) * perf(ecstore): bound and isolate cleanup execution * fix(ecstore): preserve cleanup cancellation and lease release * feat(ecstore): default isolated cleanup to CPUs 1-2 * test(ecstore): decouple budget tests from CPU affinity * upgrade version * perf(ecstore): reuse pinned directory handles for object I/O * feat(observability): add sampled local GetObject stage summaries Add an opt-in GetObject stage aggregation path that works without the OTLP metrics export. When RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED is set, io-metrics keeps per-(path, stage, object class, size bucket) sampled counts and duration totals in a bounded in-process map, and a startup task flushes them periodically to /var/log/rustfs/get-stage-summary.log, tunable via the sample rate and interval env keys. * build(deps): bump workspace dependency releases Patch releases: async-compression 0.4.49, tokio-rustls 0.26.6, xxhash-rust 0.8.19, suppaftp 12.1.1, russh-sftp 3.0.1. Minor release: hotpath 0.27.0. * fix(ecstore): fence distributed metadata cache mutations Add authenticated cluster-wide mutation fences for the opt-in DistErasure GetObject metadata cache. Guard recursive prefix and batch deletes, fail closed on configuration mismatch or uncertain quorum, and cover replay, restart, peer loss, and mixed-configuration behavior. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * refactor(metrics): consolidate PUT observers on improve/base (#8039) * perf(object): instrument storage hot paths Co-Authored-By: heihutu Co-Authored-By: zhi22915 * refactor(metrics): consolidate PUT stage observation Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: zhi22915 Co-authored-by: heihutu * build(deps): bump workspace dependency releases * ci: allowlist rustix OFlags WRONLY in typos config * refactor(storage): route ecstore symbols through storage_api boundaries * perf(ecstore): cache quorum-selected early-stop metadata (#8308) Publish eligible latest-version GET metadata after a validated read quorum when early-stop cancels the remaining fanout. Keep metadata errors, selection quorum, online read quorum, and mutation-generation checks as publication gates. Add slow-peer repeat-hit and concurrent mutation-fence regression tests. Co-authored-by: heihutu Co-authored-by: zhi22915 * chore: merge latest main into improve/base (#8332) * refactor(storage): bind rename rollback to its commit context (#8324) * refactor(storage): separate metadata scheduling from quorum decisions (#8327) * test(storage): verify rejected rollback inspection on real disks (#8331) * test(storage): make commit fault checkpoints independent (#8326) * test(storage): make commit fault checkpoints independent * test(storage): simplify commit checkpoint lookup * fix(listing): count logical prefixes before pagination (#8328) * fix(listing): count logical prefixes before pagination * test(listing): normalize the fixed corpus seed --------- Co-authored-by: Chris Co-authored-by: zhi22915 * perf(ecstore): borrow metadata cache lookup key (#8333) Avoid request-scoped Arc allocations on eligible cache lookups while preserving full key identity. Co-authored-by: zhi22915 * chore: sync improve/base with latest main (#8334) * refactor(storage): bind rename rollback to its commit context (#8324) * refactor(storage): separate metadata scheduling from quorum decisions (#8327) * test(storage): verify rejected rollback inspection on real disks (#8331) * test(storage): make commit fault checkpoints independent (#8326) * test(storage): make commit fault checkpoints independent * test(storage): simplify commit checkpoint lookup * fix(listing): count logical prefixes before pagination (#8328) * fix(listing): count logical prefixes before pagination * test(listing): normalize the fixed corpus seed * fix(scanner): isolate and coalesce dirty usage journals (#8325) * fix(scanner): isolate and coalesce dirty usage journals Bind dirty replay records to a stable node owner so peers cannot overwrite or delete another node's pending journal. Replace the unbounded command queue with bounded coalesced state and revision-aware background saves. Keep legacy records isolated and enforce replay byte limits with conservative whole-bucket or unverified fallback. Add publication-stage diagnostics and storage regressions for ownership, clearing, retry, and restart behavior. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(scanner): collapse confirmed journal clear condition Use a let-chain to retain the generation fence before removing a confirmed bucket and updating its scope byte accounting. Satisfy collapsible_if without changing journal clearing behavior. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --------- Co-authored-by: Chris Co-authored-by: heihutu Co-authored-by: zhi22915 * fix(obs): distinguish metrics by process identity (#8343) Attach process.pid to the OpenTelemetry resource and promote it for Prometheus so counter series from restarted processes remain distinct. Preserve the stable RustFS node and service instance attributes. Co-authored-by: heihutu Co-authored-by: zhi22915 * merge: sync origin/main into improve/base (#8345) * refactor(storage): bind rename rollback to its commit context (#8324) * refactor(storage): separate metadata scheduling from quorum decisions (#8327) * test(storage): verify rejected rollback inspection on real disks (#8331) * test(storage): make commit fault checkpoints independent (#8326) * test(storage): make commit fault checkpoints independent * test(storage): simplify commit checkpoint lookup * fix(listing): count logical prefixes before pagination (#8328) * fix(listing): count logical prefixes before pagination * test(listing): normalize the fixed corpus seed * fix(scanner): isolate and coalesce dirty usage journals (#8325) * fix(scanner): isolate and coalesce dirty usage journals Bind dirty replay records to a stable node owner so peers cannot overwrite or delete another node's pending journal. Replace the unbounded command queue with bounded coalesced state and revision-aware background saves. Keep legacy records isolated and enforce replay byte limits with conservative whole-bucket or unverified fallback. Add publication-stage diagnostics and storage regressions for ownership, clearing, retry, and restart behavior. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(scanner): collapse confirmed journal clear condition Use a let-chain to retain the generation fence before removing a confirmed bucket and updating its scope byte accounting. Satisfy collapsible_if without changing journal clearing behavior. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 * fix(storage): retain typed owners across borrowed writes (#8329) * fix(storage): retain typed owners across borrowed writes * fix(storage): preserve Copy write completion policy * refactor(storage): remove redundant write owner references * fix(ecstore): hide rollback backup residue in prefix listings * test(connect): isolate short HTTP fixtures in nextest * fix(protocols): enforce S3 policies for file protocol operations (#8262) * test(ecstore): enforce main core regression gates (#8338) * test(ecstore): enforce main core regression gates * test(ecstore): pin external shards and completion regressions * fix(test): restore authenticated cluster fault scenarios (#8340) * fix(test): restore authenticated cluster fault scenarios * fix(test): use the E2E storage API boundary * ci: coerce the pool timeout input to a number timeout-minutes requires a numeric value, but inputs.pool_timeout_minutes is a string, so '${{ inputs.pool_timeout_minutes || 240 }}' evaluated to the string '240' and the runner rejected the template: 'An error occurred when attempting to determine the step timeout ... Unexpected value 240' (chain run 36505625402 job at 02:25:52, standalone run 36393453114 at 07:46:37). Both pool runs since #8155 merged have therefore executed without the intended step budget. Wrap the expression in fromJSON and update the policy test that pinned the broken literal. * ci: coerce the pool timeout input to a number timeout-minutes requires a numeric value, but inputs.pool_timeout_minutes is a string, so '${{ inputs.pool_timeout_minutes || 240 }}' evaluated to the string '240' and the runner rejected the template: 'An error occurred when attempting to determine the step timeout ... Unexpected value 240' (chain run 36505625402 job at 02:25:52, standalone run 36393453114 at 07:46:37). Both pool runs since #8155 merged have therefore executed without the intended step budget. Wrap the expression in fromJSON and update the policy test that pinned the broken literal. * fix(ecstore): own listing workers through cancellation (#8342) Consolidate the three page collectors, preserve completed worker failures, and abort nested merges when their owner is dropped. Co-authored-by: heihutu Co-authored-by: zhi22915 * fix(ecstore): route commit sync through guarded helper Use the Linux-capable guarded directory sync entry point for both commit sync sites. This keeps group commit and file-sync admission behavior while ensuring the already-open commit directory remains the sync target. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): expose rename directory guard in production The commit path now acquires the directory guard in production builds, not only tests, so destination metadata reads and durable sync share the guarded context. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: Chris Co-authored-by: heihutu Co-authored-by: zhi22915 Co-authored-by: GatewayJ <835269233@qq.com> Co-authored-by: hector <42570491+majinghe@users.noreply.github.com> * fix(ecstore): remove unsafe metadata cache admission paths Keep distributed GET metadata on quorum reads and quota admission on authoritative storage. Preserve local cache invalidation, remove the experimental RPC surface, and gate diagnostic work. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(ecstore): cover cold degraded mutation admission Check first PUT and DELETE with one peer absent and no SDK retries, then verify that losing write quorum still rejects writes. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): retain guarded Linux destination metadata reads Use the existing commit directory guard when reading old xl.meta on Linux. Cover symlinked destination metadata for inline and non-inline renames without changing the split commit structure. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * chore(test): remove retired cache RPC import Co-Authored-By: heihutu Co-Authored-By: zhi22915 * merge: sync latest main into improve/base (#8349) * test: allow cleanup fixture CI headroom (#8336) * fix(s3): encode multipart listing response keys (#8347) --------- Co-authored-by: cxymds Co-authored-by: Chris Co-authored-by: zhi22915 * merge: sync main auth and scanner updates (#8352) * test: allow cleanup fixture CI headroom (#8336) * fix(s3): encode multipart listing response keys (#8347) * perf(scanner): reduce scan and checkpoint peak allocations (#8346) * perf(scanner): reduce scan and checkpoint peak allocations * test(scanner): isolate partial commit recovery scenarios * fix(auth): return ExpiredToken for retained STS credentials (#8344) * fix(auth): return ExpiredToken for retained STS credentials Validate session identity independently of credential expiry in both IAM rejection and claims validation, and preserve typed S3 errors at the caller. Keep token binding, disabled-account precedence, and service-account behavior. Add exact expiry, IAM, and HTTP/XML regression coverage. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(ecstore): synchronize the metadata slowtail hedge regression Park the slow metadata task at a test-only fault checkpoint and release healthy reads after the fault is active. Assert the spare response and cancelled-task drain without imposing a 300 ms disk/scheduler deadline. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(ilm): synchronize copy compensation transition stages Observe automatic compensation reaching the mock tier PUT checkpoint before waiting for the final metadata commit. Assert forced enqueue failure, automatic backfill, one completed tier upload, and payload integrity. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: heihutu Co-authored-by: zhi22915 --------- Co-authored-by: cxymds Co-authored-by: Chris Co-authored-by: heihutu Co-authored-by: zhi22915 * fix(ecstore): reject overlapping cleanup schedulers Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): reject overlapping cleanup schedulers Co-Authored-By: heihutu Co-Authored-By: zhi22915 * docs(ecstore): explain cleanup scheduler conflict Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(metrics): separate GET request and timeout outcomes (#8354) Co-authored-by: zhi22915 * docs: refresh security advisory lessons (#8350) * fix(heal): bound durable MRF rescheduling (#8351) Reuse stable live-owner proof anchors to avoid duplicate in-flight admission. Pace unverified completions and unavailable attempts with bounded jitter while retaining checkpoints, exact proofs, and generation isolation. Co-authored-by: heihutu Co-authored-by: zhi22915 * fix(rustfs): route recovery errors through S3 gateway Co-Authored-By: heihutu Co-Authored-By: zhi22915 * chore(deps): update workspace dependencies Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(io-metrics): use is_multiple_of for sampling Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): satisfy strict Clippy pagination checks Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(heal): avoid cloning Copy purge identity Co-Authored-By: heihutu Co-Authored-By: zhi22915 * add ignore .mimosa * fix(zip): adapt archive readers to zip 9 APIs Upgrade zip and its required dependencies alongside fallible archive-name handling in log ingestion, report bundles, bucket metadata import, and archive tests. Preserve existing skip and error propagation behavior. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): reuse quorum metadata for body GETs Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ci): align storage test stacks and peer reset classification Apply the existing bounded ECStore stack setup to the affected cross-crate storage fixtures. Validate conditional-only metadata publication separately from full body-layout cache reuse. Classify typed h2 shutdown resets as unreachable while retaining application, authentication, cancellation, and deadline distinctions. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * chore(deps): update workspace dependencies Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(ecstore): preserve omitted-disk GET recovery Co-Authored-By: heihutu Co-Authored-By: zhi22915 * chore: integrate published workspace dependency updates Preserve the concurrent dependency refresh and retain the typed h2 reset classification and bounded storage-test stack setup. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * test(ecstore): preserve snapshot size budget after fallback move * fix(test): use one barrier for compensation COPY Let the forced-enqueue helper own the PUT barrier and compensation assertions. Waiting on an outer barrier overwritten by the helper always timed out after the upload had completed. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * chore(ecstore): apply current lint fixes Co-Authored-By: heihutu Co-Authored-By: zhi22915 * chore(deps): update sha2 to 0.11.1 Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: zhi22915 Co-authored-by: GatewayJ <835269233@qq.com> Co-authored-by: heihutu Co-authored-by: Chris Co-authored-by: hector <42570491+majinghe@users.noreply.github.com> Co-authored-by: cxymds --- .config/nextest.toml | 10 + .docker/observability/prometheus.yml | 1 + .gitignore | 1 + Cargo.lock | 393 ++--- Cargo.toml | 43 +- _typos.toml | 2 + crates/config/src/observability/mod.rs | 14 + .../src/namespace_lock_quorum_test.rs | 85 ++ crates/ecstore/Cargo.toml | 5 +- crates/ecstore/src/bucket/metadata_sys.rs | 77 + .../ecstore/src/bucket/quota/reservation.rs | 97 ++ crates/ecstore/src/cluster/rpc/client.rs | 38 +- .../ecstore/src/cluster/rpc/network_probe.rs | 47 + crates/ecstore/src/cluster/rpc/remote_disk.rs | 62 +- .../src/diagnostics/admin_server_info.rs | 25 +- crates/ecstore/src/disk/cleanup_runtime.rs | 1292 +++++++++++++++++ crates/ecstore/src/disk/disk_store.rs | 379 ++++- crates/ecstore/src/disk/fs.rs | 28 - crates/ecstore/src/disk/local.rs | 521 ++++--- crates/ecstore/src/disk/local/commit.rs | 92 +- .../ecstore/src/disk/local/delete_recovery.rs | 5 +- crates/ecstore/src/disk/mod.rs | 15 + crates/ecstore/src/disk/os.rs | 909 +++++++++++- .../src/set_disk/core/io_primitives.rs | 724 ++++++++- .../src/set_disk/core/metadata_read.rs | 1 + crates/ecstore/src/set_disk/mod.rs | 129 +- crates/ecstore/src/set_disk/ops/object.rs | 1193 +++++++++++++-- crates/ecstore/src/set_disk/read.rs | 298 +++- crates/ecstore/src/store/peer.rs | 28 + .../ecstore/tests/cleanup_isolation_test.rs | 174 +++ crates/ecstore/tests/fd_relative_io_test.rs | 283 ++++ .../tests/metadata_read_lock_reuse_test.rs | 95 ++ crates/ecstore/tests/storage_api.rs | 11 + crates/io-metrics/benches/metrics_pipeline.rs | 10 +- crates/io-metrics/src/lib.rs | 663 ++++++++- crates/log-analyzer/src/ingest/mod.rs | 8 +- crates/obs/src/telemetry/resource.rs | 17 +- deny.toml | 3 - docs/README.md | 3 + docs/operations/put-cleanup-isolation.md | 195 +++ rustfs/src/admin/handlers/bucket_meta.rs | 14 +- rustfs/src/app/object/get.rs | 68 +- rustfs/src/app/object/put.rs | 91 +- rustfs/src/connect/report_bundle.rs | 6 +- rustfs/src/server/hybrid.rs | 70 +- rustfs/src/server/layer.rs | 75 +- rustfs/src/startup_observability.rs | 104 +- rustfs/src/startup_runtime_sources.rs | 8 + rustfs/src/storage/rpc/node_service/disk.rs | 108 +- rustfs/src/storage/timeout_wrapper.rs | 8 +- rustfs/tests/connect_offline_bundle.rs | 2 +- rustfs/tests/connect_perf_drive.rs | 21 +- rustfs/tests/connect_perf_network.rs | 23 +- rustfs/tests/connect_top_net.rs | 9 +- 54 files changed, 7785 insertions(+), 798 deletions(-) create mode 100644 crates/ecstore/src/disk/cleanup_runtime.rs create mode 100644 crates/ecstore/tests/cleanup_isolation_test.rs create mode 100644 crates/ecstore/tests/fd_relative_io_test.rs create mode 100644 docs/operations/put-cleanup-isolation.md diff --git a/.config/nextest.toml b/.config/nextest.toml index 3881b9036..01a1141b5 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -88,6 +88,11 @@ threads-required = "num-test-threads" filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)' setup = 'ecstore-large-stack' +# These cross-crate real-storage fixtures need the same bounded stack as ECStore tests. +[[profile.default.scripts]] +filter = '(package(rustfs) & test(/^(admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))' +setup = 'ecstore-base-stack' + [[profile.default.scripts]] filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' setup = 'ecstore-base-stack' @@ -339,6 +344,11 @@ threads-required = "num-test-threads" filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)' setup = 'ecstore-large-stack' +# These cross-crate real-storage fixtures need the same bounded stack as ECStore tests. +[[profile.ci.scripts]] +filter = '(package(rustfs) & test(/^(admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))' +setup = 'ecstore-base-stack' + [[profile.ci.scripts]] filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))' setup = 'ecstore-base-stack' diff --git a/.docker/observability/prometheus.yml b/.docker/observability/prometheus.yml index 1039e2972..4ac483e6e 100644 --- a/.docker/observability/prometheus.yml +++ b/.docker/observability/prometheus.yml @@ -61,6 +61,7 @@ scrape_configs: otlp: promote_resource_attributes: - service.instance.id + - process.pid - service.name - service.namespace - cloud.availability_zone diff --git a/.gitignore b/.gitignore index 8a26b9e94..1dad3b899 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,4 @@ worktrees/* # insta scratch files; the accepted .snap files ARE the assertions and are committed *.snap.new *.pending-snap +.mimosa \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index dc20abde7..3d4cf7f87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -791,7 +791,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -916,9 +916,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.18.1" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -927,9 +927,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" dependencies = [ "cc", "cmake", @@ -1020,7 +1020,7 @@ dependencies = [ "lru 0.18.5", "percent-encoding", "regex-lite", - "sha2 0.11.0", + "sha2 0.11.1", "tracing", "url", ] @@ -1120,7 +1120,7 @@ dependencies = [ "http 1.5.0", "p256 0.13.2", "percent-encoding", - "sha2 0.11.0", + "sha2 0.11.1", "subtle", "time", "tracing", @@ -1155,7 +1155,7 @@ dependencies = [ "md-5 0.11.0", "pin-project-lite", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "tracing", ] @@ -1518,7 +1518,7 @@ checksum = "144e573728da132683b9488acd528274c790e07fc06ff81ee29f9d8f8b1041e0" dependencies = [ "blowfish", "pbkdf2 0.13.0", - "sha2 0.11.0", + "sha2 0.11.1", ] [[package]] @@ -1561,19 +1561,13 @@ dependencies = [ [[package]] name = "bit-set" -version = "0.8.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" dependencies = [ - "bit-vec 0.8.0", + "bit-vec 0.10.1", ] -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - [[package]] name = "bit-vec" version = "0.9.1" @@ -1583,6 +1577,16 @@ dependencies = [ "serde", ] +[[package]] +name = "bit-vec" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" +dependencies = [ + "borsh", + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -1703,7 +1707,31 @@ dependencies = [ "prettyplease 0.3.0", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", +] + +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.9", ] [[package]] @@ -1950,9 +1978,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" dependencies = [ "find-msvc-tools", "jobserver", @@ -2107,7 +2135,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2199,8 +2227,8 @@ dependencies = [ "liblzma", "lz4", "memchr", - "zstd 0.14.0", - "zstd-safe 8.0.0", + "zstd 0.14.2", + "zstd-safe 8.1.0", ] [[package]] @@ -2304,9 +2332,12 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.4.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +checksum = "9eb6feb175df65d0a3ccd3e73484e940d0ac664e8034f0519fd49a93db3071d9" +dependencies = [ + "rustversion", +] [[package]] name = "convert_case" @@ -2764,7 +2795,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2808,7 +2839,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core 0.24.1", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -3310,7 +3341,7 @@ checksum = "14d6df9763c8fb968b2417d797aef4c55ac2e4a895aaef38c72c16a287aa4a40" dependencies = [ "datafusion-doc", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -3692,7 +3723,7 @@ checksum = "2e2b94854e8576378ccda7c8de8a66ed8b4e8acbd2c50ec3418ea6c8aaf4b567" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -3949,7 +3980,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3970,7 +4001,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -4053,7 +4084,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "suppaftp", "tempfile", "time", @@ -4066,7 +4097,7 @@ dependencies = [ "uuid", "walkdir", "zip", - "zstd 0.14.0", + "zstd 0.14.2", ] [[package]] @@ -4143,7 +4174,7 @@ dependencies = [ "ed25519 3.0.0", "rand_core 0.10.1", "serde", - "sha2 0.11.0", + "sha2 0.11.1", "signature 3.0.0", "subtle", "zeroize", @@ -4299,7 +4330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4581,7 +4612,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -4764,7 +4795,7 @@ dependencies = [ "rustls-pki-types", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "thiserror 2.0.21", "time", "tokio", @@ -4926,7 +4957,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "sha2 0.11.0", + "sha2 0.11.1", "thiserror 2.0.21", "tokio", "tokio-stream", @@ -5704,7 +5735,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" dependencies = [ "io-lifetimes 3.0.1", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -6021,9 +6052,9 @@ checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" [[package]] name = "lapin" -version = "4.12.0" +version = "4.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b86b2a46f42a1d565c7ba327e03725708c1ff212baa4c271cec291e265ae542" +checksum = "1f6674e18d68e810f66ea962f8080f1ded75f813c1cd3fae9ed53bab760dfcfd" dependencies = [ "amq-protocol", "async-rs", @@ -6057,7 +6088,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -6349,11 +6380,11 @@ dependencies = [ [[package]] name = "lzma-rust2" -version = "0.16.5" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca93e534d1142d1d0dcca6d25fe302508a5dfb40b302802904577725ea0b695b" +checksum = "fde178a3caf126c440fa15628147772d8ee590a39477d711353fbe2e58a73a5b" dependencies = [ - "sha2 0.11.0", + "sha2 0.11.1", ] [[package]] @@ -6536,7 +6567,7 @@ dependencies = [ "metrique-core", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -6599,7 +6630,7 @@ dependencies = [ "metrique-core", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure 0.14.0", ] @@ -6963,7 +6994,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7339,9 +7370,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" +checksum = "8c1f1a6ac9c875a4941e4c7c085b61e8f4daa326de03b4701768c3491a6ad9ee" dependencies = [ "futures-core", "futures-sink", @@ -7353,9 +7384,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b304e8d8f11326ba366a99ca5c7aaf7cd397b67268a332437175aeb42a40db" +checksum = "1f01f1c53c46ce80e5fd8cb7a646f13766fd8f315a8a5b75b9bad59a08d2c5c7" dependencies = [ "opentelemetry", "tracing", @@ -7366,9 +7397,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300" +checksum = "302249bd8ee804789faaa7b5b2cbf79bc61cb864714079c644e3828af5b7e811" dependencies = [ "async-trait", "bytes", @@ -7379,9 +7410,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344" +checksum = "0756af24841e367128ef31c831999937b8b5c5197d47914590d8f4e2e88c2e1a" dependencies = [ "flate2", "http 1.5.0", @@ -7397,9 +7428,9 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" +checksum = "d576f5dddc3299341bcd85a04be17b2564844c0c9563214f02d85f2524d1afae" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -7408,15 +7439,15 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d288b1e1bc3590052fbb8f0f635ef248dd9022cf1fbaeb6582b7ad6081a17b87" +checksum = "7a3cf639fefd3e852869114f9925b578da6dd8e99281a2089bb0e773ead46962" [[package]] name = "opentelemetry-stdout" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f71803f481d3f5ccbe477edb42a78f473613712b2327835911a9ea0ab069824" +checksum = "2afbc6661934787ce79bef1e35d0ffe9d62e945f07f664dc1283ce83292b908a" dependencies = [ "chrono", "opentelemetry", @@ -7425,9 +7456,9 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" +checksum = "372a912db2974d1990207006fd0e3aa0ef3fd4ad3be43c65cfaadf2206866061" dependencies = [ "futures-channel", "futures-executor", @@ -7546,7 +7577,7 @@ dependencies = [ "elliptic-curve 0.14.1", "primefield", "primeorder 0.14.0", - "sha2 0.11.0", + "sha2 0.11.1", ] [[package]] @@ -7572,7 +7603,7 @@ dependencies = [ "fiat-crypto 0.3.0", "primefield", "primeorder 0.14.0", - "sha2 0.11.0", + "sha2 0.11.1", ] [[package]] @@ -7586,7 +7617,7 @@ dependencies = [ "elliptic-curve 0.14.1", "primefield", "primeorder 0.14.0", - "sha2 0.11.0", + "sha2 0.11.1", ] [[package]] @@ -7612,7 +7643,7 @@ dependencies = [ "futures", "log", "rand 0.10.3", - "sha2 0.11.0", + "sha2 0.11.1", "thiserror 2.0.21", "tokio", "windows", @@ -8001,7 +8032,7 @@ dependencies = [ "pbkdf2 0.13.0", "rand_core 0.10.1", "scrypt 0.12.0", - "sha2 0.11.0", + "sha2 0.11.1", "spki 0.8.1", ] @@ -8119,7 +8150,7 @@ dependencies = [ "md-5 0.11.0", "memchr", "rand 0.10.3", - "sha2 0.11.0", + "sha2 0.11.1", "stringprep", ] @@ -8204,7 +8235,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" dependencies = [ "proc-macro2", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -8301,16 +8332,17 @@ dependencies = [ [[package]] name = "proptest" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" dependencies = [ "bit-set", - "bit-vec 0.8.0", + "bit-vec 0.10.1", "bitflags 2.13.2", + "chacha20", + "core_detect", "num-traits", - "rand 0.9.5", - "rand_chacha 0.9.0", + "rand 0.10.3", "rand_xorshift", "regex-syntax", "rusty-fork", @@ -8345,7 +8377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -8365,7 +8397,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "petgraph 0.8.3", @@ -8386,7 +8418,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -8399,7 +8431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -8726,7 +8758,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8848,11 +8880,11 @@ dependencies = [ [[package]] name = "rand_xorshift" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" dependencies = [ - "rand_core 0.9.5", + "rand_core 0.10.1", ] [[package]] @@ -9039,7 +9071,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -9257,7 +9289,7 @@ dependencies = [ "pkcs1 0.8.0-rc.4", "pkcs8 0.11.0", "rand_core 0.10.1", - "sha2 0.11.0", + "sha2 0.11.1", "signature 3.0.0", "spki 0.8.1", "zeroize", @@ -9372,7 +9404,7 @@ dependencies = [ "scrypt 0.12.0", "sec1 0.8.1", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "sha3 0.12.0", "signature 3.0.0", "spki 0.8.1", @@ -9462,7 +9494,7 @@ version = "8.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d84e8ba78bd384263e5922f084cbe1b081c3b7e69add59c8fb097b879ba968a" dependencies = [ - "sha2 0.11.0", + "sha2 0.11.1", "walkdir", ] @@ -9615,7 +9647,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "serial_test", - "sha2 0.11.0", + "sha2 0.11.1", "shadow-rs", "snap", "socket2", @@ -9643,7 +9675,7 @@ dependencies = [ "xxhash-rust", "zeroize", "zip", - "zstd 0.14.0", + "zstd 0.14.2", ] [[package]] @@ -9680,7 +9712,7 @@ dependencies = [ "md-5 0.11.0", "pretty_assertions", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "xxhash-rust", ] @@ -9730,7 +9762,7 @@ dependencies = [ "object 0.40.0", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "symbolic-demangle", ] @@ -9744,7 +9776,7 @@ dependencies = [ "rand 0.10.3", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "time", ] @@ -9763,7 +9795,7 @@ dependencies = [ "rsa 0.10.0-rc.18", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "test-case", "thiserror 2.0.21", "time", @@ -9804,6 +9836,7 @@ dependencies = [ "futures-util", "google-cloud-auth", "google-cloud-storage", + "h2", "hex-simd", "hmac 0.13.0", "hostname", @@ -9871,7 +9904,7 @@ dependencies = [ "serde", "serde_json", "serial_test", - "sha2 0.11.0", + "sha2 0.11.1", "shadow-rs", "smallvec", "temp-env", @@ -9940,7 +9973,7 @@ dependencies = [ "rustfs-utils", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tempfile", "thiserror 2.0.21", "time", @@ -9980,7 +10013,7 @@ dependencies = [ "serde", "serde_json", "serial_test", - "sha2 0.11.0", + "sha2 0.11.1", "temp-env", "tempfile", "thiserror 2.0.21", @@ -10039,7 +10072,7 @@ dependencies = [ "serde_with", "serial_test", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "subtle", "temp-env", "tempfile", @@ -10096,7 +10129,7 @@ dependencies = [ "pbkdf2 0.13.0", "rand 0.10.3", "rustls", - "sha2 0.11.0", + "sha2 0.11.1", "socket2", "thiserror 2.0.21", "tracing", @@ -10120,7 +10153,7 @@ dependencies = [ "rand 0.10.3", "rustfs-kafka", "rustls", - "sha2 0.11.0", + "sha2 0.11.1", "tokio", "tokio-rustls", "tracing", @@ -10190,7 +10223,7 @@ dependencies = [ "serde", "serde_ignored", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "subtle", "temp-env", "tempfile", @@ -10267,13 +10300,13 @@ dependencies = [ "regex", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tar", "tempfile", "thiserror 2.0.21", "walkdir", "zip", - "zstd 0.14.0", + "zstd 0.14.2", ] [[package]] @@ -10445,7 +10478,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "url", - "zstd 0.14.0", + "zstd 0.14.2", ] [[package]] @@ -10524,7 +10557,7 @@ dependencies = [ "serde", "serde_json", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "socket2", "subtle", "tempfile", @@ -10618,7 +10651,7 @@ dependencies = [ "serde", "serde_json", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "temp-env", "thiserror 2.0.21", "tokio", @@ -10646,7 +10679,7 @@ dependencies = [ "rustfs-utils", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tokio", "walkdir", ] @@ -10683,7 +10716,7 @@ dependencies = [ "s3s", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "thiserror 2.0.21", "time", "tokio", @@ -10803,7 +10836,7 @@ dependencies = [ "serde", "serde_json", "serial_test", - "sha2 0.11.0", + "sha2 0.11.1", "temp-env", "tempfile", "thiserror 2.0.21", @@ -10909,7 +10942,7 @@ dependencies = [ "s3s", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "snap", "sysinfo", "tempfile", @@ -10955,7 +10988,7 @@ dependencies = [ "rustls-pki-types", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tempfile", "thiserror 2.0.21", "tokio", @@ -11046,7 +11079,7 @@ dependencies = [ "rustix", "serde", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "siphasher", "snap", "temp-env", @@ -11056,7 +11089,7 @@ dependencies = [ "transform-stream", "url", "windows", - "zstd 0.14.0", + "zstd 0.14.2", ] [[package]] @@ -11071,7 +11104,7 @@ dependencies = [ "rustfs-tokio-tar", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tar-codec", "tar-framing", "thiserror 2.0.21", @@ -11131,7 +11164,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11214,7 +11247,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11308,7 +11341,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "smallvec", "std-next", "subtle", @@ -11372,7 +11405,7 @@ dependencies = [ "jiff", "nom 8.0.0", "serde", - "sha2 0.11.0", + "sha2 0.11.1", "smallvec", "std-next", "subtle", @@ -11437,9 +11470,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" dependencies = [ "dyn-clone", "ref-cast", @@ -11473,7 +11506,7 @@ dependencies = [ "cfg-if", "pbkdf2 0.13.0", "salsa20 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", ] [[package]] @@ -11596,7 +11629,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -11611,10 +11644,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ + "foldhash 0.2.0", "indexmap 2.14.2", "itoa", "memchr", @@ -11660,7 +11694,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -11689,7 +11723,7 @@ dependencies = [ "indexmap 2.14.2", "jiff", "schemars 0.9.0", - "schemars 1.2.2", + "schemars 1.2.3", "serde_core", "serde_json", "serde_with_macros", @@ -11705,7 +11739,7 @@ dependencies = [ "darling 0.24.1", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -11740,7 +11774,7 @@ checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -11784,9 +11818,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", "cpufeatures 0.3.1", @@ -11987,9 +12021,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.16.2" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" dependencies = [ "serde", ] @@ -12156,7 +12190,7 @@ dependencies = [ "sec1 0.8.1", "serde", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "signature 3.0.0", "ssh-cipher", "ssh-encoding", @@ -12259,9 +12293,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "suppaftp" -version = "12.1.1" +version = "12.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b161d71217da0cc91bd09c8f98ac5ad2a5e5b2735c6dfe1c6cb08b4471612f35" +checksum = "a560e9f42cc5bd4a92295eca3c7140d51eb685d688087f547f27e6426fbe97da" dependencies = [ "async-trait", "chrono", @@ -12327,9 +12361,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.6" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -12376,7 +12410,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -12485,10 +12519,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -12518,7 +12552,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -12529,7 +12563,7 @@ checksum = "f4ce35ece947bccac166e1ded639133827a953f69a48ba134d17f91108333082" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "test-case-core", ] @@ -12570,7 +12604,7 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -12707,7 +12741,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -12743,7 +12777,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c2ad44aa0ae96db89c4742212ed41645b2f597311ff6e1945542a4d9fadc2fb" dependencies = [ "rustls", - "sha2 0.11.0", + "sha2 0.11.1", "tokio", "tokio-postgres", "tokio-rustls", @@ -12785,9 +12819,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" dependencies = [ "bytes", "futures-core", @@ -12823,18 +12857,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.1.1+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.25.15+spec-1.1.0" +version = "0.25.17+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" dependencies = [ "indexmap 2.14.2", "toml_datetime", @@ -12844,9 +12878,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.3+spec-1.1.0" +version = "1.1.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" dependencies = [ "winnow", ] @@ -13223,9 +13257,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "unicode-normalization" @@ -13315,9 +13349,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.27.0" +version = "1.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -13468,7 +13502,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "wasm-bindgen-shared", ] @@ -13582,7 +13616,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -14043,24 +14077,24 @@ checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure 0.14.0", ] [[package]] name = "zerocopy" -version = "0.8.60" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6400fc4bb7426f6faac3fb07566122bb18622e3022bd2c36426a92ed489b1dbe" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.60" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99c27cc7525bad4df59a45e073c45b2c2aa820efb588234cc23a547ec69a34db" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" dependencies = [ "proc-macro2", "quote", @@ -14084,7 +14118,7 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure 0.14.0", ] @@ -14138,14 +14172,14 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "zip" -version = "8.6.0" +version = "9.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +checksum = "9e2fc56a6707efb066c40aead2409ebb1224e6dde24a074e9fbc78ced500507b" dependencies = [ "aes 0.9.3", "bzip2", @@ -14158,6 +14192,7 @@ dependencies = [ "indexmap 2.14.2", "lzma-rust2", "memchr", + "nix 0.31.3", "pbkdf2 0.13.0", "ppmd-rust", "sha1 0.11.0", @@ -14165,7 +14200,7 @@ dependencies = [ "typed-path", "zeroize", "zopfli", - "zstd 0.13.3", + "zstd 0.14.2", ] [[package]] @@ -14182,9 +14217,9 @@ checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zopfli" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +checksum = "aaf7fc5d30c28483d93805c4a5e12b05bbb52407fa67c5f8bd552374cd01fb11" dependencies = [ "bumpalo", "crc32fast", @@ -14203,11 +14238,11 @@ dependencies = [ [[package]] name = "zstd" -version = "0.14.0" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e" +checksum = "057cfd910cfac363a0ada849592624b4c9ff2e10bef504c3433810d78ed96f93" dependencies = [ - "zstd-safe 8.0.0", + "zstd-safe 8.1.0", ] [[package]] @@ -14221,18 +14256,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "8.0.0" +version = "8.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab" +checksum = "bdd44c6a7284e91f3717755b24315a302edd9153a01f753c3cba3d765e8eafac" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.1.0+zstd.1.5.7" +version = "2.1.1+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +checksum = "aeec9eaf2dffbbd09201e23bd0ffcbaa33bb8e9266a10734fd7ed90a85eca078" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 6a5791efb..a0b447ac7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -156,8 +156,9 @@ futures-lite = "2.6.1" futures-util = "0.3.34" pollster = "1.0.1" pulsar = { default-features = false, version = "6.9.0" } -lapin = { default-features = false, version = "4.12.0" } +lapin = { default-features = false, version = "4.12.1" } hyper = { version = "1.12.0" } +h2 = { version = "0.4.20" } hyper-rustls = { default-features = false, version = "0.27.10" } hyper-util = { version = "0.1.21" } http = "1.5.0" @@ -172,7 +173,7 @@ tokio = { version = "1.53.2" } tokio-rustls = { default-features = false, version = "0.26.6" } tokio-stream = { version = "0.1.19" } tokio-test = "0.4.6" -tokio-util = { version = "0.7.19" } +tokio-util = { version = "0.7.20" } tonic = { version = "0.14.6" } tonic-prost = { version = "0.14.6" } tonic-prost-build = { version = "0.14.6" } @@ -180,7 +181,7 @@ tower = { version = "0.5.3" } tower-http = { version = "0.7.1" } h3 = "0.0.8" h3-quinn = "0.0.10" -quinn = { version = "0.11.11", default-features = false, features = [ +quinn = { version = "0.11.12", default-features = false, features = [ "futures-io", "runtime-tokio", "rustls-aws-lc-rs", @@ -200,7 +201,7 @@ rmp-serde = { version = "1.3.1" } serde = { version = "1.0.229" } serde_ignored = { version = "0.1" } serde_with = { version = "3", default-features = false, features = ["macros", "std"] } -serde_json = { version = "1.0.151" } +serde_json = { version = "1.0.154" } serde_urlencoded = "0.7.1" # Cryptography and Security @@ -224,7 +225,7 @@ rustls-native-certs = "0.8" rustls-pki-types = "1.15.1" x509-parser = "0.18.1" sha1 = "0.11.0" -sha2 = "0.11.0" +sha2 = "0.11.1" subtle = "2.6" zeroize = { version = "1.9.1" } proptest = "1" @@ -268,7 +269,7 @@ crossbeam-queue = "0.3.14" crossbeam-channel = "0.5.17" crossbeam-deque = "0.8.8" crossbeam-utils = "0.8.23" -datafusion = { default-features = false, version = "55.1.0" } +datafusion = { default-features = false, version = "55.2.0" } derive_builder = "0.20.2" ed25519-dalek = "3.0.0" enumset = "1.1.14" @@ -319,14 +320,14 @@ rumqttc = { package = "rumqttc-next", version = "0.34.0" } redis = { version = "1.7.1" } rustify = { version = "0.7", default-features = false } rustix = { version = "1.1.5" } -rust-embed = { version = "8.12.0" } +rust-embed = { version = "8.13.0" } rustc-hash = { version = "2.1.3" } s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] } s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" } serial_test = "4.0.1" shadow-rs = { default-features = false, version = "2.0.0" } siphasher = "1.0.4" -smallvec = { version = "1.16.2" } +smallvec = { version = "1.16.3" } compact_str = "0.10.0" snap = "1.1.2" starshard = { version = "2.4.0" } @@ -340,12 +341,12 @@ tracing = { version = "0.1.44" } tracing-appender = "0.2.5" tracing-core = "0.1.36" tracing-error = "0.2.1" -tracing-opentelemetry = { version = "0.34" } +tracing-opentelemetry = { version = "0.34.0" } tracing-subscriber = { version = "0.3.23" } transform-stream = "0.3.1" url = "2.5.8" urlencoding = "2.1.3" -uuid = { version = "1.27.0" } +uuid = { version = "1.28.0" } vaultrs = { version = "0.8.0" } tar = "0.4.46" walkdir = "2.5.0" @@ -353,26 +354,26 @@ winapi-util = "0.1.11" windows = { version = "0.62.2" } windows-sys = "0.61.2" xxhash-rust = { version = "0.8.19" } -zip = "8.6.0" -zstd = "0.14.0" +zip = "9.0.2" +zstd = "0.14.2" # Observability and Metrics metrics = "0.24.6" metrics-util = "0.20" dial9 = { version = "0.5.3", default-features = false } -opentelemetry = { version = "0.33.0" } -opentelemetry-appender-tracing = { version = "0.33.0" } -opentelemetry-otlp = { version = "0.33.0" } -opentelemetry-proto = { version = "0.33.0", default-features = false, features = ["metrics", "gen-tonic-messages"] } -opentelemetry_sdk = { version = "0.33.0" } -opentelemetry-semantic-conventions = { version = "0.33.0" } -opentelemetry-stdout = { version = "0.33.0" } +opentelemetry = { version = "0.33.1" } +opentelemetry-appender-tracing = { version = "0.33.1" } +opentelemetry-otlp = { version = "0.33.1" } +opentelemetry-proto = { version = "0.33.1", default-features = false, features = ["metrics", "gen-tonic-messages"] } +opentelemetry_sdk = { version = "0.33.1" } +opentelemetry-semantic-conventions = { version = "0.33.1" } +opentelemetry-stdout = { version = "0.33.1" } pyroscope = { version = "2.1.1" } # FTP and SFTP libunftp = { version = "0.23.1" } unftp-core = "0.1.0" -suppaftp = { version = "12.1.1" } +suppaftp = { version = "12.2.0" } rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] } russh = { version = "0.64.1" } russh-sftp = "3.0.1" @@ -383,7 +384,7 @@ dav-server = "0.11.0" # Performance Analysis and Memory Profiling rustfs-mimalloc = { version = "0.5.6" } # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved. -hotpath = { version = "0.28.5", default-features = false } +hotpath = { version = "0.28.6", default-features = false } # Snapshot testing for output format regression detection insta = { version = "1.49" } diff --git a/_typos.toml b/_typos.toml index 7ff843137..e11c3c036 100644 --- a/_typos.toml +++ b/_typos.toml @@ -51,6 +51,8 @@ unparseable = "unparseable" # Disaster-recovery objectives: recovery time and recovery point. RTO = "RTO" rto = "rto" +# rustix OFlags variant (POSIX O_WRONLY equivalent). +WRONLY = "WRONLY" [files] extend-exclude = [] diff --git a/crates/config/src/observability/mod.rs b/crates/config/src/observability/mod.rs index 9a1acc6c4..ca001fc8b 100644 --- a/crates/config/src/observability/mod.rs +++ b/crates/config/src/observability/mod.rs @@ -51,6 +51,11 @@ pub const ENV_OBS_PROFILING_EXPORT_ENABLED: &str = "RUSTFS_OBS_PROFILING_EXPORT_ /// PUT records multiple timers and histograms when attribution is active. pub const ENV_OBS_PUT_STAGE_METRICS_ENABLED: &str = "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED"; +/// Enables sampled in-process GetObject stage summaries without OTLP histograms. +pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED"; +pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE"; +pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS"; + pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL"; pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED"; pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY"; @@ -159,6 +164,15 @@ mod tests { assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED"); assert_eq!(ENV_OBS_PROFILING_EXPORT_ENABLED, "RUSTFS_OBS_PROFILING_EXPORT_ENABLED"); assert_eq!(ENV_OBS_PUT_STAGE_METRICS_ENABLED, "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED"); + assert_eq!(ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED, "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED"); + assert_eq!( + ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE, + "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE" + ); + assert_eq!( + ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS, + "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS" + ); // Test log cleanup related env keys assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES"); assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES"); diff --git a/crates/e2e_test/src/namespace_lock_quorum_test.rs b/crates/e2e_test/src/namespace_lock_quorum_test.rs index 1edf47c2b..7f9e8c760 100644 --- a/crates/e2e_test/src/namespace_lock_quorum_test.rs +++ b/crates/e2e_test/src/namespace_lock_quorum_test.rs @@ -195,6 +195,91 @@ async fn wait_for_quorum_read_admission(clients: &[Client], bucket: &str) -> Tes } } +#[tokio::test] +async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> TestResult { + crate::common::init_logging(); + let mut cluster = RustFSTestClusterEnvironment::new(4).await?; + cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2"); + cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false"); + cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false"); + cluster.set_env("RUSTFS_GET_OBJECT_METADATA_CACHE_DISTRIBUTED_ENABLE", "false"); + cluster.start().await?; + let clients = cluster + .create_all_clients()? + .into_iter() + .map(|client| { + Client::from_conf( + client + .config() + .to_builder() + .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1)) + .build(), + ) + }) + .collect::>(); + let bucket = "cold-degraded-mutations"; + clients[0].create_bucket().bucket(bucket).send().await?; + // No object mutation may warm a process-wide configuration probe before + // the missing peer is introduced. Three shards still satisfy EC:2 writes. + cluster.stop_node(3)?; + let admission_deadline = Instant::now() + Duration::from_secs(30); + for client in &clients[..3] { + loop { + match client + .get_object() + .bucket(bucket) + .key("never-created-readiness-probe") + .send() + .await + { + Err(error) if error.as_service_error().and_then(|error| error.meta().code()) == Some("NoSuchKey") => break, + Err(error) + if error.raw_response().is_some_and(|response| response.status().as_u16() == 503) + && Instant::now() < admission_deadline => + { + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err(error) => return Err(error.into()), + Ok(_) => panic!("a cold readiness probe must not create an object"), + } + } + } + let body = b"first write while one peer is offline"; + clients[0] + .put_object() + .bucket(bucket) + .key("object") + .body(Bytes::from_static(body).into()) + .send() + .await?; + for client in &clients[..3] { + assert_quorum_object_body(client, bucket, "object", body).await?; + } + // This coordinator has not previously committed an object mutation either. + clients[1].delete_object().bucket(bucket).key("object").send().await?; + let error = clients[2] + .get_object() + .bucket(bucket) + .key("object") + .send() + .await + .expect_err("acknowledged deletion must be visible at another entrypoint"); + assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("NoSuchKey")); + + cluster.stop_node(2)?; + let error = clients[0] + .put_object() + .bucket(bucket) + .key("below-quorum") + .body(Bytes::from_static(body).into()) + .send() + .await + .expect_err("removing the cache gate must not weaken storage write quorum"); + assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable")); + cluster.stop(); + Ok(()) +} + #[tokio::test] async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult { crate::common::init_logging(); diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 5a41e01f4..2eee87cea 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -183,6 +183,7 @@ sha2 = { workspace = true } hex-simd = { workspace = true } tempfile.workspace = true hyper = { workspace = true, features = ["http2", "http1", "server"] } +h2 = { workspace = true } hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] } hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs"] } hostname.workspace = true @@ -200,8 +201,8 @@ md-5.workspace = true memmap2 = { workspace = true } libc.workspace = true # "process" adds getrlimit for the io_uring fd-cache RLIMIT_NOFILE gate -# (backlog#1178); "fs" comes from the workspace default. -rustix = { workspace = true, features = ["process", "fs"] } +# (backlog#1178); "thread" provides cleanup affinity; "fs" is a workspace default. +rustix = { workspace = true, features = ["process", "fs", "thread"] } rustfs-madmin.workspace = true reqwest = { workspace = true } aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 1dc3e43e1..187356d4f 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -178,6 +178,13 @@ pub(crate) fn object_lock_config_state_from_authoritative_metadata(bm: &BucketMe Ok(ObjectLockConfigState::ConfirmedAbsent) } +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct CachedQuotaMetadataSnapshot { + pub(crate) quota: Option, + pub(crate) bucket_incarnation: Uuid, + pub(crate) quota_revision: OffsetDateTime, +} + /// Convert the persisted serving-layer configuration into the storage-level /// [`DefaultRetention`](crate::bucket::object_lock::types::DefaultRetention) /// the WORM evaluation code consumes (rustfs/backlog#1842). A rule without a @@ -1450,6 +1457,9 @@ fn quota_config_and_incarnation_from_authority( BucketMetadataAuthority::Authoritative(metadata) if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() => { + if !metadata.quota_config_json.is_empty() && metadata.quota_config.is_none() { + return Err(Error::other("persisted bucket quota configuration is invalid")); + } Ok(( metadata.quota_config.clone(), metadata.bucket_incarnation_id, @@ -1464,6 +1474,15 @@ fn quota_config_and_incarnation_from_authority( } } +pub(crate) async fn get_cached_quota_config_and_incarnation_in( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, +) -> Result> { + let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?; + let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone(); + bucket_meta_sys.cached_quota_config_and_incarnation(bucket).await +} + pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> { let bucket_meta_sys_lock = get_bucket_metadata_sys()?; let bucket_meta_sys = bucket_meta_sys_lock.read().await; @@ -2979,6 +2998,27 @@ impl BucketMetadataSys { } } + async fn cached_quota_config_and_incarnation(&self, bucket: &str) -> Result> { + let metadata = { + let metadata_map = self.metadata_map.read().await; + metadata_map.get(bucket).cloned() + }; + let Some(metadata) = metadata else { + return Ok(None); + }; + if !metadata.bucket_incarnation_sidecar || metadata.bucket_incarnation_id.is_nil() { + return Ok(None); + } + if !metadata.quota_config_json.is_empty() && metadata.quota_config.is_none() { + return Err(Error::other("persisted bucket quota configuration is invalid")); + } + Ok(Some(CachedQuotaMetadataSnapshot { + quota: metadata.quota_config.clone(), + bucket_incarnation: metadata.bucket_incarnation_id, + quota_revision: metadata.quota_config_updated_at, + })) + } + pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> { let (bm, _) = self.get_config(bucket).await?; @@ -4883,6 +4923,43 @@ mod tests { ); } + #[tokio::test] + #[serial] + async fn quota_admission_rejects_stale_cached_no_quota() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let sys = bucket_metadata_sys_of(&store.ctx).expect("scoped metadata system"); + let bucket = "stale-no-quota-admission"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + let (stale, _) = sys.read().await.get_config(bucket).await.expect("cache initial metadata"); + store + .update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec()) + .await + .expect("publish quota to authoritative storage"); + // Model a peer that has not received the newer configuration snapshot. + sys.read().await.set(bucket.to_owned(), stale).await; + assert!( + get_cached_quota_config_and_incarnation_in(&store.ctx, bucket) + .await + .expect("read stale cache") + .expect("cached bucket identity") + .quota + .is_none() + ); + + temp_env::async_with_vars([("RUSTFS_QUOTA_BEGIN_SAFE_NO_QUOTA_CACHE_FAST_PATH", Some("true"))], async { + let error = crate::bucket::quota::reservation::begin(&store.ctx, bucket, "object", &ObjectOptions::default(), 0, 0) + .await + .err() + .expect("persisted quota must reject admission without its snapshot"); + assert!(matches!(error, Error::PartMissingOrCorrupt), "{error:?}"); + }) + .await; + } + #[tokio::test] #[serial] async fn peer_reload_clears_negative_cache_after_remote_bucket_creation() { diff --git a/crates/ecstore/src/bucket/quota/reservation.rs b/crates/ecstore/src/bucket/quota/reservation.rs index 59c51c301..b269813f8 100644 --- a/crates/ecstore/src/bucket/quota/reservation.rs +++ b/crates/ecstore/src/bucket/quota/reservation.rs @@ -850,6 +850,11 @@ pub(crate) async fn begin( let snapshot_admission = opts.quota_admission; let data_movement = opts.data_movement; if crate::bucket::utils::is_meta_bucketname(bucket) { + let meta_bucket_fast_path_started = rustfs_io_metrics::put_stage_timer(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH, + meta_bucket_fast_path_started, + ); return Ok(QuotaContext { store: None, bucket: bucket.to_string(), @@ -906,9 +911,63 @@ pub(crate) async fn begin( }); } + let metadata_lock_started = rustfs_io_metrics::put_stage_timer(); let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK, + metadata_lock_started, + ); + let cache_lookup_started = rustfs_io_metrics::put_stage_timer(); + let cached_quota_snapshot = if rustfs_io_metrics::put_stage_metrics_enabled() { + match metadata_sys::get_cached_quota_config_and_incarnation_in(ctx, bucket).await { + Ok(Some(snapshot)) => { + rustfs_io_metrics::record_put_object_quota_cache_decision( + "candidate", + quota_cache_reason(snapshot.quota.as_ref()), + ); + Some(snapshot) + } + Ok(None) => { + rustfs_io_metrics::record_put_object_quota_cache_decision("miss", "no_authoritative_cache"); + None + } + Err(_) => { + rustfs_io_metrics::record_put_object_quota_cache_decision("miss", "invalid_cached_quota"); + None + } + } + } else { + None + }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP, + cache_lookup_started, + ); + let config_read_started = rustfs_io_metrics::put_stage_timer(); let (quota, bucket_incarnation, quota_revision) = metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(ctx, bucket, opts, &metadata_guard).await?; + if let Some(cached) = cached_quota_snapshot.as_ref() { + let reason = quota_cache_reason(quota.as_ref()); + if cached.quota == quota && cached.bucket_incarnation == bucket_incarnation && cached.quota_revision == quota_revision { + rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_match", reason); + } else { + rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch", reason); + if cached.quota != quota { + rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "quota"); + } + if cached.bucket_incarnation != bucket_incarnation { + rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "bucket_incarnation"); + } + if cached.quota_revision != quota_revision { + rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "quota_revision"); + } + } + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ, + config_read_started, + ); + let lock_lost_check_started = rustfs_io_metrics::put_stage_timer(); if metadata_guard.is_lock_lost() { return Err(StorageError::NamespaceLockQuorumUnavailable { mode: "quota_config", @@ -918,6 +977,11 @@ pub(crate) async fn begin( achieved: 0, }); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK, + lock_lost_check_started, + ); + let policy_evaluate_started = rustfs_io_metrics::put_stage_timer(); if quota .as_ref() .is_some_and(|quota| quota.has_unsupported_reservation_protocol()) @@ -926,6 +990,11 @@ pub(crate) async fn begin( return Err(StorageError::PartMissingOrCorrupt); } let durable_quota = quota.as_ref().filter(|quota| quota.uses_durable_reservations()); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE, + policy_evaluate_started, + ); + let capability_proof_started = rustfs_io_metrics::put_stage_timer(); let capability_proof = if durable_quota.is_some() { Some( crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof() @@ -934,7 +1003,12 @@ pub(crate) async fn begin( } else { None }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF, + capability_proof_started, + ); let durable_quota_limit = durable_quota.and_then(|quota| quota.quota); + let snapshot_admission_started = rustfs_io_metrics::put_stage_timer(); let reservation_protocol = durable_quota.and_then(|quota| quota.reservation_protocol); let snapshot_admission = match quota.as_ref().filter(|quota| !quota.uses_durable_reservations()) { Some(quota) => match (quota.quota, snapshot_admission) { @@ -954,6 +1028,10 @@ pub(crate) async fn begin( }, None => None, }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION, + snapshot_admission_started, + ); let legacy_data_movement = durable_quota_limit.is_none() && quota.as_ref().and_then(|quota| quota.quota).is_some() && snapshot_admission.is_none() @@ -965,11 +1043,21 @@ pub(crate) async fn begin( .then(|| quota.as_ref().and_then(|quota| quota.quota)) .flatten() }); + let object_store_lookup_started = rustfs_io_metrics::put_stage_timer(); let store = if durable_quota_limit.is_some() { Some(metadata_sys::object_store_in(ctx).await?) } else { None }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP, + object_store_lookup_started, + ); + let context_build_started = rustfs_io_metrics::put_stage_timer(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD, + context_build_started, + ); Ok(QuotaContext { store, bucket: bucket.to_string(), @@ -988,6 +1076,15 @@ pub(crate) async fn begin( }) } +fn quota_cache_reason(quota: Option<&crate::bucket::quota::BucketQuota>) -> &'static str { + match quota { + None => "no_quota", + Some(quota) if quota.uses_durable_reservations() => "durable_quota", + Some(quota) if quota.quota.is_some() => "snapshot_quota", + Some(_) => "quota_without_limit", + } +} + fn quota_capability_error(bucket: &str, object: &str) -> StorageError { StorageError::NamespaceLockQuorumUnavailable { mode: "quota_capability", diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index ca781d6be..c8170b613 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -353,6 +353,13 @@ fn apply_peer_replay_response( } } +fn rename_data_grpc_stage(path: &str) -> bool { + matches!( + path, + "/node_service.NodeService/RenameData" | "/node_service.NodeService/RenameDataAtIncarnation" + ) +} + impl Service> for ReplayScopeChannel where S: Service, Response = HttpResponse>, @@ -370,6 +377,12 @@ where } fn call(&mut self, mut request: HttpRequest) -> Self::Future { + let observe_rename_data = rustfs_io_metrics::put_stage_metrics_enabled() && rename_data_grpc_stage(request.uri().path()); + let request_scope_started = if observe_rename_data { + rustfs_io_metrics::put_stage_timer() + } else { + None + }; let authenticated = self.audience.as_ref().is_some_and(|_| { request .headers() @@ -404,11 +417,30 @@ where } } } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE, + request_scope_started, + ); let audience = self.audience.clone(); let future = self.inner.call(request); Box::pin(async move { - let response = future.await?; + let response = if observe_rename_data { + rustfs_io_metrics::observe_put_stage_future( + future, + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL, + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT, + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY, + ) + .await? + } else { + future.await? + }; + let replay_response_started = if observe_rename_data { + rustfs_io_metrics::put_stage_timer() + } else { + None + }; if let (Some(audience), Some(challenge)) = (audience, challenge) { let response_state = verify_tonic_peer_replay_capabilities_response(&audience, challenge, response.headers()); if let Err(error) = &response_state @@ -426,6 +458,10 @@ where } apply_peer_replay_response(audience, sent_state, response_state); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE, + replay_response_started, + ); Ok(response) }) } diff --git a/crates/ecstore/src/cluster/rpc/network_probe.rs b/crates/ecstore/src/cluster/rpc/network_probe.rs index 765cc7322..db2221cfb 100644 --- a/crates/ecstore/src/cluster/rpc/network_probe.rs +++ b/crates/ecstore/src/cluster/rpc/network_probe.rs @@ -325,6 +325,24 @@ fn map_status(status: tonic::Status) -> NetworkPeerProbeError { match status.code() { Code::Unavailable | Code::Unknown => NetworkPeerProbeError::Unreachable, Code::DeadlineExceeded => NetworkPeerProbeError::TimedOut, + Code::Cancelled | Code::Internal => { + let mut source = std::error::Error::source(&status); + while let Some(error) = source { + if error + .downcast_ref::() + .is_some_and(|error| matches!(error.reason(), Some(h2::Reason::CANCEL | h2::Reason::NO_ERROR))) + { + return NetworkPeerProbeError::Unreachable; + } + source = if let Some(error) = error.downcast_ref::() { + // io::Error::source skips its immediate custom error. + error.get_ref().map(|inner| inner as &(dyn std::error::Error + 'static)) + } else { + error.source() + }; + } + NetworkPeerProbeError::ProtocolFailure + } _ => NetworkPeerProbeError::ProtocolFailure, } } @@ -335,6 +353,35 @@ mod tests { use crate::layout::endpoint::Endpoint; use crate::layout::endpoints::{Endpoints, PoolEndpoints}; + #[test] + fn peer_transport_resets_are_unreachable_without_reclassifying_application_errors() { + for reason in [h2::Reason::CANCEL, h2::Reason::NO_ERROR] { + let status = tonic::Status::from_error(Box::new(h2::Error::from(reason))); + assert!(matches!(status.code(), Code::Cancelled | Code::Internal)); + let code = status.code(); + assert_eq!(map_status(status), NetworkPeerProbeError::Unreachable); + let mut wrapped = tonic::Status::new(code, "wrapped transport reset"); + wrapped.set_source(Arc::new(std::io::Error::other(h2::Error::from(reason)))); + assert_eq!(map_status(wrapped), NetworkPeerProbeError::Unreachable); + } + for reason in [h2::Reason::PROTOCOL_ERROR, h2::Reason::COMPRESSION_ERROR] { + let status = tonic::Status::from_error(Box::new(h2::Error::from(reason))); + assert_eq!(map_status(status), NetworkPeerProbeError::ProtocolFailure); + } + for status in [ + tonic::Status::cancelled("peer cancelled the application request"), + tonic::Status::internal("peer rejected the probe"), + tonic::Status::unauthenticated("authentication rejected"), + tonic::Status::permission_denied("authorization rejected"), + ] { + assert_eq!(map_status(status), NetworkPeerProbeError::ProtocolFailure); + } + assert_eq!( + map_status(tonic::Status::deadline_exceeded("probe deadline elapsed")), + NetworkPeerProbeError::TimedOut + ); + } + fn pacing(duration: Duration) -> Arc { NetworkPeerProbeClient::from_endpoint_pools(&topology()) .with_diagnostic_pacing(Instant::now(), duration) diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index e067609a0..b350fd497 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -2279,6 +2279,7 @@ impl RemoteDisk { self.execute_with_timeout_for_op( "rename_data", || async { + let prepare_started = rustfs_io_metrics::put_stage_timer(); let file_info = compat_json(fi)?; let file_info_bin = encode_file_info_msgpack(fi)?; let mut client = self.get_client().await?; @@ -2306,24 +2307,67 @@ impl RemoteDisk { } else { attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?; } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE, + prepare_started, + ); - let response = if incarnation_bound { - // Older peers return Unimplemented before mutation; never downgrade. - client.rename_data_at_incarnation(request).await? - } else { - client.rename_data(request).await? - } - .into_inner(); + let rpc_started = rustfs_io_metrics::put_stage_timer(); + let response = rustfs_io_metrics::observe_put_stage_future( + async { + if incarnation_bound { + // Older peers return Unimplemented before mutation; never downgrade. + client.rename_data_at_incarnation(request).await + } else { + client.rename_data(request).await + } + }, + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT, + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT, + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY, + ) + .await?; + let response_process_started = rustfs_io_metrics::put_stage_timer(); + let response_headers_started = rustfs_io_metrics::put_stage_timer(); + let _ = response.metadata().len(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS, + response_headers_started, + ); + let into_inner_started = rustfs_io_metrics::put_stage_timer(); + let response = response.into_inner(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER, + into_inner_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC, + rpc_started, + ); + let success_check_started = rustfs_io_metrics::put_stage_timer(); if !response.success { return Err(response.error.unwrap_or_default().into()); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK, + success_check_started, + ); + let decode_started = rustfs_io_metrics::put_stage_timer(); let rename_data_resp = decode_msgpack_or_json::( &response.rename_data_resp_bin, &response.rename_data_resp, "RenameDataResp", )?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE, + decode_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS, + response_process_started, + ); Ok(rename_data_resp) }, @@ -5116,6 +5160,8 @@ mod tests { cleanup_data_dir: Some(Uuid::new_v4()), sign: Some(vec![0x14, 0x35]), old_current_size: Some(crate::disk::OldCurrentSize::Present(64 * 1024)), + old_current_source_checked: false, + old_current_source: None, }; let json = serde_json::to_string(&response).expect("legacy rename_data JSON response should encode"); let decode_before = rustfs_io_metrics::internode_metrics::global_internode_metrics().msgpack_json_decode_total_for_test(); @@ -5518,6 +5564,8 @@ mod tests { cleanup_data_dir: Some(Uuid::new_v4()), sign: Some(vec![1_u8; 32]), old_current_size: Some(crate::disk::OldCurrentSize::Present(4096)), + old_current_source_checked: false, + old_current_source: None, }; let json = serde_json::to_vec(&response).expect("rename data response json should encode"); let named_msgpack = rmp_serde::encode::to_vec_named(&response).expect("rename data response named msgpack should encode"); diff --git a/crates/ecstore/src/diagnostics/admin_server_info.rs b/crates/ecstore/src/diagnostics/admin_server_info.rs index def38f38b..0ff94e93c 100644 --- a/crates/ecstore/src/diagnostics/admin_server_info.rs +++ b/crates/ecstore/src/diagnostics/admin_server_info.rs @@ -44,7 +44,11 @@ use std::{ use time::OffsetDateTime; use tokio::time::timeout; use tonic::Request; -use tracing::warn; +use tracing::{error, trace, warn}; + +const EVENT_SERVER_PING: &str = "server_ping"; +const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +const LOG_SUBSYSTEM_DIAGNOSTICS: &str = "diagnostics"; use shadow_rs::shadow; @@ -159,11 +163,22 @@ async fn is_server_resolvable(endpoint: &Endpoint) -> Result<()> { let response: PingResponse = client.ping(request).await?.into_inner(); - let ping_response_body = flatbuffers::root::(&response.body); - if let Err(e) = ping_response_body { - eprintln!("{e}"); + if flatbuffers::root::(&response.body).is_err() { + error!( + event = EVENT_SERVER_PING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DIAGNOSTICS, + result = "invalid_response", + "Server ping response decoding failed" + ); } else { - println!("ping_resp:body(flatbuffer): {ping_response_body:?}"); + trace!( + event = EVENT_SERVER_PING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DIAGNOSTICS, + result = "success", + "Server ping completed" + ); } Ok(()) diff --git a/crates/ecstore/src/disk/cleanup_runtime.rs b/crates/ecstore/src/disk/cleanup_runtime.rs new file mode 100644 index 000000000..8293e577a --- /dev/null +++ b/crates/ecstore/src/disk/cleanup_runtime.rs @@ -0,0 +1,1292 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Admission and execution isolation for PUT, local/RPC disk cleanup and GC. +//! PUT admission precedes staging; receiver and GC disk work share a separate +//! budget so coordinators waiting for peers cannot exhaust receiver capacity. + +use crate::disk::{LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_DISK}; +use std::collections::BTreeSet; +use std::future::Future; +use std::io; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::time::Instant; +use tokio::runtime::{Handle, Runtime}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +const ENABLE: &str = "RUSTFS_CLEANUP_ISOLATE_ENABLE"; +const BUDGET: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_MAX_PENDING"; +const BLOCKING_THREADS: &str = "RUSTFS_CLEANUP_BLOCKING_THREADS"; +const ASYNC_THREADS: &str = "RUSTFS_CLEANUP_ASYNC_THREADS"; +const CPUS: &str = "RUSTFS_CLEANUP_CPUS"; +const WORKERS: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS"; +const RENAME_TAIL_CLEANUP_LANE_ENABLE: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_ENABLE"; +const DISK_PENDING: &str = "RUSTFS_CLEANUP_DISK_MAX_PENDING"; +const DISK_WORKERS: &str = "RUSTFS_CLEANUP_DISK_WORKERS"; +const GC_WORKERS: &str = "RUSTFS_CLEANUP_GC_WORKERS"; +const EVENT_CLEANUP_RUNTIME: &str = "disk_cleanup_runtime"; +pub(crate) const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json"; + +#[derive(Clone, Copy)] +enum DiskAdmission { + Request, + Guarded, +} + +#[derive(Clone)] +pub(crate) struct Reservation { + _admission: Arc, + execution: Arc, + handle: Handle, +} + +struct Admission { + _permit: OwnedSemaphorePermit, + metrics_enabled: bool, + metric: &'static str, +} + +impl Admission { + fn new(permit: OwnedSemaphorePermit, metric: &'static str) -> Self { + let metrics_enabled = rustfs_io_metrics::put_stage_metrics_enabled(); + if metrics_enabled { + metrics::gauge!(metric).increment(1.0); + } + Self { + _permit: permit, + metrics_enabled, + metric, + } + } +} + +impl Drop for Admission { + fn drop(&mut self) { + if self.metrics_enabled { + metrics::gauge!(self.metric).decrement(1.0); + } + } +} + +/// A lease release cannot queue behind work that might need the releasing +/// reader's locks. It relinquishes the lease first, then tries this disk budget. +pub(crate) struct DiskExecution { + _active: Admission, + _pending: Admission, +} + +/// Created only inside an owned scan. Steps can borrow the scan's directory +/// iterator and paths: cancelling its waiter cannot cancel a step's syscalls. +#[cfg_attr(test, derive(Default))] +pub(crate) struct GcBudget { + gates: Option<(Arc, Arc)>, +} + +impl GcBudget { + pub(crate) async fn step(&self, future: F) -> super::error::Result + where + F: Future>, + { + let Some((pending, execution)) = &self.gates else { + return future.await; + }; + // Scan producers are bounded and hold no object locks here. Unlike + // requests, they can wait for admission without an unbounded backlog. + let started = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); + let pending = pending.clone().acquire_owned().await.map_err(io::Error::other)?; + execute_disk( + Admission::new(pending, "rustfs_cleanup_disk_pending"), + None, + execution.clone(), + started, + future, + ) + .await + } +} + +fn try_disk_permit(semaphore: &Arc) -> io::Result { + semaphore.clone().try_acquire_owned().map_err(|err| { + if rustfs_io_metrics::put_stage_metrics_enabled() { + metrics::counter!("rustfs_cleanup_disk_admission_rejected_total").increment(1); + } + io::Error::new(io::ErrorKind::WouldBlock, err) + }) +} + +async fn execute_disk( + pending: Admission, + active: Option, + execution: Arc, + started: Option, + future: F, +) -> super::error::Result +where + F: Future>, +{ + let _pending = pending; + let active = match active { + Some(active) => active, + None => execution.acquire_owned().await.map_err(super::error::DiskError::other)?, + }; + let _active = Admission::new(active, "rustfs_cleanup_disk_active"); + if let Some(started) = started { + metrics::histogram!("rustfs_cleanup_disk_execution_wait_seconds").record(started.elapsed().as_secs_f64()); + } + future.await +} + +struct Config { + pending: usize, + workers: usize, + async_threads: usize, + blocking_threads: usize, + cpus: Vec, + disk_pending: usize, + disk_workers: usize, + gc_workers: usize, +} + +fn positive_env(name: &str, default: usize) -> io::Result { + match std::env::var(name) { + Ok(value) => value + .parse::() + .ok() + .filter(|v| *v > 0 && *v <= Semaphore::MAX_PERMITS) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, format!("{name} must be a positive supported count"))), + Err(std::env::VarError::NotPresent) => Ok(default), + Err(err) => Err(io::Error::new(io::ErrorKind::InvalidInput, err)), + } +} + +// Keep the guard's accepted values aligned with the lane's existing parser. +fn legacy_rename_tail_cleanup_lane_enabled(value: Option<&str>) -> bool { + matches!(value, Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")) +} + +fn parse_cpus(value: &str) -> io::Result> { + let invalid = || { + io::Error::new( + io::ErrorKind::InvalidInput, + "cleanup CPUs must be comma-separated CPU IDs or ascending ranges", + ) + }; + if value.trim().is_empty() { + return Err(invalid()); + } + let mut cpus = BTreeSet::new(); + for part in value.split(',') { + let mut ends = part.trim().split('-'); + let first = ends.next().and_then(|s| s.parse::().ok()).ok_or_else(invalid)?; + let last = match ends.next() { + Some(s) => s.parse::().map_err(|_| invalid())?, + None => first, + }; + if ends.next().is_some() || first > last || last >= 1024 { + return Err(invalid()); + } + cpus.extend(first..=last); + } + Ok(cpus.into_iter().collect()) +} + +impl Config { + fn from_env() -> io::Result { + for name in [ + "RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP", + "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER", + "RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP", + ] { + if rustfs_utils::get_env_bool(name, false) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} cannot be combined with cleanup isolation"), + )); + } + } + if legacy_rename_tail_cleanup_lane_enabled(std::env::var(RENAME_TAIL_CLEANUP_LANE_ENABLE).ok().as_deref()) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{RENAME_TAIL_CLEANUP_LANE_ENABLE} cannot be combined with cleanup isolation"), + )); + } + let workers = positive_env(WORKERS, 2)?; + let pending = positive_env(BUDGET, 1024)?; + let blocking_threads = positive_env(BLOCKING_THREADS, 4)?; + let cpus = match std::env::var(CPUS) { + Ok(value) if value.trim().eq_ignore_ascii_case("none") => Vec::new(), + Ok(value) => parse_cpus(&value)?, + Err(std::env::VarError::NotPresent) if cfg!(target_os = "linux") => vec![1, 2], + Err(std::env::VarError::NotPresent) => Vec::new(), + Err(err) => return Err(io::Error::new(io::ErrorKind::InvalidInput, err)), + }; + let available = std::thread::available_parallelism().map_or(1, usize::from); + let async_threads = positive_env(ASYNC_THREADS, available.min(if cpus.is_empty() { 2 } else { cpus.len().min(2) }))?; + let disk_workers = positive_env(DISK_WORKERS, blocking_threads)?; + Ok(Self { + pending, + workers, + async_threads, + blocking_threads, + cpus, + // Bound queueing to sixteen service waves by default. Increasing + // async coordinator concurrency does not create more OS threads. + disk_pending: positive_env(DISK_PENDING, disk_workers.saturating_mul(16).min(Semaphore::MAX_PERMITS))?, + disk_workers, + gc_workers: positive_env(GC_WORKERS, 1)?, + }) + } + + fn validate(&self) -> io::Result<()> { + if self.pending == 0 + || self.pending > Semaphore::MAX_PERMITS + || self.workers == 0 + || self.workers > self.pending + || self.async_threads == 0 + || self.async_threads > 1024 + || self.blocking_threads == 0 + || self.blocking_threads > 1024 + || self.disk_pending == 0 + || self.disk_pending > Semaphore::MAX_PERMITS + || self.disk_workers == 0 + || self.disk_workers > self.disk_pending + || self.gc_workers == 0 + || self.gc_workers > self.disk_workers + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "cleanup counts must be positive: PUT workers <= PUT pending, GC workers <= disk workers <= disk pending, async/blocking threads <= 1024", + )); + } + validate_cpus(&self.cpus) + } +} + +#[cfg(target_os = "linux")] +fn validate_cpus(cpus: &[usize]) -> io::Result<()> { + if cpus.is_empty() { + return Ok(()); + } + let allowed = rustix::thread::sched_getaffinity(None)?; + if cpus + .iter() + .any(|cpu| *cpu >= rustix::thread::CpuSet::MAX_CPU || !allowed.is_set(*cpu)) + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "cleanup CPUs must be a subset of the caller's allowed CPU set", + )); + } + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +fn validate_cpus(cpus: &[usize]) -> io::Result<()> { + if cpus.is_empty() { + Ok(()) + } else { + Err(io::Error::new(io::ErrorKind::Unsupported, "cleanup CPU affinity requires Linux")) + } +} + +fn bind_thread(cpus: &[usize]) -> io::Result<()> { + if cpus.is_empty() { + return Ok(()); + } + #[cfg(target_os = "linux")] + { + let mut set = rustix::thread::CpuSet::new(); + for cpu in cpus { + set.set(*cpu); + } + rustix::thread::sched_setaffinity(None, &set)?; + if rustix::thread::sched_getaffinity(None)? != set { + return Err(io::Error::other("cleanup CPU affinity changed during thread startup")); + } + Ok(()) + } + #[cfg(not(target_os = "linux"))] + { + validate_cpus(cpus) + } +} + +struct CleanupRuntime { + runtime: Option, + admission: Arc, + execution: Arc, + affinity_failed: Arc, + disk_admission: Arc, + disk_execution: Arc, + gc_execution: Arc, +} + +impl CleanupRuntime { + fn new(config: Config) -> io::Result { + config.validate()?; + let affinity_failed = Arc::new(AtomicBool::new(false)); + let failed = affinity_failed.clone(); + let cpus = config.cpus; + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(config.async_threads) + .max_blocking_threads(config.blocking_threads) + .thread_name("rustfs-cleanup") + .enable_all() + .on_thread_start(move || { + // Explicit fsync dispatch must not escape into the shared + // fsync runtime. This callback also runs on blocking threads. + crate::disk::os::use_current_runtime_for_fsync(); + if let Err(err) = bind_thread(&cpus) + && !failed.swap(true, Ordering::AcqRel) + { + tracing::error!(event = EVENT_CLEANUP_RUNTIME, component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK, state = "affinity_failed", error = %err, + "Cleanup thread affinity failed"); + } + }) + .build()?; + Ok(Self { + runtime: Some(runtime), + admission: Arc::new(Semaphore::new(config.pending)), + execution: Arc::new(Semaphore::new(config.workers)), + affinity_failed, + disk_admission: Arc::new(Semaphore::new(config.disk_pending)), + disk_execution: Arc::new(Semaphore::new(config.disk_workers)), + gc_execution: Arc::new(Semaphore::new(config.gc_workers)), + }) + } + + fn reserve(&self) -> io::Result { + if self.affinity_failed.load(Ordering::Acquire) { + return Err(io::Error::other("cleanup thread affinity failed; refusing new cleanup admission")); + } + let metrics_enabled = rustfs_io_metrics::put_stage_metrics_enabled(); + // Callers may already own source-reader or data-movement locks. Do + // not wait here behind a PUT whose completion requires those locks. + let permit = self.admission.clone().try_acquire_owned().map_err(|err| { + if metrics_enabled { + metrics::counter!("rustfs_cleanup_admission_rejected_total").increment(1); + } + io::Error::new(io::ErrorKind::WouldBlock, err) + })?; + let runtime = self + .runtime + .as_ref() + .ok_or_else(|| io::Error::other("cleanup runtime stopped"))?; + Ok(Reservation { + _admission: Arc::new(Admission::new(permit, "rustfs_cleanup_admitted_puts")), + execution: self.execution.clone(), + handle: runtime.handle().clone(), + }) + } + + async fn disk_operation(&self, future: F, mode: DiskAdmission) -> super::error::Result + where + F: Future> + Send + 'static, + T: Send + 'static, + { + if self.affinity_failed.load(Ordering::Acquire) && !super::os::is_isolated_io_thread() { + return Err(io::Error::other("cleanup thread affinity failed; refusing new disk cleanup admission").into()); + } + let started = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); + let pending = try_disk_permit(&self.disk_admission)?; + let pending = Admission::new(pending, "rustfs_cleanup_disk_pending"); + let runtime = self + .runtime + .as_ref() + .ok_or_else(|| io::Error::other("cleanup runtime stopped"))?; + let execution = self.disk_execution.clone(); + // An externally guarded mutation may already hold a namespace lock + // needed by an active disk job. Never queue it behind that job. + let active = if matches!(mode, DiskAdmission::Guarded) { + try_disk_permit(&execution)? + } else { + // Keep queued work in the caller's existing future. A waiter + // timeout must discard an unstarted delete, not leave a task that + // can later delete a reused path after its caller released locks. + execution.clone().acquire_owned().await.map_err(io::Error::other)? + }; + // A permit does not mean the worker has polled this operation. Use + // the result receiver's lifetime to discard work cancelled during + // runtime scheduling, before its first poll can start filesystem I/O. + let (result_tx, result_rx) = tokio::sync::oneshot::channel(); + let task = runtime.spawn(async move { + if result_tx.is_closed() { + return; + } + // After the first poll, the owned job must retain both permits + // through I/O even if the waiter cancels and closes result_rx. + let result = execute_disk(pending, Some(active), execution, started, future).await; + let _ = result_tx.send(result); + }); + match result_rx.await { + Ok(result) => result, + Err(err) => { + // Preserve a panic's JoinError rather than erasing its cause + // into a closed-channel error. + task.await.map_err(super::error::DiskError::other)?; + Err(super::error::DiskError::other(err)) + } + } + } + + fn try_disk_execution(&self) -> io::Result { + let pending = try_disk_permit(&self.disk_admission)?; + let active = try_disk_permit(&self.disk_execution)?; + Ok(DiskExecution { + _pending: Admission::new(pending, "rustfs_cleanup_disk_pending"), + _active: Admission::new(active, "rustfs_cleanup_disk_active"), + }) + } + + async fn gc_operation(&self, scan: F) -> super::error::Result<()> + where + F: FnOnce(GcBudget) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + if self.affinity_failed.load(Ordering::Acquire) { + return Err(io::Error::other("cleanup thread affinity failed; refusing new GC scan").into()); + } + let handle = self + .runtime + .as_ref() + .ok_or_else(|| io::Error::other("cleanup runtime stopped"))? + .handle(); + let execution = self.gc_execution.clone(); + let budget = GcBudget { + gates: Some((self.disk_admission.clone(), self.disk_execution.clone())), + }; + handle + .spawn(async move { + let active = execution.acquire_owned().await.map_err(super::error::DiskError::other)?; + let _active = Admission::new(active, "rustfs_cleanup_gc_active"); + scan(budget).await + }) + .await + .map_err(super::error::DiskError::other)? + } +} + +impl Drop for CleanupRuntime { + fn drop(&mut self) { + if let Some(runtime) = self.runtime.take() { + runtime.shutdown_background(); + } + } +} + +fn instance() -> io::Result> { + static INSTANCE: OnceLock, String>> = OnceLock::new(); + match INSTANCE.get_or_init(|| { + if !rustfs_utils::get_env_bool(ENABLE, false) { + return Ok(None); + } + Config::from_env() + .and_then(CleanupRuntime::new) + .map(Some) + .map_err(|err| err.to_string()) + }) { + Ok(runtime) => Ok(runtime.as_ref()), + Err(err) => Err(io::Error::other(err.clone())), + } +} + +pub(crate) fn reserve() -> io::Result> { + match instance()? { + Some(runtime) => runtime.reserve().map(Some), + None => Ok(None), + } +} + +pub(crate) fn enabled() -> io::Result { + Ok(instance()?.is_some()) +} + +/// Local cleanup entry point, also used by remote RPC receivers. This budget +/// is distinct from coordinator execution: coordinators can wait for peers +/// without exhausting the peers' capacity to service disk cleanup. +pub(crate) async fn disk_operation(future: F) -> super::error::Result +where + F: Future> + Send + 'static, + T: Send + 'static, +{ + match instance()? { + Some(runtime) => runtime.disk_operation(future, DiskAdmission::Request).await, + None => future.await, + } +} + +pub(crate) async fn guarded_disk_operation(future: F) -> super::error::Result +where + F: Future> + Send + 'static, + T: Send + 'static, +{ + match instance()? { + Some(runtime) => runtime.disk_operation(future, DiskAdmission::Guarded).await, + None => future.await, + } +} + +/// Lease release must run even under cleanup saturation, or reader tokens +/// would leak. Only its optional deferred deletion consumes disk permits. +pub(crate) async fn release_lease(future: F) -> super::error::Result<()> +where + F: Future> + Send + 'static, +{ + match instance()? { + Some(runtime) => runtime + .runtime + .as_ref() + .ok_or_else(|| io::Error::other("cleanup runtime stopped"))? + .spawn(future) + .await + .map_err(super::error::DiskError::other)?, + None => future.await, + } +} + +pub(crate) fn try_disk_execution() -> io::Result> { + instance()?.map(CleanupRuntime::try_disk_execution).transpose() +} + +/// At most gc_workers scans can produce work. Each scan yields the shared +/// disk execution budget between entries without spawning a task per entry. +pub(crate) async fn run_gc(scan: F) -> super::error::Result<()> +where + F: FnOnce(GcBudget) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, +{ + match instance()? { + Some(runtime) => runtime.gc_operation(scan).await, + None => scan(GcBudget { gates: None }).await, + } +} + +/// Dispatch only already-admitted work. Keeping the runtime handle in the +/// reservation prevents configuration errors from dropping committed cleanup. +pub(crate) fn spawn(reservation: Option, future: F) -> tokio::task::JoinHandle +where + F: Future + Send + 'static, + F::Output: Send + 'static, +{ + let started = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); + match reservation { + Some(reservation) => reservation.handle.clone().spawn(async move { + // This semaphore is never closed. Both queued work and overflow + // use this gate, and cancellation of the waiter cannot release it. + let _permit = reservation.execution.clone().acquire_owned().await; + if let Some(started) = started { + metrics::histogram!("rustfs_cleanup_execution_wait_seconds").record(started.elapsed().as_secs_f64()); + } + let result = future.await; + drop(reservation); + result + }), + None => tokio::spawn(future), + } +} + +/// Queue workers, queue overflow and synchronous failure cleanup share a gate. +/// With isolation disabled, retain the original inline polling behavior. +pub(crate) async fn run(reservation: Option, future: F) +where + F: Future + Send + 'static, +{ + if reservation.is_none() { + future.await; + } else if let Err(err) = spawn(reservation, future).await { + tracing::error!(event = EVENT_CLEANUP_RUNTIME, component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK, state = "task_failed", error = %err, + "Cleanup task failed"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicUsize; + use std::time::Duration; + use tokio::sync::oneshot; + + fn config(pending: usize, workers: usize, blocking_threads: usize) -> Config { + Config { + pending, + workers, + async_threads: workers.min(2), + blocking_threads, + cpus: Vec::new(), + disk_pending: 4, + disk_workers: 2, + gc_workers: 1, + } + } + + async fn deadline(future: F) -> F::Output { + tokio::time::timeout(Duration::from_secs(20), future) + .await + .expect("cleanup test timed out") + } + + #[test] + fn cpu_list_rejects_invalid_or_unbounded_ranges() { + assert_eq!(parse_cpus("3, 1-3,6").unwrap(), vec![1, 2, 3, 6]); + for value in ["", " ", "1,", "-1", "2-1", "1-2-3", "no", "1024", "0-18446744073709551615"] { + assert_eq!(parse_cpus(value).unwrap_err().kind(), io::ErrorKind::InvalidInput, "{value}"); + } + } + + #[test] + fn cleanup_lane_guard_matches_legacy_boolean_values() { + for (value, expected) in [ + (Some("1"), true), + (Some("true"), true), + (Some("TRUE"), true), + (Some("yes"), true), + (Some("YES"), true), + (Some("on"), true), + (Some("ON"), true), + (Some("True"), false), + (Some("enabled"), false), + (None, false), + ] { + assert_eq!(legacy_rename_tail_cleanup_lane_enabled(value), expected, "{value:?}"); + } + } + + #[test] + fn invalid_environment_is_rejected_before_admission() { + const CHILD: &str = "RUSTFS_CLEANUP_CONFIG_TEST_CHILD"; + if std::env::var_os(CHILD).is_some() { + assert!(reserve().is_err()); + return; + } + // Each configuration needs a fresh process: production intentionally + // caches configuration, and mutating process env races parallel tests. + for (name, value) in [ + (WORKERS, "0"), + (WORKERS, "5"), + (BUDGET, "invalid"), + (BLOCKING_THREADS, "0"), + (BLOCKING_THREADS, "1025"), + (ASYNC_THREADS, "0"), + (ASYNC_THREADS, "1025"), + (DISK_PENDING, "0"), + (DISK_WORKERS, "3"), + (GC_WORKERS, "3"), + (CPUS, "1-0"), + ("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP", "true"), + ("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER", "true"), + ("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP", "true"), + (RENAME_TAIL_CLEANUP_LANE_ENABLE, "true"), + ] { + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "disk::cleanup_runtime::tests::invalid_environment_is_rejected_before_admission", + "--nocapture", + ]) + .env(CHILD, "1") + .env(ENABLE, "true") + .env(WORKERS, "1") + .env(BUDGET, "4") + .env(BLOCKING_THREADS, "1") + .env(ASYNC_THREADS, "1") + .env(DISK_PENDING, "2") + .env(DISK_WORKERS, "2") + .env(GC_WORKERS, "1") + .env_remove(CPUS) + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP") + .env_remove(RENAME_TAIL_CLEANUP_LANE_ENABLE) + .env(name, value) + .output() + .unwrap(); + assert!(result.status.success(), "{name}={value}: {}", String::from_utf8_lossy(&result.stderr)); + assert!( + String::from_utf8_lossy(&result.stdout).contains("1 passed"), + "child must execute the admission assertion" + ); + } + } + + #[test] + fn invalid_budgets_fail_before_building_threads() { + for config in [config(0, 1, 1), config(1, 0, 1), config(1, 2, 1), config(1, 1, 0)] { + assert!(matches!(CleanupRuntime::new(config), Err(err) if err.kind() == io::ErrorKind::InvalidInput)); + } + } + + #[tokio::test] + async fn disk_work_keeps_budget_after_rpc_waiter_cancellation() { + let mut settings = config(1, 1, 1); + settings.disk_pending = 1; + settings.disk_workers = 1; + let runtime = Arc::new(CleanupRuntime::new(settings).unwrap()); + let receiver = runtime.clone(); + let (entered, entry) = oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let rpc = tokio::spawn(async move { + receiver + .disk_operation( + async move { + tokio::task::spawn_blocking(move || { + assert_eq!(std::thread::current().name(), Some("rustfs-cleanup")); + entered.send(()).unwrap(); + released.recv_timeout(Duration::from_secs(20)).unwrap(); + }) + .await + .unwrap(); + Ok(()) + }, + DiskAdmission::Request, + ) + .await + }); + deadline(entry).await.unwrap(); + rpc.abort(); + assert!(rpc.await.unwrap_err().is_cancelled()); + assert_eq!(runtime.disk_admission.available_permits(), 0); + assert_eq!(runtime.disk_execution.available_permits(), 0); + let ran = Arc::new(AtomicBool::new(false)); + let invoked = ran.clone(); + let rejected = runtime + .disk_operation( + async move { + invoked.store(true, Ordering::SeqCst); + Ok(()) + }, + DiskAdmission::Request, + ) + .await; + assert!(matches!(rejected, Err(super::super::error::DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock)); + assert!(!ran.load(Ordering::SeqCst), "rejected work must not execute"); + release.send(()).unwrap(); + deadline(async { + while runtime.disk_admission.available_permits() == 0 { + tokio::task::yield_now().await; + } + }) + .await; + assert_eq!(runtime.disk_execution.available_permits(), 1); + } + + #[tokio::test] + async fn cancelling_queued_disk_work_discards_it_before_execution() { + let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime")); + let occupied = runtime + .disk_execution + .clone() + .acquire_many_owned(2) + .await + .expect("occupy all execution slots"); + let receiver = runtime.clone(); + let ran = Arc::new(AtomicBool::new(false)); + let invoked = ran.clone(); + let waiter = tokio::spawn(async move { + receiver + .disk_operation( + async move { + invoked.store(true, Ordering::SeqCst); + Ok(()) + }, + DiskAdmission::Request, + ) + .await + }); + deadline(async { + while runtime.disk_admission.available_permits() == 4 { + tokio::task::yield_now().await; + } + }) + .await; + waiter.abort(); + assert!(waiter.await.expect_err("cancel queued waiter").is_cancelled()); + assert_eq!( + runtime.disk_admission.available_permits(), + 4, + "unstarted work must release pending capacity immediately" + ); + drop(occupied); + deadline(runtime.disk_operation(async { Ok(()) }, DiskAdmission::Request)) + .await + .expect("later work must progress"); + assert!(!ran.load(Ordering::SeqCst), "cancelled queued deletion must never execute later"); + } + + #[tokio::test] + async fn cancelling_disk_work_before_worker_poll_preserves_reused_path() { + let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime")); + let (entered, entry) = oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let stalled = runtime.runtime.as_ref().expect("live runtime").spawn(async move { + entered.send(()).expect("worker entered"); + released + .recv_timeout(Duration::from_secs(30)) + .expect("release stalled worker"); + }); + deadline(entry).await.expect("worker must stall before dispatch"); + let dir = tempfile::tempdir().expect("temporary disk"); + let backup = dir.path().join("xl.meta.bkp"); + std::fs::write(&backup, b"previous transaction").expect("old backup"); + { + let path = backup.clone(); + let deletion = runtime.disk_operation( + async move { + tokio::fs::remove_file(path).await?; + Ok(()) + }, + DiskAdmission::Request, + ); + tokio::pin!(deletion); + assert!(futures::poll!(&mut deletion).is_pending()); + assert_eq!(runtime.disk_execution.available_permits(), 1, "execution was admitted but not polled"); + } + std::fs::write(&backup, b"next transaction").expect("reuse path after cancelled waiter"); + release.send(()).expect("resume worker"); + deadline(stalled).await.expect("worker resumed"); + deadline(async { + while runtime.disk_admission.available_permits() != 4 { + tokio::task::yield_now().await; + } + }) + .await; + assert_eq!( + std::fs::read(backup).expect("unstarted deletion must not execute after cancellation"), + b"next transaction" + ); + assert_eq!(runtime.disk_execution.available_permits(), 2); + } + + #[tokio::test] + async fn disk_task_panic_preserves_cause_and_releases_budget() { + async fn panicking_job() -> super::super::error::Result<()> { + panic!("injected cleanup panic"); + } + let runtime = CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime"); + let error = deadline(runtime.disk_operation(panicking_job(), DiskAdmission::Request)) + .await + .expect_err("worker panic must reach its caller"); + let super::super::error::DiskError::Io(error) = error else { + panic!("worker panic must retain its I/O error source"); + }; + let cause = error + .get_ref() + .expect("panic source") + .downcast_ref::() + .expect("preserved JoinError"); + assert!(cause.is_panic()); + assert_eq!(runtime.disk_admission.available_permits(), 4); + assert_eq!(runtime.disk_execution.available_permits(), 2); + } + + #[tokio::test] + async fn coordinator_waiting_for_disk_does_not_consume_receiver_capacity() { + let mut settings = config(1, 1, 1); + settings.disk_pending = 1; + settings.disk_workers = 1; + let runtime = Arc::new(CleanupRuntime::new(settings).unwrap()); + let reservation = runtime.reserve().unwrap(); + let receiver = runtime.clone(); + let result = deadline(spawn(Some(reservation), async move { + receiver + .disk_operation(async { Ok(42) }, DiskAdmission::Request) + .await + .unwrap() + })) + .await + .unwrap(); + assert_eq!(result, 42); + } + + #[tokio::test] + async fn gc_shares_disk_budget_and_guarded_requests_do_not_wait_under_locks() { + let mut settings = config(1, 1, 1); + settings.disk_pending = 1; + settings.disk_workers = 1; + let runtime = Arc::new(CleanupRuntime::new(settings).unwrap()); + let receiver = runtime.clone(); + let (entered, entry) = oneshot::channel(); + let (release, released) = oneshot::channel(); + let rpc = tokio::spawn(async move { + receiver + .disk_operation( + async move { + entered.send(()).unwrap(); + released.await.unwrap(); + Ok(()) + }, + DiskAdmission::Request, + ) + .await + }); + deadline(entry).await.unwrap(); + let guarded = runtime.disk_operation(async { Ok(()) }, DiskAdmission::Guarded).await; + assert!(matches!(guarded, Err(super::super::error::DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock)); + let budget = GcBudget { + gates: Some((runtime.disk_admission.clone(), runtime.disk_execution.clone())), + }; + let gc = budget.step(async { Ok(7) }); + tokio::pin!(gc); + assert!( + futures::poll!(&mut gc).is_pending(), + "GC must join the shared budget rather than bypass it" + ); + release.send(()).unwrap(); + deadline(rpc).await.unwrap().unwrap(); + assert_eq!(deadline(gc).await.unwrap(), 7); + + // Also reject a guarded mutation when pending capacity is free but + // execution is occupied. This prevents namespace -> budget inversion. + let execution = runtime.disk_execution.clone().acquire_owned().await.unwrap(); + let guarded = runtime.disk_operation(async { Ok(()) }, DiskAdmission::Guarded).await; + assert!(matches!(guarded, Err(super::super::error::DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock)); + assert_eq!(runtime.disk_admission.available_permits(), 1); + assert!(matches!(runtime.try_disk_execution(), Err(err) if err.kind() == io::ErrorKind::WouldBlock)); + assert_eq!(runtime.disk_admission.available_permits(), 1); + drop(execution); + let lease_cleanup = runtime.try_disk_execution().unwrap(); + assert_eq!(runtime.disk_admission.available_permits(), 0); + assert_eq!(runtime.disk_execution.available_permits(), 0); + drop(lease_cleanup); + assert_eq!(runtime.disk_execution.available_permits(), 1); + } + + #[tokio::test] + async fn gc_scan_limit_leaves_disk_slots_available_between_entries() { + let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).unwrap()); + let first_runtime = runtime.clone(); + let (entered, entry) = oneshot::channel(); + let (release, released) = oneshot::channel(); + let first = tokio::spawn(async move { + first_runtime + .gc_operation(move |_| async move { + entered.send(()).unwrap(); + released.await.unwrap(); + Ok(()) + }) + .await + }); + deadline(entry).await.unwrap(); + assert_eq!(runtime.gc_execution.available_permits(), 0); + assert_eq!( + deadline(runtime.disk_operation(async { Ok(42) }, DiskAdmission::Request)) + .await + .unwrap(), + 42 + ); + let second = runtime.gc_operation(|_| async { Ok(()) }); + tokio::pin!(second); + assert!(futures::poll!(&mut second).is_pending()); + release.send(()).unwrap(); + deadline(first).await.unwrap().unwrap(); + deadline(second).await.unwrap(); + } + + #[tokio::test] + async fn gc_steps_borrow_scan_state_and_keep_budget_after_waiter_cancellation() { + let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime")); + let receiver = runtime.clone(); + let (entered, entry) = oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let waiter = tokio::spawn(async move { + receiver + .gc_operation(move |budget| async move { + let scan_task = tokio::task::id(); + let mut count = 0; + for _ in 0..32 { + budget + .step(async { + assert_eq!(tokio::task::id(), scan_task, "each entry must stay on the owned scan task"); + count += 1; + Ok(()) + }) + .await?; + } + assert_eq!(count, 32); + budget + .step(async move { + tokio::task::spawn_blocking(move || { + entered.send(()).expect("notify started syscall"); + released.recv_timeout(Duration::from_secs(20)).expect("release syscall"); + }) + .await + .expect("blocking cleanup"); + Ok(()) + }) + .await + }) + .await + }); + deadline(entry).await.expect("GC must start"); + waiter.abort(); + assert!(waiter.await.expect_err("cancelled waiter").is_cancelled()); + assert_eq!(runtime.gc_execution.available_permits(), 0); + assert_eq!(runtime.disk_admission.available_permits(), 3); + assert_eq!(runtime.disk_execution.available_permits(), 1); + release.send(()).expect("release blocking cleanup"); + deadline(async { + while runtime.gc_execution.available_permits() == 0 { + tokio::task::yield_now().await; + } + }) + .await; + assert_eq!(runtime.disk_admission.available_permits(), 4); + assert_eq!(runtime.disk_execution.available_permits(), 2); + } + + #[test] + fn defaults_bound_queue_waves_and_separate_threads_from_jobs() { + const CHILD: &str = "RUSTFS_CLEANUP_DEFAULTS_TEST_CHILD"; + if std::env::var_os(CHILD).is_some() { + let mut config = Config::from_env().expect("default configuration"); + let expected_cpus = match std::env::var(CPUS).as_deref() { + Ok("none") => Vec::new(), + Ok("3-4") => vec![3, 4], + _ if cfg!(target_os = "linux") => vec![1, 2], + _ => Vec::new(), + }; + assert_eq!(config.cpus, expected_cpus); + assert_eq!(config.workers, 64); + assert!(config.async_threads <= 2); + assert_eq!(config.blocking_threads, 4); + assert_eq!(config.disk_workers, 4); + assert_eq!(config.disk_pending, 64); + // Queue/thread defaults are independent of the host's CPU mask. + // The integration test exercises actual default/custom affinity. + config.cpus.clear(); + config.validate().expect("defaults must be internally consistent"); + let threads = config.async_threads; + let runtime = CleanupRuntime::new(config).expect("runtime with many logical jobs"); + assert_eq!(runtime.runtime.as_ref().expect("live runtime").metrics().num_workers(), threads); + return; + } + for cpus in [None, Some("3-4"), Some("none")] { + let mut command = std::process::Command::new(std::env::current_exe().expect("test executable")); + command + .args([ + "--exact", + "disk::cleanup_runtime::tests::defaults_bound_queue_waves_and_separate_threads_from_jobs", + ]) + .env(CHILD, "1") + .env(WORKERS, "64") + .env(BUDGET, "1024") + .env(GC_WORKERS, "1") + .env_remove(ASYNC_THREADS) + .env_remove(BLOCKING_THREADS) + .env_remove(DISK_WORKERS) + .env_remove(DISK_PENDING) + .env_remove(CPUS) + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP"); + if let Some(cpus) = cpus { + command.env(CPUS, cpus); + } + let output = command.output().expect("run defaults test in a fresh process"); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + } + } + + #[tokio::test] + async fn affinity_failure_rejects_new_admission_without_dropping_admitted_work() { + let runtime = CleanupRuntime::new(config(2, 1, 1)).unwrap(); + let reservation = runtime.reserve().unwrap(); + runtime.affinity_failed.store(true, Ordering::Release); + assert!(runtime.reserve().is_err()); + assert_eq!(deadline(spawn(Some(reservation), async { 42 })).await.unwrap(), 42); + assert_eq!(runtime.admission.available_permits(), 2); + } + + #[tokio::test] + async fn admission_is_retained_by_every_commit_and_cleanup_owner() { + let runtime = CleanupRuntime::new(config(1, 1, 1)).unwrap(); + let request = runtime.reserve().unwrap(); + let commit = request.clone(); + let (release, released) = oneshot::channel(); + let cleanup = spawn(Some(commit.clone()), async move { + released.await.unwrap(); + }); + drop(request); + drop(commit); + assert!( + matches!(runtime.reserve(), Err(err) if err.kind() == io::ErrorKind::WouldBlock), + "queued and running cleanup must retain admission without waiting on external locks" + ); + release.send(()).unwrap(); + deadline(cleanup).await.unwrap(); + let next = runtime.reserve().unwrap(); + assert_eq!(runtime.admission.available_permits(), 0); + drop(next); + assert_eq!(runtime.admission.available_permits(), 1); + } + + #[tokio::test] + async fn cancelling_cleanup_waiter_does_not_cancel_work_or_release_budget() { + let runtime = CleanupRuntime::new(config(1, 1, 1)).unwrap(); + let reservation = runtime.reserve().unwrap(); + let (entered, entry) = oneshot::channel(); + let (release, released) = oneshot::channel(); + let waiter = tokio::spawn(run(Some(reservation), async move { + entered.send(()).unwrap(); + released.await.unwrap(); + })); + deadline(entry).await.unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + assert_eq!(runtime.admission.available_permits(), 0); + assert_eq!(runtime.execution.available_permits(), 0); + release.send(()).unwrap(); + deadline(async { + loop { + if let Ok(reservation) = runtime.reserve() { + drop(reservation); + break; + } + tokio::task::yield_now().await; + } + }) + .await; + assert_eq!(runtime.admission.available_permits(), 1); + } + + #[tokio::test] + async fn worker_and_overflow_dispatch_share_one_execution_budget() { + let runtime = CleanupRuntime::new(config(8, 2, 1)).unwrap(); + let active = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let gate = Arc::new(Semaphore::new(0)); + let (entered, mut entries) = tokio::sync::mpsc::unbounded_channel(); + let mut jobs = Vec::new(); + for index in 0..8 { + let reservation = runtime.reserve().unwrap(); + let active = active.clone(); + let peak = peak.clone(); + let gate = gate.clone(); + let entered = entered.clone(); + let work = async move { + let running = active.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(running, Ordering::SeqCst); + entered.send(()).unwrap(); + gate.acquire().await.unwrap().forget(); + active.fetch_sub(1, Ordering::SeqCst); + }; + // Both a dequeued job and a queue-full continuation enter `run`; + // no-tail temporary cleanup uses direct owned dispatch. + jobs.push(if index % 2 == 0 { + tokio::spawn(run(Some(reservation), work)) + } else { + spawn(Some(reservation), work) + }); + } + deadline(entries.recv()).await.unwrap(); + deadline(entries.recv()).await.unwrap(); + assert_eq!(runtime.execution.available_permits(), 0); + assert!(entries.try_recv().is_err()); + assert_eq!(runtime.admission.available_permits(), 0); + gate.add_permits(8); + for job in jobs { + deadline(job).await.unwrap(); + } + assert_eq!(peak.load(Ordering::SeqCst), 2); + assert_eq!(runtime.admission.available_permits(), 8); + } + + #[tokio::test] + async fn cleanup_children_and_blocking_work_use_dedicated_threads() { + let runtime = CleanupRuntime::new(config(1, 1, 1)).unwrap(); + let task = spawn(Some(runtime.reserve().unwrap()), async move { + assert_eq!(std::thread::current().name(), Some("rustfs-cleanup")); + tokio::spawn(async { + assert_eq!(std::thread::current().name(), Some("rustfs-cleanup")); + tokio::task::spawn_blocking(|| { + assert_eq!(std::thread::current().name(), Some("rustfs-cleanup")); + }) + .await + .unwrap(); + }) + .await + .unwrap(); + }); + deadline(task).await.unwrap(); + assert_ne!(std::thread::current().name(), Some("rustfs-cleanup")); + } + + #[tokio::test] + async fn blocking_pool_saturation_does_not_block_foreground_pool() { + let runtime = CleanupRuntime::new(config(2, 2, 1)).unwrap(); + let (entered, entry) = oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let first = spawn(Some(runtime.reserve().unwrap()), async move { + tokio::task::spawn_blocking(move || { + entered.send(()).unwrap(); + released.recv_timeout(Duration::from_secs(20)).unwrap(); + }) + .await + .unwrap(); + }); + deadline(entry).await.unwrap(); + let (submitted, submission) = oneshot::channel(); + let (second_entered, mut second_entry) = oneshot::channel(); + let second = spawn(Some(runtime.reserve().unwrap()), async move { + let task = tokio::task::spawn_blocking(move || { + second_entered.send(()).unwrap(); + }); + submitted.send(()).unwrap(); + task.await.unwrap(); + }); + deadline(submission).await.unwrap(); + assert!(matches!(second_entry.try_recv(), Err(oneshot::error::TryRecvError::Empty))); + assert_eq!(deadline(tokio::task::spawn_blocking(|| 42)).await.unwrap(), 42); + release.send(()).unwrap(); + deadline(first).await.unwrap(); + deadline(second).await.unwrap(); + deadline(second_entry).await.unwrap(); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn affinity_covers_async_and_blocking_threads_without_changing_caller() { + let caller = rustix::thread::sched_getaffinity(None).unwrap(); + let cpu = (0..rustix::thread::CpuSet::MAX_CPU).find(|cpu| caller.is_set(*cpu)).unwrap(); + let mut config = config(1, 1, 1); + config.cpus = vec![cpu]; + let runtime = CleanupRuntime::new(config).unwrap(); + deadline(spawn(Some(runtime.reserve().unwrap()), async move { + let expected = { + let mut set = rustix::thread::CpuSet::new(); + set.set(cpu); + set + }; + assert_eq!(rustix::thread::sched_getaffinity(None).unwrap(), expected); + let actual = tokio::task::spawn_blocking(|| rustix::thread::sched_getaffinity(None).unwrap()) + .await + .unwrap(); + assert_eq!(actual, expected); + })) + .await + .unwrap(); + assert_eq!(rustix::thread::sched_getaffinity(None).unwrap(), caller); + assert!(validate_cpus(&[rustix::thread::CpuSet::MAX_CPU]).is_err()); + } + + #[cfg(not(target_os = "linux"))] + #[test] + fn explicit_cpu_affinity_is_rejected_on_unsupported_platforms() { + assert_eq!(validate_cpus(&[0]).unwrap_err().kind(), io::ErrorKind::Unsupported); + validate_cpus(&[]).unwrap(); + } +} diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 01091325b..3996c9dfd 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::disk::cleanup_runtime; use crate::disk::io_admission::{ AdmissionReader, BoxedChunkReader, DiskIoAdmission, DiskIoClass, DiskIoContext, current_disk_io_context, with_disk_io_context, with_disk_io_permit, @@ -484,10 +485,10 @@ impl LocalDiskWrapper { opts: DeleteOptions, namespace_owner: Option>, ) -> Result<()> { - self.track_disk_health_mutation( + self.track_cleanup_mutation( "delete", DiskMetricMutation::Delete, - || async { Box::pin(self.disk.delete_with_namespace_owner(volume, path, opts, namespace_owner)).await }, + || async { Box::pin(self.cleanup_delete(volume, path, opts, namespace_owner)).await }, get_max_timeout_duration(), ) .await @@ -1475,6 +1476,28 @@ impl LocalDiskWrapper { ) } + async fn cleanup_delete( + &self, + volume: &str, + path: &str, + options: DeleteOptions, + owner: Option>, + ) -> Result<()> { + if !cleanup_runtime::enabled()? { + return self.disk.delete_with_namespace_owner(volume, path, options, owner).await; + } + let disk = self.disk.clone(); + let volume = volume.to_owned(); + let path = path.to_owned(); + let guarded = owner.is_some(); + let operation = async move { disk.delete_with_namespace_owner(&volume, &path, options, owner).await }; + if guarded { + cleanup_runtime::guarded_disk_operation(operation).await + } else { + cleanup_runtime::disk_operation(operation).await + } + } + /// Run a delete under an owned coordinator task when a publication guard /// is present. This keeps the guard alive if the RPC waiter is cancelled /// while the local namespace mutation is still in progress. @@ -1493,12 +1516,13 @@ impl LocalDiskWrapper { } else { get_max_timeout_duration() }; + let cleanup_owner = external_guard.clone(); run_owned_mutation(external_guard, move || async move { operation - .track_disk_health_mutation( + .track_cleanup_mutation( "delete", DiskMetricMutation::Delete, - || async { operation.disk.delete(&volume, &path, options).await }, + || async { operation.cleanup_delete(&volume, &path, options, cleanup_owner).await }, timeout_duration, ) .await @@ -2092,6 +2116,31 @@ impl LocalDiskWrapper { self.track_disk_health_with_op("unknown", operation, timeout_duration).await } + /// Cleanup latency includes waiting behind a node-wide execution budget, + /// so a waiter deadline alone is not evidence that this disk is faulty. + /// The active disk health probe retains its original timeout policy. + async fn track_cleanup_mutation( + &self, + op: &'static str, + mutation: DiskMetricMutation, + operation: F, + timeout_duration: Duration, + ) -> Result + where + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + // Runtime initialization must not prevent control-plane release of + // reader tokens. Disk I/O entries still propagate configuration errors. + let action = if cleanup_runtime::enabled().unwrap_or(false) { + TimeoutHealthAction::IgnoreFailure + } else { + TimeoutHealthAction::MarkFailure + }; + self.track_disk_health_with_op_timeout_action_and_mutation(op, operation, timeout_duration, action, mutation) + .await + } + async fn track_disk_health_mutation( &self, op: &'static str, @@ -2589,9 +2638,37 @@ impl DiskAPI for LocalDiskWrapper { } async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> { - self.track_disk_health_with_op( + // Quota fence release is control-plane work and may wait for in-flight + // mutations. Keep its original waiter lifecycle instead of creating + // detached cleanup tasks for repeated fence-release RPCs. + if volume == super::RUSTFS_META_BUCKET && super::is_quota_mutation_fence_path(path) { + return self + .track_disk_health_with_op( + "release_snapshot_lease", + || async { self.disk.release_snapshot_lease(volume, path, token).await }, + get_max_timeout_duration(), + ) + .await; + } + self.track_cleanup_mutation( "release_snapshot_lease", - || async { self.disk.release_snapshot_lease(volume, path, token).await }, + DiskMetricMutation::None, + || async { + if self.disk.release_snapshot_lease_without_cleanup(volume, path, token).await { + return Ok(()); + } + if !cleanup_runtime::enabled().unwrap_or(false) { + // The raw release removes the token before trying the + // reclaim budget. Initialization failure therefore keeps + // the cleanup intent and data, without leaking a reader + // or falling back to unbudgeted filesystem deletion. + return self.disk.release_snapshot_lease(volume, path, token).await; + } + let disk = self.disk.clone(); + let volume = volume.to_owned(); + let path = path.to_owned(); + cleanup_runtime::release_lease(async move { disk.release_snapshot_lease(&volume, &path, token).await }).await + }, get_max_timeout_duration(), ) .await @@ -2616,10 +2693,22 @@ impl DiskAPI for LocalDiskWrapper { } else { get_max_timeout_duration() }; - self.track_disk_health_mutation( + self.track_cleanup_mutation( "delete_data_dir", DiskMetricMutation::Delete, - || async { self.disk.delete_data_dir(volume, path, opts).await }, + || async { + if !cleanup_runtime::enabled()? { + return self.disk.delete_data_dir(volume, path, opts).await; + } + let disk = self.disk.clone(); + let volume = volume.to_owned(); + let path = path.to_owned(); + cleanup_runtime::disk_operation(async move { + let _scope = scope; + disk.delete_data_dir(&volume, &path, opts).await + }) + .await + }, timeout, ) .await @@ -2866,10 +2955,10 @@ impl DiskAPI for LocalDiskWrapper { } async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { - self.track_disk_health_mutation( + self.track_cleanup_mutation( "delete", DiskMetricMutation::Delete, - || async { self.disk.delete(volume, path, opt).await }, + || async { self.cleanup_delete(volume, path, opt, None).await }, get_max_timeout_duration(), ) .await @@ -2902,11 +2991,25 @@ impl DiskAPI for LocalDiskWrapper { } async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()> { - self.track_disk_health_mutation( + let cleanup = path.rsplit('/').next() == Some(cleanup_runtime::OLD_DATA_CLEANUP_RECEIPT_FILE); + self.track_disk_health_with_op_timeout_action_and_mutation( "write_all", - DiskMetricMutation::Write, - || async { self.disk.write_all(volume, path, data).await }, + || async { + if !cleanup || !cleanup_runtime::enabled()? { + return self.disk.write_all(volume, path, data).await; + } + let disk = self.disk.clone(); + let volume = volume.to_owned(); + let path = path.to_owned(); + cleanup_runtime::disk_operation(async move { disk.write_all(&volume, &path, data).await }).await + }, get_max_timeout_duration(), + if cleanup && cleanup_runtime::enabled()? { + TimeoutHealthAction::IgnoreFailure + } else { + TimeoutHealthAction::MarkFailure + }, + DiskMetricMutation::Write, ) .await } @@ -3299,6 +3402,256 @@ mod tests { let _ = walk.await; } + #[tokio::test] + async fn invalid_cleanup_config_does_not_leak_snapshot_readers() { + const CHILD: &str = "RUSTFS_CLEANUP_INVALID_LEASE_TEST_CHILD"; + if std::env::var_os(CHILD).is_none() { + let output = tokio::task::spawn_blocking(|| { + std::process::Command::new(std::env::current_exe().expect("test executable")) + .args([ + "--exact", + "disk::disk_store::tests::invalid_cleanup_config_does_not_leak_snapshot_readers", + "--nocapture", + ]) + .env(CHILD, "1") + .env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true") + .env("RUSTFS_CLEANUP_ASYNC_THREADS", "0") + .output() + .expect("run with invalid cleanup configuration") + }) + .await + .expect("child process task"); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + return; + } + let dir = tempfile::tempdir().expect("temporary disk"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk")); + let wrapper = LocalDiskWrapper::new(disk.clone(), false); + let volume = "invalid-cleanup-config"; + let path = "object/data"; + wrapper.make_volume(volume).await.expect("bucket"); + wrapper + .write_all(volume, "object/data/part.1", Bytes::from_static(b"live shard")) + .await + .expect("write shard"); + let token = wrapper.acquire_snapshot_lease(volume, path).await.expect("GET lease"); + wrapper + .release_snapshot_lease(volume, path, token) + .await + .expect("control-only release must not depend on cleanup initialization"); + assert!(matches!( + wrapper.renew_snapshot_lease(volume, path, token).await, + Err(DiskError::FileNotFound) + )); + + let token = wrapper + .acquire_snapshot_lease(volume, path) + .await + .expect("reader of old data"); + // Register reclaim intent directly; this only updates the lease + // registry while a reader is present and performs no filesystem I/O. + assert_eq!( + disk.delete_data_dir( + volume, + path, + DeleteOptions { + recursive: true, + ..Default::default() + } + ) + .await + .expect("defer reclaim"), + DataDirDeleteStatus::Deferred + ); + assert!( + wrapper.release_snapshot_lease(volume, path, token).await.is_err(), + "invalid configuration must still reject reclamation" + ); + assert!( + matches!(wrapper.renew_snapshot_lease(volume, path, token).await, Err(DiskError::FileNotFound)), + "failed reclamation must not strand a reader token" + ); + assert_eq!( + wrapper.read_all(volume, "object/data/part.1").await.expect("retained data"), + Bytes::from_static(b"live shard"), + "configuration failure must not fall back to unbudgeted deletion" + ); + } + + #[tokio::test] + async fn rpc_cleanup_entries_share_disk_budget() { + const CHILD: &str = "RUSTFS_CLEANUP_RPC_TEST_CHILD"; + if std::env::var_os(CHILD).is_none() { + let output = tokio::task::spawn_blocking(|| { + std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "disk::disk_store::tests::rpc_cleanup_entries_share_disk_budget", + "--nocapture", + ]) + .env(CHILD, "1") + .env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true") + .env("RUSTFS_CLEANUP_ASYNC_THREADS", "1") + .env("RUSTFS_CLEANUP_DISK_MAX_PENDING", "1") + .env("RUSTFS_CLEANUP_DISK_WORKERS", "1") + .env("RUSTFS_CLEANUP_GC_WORKERS", "1") + .env("RUSTFS_CLEANUP_CPUS", "none") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP") + .output() + .unwrap() + }) + .await + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + return; + } + let dir = tempfile::tempdir().unwrap(); + let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.unwrap()); + let wrapper = LocalDiskWrapper::new(disk, false); + let volume = "cleanup-receiver"; + let path = "object/old-data"; + let body_path = format!("{path}/part.1"); + let receipt_path = format!("{path}/{}", cleanup_runtime::OLD_DATA_CLEANUP_RECEIPT_FILE); + wrapper.make_volume(volume).await.unwrap(); + wrapper + .write_all(volume, &body_path, Bytes::from_static(b"old body")) + .await + .unwrap(); + let read_lease = wrapper + .acquire_snapshot_lease(volume, path) + .await + .expect("ordinary GET lease"); + let (entered, entry) = tokio::sync::oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let occupied = tokio::spawn(cleanup_runtime::disk_operation(async move { + entered.send(()).unwrap(); + // Deliberately stall the only cleanup scheduler to prove that + // ordinary GET completion does not dispatch to it. + released + .recv_timeout(Duration::from_secs(30)) + .expect("release stalled scheduler"); + Ok(()) + })); + tokio::time::timeout(Duration::from_secs(20), entry).await.unwrap().unwrap(); + tokio::time::timeout(Duration::from_secs(5), wrapper.release_snapshot_lease(volume, path, read_lease)) + .await + .expect("ordinary GET release must progress while the cleanup scheduler is stalled") + .expect("release without reclamation"); + let fence_path = super::super::quota_mutation_fence_path(volume, "object"); + let fence = wrapper + .acquire_snapshot_lease(super::super::RUSTFS_META_BUCKET, &fence_path) + .await + .unwrap(); + wrapper + .release_snapshot_lease(super::super::RUSTFS_META_BUCKET, &fence_path, fence) + .await + .unwrap(); + let timeout = wrapper + .track_cleanup_mutation( + "delete", + DiskMetricMutation::Delete, + std::future::pending::>, + Duration::from_millis(10), + ) + .await; + assert!(matches!(timeout, Err(DiskError::Timeout))); + assert!(!wrapper.health.is_faulty(), "cleanup queue delay must not mark a healthy disk faulty"); + let options = DeleteOptions { + recursive: true, + ..Default::default() + }; + let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + // This is the same entry point used after RPC authentication and + // scanner publication-lease validation by handle_delete. + let result = wrapper + .delete_with_publication_guard(volume, path, options.clone(), Some(Arc::new(DropProbe(drops.clone())))) + .await; + assert!(matches!(result, Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock)); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert!(matches!(wrapper.delete_data_dir(volume, path, options.clone()).await, + Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock)); + assert!(matches!(wrapper.write_all(volume, &receipt_path, Bytes::from_static(b"receipt")).await, + Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock)); + assert_eq!(wrapper.read_all(volume, &body_path).await.unwrap(), Bytes::from_static(b"old body")); + release.send(()).unwrap(); + occupied.await.unwrap().unwrap(); + wrapper + .write_all(volume, &receipt_path, Bytes::from_static(b"receipt")) + .await + .unwrap(); + wrapper + .delete_with_publication_guard(volume, path, options, None) + .await + .unwrap(); + assert!(matches!(wrapper.read_all(volume, &body_path).await, Err(DiskError::FileNotFound))); + // Snapshot readers defer old-data cleanup. Saturation must release + // their token while retaining reclaim intent, rather than leaking a + // live token or performing an unbudgeted delete on the RPC runtime. + wrapper + .write_all(volume, &body_path, Bytes::from_static(b"leased body")) + .await + .unwrap(); + let lease = wrapper.acquire_snapshot_lease(volume, path).await.unwrap(); + assert_eq!( + wrapper + .delete_data_dir( + volume, + path, + DeleteOptions { + recursive: true, + ..Default::default() + } + ) + .await + .unwrap(), + DataDirDeleteStatus::Deferred + ); + let (entered, entry) = tokio::sync::oneshot::channel(); + let (release, released) = tokio::sync::oneshot::channel(); + let occupied = tokio::spawn(cleanup_runtime::disk_operation(async move { + entered.send(()).unwrap(); + released.await.unwrap(); + Ok(()) + })); + tokio::time::timeout(Duration::from_secs(20), entry).await.unwrap().unwrap(); + assert!(matches!(wrapper.release_snapshot_lease(volume, path, lease).await, + Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock)); + assert_eq!(wrapper.read_all(volume, &body_path).await.unwrap(), Bytes::from_static(b"leased body")); + release.send(()).unwrap(); + occupied.await.unwrap().unwrap(); + assert_eq!( + wrapper + .delete_data_dir( + volume, + path, + DeleteOptions { + recursive: true, + ..Default::default() + } + ) + .await + .unwrap(), + DataDirDeleteStatus::Deleted + ); + assert!(matches!(wrapper.read_all(volume, &body_path).await, Err(DiskError::FileNotFound))); + } + #[tokio::test] async fn owned_mutation_keeps_publication_guard_after_waiter_cancellation() { let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0)); diff --git a/crates/ecstore/src/disk/fs.rs b/crates/ecstore/src/disk/fs.rs index 7dbbaf14d..15a52ef7b 100644 --- a/crates/ecstore/src/disk/fs.rs +++ b/crates/ecstore/src/disk/fs.rs @@ -223,12 +223,6 @@ pub fn rename_std(from: impl AsRef, to: impl AsRef) -> io::Result<() std::fs::rename(from, to) } -#[cfg(any(not(windows), test))] -#[tracing::instrument(level = "debug", skip_all)] -pub async fn read_file(path: impl AsRef) -> io::Result> { - fs::read(path.as_ref()).await -} - // Bucket existence cache - reduces statx syscalls for repeated bucket checks /// Cache for bucket directory existence checks. @@ -614,28 +608,6 @@ mod tests { assert_eq!(content, "test content"); } - #[tokio::test] - async fn test_read_file() { - let temp_dir = TempDir::new().unwrap(); - let file_path = temp_dir.path().join("test_read.txt"); - - let test_content = b"This is test content for reading"; - tokio::fs::write(&file_path, test_content).await.unwrap(); - - // Read file - let read_content = read_file(&file_path).await.unwrap(); - assert_eq!(read_content, test_content); - } - - #[tokio::test] - async fn test_read_file_nonexistent() { - let temp_dir = TempDir::new().unwrap(); - let file_path = temp_dir.path().join("nonexistent.txt"); - - // Should fail for non-existent file - assert!(read_file(&file_path).await.is_err()); - } - #[tokio::test] async fn test_same_file() { let temp_dir = TempDir::new().unwrap(); diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 66cc3e10b..ea3a39ff8 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -221,8 +221,10 @@ struct ListingMetadataRead { has_namespace_child_candidate: bool, } -fn read_all_data_std(path: &Path) -> core::result::Result<(Vec, Option), ReadAllError> { - let mut file = std::fs::File::open(path).map_err(ReadAllError::Open)?; +fn read_all_data_std( + opened: std::io::Result, +) -> core::result::Result<(Vec, Option), ReadAllError> { + let mut file = opened.map_err(ReadAllError::Open)?; let metadata = file.metadata().map_err(|err| ReadAllError::Disk(to_file_error(err).into()))?; if metadata.is_dir() { @@ -5353,7 +5355,7 @@ pub struct LocalDisk { /// pathname is later covered by another mount. io_root: PathBuf, #[cfg(target_os = "linux")] - mount_lease: std::fs::File, + mount_lease: Arc, #[cfg(target_os = "linux")] mount_lease_mount_id: Option, /// Public path for callers that need the configured disk layout. Internal @@ -5610,24 +5612,6 @@ fn mount_id_from_mountinfo_contents(mountinfo: &str, path: &Path) -> Option } impl LocalDisk { - #[cfg(target_os = "linux")] - fn open_mount_lease(root: &Path) -> Result<(std::fs::File, PathBuf, Option)> { - use rustix::fs::{Mode, OFlags, open}; - use std::os::fd::AsRawFd as _; - - let fd = open( - root, - OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, - Mode::empty(), - ) - .map_err(std::io::Error::from) - .map_err(DiskError::from)?; - let lease = std::fs::File::from(fd); - let io_root = PathBuf::from(format!("/proc/self/fd/{}/.", lease.as_raw_fd())); - let mount_id = mount_id_for_fd(&lease); - Ok((lease, io_root, mount_id)) - } - #[cfg(not(target_os = "linux"))] fn open_mount_lease(root: &Path) -> Result { Ok(root.to_path_buf()) @@ -5711,7 +5695,11 @@ impl LocalDisk { let root = publication_root.path().to_path_buf(); #[cfg(target_os = "linux")] - let (mount_lease, io_root, mount_lease_mount_id) = Self::open_mount_lease(&root)?; + let (mount_lease, io_root, mount_lease_mount_id) = ( + publication_root.directory().clone(), + publication_root.io_path().to_path_buf(), + mount_id_for_fd(publication_root.directory()), + ); #[cfg(not(target_os = "linux"))] let io_root = Self::open_mount_lease(&root)?; @@ -5759,9 +5747,11 @@ impl LocalDisk { state = "format_path_resolved", "Local disk format path resolved" ); - let (format_data, format_meta) = read_file_exists(&io_format_path).await.inspect_err(|err| { - log_startup_disk_error("read_format_json", &io_format_path, err); - })?; + let (format_data, format_meta) = read_file_exists(&io_format_path, &publication_root) + .await + .inspect_err(|err| { + log_startup_disk_error("read_format_json", &io_format_path, err); + })?; let mut id = None; // let mut format_legacy = false; @@ -5907,7 +5897,13 @@ impl LocalDisk { let io_root = disk.io_root.clone(); let publication_root = disk.publication_root.clone(); - tokio::spawn(Self::cleanup_deleted_objects_loop(io_root, publication_root, exit_rx)); + tokio::spawn(Self::cleanup_deleted_objects_loop( + io_root, + publication_root, + #[cfg(target_os = "linux")] + disk.mount_lease.clone(), + exit_rx, + )); debug!( event = EVENT_DISK_LOCAL_STARTUP_CLEANUP, component = LOG_COMPONENT_ECSTORE, @@ -5923,30 +5919,53 @@ impl LocalDisk { async fn cleanup_deleted_objects_loop( root: PathBuf, publication_root: os::PublicationRoot, + #[cfg(target_os = "linux")] mount_lease: Arc, mut exit_rx: tokio::sync::broadcast::Receiver<()>, ) { let start_at = Instant::now() + DELETED_OBJECTS_CLEANUP_INTERVAL; let mut interval = interval_at(start_at, DELETED_OBJECTS_CLEANUP_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { tokio::select! { _ = interval.tick() => { - if let Err(err) = Self::cleanup_deleted_objects(root.clone()).await { + let root = root.clone(); + let publication_root = publication_root.clone(); + #[cfg(target_os = "linux")] + let mount_lease = mount_lease.clone(); + if let Err(err) = super::cleanup_runtime::run_gc(move |budget| async move { + // Queued/detached scans must keep /proc/self/fd paths + // attached to the original mount after LocalDisk drops. + #[cfg(target_os = "linux")] + let _mount_lease = mount_lease; + if let Err(err) = Self::cleanup_deleted_objects(root.clone(), &budget).await { + error!( + event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + task = "deleted_objects", + state = "failed", + error = ?err, + "Disk local background cleanup failed" + ); + } + if let Err(err) = Self::cleanup_stale_tmp_objects(root.clone(), &publication_root, &budget).await { + error!( + event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + task = "stale_tmp_objects", + state = "failed", + error = ?err, + "Disk local background cleanup failed" + ); + } + Ok(()) + }).await { error!( event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - task = "deleted_objects", - state = "failed", - error = ?err, - "Disk local background cleanup failed" - ); - } - if let Err(err) = Self::cleanup_stale_tmp_objects(root.clone(), &publication_root).await { - error!( - event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - task = "stale_tmp_objects", + task = "cleanup_dispatch", state = "failed", error = ?err, "Disk local background cleanup failed" @@ -6037,95 +6056,123 @@ impl LocalDisk { } } - async fn cleanup_stale_tmp_objects(root: PathBuf, publication_root: &os::PublicationRoot) -> Result<()> { - Self::cleanup_stale_tmp_objects_with_expiry(root, publication_root, STALE_TMP_OBJECT_EXPIRY).await + async fn cleanup_stale_tmp_objects( + root: PathBuf, + publication_root: &os::PublicationRoot, + budget: &super::cleanup_runtime::GcBudget, + ) -> Result<()> { + Self::cleanup_stale_tmp_objects_with_expiry(root, publication_root, STALE_TMP_OBJECT_EXPIRY, budget).await } async fn cleanup_stale_tmp_objects_with_expiry( root: PathBuf, publication_root: &os::PublicationRoot, expiry: Duration, + budget: &super::cleanup_runtime::GcBudget, ) -> Result<()> { let tmp_path = Self::meta_path(&root, RUSTFS_META_TMP_BUCKET); - let mut entries = match fs::read_dir(&tmp_path).await { + let mut entries = match budget.step(async { Ok(fs::read_dir(&tmp_path).await?) }).await { Ok(entries) => entries, Err(e) => { - if e.kind() == ErrorKind::NotFound { + if matches!(&e, DiskError::Io(err) if err.kind() == ErrorKind::NotFound) { return Ok(()); } - return Err(e.into()); + return Err(e); } }; - while let Some(entry) = entries.next_entry().await? { - let name = entry.file_name().to_string_lossy().to_string(); - if name.is_empty() || name == "." || name == ".." || name == ".trash" { - continue; + loop { + let more = budget + .step(async { + let Some(entry) = entries.next_entry().await? else { + return Ok(false); + }; + Self::cleanup_stale_tmp_entry(entry, &root, publication_root, expiry).await?; + Ok(true) + }) + .await?; + if !more { + break; } - - let file_type = entry.file_type().await?; - if !file_type.is_dir() { - continue; - } - - let Some(age) = entry - .metadata() - .await? - .modified() - .ok() - .and_then(|modified| modified.elapsed().ok()) - else { - continue; - }; - if age <= expiry { - continue; - } - - let target_path = Self::meta_path(&root, RUSTFS_META_TMP_DELETED_BUCKET).join(Uuid::new_v4().to_string()); - rename_all(entry.path(), target_path, Self::meta_path(&root, RUSTFS_META_BUCKET), publication_root).await?; } Ok(()) } - async fn cleanup_deleted_objects(root: PathBuf) -> Result<()> { + async fn cleanup_stale_tmp_entry( + entry: fs::DirEntry, + root: &Path, + publication_root: &os::PublicationRoot, + expiry: Duration, + ) -> Result<()> { + let name = entry.file_name().to_string_lossy().to_string(); + if name.is_empty() || name == "." || name == ".." || name == ".trash" || !entry.file_type().await?.is_dir() { + return Ok(()); + } + let Some(age) = entry + .metadata() + .await? + .modified() + .ok() + .and_then(|modified| modified.elapsed().ok()) + else { + return Ok(()); + }; + if age <= expiry { + return Ok(()); + } + let target_path = Self::meta_path(root, RUSTFS_META_TMP_DELETED_BUCKET).join(Uuid::new_v4().to_string()); + rename_all(entry.path(), target_path, Self::meta_path(root, RUSTFS_META_BUCKET), publication_root).await?; + Ok(()) + } + + async fn cleanup_deleted_objects(root: PathBuf, budget: &super::cleanup_runtime::GcBudget) -> Result<()> { let trash = Self::meta_path(&root, RUSTFS_META_TMP_DELETED_BUCKET); - let mut entries = match fs::read_dir(&trash).await { + let mut entries = match budget.step(async { Ok(fs::read_dir(&trash).await?) }).await { Ok(entries) => entries, Err(e) => { - if e.kind() == ErrorKind::NotFound { + if matches!(&e, DiskError::Io(err) if err.kind() == ErrorKind::NotFound) { return Ok(()); } - return Err(e.into()); + return Err(e); } }; - while let Some(entry) = entries.next_entry().await? { - let name = entry.file_name().to_string_lossy().to_string(); - if name.is_empty() || name == "." || name == ".." { - continue; - } - - let file_type = entry.file_type().await?; - - let path = trash.join(name); - - if file_type.is_dir() { - if let Err(e) = tokio::fs::remove_dir_all(path).await - && e.kind() != ErrorKind::NotFound - { - return Err(e.into()); - } - } else if let Err(e) = tokio::fs::remove_file(path).await - && e.kind() != ErrorKind::NotFound - { - return Err(e.into()); + loop { + let more = budget + .step(async { + let Some(entry) = entries.next_entry().await? else { + return Ok(false); + }; + Self::cleanup_trash_entry(entry, &trash).await?; + Ok(true) + }) + .await?; + if !more { + break; } } Ok(()) } + async fn cleanup_trash_entry(entry: fs::DirEntry, trash: &Path) -> Result<()> { + let name = entry.file_name().to_string_lossy().to_string(); + if name.is_empty() || name == "." || name == ".." { + return Ok(()); + } + let path = trash.join(name); + let result = if entry.file_type().await?.is_dir() { + fs::remove_dir_all(path).await + } else { + fs::remove_file(path).await + }; + match result { + Err(err) if err.kind() != ErrorKind::NotFound => Err(err.into()), + _ => Ok(()), + } + } + fn is_valid_volname(volname: &str) -> bool { if volname.len() < 3 { return false; @@ -6252,6 +6299,19 @@ impl LocalDisk { self.local_disk_object_path(self.io_root(), bucket, key) } + /// Only for operations that open through PublicationRoot or a commit guard: + /// containment and symlink checks happen with the actual open, off the runtime. + fn io_get_object_open_path(&self, bucket: &str, key: &str) -> Result { + #[cfg(target_os = "linux")] + { + let (bucket_path, path) = build_local_disk_object_path(self.io_root(), bucket, key); + check_local_disk_object_path_components(self.io_root(), &bucket_path, &path)?; + Ok(path) + } + #[cfg(not(target_os = "linux"))] + self.io_get_object_path(bucket, key) + } + fn io_get_bucket_path(&self, bucket: &str) -> Result { self.local_disk_bucket_path(self.io_root(), bucket) } @@ -6410,7 +6470,7 @@ impl LocalDisk { let mut meta = FileMeta::new(); if !fi.fresh { - let (buf, _) = read_file_exists(&p).await?; + let (buf, _) = read_file_exists(&p, &self.publication_root).await?; if !buf.is_empty() { let _ = meta.unmarshal_msg(&buf).map_err(|_| { meta = FileMeta::new(); @@ -6436,6 +6496,36 @@ impl LocalDisk { Ok(()) } + /// Keep the common GET completion on its caller. Only a last-reader + /// release with pending reclamation needs an owned cleanup task. Leave + /// that token untouched so the slow path owns the complete transition. + pub(in crate::disk) async fn release_snapshot_lease_without_cleanup( + &self, + volume: &str, + path: &str, + token: SnapshotLeaseToken, + ) -> bool { + let key = SnapshotLeaseKey { + volume: volume.to_owned(), + path: path.to_owned(), + }; + let mut registry = self.snapshot_leases.lock(); + let Some(entry) = registry.entries.get_mut(&key) else { + return true; + }; + if !entry.deleting + && entry.pending_delete.is_some() + && (entry.tokens.is_empty() || (entry.tokens.len() == 1 && entry.tokens.contains(&token))) + { + return false; + } + entry.tokens.remove(&token); + if entry.tokens.is_empty() && !entry.deleting { + registry.entries.remove(&key); + } + true + } + async fn delete_data_dir_with_namespace_owner( &self, volume: &str, @@ -7489,10 +7579,11 @@ impl LocalDisk { // - metadata failure -> to_file_error // - parse failure -> propagated verbatim from read_xl_meta_no_data_sync (`?`) let path = file_path.as_ref().to_path_buf(); + let root = self.publication_root.clone(); let (data, modtime) = spawn_blocking_with_disk_io_permit(io_permit, move || -> Result<(Vec, Option)> { // Read-only open, equivalent to O_RDONLY (get_readonly_options only sets read(true)). - let mut f = std::fs::File::open(&path).map_err(to_file_error)?; + let mut f = root.open_readonly(&path).map_err(to_file_error)?; let meta = f.metadata().map_err(to_file_error)?; @@ -7546,6 +7637,7 @@ impl LocalDisk { let metadata_path = self.io_get_object_path(bucket, path.to_string_lossy().as_ref()); // A part's existence check, metadata read and decode share one dispatch. // Keep open errors unmapped for the existing missing-volume fallback. + let root = self.publication_root.clone(); let result = spawn_blocking_with_disk_io_permit(io_permit, move || -> Result<_> { let part_error = |error: String| ObjectPartInfo { number: num, @@ -7558,7 +7650,7 @@ impl LocalDisk { // Invalid metadata paths remain request errors, but missing data wins // first, as it does in the serial reader. let metadata_path = metadata_path?; - Ok(read_all_data_std(&metadata_path) + Ok(read_all_data_std(root.open_readonly(&metadata_path)) .map(|(data, _)| ObjectPartInfo::unmarshal(&data).unwrap_or_else(|err| part_error(err.to_string())))) }) .await; @@ -7578,8 +7670,9 @@ impl LocalDisk { let object_dir = self.io_get_object_path(volume, object_name)?; let metadata_path = object_dir.join(STORAGE_FORMAT_FILE); let volume_dir = self.io_get_bucket_path(volume)?; + let root = self.publication_root.clone(); let result = spawn_blocking_with_disk_io_permit(io_permit, move || { - let (bytes, _) = read_all_data_std(&metadata_path)?; + let (bytes, _) = read_all_data_std(root.open_readonly(&metadata_path))?; let file_meta = FileMeta::load(&bytes).ok(); let data_dirs: HashSet = file_meta .as_ref() @@ -7677,7 +7770,8 @@ impl LocalDisk { // gating the volume fallback on the open error alone is equivalent to // the original code, where the fallback lived solely in the open match arm. let path = file_path.as_ref().to_path_buf(); - let res = spawn_blocking_with_disk_io_permit(io_permit, move || read_all_data_std(&path)) + let root = self.publication_root.clone(); + let res = spawn_blocking_with_disk_io_permit(io_permit, move || read_all_data_std(root.open_readonly(&path))) .await .map_err(DiskError::from)?; @@ -8505,16 +8599,16 @@ impl LocalDisk { os::make_dir_all(parent, skip_parent).await?; } + let root = self.publication_root.clone(); tokio::task::spawn_blocking(move || { #[cfg(test)] run_owned_file_write_before_open(&path); - let mut file = std::fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(&path) - .map_err(to_file_error)?; + let (mut file, parent_handle) = if sync == SyncMode::FileAndDir { + root.create_truncate_with_parent(&path).map_err(to_file_error)? + } else { + (root.create_truncate(&path).map_err(to_file_error)?, None) + }; std::io::Write::write_all(&mut file, buf.as_ref()).map_err(to_file_error)?; if sync != SyncMode::None { file.sync_data().map_err(to_file_error)?; @@ -8523,7 +8617,7 @@ impl LocalDisk { if sync == SyncMode::FileAndDir && let Some(parent) = path.parent() { - os::fsync_dir_std(parent).map_err(to_file_error)?; + os::fsync_directory_handle_std(parent, parent_handle.as_ref()).map_err(to_file_error)?; } } Ok::<_, std::io::Error>(()) @@ -8548,7 +8642,18 @@ impl LocalDisk { os::make_dir_all(parent, skip_parent).await?; } - let f = super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?; + let f = if mode == O_CREATE | O_WRONLY | O_TRUNC { + let path = path.as_ref().to_path_buf(); + let root = self.publication_root.clone(); + File::from_std( + tokio::task::spawn_blocking(move || root.create_truncate(&path)) + .await + .map_err(DiskError::from)? + .map_err(to_file_error)?, + ) + } else { + super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)? + }; Ok(f) } @@ -9424,9 +9529,9 @@ fn file_meta_counts_toward_limit(meta: &FileMeta) -> bool { } // Filter std::io::ErrorKind::NotFound -async fn read_file_exists(path: impl AsRef) -> Result<(Bytes, Option)> { +async fn read_file_exists(path: impl AsRef, root: &os::PublicationRoot) -> Result<(Bytes, Option)> { let p = path.as_ref(); - let (data, meta) = match read_file_all(&p).await { + let (data, meta) = match read_file_all(&p, root).await { Ok((data, meta)) => (data, Some(meta)), Err(e) => { if e == Error::FileNotFound { @@ -9445,32 +9550,28 @@ async fn read_file_exists(path: impl AsRef) -> Result<(Bytes, Option) -> Result<(Bytes, Metadata)> { - let p = path.as_ref(); - let meta = read_file_metadata(&path).await?; - - let data = fs::read(&p) - .await - .inspect_err(|err| { - log_startup_disk_io_error("read_file_all", p, err); - }) - .map_err(to_file_error)?; - - Ok((data.into(), meta)) -} - -async fn read_file_metadata(p: impl AsRef) -> Result { - let path = p.as_ref(); - let meta = fs::metadata(path) - .await - .inspect_err(|err| { - if err.kind() != ErrorKind::NotFound { - log_startup_disk_io_error("read_file_metadata", path, err); - } - }) - .map_err(to_file_error)?; - - Ok(meta) +async fn read_file_all(path: impl AsRef, root: &os::PublicationRoot) -> Result<(Bytes, Metadata)> { + let path = path.as_ref().to_path_buf(); + let root = root.clone(); + tokio::task::spawn_blocking(move || { + let result = (|| { + let mut file = root.open_readonly(&path)?; + let metadata = file.metadata()?; + let mut data = Vec::new(); + std::io::Read::read_to_end(&mut file, &mut data)?; + Ok::<_, std::io::Error>((Bytes::from(data), metadata)) + })(); + result + .inspect_err(|err| { + if err.kind() != ErrorKind::NotFound { + log_startup_disk_io_error("read_file_all", &path, err); + } + }) + .map_err(to_file_error) + .map_err(DiskError::from) + }) + .await + .map_err(DiskError::from)? } fn skip_access_checks(p: impl AsRef) -> bool { @@ -9552,13 +9653,19 @@ fn check_local_disk_valid_path(root: &Path, path: impl AsRef) -> Result<() #[cfg(target_os = "linux")] fn check_local_disk_valid_object_path_at(root: &Path, root_fd: &std::fs::File, bucket_path: &Path, path: &Path) -> Result<()> { + check_local_disk_object_path_components(root, bucket_path, path)?; + reject_local_disk_symlink_components_at(root, root_fd, &normalize_path_components(path)) +} + +#[cfg(target_os = "linux")] +fn check_local_disk_object_path_components(root: &Path, bucket_path: &Path, path: &Path) -> Result<()> { let bucket_path = normalize_path_components(bucket_path); let path = normalize_path_components(path); if !bucket_path.starts_with(root) || !path.starts_with(&bucket_path) { return Err(DiskError::InvalidPath); } - reject_local_disk_symlink_components_at(root, root_fd, &path) + Ok(()) } #[cfg(target_os = "linux")] @@ -10407,9 +10514,9 @@ impl DiskAPI for LocalDisk { } } - let p = self.io_get_object_path(volume, path)?; + let p = self.io_get_object_open_path(volume, path)?; - let (data, _) = read_file_all(&p).await?; + let (data, _) = read_file_all(&p, &self.publication_root).await?; Ok(data) } @@ -11701,8 +11808,17 @@ impl DiskAPI for LocalDisk { entry.deleting = true; opts }; + // Release the reader token even under saturation. A rejected reclaim + // retains pending_delete (and its on-disk receipt) for a later retry. + // Do not wait for execution while a releasing reader may own locks. let claim = SnapshotLeaseDeleteClaim::new(&self.snapshot_leases, key); - let result = self.delete_unleased(volume, path, &opts).await; + let result = match super::cleanup_runtime::try_disk_execution() { + Ok(permit) => { + let _permit = permit; + self.delete_unleased(volume, path, &opts).await + } + Err(err) => Err(err.into()), + }; claim.finish(result.is_ok()); result } @@ -11976,7 +12092,7 @@ impl DiskAPI for LocalDisk { }; // if req.metadata_only {} - match read_file_all(&fpath).await { + match read_file_all(&fpath, &self.publication_root).await { Ok((data, meta)) => { found += 1; @@ -13572,10 +13688,15 @@ mod test { disk.wait_for_startup_cleanup().await; assert_eq!(disk.startup_cleanup_ready.load(Ordering::Acquire), 1); - LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().join("missing-root"), &publication_root, Duration::ZERO) - .await - .expect("missing tmp path should be a cleanup no-op"); - LocalDisk::cleanup_deleted_objects(dir.path().join("missing-root")) + LocalDisk::cleanup_stale_tmp_objects_with_expiry( + dir.path().join("missing-root"), + &publication_root, + Duration::ZERO, + &crate::disk::cleanup_runtime::GcBudget::default(), + ) + .await + .expect("missing tmp path should be a cleanup no-op"); + LocalDisk::cleanup_deleted_objects(dir.path().join("missing-root"), &crate::disk::cleanup_runtime::GcBudget::default()) .await .expect("missing trash path should be a cleanup no-op"); @@ -13588,9 +13709,14 @@ mod test { fs::create_dir_all(&trash_root).await.expect("trash dir should be created"); backdate_mtime(&stale_dir, Duration::from_secs(10)); - LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().to_path_buf(), &publication_root, Duration::ZERO) - .await - .expect("stale tmp directory should move to trash"); + LocalDisk::cleanup_stale_tmp_objects_with_expiry( + dir.path().to_path_buf(), + &publication_root, + Duration::ZERO, + &crate::disk::cleanup_runtime::GcBudget::default(), + ) + .await + .expect("stale tmp directory should move to trash"); assert!(!stale_dir.exists(), "stale tmp directory should be moved away"); assert!(live_file.exists(), "plain tmp files should be ignored by stale dir cleanup"); @@ -13600,7 +13726,7 @@ mod test { fs::create_dir_all(trash_root.join("trash-dir")) .await .expect("trash dir should be created"); - LocalDisk::cleanup_deleted_objects(dir.path().to_path_buf()) + LocalDisk::cleanup_deleted_objects(dir.path().to_path_buf(), &crate::disk::cleanup_runtime::GcBudget::default()) .await .expect("trash cleanup should remove files and directories"); assert!( @@ -20705,9 +20831,14 @@ mod test { // Backdate after the write above: creating stale/data refreshes the // scanned tmp/stale directory's mtime. backdate_mtime(&tmp.join("stale"), Duration::from_secs(10)); - LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().to_path_buf(), &publication_root, Duration::ZERO) - .await - .expect("operation should succeed"); + LocalDisk::cleanup_stale_tmp_objects_with_expiry( + dir.path().to_path_buf(), + &publication_root, + Duration::ZERO, + &crate::disk::cleanup_runtime::GcBudget::default(), + ) + .await + .expect("operation should succeed"); assert!(!tmp.join("stale").exists()); assert!(trash.exists()); @@ -20734,9 +20865,14 @@ mod test { fs::write(&fresh_dir, b"temporary").await.expect("operation should succeed"); fs::write(®ular_file, b"keep").await.expect("operation should succeed"); - LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().to_path_buf(), &publication_root, Duration::from_secs(60)) - .await - .expect("operation should succeed"); + LocalDisk::cleanup_stale_tmp_objects_with_expiry( + dir.path().to_path_buf(), + &publication_root, + Duration::from_secs(60), + &crate::disk::cleanup_runtime::GcBudget::default(), + ) + .await + .expect("operation should succeed"); assert!(tmp.join("fresh").exists()); assert!(regular_file.exists()); @@ -20745,6 +20881,44 @@ mod test { assert!(entries.next_entry().await.expect("operation should succeed").is_none()); } + #[cfg(target_os = "linux")] + #[tokio::test] + async fn background_cleanup_keeps_mount_lease_after_disk_drop() { + let dir = tempfile::tempdir().unwrap(); + let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(); + let disk = LocalDisk::new(&endpoint, false).await.unwrap(); + let root = disk.io_root.clone(); + let trash_entry = LocalDisk::meta_path(&root, RUSTFS_META_TMP_DELETED_BUCKET).join("detached-gc"); + fs::create_dir_all(&trash_entry).await.unwrap(); + fs::write(trash_entry.join("part.1"), b"garbage").await.unwrap(); + let live_file = root.join("keep"); + fs::write(&live_file, b"live").await.unwrap(); + let (exit, exited) = tokio::sync::broadcast::channel(1); + tokio::time::pause(); + let cleanup = tokio::spawn(LocalDisk::cleanup_deleted_objects_loop( + root.clone(), + disk.publication_root.clone(), + disk.mount_lease.clone(), + exited, + )); + tokio::task::yield_now().await; + drop(disk); + tokio::task::yield_now().await; + assert!(live_file.exists(), "the GC owner must retain the original /proc/self/fd root"); + tokio::time::advance(DELETED_OBJECTS_CLEANUP_INTERVAL + Duration::from_secs(1)).await; + tokio::time::resume(); + timeout(Duration::from_secs(20), async { + while trash_entry.exists() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("detached cleanup must finish against the pinned mount"); + assert_eq!(fs::read(&live_file).await.unwrap(), b"live"); + exit.send(()).unwrap(); + timeout(Duration::from_secs(20), cleanup).await.unwrap().unwrap(); + } + #[tokio::test(start_paused = true)] async fn cleanup_loop_interval_does_not_tick_immediately() { let start_at = Instant::now() + DELETED_OBJECTS_CLEANUP_INTERVAL; @@ -23887,10 +24061,12 @@ mod test { #[tokio::test] async fn test_read_file_exists() { - let test_file = "./test_read_exists.txt"; + let directory = tempfile::tempdir().expect("test directory"); + let root = os::PublicationRoot::new(directory.path()).expect("publication root"); + let test_file = &directory.path().join("read_exists"); // Test non-existent file - let (data, metadata) = read_file_exists(test_file).await.expect("operation should succeed"); + let (data, metadata) = read_file_exists(test_file, &root).await.expect("operation should succeed"); assert!(data.is_empty()); assert!(metadata.is_none()); @@ -23898,7 +24074,7 @@ mod test { fs::write(test_file, b"test content").await.expect("operation should succeed"); // Test existing file - let (data, metadata) = read_file_exists(test_file).await.expect("operation should succeed"); + let (data, metadata) = read_file_exists(test_file, &root).await.expect("operation should succeed"); assert_eq!(data.as_ref(), b"test content"); assert!(metadata.is_some()); @@ -23908,14 +24084,16 @@ mod test { #[tokio::test] async fn test_read_file_all() { - let test_file = "./test_read_all.txt"; + let directory = tempfile::tempdir().expect("test directory"); + let root = os::PublicationRoot::new(directory.path()).expect("publication root"); + let test_file = &directory.path().join("read_all"); let test_content = b"test content for read_all"; // Create test file fs::write(test_file, test_content).await.expect("operation should succeed"); // Test reading file - let (data, metadata) = read_file_all(test_file).await.expect("operation should succeed"); + let (data, metadata) = read_file_all(test_file, &root).await.expect("operation should succeed"); assert_eq!(data.as_ref(), test_content); assert!(metadata.is_file()); assert_eq!(metadata.len(), test_content.len() as u64); @@ -23924,22 +24102,6 @@ mod test { let _ = fs::remove_file(test_file).await; } - #[tokio::test] - async fn test_read_file_metadata() { - let test_file = "./test_metadata.txt"; - - // Create test file - fs::write(test_file, b"test").await.expect("operation should succeed"); - - // Test reading metadata - let metadata = read_file_metadata(test_file).await.expect("operation should succeed"); - assert!(metadata.is_file()); - assert_eq!(metadata.len(), 4); // "test" is 4 bytes - - // Clean up - let _ = fs::remove_file(test_file).await; - } - #[test] fn test_is_root_path() { // Unix root path @@ -24528,9 +24690,14 @@ mod test { Bytes::from_static(b"later") ); - disk.release_snapshot_lease(volume, &data_dir, renewed) - .await - .expect("renewed lease release should succeed"); + assert!( + disk.release_snapshot_lease_without_cleanup(volume, &data_dir, renewed).await, + "a non-final reader release must not dispatch cleanup" + ); + assert!( + !disk.release_snapshot_lease_without_cleanup(volume, &data_dir, second).await, + "the final reader must retain its token for owned deferred reclamation" + ); assert!( disk.read_all(volume, &first_part).await.is_ok(), "one remaining lease must keep the data directory" diff --git a/crates/ecstore/src/disk/local/commit.rs b/crates/ecstore/src/disk/local/commit.rs index 1fa6b8803..56ef71101 100644 --- a/crates/ecstore/src/disk/local/commit.rs +++ b/crates/ecstore/src/disk/local/commit.rs @@ -47,6 +47,33 @@ use tokio::fs; use tracing::{info, warn}; use uuid::Uuid; +fn transitioned_old_current_source( + dst_meta_existed: bool, + dst_meta_unparsable: bool, + xlmeta: &FileMeta, + volume: &str, + path: &str, + version_id: Uuid, +) -> (bool, Option) { + if dst_meta_unparsable { + return (false, None); + } + if !dst_meta_existed { + return (true, None); + } + let Ok(old_current) = xlmeta.into_fileinfo(volume, path, "", false, false, true) else { + return (false, None); + }; + if old_current.version_id.unwrap_or_default() != version_id { + return (true, None); + } + if old_current.transition_status == rustfs_filemeta::TRANSITION_COMPLETE { + (true, Some(old_current)) + } else { + (true, None) + } +} + /// Hold later repair publications after admitting one baseline object. The /// fixture arms this on one replacement disk before rejoining the cluster. #[cfg(feature = "e2e-test-hooks")] @@ -214,7 +241,7 @@ pub(super) async fn lock_rename_commit_directories( publication_root: &os::PublicationRoot, mutation_lease: Arc, ) -> Result { - #[cfg(windows)] + #[cfg(any(windows, target_os = "linux"))] let result = { let source_parent = source_parent.to_path_buf(); let destination_parent = destination_parent.to_path_buf(); @@ -222,7 +249,7 @@ pub(super) async fn lock_rename_commit_directories( let publication_root = publication_root.clone(); os::run_blocking_namespace_operation(mutation_lease, move || { let result = os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root); - #[cfg(test)] + #[cfg(all(test, windows))] if result.is_ok() { run_destination_commit_directory_preparation(&destination_parent); } @@ -230,7 +257,7 @@ pub(super) async fn lock_rename_commit_directories( }) .await }; - #[cfg(not(windows))] + #[cfg(not(any(windows, target_os = "linux")))] let result = { let _ = mutation_lease; os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root) @@ -244,12 +271,12 @@ pub(super) async fn lock_rename_commit_directories( result.map_err(to_file_error).map_err(DiskError::from) } -async fn read_rename_destination_metadata( +async fn prepare_rename_destination_metadata( file_path: &Path, rename_commit_guard: &os::RenameCommitGuard, mutation_lease: Arc, ) -> Result> { - #[cfg(windows)] + #[cfg(any(windows, target_os = "linux"))] let result = { let file_path = file_path.to_path_buf(); let rename_commit_guard = rename_commit_guard.clone(); @@ -258,10 +285,10 @@ async fn read_rename_destination_metadata( }) .await }; - #[cfg(not(windows))] + #[cfg(not(any(windows, target_os = "linux")))] let _ = (rename_commit_guard, mutation_lease); - #[cfg(not(windows))] - let result = match super::super::fs::read_file(file_path).await { + #[cfg(not(any(windows, target_os = "linux")))] + let result = match fs::read(file_path).await { Ok(data) => Ok(Some(data)), Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), Err(err) => Err(err), @@ -398,7 +425,7 @@ impl LocalDisk { // xl.meta path let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?; - let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; + let dst_file_path = self.io_get_object_open_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?; // data_dir path let has_data_dir_path = { @@ -453,8 +480,7 @@ impl LocalDisk { fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?; } // Acquire the common trees before reading destination metadata. On - // Windows this pins the object directory identity across metadata - // preparation, data publication, rollback backup, and final commit. + // Windows this pins the object directory identity across publication. let rename_commit_guard = lock_rename_commit_directories( src_file_parent, dst_file_parent, @@ -472,9 +498,12 @@ impl LocalDisk { source_volume_dir: &src_volume_dir, data_paths: has_data_dir_path.as_ref(), }; - let has_dst_buf = - read_rename_destination_metadata(commit.destination_metadata, &commit.directory_guard, commit.mutation_lease.clone()) - .await?; + let has_dst_buf = prepare_rename_destination_metadata( + commit.destination_metadata, + &commit.directory_guard, + commit.mutation_lease.clone(), + ) + .await?; if no_inline { // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta @@ -508,6 +537,14 @@ impl LocalDisk { } let version_id = fi.version_id.unwrap_or_default(); + let (old_current_source_checked, old_current_source) = transitioned_old_current_source( + has_dst_buf.is_some(), + dst_meta_unparsable, + &xlmeta, + dst_volume, + dst_path, + version_id, + ); let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); let rollback_data_dir = has_old_data_dir.or_else(|| { @@ -798,7 +835,9 @@ impl LocalDisk { && let Some(parent) = dst_file_path.parent() { let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dst_dir_group_commit(parent, Some(commit.mutation_lease.clone())).await { + if let Err(err) = + os::fsync_commit_directory(parent, &commit.directory_guard, commit.mutation_lease.clone(), None).await + { rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, fsync_started, @@ -897,6 +936,8 @@ impl LocalDisk { cleanup_data_dir: has_old_data_dir, sign: version_signature, old_current_size, + old_current_source_checked, + old_current_source, }) } else { // Inline metadata preparation is blocking. The transaction lease is @@ -910,6 +951,8 @@ impl LocalDisk { None }; let dst_path_for_failpoint = dst_path.to_string(); + let dst_volume_for_source = dst_volume.to_string(); + let dst_path_for_source = dst_path.to_string(); #[cfg(windows)] let source_parent = src_file_parent.to_path_buf(); let rename_commit_guard_for_preparation = commit.directory_guard.clone(); @@ -954,6 +997,14 @@ impl LocalDisk { }; let version_id = fi.version_id.unwrap_or_default(); + let (old_current_source_checked, old_current_source) = transitioned_old_current_source( + has_dst_buf.is_some(), + dst_meta_unparsable, + &xlmeta, + &dst_volume_for_source, + &dst_path_for_source, + version_id, + ); let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id)); let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok(); let rollback_data_dir = old_data_dir.or_else(|| { @@ -1003,6 +1054,8 @@ impl LocalDisk { old_data_dir, version_signature, old_current_size, + old_current_source_checked, + old_current_source, staged_rollback_path, has_dst_buf.is_none(), prepared_metadata_source, @@ -1022,6 +1075,8 @@ impl LocalDisk { cleanup_data_dir, version_signature, old_current_size, + old_current_source_checked, + old_current_source, mut local_rollback_path, destination_was_absent, prepared_metadata_source, @@ -1147,10 +1202,11 @@ impl LocalDisk { && let Some(dst_parent) = dst_file_path.parent() { let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit( + if let Err(err) = os::fsync_commit_directory( dst_parent, + &commit.directory_guard, commit.mutation_lease.clone(), - admission, + Some(admission), ) .await { @@ -1262,6 +1318,8 @@ impl LocalDisk { cleanup_data_dir, sign: version_signature, old_current_size, + old_current_source_checked, + old_current_source, }) } } diff --git a/crates/ecstore/src/disk/local/delete_recovery.rs b/crates/ecstore/src/disk/local/delete_recovery.rs index f8f4e3485..00f268278 100644 --- a/crates/ecstore/src/disk/local/delete_recovery.rs +++ b/crates/ecstore/src/disk/local/delete_recovery.rs @@ -1953,10 +1953,7 @@ async fn barrier(volume: &str, object: &str, transaction: Uuid, stage: &str, dri .map_err(to_file_error)? == b"io_error" { - return Err(DiskError::Io(std::io::Error::new( - ErrorKind::Other, - "injected delete recovery barrier failure", - ))); + return Err(DiskError::Io(std::io::Error::other("injected delete recovery barrier failure"))); } Ok(()) } diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index c1166f405..0b90fe6a7 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -14,6 +14,7 @@ // #730: disk abstractions still carry staged health and direct-I/O migration paths. +pub(crate) mod cleanup_runtime; pub mod disk_store; pub mod endpoint; pub mod error; @@ -1776,6 +1777,14 @@ pub struct RenameDataResp { /// "cannot vote", never as `Absent`. #[serde(default)] pub old_current_size: Option, + /// Whether this disk decoded enough of the overwritten current version to + /// decide if a transitioned source has to be returned. + #[serde(default)] + pub old_current_source_checked: bool, + /// Transitioned old-current source used by PUT free-version cleanup receipt. + /// Empty when the old current is not transitioned or no source was known. + #[serde(default)] + pub old_current_source: Option, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -2517,6 +2526,8 @@ mod tests { cleanup_data_dir: Some(uuid), sign: Some(signature.clone()), old_current_size: Some(OldCurrentSize::Present(42)), + old_current_source_checked: false, + old_current_source: None, }; assert_eq!(resp.old_data_dir, Some(uuid)); @@ -2537,6 +2548,8 @@ mod tests { cleanup_data_dir: Some(Uuid::new_v4()), sign: Some(vec![0x01, 0x02, 0x03]), old_current_size, + old_current_source_checked: false, + old_current_source: None, }; let encoded = rmp_serde::encode::to_vec_named(&resp).expect("named msgpack should encode"); @@ -2574,6 +2587,8 @@ mod tests { assert_eq!(decoded.cleanup_data_dir, None); assert_eq!(decoded.sign, legacy.sign); assert_eq!(decoded.old_current_size, None); + assert!(!decoded.old_current_source_checked); + assert!(decoded.old_current_source.is_none()); } /// Test constants diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index 14a56b598..9c94610be 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -486,18 +486,25 @@ pub(crate) mod prepare_rename_test_hooks { /// Fsync a directory so recently created or renamed entries survive power loss. /// No-op on non-Unix platforms where directories cannot be opened for syncing. pub fn fsync_dir_std(dir: impl AsRef) -> io::Result<()> { + fsync_directory_handle_std(dir.as_ref(), None) +} + +pub(crate) fn fsync_directory_handle_std(dir: &Path, opened: Option<&std::fs::File>) -> io::Result<()> { #[cfg(test)] - fsync_dir_recorder::record(dir.as_ref()); + fsync_dir_recorder::record(dir); #[cfg(unix)] { #[cfg(test)] - if let Some(kind) = fsync_dir_recorder::take_failure(dir.as_ref()) { + if let Some(kind) = fsync_dir_recorder::take_failure(dir) { return Err(io::Error::from(kind)); } - std::fs::File::open(dir.as_ref())?.sync_all()?; + match opened { + Some(file) => file.sync_all()?, + None => std::fs::File::open(dir)?.sync_all()?, + } } #[cfg(not(unix))] - let _ = dir; + let _ = (dir, opened); Ok(()) } @@ -670,6 +677,8 @@ fn file_fdatasync_group_commit_wait() -> Duration { #[derive(Clone, Eq, Hash, PartialEq)] struct DstDirFsyncGroupKey { canonical_path: PathBuf, + // A batch must execute in the same resource class as its producers. + isolated: bool, #[cfg(unix)] dev: u64, #[cfg(unix)] @@ -686,13 +695,17 @@ impl DstDirFsyncGroupKey { use std::os::unix::fs::MetadataExt; Ok(Self { canonical_path, + isolated: FSYNC_ON_CURRENT_RUNTIME.get(), dev: metadata.dev(), ino: metadata.ino(), }) } #[cfg(not(unix))] { - Ok(Self { canonical_path }) + Ok(Self { + canonical_path, + isolated: FSYNC_ON_CURRENT_RUNTIME.get(), + }) } } } @@ -729,6 +742,17 @@ impl OpenedDstDirFsyncGroup { Ok(Self { key, dir }) } } + + #[cfg(target_os = "linux")] + fn from_file(dir: &Path, file: Arc) -> io::Result { + let key = DstDirFsyncGroupKey::from_metadata(dir.canonicalize()?, file.metadata()?)?; + Ok(Self { + #[cfg(test)] + dir: key.canonical_path.clone(), + key, + dir_file: file, + }) + } } struct DstDirFsyncWaiter { @@ -1002,19 +1026,45 @@ fn run_dst_dir_fsync_group_worker(group: Arc) -> impl std::fut } } +#[cfg(any(test, not(target_os = "linux")))] async fn fsync_dst_dir_group_commit_with_enabled( dir: impl AsRef, enabled: bool, namespace_owner: Option>, +) -> io::Result<()> { + fsync_dst_dir_group_commit_opened(dir.as_ref(), enabled, namespace_owner, None).await +} + +async fn fsync_dst_dir_group_commit_opened( + dir: &Path, + enabled: bool, + namespace_owner: Option>, + opened: Option>, ) -> io::Result<()> { if !enabled { - return fsync_dir_with_owner(dir.as_ref(), namespace_owner).await; + if let Some(file) = opened { + let dir = dir.to_path_buf(); + return fsync_spawn_blocking(move || { + let _owner = namespace_owner; + fsync_directory_handle_std(&dir, Some(&file)) + }) + .await?; + } + return fsync_dir_with_owner(dir, namespace_owner).await; } - let dir = dir.as_ref().to_path_buf(); - let opened = tokio::task::spawn_blocking(move || OpenedDstDirFsyncGroup::open(&dir)) - .await - .map_err(|err| io::Error::other(format!("blocking dst dir group open failed: {err}")))??; + let dir = dir.to_path_buf(); + let opened = tokio::task::spawn_blocking(move || { + #[cfg(target_os = "linux")] + if let Some(file) = opened { + return OpenedDstDirFsyncGroup::from_file(&dir, file); + } + #[cfg(not(target_os = "linux"))] + let _ = opened; + OpenedDstDirFsyncGroup::open(&dir) + }) + .await + .map_err(|err| io::Error::other(format!("blocking dst dir group open failed: {err}")))??; let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened, namespace_owner)?; if let Some(group) = worker { tokio::spawn(run_dst_dir_fsync_group_worker(group)); @@ -1027,6 +1077,49 @@ async fn fsync_dst_dir_group_commit_with_enabled( } } +/// Reuse the published directory without bypassing group commit or sync admission. +pub(crate) async fn fsync_commit_directory( + dir: &Path, + guard: &RenameCommitGuard, + lease: Arc, + admission: Option<&FileSyncAdmission>, +) -> io::Result<()> { + #[cfg(target_os = "linux")] + { + if dir != guard.directories.destination_path { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "sync directory does not match commit guard")); + } + let file = guard + .directories + .destination + .get() + .ok_or_else(|| io::Error::other("commit directory was not published"))? + .file + .clone(); + if !dst_dir_fsync_group_commit_enabled() + && let Some(admission) = admission + { + let dir = dir.to_path_buf(); + return run_blocking_namespace_file_sync_operation(lease, admission, move || { + #[cfg(test)] + fsync_dir_recorder::record_limited(&dir); + fsync_directory_handle_std(&dir, Some(&file)) + }) + .await; + } + fsync_dst_dir_group_commit_opened(dir, dst_dir_fsync_group_commit_enabled(), Some(lease), Some(file)).await + } + #[cfg(not(target_os = "linux"))] + { + let _ = guard; + match admission { + Some(admission) => fsync_dst_dir_group_commit_or_namespace_file_sync_limit(dir, lease, admission).await, + None => fsync_dst_dir_group_commit(dir, Some(lease)).await, + } + } +} + +#[cfg(not(target_os = "linux"))] pub(crate) async fn fsync_dst_dir_group_commit( dir: impl AsRef, namespace_owner: Option>, @@ -1034,6 +1127,7 @@ pub(crate) async fn fsync_dst_dir_group_commit( fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled(), namespace_owner).await } +#[cfg(not(target_os = "linux"))] pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit( dir: impl AsRef, lease: Arc, @@ -1061,7 +1155,7 @@ fn clear_dst_dir_fsync_group_commit_for_test() { DST_DIR_FSYNC_GROUP_COMMIT.clear_for_test(); } -type FileFdatasyncGroupKey = usize; +type FileFdatasyncGroupKey = (usize, bool); struct FileFdatasyncWaiter { files: Vec, @@ -1133,7 +1227,7 @@ impl FileFdatasyncGroupCommit { )); } let (result_tx, result_rx) = oneshot::channel(); - let key = Arc::as_ptr(&disk_permits) as FileFdatasyncGroupKey; + let key = (Arc::as_ptr(&disk_permits) as usize, FSYNC_ON_CURRENT_RUNTIME.get()); let mut registry = self.inner.lock(); registry.groups.retain(|_, group| group.disk_permits.strong_count() > 0); if registry.total_waiters >= MAX_FILE_FDATASYNC_WAITERS { @@ -1371,9 +1465,26 @@ static FSYNC_RUNTIME: LazyLock> = LazyLock::new( } }); +thread_local! { + static FSYNC_ON_CURRENT_RUNTIME: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Called only when starting threads belonging to an isolated I/O runtime. +/// Async and blocking threads use the same runtime's blocking budget for fsync. +pub(crate) fn use_current_runtime_for_fsync() { + FSYNC_ON_CURRENT_RUNTIME.set(true); +} + +pub(super) fn is_isolated_io_thread() -> bool { + FSYNC_ON_CURRENT_RUNTIME.get() +} + /// Spawn a blocking task on the fsync-dedicated runtime if configured, /// otherwise fall back to the main tokio blocking pool. fn fsync_spawn_blocking(f: impl FnOnce() -> T + Send + 'static) -> tokio::task::JoinHandle { + if FSYNC_ON_CURRENT_RUNTIME.get() { + return tokio::task::spawn_blocking(f); + } match FSYNC_RUNTIME.as_ref() { Some(rt) => rt.spawn_blocking(f), None => tokio::task::spawn_blocking(f), @@ -2268,6 +2379,20 @@ pub(crate) fn create_prepared_rename_source_with_commit_guard( #[cfg(not(windows))] { let _ = (dst_file_path, commit_guard); + #[cfg(target_os = "linux")] + let source = { + use rustix::fs::{Mode, OFlags}; + let directories = &commit_guard.directories; + let name = directories.file_name(src_file_path, true)?; + directories.file_name(dst_file_path, false)?; + open_linux_relative( + &directories.source.file, + Path::new(name), + OFlags::CREATE | OFlags::TRUNC | OFlags::WRONLY, + Mode::from_raw_mode(0o666), + )? + }; + #[cfg(not(target_os = "linux"))] let source = std::fs::OpenOptions::new() .create(true) .write(true) @@ -2288,18 +2413,39 @@ pub(crate) fn create_prepared_rename_source_with_commit_guard( } } -#[cfg(windows)] +#[cfg(any(windows, target_os = "linux"))] pub(crate) fn read_destination_file_with_commit_guard( file_path: &Path, commit_guard: &RenameCommitGuard, ) -> io::Result>> { - if file_path.parent() != Some(commit_guard.destination_parent.as_path()) { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "destination file parent does not match its commit guard", - )); + #[cfg(target_os = "linux")] + { + use rustix::fs::{Mode, OFlags}; + let directories = &commit_guard.directories; + let name = directories.file_name(file_path, false)?; + let Some(parent) = directories.destination.get() else { + return Ok(None); + }; + return match open_linux_relative(&parent.file, Path::new(name), OFlags::RDONLY, Mode::empty()) { + Ok(mut file) => { + let mut data = Vec::new(); + std::io::Read::read_to_end(&mut file, &mut data)?; + Ok(Some(data)) + } + Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(err), + }; + } + #[cfg(windows)] + { + if file_path.parent() != Some(commit_guard.destination_parent.as_path()) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "destination file parent does not match its commit guard", + )); + } + read_windows_relative_file(file_path, &commit_guard.destination_parent_guard) } - read_windows_relative_file(file_path, &commit_guard.destination_parent_guard) } #[cfg(windows)] @@ -2337,23 +2483,40 @@ pub(crate) async fn rename_all_with_prepared_source( dst_file_path: impl AsRef, base_dir: impl AsRef, publication_root: &PublicationRoot, - _commit_guard: &RenameCommitGuard, + commit_guard: &RenameCommitGuard, lease: Arc, ) -> Result<()> { let src_file_path = src_file_path.as_ref().to_path_buf(); let dst_file_path = dst_file_path.as_ref().to_path_buf(); let base_dir = base_dir.as_ref().to_path_buf(); + #[cfg(not(target_os = "linux"))] let publication_root = publication_root.clone(); + #[cfg(target_os = "linux")] + let _ = publication_root; + let commit_guard = commit_guard.clone(); let operation = { let src_file_path = src_file_path.clone(); let dst_file_path = dst_file_path.clone(); + #[cfg(not(target_os = "linux"))] let base_dir = base_dir.clone(); move || { - validate_prepared_rename_source(&prepared_source, &src_file_path)?; - let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?; - #[cfg(any(test, feature = "test-util"))] - prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path); - rename_prepared(&src_file_path, &dst_file_path, &preparation) + #[cfg(target_os = "linux")] + { + #[cfg(any(test, feature = "test-util"))] + prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path); + return commit_guard + .directories + .rename(&src_file_path, &dst_file_path, Some(&prepared_source)); + } + #[cfg(not(target_os = "linux"))] + { + let _ = commit_guard; + validate_prepared_rename_source(&prepared_source, &src_file_path)?; + let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?; + #[cfg(any(test, feature = "test-util"))] + prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path); + rename_prepared(&src_file_path, &dst_file_path, &preparation) + } } }; let result = run_blocking_namespace_operation(lease, operation).await; @@ -2370,10 +2533,35 @@ pub(crate) async fn rename_all_with_commit_guard( dst_file_path: impl AsRef, base_dir: impl AsRef, publication_root: &PublicationRoot, - _commit_guard: &RenameCommitGuard, + commit_guard: &RenameCommitGuard, lease: Arc, ) -> Result<()> { - rename_all_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await + #[cfg(target_os = "linux")] + { + let source = src_file_path.as_ref().to_path_buf(); + let destination = dst_file_path.as_ref().to_path_buf(); + let guard = commit_guard.clone(); + let _ = (base_dir, publication_root); + return run_blocking_namespace_operation(lease, move || { + guard.directories.prepare_destination()?; + #[cfg(test)] + prepared_publication_test_hooks::run_rename_destination(&source, &destination); + #[cfg(any(test, feature = "test-util"))] + { + prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &source); + prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &destination); + } + guard.directories.rename(&source, &destination, None) + }) + .await + .map_err(to_file_error) + .map_err(DiskError::from); + } + #[cfg(not(target_os = "linux"))] + { + let _ = commit_guard; + rename_all_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await + } } #[tracing::instrument(level = "debug", skip_all)] @@ -2501,7 +2689,7 @@ async fn reliable_rename_inner_with_lease( result } -#[cfg(not(windows))] +#[cfg(not(any(windows, target_os = "linux")))] fn validate_prepared_rename_source(prepared_source: &PreparedRenameSource, src_file_path: &Path) -> io::Result<()> { if prepared_source.path != src_file_path { return Err(io::Error::new( @@ -2701,6 +2889,8 @@ pub(crate) async fn fsync_dir_with_namespace_file_sync_limit( struct RenamePreparation { parent_guard: Option, + #[cfg(target_os = "linux")] + publication_root: PublicationRoot, #[cfg(windows)] _source_parent_guard: ExistingBaseDirectoryGuard, #[cfg(windows)] @@ -2709,15 +2899,32 @@ struct RenamePreparation { #[cfg(not(windows))] fn prepare_rename_with_retry( - src_file_path: &Path, + _src_file_path: &Path, dst_file_path: &Path, base_dir: &Path, publication_root: &PublicationRoot, ) -> io::Result { - let prune_budget = prepare_prune_budget(dst_file_path, base_dir); + let parent_guard = dst_file_path + .parent() + .map(|parent| prepare_destination_parent_with_retry(parent, base_dir, publication_root)) + .transpose()?; + Ok(RenamePreparation { + parent_guard, + #[cfg(target_os = "linux")] + publication_root: publication_root.clone(), + }) +} + +#[cfg(not(windows))] +fn prepare_destination_parent_with_retry( + parent: &Path, + base_dir: &Path, + publication_root: &PublicationRoot, +) -> io::Result { + let prune_budget = parent_prune_budget(parent, base_dir); let mut attempt = 0; loop { - match prepare_rename(src_file_path, dst_file_path, base_dir, publication_root) { + match mkdir_all_below_existing_base_std(parent, base_dir, publication_root) { Ok(preparation) => return Ok(preparation), Err(err) if should_retry_prepare(&err, attempt, prune_budget) => { attempt += 1; @@ -2807,20 +3014,6 @@ fn prepare_rename_with_retry( }) } -#[cfg(not(windows))] -fn prepare_rename( - _src_file_path: &Path, - dst_file_path: &Path, - base_dir: &Path, - publication_root: &PublicationRoot, -) -> io::Result { - let parent_guard = dst_file_path - .parent() - .map(|parent| mkdir_all_below_existing_base_std(parent, base_dir, publication_root)) - .transpose()?; - Ok(RenamePreparation { parent_guard }) -} - /// Publish a prepared rename. The retry budget starts fresh here: preparation /// keeps its own counter, so a chain rebuilt after a concurrent prune must not /// cost the rename its one retry. @@ -2836,9 +3029,23 @@ fn rename_prepared(_src_file_path: &Path, dst_file_path: &Path, preparation: &Re #[cfg(not(windows))] { + // A missing source parent is not a destination-prune race. Open it once + // outside both retry loops, retaining it through any rename retry. + #[cfg(target_os = "linux")] + let source_parent = preparation.publication_root.open_directory( + _src_file_path + .parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "missing rename source parent"))?, + )?; let mut attempt = 0; loop { - let rename_result = rename_into_existing_parent(_src_file_path, dst_file_path, preparation.parent_guard.as_ref()); + let rename_result = rename_into_existing_parent( + _src_file_path, + dst_file_path, + preparation.parent_guard.as_ref(), + #[cfg(target_os = "linux")] + &source_parent, + ); match rename_result { Ok(()) => return Ok(()), Err(err) if should_retry_rename(&err, attempt) => { @@ -2877,8 +3084,9 @@ fn rename_into_existing_parent( src_file_path: &Path, dst_file_path: &Path, parent_guard: Option<&ExistingBaseDirectoryGuard>, + #[cfg(target_os = "linux")] source_parent: &std::fs::File, ) -> io::Result<()> { - use rustix::fs::{Mode, OFlags, open, renameat}; + use rustix::fs::renameat; let Some(parent_guard) = parent_guard else { let rename_started = rustfs_io_metrics::put_stage_timer(); @@ -2889,6 +3097,7 @@ fn rename_into_existing_parent( ); return result; }; + #[cfg(not(target_os = "linux"))] let src_parent = src_file_path .parent() .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a parent directory"))?; @@ -2898,10 +3107,11 @@ fn rename_into_existing_parent( let dst_name = dst_file_path .file_name() .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a file name"))?; - let src_parent = open( + #[cfg(not(target_os = "linux"))] + let src_parent = rustix::fs::open( src_parent, - OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, - Mode::empty(), + rustix::fs::OFlags::RDONLY | rustix::fs::OFlags::DIRECTORY | rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::CLOEXEC, + rustix::fs::Mode::empty(), ) .map_err(io::Error::from)?; let dst_parent = parent_guard @@ -2909,6 +3119,9 @@ fn rename_into_existing_parent( .ok_or_else(|| io::Error::other("rename destination parent guard is empty"))?; let rename_started = rustfs_io_metrics::put_stage_timer(); + #[cfg(target_os = "linux")] + let result = renameat(source_parent, src_name, dst_parent, dst_name).map_err(io::Error::from); + #[cfg(not(target_os = "linux"))] let result = renameat(&src_parent, src_name, dst_parent, dst_name).map_err(io::Error::from); rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL, @@ -3091,16 +3304,18 @@ struct WindowsDirectoryHandle { /// Stable root for namespace-changing disk operations. /// -/// Windows opens the configured endpoint once and keeps that directory identity -/// pinned for the lifetime of the disk. Publication then resolves every source -/// and destination component relative to this handle instead of re-entering the -/// mutable pathname namespace. Other platforms retain the path so callers use a -/// uniform API while their existing `openat`/`renameat` guards remain unchanged. +/// Linux and Windows pin the configured directory for the lifetime of the disk. +/// Linux also exposes a procfs alias for operations not yet using relative IO; +/// both access forms share one root handle and therefore one mount identity. #[derive(Clone)] pub(crate) struct PublicationRoot { - path: PathBuf, + path: Arc, + #[cfg(target_os = "linux")] + directory: Arc, + #[cfg(target_os = "linux")] + io_path: Arc, #[cfg(windows)] - configured_path: PathBuf, + configured_path: Arc, #[cfg(windows)] directory: WindowsDirectoryHandle, } @@ -3114,15 +3329,34 @@ impl PublicationRoot { #[cfg(windows)] let (resolved_path, directory) = open_windows_publication_root(path)?; + #[cfg(target_os = "linux")] + let directory = { + use rustix::fs::{Mode, OFlags, open}; + Arc::new(std::fs::File::from(open( + path, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + )?)) + }; + #[cfg(target_os = "linux")] + let io_path = { + use std::os::fd::AsRawFd; + Arc::new(PathBuf::from(format!("/proc/self/fd/{}/.", directory.as_raw_fd()))) + }; + Ok(Self { #[cfg(not(windows))] - path: path.to_path_buf(), + path: Arc::new(path.to_path_buf()), #[cfg(windows)] - path: resolved_path, + path: Arc::new(resolved_path), #[cfg(windows)] - configured_path: path.to_path_buf(), + configured_path: Arc::new(path.to_path_buf()), #[cfg(windows)] directory, + #[cfg(target_os = "linux")] + directory, + #[cfg(target_os = "linux")] + io_path, }) } @@ -3130,15 +3364,170 @@ impl PublicationRoot { &self.path } + #[cfg(target_os = "linux")] + pub(crate) fn directory(&self) -> &Arc { + &self.directory + } + + #[cfg(target_os = "linux")] + pub(crate) fn io_path(&self) -> &Path { + &self.io_path + } + + pub(crate) fn open_readonly(&self, path: &Path) -> io::Result { + #[cfg(target_os = "linux")] + return self.open_relative(path, rustix::fs::OFlags::RDONLY, rustix::fs::Mode::empty()); + #[cfg(not(target_os = "linux"))] + std::fs::File::open(path) + } + + pub(crate) fn create_truncate(&self, path: &Path) -> io::Result { + #[cfg(target_os = "linux")] + { + use rustix::fs::{Mode, OFlags}; + self.open_relative(path, OFlags::CREATE | OFlags::WRONLY | OFlags::TRUNC, Mode::from_raw_mode(0o666)) + } + #[cfg(not(target_os = "linux"))] + std::fs::OpenOptions::new().create(true).write(true).truncate(true).open(path) + } + + pub(crate) fn create_truncate_with_parent(&self, path: &Path) -> io::Result<(std::fs::File, Option)> { + #[cfg(target_os = "linux")] + { + use rustix::fs::{Mode, OFlags}; + let parent = self.open_directory( + path.parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "missing file parent"))?, + )?; + let name = path + .file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "missing file name"))?; + let file = open_linux_relative( + &parent, + Path::new(name), + OFlags::CREATE | OFlags::WRONLY | OFlags::TRUNC, + Mode::from_raw_mode(0o666), + )?; + Ok((file, Some(parent))) + } + #[cfg(not(target_os = "linux"))] + Ok((self.create_truncate(path)?, None)) + } + + #[cfg(target_os = "linux")] + fn open_relative(&self, path: &Path, flags: rustix::fs::OFlags, mode: rustix::fs::Mode) -> io::Result { + let relative = path + .strip_prefix(self.io_path.as_path()) + .or_else(|_| path.strip_prefix(self.path.as_path())) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path must remain below its publication root"))?; + open_linux_relative(&self.directory, relative, flags, mode) + } + + #[cfg(target_os = "linux")] + fn open_directory(&self, path: &Path) -> io::Result { + self.open_relative( + path, + rustix::fs::OFlags::RDONLY | rustix::fs::OFlags::DIRECTORY, + rustix::fs::Mode::empty(), + ) + } + #[cfg(windows)] fn relative_path<'a>(&self, path: &'a Path) -> io::Result<&'a Path> { // The configured path only derives a suffix; traversal stays rooted at the pinned directory handle. - path.strip_prefix(&self.path) - .or_else(|_| path.strip_prefix(&self.configured_path)) + path.strip_prefix(self.path.as_path()) + .or_else(|_| path.strip_prefix(self.configured_path.as_path())) .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path must remain below its publication root")) } } +/// Open the actual IO handle with the same containment policy as path validation. +/// Older kernels retain fd-relative traversal; only ENOSYS enables the fallback. +#[cfg(target_os = "linux")] +fn open_linux_relative( + directory: &std::fs::File, + relative: &Path, + flags: rustix::fs::OFlags, + mode: rustix::fs::Mode, +) -> io::Result { + use rustix::fs::{OFlags, ResolveFlags, openat2}; + if relative + .components() + .any(|component| !matches!(component, Component::Normal(_) | Component::CurDir)) + { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "invalid relative disk path")); + } + let relative = if relative.as_os_str().is_empty() { + Path::new(".") + } else { + relative + }; + let flags = flags | OFlags::CLOEXEC | OFlags::NOFOLLOW; + match openat2(directory, relative, flags, mode, ResolveFlags::BENEATH | ResolveFlags::NO_SYMLINKS) { + Ok(fd) => Ok(fd.into()), + Err(rustix::io::Errno::NOSYS) => open_linux_relative_legacy(directory, relative, flags, mode), + Err(rustix::io::Errno::LOOP | rustix::io::Errno::XDEV) => Err(DiskError::InvalidPath.into()), + Err(err) => Err(err.into()), + } +} + +#[cfg(target_os = "linux")] +fn open_linux_relative_legacy( + directory: &std::fs::File, + relative: &Path, + flags: rustix::fs::OFlags, + mode: rustix::fs::Mode, +) -> io::Result { + use rustix::fs::OFlags; + + let mut parent = None; + let mut components = relative + .components() + .filter_map(|component| match component { + Component::Normal(name) => Some(name), + _ => None, + }) + .peekable(); + while let Some(name) = components.next() { + let current = parent.as_ref().unwrap_or(directory); + if components.peek().is_none() { + return open_linux_relative_component(current, name, flags, mode); + } + parent = Some(open_linux_relative_component( + current, + name, + OFlags::PATH | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + rustix::fs::Mode::empty(), + )?); + } + open_linux_relative_component(directory, std::ffi::OsStr::new("."), flags, mode) +} + +#[cfg(target_os = "linux")] +fn open_linux_relative_component( + directory: &std::fs::File, + name: &std::ffi::OsStr, + flags: rustix::fs::OFlags, + mode: rustix::fs::Mode, +) -> io::Result { + use rustix::fs::{AtFlags, FileType, openat, statat}; + use rustix::io::Errno; + match openat(directory, name, flags, mode) { + Ok(fd) => Ok(fd.into()), + Err(Errno::LOOP) => Err(DiskError::InvalidPath.into()), + // O_DIRECTORY | O_NOFOLLOW reports ENOTDIR for an intermediate link. + // Inspect only this failure so ordinary non-directory errors retain + // their classification without adding a stat to successful opens. + Err(Errno::NOTDIR) + if statat(directory, name, AtFlags::SYMLINK_NOFOLLOW) + .is_ok_and(|metadata| FileType::from_raw_mode(metadata.st_mode) == FileType::Symlink) => + { + Err(DiskError::InvalidPath.into()) + } + Err(err) => Err(err.into()), + } +} + #[cfg(windows)] #[derive(Clone)] pub(crate) struct ExistingBaseDirectoryGuard { @@ -3176,6 +3565,8 @@ pub(crate) type ExistingBaseDirectoryGuard = (); #[derive(Clone)] pub(crate) struct RenameCommitGuard { + #[cfg(target_os = "linux")] + directories: Arc, #[cfg(windows)] source_parent: PathBuf, #[cfg(windows)] @@ -3186,6 +3577,108 @@ pub(crate) struct RenameCommitGuard { destination_parent_guard: ExistingBaseDirectoryGuard, } +#[cfg(target_os = "linux")] +struct LinuxCommitDirectory { + file: Arc, + device: u64, + inode: u64, +} + +#[cfg(target_os = "linux")] +impl LinuxCommitDirectory { + fn new(file: std::fs::File) -> io::Result { + use std::os::unix::fs::MetadataExt; + let metadata = file.metadata()?; + Ok(Self { + file: Arc::new(file), + device: metadata.dev(), + inode: metadata.ino(), + }) + } +} + +#[cfg(target_os = "linux")] +struct LinuxCommitDirectories { + root: PublicationRoot, + source_path: PathBuf, + destination_path: PathBuf, + source: LinuxCommitDirectory, + destination: std::sync::OnceLock, + destination_base: PathBuf, +} + +#[cfg(target_os = "linux")] +impl LinuxCommitDirectories { + fn prepare_destination(&self) -> io::Result<&LinuxCommitDirectory> { + if let Some(directory) = self.destination.get() { + return Ok(directory); + } + let mut parents = prepare_destination_parent_with_retry(&self.destination_path, &self.destination_base, &self.root)?; + let directory = std::fs::File::from( + parents + .pop() + .ok_or_else(|| io::Error::other("missing destination directory guard"))?, + ); + let directory = LinuxCommitDirectory::new(directory)?; + Ok(self.destination.get_or_init(|| directory)) + } + + fn file_name<'a>(&self, path: &'a Path, source: bool) -> io::Result<&'a std::ffi::OsStr> { + let parent = if source { &self.source_path } else { &self.destination_path }; + if path.parent() != Some(parent) { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "file parent does not match its commit guard")); + } + path.file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "missing file name")) + } + + fn validate(&self) -> io::Result<()> { + use std::os::unix::fs::MetadataExt; + // A pinned directory may have been unlinked or replaced. Do not report a + // successful publication into a detached object tree. + for (path, directory) in [ + (&self.source_path, &self.source), + (&self.destination_path, self.prepare_destination()?), + ] { + let current = self.root.open_directory(path)?.metadata()?; + if (current.dev(), current.ino()) != (directory.device, directory.inode) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "commit directory identity changed")); + } + } + Ok(()) + } + + fn rename(&self, source: &Path, destination: &Path, prepared: Option<&PreparedRenameSource>) -> io::Result<()> { + let source_name = self.file_name(source, true)?; + let destination_name = self.file_name(destination, false)?; + let destination_parent = self.prepare_destination()?; + self.validate()?; + let mut attempt = 0; + loop { + if let Some(prepared) = prepared { + let metadata = rustix::fs::statat(&*self.source.file, source_name, rustix::fs::AtFlags::SYMLINK_NOFOLLOW)?; + if prepared.path != source || metadata.st_dev != prepared.device || metadata.st_ino != prepared.inode { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "rename source identity changed while publication was prepared", + )); + } + } + let started = rustfs_io_metrics::put_stage_timer(); + let result = rustix::fs::renameat(&*self.source.file, source_name, &*destination_parent.file, destination_name) + .map_err(io::Error::from); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL, + started, + ); + match result { + Err(err) if should_retry_rename(&err, attempt) => attempt += 1, + result => return result, + } + } + } +} + pub(crate) struct RenameDestinationPathGuard { #[cfg(windows)] directory: PathBuf, @@ -3385,6 +3878,29 @@ pub(crate) fn prepare_rename_commit_guard( destination_base: &Path, publication_root: &PublicationRoot, ) -> io::Result { + #[cfg(target_os = "linux")] + { + existing_base_directory_suffix(destination_parent, destination_base)?; + let source = LinuxCommitDirectory::new(publication_root.open_directory(source_parent)?)?; + let destination = std::sync::OnceLock::new(); + match publication_root.open_directory(destination_parent) { + Ok(directory) => { + let _ = destination.set(LinuxCommitDirectory::new(directory)?); + } + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => return Err(err), + } + return Ok(RenameCommitGuard { + directories: Arc::new(LinuxCommitDirectories { + root: publication_root.clone(), + source_path: source_parent.to_path_buf(), + destination_path: destination_parent.to_path_buf(), + source, + destination, + destination_base: destination_base.to_path_buf(), + }), + }); + } #[cfg(windows)] { // A same-directory rename must use one shared-write parent handle: @@ -3413,7 +3929,7 @@ pub(crate) fn prepare_rename_commit_guard( }) } - #[cfg(not(windows))] + #[cfg(not(any(windows, target_os = "linux")))] { let _ = (source_parent, destination_parent, destination_base, publication_root); Ok(RenameCommitGuard {}) @@ -4209,11 +4725,7 @@ fn windows_rename_source_is_allowed(attributes: u32, reparse_tag: u32) -> bool { attributes & FILE_ATTRIBUTE_REPARSE_POINT == 0 || reparse_tag == IO_REPARSE_TAG_DEDUP } -pub(crate) fn mkdir_all_below_existing_base_std( - dir_path: &Path, - base_dir: &Path, - publication_root: &PublicationRoot, -) -> io::Result { +fn existing_base_directory_suffix<'a>(dir_path: &'a Path, base_dir: &Path) -> io::Result<&'a Path> { let relative = dir_path .strip_prefix(base_dir) .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must remain below its base directory"))?; @@ -4226,15 +4738,31 @@ pub(crate) fn mkdir_all_below_existing_base_std( } } + Ok(relative) +} + +pub(crate) fn mkdir_all_below_existing_base_std( + dir_path: &Path, + base_dir: &Path, + publication_root: &PublicationRoot, +) -> io::Result { + let relative = existing_base_directory_suffix(dir_path, base_dir)?; + #[cfg(unix)] { - let _ = publication_root; - use rustix::fs::{Mode, OFlags, mkdirat, open, openat}; + use rustix::fs::{Mode, OFlags, mkdirat, openat}; use rustix::io::Errno; let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC; let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO; - let mut parents = vec![open(base_dir, flags, Mode::empty()).map_err(io::Error::from)?]; + #[cfg(target_os = "linux")] + let base = publication_root.open_directory(base_dir)?.into(); + #[cfg(not(target_os = "linux"))] + let base = { + let _ = publication_root; + rustix::fs::open(base_dir, flags, Mode::empty()).map_err(io::Error::from)? + }; + let mut parents = vec![base]; #[cfg(test)] let mut walked_path = base_dir.to_path_buf(); @@ -4354,10 +4882,18 @@ fn should_retry_rename(err: &io::Error, attempt: usize) -> bool { /// steal an attempt by making that same upward progress. A destination whose /// parent *is* the base gets a budget of zero, keeping `NotFound` immediately /// terminal for speculative cleanup renames. +#[cfg(any(windows, test))] fn prepare_prune_budget(dst_file_path: &Path, base_dir: &Path) -> usize { dst_file_path .parent() - .and_then(|parent| parent.strip_prefix(base_dir).ok()) + .map(|parent| parent_prune_budget(parent, base_dir)) + .unwrap_or(0) +} + +fn parent_prune_budget(parent: &Path, base_dir: &Path) -> usize { + parent + .strip_prefix(base_dir) + .ok() .map(|relative| relative.components().count()) .unwrap_or(0) } @@ -4484,6 +5020,60 @@ mod tests { use tempfile::tempdir; use tracing_subscriber::fmt::MakeWriter; + #[test] + fn fsync_groups_do_not_mix_cleanup_and_foreground_workers() { + let dir = tempdir().unwrap(); + let dst_groups = Arc::new(DstDirFsyncGroupCommit::default()); + let file_groups = Arc::new(FileFdatasyncGroupCommit::default()); + let disk_permits = Arc::new(Semaphore::new(1)); + let (_, dst_worker) = dst_groups.enqueue_for_test(dir.path()).unwrap(); + let (_, file_worker) = file_groups + .enqueue(disk_permits.clone(), vec![dir.path().join("one")]) + .unwrap(); + assert!(dst_worker.is_some()); + assert!(file_worker.is_some()); + let cleanup_dst_groups = dst_groups.clone(); + let cleanup_file_groups = file_groups.clone(); + std::thread::spawn(move || { + use_current_runtime_for_fsync(); + let (_, dst_worker) = cleanup_dst_groups.enqueue_for_test(dir.path()).unwrap(); + let (_, file_worker) = cleanup_file_groups + .enqueue(disk_permits, vec![dir.path().join("two")]) + .unwrap(); + assert!(dst_worker.is_some(), "cleanup must not join a foreground directory worker"); + assert!(file_worker.is_some(), "cleanup must not join a foreground file worker"); + }) + .join() + .unwrap(); + assert_eq!(dst_groups.inner.lock().groups.len(), 2); + assert_eq!(file_groups.inner.lock().groups.len(), 2); + } + + #[tokio::test] + async fn isolated_runtime_keeps_fsync_on_its_own_blocking_pool() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .max_blocking_threads(1) + .thread_name("isolated-fsync-test") + .on_thread_start(use_current_runtime_for_fsync) + .enable_all() + .build() + .unwrap(); + let result = runtime + .spawn(async { + fsync_spawn_blocking(|| { + assert_eq!(std::thread::current().name(), Some("isolated-fsync-test")); + }) + .await + .unwrap(); + }) + .await; + // A runtime cannot be dropped from another runtime's async context. + runtime.shutdown_background(); + result.unwrap(); + assert!(!FSYNC_ON_CURRENT_RUNTIME.get(), "the calling runtime must retain normal fsync routing"); + } + fn file_sync_limiter() -> Arc { Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS)) } @@ -4493,7 +5083,11 @@ mod tests { .first() .expect("test publication root requires at least one path") .to_path_buf(); - while !paths.iter().all(|path| path.starts_with(&common)) { + // The disk root must exist independently of a missing or symlinked + // rename base; those are inputs whose rejection the tests exercise. + while !paths.iter().all(|path| path.starts_with(&common)) + || !std::fs::symlink_metadata(&common).is_ok_and(|metadata| metadata.is_dir()) + { assert!(common.pop(), "test paths must share an absolute root"); } PublicationRoot::new(&common).expect("test publication root should open") @@ -7737,3 +8331,172 @@ mod tests { assert_eq!(err.kind(), io::ErrorKind::NotFound); } } + +#[cfg(all(test, target_os = "linux"))] +mod fd_relative_tests { + use super::*; + use std::io::{Read, Write}; + + #[tokio::test] + async fn guarded_first_publication_recovers_from_a_concurrent_parent_prune() { + let temp = tempfile::tempdir().expect("temporary disk"); + let root = PublicationRoot::new(temp.path()).expect("pinned root"); + let source = temp.path().join("tmp"); + let base = temp.path().join("bucket"); + let shared = base.join("prefix"); + let destination = shared.join("object"); + std::fs::create_dir(&source).expect("staging directory"); + std::fs::create_dir_all(&shared).expect("shared prefix"); + let guard = prepare_rename_commit_guard(&source, &destination, &base, &root).expect("commit directories"); + let src = source.join("xl.meta"); + let dst = destination.join("xl.meta"); + let mut staged = create_prepared_rename_source_with_commit_guard(&src, &dst, &guard).expect("staged metadata"); + staged.write_all(b"metadata", true).expect("write metadata"); + let pruned = shared.clone(); + prepare_rename_test_hooks::queue_after_component_opened(&shared, move || { + std::fs::remove_dir(pruned).expect("prune empty prefix"); + }); + let lease = acquire_namespace_mutation_lease(&dst).await; + rename_all_with_prepared_source(staged, &src, &dst, &base, &root, &guard, lease) + .await + .expect("publication must rebuild the pruned parent"); + assert_eq!(std::fs::read(dst).expect("published metadata"), b"metadata"); + assert!(!src.exists()); + } + + #[test] + fn legacy_relative_open_rejects_symlinks_and_opens_existing_components() { + use rustix::fs::{Mode, OFlags}; + let temp = tempfile::tempdir().expect("temporary disk"); + std::fs::create_dir(temp.path().join("parent")).expect("parent directory"); + std::fs::write(temp.path().join("parent/data"), b"data").expect("file data"); + std::os::unix::fs::symlink("parent", temp.path().join("link")).expect("directory link"); + std::os::unix::fs::symlink("data", temp.path().join("parent/link")).expect("file link"); + let root = PublicationRoot::new(temp.path()).expect("pinned root"); + let flags = OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC; + let mut bytes = Vec::new(); + open_linux_relative_legacy(root.directory(), Path::new("parent/data"), flags, Mode::empty()) + .expect("legacy open") + .read_to_end(&mut bytes) + .expect("legacy read"); + assert_eq!(bytes, b"data"); + for relative in ["link/data", "parent/link"] { + assert!(open_linux_relative_legacy(root.directory(), Path::new(relative), flags, Mode::empty()).is_err()); + } + } + + #[test] + fn root_relative_io_retains_mount_identity_after_path_replacement() { + let temp = tempfile::tempdir().expect("temporary disk"); + let disk = temp.path().join("disk"); + std::fs::create_dir(&disk).expect("disk root"); + std::fs::write(disk.join("data"), b"original").expect("original data"); + let root = PublicationRoot::new(&disk).expect("pinned root"); + let original = temp.path().join("original"); + std::fs::rename(&disk, &original).expect("move root"); + std::fs::create_dir(&disk).expect("replacement root"); + std::fs::write(disk.join("data"), b"replacement").expect("replacement data"); + for path in [disk.join("data"), root.io_path().join("data")] { + let mut actual = Vec::new(); + root.open_readonly(&path) + .expect("pinned read") + .read_to_end(&mut actual) + .expect("read bytes"); + assert_eq!(actual, b"original"); + } + let (mut file, parent) = root.create_truncate_with_parent(&disk.join("new")).expect("pinned creation"); + file.write_all(b"new").expect("write bytes"); + file.sync_data().expect("file sync"); + fsync_directory_handle_std(&disk, parent.as_ref()).expect("pinned parent sync"); + assert_eq!(std::fs::read(original.join("new")).expect("original disk file"), b"new"); + assert!(!disk.join("new").exists()); + } + + #[test] + fn root_relative_io_rejects_links_and_parent_escape_without_truncation() { + let temp = tempfile::tempdir().expect("temporary disk"); + let disk = temp.path().join("disk"); + std::fs::create_dir(&disk).expect("disk root"); + let outside = temp.path().join("outside"); + std::fs::write(&outside, b"preserved").expect("outside data"); + std::os::unix::fs::symlink(&outside, disk.join("link")).expect("file link"); + std::os::unix::fs::symlink(temp.path(), disk.join("parent")).expect("directory link"); + let root = PublicationRoot::new(&disk).expect("pinned root"); + for path in [disk.join("link"), disk.join("parent/outside"), disk.join("../outside")] { + assert!(root.open_readonly(&path).is_err(), "unsafe read: {path:?}"); + assert!(root.create_truncate(&path).is_err(), "unsafe write: {path:?}"); + } + assert_eq!(std::fs::read(&outside).expect("outside remains readable"), b"preserved"); + } + + #[tokio::test] + async fn guarded_publication_reuses_parents_and_syncs_without_changing_bytes() { + for grouped in [false, true] { + let _grouping = set_dst_dir_fsync_group_commit_for_test(grouped); + let temp = tempfile::tempdir().expect("temporary disk"); + let root = PublicationRoot::new(temp.path()).expect("pinned root"); + let source = temp.path().join("tmp"); + let destination = temp.path().join("bucket/object"); + let base = temp.path().join("bucket"); + std::fs::create_dir(&source).expect("staging directory"); + std::fs::create_dir(&base).expect("bucket directory"); + let guard = prepare_rename_commit_guard(&source, &destination, &base, &root).expect("commit directories"); + assert!(!destination.exists(), "preparation must not create the object directory"); + let src = source.join("xl.meta"); + let dst = destination.join("xl.meta"); + assert!( + read_destination_file_with_commit_guard(&dst, &guard) + .expect("absent read") + .is_none() + ); + let mut staged = create_prepared_rename_source_with_commit_guard(&src, &dst, &guard).expect("staged metadata"); + staged.write_all(b"metadata", true).expect("write metadata"); + let lease = acquire_namespace_mutation_lease(&dst).await; + rename_all_with_prepared_source(staged, &src, &dst, &base, &root, &guard, lease.clone()) + .await + .expect("publish"); + fsync_commit_directory(&destination, &guard, lease, None) + .await + .expect("directory sync"); + assert_eq!( + read_destination_file_with_commit_guard(&dst, &guard).expect("committed read"), + Some(b"metadata".to_vec()) + ); + assert!(!src.exists()); + assert_eq!(std::fs::read_dir(&destination).expect("object directory").count(), 1); + } + } + + #[tokio::test] + async fn guarded_publication_rejects_replaced_directory_and_source() { + for replace_directory in [false, true] { + let temp = tempfile::tempdir().expect("temporary disk"); + let root = PublicationRoot::new(temp.path()).expect("pinned root"); + let source = temp.path().join("tmp"); + let destination = temp.path().join("object"); + std::fs::create_dir(&source).expect("staging directory"); + std::fs::create_dir(&destination).expect("existing object directory"); + let guard = prepare_rename_commit_guard(&source, &destination, temp.path(), &root).expect("commit directories"); + let src = source.join("xl.meta"); + let dst = destination.join("xl.meta"); + std::fs::write(&dst, b"old").expect("old metadata"); + let mut staged = create_prepared_rename_source_with_commit_guard(&src, &dst, &guard).expect("staged metadata"); + staged.write_all(b"new", false).expect("new metadata"); + if replace_directory { + std::fs::rename(&destination, temp.path().join("detached")).expect("replace destination"); + std::fs::create_dir(&destination).expect("replacement directory"); + std::fs::write(&dst, b"old").expect("replacement metadata"); + } else { + std::fs::rename(&src, source.join("detached")).expect("replace staged file"); + std::fs::write(&src, b"unrelated").expect("replacement file"); + } + let lease = acquire_namespace_mutation_lease(&dst).await; + assert!( + rename_all_with_prepared_source(staged, &src, &dst, temp.path(), &root, &guard, lease) + .await + .is_err() + ); + assert_eq!(std::fs::read(&dst).expect("old metadata survives"), b"old"); + } + } +} diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 72af3fe60..2b99bd9c1 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -145,7 +145,7 @@ use std::{ time::{Duration, Instant}, }; use tokio::io::{AsyncRead, ReadBuf}; -use tokio::sync::RwLock; +use tokio::sync::{RwLock, oneshot}; use tokio::task::JoinSet; pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read"; @@ -2460,6 +2460,9 @@ pub(in crate::set_disk) struct RenameDataCommit { pub(in crate::set_disk) data_dir: Option, pub(in crate::set_disk) cleanup_disks: Vec>, pub(in crate::set_disk) old_current_size: Option, + pub(in crate::set_disk) old_current_source_checked: bool, + pub(in crate::set_disk) old_current_source_all_successful_checked: bool, + pub(in crate::set_disk) old_current_source: Option, pub(in crate::set_disk) committed_file_info: FileInfo, pub(in crate::set_disk) tail_drain: Option>>, } @@ -2475,6 +2478,500 @@ pub(in crate::set_disk) struct RenameTailOutcome { pub(in crate::set_disk) cleanup: Vec, } +const RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP"; +const RENAME_TAIL_CLEANUP_DEFER_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ENABLE"; +const RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY"; +const RENAME_TAIL_CLEANUP_DEFER_WORKERS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS"; +const RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY_MS"; +const RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW_MS"; +const RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_ENABLE_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_ENABLE"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_MIN_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_MIN"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_LOW_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_LOW"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_HIGH_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_HIGH"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS"; +const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS"; +const RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER"; +const RENAME_FOREGROUND_ACTIVE_LANES_ENV: &str = "RUSTFS_PUT_RENAME_FOREGROUND_ACTIVE_LANES"; +const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY: usize = 1024; +const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_WORKERS: usize = 2; +const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX: usize = 1; +const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW: usize = 128; +const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH: usize = 512; +const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS: u64 = 500; +const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS: u64 = 2_000; +const DEFAULT_RENAME_FOREGROUND_ACTIVE_LANES: usize = 1024; + +fn rename_tail_cleanup_counterfactual_skip_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP_ENV).ok().as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn rename_tail_cleanup_defer_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(RENAME_TAIL_CLEANUP_DEFER_ENV).ok().as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn rename_tail_cleanup_defer_queue_capacity() -> usize { + static CAPACITY: OnceLock = OnceLock::new(); + *CAPACITY.get_or_init(|| { + rustfs_utils::get_env_usize( + RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY_ENV, + DEFAULT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY, + ) + .max(1) + }) +} + +fn rename_tail_cleanup_defer_workers() -> usize { + static WORKERS: OnceLock = OnceLock::new(); + *WORKERS.get_or_init(|| { + rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_WORKERS_ENV, DEFAULT_RENAME_TAIL_CLEANUP_DEFER_WORKERS).max(1) + }) +} + +fn rename_tail_cleanup_defer_worker_delay() -> Duration { + static DELAY: OnceLock = OnceLock::new(); + *DELAY.get_or_init(|| Duration::from_millis(rustfs_utils::get_env_u64(RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY_MS_ENV, 0))) +} + +fn rename_tail_cleanup_defer_batch_window() -> Duration { + static WINDOW: OnceLock = OnceLock::new(); + *WINDOW.get_or_init(|| Duration::from_millis(rustfs_utils::get_env_u64(RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW_MS_ENV, 0))) +} + +fn rename_tail_cleanup_defer_batch_max() -> usize { + static BATCH_MAX: OnceLock = OnceLock::new(); + *BATCH_MAX.get_or_init(|| { + rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX_ENV, DEFAULT_RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX).max(1) + }) +} + +fn rename_tail_cleanup_defer_adaptive_batch_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_ENABLE_ENV) + .ok() + .as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn rename_tail_cleanup_defer_adaptive_batch_min() -> usize { + static BATCH_MIN: OnceLock = OnceLock::new(); + *BATCH_MIN.get_or_init(|| rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_MIN_ENV, 1).max(1)) +} + +fn rename_tail_cleanup_defer_adaptive_batch_low() -> usize { + static BATCH_LOW: OnceLock = OnceLock::new(); + *BATCH_LOW.get_or_init(|| { + rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_LOW_ENV, rename_tail_cleanup_defer_batch_max()) + .max(1) + }) +} + +fn rename_tail_cleanup_defer_adaptive_batch_high() -> usize { + static BATCH_HIGH: OnceLock = OnceLock::new(); + *BATCH_HIGH.get_or_init(|| { + rustfs_utils::get_env_usize( + RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_HIGH_ENV, + rename_tail_cleanup_defer_adaptive_batch_low().max(rename_tail_cleanup_defer_batch_max()), + ) + .max(1) + }) +} + +fn rename_tail_cleanup_defer_adaptive_queue_low() -> usize { + static QUEUE_LOW: OnceLock = OnceLock::new(); + *QUEUE_LOW.get_or_init(|| { + rustfs_utils::get_env_usize( + RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW_ENV, + DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW, + ) + }) +} + +fn rename_tail_cleanup_defer_adaptive_queue_high() -> usize { + static QUEUE_HIGH: OnceLock = OnceLock::new(); + *QUEUE_HIGH.get_or_init(|| { + rustfs_utils::get_env_usize( + RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH_ENV, + DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH, + ) + }) +} + +fn rename_tail_cleanup_defer_adaptive_age_low() -> Duration { + static AGE_LOW: OnceLock = OnceLock::new(); + *AGE_LOW.get_or_init(|| { + Duration::from_millis(rustfs_utils::get_env_u64( + RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS_ENV, + DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS, + )) + }) +} + +fn rename_tail_cleanup_defer_adaptive_age_high() -> Duration { + static AGE_HIGH: OnceLock = OnceLock::new(); + *AGE_HIGH.get_or_init(|| { + Duration::from_millis(rustfs_utils::get_env_u64( + RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS_ENV, + DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS, + )) + }) +} + +fn rename_tail_cleanup_defer_batch_target(queued_depth: usize, oldest_age: Option) -> usize { + let fixed = rename_tail_cleanup_defer_batch_max(); + if !rename_tail_cleanup_defer_adaptive_batch_enabled() { + return fixed; + } + + let min = rename_tail_cleanup_defer_adaptive_batch_min(); + let low = rename_tail_cleanup_defer_adaptive_batch_low().max(min); + let high = rename_tail_cleanup_defer_adaptive_batch_high().max(low); + let queue_low = rename_tail_cleanup_defer_adaptive_queue_low(); + let queue_high = rename_tail_cleanup_defer_adaptive_queue_high().max(queue_low); + let age_low = rename_tail_cleanup_defer_adaptive_age_low(); + let age_high = rename_tail_cleanup_defer_adaptive_age_high().max(age_low); + + let age = oldest_age.unwrap_or_default(); + if queued_depth >= queue_high || age >= age_high { + high + } else if queued_depth >= queue_low || age >= age_low { + low + } else { + min + } +} + +fn rename_tail_cleanup_defer_hold_worker_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER_ENV).ok().as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn rename_foreground_active_lanes() -> &'static Vec { + static LANES: OnceLock> = OnceLock::new(); + LANES.get_or_init(|| { + let lanes = + rustfs_utils::get_env_usize(RENAME_FOREGROUND_ACTIVE_LANES_ENV, DEFAULT_RENAME_FOREGROUND_ACTIVE_LANES).max(1); + (0..lanes).map(|_| AtomicUsize::new(0)).collect() + }) +} + +pub(in crate::set_disk) fn rename_foreground_active_count(disk_index: usize) -> usize { + let lanes = rename_foreground_active_lanes(); + lanes[disk_index % lanes.len()].load(Ordering::Relaxed) +} + +struct RenameForegroundActiveGuard { + disk_index: usize, +} + +impl Drop for RenameForegroundActiveGuard { + fn drop(&mut self) { + let lanes = rename_foreground_active_lanes(); + lanes[self.disk_index % lanes.len()].fetch_sub(1, Ordering::Relaxed); + } +} + +fn rename_foreground_active_guard(disk_index: usize) -> RenameForegroundActiveGuard { + let lanes = rename_foreground_active_lanes(); + let active = lanes[disk_index % lanes.len()].fetch_add(1, Ordering::Relaxed) + 1; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_FOREGROUND_ACTIVE, + active as f64, + ); + RenameForegroundActiveGuard { disk_index } +} + +type RenameTailCleanupFuture = Pin + Send + 'static>>; + +struct RenameTailCleanupJob { + enqueued_at: Instant, + targets: usize, + future: RenameTailCleanupFuture, +} + +struct RenameTailCleanupQueue { + sender: tokio::sync::mpsc::Sender, + queued: Arc, + oldest_enqueued_at: Arc>>, +} + +async fn run_rename_tail_cleanup_job(job: RenameTailCleanupJob) { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_START_DELAY, + job.enqueued_at.elapsed().as_secs_f64() * 1000.0, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_TARGETS, + job.targets as f64, + ); + let worker_delay = rename_tail_cleanup_defer_worker_delay(); + if !worker_delay.is_zero() { + let delay_started = rustfs_io_metrics::put_stage_timer(); + tokio::time::sleep(worker_delay).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY, + delay_started, + ); + } + let cleanup_started = rustfs_io_metrics::put_stage_timer(); + job.future.await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DURATION, + cleanup_started, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED, + 1.0, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED_LAG, + job.enqueued_at.elapsed().as_secs_f64() * 1000.0, + ); +} + +impl RenameTailCleanupQueue { + fn new() -> Self { + let (sender, receiver) = tokio::sync::mpsc::channel::(rename_tail_cleanup_defer_queue_capacity()); + let queued = Arc::new(AtomicUsize::new(0)); + let active_workers = Arc::new(AtomicUsize::new(0)); + let oldest_enqueued_at = Arc::new(tokio::sync::Mutex::new(None::)); + let receiver = Arc::new(tokio::sync::Mutex::new(receiver)); + if !rename_tail_cleanup_defer_hold_worker_enabled() { + for _ in 0..rename_tail_cleanup_defer_workers() { + let receiver = Arc::clone(&receiver); + let queued = Arc::clone(&queued); + let active_workers = Arc::clone(&active_workers); + let oldest_enqueued_at = Arc::clone(&oldest_enqueued_at); + tokio::spawn(async move { + loop { + let queued_before_poll = queued.load(Ordering::Relaxed); + let worker_poll_started = rustfs_io_metrics::put_stage_timer(); + let Some(job) = receiver.lock().await.recv().await else { + return; + }; + if queued_before_poll == 0 { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_IDLE_WAIT, + worker_poll_started, + ); + } else { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_POLL_WAIT, + worker_poll_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_JOB_WAIT, + worker_poll_started, + ); + } + let mut batch = vec![job]; + let batch_window = rename_tail_cleanup_defer_batch_window(); + let queued_snapshot = queued.load(Ordering::Relaxed); + let oldest_snapshot = *oldest_enqueued_at.lock().await; + let batch_max = rename_tail_cleanup_defer_batch_target( + queued_snapshot, + oldest_snapshot.map(|oldest| oldest.elapsed()), + ); + if batch_max > 1 && !batch_window.is_zero() { + let window_started = rustfs_io_metrics::put_stage_timer(); + tokio::time::sleep(batch_window).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW, + window_started, + ); + } + while batch.len() < batch_max { + let next = { + let mut locked = receiver.lock().await; + locked.try_recv().ok() + }; + let Some(next) = next else { + break; + }; + batch.push(next); + } + let queued_after_dequeue = queued.fetch_sub(batch.len(), Ordering::Relaxed).saturating_sub(batch.len()); + if queued_after_dequeue == 0 { + *oldest_enqueued_at.lock().await = None; + } + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH, + queued_after_dequeue as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE, + queued_after_dequeue as f64, + ); + if let Some(oldest) = *oldest_enqueued_at.lock().await { + let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE, + oldest_age_ms, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE, + oldest_age_ms, + ); + } + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_SIZE, + batch.len() as f64, + ); + if batch.len() > 1 { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BURST_BATCH_SIZE, + batch.len() as f64, + ); + } + let active = active_workers.fetch_add(1, Ordering::Relaxed) + 1; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_ACTIVE, + active as f64, + ); + for job in batch { + run_rename_tail_cleanup_job(job).await; + } + active_workers.fetch_sub(1, Ordering::Relaxed); + } + }); + } + } + Self { + sender, + queued, + oldest_enqueued_at, + } + } + + async fn try_enqueue(&self, job: RenameTailCleanupJob) -> std::result::Result<(), RenameTailCleanupJob> { + let queue_push_started = rustfs_io_metrics::put_stage_timer(); + { + let mut oldest = self.oldest_enqueued_at.lock().await; + if oldest.is_none() { + *oldest = Some(job.enqueued_at); + } + } + let queued_after_enqueue = self.queued.fetch_add(1, Ordering::Relaxed) + 1; + if rename_tail_cleanup_defer_hold_worker_enabled() { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED, + 1.0, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER, + 1.0, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH, + queued_after_enqueue as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE, + queued_after_enqueue as f64, + ); + if let Some(oldest) = *self.oldest_enqueued_at.lock().await { + let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE, + oldest_age_ms, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE, + oldest_age_ms, + ); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH, + queue_push_started, + ); + return Ok(()); + } + match self.sender.try_send(job) { + Ok(()) => { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED, + 1.0, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH, + queued_after_enqueue as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE, + queued_after_enqueue as f64, + ); + if let Some(oldest) = *self.oldest_enqueued_at.lock().await { + let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE, + oldest_age_ms, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE, + oldest_age_ms, + ); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH, + queue_push_started, + ); + Ok(()) + } + Err(err) => { + self.queued.fetch_sub(1, Ordering::Relaxed); + if self.queued.load(Ordering::Relaxed) == 0 { + *self.oldest_enqueued_at.lock().await = None; + } + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_FALLBACK_SYNC, + 1.0, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH, + self.queued.load(Ordering::Relaxed) as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE, + self.queued.load(Ordering::Relaxed) as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH, + queue_push_started, + ); + Err(err.into_inner()) + } + } + } +} + +fn rename_tail_cleanup_queue() -> &'static RenameTailCleanupQueue { + static QUEUE: OnceLock = OnceLock::new(); + QUEUE.get_or_init(RenameTailCleanupQueue::new) +} + const EVENT_SET_DISK_RENAME_ROLLBACK: &str = "set_disk_rename_rollback"; #[derive(Clone, Copy)] @@ -2569,10 +3066,9 @@ async fn inspect_incomplete_rename_rollback( request.recreate_missing = Some(false); request.update_parity = Some(false); request.recursive = Some(false); - match tokio::time::timeout(Duration::from_secs(1), submitter(request)).await { - Ok(result) => result, - Err(_) => ReadRepairAdmissionOutcome::Failed("rollback inspection admission timed out".to_string()), - } + tokio::time::timeout(Duration::from_secs(1), submitter(request)) + .await + .unwrap_or_else(|_| ReadRepairAdmissionOutcome::Failed("rollback inspection admission timed out".to_string())) } fn rename_rollback_task_outcome( @@ -2859,21 +3355,24 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal< SubmitFuture, >( tail_drain: tokio::task::JoinHandle>, - guard_release: tokio::sync::oneshot::Receiver, + guard_release: oneshot::Receiver, guards: Guards, request: rustfs_heal_contracts::heal_channel::HealChannelRequest, finalize: Finalize, cleanup: Cleanup, submit: Submit, ) where - Guards: Send, + Guards: Send + 'static, Finalize: FnOnce() -> FinalizeFuture + Send, FinalizeFuture: Future + Send, Cleanup: FnOnce(Guards, Vec) -> CleanupFuture + Send, CleanupFuture: Future + Send, + Cleanup: 'static, + CleanupFuture: 'static, Submit: FnOnce(rustfs_heal_contracts::heal_channel::HealChannelRequest) -> SubmitFuture + Send, SubmitFuture: Future + Send, { + let tail_async_started = rustfs_io_metrics::put_stage_timer(); let (needs_heal, tail_cleanup, tail_complete) = match tail_drain.await { Ok(Some(outcome)) => (outcome.convergence.needs_heal(), outcome.cleanup, true), Ok(None) => { @@ -2902,16 +3401,80 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal< (true, Vec::new(), false) } }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT, + tail_async_started, + ); // A dropped sender means the request continuation was cancelled. Cleanup // must still run; only the explicit `false` used by the hard-crash harness // suppresses all in-process continuation work. + let guard_release_wait_started = rustfs_io_metrics::put_stage_timer(); if matches!(guard_release.await, Ok(false)) { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT, + guard_release_wait_started, + ); drop(guards); return; } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT, + guard_release_wait_started, + ); + let finalize_started = rustfs_io_metrics::put_stage_timer(); finalize().await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_FINALIZE, + finalize_started, + ); if tail_complete { - cleanup(guards, tail_cleanup).await; + let cleanup_submit_started = rustfs_io_metrics::put_stage_timer(); + if rename_tail_cleanup_counterfactual_skip_enabled() { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP, + 1.0, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT, + cleanup_submit_started, + ); + drop(guards); + } else if rename_tail_cleanup_defer_enabled() { + let targets = tail_cleanup.len(); + let job = RenameTailCleanupJob { + enqueued_at: Instant::now(), + targets, + future: Box::pin(cleanup(guards, tail_cleanup)), + }; + if let Err(job) = rename_tail_cleanup_queue().try_enqueue(job).await { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT, + cleanup_submit_started, + ); + let cleanup_started = rustfs_io_metrics::put_stage_timer(); + job.future.await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP, + cleanup_started, + ); + } else { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT, + cleanup_submit_started, + ); + } + } else { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT, + cleanup_submit_started, + ); + let cleanup_started = rustfs_io_metrics::put_stage_timer(); + cleanup(guards, tail_cleanup).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP, + cleanup_started, + ); + } } else { // The outer coordinator may fail while a disk rename still owns the // staging source. Preserve it for later healing/GC instead of racing @@ -2919,7 +3482,12 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal< drop(guards); } if needs_heal { + let heal_submit_started = rustfs_io_metrics::put_stage_timer(); submit(request).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_HEAL_SUBMIT, + heal_submit_started, + ); } } @@ -2973,8 +3541,8 @@ impl SetDisks { bucket: &str, object: &str, write_quorum: usize, - ) -> crate::error::Result<(Vec>, Vec>)> { - let fence_path = crate::disk::quota_mutation_fence_path(bucket, object); + ) -> Result<(Vec>, Vec>)> { + let fence_path = disk::quota_mutation_fence_path(bucket, object); let results = join_all(disks.iter().map(|disk| { let disk = disk.clone(); let fence_path = fence_path.clone(); @@ -3016,8 +3584,8 @@ impl SetDisks { bucket: &str, object: &str, write_quorum: usize, - ) -> crate::error::Result<()> { - let fence_path = crate::disk::quota_mutation_fence_path(bucket, object); + ) -> Result<()> { + let fence_path = disk::quota_mutation_fence_path(bucket, object); let results = join_all(disks.iter().zip(tokens).filter_map(|(disk, token)| { let disk = disk.as_ref()?.clone(); let token = (*token)?; @@ -3046,11 +3614,25 @@ impl SetDisks { errs: &[Option], cleanup_data_dirs: &[Option], old_current_sizes: &[Option], + old_current_source_checked: &[bool], + old_current_sources: &[Option], write_quorum: usize, ) -> disk::error::Result { let data_dir = Self::reduce_common_data_dir(cleanup_data_dirs, write_quorum); let convergence = Self::classify_rename_convergence(disk_versions, errs); let old_current_size = Self::reduce_common_old_current_size(old_current_sizes, write_quorum); + let old_current_source_quorum_checked = + old_current_source_checked.iter().filter(|checked| **checked).count() >= write_quorum; + let old_current_source_all_successful_checked = errs + .iter() + .enumerate() + .filter(|(_, err)| err.is_none()) + .all(|(idx, _)| old_current_source_checked.get(idx).copied().unwrap_or(false)); + let old_current_source = if old_current_source_quorum_checked { + Self::reduce_common_old_current_source(old_current_sources, write_quorum) + } else { + None + }; let online_disks = Self::eval_disks(disks, errs); let committed_slot = online_disks.iter().position(Option::is_some).ok_or(DiskError::Unexpected)?; let committed_file_info = file_infos.get(committed_slot).cloned().ok_or(DiskError::Unexpected)?; @@ -3078,6 +3660,9 @@ impl SetDisks { data_dir, cleanup_disks, old_current_size, + old_current_source_checked: old_current_source_quorum_checked, + old_current_source_all_successful_checked, + old_current_source, committed_file_info, tail_drain: None, }) @@ -3195,9 +3780,10 @@ impl SetDisks { let src_object = Arc::new(src_object.to_string()); let dst_bucket = Arc::new(dst_bucket.to_string()); let dst_object = Arc::new(dst_object.to_string()); - let (commit_tx, commit_rx) = tokio::sync::oneshot::channel(); + let (commit_tx, commit_rx) = oneshot::channel(); let coordinator_failure_receipt = rollback_receipt.clone(); + let early_ack_spawn_to_quorum_wait_started = rustfs_io_metrics::put_stage_timer(); let tail_drain = tokio::spawn({ let fanout_src_bucket = src_bucket.clone(); let fanout_src_object = src_object.clone(); @@ -3230,6 +3816,11 @@ impl SetDisks { let Some(disk) = disk else { return Err(DiskError::DiskNotFound); }; + let disk_wait_location_stage = if disk.is_local() { + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL + } else { + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE + }; let is_delete_marker = file_info.is_canonical_delete_marker(); // Clone FileInfo and set erasure.index for this disk @@ -3249,6 +3840,7 @@ impl SetDisks { let disk_wait_started = rustfs_io_metrics::put_stage_timer(); dispatch_state = RenameDispatchState::MayHavePublished; + let _foreground_active_guard = rename_foreground_active_guard(i); let observed = disk .rename_data_borrowed_with_fence_observed( &src_bucket, @@ -3256,7 +3848,7 @@ impl SetDisks { &file_info, &dst_bucket, &dst_object, - crate::disk::RenameDataGuards { + disk::RenameDataGuards { scanner_publication_lease_token, namespace_owner: namespace_commit_guard .clone() @@ -3277,6 +3869,7 @@ impl SetDisks { rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT, duration_ms, ); + rustfs_io_metrics::record_put_object_stage_duration(disk_wait_location_stage, duration_ms); let position = if result.is_ok() { let rank = successful_rename_completion_rank .as_ref() @@ -3315,9 +3908,12 @@ impl SetDisks { let mut data_dirs = vec![None; disk_count]; let mut cleanup_data_dirs = vec![None; disk_count]; let mut old_current_sizes = vec![None; disk_count]; + let mut old_current_source_checked = vec![false; disk_count]; + let mut old_current_sources = vec![None; disk_count]; let mut capacity_scope_generation = None; let mut cleanup_at_snapshot = vec![false; disk_count]; + let mut tail_after_quorum_started = None; while let Some(joined) = tasks.join_next().await { results_seen += 1; match joined { @@ -3327,6 +3923,8 @@ impl SetDisks { cleanup_data_dirs[idx] = res.cleanup_data_dir; disk_versions[idx] = res.sign; old_current_sizes[idx] = res.old_current_size; + old_current_source_checked[idx] = res.old_current_source_checked; + old_current_sources[idx] = res.old_current_source; errs[idx] = None; success_count += 1; } @@ -3352,6 +3950,8 @@ impl SetDisks { &errs, &cleanup_data_dirs, &old_current_sizes, + &old_current_source_checked, + &old_current_sources, write_quorum, ); if let Ok(commit) = snapshot_commit.as_ref() { @@ -3362,8 +3962,13 @@ impl SetDisks { let _ = commit_tx.send(snapshot_commit); } sent_commit = true; + tail_after_quorum_started = rustfs_io_metrics::put_stage_timer(); } } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_AFTER_QUORUM_WAIT, + tail_after_quorum_started, + ); #[cfg(test)] rollback_fault_injection::after_fanout(&fanout_dst_object); @@ -3483,6 +4088,10 @@ impl SetDisks { } }); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT, + early_ack_spawn_to_quorum_wait_started, + ); let quorum_wait_started = rustfs_io_metrics::put_stage_timer(); let commit = match commit_rx.await { Ok(commit) => commit, @@ -3495,8 +4104,13 @@ impl SetDisks { rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT, quorum_wait_started, ); + let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer(); commit.map(|mut commit| { commit.tail_drain = Some(tail_drain); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER, + attach_tail_owner_started, + ); commit }) } @@ -3619,6 +4233,11 @@ impl SetDisks { let Some(disk) = disk else { return Err(DiskError::DiskNotFound); }; + let disk_wait_location_stage = if disk.is_local() { + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL + } else { + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE + }; let is_delete_marker = file_info.is_canonical_delete_marker(); let mut local_file_info; @@ -3652,6 +4271,7 @@ impl SetDisks { let disk_wait_started = rustfs_io_metrics::put_stage_timer(); dispatch_state = RenameDispatchState::MayHavePublished; + let _foreground_active_guard = rename_foreground_active_guard(i); let observed = disk .rename_data_borrowed_with_fence_observed( &src_bucket, @@ -3659,7 +4279,7 @@ impl SetDisks { file_info, &dst_bucket, &dst_object, - crate::disk::RenameDataGuards { + disk::RenameDataGuards { scanner_publication_lease_token, namespace_owner: namespace_commit_guard .clone() @@ -3680,6 +4300,7 @@ impl SetDisks { rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT, duration_ms, ); + rustfs_io_metrics::record_put_object_stage_duration(disk_wait_location_stage, duration_ms); let position = if result.is_ok() { let rank = successful_rename_completion_rank .as_ref() @@ -3715,9 +4336,16 @@ impl SetDisks { let mut data_dirs = vec![None; disk_count]; let mut cleanup_data_dirs = vec![None; disk_count]; let mut old_current_sizes = vec![None; disk_count]; + let mut old_current_source_checked = vec![false; disk_count]; + let mut old_current_sources = vec![None; disk_count]; let quorum_wait_started = rustfs_io_metrics::put_stage_timer(); + let sync_full_fanout_started = rustfs_io_metrics::put_stage_timer(); let fanout_result = fanout.await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_SYNC_FULL_FANOUT_WAIT, + sync_full_fanout_started, + ); rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT, quorum_wait_started, @@ -3758,6 +4386,8 @@ impl SetDisks { cleanup_data_dirs[idx] = res.cleanup_data_dir; disk_versions[idx].clone_from(&res.sign); old_current_sizes[idx] = res.old_current_size; + old_current_source_checked[idx] = res.old_current_source_checked; + old_current_sources[idx] = res.old_current_source.clone(); errs.push(None); } Ok(Err(e)) => { @@ -3894,6 +4524,18 @@ impl SetDisks { let data_dir = Self::reduce_common_data_dir(&cleanup_data_dirs, write_quorum); let convergence = Self::classify_rename_convergence(&disk_versions, &errs); let old_current_size = Self::reduce_common_old_current_size(&old_current_sizes, write_quorum); + let old_current_source_quorum_checked = + old_current_source_checked.iter().filter(|checked| **checked).count() >= write_quorum; + let old_current_source_all_successful_checked = errs + .iter() + .enumerate() + .filter(|(_, err)| err.is_none()) + .all(|(idx, _)| old_current_source_checked.get(idx).copied().unwrap_or(false)); + let old_current_source = if old_current_source_quorum_checked { + Self::reduce_common_old_current_source(&old_current_sources, write_quorum) + } else { + None + }; let online_disks = Self::eval_disks(disks, &errs); let committed_slot = online_disks.iter().position(Option::is_some).ok_or(DiskError::Unexpected)?; let committed_file_info = std::mem::take(&mut file_infos[committed_slot]); @@ -3921,11 +4563,34 @@ impl SetDisks { data_dir, cleanup_disks, old_current_size, + old_current_source_checked: old_current_source_quorum_checked, + old_current_source_all_successful_checked, + old_current_source, committed_file_info, tail_drain: None, }) } + pub(in crate::set_disk) fn reduce_common_old_current_source( + old_current_sources: &[Option], + write_quorum: usize, + ) -> Option { + let mut counts: HashMap<[u8; 32], (usize, FileInfo)> = HashMap::new(); + + for source in old_current_sources.iter().flatten() { + let entry = counts + .entry(Self::file_info_quorum_hash(source)) + .or_insert_with(|| (0, source.clone())); + entry.0 += 1; + } + + counts + .into_values() + .filter(|(count, _)| *count >= write_quorum) + .max_by_key(|(count, _)| *count) + .map(|(_, source)| source) + } + /// rustfs/backlog#1009: reduce the per-disk observations of the /// destination's previous current version to one set-level value, mirroring /// `reduce_common_data_dir`: the observation reported by at least @@ -4991,6 +5656,7 @@ impl SetDisks { fi.set_tier_free_version_id(&Uuid::new_v4().to_string()); let disks = self.get_disks_internal().await; + self.invalidate_get_object_metadata_cache(bucket, object).await; let mut futures = Vec::with_capacity(disks.len()); for (disk_index, disk_op) in disks.iter().enumerate() { @@ -5071,6 +5737,9 @@ impl SetDisks { } else { false }; + if absent { + self.invalidate_get_object_metadata_cache(bucket, object).await; + } Ok((m, absent)) } @@ -6566,9 +7235,9 @@ mod tests { ); scope.try_begin().expect("delete scope should enter flight"); let scope_guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone()); - let (started_tx, started_rx) = tokio::sync::oneshot::channel(); - let (release_tx, release_rx) = tokio::sync::oneshot::channel(); - let (finished_tx, finished_rx) = tokio::sync::oneshot::channel(); + let (started_tx, started_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + let (finished_tx, finished_rx) = oneshot::channel(); let waiter = tokio::spawn(run_scanner_publication_delete_owner(Some(scope.clone()), move || async move { started_tx.send(()).expect("delete owner should start"); @@ -9026,7 +9695,7 @@ mod tests { #[allow(unreachable_code)] None }); - let (release, released) = tokio::sync::oneshot::channel::(); + let (release, released) = oneshot::channel::(); drop(release); finish_rename_tail_heal( @@ -10314,7 +10983,7 @@ mod tests { info.size = 11; info.parts.clear(); info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None); - let crate::disk::Disk::Local(local) = disk.as_ref() else { + let disk::Disk::Local(local) = disk.as_ref() else { panic!("physical publication fixture requires local disks"); }; // Linux IO paths use a mount FD, which is also the namespace lock key. @@ -10322,7 +10991,7 @@ mod tests { .get_disk() .get_object_path_for_io(bucket, object) .expect("the publication path must resolve through the disk's mount lease"); - let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (entered_tx, entered_rx) = oneshot::channel(); let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); hooks.push(os::prepared_publication_test_hooks::install( &destination.join(STORAGE_FORMAT_FILE), @@ -10508,11 +11177,11 @@ mod tests { info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None); } let disk = disks[3].as_ref().expect("tail disk"); - let crate::disk::Disk::Local(local) = disk.as_ref() else { + let disk::Disk::Local(local) = disk.as_ref() else { panic!("local fixture"); }; let destination = local.get_disk().get_object_path_for_io(bucket, object).expect("tail IO path"); - let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (entered_tx, entered_rx) = oneshot::channel(); let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); let _hook = os::prepared_publication_test_hooks::install(&destination.join(STORAGE_FORMAT_FILE), move || { let _ = entered_tx.send(()); @@ -10524,7 +11193,7 @@ mod tests { let receipt = RenameRollbackReceipt::default(); let caller_receipt = receipt.clone(); let caller_disks = disks.clone(); - let (ack_tx, ack_rx) = tokio::sync::oneshot::channel(); + let (ack_tx, ack_rx) = oneshot::channel(); let caller = tokio::spawn(async move { let commit = SetDisks::rename_data_owned_with_fence( &caller_disks, @@ -12001,6 +12670,9 @@ mod tests { let bucket = "dangling-invalid-meta-bucket"; let object = "object"; let set = io_primitives_test_set(vec![None, None, None, None], 1).await; + let cached_generation = set + .get_object_metadata_cache_generation(bucket, object) + .expect("local cache generation should be enabled"); let mut data_errs_by_part = HashMap::new(); data_errs_by_part.insert( 1, @@ -12035,6 +12707,10 @@ mod tests { .expect("invalid metadata should be cleanable when part results prove the object is dangling"); assert!(!deleted.is_valid()); + assert!( + !set.is_get_object_metadata_cache_generation_current(cached_generation), + "dangling cleanup must retire locally cached metadata before deletion" + ); } #[tokio::test] diff --git a/crates/ecstore/src/set_disk/core/metadata_read.rs b/crates/ecstore/src/set_disk/core/metadata_read.rs index 37e727395..39e061c61 100644 --- a/crates/ecstore/src/set_disk/core/metadata_read.rs +++ b/crates/ecstore/src/set_disk/core/metadata_read.rs @@ -576,6 +576,7 @@ impl MetadataFanoutDiagnostics { pub(in crate::set_disk) fn record_quorum_candidate_latency(&self, path: &'static str, read_quorum: usize) { if let Some(latency) = self.quorum_candidate_latency(read_quorum) { + rustfs_io_metrics::record_get_object_metadata_quorum_result(path, "reached"); rustfs_io_metrics::record_get_object_quorum_reached_latency(path, latency.as_secs_f64()); } } diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 58d18b5a7..cd736e932 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -166,7 +166,7 @@ use sha2::Sha256; use std::hash::{BuildHasher, Hash, Hasher}; use std::mem::{self}; use std::pin::Pin; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, OnceLock}; use std::task::{Context, Poll}; use std::time::{Instant, SystemTime, UNIX_EPOCH}; @@ -410,6 +410,12 @@ impl Drop for ObjectLockDiagGuard { } let hold = self.acquired_at.elapsed(); + if self.op == "put_object_commit" && self.mode == "write" && rustfs_io_metrics::put_stage_metrics_enabled() { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD, + hold.as_secs_f64() * 1000.0, + ); + } record_object_lock_diag_hold_duration(self.op, self.mode, hold); let threshold = get_object_lock_diag_slow_hold_threshold(); if hold >= threshold { @@ -925,16 +931,21 @@ pub(in crate::set_disk) struct GetObjectFileInfo { owned: Option, shared: Option>, condition_stopped: bool, + late_metadata_fanout: Option, } struct OwnedGetObjectFileInfo { fi: FileInfo, parts_metadata: Vec, online_disks: Vec>, - late_metadata_fanout_disks: Option>>, } -type OwnedGetObjectFileInfoParts = (FileInfo, Vec, Vec>, Option>>); +enum LateMetadataFanout { + CapturedDisks(Vec>), + RefreshCurrentSet, +} + +type OwnedGetObjectFileInfoParts = (FileInfo, Vec, Vec>, Option); impl GetObjectFileInfo { fn owned(fi: FileInfo, parts_metadata: Vec, online_disks: Vec>) -> Self { @@ -943,10 +954,10 @@ impl GetObjectFileInfo { fi, parts_metadata, online_disks, - late_metadata_fanout_disks: None, }), shared: None, condition_stopped: false, + late_metadata_fanout: None, } } @@ -956,16 +967,9 @@ impl GetObjectFileInfo { online_disks: Vec>, late_metadata_fanout_disks: Vec>, ) -> Self { - Self { - owned: Some(OwnedGetObjectFileInfo { - fi, - parts_metadata, - online_disks, - late_metadata_fanout_disks: Some(late_metadata_fanout_disks), - }), - shared: None, - condition_stopped: false, - } + let mut snapshot = Self::owned(fi, parts_metadata, online_disks); + snapshot.late_metadata_fanout = Some(LateMetadataFanout::CapturedDisks(late_metadata_fanout_disks)); + snapshot } fn shared(entry: Arc) -> Self { @@ -974,9 +978,16 @@ impl GetObjectFileInfo { owned: None, shared: Some(entry), condition_stopped, + late_metadata_fanout: None, } } + fn shared_with_late_metadata_refresh(entry: Arc) -> Self { + let mut snapshot = Self::shared(entry); + snapshot.late_metadata_fanout = Some(LateMetadataFanout::RefreshCurrentSet); + snapshot + } + fn fi(&self) -> &FileInfo { match (&self.owned, &self.shared) { (Some(snapshot), None) => &snapshot.fi, @@ -1002,9 +1013,7 @@ impl GetObjectFileInfo { } fn has_late_metadata_fanout(&self) -> bool { - self.owned - .as_ref() - .is_some_and(|snapshot| snapshot.late_metadata_fanout_disks.is_some()) + self.late_metadata_fanout.is_some() } fn into_owned(self) -> (FileInfo, Vec, Vec>) { @@ -1013,16 +1022,22 @@ impl GetObjectFileInfo { } fn into_owned_with_late_metadata_fanout(self) -> OwnedGetObjectFileInfoParts { - match (self.owned, self.shared) { - (Some(snapshot), None) => ( - snapshot.fi, - snapshot.parts_metadata, - snapshot.online_disks, - snapshot.late_metadata_fanout_disks, - ), + let Self { + owned, + shared, + late_metadata_fanout, + .. + } = self; + match (owned, shared) { + (Some(snapshot), None) => (snapshot.fi, snapshot.parts_metadata, snapshot.online_disks, late_metadata_fanout), (None, Some(entry)) => match Arc::try_unwrap(entry) { - Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks, None), - Err(entry) => (entry.fi.clone(), entry.parts_metadata.clone(), entry.online_disks.clone(), None), + Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks, late_metadata_fanout), + Err(entry) => ( + entry.fi.clone(), + entry.parts_metadata.clone(), + entry.online_disks.clone(), + late_metadata_fanout, + ), }, _ => unreachable!("GET metadata snapshot representation must be exclusive"), } @@ -1413,11 +1428,11 @@ mod prepared_get_object_metadata_tests { let snapshot = GetObjectFileInfo::shared(Arc::clone(&cached)); assert!(snapshot.has_valid_representation()); - assert!(std::mem::size_of::() >= std::mem::size_of::()); - assert!( - std::mem::size_of::() - <= std::mem::size_of::() + 2 * std::mem::size_of::() - ); + // Keep the original payload budget after moving the captured disks out + // of OwnedGetObjectFileInfo. + let owned_payload_size = std::mem::size_of::<(OwnedGetObjectFileInfo, Option>>)>(); + assert!(std::mem::size_of::() >= owned_payload_size); + assert!(std::mem::size_of::() <= owned_payload_size + 2 * std::mem::size_of::()); assert_eq!(Arc::strong_count(&cached), 2, "a cache hit must add one snapshot reference"); assert_eq!(snapshot.fi().name, "object"); assert_eq!(snapshot.parts_metadata().len(), 1); @@ -4337,6 +4352,9 @@ pub struct SetDisks { /// writes skip the global registry mutex (backlog#1315). `Arc` so clones of /// a set share one generation marker. capacity_dirty_generation: Arc, + /// Per-set proof that successful rename_data peers return the old-current + /// source capability needed before PUT may skip its pre-rename lookup. + tier_free_version_rename_data_source_capable: Arc, /// Orphan prefixes whose last purge scan met data that can never be /// purged by listing (residue without a committed marker, an in-flight /// write), keyed by `bucket/prefix` with the time of that scan. Empty @@ -4592,6 +4610,14 @@ struct GetObjectMetadataCacheKey { hash: u64, } +#[derive(Clone, Copy, Debug)] +struct GetObjectMetadataCacheLookupKey<'a> { + bucket: &'a str, + object: &'a str, + generation: u64, + hash: u64, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct GetObjectMetadataCacheGeneration { index: usize, @@ -4680,6 +4706,33 @@ impl GetObjectMetadataCacheKey { } } +impl<'a> GetObjectMetadataCacheLookupKey<'a> { + fn new(bucket: &'a str, object: &'a str, generation: GetObjectMetadataCacheGeneration) -> Self { + Self { + bucket, + object, + generation: generation.value, + hash: generation.hash, + } + } +} + +impl Hash for GetObjectMetadataCacheLookupKey<'_> { + fn hash(&self, state: &mut H) { + self.hash.hash(state); + self.generation.hash(state); + } +} + +impl moka::Equivalent for GetObjectMetadataCacheLookupKey<'_> { + fn equivalent(&self, key: &GetObjectMetadataCacheKey) -> bool { + self.bucket == key.bucket.as_ref() + && self.object == key.object.as_ref() + && self.generation == key.generation + && self.hash == key.hash + } +} + impl Hash for GetObjectMetadataCacheKey { fn hash(&self, state: &mut H) { self.hash.hash(state); @@ -4690,7 +4743,9 @@ impl Hash for GetObjectMetadataCacheKey { #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum GetObjectMetadataCacheCapability { Full, - // Quorum-validated metadata without a complete body or write-target layout. + // A quorum-selected layout can serve body reads, but is not a mutation layout. + QuorumSelected, + // Quorum-validated metadata used only to evaluate a terminal read condition. ConditionOnly, } @@ -5233,6 +5288,7 @@ impl SetDisks { ctx, capacity_scope_cache: Arc::new(std::sync::RwLock::new(CapacityScopeCache::default())), capacity_dirty_generation: Arc::new(AtomicU64::new(u64::MAX)), + tier_free_version_rename_data_source_capable: Arc::new(AtomicBool::new(false)), orphan_purge_backoff: Arc::new(std::sync::Mutex::new(HashMap::new())), #[cfg(test)] storage_class_config_override: Arc::new(std::sync::RwLock::new(None)), @@ -5247,6 +5303,15 @@ impl SetDisks { &self.ctx } + pub(in crate::set_disk) fn tier_free_version_rename_data_source_capable(&self) -> bool { + self.tier_free_version_rename_data_source_capable.load(Ordering::Acquire) + } + + pub(in crate::set_disk) fn observe_tier_free_version_rename_data_source_capability(&self, capable: bool) { + self.tier_free_version_rename_data_source_capable + .store(capable, Ordering::Release); + } + #[cfg(test)] pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc) { self.local_lock_manager = ctx.lock_manager(); diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 6f5ae956e..0365d0aa2 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -20,6 +20,7 @@ //! SetDisks core (io_primitives) via inherent calls. use crate::core::pools::DecommissionCapacityAdmission; +use crate::disk::cleanup_runtime::{self, OLD_DATA_CLEANUP_RECEIPT_FILE}; use rustfs_common::mrf_channel::MrfDeleteMarkerPurge; use rustfs_filemeta::metadata_keys; @@ -36,9 +37,9 @@ use super::super::{ GET_STAGE_LOCK_ACQUIRE, GET_STAGE_METADATA, GET_STAGE_OBJECT_INFO, GET_STAGE_PATH_DECISION, GET_STAGE_READER_SETUP, GenericError, GetCodecStreamingDecision, GetCodecStreamingFallbackReason, GetDirectMemoryDecision, GetObjectReader, HTTPRangeSpec, HashAlgorithm, HashMap, HashReader, HashSet, HeaderMap, HealChannelPriority, InstanceContext, Instant, - LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, OBJECT_OP_IGNORED_ERRS, ObjectApiError, ObjectInfo, ObjectKey, - ObjectLockConfigSnapshot, ObjectLockConfigState, ObjectOptions, ObjectReader, ObjectToDelete, OffsetDateTime, Ordering, Pin, - PutObjReader, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, ReadPathPlan, ReaderImpl, ReplicateDecision, + LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, LateMetadataFanout, OBJECT_OP_IGNORED_ERRS, ObjectApiError, ObjectInfo, + ObjectKey, ObjectLockConfigSnapshot, ObjectLockConfigState, ObjectOptions, ObjectReader, ObjectToDelete, OffsetDateTime, + Ordering, Pin, PutObjReader, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, ReadPathPlan, ReaderImpl, ReplicateDecision, ReplicationObjectBridge, Result, SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS, SLASH_SEPARATOR, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks, SmallWritePath, StorageError, TRANSITION_COMPLETE, UpdateMetadataOpts, Uuid, WriteLayout, adaptive_duplex_buffer_size, @@ -93,6 +94,392 @@ impl<'a> DeleteCommitContext<'a> { } } +const TIER_FREE_VERSION_SOURCE_LOOKUP_COUNTERFACTUAL_SKIP_ENV: &str = + "RUSTFS_PUT_TIER_FREE_VERSION_SOURCE_LOOKUP_COUNTERFACTUAL_SKIP"; +const TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ENV: &str = "RUSTFS_PUT_TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA"; +const TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ASSUME_HOMOGENEOUS_ENV: &str = + "RUSTFS_PUT_TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ASSUME_HOMOGENEOUS"; +const RENAME_TAIL_CLEANUP_PER_DISK_LIMIT_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_PER_DISK_LIMIT"; +const RENAME_TAIL_CLEANUP_PER_DISK_LANES_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_PER_DISK_LANES"; +const RENAME_TAIL_CLEANUP_LANE_ENABLE_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_ENABLE"; +const RENAME_TAIL_CLEANUP_LANE_CAPACITY_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_CAPACITY"; +const RENAME_TAIL_CLEANUP_LANE_BATCH_MAX_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_BATCH_MAX"; +const RENAME_TAIL_CLEANUP_LANE_BATCH_WINDOW_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_BATCH_WINDOW_MS"; +const RENAME_TAIL_CLEANUP_LANE_ACTIVE_WINDOW_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_ACTIVE_WINDOW_MS"; +const RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP"; +const DEFAULT_RENAME_TAIL_CLEANUP_PER_DISK_LANES: usize = 1024; +const DEFAULT_RENAME_TAIL_CLEANUP_LANE_CAPACITY: usize = 1024; +const DEFAULT_RENAME_TAIL_CLEANUP_LANE_BATCH_MAX: usize = 1; + +fn tier_free_version_source_lookup_counterfactual_skip_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(TIER_FREE_VERSION_SOURCE_LOOKUP_COUNTERFACTUAL_SKIP_ENV) + .ok() + .as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn rename_tail_cleanup_zero_target_tmp_delete_skip_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP_ENV) + .ok() + .as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn tier_free_version_source_reuse_rename_data_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ENV).ok().as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn tier_free_version_source_reuse_rename_data_assume_homogeneous_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ASSUME_HOMOGENEOUS_ENV) + .ok() + .as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn rename_tail_cleanup_per_disk_limit() -> usize { + static LIMIT: std::sync::OnceLock = std::sync::OnceLock::new(); + *LIMIT.get_or_init(|| rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_PER_DISK_LIMIT_ENV, 0)) +} + +fn rename_tail_cleanup_per_disk_lanes() -> usize { + static LANES: std::sync::OnceLock = std::sync::OnceLock::new(); + *LANES.get_or_init(|| { + rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_PER_DISK_LANES_ENV, DEFAULT_RENAME_TAIL_CLEANUP_PER_DISK_LANES).max(1) + }) +} + +fn rename_tail_cleanup_foreground_yield() -> Duration { + static YIELD: std::sync::OnceLock = std::sync::OnceLock::new(); + *YIELD + .get_or_init(|| Duration::from_millis(rustfs_utils::get_env_u64("RUSTFS_PUT_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD_MS", 0))) +} + +fn rename_tail_cleanup_foreground_active_yield() -> Duration { + static YIELD: std::sync::OnceLock = std::sync::OnceLock::new(); + *YIELD.get_or_init(|| { + Duration::from_millis(rustfs_utils::get_env_u64("RUSTFS_PUT_RENAME_TAIL_CLEANUP_FOREGROUND_ACTIVE_YIELD_MS", 0)) + }) +} + +fn rename_tail_cleanup_lane_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + matches!( + std::env::var(RENAME_TAIL_CLEANUP_LANE_ENABLE_ENV).ok().as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) + }) +} + +fn rename_tail_cleanup_lane_capacity() -> usize { + static CAPACITY: std::sync::OnceLock = std::sync::OnceLock::new(); + *CAPACITY.get_or_init(|| { + rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_LANE_CAPACITY_ENV, DEFAULT_RENAME_TAIL_CLEANUP_LANE_CAPACITY).max(1) + }) +} + +fn rename_tail_cleanup_lane_batch_max() -> usize { + static BATCH_MAX: std::sync::OnceLock = std::sync::OnceLock::new(); + *BATCH_MAX.get_or_init(|| { + rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_LANE_BATCH_MAX_ENV, DEFAULT_RENAME_TAIL_CLEANUP_LANE_BATCH_MAX).max(1) + }) +} + +fn rename_tail_cleanup_lane_batch_window() -> Duration { + static WINDOW: std::sync::OnceLock = std::sync::OnceLock::new(); + *WINDOW.get_or_init(|| Duration::from_millis(rustfs_utils::get_env_u64(RENAME_TAIL_CLEANUP_LANE_BATCH_WINDOW_MS_ENV, 0))) +} + +fn rename_tail_cleanup_lane_active_window() -> Duration { + static WINDOW: std::sync::OnceLock = std::sync::OnceLock::new(); + *WINDOW.get_or_init(|| Duration::from_millis(rustfs_utils::get_env_u64(RENAME_TAIL_CLEANUP_LANE_ACTIVE_WINDOW_MS_ENV, 0))) +} + +async fn rename_tail_cleanup_per_disk_permit(disk_index: usize) -> Option { + let limit = rename_tail_cleanup_per_disk_limit(); + if limit == 0 { + return None; + } + static LANES: std::sync::OnceLock>> = std::sync::OnceLock::new(); + let lanes = LANES.get_or_init(|| { + (0..rename_tail_cleanup_per_disk_lanes()) + .map(|_| Arc::new(tokio::sync::Semaphore::new(limit))) + .collect() + }); + let semaphore = Arc::clone(&lanes[disk_index % lanes.len()]); + let wait_started = rustfs_io_metrics::put_stage_timer(); + let permit = semaphore.clone().acquire_owned().await.ok(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_PERMIT_WAIT, + wait_started, + ); + if let Some(semaphore_permit) = permit { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_INFLIGHT, + limit.saturating_sub(semaphore.available_permits()) as f64, + ); + Some(semaphore_permit) + } else { + None + } +} + +type RenameTailCleanupLaneFuture = Pin + Send + 'static>>; + +struct RenameTailCleanupLaneJob { + disk_index: usize, + future: RenameTailCleanupLaneFuture, + done: tokio::sync::oneshot::Sender<()>, +} + +struct RenameTailCleanupLane { + sender: tokio::sync::mpsc::Sender, + queued: Arc, + oldest_enqueued_at: Arc>>, +} + +struct RenameTailCleanupLaneQueue { + lanes: Vec, +} + +async fn run_rename_tail_cleanup_lane_job(job: RenameTailCleanupLaneJob) { + let service_started = rustfs_io_metrics::put_stage_timer(); + job.future.await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_SERVICE_TIME, + service_started, + ); + let _ = job.done.send(()); +} + +impl RenameTailCleanupLaneQueue { + fn new() -> Self { + let lanes = (0..rename_tail_cleanup_per_disk_lanes()) + .map(|_| { + let (sender, mut receiver) = + tokio::sync::mpsc::channel::(rename_tail_cleanup_lane_capacity()); + let queued = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let active_workers = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let oldest_enqueued_at = Arc::new(tokio::sync::Mutex::new(None::)); + let worker_queued = Arc::clone(&queued); + let worker_active = Arc::clone(&active_workers); + let worker_oldest_enqueued_at = Arc::clone(&oldest_enqueued_at); + tokio::spawn(async move { + loop { + let queued_before_poll = worker_queued.load(Ordering::Relaxed); + let worker_poll_started = rustfs_io_metrics::put_stage_timer(); + let Some(job) = receiver.recv().await else { + return; + }; + if queued_before_poll == 0 { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_IDLE_WAIT, + worker_poll_started, + ); + } else { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_POLL_WAIT, + worker_poll_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_JOB_WAIT, + worker_poll_started, + ); + } + let mut batch = vec![job]; + let batch_window = rename_tail_cleanup_lane_batch_window(); + let batch_max = rename_tail_cleanup_lane_batch_max(); + if batch_max > 1 && !batch_window.is_zero() { + tokio::time::sleep(batch_window).await; + } + while batch.len() < batch_max { + let Some(next) = receiver.try_recv().ok() else { + break; + }; + batch.push(next); + } + let queued_after_dequeue = worker_queued + .fetch_sub(batch.len(), Ordering::Relaxed) + .saturating_sub(batch.len()); + if queued_after_dequeue == 0 { + *worker_oldest_enqueued_at.lock().await = None; + } + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH, + queued_after_dequeue as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE, + queued_after_dequeue as f64, + ); + if let Some(oldest) = *worker_oldest_enqueued_at.lock().await { + let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE, + oldest_age_ms, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE_MAX_SAMPLE, + oldest_age_ms, + ); + } + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BATCH_SIZE, + batch.len() as f64, + ); + if batch.len() > 1 { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BURST_BATCH_SIZE, + batch.len() as f64, + ); + } + let active = worker_active.fetch_add(1, Ordering::Relaxed) + 1; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_ACTIVE, + active as f64, + ); + let active_window = rename_tail_cleanup_lane_active_window(); + if !active_window.is_zero() { + let mut deferred = Vec::new(); + let mut ready = Vec::new(); + for job in batch { + let active = rename_foreground_active_count(job.disk_index); + if active > 0 { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_FOREGROUND_ACTIVE_HIT, + active as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_REQUEUE, + 1.0, + ); + deferred.push(job); + } else { + ready.push(job); + } + } + for job in ready { + run_rename_tail_cleanup_lane_job(job).await; + } + if !deferred.is_empty() { + tokio::time::sleep(active_window).await; + for job in deferred { + run_rename_tail_cleanup_lane_job(job).await; + } + } + } else { + for job in batch { + run_rename_tail_cleanup_lane_job(job).await; + } + } + worker_active.fetch_sub(1, Ordering::Relaxed); + } + }); + RenameTailCleanupLane { + sender, + queued, + oldest_enqueued_at, + } + }) + .collect(); + Self { lanes } + } + + async fn enqueue( + &self, + disk_index: usize, + future: RenameTailCleanupLaneFuture, + ) -> Option> { + let lane = &self.lanes[disk_index % self.lanes.len()]; + let enqueued_at = Instant::now(); + let queue_push_started = rustfs_io_metrics::put_stage_timer(); + { + let mut oldest = lane.oldest_enqueued_at.lock().await; + if oldest.is_none() { + *oldest = Some(enqueued_at); + } + } + let queued_after_enqueue = lane.queued.fetch_add(1, Ordering::Relaxed) + 1; + let (done, done_rx) = tokio::sync::oneshot::channel(); + let job = RenameTailCleanupLaneJob { + disk_index, + future, + done, + }; + match lane.sender.try_send(job) { + Ok(()) => { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH, + queued_after_enqueue as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE, + queued_after_enqueue as f64, + ); + if let Some(oldest) = *lane.oldest_enqueued_at.lock().await { + let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE, + oldest_age_ms, + ); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE_MAX_SAMPLE, + oldest_age_ms, + ); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_PUSH, + queue_push_started, + ); + Some(done_rx) + } + Err(err) => { + lane.queued.fetch_sub(1, Ordering::Relaxed); + if lane.queued.load(Ordering::Relaxed) == 0 { + *lane.oldest_enqueued_at.lock().await = None; + } + let job = err.into_inner(); + run_rename_tail_cleanup_lane_job(job).await; + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE, + lane.queued.load(Ordering::Relaxed) as f64, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_PUSH, + queue_push_started, + ); + None + } + } + } +} + +fn rename_tail_cleanup_lane_queue() -> &'static RenameTailCleanupLaneQueue { + static QUEUE: std::sync::OnceLock = std::sync::OnceLock::new(); + QUEUE.get_or_init(RenameTailCleanupLaneQueue::new) +} + fn delete_marker_purge_candidate( file_info: &FileInfo, expected_bucket_incarnation_id: Option, @@ -358,7 +745,9 @@ use crate::object_api::{ }; use crate::services::notification_sys::RemoteVersionStateFleetProofToken; use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata}; -use crate::set_disk::core::io_primitives::{RenameRollbackReceipt, RenameTailCleanup, finish_rename_tail_heal}; +use crate::set_disk::core::io_primitives::{ + RenameRollbackReceipt, RenameTailCleanup, finish_rename_tail_heal, rename_foreground_active_count, +}; #[cfg(test)] use crate::storage_api_contracts::namespace::NamespaceLocking; #[cfg(test)] @@ -528,7 +917,6 @@ async fn acquire_single_tier_delete_lease(opts: &ObjectOptions, source: &ObjectI Ok(Some(lease)) } -const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json"; const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024; const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256; @@ -2809,9 +3197,9 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { let object_size = usize::try_from(object_info.size) .map_err(|_| to_object_err(Error::other("two-phase GET object size is invalid"), vec![bucket, object]))?; let mut output = Vec::with_capacity(object_size); - let (fi, files, disks, late_metadata_fanout_disks) = snapshot.into_owned_with_late_metadata_fanout(); + let (fi, files, disks, late_metadata_fanout) = snapshot.into_owned_with_late_metadata_fanout(); let expected_identity = super::super::read::LateMetadataIdentity::from_file_info(&fi); - let late_metadata_fanout_disks = late_metadata_fanout_disks.ok_or_else(|| { + let late_metadata_fanout = late_metadata_fanout.ok_or_else(|| { to_object_err(Error::other("two-phase GET fallback context is missing"), vec![bucket, object]) })?; let initial_result = Self::get_object_with_fileinfo( @@ -2836,6 +3224,10 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { ) .await; if prepare_late_materialized_retry(&initial_result, &mut output, object_size) { + let late_metadata_fanout_disks = match late_metadata_fanout { + LateMetadataFanout::CapturedDisks(disks) => disks, + LateMetadataFanout::RefreshCurrentSet => self.get_disks_for_data_read().await, + }; #[cfg(test)] super::super::rename_fanout_barrier::checkpoint( object, @@ -3356,6 +3748,19 @@ pub(in crate::set_disk) fn merge_replication_metadata_lww( } impl SetDisks { + /// Failure cleanup stays awaited, but shares the same execution budget as + /// successful PUT continuations when isolation is enabled. + async fn delete_put_tmp_with_budget(&self, tmp_dir: &str, reservation: &Option) -> Result<()> { + if reservation.is_none() { + return self.delete_all(RUSTFS_META_TMP_BUCKET, tmp_dir).await; + } + let set = self.clone(); + let tmp_dir = tmp_dir.to_owned(); + cleanup_runtime::spawn(reservation.clone(), async move { set.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await }) + .await + .map_err(Error::other)? + } + async fn reclaim_missing_delete_residue( &self, bucket: &str, @@ -3434,26 +3839,107 @@ impl SetDisks { committed_data_dir: Option, epoch: Option, ) { + if rename_tail_cleanup_lane_enabled() { + let bucket = bucket.to_string(); + let object = object.to_string(); + let mut completions = Vec::with_capacity(targets.len()); + for target in targets { + let cleanup_set = self.clone(); + let cleanup_bucket = bucket.clone(); + let cleanup_object = object.clone(); + let disk_index = target.disk_index; + let future = Box::pin(async move { + cleanup_set + .cleanup_rename_tail_target(target, &cleanup_bucket, &cleanup_object, committed_data_dir, epoch) + .await; + }); + if let Some(done) = rename_tail_cleanup_lane_queue().enqueue(disk_index, future).await { + completions.push(done); + } + } + for done in completions { + let _ = done.await; + } + return; + } + for target in targets { - let mut disks = vec![None; self.set_drive_count]; - disks[target.disk_index] = Some(target.disk); - self.persist_old_data_cleanup_receipts(&disks, bucket, object, target.old_data_dir, committed_data_dir, epoch) - .await; - let cleanup = self - .commit_rename_data_dir_and_mark_capacity( - &disks, - bucket, - object, - &target.old_data_dir.to_string(), - &committed_data_dir.unwrap_or_default().to_string(), - 1, - ) - .await; - self.report_old_data_dir_cleanup(bucket, object, &target.old_data_dir.to_string(), &cleanup) + self.cleanup_rename_tail_target(target, bucket, object, committed_data_dir, epoch) .await; } } + async fn cleanup_rename_tail_target( + &self, + target: RenameTailCleanup, + bucket: &str, + object: &str, + committed_data_dir: Option, + epoch: Option, + ) { + let foreground_yield = rename_tail_cleanup_foreground_yield(); + if !foreground_yield.is_zero() { + let yield_started = rustfs_io_metrics::put_stage_timer(); + tokio::time::sleep(foreground_yield).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD, + yield_started, + ); + } + let active_yield = rename_tail_cleanup_foreground_active_yield(); + if !active_yield.is_zero() { + let active = rename_foreground_active_count(target.disk_index); + if active > 0 { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFERRED_FOREGROUND_ACTIVE, + active as f64, + ); + let yield_started = rustfs_io_metrics::put_stage_timer(); + tokio::time::sleep(active_yield).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD, + yield_started, + ); + } + } + let _cleanup_per_disk_permit = rename_tail_cleanup_per_disk_permit(target.disk_index).await; + let mut disks = vec![None; self.set_drive_count]; + disks[target.disk_index] = Some(target.disk); + let receipt_started = rustfs_io_metrics::put_stage_timer(); + self.persist_old_data_cleanup_receipts(&disks, bucket, object, target.old_data_dir, committed_data_dir, epoch) + .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_RECEIPT, + receipt_started, + ); + let commit_capacity_started = rustfs_io_metrics::put_stage_timer(); + let cleanup = self + .commit_rename_data_dir_and_mark_capacity( + &disks, + bucket, + object, + &target.old_data_dir.to_string(), + &committed_data_dir.unwrap_or_default().to_string(), + 1, + ) + .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COMMIT_CAPACITY, + commit_capacity_started, + ); + let report_started = rustfs_io_metrics::put_stage_timer(); + self.report_old_data_dir_cleanup(bucket, object, &target.old_data_dir.to_string(), &cleanup) + .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_REPORT, + report_started, + ); + } + + fn needs_old_data_cleanup_receipt(old_data_dir: Uuid, committed_data_dir: Option, epoch: Option) -> bool { + epoch.is_some() && committed_data_dir != Some(old_data_dir) + } + pub(in crate::set_disk) async fn persist_old_data_cleanup_receipts( &self, disks: &[Option], @@ -3463,10 +3949,10 @@ impl SetDisks { committed_data_dir: Option, epoch: Option, ) { - let Some(epoch) = epoch else { return }; - if committed_data_dir == Some(old_data_dir) { + if !Self::needs_old_data_cleanup_receipt(old_data_dir, committed_data_dir, epoch) { return; } + let Some(epoch) = epoch else { return }; let receipt = OldDataCleanupReceipt::new(epoch, old_data_dir, committed_data_dir); let Ok(encoded) = receipt.encode() else { return; @@ -3652,6 +4138,23 @@ impl SetDisks { .scanner_publication_commit_scope .clone() .map(ScannerPublicationCommitScopeGuard::new); + // Reserve before staging. Some callers already hold external locks, + // so admission must not wait for another PUT to release its budget. + // Commit/cleanup owners retain clones across cancellation and early ACK. + if opts + .put_object_cancellation + .as_ref() + .is_some_and(CancellationToken::is_cancelled) + { + return Err(StorageError::OperationCanceled); + } + let cleanup_reservation = cleanup_runtime::reserve().map_err(|err| { + if err.kind() == std::io::ErrorKind::WouldBlock { + StorageError::SlowDown + } else { + Error::other(err) + } + })?; let storage_class_config = self.storage_class_config_snapshot(); self.invalidate_get_object_metadata_cache(bucket, object).await; @@ -3771,7 +4274,13 @@ impl SetDisks { }; let mut tmp_cleanup_owned = false; let rollback_receipt = RenameRollbackReceipt::default(); + let operation_first_poll_delay_started = rustfs_io_metrics::put_stage_timer(); let operation = async { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY, + operation_first_poll_delay_started, + ); + let operation_body_to_commit_submit_started = rustfs_io_metrics::put_stage_timer(); let erasure = Arc::new(erasure_from_file_info(&fi, false)?); let put_object_size = known_put_object_storage_size(data.size()); @@ -4209,6 +4718,8 @@ impl SetDisks { } #[cfg(any(test, feature = "test-util"))] pause_put_object_commit(bucket, object, PutObjectCommitPause::AfterNamespace).await; + let namespace_lock_acquired_to_commit_ready_started = rustfs_io_metrics::put_stage_timer(); + let commit_ready_precondition_and_timestamp_started = rustfs_io_metrics::put_stage_timer(); if opts.http_preconditions.is_some() && let Some(err) = self.check_write_precondition(bucket, object, opts).await @@ -4229,8 +4740,13 @@ impl SetDisks { } } } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_PRECONDITION_AND_TIMESTAMP, + commit_ready_precondition_and_timestamp_started, + ); if let Some(expected) = opts.expected_current_version_id.as_deref() { + let expected_version_lookup_started = rustfs_io_metrics::put_stage_timer(); let current = self .get_object_info( bucket, @@ -4253,11 +4769,16 @@ impl SetDisks { if current.version_id.map(|version| version.to_string()).as_deref() != Some(expected) { return Err(StorageError::PreconditionFailed); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPECTED_VERSION_LOOKUP, + expected_version_lookup_started, + ); } if let Some(version_id) = opts.version_id.as_deref() && !is_meta_bucketname(bucket) { + let explicit_version_lookup_started = rustfs_io_metrics::put_stage_timer(); let current = self .get_object_info( bucket, @@ -4310,13 +4831,23 @@ impl SetDisks { Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {} Err(err) => return Err(err), } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPLICIT_VERSION_LOOKUP, + explicit_version_lookup_started, + ); } + let restore_verify_started = rustfs_io_metrics::put_stage_timer(); self.require_current_restore_operation_id(bucket, object, opts, expected_restore_operation_id, "put_object_commit") .await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_RESTORE_VERIFY, + restore_verify_started, + ); // Fence every commit-time read before entering rename_data. Once // rename_data returns Ok the write is durable and must not be aborted. + let fence_verify_started = rustfs_io_metrics::put_stage_timer(); if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) || publication_commit_guard .as_ref() @@ -4360,6 +4891,7 @@ impl SetDisks { }); } + let decommission_capacity_fence_started = rustfs_io_metrics::put_stage_timer(); if decommission_capacity_guard.is_none() && let Some(store) = opts.decommission_capacity_admission.as_ref() { @@ -4369,6 +4901,10 @@ impl SetDisks { .await?, ); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_DECOMMISSION_CAPACITY_FENCE, + decommission_capacity_fence_started, + ); // The object namespace is acquired above, after the input stream // has been fully staged. Only then admit the local publication @@ -4386,21 +4922,36 @@ impl SetDisks { achieved: 0, }); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_FENCE_VERIFY, + fence_verify_started, + ); let transaction_fencing_proof = object_transaction_fencing_fleet_proof(); if object_transaction_fencing_requested() && transaction_fencing_proof.is_none() { return Err(Error::other("object transaction fencing requires a live fleet capability proof")); } + let transaction_epoch_fence_read_started = rustfs_io_metrics::put_stage_timer(); let transaction_epoch_fence = if transaction_fencing_proof.is_some() { Some(read_object_transaction_epoch_fence(self, bucket, object).await?) } else { None }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_TRANSACTION_EPOCH_FENCE_READ, + transaction_epoch_fence_read_started, + ); + let quota_begin_started = rustfs_io_metrics::put_stage_timer(); let quota_context = reservation::begin(&self.ctx, bucket, object, opts, self.pool_index, self.set_index).await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_BEGIN, + quota_begin_started, + ); let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some(); let mut replication_quota_size = None; + let metadata_adjust_started = rustfs_io_metrics::put_stage_timer(); if opts.replication_request { if quota_context.is_enforced() && opts.preserve_ciphertext { return Err(Error::PartMissingOrCorrupt); @@ -4464,7 +5015,12 @@ impl SetDisks { } } } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_METADATA_ADJUST, + metadata_adjust_started, + ); + let quota_reserve_started = rustfs_io_metrics::put_stage_timer(); let (quota_old_size, quota_new_size) = if quota_context.is_enforced() { let new_size = match replication_quota_size { Some(size) => size, @@ -4482,8 +5038,13 @@ impl SetDisks { (0, 0) }; let quota_reservation = quota_context.reserve(quota_old_size, quota_new_size).await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_RESERVE, + quota_reserve_started, + ); let (commit_disks, quota_fence_tokens) = if quota_mutation_fence { - match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await { + let quota_mutation_fence_prepare_started = rustfs_io_metrics::put_stage_timer(); + let prepared = match Self::prepare_quota_mutation_fences(&shuffle_disks, bucket, object, write_quorum).await { Ok((disks, tokens)) => { for (metadata, token) in parts_metadatas.iter_mut().zip(tokens.iter().copied()) { if let Some(token) = token { @@ -4500,10 +5061,16 @@ impl SetDisks { quota_reservation.abort().await; return Err(err); } - } + }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_MUTATION_FENCE_PREPARE, + quota_mutation_fence_prepare_started, + ); + prepared } else { (shuffle_disks.clone(), vec![None; shuffle_disks.len()]) }; + let commit_context_prepare_started = rustfs_io_metrics::put_stage_timer(); let transaction_epoch = transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&commit_disks, &mut parts_metadatas)); @@ -4550,10 +5117,26 @@ impl SetDisks { let commit_tier_free_version_receipt_sink = opts.tier_free_version_receipt_sink.clone(); let commit_skip_free_version = opts.skip_free_version; let request_cancellation = operation_cancellation.clone(); + let commit_cleanup_reservation = cleanup_reservation.clone(); tmp_cleanup_owned = true; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_READY_CONTEXT_PREPARE, + commit_context_prepare_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY, + namespace_lock_acquired_to_commit_ready_started, + ); - let commit = move |cancellation: Option| async move { + let commit = move |commit_submit_to_closure_enter_started: Option, + cancellation: Option| async move { let write_context = write_context; + let commit_closure_total_started = rustfs_io_metrics::put_stage_timer(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER, + commit_submit_to_closure_enter_started, + ); + let commit_closure_enter_to_first_await_started = rustfs_io_metrics::put_stage_timer(); let mut _object_lock_guard = commit_object_lock_guard; let mut _decommission_object_lock_guard = commit_decommission_object_lock_guard; let mut _publication_guard = commit_publication_guard; @@ -4561,12 +5144,20 @@ impl SetDisks { let mut _decommission_capacity_guard = commit_decommission_capacity_guard; let mut quota_reservation = quota_reservation; let rename_stage_start = Instant::now(); + let commit_before_rename_started = rustfs_io_metrics::put_stage_timer(); + let lock_held_pre_rename_prepare_started = rustfs_io_metrics::put_stage_timer(); let pre_rename = async { #[cfg(any(test, feature = "test-util"))] pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterQuotaReservation).await; + let quota_mark_commit_started = rustfs_io_metrics::put_stage_timer(); quota_reservation.mark_commit_started().await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_QUOTA_MARK_COMMIT_STARTED, + quota_mark_commit_started, + ); #[cfg(any(test, feature = "test-util"))] pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::BeforeQuotaRename).await; + let first_fence_check_started = rustfs_io_metrics::put_stage_timer(); if quota_reservation.is_lock_lost() || !quota_reservation.capability_proof_matches() || write_context.is_lock_lost() @@ -4596,6 +5187,10 @@ impl SetDisks { achieved: 0, }); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_FIRST_FENCE_CHECK, + first_fence_check_started, + ); let restore_opts = ObjectOptions { version_id: commit_version_id.clone(), versioned: commit_versioned, @@ -4603,6 +5198,7 @@ impl SetDisks { no_lock: true, ..Default::default() }; + let restore_verify_started = rustfs_io_metrics::put_stage_timer(); commit_set .require_current_restore_operation_id( &commit_bucket, @@ -4612,6 +5208,10 @@ impl SetDisks { "put_object_quota_reservation", ) .await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_RESTORE_VERIFY, + restore_verify_started, + ); if let Some(proof) = transaction_fencing_proof.as_ref() && !object_transaction_fencing_fleet_proof_matches(proof) { @@ -4625,8 +5225,14 @@ impl SetDisks { PutObjectCommitPause::BeforeTransactionEpochVerify, ) .await; + let transaction_fence_verify_started = rustfs_io_metrics::put_stage_timer(); verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TRANSACTION_FENCE_VERIFY, + transaction_fence_verify_started, + ); } + let second_fence_check_started = rustfs_io_metrics::put_stage_timer(); if quota_reservation.is_lock_lost() || !quota_reservation.capability_proof_matches() || write_context.is_lock_lost() @@ -4656,11 +5262,20 @@ impl SetDisks { achieved: 0, }); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SECOND_FENCE_CHECK, + second_fence_check_started, + ); Ok(()) }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT, + commit_closure_enter_to_first_await_started, + ); + let pre_rename_await_total_started = rustfs_io_metrics::put_stage_timer(); let mut pre_rename_result = if cancellation.is_some() || request_cancellation.is_some() { tokio::select! { - biased; + biased; _ = wait_for_put_object_commit_cancellation(cancellation.as_ref(), request_cancellation.as_ref()) => { Err(StorageError::OperationCanceled) }, @@ -4669,6 +5284,12 @@ impl SetDisks { } else { pre_rename.await }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_AWAIT_TOTAL, + pre_rename_await_total_started, + ); + let pre_rename_post_await_to_rename_started = rustfs_io_metrics::put_stage_timer(); + let scanner_scope_check_started = rustfs_io_metrics::put_stage_timer(); if pre_rename_result.is_ok() && let Some(scope) = commit_scanner_publication_scope.as_ref() && let Err(err) = scope.try_begin() @@ -4683,6 +5304,10 @@ impl SetDisks { let _ = scope.mark_indeterminate(); pre_rename_result = Err(StorageError::OperationCanceled); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SCANNER_SCOPE_CHECK, + scanner_scope_check_started, + ); if let Err(err) = pre_rename_result { SetDisks::abort_quota_reservation_after_fence( quota_reservation, @@ -4694,7 +5319,10 @@ impl SetDisks { quota_mutation_fence, ) .await; - if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await { + if let Err(cleanup_err) = commit_set + .delete_put_tmp_with_budget(&commit_tmp_dir, &commit_cleanup_reservation) + .await + { warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { warn!( @@ -4707,43 +5335,72 @@ impl SetDisks { } return Err(err); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_PRE_RENAME_PREPARE, + lock_held_pre_rename_prepare_started, + ); - let put_tier_free_version_source = - if commit_put_tier_free_version_id.is_some() && commit_tier_free_version_receipt_sink.is_some() { - match commit_set - .get_object_info( - &commit_bucket, - &commit_object, - &ObjectOptions { - no_lock: true, - metadata_cache_safe: false, - versioned: commit_versioned, - version_suspended: commit_version_suspended, - ..Default::default() - }, - ) - .await - { - Ok(source) => Some(source), - Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => None, - Err(err) => { - debug!( - event = EVENT_LIFECYCLE_TRANSITIONED_DELETE_CLEANUP_OWNER, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_SET_DISK, - bucket = %commit_bucket, - object = %commit_object, - error = ?err, - "Skipped opportunistic tier free-version receipt source capture" - ); - None - } + let tier_free_version_source_lookup_started = rustfs_io_metrics::put_stage_timer(); + let reuse_rename_data_source_for_tier_free_version = tier_free_version_source_reuse_rename_data_enabled() + && (commit_set.tier_free_version_rename_data_source_capable() + || tier_free_version_source_reuse_rename_data_assume_homogeneous_enabled()); + let mut put_tier_free_version_source = if commit_put_tier_free_version_id.is_some() + && commit_tier_free_version_receipt_sink.is_some() + && !tier_free_version_source_lookup_counterfactual_skip_enabled() + && !reuse_rename_data_source_for_tier_free_version + { + match commit_set + .get_object_info( + &commit_bucket, + &commit_object, + &ObjectOptions { + no_lock: true, + metadata_cache_safe: false, + versioned: commit_versioned, + version_suspended: commit_version_suspended, + ..Default::default() + }, + ) + .await + { + Ok(source) => Some(source), + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => None, + Err(err) => { + debug!( + event = EVENT_LIFECYCLE_TRANSITIONED_DELETE_CLEANUP_OWNER, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket = %commit_bucket, + object = %commit_object, + error = ?err, + "Skipped opportunistic tier free-version receipt source capture" + ); + None } - } else { - None - }; + } + } else { + None + }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TIER_FREE_VERSION_SOURCE_LOOKUP, + tier_free_version_source_lookup_started, + ); + let assign_rename_indexes_started = rustfs_io_metrics::put_stage_timer(); Self::assign_rename_data_indexes(&mut parts_metadatas); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_ASSIGN_RENAME_INDEXES, + assign_rename_indexes_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_POST_AWAIT_TO_RENAME, + pre_rename_post_await_to_rename_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME, + commit_before_rename_started, + ); + let lock_held_rename_started = rustfs_io_metrics::put_stage_timer(); let mut rename_result = SetDisks::rename_data_owned_with_fence( &commit_disks, (RUSTFS_META_TMP_BUCKET, commit_tmp_dir.as_str()), @@ -4761,6 +5418,12 @@ impl SetDisks { ), ) .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME, + lock_held_rename_started, + ); + let commit_after_rename_to_return_started = rustfs_io_metrics::put_stage_timer(); + let lock_held_rename_result_to_tail_handoff_started = rustfs_io_metrics::put_stage_timer(); if let Some(scope) = commit_scanner_publication_scope.as_ref() { if rename_result.is_ok() { let _ = scope.mark_committed(); @@ -4775,7 +5438,15 @@ impl SetDisks { let mut rename_guard_release = None; let mut needs_immediate_heal = false; let mut tail_owns_tmp_cleanup = false; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME_RESULT_TO_TAIL_HANDOFF, + lock_held_rename_result_to_tail_handoff_started, + ); + let lock_held_tail_handoff_started = rustfs_io_metrics::put_stage_timer(); if let Ok(rename_commit) = rename_result.as_mut() { + commit_set.observe_tier_free_version_rename_data_source_capability( + rename_commit.tail_drain.is_none() && rename_commit.old_current_source_all_successful_checked, + ); commit_set.record_capacity_scope_if_needed(commit_capacity_scope_token, &rename_commit.capacity_disks); // Install the tail watcher before any post-commit await. The // latch keeps namespace guards through their prior handoff point. @@ -4810,6 +5481,7 @@ impl SetDisks { let fence_object = commit_object.clone(); let (guard_release_tx, guard_release_rx) = tokio::sync::oneshot::channel(); rename_guard_release = Some(guard_release_tx); + let cleanup_reservation = commit_cleanup_reservation.clone(); tokio::spawn(finish_rename_tail_heal( rename_tail_drain, guard_release_rx, @@ -4842,36 +5514,70 @@ impl SetDisks { decommission_object_lock_guard, decommission_capacity_guard, ), - targets| async move { + targets| { drop(object_lock_guard); drop(write_context); drop(publication_guard); drop(bucket_lifecycle_guard); - cleanup_set - .cleanup_rename_tail( - targets, - &cleanup_bucket, - &cleanup_object, - committed_data_dir, - transaction_epoch, - ) - .await; - if let Err(err) = cleanup_set.delete_all(RUSTFS_META_TMP_BUCKET, &cleanup_tmp_dir).await { - warn!(tmp_dir = %cleanup_tmp_dir, error = ?err, "failed to cleanup put_object temporary data"); - } else if issue3031_diag_enabled() { - warn!( - target: "rustfs_ecstore::set_disk", - tmp_dir = %cleanup_tmp_dir, - "issue3031_put_object_tmp_cleanup_done" + cleanup_runtime::run(cleanup_reservation, async move { + let zero_target_cleanup = targets.is_empty(); + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COUNT, + targets.len() as f64, ); - } - drop(decommission_object_lock_guard); - drop(decommission_capacity_guard); + if zero_target_cleanup { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_ZERO_TARGET, + 1.0, + ); + } + cleanup_set + .cleanup_rename_tail( + targets, + &cleanup_bucket, + &cleanup_object, + committed_data_dir, + transaction_epoch, + ) + .await; + let tmp_delete_started = rustfs_io_metrics::put_stage_timer(); + if zero_target_cleanup && rename_tail_cleanup_zero_target_tmp_delete_skip_enabled() { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE_COUNTERFACTUAL_SKIP, + 1.0, + ); + } else if let Err(err) = + cleanup_set.delete_all(RUSTFS_META_TMP_BUCKET, &cleanup_tmp_dir).await + { + warn!(tmp_dir = %cleanup_tmp_dir, error = ?err, "failed to cleanup put_object temporary data"); + } else if issue3031_diag_enabled() { + warn!( + target: "rustfs_ecstore::set_disk", + tmp_dir = %cleanup_tmp_dir, + "issue3031_put_object_tmp_cleanup_done" + ); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE, + tmp_delete_started, + ); + let guard_drop_started = rustfs_io_metrics::put_stage_timer(); + drop(decommission_object_lock_guard); + drop(decommission_capacity_guard); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DECOMMISSION_GUARD_DROP, + guard_drop_started, + ); + }) }, |request| async move { heal_set.submit_rename_tail_heal(request).await }, )); } } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_TAIL_HANDOFF, + lock_held_tail_handoff_started, + ); if !tail_owns_tmp_cleanup { drop(_decommission_capacity_guard.take()); } @@ -4880,6 +5586,11 @@ impl SetDisks { pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameHandoff).await; } if quota_mutation_fence && !tail_owns_tmp_cleanup { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_TRIGGERED, + 1.0, + ); + let quota_fence_release_started = rustfs_io_metrics::put_stage_timer(); let _ = SetDisks::release_quota_mutation_fences( &commit_disks, "a_fence_tokens, @@ -4888,9 +5599,23 @@ impl SetDisks { write_quorum, ) .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE, + quota_fence_release_started, + ); + } else { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_SKIPPED, + 1.0, + ); } if rename_result.is_ok() { + let quota_commit_started = rustfs_io_metrics::put_stage_timer(); quota_reservation.commit().await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_COMMIT, + quota_commit_started, + ); } let rename_commit = match rename_result { Ok(commit) => commit, @@ -4900,7 +5625,10 @@ impl SetDisks { // rollback backup for recovery; cleanup is unsafe. return Err(err.into()); } - if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await { + if let Err(cleanup_err) = commit_set + .delete_put_tmp_with_budget(&commit_tmp_dir, &commit_cleanup_reservation) + .await + { warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { warn!( @@ -4918,7 +5646,17 @@ impl SetDisks { let op_old_dir = rename_commit.data_dir; let cleanup_disks = rename_commit.cleanup_disks; let old_current_size = rename_commit.old_current_size; + if put_tier_free_version_source.is_none() + && reuse_rename_data_source_for_tier_free_version + && rename_commit.old_current_source_checked + { + put_tier_free_version_source = rename_commit + .old_current_source + .as_ref() + .map(|source| build_get_object_info(source, &commit_bucket, &commit_object, commit_versioned)); + } let mut fi = rename_commit.committed_file_info; + let lock_held_post_rename_started = rustfs_io_metrics::put_stage_timer(); if let (Some(source), Some(free_version_id)) = (put_tier_free_version_source.as_ref(), commit_put_tier_free_version_id) && transitioned_delete_publishes_free_version(source, &fi, commit_skip_free_version) @@ -4934,6 +5672,11 @@ impl SetDisks { } if needs_immediate_heal { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_TRIGGERED, + 1.0, + ); + let heal_submit_started = rustfs_io_metrics::put_stage_timer(); let mut request = rustfs_heal_contracts::heal_channel::create_heal_request_with_options( commit_bucket.clone(), Some(commit_object.clone()), @@ -4947,28 +5690,83 @@ impl SetDisks { .map(|version_id| version_id.to_string()); request.expected_bucket_incarnation_id = source_bucket_incarnation_id; commit_set.submit_rename_tail_heal(request).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT, + heal_submit_started, + ); + } else { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_SKIPPED, + 1.0, + ); } let rename_stage_elapsed = rename_stage_start.elapsed(); let rename_stage_ms = rename_stage_elapsed.as_millis() as u64; - if let Some(old_dir) = op_old_dir { - commit_set - .persist_old_data_cleanup_receipts( - &cleanup_disks, - &commit_bucket, - &commit_object, - old_dir, - committed_data_dir, - transaction_epoch, - ) + if let Some(old_dir) = op_old_dir + && Self::needs_old_data_cleanup_receipt(old_dir, committed_data_dir, transaction_epoch) + { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_TRIGGERED, + 1.0, + ); + let cleanup_receipt_started = rustfs_io_metrics::put_stage_timer(); + if commit_cleanup_reservation.is_none() { + commit_set + .persist_old_data_cleanup_receipts( + &cleanup_disks, + &commit_bucket, + &commit_object, + old_dir, + committed_data_dir, + transaction_epoch, + ) + .await; + } else { + let cleanup_set = commit_set.clone(); + let receipt_disks = cleanup_disks.clone(); + let receipt_bucket = commit_bucket.clone(); + let receipt_object = commit_object.clone(); + cleanup_runtime::run(commit_cleanup_reservation.clone(), async move { + cleanup_set + .persist_old_data_cleanup_receipts( + &receipt_disks, + &receipt_bucket, + &receipt_object, + old_dir, + committed_data_dir, + transaction_epoch, + ) + .await; + }) .await; + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT, + cleanup_receipt_started, + ); + } else { + rustfs_io_metrics::record_put_object_stage_duration( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_SKIPPED, + 1.0, + ); } + let metadata_invalidate_started = rustfs_io_metrics::put_stage_timer(); commit_set .invalidate_get_object_metadata_cache(&commit_bucket, &commit_object) .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_METADATA_INVALIDATE, + metadata_invalidate_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_RENAME, + lock_held_post_rename_started, + ); + let lock_held_guard_release_started = rustfs_io_metrics::put_stage_timer(); if let Some(release) = rename_guard_release.take() { let _ = release.send(true); } @@ -4977,6 +5775,10 @@ impl SetDisks { drop(_object_lock_guard.take()); drop(_publication_guard.take()); drop(_bucket_lifecycle_guard.take()); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_GUARD_RELEASE, + lock_held_guard_release_started, + ); rustfs_io_metrics::record_put_object_stage_duration("set_disk_rename", duration_millis_f64(rename_stage_elapsed)); if (rename_stage_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS { @@ -5004,16 +5806,41 @@ impl SetDisks { // here must not negate an already-committed, durable write, so we // deliberately do NOT `?`-propagate it into a 503. On residue the // report path emits the leak metric and enqueues a heal. - let cleanup = commit_set - .commit_rename_data_dir_and_mark_capacity( - &cleanup_disks, - &commit_bucket, - &commit_object, - &old_dir.to_string(), - &committed_dir, - write_quorum, - ) + if commit_cleanup_reservation.is_none() { + let cleanup = commit_set + .commit_rename_data_dir_and_mark_capacity( + &cleanup_disks, + &commit_bucket, + &commit_object, + &old_dir.to_string(), + &committed_dir, + write_quorum, + ) + .await; + commit_set + .report_old_data_dir_cleanup(&commit_bucket, &commit_object, &old_dir.to_string(), &cleanup) + .await; + } else { + let cleanup_set = commit_set.clone(); + let cleanup_bucket = commit_bucket.clone(); + let cleanup_object = commit_object.clone(); + cleanup_runtime::run(commit_cleanup_reservation.clone(), async move { + let cleanup = cleanup_set + .commit_rename_data_dir_and_mark_capacity( + &cleanup_disks, + &cleanup_bucket, + &cleanup_object, + &old_dir.to_string(), + &committed_dir, + write_quorum, + ) + .await; + cleanup_set + .report_old_data_dir_cleanup(&cleanup_bucket, &cleanup_object, &old_dir.to_string(), &cleanup) + .await; + }) .await; + } let cleanup_elapsed = cleanup_stage_start.elapsed(); let cleanup_ms = cleanup_elapsed.as_millis() as u64; cleanup_stage_ms = Some(cleanup_ms); @@ -5021,9 +5848,6 @@ impl SetDisks { "set_disk_old_data_cleanup", duration_millis_f64(cleanup_elapsed), ); - commit_set - .report_old_data_dir_cleanup(&commit_bucket, &commit_object, &old_dir.to_string(), &cleanup) - .await; if (cleanup_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS { warn!( event = EVENT_SET_DISK_COMMIT_TAIL_SLOW, @@ -5103,7 +5927,7 @@ impl SetDisks { if !tail_owns_tmp_cleanup { let cleanup_set = commit_set.clone(); let cleanup_tmp_dir = commit_tmp_dir.clone(); - tokio::spawn(async move { + cleanup_runtime::spawn(commit_cleanup_reservation.clone(), async move { if let Err(err) = cleanup_set.delete_all(RUSTFS_META_TMP_BUCKET, &cleanup_tmp_dir).await { warn!(tmp_dir = %cleanup_tmp_dir, error = ?err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { @@ -5116,15 +5940,35 @@ impl SetDisks { }); } - Ok(( - ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned), - old_current_size, - )) + let object_info_build_started = rustfs_io_metrics::put_stage_timer(); + let object_info = ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_OBJECT_INFO_BUILD, + object_info_build_started, + ); + let post_object_info_to_return_started = rustfs_io_metrics::put_stage_timer(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_OBJECT_INFO_TO_RETURN, + post_object_info_to_return_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN, + commit_after_rename_to_return_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL, + commit_closure_total_started, + ); + Ok((object_info, old_current_size)) }; if let Some(handoff) = commit_cancellation_handoff_tx.take() { let _ = handoff.send(()); } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT, + operation_body_to_commit_submit_started, + ); if detach_commit_owner { if let Some(scope_outcome_guard) = scope_outcome_guard.as_mut() { // The spawned commit closure owns the scope clone and is @@ -5133,15 +5977,40 @@ impl SetDisks { } let mut cancellation = PutObjectCommitCancellation::new(); let child_token = cancellation.child_token(); - let result = tokio::spawn(async move { Box::pin(commit(Some(child_token))).await }) + let commit_task_wait_started = rustfs_io_metrics::put_stage_timer(); + let commit_submit_to_closure_enter_started = rustfs_io_metrics::put_stage_timer(); + let result = + tokio::spawn( + async move { Box::pin(commit(commit_submit_to_closure_enter_started, Some(child_token))).await }, + ) .await .map_err(|err| Error::other(format!("put_object commit task failed: {err}")))?; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT, + commit_task_wait_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL, + commit_task_wait_started, + ); cancellation.disarm(); result } else { - Box::pin(commit(None)).await + let commit_task_wait_started = rustfs_io_metrics::put_stage_timer(); + let commit_submit_to_closure_enter_started = rustfs_io_metrics::put_stage_timer(); + let result = Box::pin(commit(commit_submit_to_closure_enter_started, None)).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT, + commit_task_wait_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL, + commit_task_wait_started, + ); + result } }; + let operation_select_wait_started = rustfs_io_metrics::put_stage_timer(); let result: Result<(ObjectInfo, Option)> = if let Some((cancellation, mut handoff_rx)) = cancellation_wait { tokio::pin!(operation); @@ -5154,6 +6023,19 @@ impl SetDisks { } else { operation.await }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT, + operation_select_wait_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL, + operation_select_wait_started, + ); + let operation_result_return_started = rustfs_io_metrics::put_stage_timer(); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN, + operation_result_return_started, + ); if issue3031_diag_enabled() && let Err(err) = &result @@ -5201,7 +6083,7 @@ impl SetDisks { // entry is reclaimed by cleanup_stale_tmp_objects (24h expiry, // 5-minute background loop). let set_disks = self.clone(); - tokio::spawn(async move { + cleanup_runtime::spawn(cleanup_reservation.clone(), async move { if let Err(err) = set_disks.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await { warn!(tmp_dir = %tmp_dir, error = ?err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { @@ -5216,7 +6098,7 @@ impl SetDisks { // Failure path (quorum loss / rollback): keep the cleanup inline so // a failed PUT never returns while its tmp shards are still on disk // (state-residue hardening tracked by backlog#864 / backlog#898). - if let Err(err) = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await { + if let Err(err) = self.delete_put_tmp_with_budget(&tmp_dir, &cleanup_reservation).await { warn!(tmp_dir = %tmp_dir, error = ?err, "failed to cleanup put_object temporary data"); } else if issue3031_diag_enabled() { warn!( @@ -19412,6 +20294,73 @@ mod put_object_tmp_cleanup_tests { ); } + #[tokio::test] + async fn put_cleanup_admission_rejects_before_staging_with_an_external_lock() { + const CHILD: &str = "RUSTFS_CLEANUP_ADMISSION_TEST_CHILD"; + if std::env::var_os(CHILD).is_none() { + let output = tokio::task::spawn_blocking(|| { + std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "set_disk::ops::object::put_object_tmp_cleanup_tests::put_cleanup_admission_rejects_before_staging_with_an_external_lock", + "--nocapture", + ]) + .env(CHILD, "1") + .env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true") + .env("RUSTFS_PUT_RENAME_TAIL_CLEANUP_MAX_PENDING", "1") + .env("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS", "1") + .env("RUSTFS_CLEANUP_CPUS", "none") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP") + .output() + .unwrap() + }) + .await + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + return; + } + let (dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "cleanup-admission-bucket"; + let object = "externally-locked"; + for disk in &disks { + disk.make_volume(bucket).await.unwrap(); + } + let occupied = cleanup_runtime::reserve().unwrap().expect("isolation enabled in child"); + let external_lock = set.acquire_write_lock_diag("admission_test", bucket, object).await.unwrap(); + let mut reader = PutObjReader::from_vec(vec![7u8; TEST_OBJECT_SIZE]); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + let result = tokio::time::timeout(Duration::from_secs(20), set.put_object(bucket, object, &mut reader, &opts)) + .await + .expect("admission must not wait while the caller owns locks"); + assert!(matches!(result, Err(StorageError::SlowDown))); + assert!(non_trash_tmp_entries(&dirs).await.is_empty(), "rejection must not stage shards"); + drop(external_lock); + drop(occupied); + let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]); + set.put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .unwrap(); + wait_for_tmp_workspace_to_drain(&dirs, "admitted PUT must eventually clean staging").await; + let mut actual = set + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .unwrap(); + let mut body = Vec::new(); + actual.stream.read_to_end(&mut body).await.unwrap(); + assert_eq!(body, vec![9u8; TEST_OBJECT_SIZE]); + } + #[tokio::test] async fn put_object_success_eventually_cleans_tmp_workspace() { let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; @@ -24239,6 +25188,22 @@ mod single_delete_namespace_owner_tests { } } +#[cfg(test)] +mod cleanup_receipt_admission_tests { + use super::*; + + #[test] + fn receipt_admission_skips_only_work_that_the_writer_would_skip() { + let old = Uuid::new_v4(); + let current = Uuid::new_v4(); + let epoch = Uuid::new_v4(); + assert!(!SetDisks::needs_old_data_cleanup_receipt(old, Some(current), None)); + assert!(!SetDisks::needs_old_data_cleanup_receipt(old, Some(old), Some(epoch))); + assert!(SetDisks::needs_old_data_cleanup_receipt(old, Some(current), Some(epoch))); + assert!(SetDisks::needs_old_data_cleanup_receipt(old, None, Some(epoch))); + } +} + #[cfg(test)] mod crash_delete_retry_tests { use super::*; diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 172e40140..b2d116309 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -15,9 +15,9 @@ use super::{ Arc, Bytes, DiskError, DiskOption, DiskStore, Endpoint, ErasureCache, Error, FileInfo, GetCodecStreamingFallbackReason, GetObjectFileInfo, GetObjectMetadataCacheCapability, GetObjectMetadataCacheEntry, GetObjectMetadataCacheGeneration, - GetObjectMetadataCacheKey, GetObjectReadPolicy, HashAlgorithm, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, - OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectOptions, RUSTFS_META_BUCKET, ReadOptions, Result, SetDisks, StorageError, - adaptive_duplex_buffer_size, build_get_codec_streaming_decode_engine, build_inline_bitrot_readers_from_refs, + GetObjectMetadataCacheKey, GetObjectMetadataCacheLookupKey, GetObjectReadPolicy, HashAlgorithm, LOG_COMPONENT_ECSTORE, + LOG_SUBSYSTEM_SET_DISK, OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectOptions, RUSTFS_META_BUCKET, ReadOptions, Result, SetDisks, + StorageError, adaptive_duplex_buffer_size, build_get_codec_streaming_decode_engine, build_inline_bitrot_readers_from_refs, collect_inline_data_shard_fileinfos_by_index, debug, error, get_codec_streaming_metrics_path, get_codec_streaming_multipart_max_parts, get_object_read_policy, is_codec_streaming_multipart_enabled, is_multipart_reader_setup_prefetch_enabled, object_fits_single_block, reduce_read_quorum_errs, to_object_err, @@ -208,6 +208,16 @@ use tokio::time::timeout; #[cfg(test)] use uuid::Uuid; +fn metadata_quorum_outcome(error: &StorageError) -> &'static str { + if error.is_not_found() { + "not_found" + } else if error.is_quorum_error() { + "not_reached" + } else { + "unavailable" + } +} + pub(super) struct GetObjectDownstreamWriter { inner: W, } @@ -400,7 +410,7 @@ impl SetDisks { let Some(generation) = self.get_object_metadata_cache_generation(bucket, object) else { return MetadataCacheLookup::Miss; }; - let key = GetObjectMetadataCacheKey::new(bucket, object, generation); + let key = GetObjectMetadataCacheLookupKey::new(bucket, object, generation); // moka handles TTL expiry automatically; no is_fresh() check needed let Some(entry) = self.get_object_metadata_cache.get(&key).await else { return MetadataCacheLookup::Miss; @@ -462,14 +472,14 @@ impl SetDisks { entry: Arc, after_insert: impl FnOnce(), ) { - // Every publisher shares Moka's per-key compute lock. A conditional - // snapshot cannot downgrade a full layout or extend its original TTL. + // Every publisher shares Moka's per-key compute lock. A partial snapshot + // cannot downgrade a full layout or extend its original TTL. self.get_object_metadata_cache .entry(key.clone()) .and_compute_with(|current| { std::future::ready( if !self.is_get_object_metadata_cache_generation_current(generation) - || (entry.capability == GetObjectMetadataCacheCapability::ConditionOnly + || (entry.capability != GetObjectMetadataCacheCapability::Full && current .as_ref() .is_some_and(|existing| existing.value().capability == GetObjectMetadataCacheCapability::Full)) @@ -602,6 +612,11 @@ impl SetDisks { purpose: MetadataReadPurpose, ) -> Result { let vid = opts.version_id.clone().unwrap_or_default(); + let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) { + GET_OBJECT_PATH_INTERNAL_META + } else { + GET_OBJECT_PATH_LEGACY_DUPLEX + }; let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); let metadata_cache_lookup_start = get_stage_timer_if_enabled(stage_metrics_enabled); @@ -617,12 +632,17 @@ impl SetDisks { match self.lookup_cached_get_object_fileinfo(bucket, object).await { MetadataCacheLookup::Hit(cached) if cached.capability == GetObjectMetadataCacheCapability::Full + || (cached.capability == GetObjectMetadataCacheCapability::QuorumSelected + && read_data + && allow_early_stop) || (purpose == MetadataReadPurpose::GetObjectReader + && cached.capability == GetObjectMetadataCacheCapability::ConditionOnly && allow_early_stop && crate::object_api::get_object_read_condition_is_terminal( crate::object_api::GetObjectReadMetadata::from_file_info(bucket, object, &cached.fi), )) => { + rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "reached"); rustfs_io_metrics::record_get_object_metadata_cache_decision( GET_OBJECT_PATH_SET_DISK, GET_METADATA_CACHE_DECISION_HIT, @@ -633,7 +653,15 @@ impl SetDisks { GET_STAGE_METADATA_CACHE_LOOKUP, metadata_cache_lookup_start, ); - return Ok(GetObjectFileInfo::shared(cached)); + let snapshot = if cached.capability == GetObjectMetadataCacheCapability::QuorumSelected + && non_inline_data_read_early_stop_allowed(read_data, bucket, object) + && late_materialization_candidate_is_safe(&cached.fi) + { + GetObjectFileInfo::shared_with_late_metadata_refresh(cached) + } else { + GetObjectFileInfo::shared(cached) + }; + return Ok(snapshot); } MetadataCacheLookup::Hit(_) => { // Keep the conditional entry for other requests; this read @@ -683,7 +711,7 @@ impl SetDisks { let allow_read_version_coalescing = purpose == MetadataReadPurpose::GetObjectReader && !crate::bucket::utils::is_meta_bucketname(bucket) && crate::runtime::global::get_metadata_read_version_coalescing_service_ready(); - let metadata_read = if allow_read_version_coalescing { + let metadata_read_result = if allow_read_version_coalescing { Self::read_metadata_for_get_object( &disks, "", @@ -695,7 +723,7 @@ impl SetDisks { allow_early_stop, self.default_parity_count, ) - .await? + .await } else { Self::read_metadata_observed( &disks, @@ -709,16 +737,19 @@ impl SetDisks { allow_early_stop, self.default_parity_count, ) - .await? + .await + }; + let metadata_read = match metadata_read_result { + Ok(metadata_read) => metadata_read, + Err(err) => { + let err = StorageError::from(err); + rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, metadata_quorum_outcome(&err)); + return Err(err); + } }; let metadata_fanout_complete = metadata_read.is_complete(); let condition_stopped = metadata_read.condition_stopped; let (mut parts_metadata, errs, metadata_fanout_diagnostics) = metadata_read.into_legacy(); - let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) { - GET_OBJECT_PATH_INTERNAL_META - } else { - GET_OBJECT_PATH_LEGACY_DUPLEX - }; metadata_fanout_diagnostics.record(metadata_metrics_path); // warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata); // warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs); @@ -731,6 +762,7 @@ impl SetDisks { { Ok(v) => v, Err(e) => { + rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, metadata_quorum_outcome(&e)); // error!("Self::object_quorum_from_meta: {:?}, bucket: {}, object: {}", &e, bucket, object); record_get_stage_duration_if_enabled( GET_OBJECT_PATH_SET_DISK, @@ -745,6 +777,7 @@ impl SetDisks { let write_quorum = usize::try_from(write_quorum) .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?; if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { + rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "not_reached"); error!("reduce_read_quorum_errs: {:?}, bucket: {}, object: {}", &err, bucket, object); record_get_stage_duration_if_enabled( GET_OBJECT_PATH_SET_DISK, @@ -755,7 +788,17 @@ impl SetDisks { } let (op_online_disks, mut fi, fileinfo_selection_quorum) = - Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum)?; + match Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum) { + Ok(selection) => selection, + Err(err) => { + let err = StorageError::from(err); + rustfs_io_metrics::record_get_object_metadata_quorum_result( + metadata_metrics_path, + metadata_quorum_outcome(&err), + ); + return Err(err); + } + }; let include_part_checksums = opts.include_part_checksums || opts.part_number.is_some() || opts.data_movement || opts.raw_data_movement_read; if include_part_checksums { @@ -778,7 +821,16 @@ impl SetDisks { "metadata_read_error", ) .await; - } else if use_metadata_cache && (metadata_fanout_complete || condition_stopped) { + } else if use_metadata_cache + && (metadata_fanout_complete + || condition_stopped + || (read_data + && allow_early_stop + && metadata_fanout_diagnostics.valid_responses() >= read_quorum + && errs.iter().all(Option::is_none) + && fileinfo_selection_quorum >= read_quorum + && op_online_disks.iter().filter(|disk| disk.is_some()).count() >= read_quorum)) + { #[cfg(test)] metadata_cache_tests::wait_before_metadata_cache_publish(bucket, object).await; self.cache_get_object_fileinfo( @@ -788,8 +840,10 @@ impl SetDisks { &fi, if metadata_fanout_complete { GetObjectMetadataCacheCapability::Full - } else { + } else if condition_stopped { GetObjectMetadataCacheCapability::ConditionOnly + } else { + GetObjectMetadataCacheCapability::QuorumSelected }, ), &parts_metadata, @@ -2650,6 +2704,7 @@ fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOpti #[cfg(test)] mod metadata_cache_tests { use super::*; + use crate::set_disk::core::io_primitives::disk_call_counters; use rustfs_heal_contracts::heal_channel::HealAdmissionDropReason; use serial_test::serial; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -4009,7 +4064,7 @@ mod metadata_cache_tests { } #[tokio::test] - #[serial(metadata_cache_publish_barrier)] + #[serial(metadata_cache_early_stop)] async fn metadata_cache_production_fanout_cannot_publish_after_invalidation() { // Isolated context: an ambient DistErasure window (another test's // SetupTypeGuard) would bypass metadata-cache publication entirely @@ -4063,6 +4118,191 @@ mod metadata_cache_tests { ); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial(metadata_cache_early_stop)] + async fn metadata_cache_publishes_quorum_selected_early_stop_result() { + use tokio::io::AsyncReadExt; + + let isolated_ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + isolated_ctx + .update_erasure_type(crate::layout::endpoints::SetupType::Erasure) + .await; + let (dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, isolated_ctx).await; + let bucket = "metadata-cache-early-stop"; + let object = "objects/repeated-key"; + let disks = set.disks.read().await.clone(); + for disk in disks.iter().flatten() { + disk.make_volume(bucket) + .await + .expect("test bucket should be created on each disk"); + } + let expected_body = vec![0x5a; 1024 * 1024]; + set.put_object( + bucket, + object, + &mut PutObjReader::from_vec(expected_body.clone()), + &ObjectOptions::default(), + ) + .await + .expect("test object should be written before enabling the slow-tail fault"); + + let calls = disk_call_counters::observe(object); + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("false")), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS", Some("500")), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS", Some("3")), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET", Some(bucket)), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX", Some("objects/")), + ], + async { + let first = tokio::time::timeout( + Duration::from_millis(400), + set.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true), + ) + .await + .expect("first metadata read should stop before the delayed peer") + .expect("read quorum should provide valid object metadata"); + assert!(first.owned.is_some(), "the first lookup should resolve through disk metadata"); + assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 4); + assert_eq!(calls.total(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT), 1); + + let second = set + .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true) + .await + .expect("second lookup should use the quorum-published metadata cache entry"); + assert!(second.shared_entry().is_some(), "the repeated GET should hit the published cache entry"); + assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 4); + + let cached = second.shared_entry().expect("second lookup should share cached metadata"); + assert_eq!(cached.fi.name, object); + let cached_online_count = cached.online_disks.iter().filter(|disk| disk.is_some()).count(); + assert_eq!(cached_online_count, 3, "the warm cache should contain three of four disks"); + assert_eq!(cached.capability, GetObjectMetadataCacheCapability::QuorumSelected); + + let data_dir = cached + .fi + .data_dir + .expect("quorum cache retains the object data directory") + .to_string(); + let selected_disk_indices = cached + .online_disks + .iter() + .enumerate() + .filter_map(|(index, disk)| disk.as_ref().map(|_| index)) + .take(2) + .collect::>(); + assert_eq!(selected_disk_indices.len(), 2, "the warmed cache should name enough selected shards"); + for index in selected_disk_indices { + tokio::fs::remove_file(dirs[index].path().join(bucket).join(object).join(&data_dir).join("part.1")) + .await + .expect("simulate loss of two cached selected shards"); + } + + let mut reader = set + .get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("a cached partial layout must recover through omitted disks"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("the GET should decode after refreshing the omitted disks"); + assert_eq!(body, expected_body, "recovered body must be byte-exact"); + }, + ) + .await; + + let full_layout = set + .get_object_fileinfo_gated(bucket, object, &ObjectOptions::default(), false, false) + .await + .expect("read-before-write consumers must fetch a complete layout"); + assert!( + full_layout.shared_entry().is_none(), + "a quorum-selected partial layout cannot satisfy mutation consumers" + ); + assert!( + calls.total(disk_call_counters::KIND_READ_VERSION) > 4, + "mutation consumers must independently fan out after a quorum-only cache hit" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial(metadata_cache_early_stop)] + async fn metadata_cache_early_stop_publication_is_fenced_by_invalidation() { + let isolated_ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + isolated_ctx + .update_erasure_type(crate::layout::endpoints::SetupType::Erasure) + .await; + let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, isolated_ctx).await; + let bucket = "metadata-cache-early-stop-fence"; + let object = "objects/repeated-key"; + let disks = set.disks.read().await.clone(); + for disk in disks.iter().flatten() { + disk.make_volume(bucket) + .await + .expect("test bucket should be created on each disk"); + } + set.put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![0x5a; 1024 * 1024]), + &ObjectOptions::default(), + ) + .await + .expect("test object should be written before enabling the slow-tail fault"); + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("false")), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS", Some("500")), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS", Some("3")), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET", Some(bucket)), + ("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX", Some("objects/")), + ], + async { + let barrier = MetadataCachePublishBarrier::install(bucket, object); + let stale_generation = set + .get_object_metadata_cache_generation(bucket, object) + .expect("metadata cache generation should be active"); + let reader_set = Arc::clone(&set); + let read = tokio::spawn(async move { + reader_set + .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true) + .await + }); + barrier.wait_until_paused().await; + set.invalidate_get_object_metadata_cache(bucket, object).await; + barrier.release(); + + let snapshot = read + .await + .expect("early-stop metadata read should not panic") + .expect("read quorum should resolve the selected FileInfo"); + assert!(snapshot.owned.is_some()); + assert!( + set.get_object_metadata_cache + .get(&GetObjectMetadataCacheKey::new(bucket, object, stale_generation)) + .await + .is_none(), + "early-stop publication must not retain an entry under the retired generation" + ); + assert!( + set.cached_get_object_fileinfo(bucket, object).await.is_none(), + "invalidation must reject the quorum-selected early-stop publication" + ); + }, + ) + .await; + } + #[tokio::test] #[serial(metadata_cache_publish_barrier, body_cache_hook)] async fn conditional_inline_hedge_caches_only_terminal_metadata() { @@ -5082,12 +5322,15 @@ mod metadata_cache_tests { .get_object_metadata_cache_generation("bucket-a", "object-a") .expect("metadata cache generation should be active"); let first_key = GetObjectMetadataCacheKey::new("bucket-a", "object-a", generation); + let first_lookup = GetObjectMetadataCacheLookupKey::new("bucket-a", "object-a", generation); let second_key = GetObjectMetadataCacheKey { bucket: Arc::from("bucket-b"), object: Arc::from("object-b"), generation: generation.value, hash: generation.hash, }; + let second_lookup = GetObjectMetadataCacheLookupKey::new("bucket-b", "object-b", generation); + let unrelated_lookup = GetObjectMetadataCacheLookupKey::new("bucket-c", "object-c", generation); let first_fi = valid_test_fileinfo("object-a"); let second_fi = valid_test_fileinfo("object-b"); let entry = |fi: FileInfo| { @@ -5108,7 +5351,7 @@ mod metadata_cache_tests { assert_eq!( set.get_object_metadata_cache - .get(&first_key) + .get(&first_lookup) .await .expect("first colliding entry should remain addressable") .fi @@ -5117,13 +5360,14 @@ mod metadata_cache_tests { ); assert_eq!( set.get_object_metadata_cache - .get(&second_key) + .get(&second_lookup) .await .expect("second colliding entry should remain addressable") .fi .name, "object-b" ); + assert!(set.get_object_metadata_cache.get(&unrelated_lookup).await.is_none()); } #[tokio::test] @@ -5290,6 +5534,16 @@ mod tests { const CODEC_STREAMING_TEST_OBJECT: &str = "object"; static CAPTURED_READ_REPAIR_CALLS: AtomicUsize = AtomicUsize::new(0); + #[test] + fn metadata_quorum_outcome_uses_bounded_error_categories() { + assert_eq!(metadata_quorum_outcome(&StorageError::ErasureReadQuorum), "not_reached"); + assert_eq!(metadata_quorum_outcome(&StorageError::FileNotFound), "not_found"); + assert_eq!( + metadata_quorum_outcome(&StorageError::Io(std::io::Error::other("disk unavailable"))), + "unavailable" + ); + } + fn capture_read_repair_submitter( _request: rustfs_heal_contracts::heal_channel::HealChannelRequest, ) -> ReadRepairAdmissionFuture { diff --git a/crates/ecstore/src/store/peer.rs b/crates/ecstore/src/store/peer.rs index 655492c66..cc84c9cf7 100644 --- a/crates/ecstore/src/store/peer.rs +++ b/crates/ecstore/src/store/peer.rs @@ -18,6 +18,11 @@ use crate::disk::error::{DiskError, Result as DiskResult}; use crate::disk::{DeleteOptions, Disk, RenameDataGuards, RenameDataResp}; use crate::runtime::instance::{InstanceContext, NamespaceCommitGuard}; use crate::runtime::sources as runtime_sources; +use rustfs_io_metrics::{ + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME, + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD, + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL, +}; use tracing::{debug, error}; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; @@ -331,15 +336,38 @@ async fn rename_local_data_with_ctx( destination: (&str, &str), mut guards: RenameDataGuards, ) -> DiskResult { + let handler_started = rustfs_io_metrics::put_stage_timer(); + let lookup_started = rustfs_io_metrics::put_stage_timer(); let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?; + let lookup_ms = lookup_started.map(|started| started.elapsed().as_secs_f64() * 1000.0); + if let Some(duration_ms) = lookup_ms { + rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, duration_ms); + } let mutates_namespace = !is_meta_bucketname(source.0) || !is_meta_bucketname(destination.0); + let admit_started = rustfs_io_metrics::put_stage_timer(); let owner = admit_local_disk(ctx, &disk, disk_id, mutates_namespace).await?; + let admit_ms = admit_started.map(|started| started.elapsed().as_secs_f64() * 1000.0); + if let Some(duration_ms) = admit_ms { + rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, duration_ms); + } guards.namespace_owner = owner.as_ref().map(|owner| owner.clone() as Arc); + let local_rename_started = rustfs_io_metrics::put_stage_timer(); let result = disk .rename_data_borrowed_with_fence_observed(source.0, source.1, fi, destination.0, destination.1, guards) .await .result; + let local_rename_ms = local_rename_started.map(|started| started.elapsed().as_secs_f64() * 1000.0); + if let Some(duration_ms) = local_rename_ms { + rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME, duration_ms); + } drop(owner); + if let Some(handler_started) = handler_started { + let handler_ms = handler_started.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL, handler_ms); + let child_ms = lookup_ms.unwrap_or(0.0) + admit_ms.unwrap_or(0.0) + local_rename_ms.unwrap_or(0.0); + let overhead_ms = (handler_ms - child_ms).max(0.0); + rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD, overhead_ms); + } result } diff --git a/crates/ecstore/tests/cleanup_isolation_test.rs b/crates/ecstore/tests/cleanup_isolation_test.rs new file mode 100644 index 000000000..663681445 --- /dev/null +++ b/crates/ecstore/tests/cleanup_isolation_test.rs @@ -0,0 +1,174 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(target_os = "linux")] + +use bytes::Bytes; +mod storage_api; + +use std::time::Duration; +use storage_api::cleanup_isolation::{DeleteOptions, DiskAPI, DiskError, DiskOption, Endpoint, new_disk}; + +#[tokio::test] +async fn cleanup_disk_and_periodic_gc_use_affinity_without_changing_foreground() { + const CHILD: &str = "RUSTFS_CLEANUP_LINUX_TEST_CHILD"; + let foreground = rustix::thread::sched_getaffinity(None).unwrap(); + let cpu = (0..rustix::thread::CpuSet::MAX_CPU) + .find(|cpu| foreground.is_set(*cpu)) + .expect("the test process must have an allowed CPU"); + if std::env::var_os(CHILD).is_none() { + for mode in ["custom", "default", "none", "restricted"] { + if mode == "default" && (!foreground.is_set(1) || !foreground.is_set(2)) { + continue; + } + let output = tokio::task::spawn_blocking(move || { + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + "cleanup_disk_and_periodic_gc_use_affinity_without_changing_foreground", + "--nocapture", + ]) + .env(CHILD, mode) + .env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true") + .env("RUSTFS_PUT_RENAME_TAIL_CLEANUP_MAX_PENDING", "4") + .env("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS", "1") + .env("RUSTFS_CLEANUP_DISK_MAX_PENDING", "4") + .env("RUSTFS_CLEANUP_DISK_WORKERS", "1") + .env("RUSTFS_CLEANUP_GC_WORKERS", "1") + .env("RUSTFS_CLEANUP_BLOCKING_THREADS", "1") + .env("RUSTFS_CLEANUP_ASYNC_THREADS", "1") + .env_remove("RUSTFS_CLEANUP_CPUS") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER") + .env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP"); + match mode { + "custom" => { + command.env("RUSTFS_CLEANUP_CPUS", cpu.to_string()); + } + "none" => { + command.env("RUSTFS_CLEANUP_CPUS", "none"); + } + _ => {} + } + command.output().unwrap() + }) + .await + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + } + return; + } + + let mode = std::env::var(CHILD).unwrap(); + if mode == "restricted" { + let mut restricted = rustix::thread::CpuSet::new(); + restricted.set(cpu); + rustix::thread::sched_setaffinity(None, &restricted).expect("restrict initialization to one allowed CPU"); + } + let dir = tempfile::tempdir().unwrap(); + let disk = new_disk(&Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(), &DiskOption::default()) + .await + .unwrap(); + let bucket = "cleanup-affinity"; + disk.make_volume(bucket).await.unwrap(); + disk.write_all(bucket, "old/part.1", Bytes::from_static(b"garbage")) + .await + .unwrap(); + let receipt = disk + .write_all(bucket, "old/.rustfs-old-data-cleanup-receipt.json", Bytes::from_static(b"receipt")) + .await; + if mode == "restricted" { + assert!( + receipt + .expect_err("unavailable default CPUs must reject cleanup") + .to_string() + .contains("allowed CPU set") + ); + assert_eq!(disk.read_all(bucket, "old/part.1").await.unwrap(), Bytes::from_static(b"garbage")); + return; + } + receipt.unwrap(); + let expected_mask = match mode.as_str() { + "default" => "1-2".to_owned(), + "none" => std::fs::read_to_string("/proc/thread-self/status") + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("Cpus_allowed_list:")) + .unwrap() + .trim() + .to_owned(), + _ => cpu.to_string(), + }; + disk.write_all(bucket, "live/part.1", Bytes::from_static(b"keep")) + .await + .unwrap(); + disk.delete( + bucket, + "old", + DeleteOptions { + recursive: true, + ..Default::default() + }, + ) + .await + .unwrap(); + assert!(matches!(disk.read_all(bucket, "old/part.1").await, Err(DiskError::FileNotFound))); + + // One async worker and one lazily started blocking worker. Inspect the + // real process threads, including the threads executing filesystem work. + let mut cleanup_threads = 0; + for entry in std::fs::read_dir("/proc/self/task").unwrap() { + let task = entry.unwrap().path(); + let Ok(name) = std::fs::read_to_string(task.join("comm")) else { + continue; + }; + if name.trim() != "rustfs-cleanup" { + continue; + } + cleanup_threads += 1; + let status = std::fs::read_to_string(task.join("status")).unwrap(); + let mask = status + .lines() + .find_map(|line| line.strip_prefix("Cpus_allowed_list:")) + .unwrap() + .trim(); + assert_eq!(mask, expected_mask); + } + assert_eq!(cleanup_threads, 2, "async and blocking workers must both use the configured runtime"); + assert_eq!(rustix::thread::sched_getaffinity(None).unwrap(), foreground); + + let trash = dir.path().join(".rustfs.sys/tmp/.trash"); + assert!(std::fs::read_dir(&trash).unwrap().next().is_some()); + // Advance the foreground timer that dispatches the five-minute periodic + // scan; the dedicated runtime then performs the actual physical removal. + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(301)).await; + tokio::time::resume(); + tokio::time::timeout(Duration::from_secs(20), async { + while std::fs::read_dir(&trash).unwrap().next().is_some() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("periodic GC must physically remove trash"); + assert_eq!(disk.read_all(bucket, "live/part.1").await.unwrap(), Bytes::from_static(b"keep")); + assert_eq!(rustix::thread::sched_getaffinity(None).unwrap(), foreground); +} diff --git a/crates/ecstore/tests/fd_relative_io_test.rs b/crates/ecstore/tests/fd_relative_io_test.rs new file mode 100644 index 000000000..fa19deb52 --- /dev/null +++ b/crates/ecstore/tests/fd_relative_io_test.rs @@ -0,0 +1,283 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(all(target_os = "linux", feature = "test-util"))] + +mod storage_api; + +use bytes::Bytes; +use rustfs_filemeta::{FileInfo, ObjectPartInfo}; +use std::{path::Path, time::Duration}; +use storage_api::fd_relative::{ + DiskAPI, DiskError, DiskOption, DiskStore, Endpoint, LocalPublicationPause, LocalPublicationStage, ReadOptions, new_disk, +}; +use time::OffsetDateTime; +use uuid::Uuid; + +const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp"; + +async fn disk_at(path: &Path) -> DiskStore { + new_disk( + &Endpoint::try_from(path.to_str().expect("UTF-8 test path")).expect("endpoint"), + &DiskOption::default(), + ) + .await + .expect("local disk") +} + +fn metadata(data: Bytes, inline: bool, directory: Uuid) -> FileInfo { + let mut fi = FileInfo::new("object", 1, 0); + fi.erasure.index = 1; + fi.version_id = Some(Uuid::nil()); + fi.data_dir = Some(directory); + fi.size = i64::try_from(data.len()).expect("test data size"); + fi.parts = vec![ObjectPartInfo { + number: 1, + size: data.len(), + actual_size: fi.size, + ..Default::default() + }]; + fi.mod_time = Some(OffsetDateTime::now_utc()); + fi.data = inline.then_some(data); + if inline { + fi.set_inline_data(); + } + fi +} + +async fn put(disk: &DiskStore, staging: &str, data: Bytes, inline: bool) -> Uuid { + let directory = Uuid::new_v4(); + if !inline { + disk.write_all(RUSTFS_META_TMP_BUCKET, &format!("{staging}/{directory}/part.1"), data.clone()) + .await + .expect("stage shard"); + } + disk.rename_data(RUSTFS_META_TMP_BUCKET, staging, metadata(data, inline, directory), "bucket", "object") + .await + .expect("commit object"); + directory +} + +#[tokio::test] +async fn pinned_root_reads_and_writes_original_disk_after_path_replacement() { + let temp = tempfile::tempdir().expect("test directory"); + let root = temp.path().join("disk"); + std::fs::create_dir(&root).expect("disk root"); + let disk = disk_at(&root).await; + disk.make_volume("bucket").await.expect("bucket"); + disk.write_all("bucket", "metadata", Bytes::from_static(b"original")) + .await + .expect("initial write"); + let moved = temp.path().join("original"); + std::fs::rename(&root, &moved).expect("move mount path"); + std::fs::create_dir_all(root.join("bucket")).expect("replacement disk"); + std::fs::write(root.join("bucket/metadata"), b"replacement").expect("replacement sentinel"); + assert_eq!( + disk.read_all("bucket", "metadata").await.expect("pinned read"), + Bytes::from_static(b"original") + ); + disk.write_all("bucket", "metadata", Bytes::from_static(b"updated")) + .await + .expect("pinned write"); + assert_eq!(std::fs::read(moved.join("bucket/metadata")).expect("original disk"), b"updated"); + assert_eq!(std::fs::read(root.join("bucket/metadata")).expect("replacement disk"), b"replacement"); +} + +#[tokio::test] +async fn relative_reads_and_writes_reject_symlink_escape() { + let temp = tempfile::tempdir().expect("test directory"); + let disk = disk_at(temp.path()).await; + disk.make_volume("bucket").await.expect("bucket"); + let outside = tempfile::tempdir().expect("outside directory"); + std::fs::write(outside.path().join("sentinel"), b"preserved").expect("sentinel"); + std::os::unix::fs::symlink(outside.path(), temp.path().join("bucket/link")).expect("directory link"); + std::os::unix::fs::symlink(outside.path().join("sentinel"), temp.path().join("bucket/file-link")).expect("file link"); + for path in ["link/sentinel", "file-link"] { + assert!(matches!(disk.read_all("bucket", path).await, Err(DiskError::InvalidPath))); + assert!(matches!( + disk.write_all("bucket", path, Bytes::from_static(b"unsafe")).await, + Err(DiskError::InvalidPath) + )); + } + assert_eq!(std::fs::read(outside.path().join("sentinel")).expect("outside bytes"), b"preserved"); +} + +#[tokio::test] +async fn inline_and_non_inline_overwrites_remain_readable_after_reopen() { + for inline in [true, false] { + let temp = tempfile::tempdir().expect("test directory"); + let disk = disk_at(temp.path()).await; + disk.make_volume("bucket").await.expect("bucket"); + put(&disk, "first", Bytes::from_static(b"old bytes"), inline).await; + let directory = put(&disk, "second", Bytes::from_static(b"new bytes"), inline).await; + drop(disk); + let disk = disk_at(temp.path()).await; + let fi = disk + .read_version( + "bucket", + "bucket", + "object", + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("committed metadata after reopen"); + assert_eq!(fi.data_dir, Some(directory)); + if inline { + assert_eq!(fi.data, Some(Bytes::from_static(b"new bytes"))); + } else { + assert_eq!( + disk.read_all("bucket", &format!("object/{directory}/part.1")) + .await + .expect("shard bytes"), + Bytes::from_static(b"new bytes") + ); + } + } +} + +#[tokio::test] +async fn rename_rejects_symlinked_destination_metadata() { + for inline in [true, false] { + let temp = tempfile::tempdir().expect("disk directory"); + let outside = tempfile::tempdir().expect("outside directory"); + let disk = disk_at(temp.path()).await; + disk.make_volume("bucket").await.expect("bucket"); + put(&disk, "seed", Bytes::from_static(b"old object"), inline).await; + let destination = temp.path().join("bucket/object/xl.meta"); + let original = std::fs::read(&destination).expect("valid existing metadata"); + let external_metadata = outside.path().join("xl.meta"); + std::fs::write(&external_metadata, &original).expect("external valid metadata"); + std::fs::remove_file(&destination).expect("replace destination metadata"); + std::os::unix::fs::symlink(&external_metadata, &destination).expect("metadata escape link"); + + let directory = Uuid::new_v4(); + let data = Bytes::from_static(b"replacement"); + if !inline { + disk.write_all(RUSTFS_META_TMP_BUCKET, &format!("replacement/{directory}/part.1"), data.clone()) + .await + .expect("stage non-inline data"); + } + let result = disk + .rename_data( + RUSTFS_META_TMP_BUCKET, + "replacement", + metadata(data, inline, directory), + "bucket", + "object", + ) + .await; + assert!( + matches!(result, Err(DiskError::InvalidPath)), + "symlinked metadata must be rejected: {result:?}" + ); + assert_eq!(std::fs::read_link(&destination).expect("link must remain"), external_metadata); + assert_eq!(std::fs::read(&external_metadata).expect("external metadata must survive"), original); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn prepared_publication_rejects_source_or_directory_replacement() { + for replace_directory in [false, true] { + let temp = tempfile::tempdir().expect("test directory"); + let disk = disk_at(temp.path()).await; + disk.make_volume("bucket").await.expect("bucket"); + put(&disk, "first", Bytes::from_static(b"old bytes"), true).await; + let old = std::fs::read(temp.path().join("bucket/object/xl.meta")).expect("old metadata"); + let mut pause = LocalPublicationPause::install(&disk, "bucket", "object/xl.meta", LocalPublicationStage::PreparedRename) + .expect("publication pause"); + let writer = disk.clone(); + let task = tokio::spawn(async move { + writer + .rename_data( + RUSTFS_META_TMP_BUCKET, + "second", + metadata(Bytes::from_static(b"new bytes"), true, Uuid::new_v4()), + "bucket", + "object", + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), pause.entered()) + .await + .expect("publication reached pause") + .expect("pause entered"); + if replace_directory { + std::fs::rename(temp.path().join("bucket/object"), temp.path().join("detached")).expect("detach object"); + std::fs::create_dir(temp.path().join("bucket/object")).expect("replacement directory"); + std::fs::write(temp.path().join("bucket/object/xl.meta"), &old).expect("replacement metadata"); + } else { + let src = temp.path().join(RUSTFS_META_TMP_BUCKET).join("second/xl.meta"); + std::fs::rename(&src, src.with_extension("detached")).expect("detach staged metadata"); + std::fs::write(src, b"unrelated").expect("replacement staged file"); + } + drop(pause); + assert!(task.await.expect("writer task").is_err()); + assert_eq!( + std::fs::read(temp.path().join("bucket/object/xl.meta")).expect("old metadata survives"), + old + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancelled_waiter_keeps_publication_serialized_until_physical_completion() { + let temp = tempfile::tempdir().expect("test directory"); + let disk = disk_at(temp.path()).await; + disk.make_volume("bucket").await.expect("bucket"); + put(&disk, "first", Bytes::from_static(b"old bytes"), true).await; + let mut pause = LocalPublicationPause::install(&disk, "bucket", "object/xl.meta", LocalPublicationStage::PreparedRename) + .expect("publication pause"); + let writer = disk.clone(); + let pending = tokio::spawn(async move { put(&writer, "shared-stage", Bytes::from_static(b"cancelled waiter"), true).await }); + tokio::time::timeout(Duration::from_secs(30), pause.entered()) + .await + .expect("publication reached pause") + .expect("pause entered"); + pending.abort(); + assert!(pending.await.expect_err("waiter aborted").is_cancelled()); + let writer = disk.clone(); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let mut retry = tokio::spawn(async move { + let _ = started_tx.send(()); + put(&writer, "shared-stage", Bytes::from_static(b"latest bytes"), true).await + }); + started_rx.await.expect("retry started"); + assert!( + tokio::time::timeout(Duration::from_millis(200), &mut retry).await.is_err(), + "a retry must not publish while the cancelled writer still owns physical IO" + ); + drop(pause); + tokio::time::timeout(Duration::from_secs(30), retry) + .await + .expect("retry completed") + .expect("retry task"); + let fi = disk + .read_version( + "bucket", + "bucket", + "object", + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("latest metadata"); + assert_eq!(fi.data, Some(Bytes::from_static(b"latest bytes"))); +} diff --git a/crates/ecstore/tests/metadata_read_lock_reuse_test.rs b/crates/ecstore/tests/metadata_read_lock_reuse_test.rs index 6433bf0cd..c37949477 100644 --- a/crates/ecstore/tests/metadata_read_lock_reuse_test.rs +++ b/crates/ecstore/tests/metadata_read_lock_reuse_test.rs @@ -18,6 +18,7 @@ mod storage_api; use std::sync::Arc; use std::time::Duration; +use storage_api::contract_compat::ECStore; use storage_api::metadata_lock::{ BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys, @@ -170,6 +171,100 @@ async fn reused_metadata_lock_still_enforces_quota() { .expect("acquire writer"); } +#[tokio::test] +async fn authoritative_quota_read_does_not_ignore_later_quota() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "authoritative-quota-quota-set"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + + let mut first = PutObjReader::from_vec(b"cache no quota".to_vec()); + store + .put_object(bucket, "warm", &mut first, &ObjectOptions::default()) + .await + .expect("warm no-quota metadata cache"); + store + .update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec()) + .await + .expect("set quota after no-quota cache was observed"); + + let mut opts = quota_snapshot_options(&store, bucket).await; + assert!(opts.set_quota_admission(0, 1)); + let mut data = PutObjReader::from_vec(b"too large".to_vec()); + let error = store + .put_object(bucket, "object", &mut data, &opts) + .await + .expect_err("quota set after a no-quota cache hit must still reject growth"); + assert!(matches!(error, Error::QuotaExceeded { limit: 1, .. }), "{error:?}"); +} + +#[tokio::test] +async fn authoritative_quota_read_allows_write_after_quota_is_cleared() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "authoritative-quota-quota-cleared"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + store + .update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec()) + .await + .expect("set quota"); + store + .update_bucket_metadata_config(bucket, "quota.json", Vec::new()) + .await + .expect("clear quota"); + + let mut data = PutObjReader::from_vec(b"allowed after clear".to_vec()); + store + .put_object(bucket, "object", &mut data, &ObjectOptions::default()) + .await + .expect("authoritatively cleared quota should allow writes"); + + let mut reader = store + .get_object_reader(bucket, "object", None, Default::default(), &ObjectOptions::default()) + .await + .expect("read object written after quota clear"); + let mut bytes = Vec::new(); + reader.stream.read_to_end(&mut bytes).await.expect("read body"); + assert_eq!(bytes, b"allowed after clear"); +} + +#[tokio::test] +async fn authoritative_quota_read_fails_closed_on_invalid_quota_json() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + let bucket = "authoritative-quota-invalid-quota"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + store + .update_bucket_metadata_config(bucket, "quota.json", b"{not-json".to_vec()) + .await + .expect("persist invalid quota bytes for fail-closed read"); + + let mut data = PutObjReader::from_vec(b"must fail".to_vec()); + store + .put_object(bucket, "object", &mut data, &ObjectOptions::default()) + .await + .expect_err("invalid quota JSON must not degrade to no-quota"); +} + +async fn quota_snapshot_options(store: &Arc, bucket: &str) -> ObjectOptions { + ObjectOptions { + versioned: true, + version_id: Some(Uuid::new_v4().to_string()), + expected_bucket_incarnation_id: Some(store.bucket_incarnation_id(bucket).await.expect("load incarnation")), + object_lock_config_snapshot: Some(store.object_lock_config_snapshot(bucket).await.expect("capture snapshot")), + ..Default::default() + } +} + #[tokio::test] async fn cancelled_put_releases_shared_metadata_guard() { let (_dirs, store) = isolated_store_over_temp_disks().await; diff --git a/crates/ecstore/tests/storage_api.rs b/crates/ecstore/tests/storage_api.rs index acfd5c85e..d65582402 100644 --- a/crates/ecstore/tests/storage_api.rs +++ b/crates/ecstore/tests/storage_api.rs @@ -61,6 +61,17 @@ pub(crate) mod minio_generated_read { }; } +pub(crate) mod cleanup_isolation { + pub(crate) use super::{DiskAPI, DiskOption, Endpoint, new_disk}; + pub(crate) use rustfs_ecstore::api::disk::{DeleteOptions, DiskError}; +} + +#[cfg(all(feature = "test-util", target_os = "linux"))] +pub(crate) mod fd_relative { + pub(crate) use super::{DiskAPI, DiskOption, DiskStore, Endpoint, new_disk}; + pub(crate) use rustfs_ecstore::api::disk::{DiskError, LocalPublicationPause, LocalPublicationStage, ReadOptions}; +} + pub(crate) mod pinned_erasure_fixtures { pub(crate) use super::storage_contracts::{BucketOperations, HTTPRangeSpec, MakeBucketOptions, ObjectIO}; pub(crate) use super::{ECStore, Endpoint, Erasure, Error, ObjectOptions}; diff --git a/crates/io-metrics/benches/metrics_pipeline.rs b/crates/io-metrics/benches/metrics_pipeline.rs index fb1aa6857..3f213e586 100644 --- a/crates/io-metrics/benches/metrics_pipeline.rs +++ b/crates/io-metrics/benches/metrics_pipeline.rs @@ -1,13 +1,15 @@ use criterion::{Criterion, criterion_group, criterion_main}; -use rustfs_io_metrics::{MetricsCollector, PerformanceMetrics, record_get_object_request_started}; +use rustfs_io_metrics::{MetricsCollector, PerformanceMetrics, record_get_object_timeout_operation_start}; use rustfs_io_metrics::{record_s3_op, s3_http_metrics::S3HttpRequestGuard}; use rustfs_s3_ops::S3Operation; use std::hint::black_box; use std::sync::Arc; use std::time::Duration; -fn bench_record_get_object_request_started(c: &mut Criterion) { - c.bench_function("record_get_object_request_started", |b| b.iter(record_get_object_request_started)); +fn bench_record_get_object_timeout_operation_start(c: &mut Criterion) { + c.bench_function("record_get_object_timeout_operation_start", |b| { + b.iter(record_get_object_timeout_operation_start) + }); } fn bench_s3_http_outcomes(c: &mut Criterion) { @@ -49,7 +51,7 @@ fn bench_metrics_collector_record_io_operation(c: &mut Criterion) { criterion_group!( benches, - bench_record_get_object_request_started, + bench_record_get_object_timeout_operation_start, bench_s3_http_outcomes, bench_update_concurrent_requests, bench_metrics_collector_record_io_operation diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index f499c0f02..45e089bab 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -49,8 +49,9 @@ #[macro_use] extern crate metrics; +use std::collections::HashMap; use std::sync::{ - Mutex, + LazyLock, Mutex, atomic::{AtomicBool, AtomicU64, Ordering}, }; @@ -61,6 +62,25 @@ use std::sync::{ /// Enabled only through an explicit runtime opt-in. static PUT_STAGE_METRICS_ENABLED: AtomicBool = AtomicBool::new(false); static GET_STAGE_METRICS_ENABLED: AtomicBool = AtomicBool::new(false); +static GET_STAGE_LOCAL_SUMMARY_ENABLED: AtomicBool = AtomicBool::new(false); +static GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE: AtomicU64 = AtomicU64::new(64); +static GET_STAGE_LOCAL_SUMMARY_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +type GetStageLocalSummaryKey = (&'static str, &'static str, &'static str, &'static str); + +/// Sampled local aggregate for one bounded GetObject stage label set. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct GetStageLocalSummary { + pub path: &'static str, + pub stage: &'static str, + pub object_class: &'static str, + pub size_bucket: &'static str, + pub sampled_count: u64, + pub duration_nanoseconds: u64, +} + +static GET_STAGE_LOCAL_SUMMARY: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); /// Global switch for all remaining (non GET/PUT-stage) metric emission in this /// crate's free `record_*` functions — I/O scheduler, bytes-pool, zero-copy, @@ -87,6 +107,70 @@ pub fn set_get_stage_metrics_enabled(enabled: bool) { GET_STAGE_METRICS_ENABLED.store(enabled, Ordering::Relaxed); } +pub fn set_get_stage_local_summary_enabled(enabled: bool) { + GET_STAGE_LOCAL_SUMMARY_ENABLED.store(enabled, Ordering::Relaxed); +} + +pub fn set_get_stage_local_summary_sample_rate(sample_rate: u64) { + GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE.store(sample_rate.max(1), Ordering::Relaxed); +} + +pub fn get_stage_local_summary_enabled() -> bool { + GET_STAGE_LOCAL_SUMMARY_ENABLED.load(Ordering::Relaxed) +} + +pub fn get_stage_local_summary_sample_rate() -> u64 { + GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE.load(Ordering::Relaxed) +} + +pub fn take_get_stage_local_summary() -> Vec { + let Ok(mut summary) = GET_STAGE_LOCAL_SUMMARY.lock() else { + return Vec::new(); + }; + let mut rows = summary.drain().map(|(_, row)| row).collect::>(); + rows.sort_unstable_by_key(|row| (row.path, row.stage, row.object_class, row.size_bucket)); + rows +} + +fn record_get_stage_local_summary( + path: &'static str, + stage: &'static str, + object_class: &'static str, + size_bucket: &'static str, + duration_secs: f64, +) { + let sample_rate = get_stage_local_summary_sample_rate(); + if sample_rate > 1 + && !GET_STAGE_LOCAL_SUMMARY_SEQUENCE + .fetch_add(1, Ordering::Relaxed) + .is_multiple_of(sample_rate) + { + return; + } + + let Ok(duration) = std::time::Duration::try_from_secs_f64(duration_secs) else { + return; + }; + let Ok(duration_nanoseconds) = u64::try_from(duration.as_nanos()) else { + return; + }; + + let key = (path, stage, object_class, size_bucket); + let Ok(mut summary) = GET_STAGE_LOCAL_SUMMARY.lock() else { + return; + }; + let row = summary.entry(key).or_insert_with(|| GetStageLocalSummary { + path, + stage, + object_class, + size_bucket, + sampled_count: 0, + duration_nanoseconds: 0, + }); + row.sampled_count = row.sampled_count.saturating_add(1); + row.duration_nanoseconds = row.duration_nanoseconds.saturating_add(duration_nanoseconds); +} + /// Enable or disable general (non GET/PUT-stage) metric emission. /// /// Called once during startup, typically gated by `rustfs_obs::observability_metric_enabled()`. @@ -110,8 +194,210 @@ pub fn put_stage_timer() -> Option { } pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT: &str = "put_object_commit_namespace_lock_wait"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD: &str = "put_object_commit_namespace_lock_held"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME: &str = "put_object_commit_namespace_lock_held_rename"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_PRE_RENAME_PREPARE: &str = + "put_object_commit_namespace_lock_held_pre_rename_prepare"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME_RESULT_TO_TAIL_HANDOFF: &str = + "put_object_commit_namespace_lock_held_rename_result_to_tail_handoff"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_TAIL_HANDOFF: &str = + "put_object_commit_namespace_lock_held_tail_handoff"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_RENAME: &str = "put_object_commit_namespace_lock_held_post_rename"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_GUARD_RELEASE: &str = + "put_object_commit_namespace_lock_held_guard_release"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE: &str = + "put_object_commit_namespace_lock_held_quota_fence_release"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_COMMIT: &str = + "put_object_commit_namespace_lock_held_quota_commit"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT: &str = "put_object_commit_namespace_lock_held_heal_submit"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT: &str = + "put_object_commit_namespace_lock_held_cleanup_receipt"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_METADATA_INVALIDATE: &str = + "put_object_commit_namespace_lock_held_metadata_invalidate"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_OBJECT_INFO_BUILD: &str = + "put_object_commit_namespace_lock_held_object_info_build"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_OBJECT_INFO_TO_RETURN: &str = + "put_object_commit_namespace_lock_held_post_object_info_to_return"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT: &str = "put_object_commit_task_wait"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL: &str = "put_object_commit_future_total"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT: &str = "put_object_commit_operation_select_wait"; +pub const PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL: &str = "put_object_operation_await_total"; +pub const PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT: &str = "put_object_operation_body_to_commit_submit"; +pub const PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN: &str = "put_object_operation_result_return"; +pub const PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY: &str = "put_object_operation_first_poll_delay"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY: &str = + "put_object_commit_namespace_lock_acquired_to_commit_ready"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_PRECONDITION_AND_TIMESTAMP: &str = + "put_object_commit_ready_precondition_and_timestamp"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPECTED_VERSION_LOOKUP: &str = "put_object_commit_ready_expected_version_lookup"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPLICIT_VERSION_LOOKUP: &str = "put_object_commit_ready_explicit_version_lookup"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_RESTORE_VERIFY: &str = "put_object_commit_ready_restore_verify"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_FENCE_VERIFY: &str = "put_object_commit_ready_fence_verify"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_DECOMMISSION_CAPACITY_FENCE: &str = + "put_object_commit_ready_decommission_capacity_fence"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_TRANSACTION_EPOCH_FENCE_READ: &str = + "put_object_commit_ready_transaction_epoch_fence_read"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_BEGIN: &str = "put_object_commit_ready_quota_begin"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_METADATA_ADJUST: &str = "put_object_commit_ready_metadata_adjust"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_RESERVE: &str = "put_object_commit_ready_quota_reserve"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_MUTATION_FENCE_PREPARE: &str = + "put_object_commit_ready_quota_mutation_fence_prepare"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_CONTEXT_PREPARE: &str = "put_object_commit_ready_context_prepare"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH: &str = "put_object_quota_begin_meta_bucket_fast_path"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK: &str = "put_object_quota_begin_metadata_lock"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP: &str = "put_object_quota_begin_cache_lookup"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ: &str = "put_object_quota_begin_config_read"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK: &str = "put_object_quota_begin_lock_lost_check"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE: &str = "put_object_quota_begin_policy_evaluate"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF: &str = "put_object_quota_begin_capability_proof"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION: &str = "put_object_quota_begin_snapshot_admission"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP: &str = "put_object_quota_begin_object_store_lookup"; +pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD: &str = "put_object_quota_begin_context_build"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER: &str = "put_object_commit_submit_to_closure_enter"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT: &str = "put_object_commit_closure_enter_to_first_await"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL: &str = "put_object_commit_closure_total"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME: &str = "put_object_commit_before_rename"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_AWAIT_TOTAL: &str = "put_object_commit_pre_rename_await_total"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_QUOTA_MARK_COMMIT_STARTED: &str = + "put_object_commit_pre_rename_quota_mark_commit_started"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_FIRST_FENCE_CHECK: &str = "put_object_commit_pre_rename_first_fence_check"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_RESTORE_VERIFY: &str = "put_object_commit_pre_rename_restore_verify"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TRANSACTION_FENCE_VERIFY: &str = + "put_object_commit_pre_rename_transaction_fence_verify"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SECOND_FENCE_CHECK: &str = "put_object_commit_pre_rename_second_fence_check"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SCANNER_SCOPE_CHECK: &str = "put_object_commit_pre_rename_scanner_scope_check"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TIER_FREE_VERSION_SOURCE_LOOKUP: &str = + "put_object_commit_pre_rename_tier_free_version_source_lookup"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_ASSIGN_RENAME_INDEXES: &str = + "put_object_commit_pre_rename_assign_rename_indexes"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_POST_AWAIT_TO_RENAME: &str = "put_object_commit_pre_rename_post_await_to_rename"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN: &str = "put_object_commit_after_rename_to_return"; pub const PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT: &str = "set_disk_rename_quorum_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_SYNC_FULL_FANOUT_WAIT: &str = "set_disk_rename_sync_full_fanout_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT: &str = "set_disk_rename_early_ack_spawn_to_quorum_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER: &str = "set_disk_rename_early_ack_attach_tail_owner"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_AFTER_QUORUM_WAIT: &str = "set_disk_rename_tail_after_quorum_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT: &str = "set_disk_rename_tail_async_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT: &str = "set_disk_rename_tail_guard_release_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_FINALIZE: &str = "set_disk_rename_tail_finalize"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP: &str = "set_disk_rename_tail_cleanup"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT: &str = "set_disk_rename_tail_cleanup_submit"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP: &str = "set_disk_rename_tail_cleanup_counterfactual_skip"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED: &str = "set_disk_rename_tail_cleanup_defer_enqueued"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_FALLBACK_SYNC: &str = "set_disk_rename_tail_cleanup_defer_fallback_sync"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH: &str = "set_disk_rename_tail_cleanup_defer_queue_push"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH: &str = "set_disk_rename_tail_cleanup_defer_queue_depth"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE: &str = + "set_disk_rename_tail_cleanup_defer_queue_depth_max_sample"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE: &str = "set_disk_rename_tail_cleanup_defer_oldest_age"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE: &str = + "set_disk_rename_tail_cleanup_defer_oldest_age_max_sample"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_TARGETS: &str = "set_disk_rename_tail_cleanup_defer_targets"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_START_DELAY: &str = "set_disk_rename_tail_cleanup_defer_start_delay"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED_LAG: &str = "set_disk_rename_tail_cleanup_defer_completed_lag"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER: &str = "set_disk_rename_tail_cleanup_defer_hold_worker"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY: &str = "set_disk_rename_tail_cleanup_defer_worker_delay"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_PERMIT_WAIT: &str = "set_disk_rename_tail_cleanup_per_disk_permit_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_INFLIGHT: &str = "set_disk_rename_tail_cleanup_per_disk_inflight"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD: &str = "set_disk_rename_tail_cleanup_foreground_yield"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFERRED_FOREGROUND_ACTIVE: &str = + "set_disk_rename_tail_cleanup_deferred_foreground_active"; +pub const PUT_STAGE_SET_DISK_RENAME_FOREGROUND_ACTIVE: &str = "set_disk_rename_foreground_active"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW: &str = "set_disk_rename_tail_cleanup_defer_batch_window"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_SIZE: &str = "set_disk_rename_tail_cleanup_defer_batch_size"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BURST_BATCH_SIZE: &str = + "set_disk_rename_tail_cleanup_defer_burst_batch_size"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_ACTIVE: &str = "set_disk_rename_tail_cleanup_defer_worker_active"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_POLL_WAIT: &str = + "set_disk_rename_tail_cleanup_defer_worker_poll_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_IDLE_WAIT: &str = + "set_disk_rename_tail_cleanup_defer_worker_idle_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_JOB_WAIT: &str = + "set_disk_rename_tail_cleanup_defer_worker_job_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED: &str = "set_disk_rename_tail_cleanup_defer_completed"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DURATION: &str = + "set_disk_rename_tail_cleanup_defer_worker_duration"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_RECEIPT: &str = "set_disk_rename_tail_cleanup_target_receipt"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COMMIT_CAPACITY: &str = + "set_disk_rename_tail_cleanup_target_commit_capacity"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_REPORT: &str = "set_disk_rename_tail_cleanup_target_report"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COUNT: &str = "set_disk_rename_tail_cleanup_target_count"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_ZERO_TARGET: &str = "set_disk_rename_tail_cleanup_zero_target"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE: &str = "set_disk_rename_tail_cleanup_tmp_delete"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE_COUNTERFACTUAL_SKIP: &str = + "set_disk_rename_tail_cleanup_tmp_delete_counterfactual_skip"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DECOMMISSION_GUARD_DROP: &str = + "set_disk_rename_tail_cleanup_decommission_guard_drop"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH: &str = "set_disk_rename_tail_cleanup_lane_queue_depth"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_PUSH: &str = "set_disk_rename_tail_cleanup_lane_queue_push"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE: &str = + "set_disk_rename_tail_cleanup_lane_queue_depth_max_sample"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE: &str = "set_disk_rename_tail_cleanup_lane_oldest_age"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE_MAX_SAMPLE: &str = + "set_disk_rename_tail_cleanup_lane_oldest_age_max_sample"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BATCH_SIZE: &str = "set_disk_rename_tail_cleanup_lane_batch_size"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BURST_BATCH_SIZE: &str = + "set_disk_rename_tail_cleanup_lane_burst_batch_size"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_ACTIVE: &str = "set_disk_rename_tail_cleanup_lane_worker_active"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_POLL_WAIT: &str = + "set_disk_rename_tail_cleanup_lane_worker_poll_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_IDLE_WAIT: &str = + "set_disk_rename_tail_cleanup_lane_worker_idle_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_JOB_WAIT: &str = "set_disk_rename_tail_cleanup_lane_worker_job_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_SERVICE_TIME: &str = "set_disk_rename_tail_cleanup_lane_service_time"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_REQUEUE: &str = "set_disk_rename_tail_cleanup_lane_requeue"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_FOREGROUND_ACTIVE_HIT: &str = + "set_disk_rename_tail_cleanup_lane_foreground_active_hit"; +pub const PUT_STAGE_SET_DISK_RENAME_TAIL_HEAL_SUBMIT: &str = "set_disk_rename_tail_heal_submit"; pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT: &str = "set_disk_rename_disk_wait"; +pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL: &str = "set_disk_rename_disk_wait_local"; +pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE: &str = "set_disk_rename_disk_wait_remote"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE: &str = "set_disk_rename_remote_client_prepare"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC: &str = "set_disk_rename_remote_client_rpc"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT: &str = "set_disk_rename_remote_client_rpc_await"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT: &str = + "set_disk_rename_remote_client_rpc_await_poll_pending_count"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY: &str = + "set_disk_rename_remote_client_rpc_await_first_pending_to_ready"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE: &str = "set_disk_rename_remote_client_request_scope"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL: &str = "set_disk_rename_remote_client_transport_call"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT: &str = + "set_disk_rename_remote_client_transport_call_poll_pending_count"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY: &str = + "set_disk_rename_remote_client_transport_call_first_pending_to_ready"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE: &str = "set_disk_rename_remote_client_replay_response"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER: &str = "set_disk_rename_remote_client_into_inner"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK: &str = "set_disk_rename_remote_client_success_check"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE: &str = "set_disk_rename_remote_client_decode"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS: &str = "set_disk_rename_remote_client_response_process"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS: &str = "set_disk_rename_remote_client_response_headers"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE: &str = "set_disk_rename_remote_service_response_encode"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_TRIGGERED: &str = + "put_object_commit_namespace_lock_held_quota_fence_release_triggered"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_SKIPPED: &str = + "put_object_commit_namespace_lock_held_quota_fence_release_skipped"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_TRIGGERED: &str = + "put_object_commit_namespace_lock_held_heal_submit_triggered"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_SKIPPED: &str = + "put_object_commit_namespace_lock_held_heal_submit_skipped"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_TRIGGERED: &str = + "put_object_commit_namespace_lock_held_cleanup_receipt_triggered"; +pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_SKIPPED: &str = + "put_object_commit_namespace_lock_held_cleanup_receipt_skipped"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL: &str = "set_disk_rename_remote_handler_total"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD: &str = "set_disk_rename_remote_handler_overhead"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP: &str = "set_disk_rename_remote_handler_lookup"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT: &str = "set_disk_rename_remote_handler_admit"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME: &str = "set_disk_rename_remote_handler_local_rename"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY: &str = "set_disk_rename_remote_service_entry"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER: &str = "set_disk_rename_remote_service_before_handler"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT: &str = "set_disk_rename_remote_service_handler_await"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER: &str = "set_disk_rename_remote_service_after_handler"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD: &str = "set_disk_rename_remote_service_response_build"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN: &str = "set_disk_rename_remote_service_response_return"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE: &str = "set_disk_rename_remote_transport_service_future"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME: &str = "set_disk_rename_remote_transport_first_frame"; +pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE: &str = "set_disk_rename_remote_transport_body_complete"; pub const PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_file_sync_permit_wait"; pub const PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_global_file_sync_permit_wait"; pub const PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC: &str = "set_disk_rename_file_fdatasync"; @@ -460,7 +746,16 @@ pub fn record_get_object_request_start(concurrent_requests: usize) { gauge!("rustfs_io_get_object_concurrent_requests").set(concurrent_requests as f64); } -/// Record GetObject request start without concurrency context. +/// Record one timeout-wrapped GetObject storage operation start. +#[inline(always)] +pub fn record_get_object_timeout_operation_start() { + if !get_stage_metrics_enabled() { + return; + } + counter!("rustfs_io_get_object_timeout_operations_total").increment(1); +} + +/// Record a GetObject request start without concurrency context. #[inline(always)] pub fn record_get_object_request_started() { if !get_stage_metrics_enabled() { @@ -479,6 +774,16 @@ pub fn record_get_object_request_result(status: &str, duration_secs: f64) { histogram!("rustfs_io_get_object_request_duration_seconds", "status" => status.to_string()).record(duration_secs); } +/// Record the result of one timeout-wrapped GetObject storage operation. +#[inline(always)] +pub fn record_get_object_timeout_operation_result(status: &str, duration_secs: f64) { + if !get_stage_metrics_enabled() { + return; + } + counter!("rustfs_io_get_object_timeout_operation_results_total", "status" => status.to_string()).increment(1); + histogram!("rustfs_io_get_object_timeout_operation_duration_seconds", "status" => status.to_string()).record(duration_secs); +} + /// Record PutObject request start. #[inline(always)] pub fn record_put_object_request_start(concurrent_requests: usize) { @@ -760,6 +1065,10 @@ pub fn record_get_object_io_state( /// Record GetObject phase duration for the current read path. #[inline(always)] pub fn record_get_object_stage_duration(path: &'static str, stage: &'static str, duration_secs: f64) { + if get_stage_local_summary_enabled() { + record_get_stage_local_summary(path, stage, "", "", duration_secs); + return; + } if !get_stage_metrics_enabled() { return; } @@ -775,6 +1084,10 @@ pub fn record_get_object_stage_duration_by_size( size_bucket: &'static str, duration_secs: f64, ) { + if get_stage_local_summary_enabled() { + record_get_stage_local_summary(path, stage, object_class, size_bucket, duration_secs); + return; + } if !get_stage_metrics_enabled() { return; } @@ -827,6 +1140,15 @@ pub fn record_get_object_metadata_response(path: &'static str, outcome: &'static counter!("rustfs_io_get_object_metadata_response_total", "path" => path, "outcome" => outcome).increment(1); } +/// Record whether metadata resolution selected an object identity at read quorum. +#[inline(always)] +pub fn record_get_object_metadata_quorum_result(path: &'static str, outcome: &'static str) { + if !get_stage_metrics_enabled() { + return; + } + counter!("rustfs_io_get_object_metadata_quorum_results_total", "path" => path, "outcome" => outcome).increment(1); +} + /// Record one bounded metadata cache decision. #[inline(always)] pub fn record_get_object_metadata_cache_decision(path: &'static str, decision: &'static str, reason: &'static str) { @@ -1668,6 +1990,10 @@ pub fn record_get_object_metadata_phase_duration(duration_secs: f64) { /// Record metadata phase duration with early-stop state label. #[inline(always)] pub fn record_get_object_metadata_phase_duration_with_early_stop(duration_secs: f64, early_stop_active: &'static str) { + if get_stage_local_summary_enabled() { + record_get_stage_local_summary("legacy_duplex", "metadata", early_stop_active, "", duration_secs); + return; + } if !get_stage_metrics_enabled() { return; } @@ -2073,6 +2399,57 @@ pub fn record_put_object_stage_duration_from(stage: &'static str, started_at: Op } } +#[inline(always)] +pub fn record_put_object_stage_count(stage: &'static str, count: u64) { + if !put_stage_metrics_enabled() { + return; + } + counter!("rustfs_s3_put_object_stage_count", "stage" => stage).increment(count); +} + +/// Observe a future's latency and wakeup behavior only while detailed PUT +/// attribution is enabled. The disabled path awaits the original future +/// directly, so normal remote mutation RPCs do not gain a polling wrapper. +pub async fn observe_put_stage_future( + future: F, + duration_stage: &'static str, + pending_count_stage: &'static str, + first_pending_to_ready_stage: &'static str, +) -> F::Output +where + F: std::future::Future, +{ + if !put_stage_metrics_enabled() { + return future.await; + } + + let duration_started = put_stage_timer(); + let mut first_pending_at = None; + let mut pending_count = 0_u64; + let mut future = std::pin::pin!(future); + let output = std::future::poll_fn(|cx| match future.as_mut().poll(cx) { + std::task::Poll::Ready(output) => std::task::Poll::Ready(output), + std::task::Poll::Pending => { + pending_count = pending_count.saturating_add(1); + first_pending_at.get_or_insert_with(std::time::Instant::now); + std::task::Poll::Pending + } + }) + .await; + record_put_object_stage_duration_from(duration_stage, duration_started); + record_put_object_stage_count(pending_count_stage, pending_count); + record_put_object_stage_duration_from(first_pending_to_ready_stage, first_pending_at); + output +} + +#[inline(always)] +pub fn record_put_object_quota_cache_decision(decision: &'static str, reason: &'static str) { + if !put_stage_metrics_enabled() { + return; + } + counter!("rustfs_s3_put_object_quota_cache_total", "decision" => decision, "reason" => reason).increment(1); +} + #[inline(always)] pub fn record_put_object_commit_lock_admission(budget: &'static str, outcome: &'static str) { if !put_stage_metrics_enabled() { @@ -3219,8 +3596,162 @@ mod tests { assert_eq!(PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT, "put_object_commit_namespace_lock_wait"); let stages = [ PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_PRE_RENAME_PREPARE, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME_RESULT_TO_TAIL_HANDOFF, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_TAIL_HANDOFF, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_RENAME, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_GUARD_RELEASE, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_COMMIT, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_METADATA_INVALIDATE, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_OBJECT_INFO_BUILD, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_OBJECT_INFO_TO_RETURN, + PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT, + PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL, + PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT, + PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL, + PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT, + PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN, + PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_PRECONDITION_AND_TIMESTAMP, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPECTED_VERSION_LOOKUP, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPLICIT_VERSION_LOOKUP, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_RESTORE_VERIFY, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_FENCE_VERIFY, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_DECOMMISSION_CAPACITY_FENCE, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_TRANSACTION_EPOCH_FENCE_READ, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_BEGIN, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_METADATA_ADJUST, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_RESERVE, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_MUTATION_FENCE_PREPARE, + PUT_STAGE_PUT_OBJECT_COMMIT_READY_CONTEXT_PREPARE, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP, + PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD, + PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER, + PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT, + PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL, + PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_AWAIT_TOTAL, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_QUOTA_MARK_COMMIT_STARTED, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_FIRST_FENCE_CHECK, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_RESTORE_VERIFY, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TRANSACTION_FENCE_VERIFY, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SECOND_FENCE_CHECK, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SCANNER_SCOPE_CHECK, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TIER_FREE_VERSION_SOURCE_LOOKUP, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_ASSIGN_RENAME_INDEXES, + PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_POST_AWAIT_TO_RENAME, + PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN, PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT, + PUT_STAGE_SET_DISK_RENAME_SYNC_FULL_FANOUT_WAIT, + PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT, + PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER, + PUT_STAGE_SET_DISK_RENAME_TAIL_AFTER_QUORUM_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_FINALIZE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_FALLBACK_SYNC, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_TARGETS, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_START_DELAY, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED_LAG, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_PERMIT_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_INFLIGHT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFERRED_FOREGROUND_ACTIVE, + PUT_STAGE_SET_DISK_RENAME_FOREGROUND_ACTIVE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_SIZE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BURST_BATCH_SIZE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_ACTIVE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_POLL_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_IDLE_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_JOB_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DURATION, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_RECEIPT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COMMIT_CAPACITY, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_REPORT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TARGET_COUNT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_ZERO_TARGET, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE_COUNTERFACTUAL_SKIP, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DECOMMISSION_GUARD_DROP, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_PUSH, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE_MAX_SAMPLE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BATCH_SIZE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BURST_BATCH_SIZE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_ACTIVE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_POLL_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_IDLE_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_JOB_WAIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_SERVICE_TIME, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_REQUEUE, + PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_FOREGROUND_ACTIVE_HIT, + PUT_STAGE_SET_DISK_RENAME_TAIL_HEAL_SUBMIT, PUT_STAGE_SET_DISK_RENAME_DISK_WAIT, + PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL, + PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS, + PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_TRIGGERED, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_SKIPPED, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_TRIGGERED, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_SKIPPED, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_TRIGGERED, + PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_SKIPPED, + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL, + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD, + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, + PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME, + PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY, + PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER, + PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT, + PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER, + PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD, + PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN, + PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE, + PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME, + PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE, PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT, PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT, PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC, @@ -3233,11 +3764,36 @@ mod tests { let unique = stages.iter().copied().collect::>(); assert_eq!(unique.len(), stages.len()); assert!(stages.iter().all(|stage| { - (stage.starts_with("set_disk_rename_") || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT) + (stage.starts_with("set_disk_rename_") + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT + || *stage == PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL + || *stage == PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT + || *stage == PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN + || *stage == PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY + || stage.starts_with("put_object_commit_ready_") + || stage.starts_with("put_object_quota_begin_") + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME + || stage.starts_with("put_object_commit_pre_rename_") + || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN + || stage.starts_with("put_object_commit_namespace_lock_held_")) && !stage.contains('/') && !stage.contains('{') })); + let count_stages = [ + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT, + PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT, + ]; + assert_eq!(count_stages.iter().copied().collect::>().len(), count_stages.len()); + let recorder = DebuggingRecorder::new(); let snapshotter = recorder.snapshotter(); metrics::with_local_recorder(&recorder, || { @@ -3245,16 +3801,25 @@ mod tests { for stage in stages { record_put_object_stage_duration(stage, 1.0); } + for stage in count_stages { + record_put_object_stage_count(stage, 1); + } set_put_stage_metrics_enabled(true); for stage in stages { record_put_object_stage_duration(stage, 1.0); } + for stage in count_stages { + record_put_object_stage_count(stage, 1); + } set_put_stage_metrics_enabled(false); }); - let recorded = snapshotter - .snapshot() - .into_vec() + let rows = snapshotter.snapshot().into_vec(); + assert_eq!( + counter_total(&rows, "rustfs_s3_put_object_stage_count"), + Some(u64::try_from(count_stages.len()).expect("count stage length fits u64")), + ); + let recorded = rows .into_iter() .filter(|(composite, _, _, _)| { composite.kind() == MetricKind::Histogram && composite.key().name() == "rustfs_s3_put_object_stage_duration_ms" @@ -3349,6 +3914,63 @@ mod tests { ]))); } + #[test] + fn put_stage_future_observer_preserves_outputs_and_counts_pending() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .expect("build current-thread runtime for observer test"); + let duration_stage = PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT; + let count_stage = PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT; + let pending_stage = PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY; + + metrics::with_local_recorder(&recorder, || { + set_put_stage_metrics_enabled(false); + let output = runtime.block_on(observe_put_stage_future( + std::future::ready(42), + duration_stage, + count_stage, + pending_stage, + )); + assert_eq!(output, 42, "disabled observer must preserve the future output"); + }); + assert!(snapshotter.snapshot().into_vec().is_empty(), "disabled observer must emit no metrics"); + + metrics::with_local_recorder(&recorder, || { + set_put_stage_metrics_enabled(true); + let mut pending = 2; + let future = std::future::poll_fn(|cx| { + if pending > 0 { + pending -= 1; + cx.waker().wake_by_ref(); + std::task::Poll::Pending + } else { + std::task::Poll::Ready(Err::<(), _>("original error")) + } + }); + let output = runtime.block_on(observe_put_stage_future(future, duration_stage, count_stage, pending_stage)); + set_put_stage_metrics_enabled(false); + assert_eq!(output, Err("original error"), "enabled observer must preserve errors"); + }); + + let rows = snapshotter.snapshot().into_vec(); + assert_eq!(counter_total(&rows, "rustfs_s3_put_object_stage_count"), Some(2)); + let count_labels = rows + .iter() + .filter(|(composite, _, _, _)| composite.key().name() == "rustfs_s3_put_object_stage_count") + .flat_map(|(composite, _, _, _)| composite.key().labels().map(|label| label.value().to_string())) + .collect::>(); + assert_eq!(count_labels, vec![count_stage.to_string()]); + let durations = rows + .iter() + .filter(|(composite, _, _, _)| composite.key().name() == "rustfs_s3_put_object_stage_duration_ms") + .flat_map(|(composite, _, _, _)| composite.key().labels().map(|label| label.value().to_string())) + .collect::>(); + assert_eq!(durations, HashSet::from([duration_stage.to_string(), pending_stage.to_string()])); + } + #[test] fn put_rename_code_level_metrics_are_static_and_gated() { let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); @@ -3578,6 +4200,35 @@ mod tests { set_put_stage_metrics_enabled(false); } + #[test] + fn get_timeout_operation_and_metadata_quorum_metrics_have_separate_scopes() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + metrics::with_local_recorder(&recorder, || { + set_get_stage_metrics_enabled(true); + record_get_object_request_start(1); + record_get_object_request_result("ok", 0.01); + record_get_object_timeout_operation_start(); + record_get_object_timeout_operation_result("success", 0.005); + record_get_object_metadata_quorum_result("legacy_duplex", "reached"); + record_get_object_metadata_quorum_result("legacy_duplex", "not_reached"); + set_get_stage_metrics_enabled(false); + }); + + let rows = snapshotter.snapshot().into_vec(); + assert_eq!(counter_total(&rows, "rustfs_io_get_object_requests_total"), Some(1)); + assert_eq!(counter_total(&rows, "rustfs_io_get_object_request_results_total"), Some(1)); + assert_eq!(counter_total(&rows, "rustfs_io_get_object_timeout_operations_total"), Some(1)); + assert_eq!(counter_total(&rows, "rustfs_io_get_object_timeout_operation_results_total"), Some(1)); + assert_eq!(counter_total(&rows, "rustfs_io_get_object_metadata_quorum_results_total"), Some(2)); + assert_eq!( + histogram_samples(&rows, "rustfs_io_get_object_timeout_operation_duration_seconds"), + vec![0.005] + ); + } + #[test] fn test_record_get_object_path_and_stage() { let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); diff --git a/crates/log-analyzer/src/ingest/mod.rs b/crates/log-analyzer/src/ingest/mod.rs index 2ad960479..ba8b97b27 100644 --- a/crates/log-analyzer/src/ingest/mod.rs +++ b/crates/log-analyzer/src/ingest/mod.rs @@ -307,7 +307,13 @@ fn walk_zip_entries( if entry.is_dir() { continue; } - let entry_path = entry.name().to_string(); + let entry_path = match entry.name() { + Ok(name) => name.into_owned(), + Err(_) => { + ctx.skip(format!("{provenance}!"), SkipReason::Unreadable); + continue; + } + }; let child_provenance = format!("{provenance}!{entry_path}"); let child_node = node_from_entry_path(&entry_path).or_else(|| node.clone()); let child_name = leaf_name(&entry_path).to_string(); diff --git a/crates/obs/src/telemetry/resource.rs b/crates/obs/src/telemetry/resource.rs index 8a7d003ab..5f9c4aa33 100644 --- a/crates/obs/src/telemetry/resource.rs +++ b/crates/obs/src/telemetry/resource.rs @@ -16,9 +16,9 @@ //! //! A `Resource` describes the entity producing telemetry data. The resource //! built here includes the service name, service version, deployment -//! environment, the stable RustFS node identity, and the local machine IP -//! address so that data can be correlated across services in a distributed -//! system. +//! environment, the stable RustFS node identity, the process ID, and the local +//! machine IP address so telemetry series can be correlated across service +//! restarts and nodes. use crate::config::OtelConfig; use crate::node_identity::{RUSTFS_NODE_ATTRIBUTE, local_node_identity}; @@ -27,7 +27,7 @@ use opentelemetry_sdk::Resource; use opentelemetry_semantic_conventions::{ SCHEMA_URL, attribute::{ - DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_INSTANCE_ID as OTEL_SERVICE_INSTANCE_ID, + DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, PROCESS_PID, SERVICE_INSTANCE_ID as OTEL_SERVICE_INSTANCE_ID, SERVICE_VERSION as OTEL_SERVICE_VERSION, }, }; @@ -48,6 +48,8 @@ use std::borrow::Cow; /// - `network.local.address` — the primary local IP of the current host, /// useful as an operational fallback when the stable node name is not yet /// initialized. +/// - `process.pid` — distinguishes telemetry emitted by different processes +/// running on the same node. /// /// All attributes are attached to the resource using the semantic conventions /// schema URL to ensure compatibility with standard OTLP backends. @@ -70,6 +72,7 @@ pub(super) fn build_resource(config: &OtelConfig) -> Resource { KeyValue::new(RUSTFS_NODE_ATTRIBUTE, node_identity.clone()), KeyValue::new(OTEL_SERVICE_INSTANCE_ID, node_identity), KeyValue::new(NETWORK_LOCAL_ADDRESS, local_ip), + KeyValue::new(PROCESS_PID, i64::from(std::process::id())), ], SCHEMA_URL, ) @@ -117,6 +120,12 @@ mod tests { .map(|value| value.to_string()), Some("node1:9000".to_string()) ); + assert_eq!( + resource + .get(&Key::from_static_str(PROCESS_PID)) + .map(|value| value.to_string()), + Some(std::process::id().to_string()) + ); rustfs_common::set_global_local_node_name(&previous).await; } diff --git a/deny.toml b/deny.toml index b810318e7..1a84bdbbb 100644 --- a/deny.toml +++ b/deny.toml @@ -19,9 +19,6 @@ no-default-features = false version = 2 yanked = "deny" ignore = [ - # `paste 1.0.15` — unmaintained. No direct dependency. - # owner: rustfs-maintainers review: 2026-07 - { id = "RUSTSEC-2024-0436", reason = "paste unmaintained; transitive only; tracked for upgrade" }, # `rsa` Marvin timing sidechannel (RUSTSEC-2023-0071). Pulled in via # `openidconnect` (transitive) and historically used directly. No upstream # fix is available yet. Tracked separately for follow-up; remove this diff --git a/docs/README.md b/docs/README.md index d30e3758a..958dbda58 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,6 +8,9 @@ collection: ## Operations +For PUT cleanup admission, blocking-pool isolation, and optional CPU affinity, +see [PUT cleanup isolation](operations/put-cleanup-isolation.md). + - [External certificate issuance and TLS rotation](operations/tls-certificate-rotation.md) — publication, native reload, trust separation and failure handling. - [Architecture guard troubleshooting](operations/architecture-guard-troubleshooting.md) — diagnose repository guard failures. diff --git a/docs/operations/put-cleanup-isolation.md b/docs/operations/put-cleanup-isolation.md new file mode 100644 index 000000000..a9adaca5b --- /dev/null +++ b/docs/operations/put-cleanup-isolation.md @@ -0,0 +1,195 @@ +# PUT and disk cleanup isolation + +Cleanup can use process-wide admission budgets and a dedicated Tokio runtime. +This is opt-in. It preserves cleanup receipts, eligible-data moves to trash, +PUT temporary workspace cleanup, and periodic physical trash removal. It does +not disable healing or skip cleanup. + +## Configuration + +| Variable | Default | Meaning when isolation is enabled | +| --- | --- | --- | +| `RUSTFS_CLEANUP_ISOLATE_ENABLE` | `false` | Enable admission and execution isolation on this node. | +| `RUSTFS_PUT_RENAME_TAIL_CLEANUP_MAX_PENDING` | `1024` | Maximum admitted PUT ownership groups, including surviving commit and cleanup owners. | +| `RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS` | `2` | Maximum active PUT cleanup continuations. Also controls existing deferred queue workers; does not determine OS thread count. | +| `RUSTFS_CLEANUP_ASYNC_THREADS` | up to `2` | Dedicated async worker threads, limited by available parallelism and an explicit CPU mask by default. Independent of logical job concurrency. | +| `RUSTFS_CLEANUP_DISK_MAX_PENDING` | `16 × disk workers` (`64`) | Maximum admitted local disk cleanup operations, including queued and running operations. Shared by local requests, incoming RPC cleanup and periodic GC steps. | +| `RUSTFS_CLEANUP_DISK_WORKERS` | blocking threads (`4`) | Maximum active disk cleanup operations within that budget. | +| `RUSTFS_CLEANUP_GC_WORKERS` | `1` | Maximum periodic disk scans producing GC steps. | +| `RUSTFS_CLEANUP_BLOCKING_THREADS` | `4` | Maximum additional blocking threads on the cleanup runtime. | +| `RUSTFS_CLEANUP_CPUS` | `1-2` on Linux; unbound elsewhere | CPU affinity for cleanup async and blocking threads. Override with a CPU list such as `4-5,8`, or `none` to disable affinity while keeping isolation. | +| `RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ENABLE` | `false` | Use the existing bounded deferred queue before coordinator execution. Isolation also works without this queue. | +| `RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY` | `1024` | Capacity of that existing queue; this is not the total admission budget. | + +Counts must be positive. PUT workers must not exceed PUT pending capacity; +GC workers must not exceed disk workers, which must not exceed disk pending +capacity. Async and blocking thread counts must not exceed 1024. CPU IDs must be supported by the affinity implementation and belong +to the initiating thread's allowed CPU set. Non-Linux hosts reject explicit CPU +masks. Configuration is read once at first use, including disk cleanup or GC; +changes require a restart. Invalid configuration rejects new cleanup admission. +PUT checks configuration before staging data. + +Isolation remains disabled by default. When enabling it on Linux, the default +mask requires logical CPU IDs 1 and 2 to be available inside the process/container +CPU set. On smaller or restricted CPU sets, configure permitted CPU IDs or `none` +before enabling isolation. An unavailable default mask is an error; it does not +silently select different CPUs. Configure each node separately. + +Do not combine isolation with cleanup counterfactuals +`RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP`, +`RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER`, or +`RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP`. These combinations +are rejected because they deliberately suppress cleanup or stop queue progress. + +Do not enable `RUSTFS_PUT_RENAME_TAIL_CLEANUP_LANE_ENABLE` together with +`RUSTFS_CLEANUP_ISOLATE_ENABLE`. The legacy per-disk lane adds a second queue +inside the isolated coordinator budget, so a slow lane can hold coordinator +capacity while unrelated cleanup waits. This combination is rejected before +PUT admission; either scheduler can still be enabled on its own. + +## Ownership and execution + +PUT attempts admission before staging. Full admission returns `SlowDown`, so +clients should retry with backoff. Admission does not wait: COPY readers and +data-movement callers can already hold locks needed by admitted PUTs. No body +is ingested or tmp data staged by a rejected PUT. + +The request, detached commit owner, and cleanup continuations share one +reservation. An early ACK or cancelled waiter cannot release it while an owner +still needs cleanup. After the existing rename-convergence and guard-release +handoff, deferred workers, queue overflow and direct continuations all acquire +the same coordinator execution semaphore. Queue overflow cannot bypass that +limit. Awaited failure cleanup and no-tail tmp cleanup use the same budget. +Full-tail receipt persistence remains awaited at its existing handoff point; +old-data reclamation remains after the existing guard release. Incomplete +rollback still preserves staging for recovery. + +Each local disk operation has separate pending and active permits. The wrapper +below RPC authentication uses this gate for low-level `delete`, `delete_data_dir` +and old-data cleanup receipt `write_all`. Local PUT fan-out and incoming peer +RPCs therefore share the same node-local disk budget. Receipt classification +uses the internal receipt filename; ordinary writes retain their current path. +RPC schemas and authentication are unchanged. Enable isolation on every node +to isolate peer execution; settings and CPU masks are not transmitted by RPC. + +Coordinator and receiver permits are deliberately distinct: a coordinator can +wait for peer RPCs without occupying the slots those peers need to service disk +cleanup. Full disk admission returns an I/O `WouldBlock` error through the +existing disk error path. A mutation carrying an external publication or +namespace owner also refuses a full active budget instead of waiting while it +holds a lock. Existing cleanup error handling and receipt recovery still apply. + +Ordinary snapshot-lease release stays on the caller and only updates the in-memory +registry. It does not allocate a cleanup task or wait for the cleanup scheduler. +Invalid cleanup configuration also must not strand a reader token: release still +updates the registry, while any deferred reclamation reports the initialization +error and keeps both its intent and data. It does not fall back to unbudgeted I/O. +If the last release triggers deferred old-data +reclamation, that reclamation tries the same pending and active disk permits. +Only this case dispatches an owned task to the cleanup runtime. +Saturation leaves the token released and the cleanup intent/receipt available +for retry; it does not strand a live reader token or bypass the budget. Quota +fence release keeps its original receiver lifecycle and does not acquire disk +execution permits, since it can wait for mutations that already own them. + +An admitted request waits for disk execution capacity in the caller's existing +future. Cancellation or timeout discards this unstarted operation and releases +pending capacity; it cannot later start deleting a reused path. After execution +capacity is acquired, the worker also checks whether its result receiver closed +before its first poll. A job cancelled while waiting for the runtime to schedule +it is discarded when the worker polls it, releasing its permits without starting +I/O. Once the worker starts the operation, the owned job retains its permits and +disk reference until completion. Cancelling or timing out its waiter does not +free disk capacity while that started job still runs. +This also retains the disk's mount ownership during detached execution. Cleanup +waiter timeouts include queue delay and do not by themselves mark a disk faulty; +the existing active disk-health probe retains its timeout policy. + +Periodic trash and stale-tmp scans have a separate scan limit. Each scan acquires +the shared disk budget to open its directory and then for each entry, releasing +it between entries. The steps borrow the scan's iterator and paths and run in +the same owned task, without a spawn/join or path clones per entry. A scan cannot occupy a disk execution slot while waiting +for another scan slot. GC waits for admission without holding object locks; +request admission is bounded without creating waiting producer tasks. A single +trash subtree is still removed as one operation, so a very large subtree can +hold a slot for its entire removal. Queued scans retain the original mount +handle even when the corresponding disk object is dropped. +Missed periodic ticks are skipped so a slow scan does not cause a burst of +catch-up scans. + +Cleanup runs on `rustfs-cleanup` threads. Nested Tokio tasks, Tokio filesystem +work and ordinary `spawn_blocking` use that runtime. Explicit fsync dispatch +also stays on its blocking pool. File/directory group-commit keys separate +cleanup and foreground producers, preventing batches from moving into the +other class's runtime. Fsync durability settings remain unchanged. + +The blocking limit bounds simultaneous blocking execution, not Tokio's internal +queue or kernel I/O. Disk admission bounds selected operations, not bytes, +syscall count or device IOPS. Existing synchronous path probes and disk-full +delete fallbacks can still run on the dedicated async workers, under the disk +operation budget. All classes still share the filesystem, journal, +page cache and devices with foreground work. + +If CPU binding fails at thread startup, new admissions fail and the failure is +logged; already-admitted cleanup can finish. Affinity does not reserve exclusive +cores, move foreground threads elsewhere, change cgroups, or control kernel +workers and interrupt affinity. + +## Scope and recovery + +The limits are per process, not cluster-wide quotas. This is not a durable work +queue. Existing cleanup receipts and tmp/trash recovery retain their role after +process exit. The option does not bound all storage mutations: metadata/version +DELETE transactions, heal/scanner coordination and +multipart orchestration retain their scheduling. Calls from those paths that +reach the selected disk wrapper entries do share the disk budget; internal +LocalDisk operations that bypass those entries do not. + +## Measurement and tuning + +On Linux, `cargo test -p rustfs-ecstore --test cleanup_isolation_test` exercises +the public disk facade, actual thread affinity, receipt/delete execution and +periodic physical trash removal with temporary disks. It does not benchmark +throughput. + +With PUT stage metrics enabled, observe: + +- `rustfs_cleanup_admitted_puts` and `rustfs_cleanup_admission_rejected_total`; +- `rustfs_cleanup_execution_wait_seconds`; +- `rustfs_cleanup_disk_pending`, `rustfs_cleanup_disk_active`, and + `rustfs_cleanup_disk_admission_rejected_total`; +- `rustfs_cleanup_disk_execution_wait_seconds` and `rustfs_cleanup_gc_active`; +- existing deferred queue depth, age, overflow, completion and service time. + +Compare isolation disabled, budget/pool isolation without CPU affinity, and +isolation with affinity. Keep durability, object sizes, overwrite ratio, +concurrency and cluster placement fixed. Measure PUT/GET throughput and p95/p99 +latency, rejections, queue age, receipt/trash growth, per-disk I/O latency, CPU +usage and context switches. Include a long steady-state interval covering +physical GC, and verify backlog drains afterward. + +Start with small worker and blocking-thread limits, then increase only when +cleanup cannot keep up without harming foreground latency. Too few workers or +CPUs can increase rejections and lower PUT throughput. Larger queues do not +increase sustainable device throughput. Affinity helps only when CPU scheduling +contention matters to the measured workload; no throughput gain is guaranteed. + +The defaults protect a node; they are not a claim of optimal production throughput. +The default disk queue permits sixteen service waves, rather than 256 waves with +1024 pending operations and four workers. For illustration, at a uniform 100 ms +per operation, these bounds correspond to roughly 1.6 s versus 25.6 s to drain a +full queue with no new arrivals. Real service times, stalls and disk fan-out vary; +there is no queue latency guarantee. The ordinary disk waiter timeout defaults to +30 s and does not cancel already-admitted work. + +Size pending capacity for measured arrival bursts and local/peer fan-out, not +only for PUT request concurrency. For example, two coordinator jobs each touching +16 local disks can submit 32 disk operations before incoming peer requests and +GC are counted. Sixty-four slots need not absorb every distributed burst. Increase +the explicit limit when measured rejection rates require it, while keeping queue +age comfortably below waiter deadlines. On many-disk nodes, measure whether four +active operations underutilize devices before increasing disk/blocking workers. +The 1024 PUT admission limit includes uploading requests and detached owners; +it is an object-count bound, not a memory or byte budget. The older deferred queue +remains optional and disabled by default; its overflow path cannot bypass the +new coordinator or disk limits. No unbounded foreground fallback is introduced. diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index 1a23d64dd..8641d2c05 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -525,10 +525,13 @@ impl Operation for ImportBucketMetadata { // First pass: read all file contents into memory let mut file_contents = Vec::new(); for i in 0..zip_reader.len() { - let mut file = zip_reader + let (mut file, file_path) = zip_reader .by_index(i) + .and_then(|file| { + let path = file.name()?.into_owned(); + Ok((file, path)) + }) .map_err(|e| s3_error!(InternalError, "failed to read archive entry: {e}"))?; - let file_path = file.name().to_string(); let mut content = Vec::new(); file.read_to_end(&mut content) @@ -1564,7 +1567,7 @@ mod backup_zip_compatibility_tests { let mut content = Vec::new(); file.read_to_end(&mut content).expect("read diagnostic config"); assert!(!content.windows(RAW_SECRET.len()).any(|window| window == RAW_SECRET)); - files.insert(file.name().to_string(), content); + files.insert(file.name().expect("diagnostic entry name").into_owned(), content); } assert!(!files.contains_key(&format!("_diagnostic/{BUCKET}/{config_file}"))); assert_eq!(files[&format!("_diagnostic/{BUCKET}/{BUCKET_VERSIONING_CONFIG}")], VERSIONING_XML); @@ -1701,8 +1704,9 @@ mod backup_zip_compatibility_tests { let mut content = Vec::new(); file.read_to_end(&mut content).expect("read diagnostic config"); assert!(!content.windows(SECRET.len()).any(|window| window == SECRET.as_bytes())); - assert!(file.name() == DIAGNOSTIC_EXPORT_MANIFEST || file.name().starts_with("_diagnostic/")); - files.insert(file.name().to_string(), content); + let file_path = file.name().expect("diagnostic entry name").into_owned(); + assert!(file_path == DIAGNOSTIC_EXPORT_MANIFEST || file_path.starts_with("_diagnostic/")); + files.insert(file_path, content); } assert_eq!(files[&format!("_diagnostic/{HEALTHY}/{BUCKET_VERSIONING_CONFIG}")], VERSIONING_XML); assert!(files.contains_key(&format!("_diagnostic/{UNREADABLE}/{BUCKET_VERSIONING_CONFIG}"))); diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs index b26b72103..eab0a72bf 100644 --- a/rustfs/src/app/object/get.rs +++ b/rustfs/src/app/object/get.rs @@ -277,6 +277,16 @@ const GET_OBJECT_STAGE_LIFECYCLE_EXPIRATION: &str = "lifecycle_expiration"; const GET_OBJECT_STAGE_METADATA_FILTER: &str = "metadata_filter"; +const GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT: &str = "body_cache_hook_hit"; +const GET_OBJECT_STAGE_BODY_CACHE_LOOKUP: &str = "body_cache_lookup"; +const GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF: &str = "body_cache_fill_handoff"; +const GET_OBJECT_STAGE_BODY_BUFFERED_HANDOFF: &str = "body_buffered_handoff"; +const GET_OBJECT_STAGE_OUTPUT_STRUCT_BUILD: &str = "output_struct_build"; +const GET_OBJECT_STAGE_PREPARE_READ_EXECUTION: &str = "prepare_read_execution"; +const GET_OBJECT_STAGE_RESPONSE_CORS_WRAP: &str = "response_cors_wrap"; +const GET_OBJECT_STAGE_RESPONSE_HEADER_INJECT: &str = "response_header_inject"; +const GET_OBJECT_STAGE_RESPONSE_EVENT_COMPLETE: &str = "response_event_complete"; + const GET_OBJECT_STREAM_WARN_THRESHOLD: Duration = Duration::from_secs(5); static GET_OBJECT_BUFFER_THRESHOLD_WARNED: AtomicBool = AtomicBool::new(false); @@ -555,10 +565,10 @@ fn get_object_stream_failure_reason(error_class: &'static str) -> &'static str { } } -/// A mid-stream reopen can recover only when another representation of the -/// committed object may exist. A single-disk, no-parity object has no alternate -/// shard after a bitrot/read-quorum failure, so retrying the same read only -/// repeats the failure and adds latency. +/// A mid-stream reopen can only recover when another representation of the +/// committed object may exist. Single-disk, no-parity objects have no alternate +/// shard to discover after a bitrot/read-quorum failure, so arming the normal +/// three-attempt resume loop only repeats the same failing read and adds delay. fn should_attach_get_object_resume(info: &ObjectInfo) -> bool { info.parity_blocks > 0 || info.data_blocks > 1 || !info.transitioned_object.tier.is_empty() } @@ -3470,12 +3480,15 @@ impl DefaultObjectUsecase { // already supplied this body, the request-level plan was built before // the authoritative lookup. Serve it without planning a second time. if cache_hook_served && let Some(bytes) = buffered_body.take() { - return Ok(Self::build_memory_bytes_blob( + let hook_hit_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); + let body = Self::build_memory_bytes_blob( bytes, response_content_length, GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE, lifecycle, - )); + ); + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT, hook_hit_start); + return Ok(body); } if !cache_fill_allowed { @@ -3516,20 +3529,27 @@ impl DefaultObjectUsecase { // authoritative because it ran after fresh metadata resolution, so the // app layer skips its own lookup and only uses the plan to fill. if !cache_hook_probed { - match lookup_get_object_body_cache_hit(cache_adapter, &cache_plan).await { + let cache_lookup_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); + let cache_lookup = lookup_get_object_body_cache_hit(cache_adapter, &cache_plan).await; + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_LOOKUP, cache_lookup_start); + match cache_lookup { GetObjectBodyCacheLookup::Hit(bytes) => { - return Ok(Self::build_memory_bytes_blob( + let cache_hit_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); + let body = Self::build_memory_bytes_blob( bytes, response_content_length, GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE, lifecycle, - )); + ); + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT, cache_hit_start); + return Ok(body); } GetObjectBodyCacheLookup::Disabled | GetObjectBodyCacheLookup::Skip | GetObjectBodyCacheLookup::Miss => {} } } if let Some(buffered_body) = buffered_body { + let cache_fill_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); // ODC-15: the body is already fully in hand, so keep the fill off the // response's critical path. For a cacheable plan, run the fill in a // detached task (Bytes is a cheap clone) and return immediately. For @@ -3553,13 +3573,17 @@ impl DefaultObjectUsecase { } let _ = fill_get_object_body_cache_from_buffered_body(cache_adapter, &cache_plan, &buffered_body).await; } + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF, cache_fill_handoff_start); - return Ok(Self::build_memory_bytes_blob( + let buffered_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); + let body = Self::build_memory_bytes_blob( buffered_body, response_content_length, GET_MEMORY_BODY_SOURCE_BUFFERED_BODY, lifecycle, - )); + ); + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_BUFFERED_HANDOFF, buffered_handoff_start); + return Ok(body); } let should_materialize_for_cache = cache_adapter.materialize_fill_enabled() @@ -3613,6 +3637,7 @@ impl DefaultObjectUsecase { { Ok(buf) => { let bytes = Bytes::from(buf); + let cache_fill_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); // ODC-15: fill off the response's critical path (see the // buffered-body branch above). let cache_adapter = cache_adapter.clone(); @@ -3625,6 +3650,10 @@ impl DefaultObjectUsecase { } let _ = fill_get_object_body_cache_from_materialized_body(&cache_adapter, &cache_plan, &fill_bytes).await; }); + record_get_object_s3_handler_stage_duration( + GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF, + cache_fill_handoff_start, + ); return Ok(Self::build_memory_bytes_blob( bytes, @@ -3763,13 +3792,19 @@ impl DefaultObjectUsecase { None => helper, }; let helper = helper.version_id(version_id_for_event); + let cors_wrap_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let mut response = wrap_response_with_cors(bucket, method, headers, output).await; + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_CORS_WRAP, cors_wrap_start); + let header_inject_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); inject_accept_ranges_header(&mut response.headers); // Emit XXHash3/64/128 and SHA-512 checksums that s3s GetObjectOutput cannot // carry (#1257). This is the download-side integrity path AWS SDKs verify. inject_additional_checksum_headers(&mut response.headers, &extra_checksum_headers); + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_HEADER_INJECT, header_inject_start); let result = Ok(response); + let event_complete_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let _ = helper.complete(&result); + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_EVENT_COMPLETE, event_complete_start); result } @@ -3879,9 +3914,8 @@ impl DefaultObjectUsecase { let metadata_filter_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let metadata = filter_object_metadata(&info.user_defined); record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_METADATA_FILTER, metadata_filter_start); - + let output_struct_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let parts_count = Self::get_object_parts_count(part_number, &info); - let output = GetObjectOutput { body: Some(body), content_length: Some(response_content_length), @@ -3915,6 +3949,7 @@ impl DefaultObjectUsecase { storage_class, ..Default::default() }; + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_OUTPUT_STRUCT_BUILD, output_struct_start); Ok(GetObjectOutputContext { output, @@ -4217,6 +4252,7 @@ impl DefaultObjectUsecase { let manager = get_concurrency_manager(); + let prepare_read_start = stage_metrics_enabled.then(std::time::Instant::now); let mut prepared_read = self .prepare_get_object_read_execution( &req, @@ -4232,6 +4268,7 @@ impl DefaultObjectUsecase { object_traffic_health.clone(), ) .await; + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_PREPARE_READ_EXECUTION, prepare_read_start); // An object missing locally (and only missing — other errors keep // their semantics) may still be served by a remote copy. if let Err(err) = &prepared_read @@ -4275,6 +4312,7 @@ impl DefaultObjectUsecase { return Self::complete_get_object_error(helper.version_id(version_id_for_event), err); } Some(OdmGetOutcome::RetryLocal) => { + let prepare_read_retry_start = stage_metrics_enabled.then(std::time::Instant::now); prepared_read = self .prepare_get_object_read_execution( &req, @@ -4290,6 +4328,10 @@ impl DefaultObjectUsecase { object_traffic_health, ) .await; + record_get_object_s3_handler_stage_duration( + GET_OBJECT_STAGE_PREPARE_READ_EXECUTION, + prepare_read_retry_start, + ); } } } diff --git a/rustfs/src/app/object/put.rs b/rustfs/src/app/object/put.rs index 9bff39cf2..d182ab1bf 100644 --- a/rustfs/src/app/object/put.rs +++ b/rustfs/src/app/object/put.rs @@ -1788,6 +1788,7 @@ impl DefaultObjectUsecase { )?; } + let checksum_input_stage_start = put_stage_metrics_enabled.then(Instant::now); let mut md5hex = match content_md5 { Some(PutObjectContentMd5::Base64(base64_md5)) => { let md5 = base64_simd::STANDARD @@ -1800,12 +1801,15 @@ impl DefaultObjectUsecase { }; let mut sha256hex = get_content_sha256_with_query(headers, query); + rustfs_io_metrics::record_put_object_stage_duration_from("app_checksum_input_prepare", checksum_input_stage_start); let mut write_plan = WritePlan::new(); // Additional-checksum (XXHash3/64/128, SHA-512) values to echo on the PutObject // response (#1256); captured at want_checksum set points before opts is moved. let mut put_extra_checksum_headers: Vec<(&'static str, String)> = Vec::new(); + let reader_prepare_stage_start = put_stage_metrics_enabled.then(Instant::now); let mut reader = if should_compress { + let compressed_reader_stage_start = put_stage_metrics_enabled.then(Instant::now); let body = tokio::io::BufReader::with_capacity( buffer_size, StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))), @@ -1828,51 +1832,98 @@ impl DefaultObjectUsecase { size = HashReader::SIZE_PRESERVE_LAYER; write_plan = write_plan.with_compression(algorithm); + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_reader_prepare_compressed", + compressed_reader_stage_start, + ); hrd } else { if use_zero_copy_eager_put_path { let zero_copy_start = std::time::Instant::now(); + let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now); let eager_body = read_zero_copy_put_body_exact(body, body_size as usize).await?; + rustfs_io_metrics::record_put_object_stage_duration_from("app_body_eager_read_zero_copy", eager_read_stage_start); rustfs_io_metrics::record_zero_copy_write(body_size as usize, zero_copy_start.elapsed().as_secs_f64() * 1000.0); - HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? + let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now); + let reader = + HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_hash_reader_setup_zero_copy", + hash_reader_stage_start, + ); + reader } else if use_empty_or_small_eager_put_path { if (body_size as usize) <= POOL_BYPASS_MAX_SIZE { // Bypass BytesPool for very small objects to avoid Small-tier // Mutex contention under high concurrency. Direct allocation // for ≤4KiB is negligible cost. + let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now); let eager_body = read_small_put_body_exact_direct( StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))), body_size as usize, ) .await?; - HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_body_eager_read_small_direct", + eager_read_stage_start, + ); + let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now); + let reader = HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false) + .map_err(ApiError::from)?; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_hash_reader_setup_small_direct", + hash_reader_stage_start, + ); + reader } else { let pool = get_concurrency_manager().bytes_pool(); + let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now); let eager_body = read_small_put_body_exact_pooled( StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))), body_size as usize, pool.as_ref(), ) .await?; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_body_eager_read_small_pooled", + eager_read_stage_start, + ); + let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now); let eager_reader = PooledBufferReader::new(eager_body, body_size as usize); - HashReader::from_stream(eager_reader, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? + let reader = HashReader::from_stream(eager_reader, size, actual_size, md5hex, sha256hex, false) + .map_err(ApiError::from)?; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_hash_reader_setup_small_pooled", + hash_reader_stage_start, + ); + reader } } else { + let streaming_reader_stage_start = put_stage_metrics_enabled.then(Instant::now); let body = tokio::io::BufReader::with_capacity( buffer_size, StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))), ); - HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? + let reader = + HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_reader_prepare_streaming", + streaming_reader_stage_start, + ); + reader } }; + rustfs_io_metrics::record_put_object_stage_duration_from("app_reader_prepare", reader_prepare_stage_start); if size >= 0 { + let checksum_setup_stage_start = put_stage_metrics_enabled.then(Instant::now); if let Err(err) = reader.add_checksum(headers, trailer_source(trailing_headers.clone()), false) { return Err(ApiError::from(err).into()); } opts.want_checksum = reader.checksum(); put_extra_checksum_headers = additional_checksum_echo_pairs(&opts.want_checksum); + rustfs_io_metrics::record_put_object_stage_duration_from("app_reader_checksum_setup", checksum_setup_stage_start); } rustfs_io_metrics::record_put_object_path(put_path); rustfs_io_metrics::record_put_object_stage_duration_from("ingress_prepare", ingress_stage_start); @@ -2113,17 +2164,36 @@ impl DefaultObjectUsecase { wait_for_put_post_store_test_hook(&bucket).await; let post_store_stage_start = put_stage_metrics_enabled.then(Instant::now); + let transition_enqueue_stage_start = put_stage_metrics_enabled.then(Instant::now); maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await; - let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_post_store_transition_enqueue", + transition_enqueue_stage_start, + ); + let cache_invalidate_stage_start = put_stage_metrics_enabled.then(Instant::now); + let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_post_store_data_cache_invalidate", + cache_invalidate_stage_start, + ); + + let versioning_lookup_stage_start = put_stage_metrics_enabled.then(Instant::now); let put_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_post_store_versioning_lookup", + versioning_lookup_stage_start, + ); // Fast in-memory update for immediate quota and admin usage consistency. // The previous current size comes from the prelookup when it ran, // otherwise from the rename_data backfill (rustfs/backlog#1009); the // backfill reproduces the lookup's observation bit for bit (latest // version's ObjectInfo.size — 0 for a delete-marker latest — or // not-found → None). + let quota_size_stage_start = put_stage_metrics_enabled.then(Instant::now); let committed_size = quota_accounting_object_size(&obj_info, quota_enabled)?; + rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_quota_size", quota_size_stage_start); + let usage_memory_stage_start = put_stage_metrics_enabled.then(Instant::now); match prelookup_previous_current_size.or_else(|| previous_current_size_from_backfill(backfilled_old_current_size)) { Some(previous_current_size) => { @@ -2149,16 +2219,27 @@ impl DefaultObjectUsecase { record_bucket_object_write_unknown_previous_memory(&bucket, committed_size, put_versioned).await; } } + rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_usage_memory", usage_memory_stage_start); if dsc.replicate_any() { + let replication_schedule_stage_start = put_stage_metrics_enabled.then(Instant::now); schedule_object_replication(obj_info.clone(), store, dsc).await; + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_post_store_replication_schedule", + replication_schedule_stage_start, + ); } + let dirty_usage_stage_start = put_stage_metrics_enabled.then(Instant::now); rustfs_scanner::record_dirty_usage_object_from_producer( &bucket, &key, rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject, ); + rustfs_io_metrics::record_put_object_stage_duration_from( + "app_post_store_dirty_usage_record", + dirty_usage_stage_start, + ); rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_bookkeeping", post_store_stage_start); let capacity_update_stage_start = put_stage_metrics_enabled.then(Instant::now); diff --git a/rustfs/src/connect/report_bundle.rs b/rustfs/src/connect/report_bundle.rs index 890629ac2..454626261 100644 --- a/rustfs/src/connect/report_bundle.rs +++ b/rustfs/src/connect/report_bundle.rs @@ -76,7 +76,11 @@ pub(crate) fn upload_source( Err(_) => return original_source(path, generated_bundle_uid), }; let names = (0..archive.len()) - .map(|index| archive.by_index(index).map(|entry| entry.name().to_owned())) + .map(|index| { + archive + .by_index(index) + .and_then(|entry| entry.name().map(|name| name.into_owned())) + }) .collect::, _>>()?; if names diff --git a/rustfs/src/server/hybrid.rs b/rustfs/src/server/hybrid.rs index c20d47215..79686946e 100644 --- a/rustfs/src/server/hybrid.rs +++ b/rustfs/src/server/hybrid.rs @@ -19,6 +19,7 @@ use hyper::{Request, Response}; use pin_project_lite::pin_project; use std::pin::Pin; use std::task::{Context, Poll}; +use std::time::Instant; use tower::Service; type BoxError = Box; @@ -35,6 +36,13 @@ pub(crate) fn is_grpc_request(req: &Request) -> bool { ) } +fn rename_data_grpc_stage(path: &str) -> bool { + matches!( + path, + "/node_service.NodeService/RenameData" | "/node_service.NodeService/RenameDataAtIncarnation" + ) +} + /// The service that can serve both gRPC and REST HTTP Requests #[derive(Clone)] pub struct HybridService { @@ -71,8 +79,11 @@ where /// as a REST request fn call(&mut self, req: Request) -> Self::Future { if is_grpc_request(&req) { + let observe_rename_data = rustfs_io_metrics::put_stage_metrics_enabled() && rename_data_grpc_stage(req.uri().path()); HybridFuture::Grpc { grpc_future: self.grpc.call(req), + observe_rename_data, + service_future_started: observe_rename_data.then(Instant::now), } } else { HybridFuture::Rest { @@ -93,7 +104,10 @@ pin_project! { }, Grpc { #[pin] - grpc_body: GrpcBody + grpc_body: GrpcBody, + first_frame_started: Option, + body_started: Option, + first_frame_recorded: bool, }, } } @@ -123,21 +137,44 @@ where fn is_end_stream(&self) -> bool { match self { Self::Rest { rest_body } => rest_body.is_end_stream(), - Self::Grpc { grpc_body } => grpc_body.is_end_stream(), + Self::Grpc { grpc_body, .. } => grpc_body.is_end_stream(), } } fn poll_frame(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll, Self::Error>>> { match self.project() { HybridBodyProj::Rest { rest_body } => rest_body.poll_frame(cx).map_err(Into::into), - HybridBodyProj::Grpc { grpc_body } => grpc_body.poll_frame(cx).map_err(Into::into), + HybridBodyProj::Grpc { + mut grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + } => { + let poll = grpc_body.as_mut().poll_frame(cx).map_err(Into::into); + if !*first_frame_recorded && let Poll::Ready(_) = &poll { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME, + first_frame_started.take(), + ); + *first_frame_recorded = true; + } + let body_complete = matches!(&poll, Poll::Ready(None)) + || (matches!(&poll, Poll::Ready(Some(Ok(_)))) && grpc_body.as_ref().get_ref().is_end_stream()); + if body_complete { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE, + body_started.take(), + ); + } + poll + } } } fn size_hint(&self) -> http_body::SizeHint { match self { Self::Rest { rest_body } => rest_body.size_hint(), - Self::Grpc { grpc_body } => grpc_body.size_hint(), + Self::Grpc { grpc_body, .. } => grpc_body.size_hint(), } } } @@ -154,6 +191,8 @@ pin_project! { Grpc { #[pin] grpc_future: GrpcFuture, + observe_rename_data: bool, + service_future_started: Option, }, } } @@ -174,8 +213,27 @@ where Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())), Poll::Pending => Poll::Pending, }, - HybridFutureProj::Grpc { grpc_future } => match grpc_future.poll(cx) { - Poll::Ready(Ok(res)) => Poll::Ready(Ok(res.map(|grpc_body| HybridBody::Grpc { grpc_body }))), + HybridFutureProj::Grpc { + grpc_future, + observe_rename_data, + service_future_started, + } => match grpc_future.poll(cx) { + Poll::Ready(Ok(res)) => { + if *observe_rename_data { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE, + service_future_started.take(), + ); + } + let first_frame_started = (*observe_rename_data).then(Instant::now); + let body_started = (*observe_rename_data).then(Instant::now); + Poll::Ready(Ok(res.map(|grpc_body| HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded: !*observe_rename_data, + }))) + } Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())), Poll::Pending => Poll::Pending, }, diff --git a/rustfs/src/server/layer.rs b/rustfs/src/server/layer.rs index cff0e72a8..1cbbeefbc 100644 --- a/rustfs/src/server/layer.rs +++ b/rustfs/src/server/layer.rs @@ -863,7 +863,20 @@ where } Response::from_parts(parts, HybridBody::Rest { rest_body }) } - HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }), + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + } => Response::from_parts( + parts, + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + }, + ), }; Ok(response) @@ -1012,7 +1025,20 @@ where } Response::from_parts(parts, HybridBody::Rest { rest_body }) } - HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }), + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + } => Response::from_parts( + parts, + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + }, + ), }; Ok(response) @@ -1163,7 +1189,20 @@ where Response::from_parts(parts, HybridBody::Rest { rest_body }) } - HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }), + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + } => Response::from_parts( + parts, + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + }, + ), }; Ok(response) @@ -1234,7 +1273,20 @@ where }, ) } - HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }), + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + } => Response::from_parts( + parts, + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + }, + ), }; Ok(response) @@ -1302,7 +1354,20 @@ where rest_body: RestBody::from(Bytes::new()), }, ), - HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }), + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + } => Response::from_parts( + parts, + HybridBody::Grpc { + grpc_body, + first_frame_started, + body_started, + first_frame_recorded, + }, + ), }; Ok(response) diff --git a/rustfs/src/startup_observability.rs b/rustfs/src/startup_observability.rs index 01ec050b8..114b1a808 100644 --- a/rustfs/src/startup_observability.rs +++ b/rustfs/src/startup_observability.rs @@ -13,12 +13,16 @@ // limitations under the License. use std::sync::Arc; +use std::time::Duration; use crate::init::{init_auto_tuner, init_update_check, print_server_info}; use crate::startup_runtime_sources; use crate::storage_api::startup::storage::{ECStore, init_compression_total_memory_from_backend}; +use tokio::io::AsyncWriteExt; use tokio_util::sync::CancellationToken; +const GET_STAGE_LOCAL_SUMMARY_FILE: &str = "/var/log/rustfs/get-stage-summary.log"; + pub(crate) async fn init_observability_runtime(store: Arc, ctx: CancellationToken) { print_server_info(); init_update_check(); @@ -27,6 +31,10 @@ pub(crate) async fn init_observability_runtime(store: Arc, ctx: Cancell let metrics_enabled = startup_runtime_sources::observability_metric_enabled(); configure_metric_gates(metrics_enabled); + if rustfs_io_metrics::get_stage_local_summary_enabled() { + init_get_stage_local_summary_runtime(ctx.clone()); + } + if metrics_enabled { // Load persisted compression stats into memory early, before any PUTs can occur. init_compression_total_memory_from_backend(store).await; @@ -37,21 +45,115 @@ pub(crate) async fn init_observability_runtime(store: Arc, ctx: Cancell } fn configure_metric_gates(metrics_enabled: bool) { + let local_summary_enabled = !metrics_enabled + && rustfs_utils::get_env_bool(rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED, false); + let sample_rate = + rustfs_utils::get_env_u64(rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE, 64).max(1); + startup_runtime_sources::set_get_stage_local_summary_enabled(local_summary_enabled); + startup_runtime_sources::set_get_stage_local_summary_sample_rate(sample_rate); let put_stage_metrics_enabled = metrics_enabled && rustfs_utils::get_env_bool( rustfs_config::observability::ENV_OBS_PUT_STAGE_METRICS_ENABLED, rustfs_config::DEFAULT_OBS_PUT_STAGE_METRICS_ENABLED, ); startup_runtime_sources::set_put_stage_metrics_enabled(put_stage_metrics_enabled); - startup_runtime_sources::set_get_stage_metrics_enabled(metrics_enabled); + startup_runtime_sources::set_get_stage_metrics_enabled(metrics_enabled || local_summary_enabled); startup_runtime_sources::set_metrics_enabled(metrics_enabled); } +fn init_get_stage_local_summary_runtime(ctx: CancellationToken) { + let interval_secs = + rustfs_utils::get_env_u64(rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS, 15).max(1); + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); + interval.tick().await; + loop { + tokio::select! { + _ = ctx.cancelled() => break, + _ = interval.tick() => { + let rows = rustfs_io_metrics::take_get_stage_local_summary(); + if rows.is_empty() { + continue; + } + let summary = rows.iter().map(|row| { + format!("{}|{}|{}|{}|{}|{}", row.path, row.stage, row.object_class, row.size_bucket, row.sampled_count, row.duration_nanoseconds) + }).collect::>().join(";"); + let timestamp_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + let line = format!( + "timestamp_ms={timestamp_ms}\tsample_rate={}\tstage_count={}\tstages={summary}\n", + rustfs_io_metrics::get_stage_local_summary_sample_rate(), + rows.len(), + ); + let Ok(mut file) = tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(GET_STAGE_LOCAL_SUMMARY_FILE) + .await + else { + break; + }; + if file.write_all(line.as_bytes()).await.is_err() { + break; + } + } + } + } + }); +} + #[cfg(test)] mod tests { use super::*; const PUT_STAGE_ENV: &str = rustfs_config::observability::ENV_OBS_PUT_STAGE_METRICS_ENABLED; + const LOCAL_SUMMARY_ENV: &str = rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED; + const LOCAL_SUMMARY_RATE_ENV: &str = rustfs_config::observability::ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE; + + #[test] + #[serial_test::serial] + fn get_stage_local_summary_can_run_without_metrics_export() { + let previous_metrics = rustfs_io_metrics::metrics_enabled(); + let previous_get_stages = rustfs_io_metrics::get_stage_metrics_enabled(); + let previous_local_summary = rustfs_io_metrics::get_stage_local_summary_enabled(); + let previous_sample_rate = rustfs_io_metrics::get_stage_local_summary_sample_rate(); + + temp_env::with_var(LOCAL_SUMMARY_ENV, Some("true"), || { + temp_env::with_var(LOCAL_SUMMARY_RATE_ENV, Some("1"), || { + configure_metric_gates(false); + assert!(!rustfs_io_metrics::metrics_enabled()); + assert!(rustfs_io_metrics::get_stage_metrics_enabled()); + assert!(rustfs_io_metrics::get_stage_local_summary_enabled()); + rustfs_io_metrics::take_get_stage_local_summary(); + + rustfs_io_metrics::record_get_object_stage_duration("legacy_duplex", "metadata", 0.001); + rustfs_io_metrics::record_get_object_stage_duration("legacy_duplex", "metadata", 0.003); + rustfs_io_metrics::record_get_object_stage_duration_by_size( + "legacy_duplex", + "inline_direct.decode", + "plain_single_part", + "le_4kib", + 0.002, + ); + + let rows = rustfs_io_metrics::take_get_stage_local_summary(); + let metadata = rows + .iter() + .find(|row| row.path == "legacy_duplex" && row.stage == "metadata") + .expect("local summary should retain sampled metadata observations"); + assert_eq!(metadata.sampled_count, 2); + assert_eq!(metadata.duration_nanoseconds, 4_000_000); + assert!(rows.iter().any(|row| row.size_bucket == "le_4kib")); + }); + }); + + startup_runtime_sources::set_metrics_enabled(previous_metrics); + startup_runtime_sources::set_get_stage_metrics_enabled(previous_get_stages); + startup_runtime_sources::set_get_stage_local_summary_enabled(previous_local_summary); + startup_runtime_sources::set_get_stage_local_summary_sample_rate(previous_sample_rate); + } #[test] #[serial_test::serial] diff --git a/rustfs/src/startup_runtime_sources.rs b/rustfs/src/startup_runtime_sources.rs index 73501a592..f69546688 100644 --- a/rustfs/src/startup_runtime_sources.rs +++ b/rustfs/src/startup_runtime_sources.rs @@ -99,6 +99,14 @@ pub(crate) fn set_get_stage_metrics_enabled(enabled: bool) { rustfs_io_metrics::set_get_stage_metrics_enabled(enabled); } +pub(crate) fn set_get_stage_local_summary_enabled(enabled: bool) { + rustfs_io_metrics::set_get_stage_local_summary_enabled(enabled); +} + +pub(crate) fn set_get_stage_local_summary_sample_rate(sample_rate: u64) { + rustfs_io_metrics::set_get_stage_local_summary_sample_rate(sample_rate); +} + pub(crate) fn set_metrics_enabled(enabled: bool) { rustfs_io_metrics::set_metrics_enabled(enabled); } diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index 22c1a88c1..c11c3d350 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -1365,6 +1365,8 @@ impl NodeService { &self, request: Request, ) -> Result, Status> { + let service_return_started = rustfs_io_metrics::put_stage_timer(); + let service_entry_started = rustfs_io_metrics::put_stage_timer(); if !request.get_ref().scanner_publication_lease_token.is_empty() || !request.get_ref().bucket_incarnation_id.is_empty() { let has_body_digest = request .metadata() @@ -1372,14 +1374,37 @@ impl NodeService { .and_then(|value| value.to_str().ok()) .is_some_and(|value| value != "UNSIGNED-PAYLOAD"); if !has_body_digest { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY, + service_entry_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN, + service_return_started, + ); return Err(Status::permission_denied("scanner publication lease rename requires a body-bound digest")); } } - verify_disk_mutation_digest( + if let Err(err) = verify_disk_mutation_digest( &request, rustfs_protos::canonical_rename_data_request_body(request.get_ref()), "rename_data", - )?; + ) { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY, + service_entry_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN, + service_return_started, + ); + return Err(err); + } + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY, + service_entry_started, + ); + let before_handler_started = rustfs_io_metrics::put_stage_timer(); let request = request.into_inner(); let target = self.local_mutation_target(); let bucket_incarnation = if request.bucket_incarnation_id.is_empty() { @@ -1390,6 +1415,14 @@ impl NodeService { .filter(|id| !id.is_nil()) .ok_or_else(|| Status::invalid_argument("bucket incarnation must be a non-nil UUID"))?; if !request.scanner_publication_lease_token.is_empty() { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER, + before_handler_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN, + service_return_started, + ); return Err(Status::invalid_argument( "heal incarnation and scanner publication lease cannot be combined", )); @@ -1401,12 +1434,26 @@ impl NodeService { let decoded_file_info = match decode_rename_data_request_file_info(&request.file_info_bin, &request.file_info) { Ok(file_info) => file_info, Err(err) => { - return Ok(Response::new(RenameDataResponse { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER, + before_handler_started, + ); + let response_build_started = rustfs_io_metrics::put_stage_timer(); + let response = Response::new(RenameDataResponse { success: false, rename_data_resp: String::new(), rename_data_resp_bin: Vec::new().into(), error: Some(DiskError::other(format!("decode FileInfo failed: {err}")).into()), - })); + }); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD, + response_build_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN, + service_return_started, + ); + return Ok(response); } }; let scanner_publication_lease_token = if request.scanner_publication_lease_token.is_empty() { @@ -1415,6 +1462,14 @@ impl NodeService { let token = Uuid::from_slice(&request.scanner_publication_lease_token) .map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?; if token.is_nil() { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER, + before_handler_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN, + service_return_started, + ); return Err(Status::invalid_argument("scanner publication lease token must not be nil")); } Some(token) @@ -1422,6 +1477,11 @@ impl NodeService { let request_decoded_from_msgpack = decoded_file_info.from_msgpack; #[cfg(feature = "e2e-test-hooks")] let observation = startup_cas_rename_observation(&target, &request, &decoded_file_info.value); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER, + before_handler_started, + ); + let handler_await_started = rustfs_io_metrics::put_stage_timer(); let result = target .rename_local_data( &request.disk, @@ -1432,6 +1492,11 @@ impl NodeService { bucket_incarnation, ) .await; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT, + handler_await_started, + ); + let after_handler_started = rustfs_io_metrics::put_stage_timer(); #[cfg(feature = "e2e-test-hooks")] if let Some(mut observation) = observation { observation["ok"] = serde_json::json!(result.is_ok()); @@ -1439,28 +1504,47 @@ impl NodeService { let line = format!("RUSTFS_E2E_STARTUP_CAS {observation}\n"); let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes()); } - match result { + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER, + after_handler_started, + ); + let response_build_started = rustfs_io_metrics::put_stage_timer(); + let response_encode_started = rustfs_io_metrics::put_stage_timer(); + let response = match result { Ok(rename_data_resp) => match encode_rename_data_response_payloads(&rename_data_resp, request_decoded_from_msgpack) { - Ok((rename_data_resp, rename_data_resp_bin)) => Ok(Response::new(RenameDataResponse { + Ok((rename_data_resp, rename_data_resp_bin)) => Response::new(RenameDataResponse { success: true, rename_data_resp, rename_data_resp_bin: rename_data_resp_bin.into(), error: None, - })), - Err(err) => Ok(Response::new(RenameDataResponse { + }), + Err(err) => Response::new(RenameDataResponse { success: false, rename_data_resp: String::new(), rename_data_resp_bin: Vec::new().into(), error: Some(err.into()), - })), + }), }, - Err(err) => Ok(Response::new(RenameDataResponse { + Err(err) => Response::new(RenameDataResponse { success: false, rename_data_resp: String::new(), rename_data_resp_bin: Vec::new().into(), error: Some(err.into()), - })), - } + }), + }; + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE, + response_encode_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD, + response_build_started, + ); + rustfs_io_metrics::record_put_object_stage_duration_from( + rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN, + service_return_started, + ); + Ok(response) } pub(super) async fn handle_list_dir(&self, request: Request) -> Result, Status> { diff --git a/rustfs/src/storage/timeout_wrapper.rs b/rustfs/src/storage/timeout_wrapper.rs index f0ca0f882..94f0bed24 100644 --- a/rustfs/src/storage/timeout_wrapper.rs +++ b/rustfs/src/storage/timeout_wrapper.rs @@ -412,21 +412,21 @@ impl RequestTimeoutWrapper { debug!(timeout_secs = timeout_duration.as_secs(), "Starting timed operation"); - rustfs_io_metrics::record_get_object_request_started(); + rustfs_io_metrics::record_get_object_timeout_operation_start(); let cancel_token_for_op = self.cancel_token.clone(); match tokio::time::timeout(timeout_duration, operation(cancel_token_for_op)).await { Ok(Ok(result)) => { let elapsed = self.elapsed(); - rustfs_io_metrics::record_get_object_request_result("success", elapsed.as_secs_f64()); + rustfs_io_metrics::record_get_object_timeout_operation_result("success", elapsed.as_secs_f64()); debug!(elapsed_ms = elapsed.as_millis(), "Operation completed successfully"); TimedGetObjectResult::Success(result) } Ok(Err(e)) => { let elapsed = self.elapsed(); - rustfs_io_metrics::record_get_object_request_result("error", elapsed.as_secs_f64()); + rustfs_io_metrics::record_get_object_timeout_operation_result("error", elapsed.as_secs_f64()); debug!(elapsed_ms = elapsed.as_millis(), "Operation failed with error"); TimedGetObjectResult::Error(e) @@ -436,7 +436,7 @@ impl RequestTimeoutWrapper { self.cancel_token.cancel(); rustfs_io_metrics::record_get_object_timeout(None, Some(elapsed.as_secs_f64())); - rustfs_io_metrics::record_get_object_request_result("timeout", elapsed.as_secs_f64()); + rustfs_io_metrics::record_get_object_timeout_operation_result("timeout", elapsed.as_secs_f64()); warn!( timeout_secs = timeout_duration.as_secs(), diff --git a/rustfs/tests/connect_offline_bundle.rs b/rustfs/tests/connect_offline_bundle.rs index 9146e25b1..88cb394bf 100644 --- a/rustfs/tests/connect_offline_bundle.rs +++ b/rustfs/tests/connect_offline_bundle.rs @@ -155,7 +155,7 @@ fn connect_offline_bundle_is_deterministic_bounded_and_signed_over_exact_manifes assert_eq!(archive.len(), PATHS.len()); for (index, expected_path) in PATHS.into_iter().enumerate() { let file = archive.by_index(index).expect("archive member"); - assert_eq!(file.name(), expected_path); + assert_eq!(file.name().expect("archive entry name"), expected_path); assert_eq!(file.compression(), CompressionMethod::Stored); assert_eq!(file.last_modified(), Some(zip::DateTime::DEFAULT)); assert_eq!(file.unix_mode().map(|mode| mode & 0o777), Some(0o600)); diff --git a/rustfs/tests/connect_perf_drive.rs b/rustfs/tests/connect_perf_drive.rs index 43a3ba919..a3c88ec81 100644 --- a/rustfs/tests/connect_perf_drive.rs +++ b/rustfs/tests/connect_perf_drive.rs @@ -389,9 +389,15 @@ async fn successful_result_has_signed_bounded_private_offline_export() { let mut archive = zip::ZipArchive::new(Cursor::new(&export.archive_bytes)).expect("three-file archive"); assert_eq!(archive.len(), 3); - assert_eq!(archive.by_index(0).expect("envelope").name(), "envelope.json"); - assert_eq!(archive.by_index(1).expect("signature").name(), "envelope.sig"); - assert_eq!(archive.by_index(2).expect("result").name(), "result.json"); + assert_eq!( + archive.by_index(0).expect("envelope").name().expect("archive entry name"), + "envelope.json" + ); + assert_eq!( + archive.by_index(1).expect("signature").name().expect("archive entry name"), + "envelope.sig" + ); + assert_eq!(archive.by_index(2).expect("result").name().expect("archive entry name"), "result.json"); let envelope: serde_json::Value = serde_json::from_slice(&export.envelope_json).expect("envelope JSON"); assert_eq!(envelope["toolId"], "performance.drive"); @@ -535,7 +541,14 @@ fn production_cli_measures_and_signs_exact_binary_provenance() { let archive_bytes = fs::read(&output).expect("saved archive"); let mut archive = zip::ZipArchive::new(Cursor::new(archive_bytes.as_slice())).expect("signed archive"); let names = (0..archive.len()) - .map(|index| archive.by_index(index).expect("archive member").name().to_owned()) + .map(|index| { + archive + .by_index(index) + .expect("archive member") + .name() + .expect("archive entry name") + .into_owned() + }) .collect::>(); assert_eq!(names, ["envelope.json", "envelope.sig", "result.json"]); let envelope_bytes = read_archive_member(&mut archive, "envelope.json"); diff --git a/rustfs/tests/connect_perf_network.rs b/rustfs/tests/connect_perf_network.rs index 4136a9ca8..6ada066b8 100644 --- a/rustfs/tests/connect_perf_network.rs +++ b/rustfs/tests/connect_perf_network.rs @@ -669,9 +669,26 @@ async fn successful_result_has_signed_bounded_private_offline_export() { assert_eq!(export.archive_sha256, hex(&Sha256::digest(&export.archive_bytes))); let mut archive = zip::ZipArchive::new(Cursor::new(&export.archive_bytes)).expect("three-file archive"); assert_eq!(archive.len(), 3); - assert_eq!(archive.by_index(0).expect("envelope entry").name(), "envelope.json"); - assert_eq!(archive.by_index(1).expect("signature entry").name(), "envelope.sig"); - assert_eq!(archive.by_index(2).expect("result entry").name(), "result.json"); + assert_eq!( + archive + .by_index(0) + .expect("envelope entry") + .name() + .expect("archive entry name"), + "envelope.json" + ); + assert_eq!( + archive + .by_index(1) + .expect("signature entry") + .name() + .expect("archive entry name"), + "envelope.sig" + ); + assert_eq!( + archive.by_index(2).expect("result entry").name().expect("archive entry name"), + "result.json" + ); let envelope: serde_json::Value = serde_json::from_slice(&export.envelope_json).expect("envelope JSON"); assert_eq!(envelope["toolId"], "performance.network"); diff --git a/rustfs/tests/connect_top_net.rs b/rustfs/tests/connect_top_net.rs index 294cd3642..0da443a39 100644 --- a/rustfs/tests/connect_top_net.rs +++ b/rustfs/tests/connect_top_net.rs @@ -399,7 +399,14 @@ fn production_cli_exports_top_net_and_fails_closed_for_unavailable_and_invalid_r assert_eq!(fs::metadata(&net_output).expect("output metadata").permissions().mode() & 0o777, 0o600); let mut archive = ZipArchive::new(Cursor::new(bytes)).expect("top.net archive"); let names = (0..archive.len()) - .map(|index| archive.by_index(index).expect("archive member").name().to_owned()) + .map(|index| { + archive + .by_index(index) + .expect("archive member") + .name() + .expect("archive entry name") + .into_owned() + }) .collect::>(); assert_eq!(names, ["envelope.json", "envelope.sig", "result.json"]); From bd83bd3f6b1f1821e0549808fa722e3d9cc8c4fb Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Tue, 13 Oct 2026 17:12:00 +0800 Subject: [PATCH 011/143] fix(table-catalog): allow commits after object-backed rename (#8508) --- .../src/admin/handlers/table_catalog/tests.rs | 259 ++++++++++++++++-- rustfs/src/table_catalog/store/object.rs | 2 +- 2 files changed, 241 insertions(+), 20 deletions(-) diff --git a/rustfs/src/admin/handlers/table_catalog/tests.rs b/rustfs/src/admin/handlers/table_catalog/tests.rs index 9ce007d95..56b4f2a6d 100644 --- a/rustfs/src/admin/handlers/table_catalog/tests.rs +++ b/rustfs/src/admin/handlers/table_catalog/tests.rs @@ -4211,24 +4211,46 @@ fn table_metadata_file_name_scoping_is_bounded_and_identity_sensitive() { #[tokio::test] async fn renamed_and_recreated_tables_with_the_same_commit_id_use_disjoint_metadata_files() { + assert_renamed_and_recreated_table_commits(crate::table_catalog::TableCatalogBackingMode::DurableStrong, "curated").await; +} + +#[tokio::test] +async fn object_catalog_cross_namespace_rename_accepts_isolated_standard_commits() { + assert_renamed_and_recreated_table_commits(crate::table_catalog::TableCatalogBackingMode::ObjectBacked, "curated").await; +} + +#[tokio::test] +async fn object_catalog_same_namespace_rename_accepts_isolated_standard_commits() { + assert_renamed_and_recreated_table_commits(crate::table_catalog::TableCatalogBackingMode::ObjectBacked, "analytics").await; +} + +#[tokio::test] +async fn strong_catalog_same_namespace_rename_accepts_isolated_standard_commits() { + assert_renamed_and_recreated_table_commits(crate::table_catalog::TableCatalogBackingMode::DurableStrong, "analytics").await; +} + +async fn assert_renamed_and_recreated_table_commits(mode: crate::table_catalog::TableCatalogBackingMode, destination: &str) { let metadata_backend = TestTableCatalogObjectBackend::content_addressed(); - let store = crate::table_catalog::StrongTableCatalogStore::new(metadata_backend.clone()); + let store = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(metadata_backend.clone(), mode); let source_namespace = crate::table_catalog::Namespace::parse("analytics").expect("source namespace should parse"); - let destination_namespace = crate::table_catalog::Namespace::parse("curated").expect("destination namespace should parse"); + let destination_namespace = crate::table_catalog::Namespace::parse(destination).expect("destination namespace should parse"); create_standard_events_table(&store, &metadata_backend, &source_namespace).await; - create_namespace_response( - &store, - "warehouse", - CreateNamespaceRequest { - namespace: vec!["curated".to_string()], - properties: BTreeMap::new(), - }, - true, - ) - .await - .expect("destination namespace should be created"); + let original = store.load_table("warehouse", "analytics", "events").await.unwrap().unwrap(); + if source_namespace != destination_namespace { + create_namespace_response( + &store, + "warehouse", + CreateNamespaceRequest { + namespace: vec![destination.to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("destination namespace should be created"); + } store - .rename_table("warehouse", "analytics", "events", "curated", "events_v2") + .rename_table("warehouse", "analytics", "events", destination, "events_v2") .await .expect("table should rename"); @@ -4249,7 +4271,7 @@ async fn renamed_and_recreated_tables_with_the_same_commit_id_use_disjoint_metad .expect("source identifier should be reusable"); let renamed = store - .load_table("warehouse", "curated", "events_v2") + .load_table("warehouse", destination, "events_v2") .await .expect("renamed table lookup should succeed") .expect("renamed table should exist"); @@ -4259,18 +4281,196 @@ async fn renamed_and_recreated_tables_with_the_same_commit_id_use_disjoint_metad .expect("recreated table lookup should succeed") .expect("recreated table should exist"); assert_ne!(renamed.table_id, recreated.table_id); + assert_ne!(renamed.table_uuid, recreated.table_uuid); + assert_ne!(renamed.warehouse_location, recreated.warehouse_location); + assert_eq!(renamed.table_id, original.table_id); + assert_eq!(renamed.table_uuid, original.table_uuid); + assert_eq!(renamed.warehouse_location, original.warehouse_location); + assert_eq!(renamed.metadata_location, original.metadata_location); + assert_eq!(renamed.version_token, original.version_token); + assert_eq!(renamed.generation, original.generation); let commit_id = "11111111-1111-4111-8111-111111111111"; + let renamed_request = || { + let mut request = standard_property_commit_request(commit_id, &renamed.table_uuid, "curated"); + request.idempotency_key = Some("renamed-request".to_string()); + request + }; let renamed_commit = standard_commit_table_response( &store, &trusted_table_commit_backend(&metadata_backend), "warehouse", &destination_namespace, "events_v2", - standard_property_commit_request(commit_id, &renamed.table_uuid, "curated"), + renamed_request(), ) .await .expect("renamed table commit should succeed"); + let committed = store + .load_table("warehouse", destination, "events_v2") + .await + .unwrap() + .unwrap(); + assert_eq!(committed.table_id, renamed.table_id); + assert_eq!(committed.table_uuid, renamed.table_uuid); + assert_eq!(committed.warehouse_location, renamed.warehouse_location); + assert_eq!(committed.generation, renamed.generation + 1); + assert_ne!(committed.version_token, renamed.version_token); + assert_ne!(committed.metadata_location, renamed.metadata_location); + assert_eq!(renamed_commit.metadata["table-uuid"], renamed.table_uuid); + assert_eq!(renamed_commit.metadata["location"], renamed.warehouse_location); + assert_eq!(renamed_commit.metadata["properties"]["owner"], "curated"); + let receipt = store + .get_commit_by_id("warehouse", &renamed.table_id, commit_id) + .await + .unwrap() + .unwrap(); + assert_eq!(receipt.status, crate::table_catalog::CommitLogStatus::Committed); + assert_eq!(receipt.new_version_token, committed.version_token); + assert_eq!(receipt.new_metadata_location, committed.metadata_location); + assert_eq!( + store + .get_commit_by_idempotency_key("warehouse", &renamed.table_id, receipt.idempotency_key.as_deref().unwrap()) + .await + .unwrap(), + Some(receipt.clone()) + ); + let replay = standard_commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &destination_namespace, + "events_v2", + renamed_request(), + ) + .await + .expect("same commit ID should replay after rename"); + assert_eq!(replay.metadata_location, renamed_commit.metadata_location); + assert_eq!(replay.metadata, renamed_commit.metadata); + assert_eq!( + store.load_table("warehouse", destination, "events_v2").await.unwrap(), + Some(committed.clone()) + ); + + let stale_request = crate::table_catalog::TableCommitRequest { + table_bucket: "warehouse".to_string(), + namespace: destination.to_string(), + table: "events_v2".to_string(), + commit_id: "stale-after-rename".to_string(), + idempotency_key: Some("stale-after-rename".to_string()), + operation: "append".to_string(), + expected_version_token: renamed.version_token.clone(), + expected_metadata_location: renamed.metadata_location.clone(), + new_metadata_location: committed.metadata_location.clone(), + requirements: Vec::new(), + writer: None, + }; + for metadata_only in [false, true] { + let mut request = stale_request.clone(); + if metadata_only { + request.expected_version_token = committed.version_token.clone(); + request.commit_id = "stale-metadata-after-rename".to_string(); + request.idempotency_key = Some(request.commit_id.clone()); + } + assert!(matches!( + store.commit_table(request.clone()).await, + Err(crate::table_catalog::TableCatalogStoreError::Conflict(_)) + )); + assert!( + store + .get_commit_by_id("warehouse", &renamed.table_id, &request.commit_id) + .await + .unwrap() + .is_none() + ); + assert!( + store + .get_commit_by_idempotency_key("warehouse", &renamed.table_id, request.idempotency_key.as_deref().unwrap()) + .await + .unwrap() + .is_none() + ); + assert_eq!( + store.load_table("warehouse", destination, "events_v2").await.unwrap(), + Some(committed.clone()) + ); + } + let current_namespace_path = crate::table_catalog::default_table_metadata_dir_path( + &destination_namespace, + &crate::table_catalog::IdentifierSegment::parse("events_v2").unwrap(), + ); + let foreign_namespace_path = crate::table_catalog::default_table_metadata_dir_path( + &source_namespace, + &crate::table_catalog::IdentifierSegment::parse("foreign").unwrap(), + ); + let retained_directory = committed.metadata_location.rsplit_once('/').unwrap().0; + let persisted_metadata = read_table_metadata_json(&metadata_backend, "warehouse", &committed.metadata_location) + .await + .unwrap(); + for (index, location) in [ + format!("{current_namespace_path}/00003.metadata.json"), + format!("{foreign_namespace_path}/00003.metadata.json"), + format!("s3://other-warehouse/{foreign_namespace_path}/00003.metadata.json"), + format!("{retained_directory}/../00003.metadata.json"), + format!("{retained_directory}/child/00003.metadata.json"), + format!("{retained_directory}/%2e%2e.json"), + ] + .into_iter() + .enumerate() + { + if index < 2 { + metadata_backend + .put_json("warehouse", &location, persisted_metadata.clone()) + .await; + } + let mut request = stale_request.clone(); + request.commit_id = format!("invalid-renamed-path-{index}"); + request.idempotency_key = Some(request.commit_id.clone()); + request.expected_version_token = committed.version_token.clone(); + request.expected_metadata_location = committed.metadata_location.clone(); + request.new_metadata_location = location; + assert!(matches!( + store.commit_table(request.clone()).await, + Err(crate::table_catalog::TableCatalogStoreError::Invalid(_)) + )); + assert!( + store + .get_commit_by_id("warehouse", &renamed.table_id, &request.commit_id) + .await + .unwrap() + .is_none() + ); + assert!( + store + .get_commit_by_idempotency_key("warehouse", &renamed.table_id, request.idempotency_key.as_deref().unwrap()) + .await + .unwrap() + .is_none() + ); + assert_eq!( + store.load_table("warehouse", destination, "events_v2").await.unwrap(), + Some(committed.clone()) + ); + assert_eq!( + store.load_table("warehouse", "analytics", "events").await.unwrap(), + Some(recreated.clone()) + ); + } + let wrong_identity = standard_commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &destination_namespace, + "events_v2", + standard_property_commit_request("22222222-2222-4222-8222-222222222222", &recreated.table_uuid, "wrong-identity"), + ) + .await + .expect_err("recreated source identity cannot commit to the renamed table"); + assert_eq!(wrong_identity.status_code(), Some(StatusCode::CONFLICT)); + assert_eq!( + store.load_table("warehouse", destination, "events_v2").await.unwrap(), + Some(committed.clone()) + ); let recreated_commit = standard_commit_table_response( &store, &trusted_table_commit_backend(&metadata_backend), @@ -4300,7 +4500,7 @@ async fn renamed_and_recreated_tables_with_the_same_commit_id_use_disjoint_metad assert!( !renamed_commit .metadata_location - .contains("/namespaces/curated/tables/events_v2/") + .contains(&format!("/namespaces/{destination}/tables/events_v2/")) ); assert!( recreated_commit @@ -4309,8 +4509,8 @@ async fn renamed_and_recreated_tables_with_the_same_commit_id_use_disjoint_metad ); assert_eq!(recreated_commit.metadata["table-uuid"], recreated.table_uuid); assert_eq!(recreated_commit.metadata["properties"]["owner"], "analytics"); - for metadata_location in [renamed_commit.metadata_location, recreated_commit.metadata_location] { - let object_key = test_snapshot_object_key("warehouse", &metadata_location); + for metadata_location in [&renamed_commit.metadata_location, &recreated_commit.metadata_location] { + let object_key = test_snapshot_object_key("warehouse", metadata_location); assert!( metadata_backend .object_exists("warehouse", &object_key) @@ -4318,6 +4518,27 @@ async fn renamed_and_recreated_tables_with_the_same_commit_id_use_disjoint_metad .expect("metadata lookup should succeed") ); } + let restarted = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(metadata_backend, mode); + assert_eq!( + restarted.load_table("warehouse", destination, "events_v2").await.unwrap(), + Some(committed) + ); + let recreated_after = restarted + .load_table("warehouse", "analytics", "events") + .await + .unwrap() + .unwrap(); + assert_eq!(recreated_after.table_id, recreated.table_id); + assert_eq!(recreated_after.table_uuid, recreated.table_uuid); + assert_eq!(recreated_after.warehouse_location, recreated.warehouse_location); + assert_eq!(recreated_after.generation, recreated.generation + 1); + assert_eq!( + restarted + .get_commit_by_id("warehouse", &renamed.table_id, commit_id) + .await + .unwrap(), + Some(receipt) + ); } #[tokio::test] diff --git a/rustfs/src/table_catalog/store/object.rs b/rustfs/src/table_catalog/store/object.rs index f62b2b93f..c4177251f 100644 --- a/rustfs/src/table_catalog/store/object.rs +++ b/rustfs/src/table_catalog/store/object.rs @@ -5875,7 +5875,7 @@ where )), ); } - if !is_valid_table_metadata_location(&namespace, &table, &request.new_metadata_location) { + if !is_valid_table_metadata_location_for_entry(¤t, &request.new_metadata_location) { return table_commit_result( &request.table_bucket, &request.namespace, From 9ca0e9b7179b8be42da448f9f119de1800f198ab Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 18:00:44 +0800 Subject: [PATCH 012/143] fix(obs): isolate bucket usage dashboards by cluster (#8510) Keep same-name buckets in independent clusters separate while deduplicating replicated usage observations. Document the existing scanner-backed exporter and cover its gauge contract and cluster-scoped queries. Co-authored-by: heihutu Co-authored-by: zhi22915 --- .../grafana/dashboards/rustfs.json | 4 +- .../observability/tests/bucket-usage.test.yml | 48 +++++++ crates/obs/README.md | 4 +- crates/obs/src/global.rs | 23 +++- crates/obs/src/metrics/storage_snapshot.rs | 63 +++++++++ docs/README.md | 3 + docs/operations/bucket-usage-metrics.md | 122 ++++++++++++++++++ 7 files changed, 261 insertions(+), 6 deletions(-) create mode 100644 .docker/observability/tests/bucket-usage.test.yml create mode 100644 docs/operations/bucket-usage-metrics.md diff --git a/.docker/observability/grafana/dashboards/rustfs.json b/.docker/observability/grafana/dashboards/rustfs.json index 15fbdc34b..1ca9a90a7 100644 --- a/.docker/observability/grafana/dashboards/rustfs.json +++ b/.docker/observability/grafana/dashboards/rustfs.json @@ -1748,7 +1748,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max by (job, bucket) (rustfs_cluster_usage_buckets_objects_count{job=~\"$job\", bucket=~\"$bucket\"})", + "expr": "max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id=\"$storage_cluster\", job=~\"$job\", bucket=~\"$bucket\"})", "legendFormat": "{{bucket}}", "range": true, "refId": "A" @@ -1848,7 +1848,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max by (job, bucket) (rustfs_cluster_usage_buckets_total_bytes{job=~\"$job\", bucket=~\"$bucket\"})", + "expr": "max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id=\"$storage_cluster\", job=~\"$job\", bucket=~\"$bucket\"})", "legendFormat": "{{bucket}}", "range": true, "refId": "A" diff --git a/.docker/observability/tests/bucket-usage.test.yml b/.docker/observability/tests/bucket-usage.test.yml new file mode 100644 index 000000000..d886a7dc9 --- /dev/null +++ b/.docker/observability/tests/bucket-usage.test.yml @@ -0,0 +1,48 @@ +evaluation_interval: 1m +tests: +- name: bucket usage keeps clusters separate and deduplicates reporting nodes + interval: 1m + input_series: + - series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a1"} + values: '4096 1024' + - series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a2"} + values: '4096 1024' + - series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="b",job="rustfs",bucket="shared",service_instance_id="b1"} + values: '65536 65536' + - series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job="rustfs",bucket="empty",service_instance_id="a1"} + values: '0 0' + - series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a1"} + values: '3 1' + - series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a2"} + values: '3 1' + - series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="b",job="rustfs",bucket="shared",service_instance_id="b1"} + values: '70 70' + - series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job="rustfs",bucket="empty",service_instance_id="a1"} + values: '0 0' + promql_expr_test: + - expr: max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job=~"rustfs",bucket=~".*"}) + eval_time: 1m + exp_samples: + - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="shared"}' + value: 1024 + - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="empty"}' + value: 0 + - expr: max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job=~"rustfs",bucket=~".*"}) + eval_time: 1m + exp_samples: + - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="shared"}' + value: 1 + - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="empty"}' + value: 0 + - expr: max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_total_bytes) + eval_time: 1m + exp_samples: + - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="shared"}' + value: 1024 + - labels: '{rustfs_cluster_id="a",job="rustfs",bucket="empty"}' + value: 0 + - labels: '{rustfs_cluster_id="b",job="rustfs",bucket="shared"}' + value: 65536 + - expr: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",bucket="unknown"} + eval_time: 1m + exp_samples: [] diff --git a/crates/obs/README.md b/crates/obs/README.md index a66fc5dea..0a1a9caaf 100644 --- a/crates/obs/README.md +++ b/crates/obs/README.md @@ -127,7 +127,9 @@ All configuration is read from environment variables at startup. `RUSTFS_OBS_TLS_CA_FILE` is read only while OTLP exporters are initialized. Its PEM certificates are added to, not substituted for, the platform trust roots, and TLS chain, expiry, and hostname checks remain enabled. Restart RustFS after rotating the bundle. A missing, unreadable, non-absolute, empty, or invalid bundle fails OTLP initialization. -The export interval is separate from application metric collection. Without interval overrides, node/disk metrics refresh every 60 seconds (`RUSTFS_METRICS_NODE_INTERVAL_SEC`), cluster metrics every 60 seconds (`RUSTFS_METRICS_CLUSTER_INTERVAL_SEC`), and per-bucket metrics every 300 seconds (`RUSTFS_METRICS_BUCKET_INTERVAL_SEC`). See the [collection defaults](src/metrics/config.rs) and [interval configuration](src/metrics/scheduler.rs) for collector-specific and global overrides. +The export interval is separate from application metric collection. Without interval overrides, node/disk metrics refresh every 60 seconds (`RUSTFS_METRICS_NODE_INTERVAL_SEC`), cluster metrics including scanner bucket usage every 60 seconds (`RUSTFS_METRICS_CLUSTER_INTERVAL_SEC`), and the separate bucket collector every 300 seconds (`RUSTFS_METRICS_BUCKET_INTERVAL_SEC`). See the [collection defaults](src/metrics/config.rs) and [interval configuration](src/metrics/scheduler.rs) for collector-specific and global overrides. + +For bucket size and object-count metric names, all-bucket PromQL queries, accounting semantics, and scanner freshness, see [Bucket usage metrics](../../docs/operations/bucket-usage-metrics.md). With OTLP → Collector → Prometheus, the time from a failure to an alert also includes underlying health detection, collection and export work, transport and Collector batching, Prometheus `scrape_interval` and `evaluation_interval`, and any alert rule `for` duration. A shorter scrape interval cannot refresh a value that RustFS has not yet collected and exported. These periodic intervals do not guarantee a fixed detection or alert latency; see the [Prometheus alert rule documentation](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) for the role of `for`. diff --git a/crates/obs/src/global.rs b/crates/obs/src/global.rs index 08f1e83b0..a413a5d01 100644 --- a/crates/obs/src/global.rs +++ b/crates/obs/src/global.rs @@ -260,9 +260,26 @@ mod tests { } #[test] - fn test_grafana_dashboard_deduplicates_cluster_bucket_usage() { - assert!(GRAFANA_DASHBOARD.contains("max by (job, bucket) (rustfs_cluster_usage_buckets_objects_count")); - assert!(GRAFANA_DASHBOARD.contains("max by (job, bucket) (rustfs_cluster_usage_buckets_total_bytes")); + fn test_grafana_dashboard_scopes_and_deduplicates_cluster_bucket_usage() { + let dashboard: serde_json::Value = serde_json::from_str(GRAFANA_DASHBOARD).expect("dashboard must be valid JSON"); + for (title, metric) in [ + ("Bucket Objects", "rustfs_cluster_usage_buckets_objects_count"), + ("Bucket Size", "rustfs_cluster_usage_buckets_total_bytes"), + ] { + let panel = dashboard["panels"] + .as_array() + .expect("dashboard panels") + .iter() + .find(|panel| panel["title"] == title) + .expect("bucket usage panel"); + assert_eq!( + panel["targets"][0]["expr"], + format!( + "max by (rustfs_cluster_id, job, bucket) ({metric}{{rustfs_cluster_id=\"$storage_cluster\", job=~\"$job\", bucket=~\"$bucket\"}})" + ), + "{title} must select one cluster and deduplicate its reporting nodes" + ); + } assert!(!GRAFANA_DASHBOARD.contains("sum by (bucket) (rustfs_bucket_api_objects_total")); assert!(!GRAFANA_DASHBOARD.contains("sum by (bucket) (rustfs_bucket_api_usage_bytes")); } diff --git a/crates/obs/src/metrics/storage_snapshot.rs b/crates/obs/src/metrics/storage_snapshot.rs index 8977403e9..d3e1a74ee 100644 --- a/crates/obs/src/metrics/storage_snapshot.rs +++ b/crates/obs/src/metrics/storage_snapshot.rs @@ -408,6 +408,69 @@ mod tests { provider.shutdown().unwrap(); } + #[test] + fn actual_sdk_exports_bucket_size_and_object_count_as_current_gauges() { + use crate::metrics::collectors::{BucketUsageStats, collect_bucket_usage_metrics}; + + let (provider, exporter, mut snapshot) = setup_unscoped(); + let mut buckets = vec![ + BucketUsageStats { + bucket: "client-a".into(), + total_bytes: 4096, + objects_count: 3, + ..Default::default() + }, + BucketUsageStats { + bucket: "client-b".into(), + total_bytes: 8192, + objects_count: 7, + ..Default::default() + }, + BucketUsageStats { + bucket: "empty".into(), + ..Default::default() + }, + ]; + + for expected in [ + vec![("client-a", 4096.0, 3.0), ("client-b", 8192.0, 7.0), ("empty", 0.0, 0.0)], + vec![("client-a", 1024.0, 1.0), ("empty", 0.0, 0.0)], + ] { + let metrics = collect_bucket_usage_metrics(&buckets); + assert!( + metrics.iter().all(|metric| metric.metric_type == MetricType::Gauge), + "bucket inventory values must decrease without counter semantics" + ); + snapshot.replace(metrics).expect("publish current bucket usage"); + let points = flush(&provider, &exporter); + for (name, index) in [ + ("rustfs_cluster_usage_buckets_total_bytes", 0), + ("rustfs_cluster_usage_buckets_objects_count", 1), + ] { + let mut actual = points + .iter() + .filter(|point| point.name == name) + .map(|point| { + assert_eq!(point.scope, "service-test"); + assert_eq!(point.attributes.len(), 1, "usage is cluster-wide, not a per-node count"); + assert_eq!(point.attributes[0].key.as_str(), "bucket"); + (point.attributes[0].value.to_string(), point.value) + }) + .collect::>(); + actual.sort_by(|a, b| a.0.cmp(&b.0)); + let values = expected + .iter() + .map(|(bucket, bytes, objects)| (bucket.to_string(), if index == 0 { *bytes } else { *objects })) + .collect::>(); + assert_eq!(actual, values, "{name} must preserve each bucket's current value"); + } + buckets.remove(1); + buckets[0].total_bytes = 1024; + buckets[0].objects_count = 1; + } + provider.shutdown().expect("shut down bucket usage test provider"); + } + #[test] fn failed_or_stalled_collection_cannot_refresh_last_success_or_export_old_points() { let (provider, exporter, mut snapshot) = setup(); diff --git a/docs/README.md b/docs/README.md index 958dbda58..c32761e35 100644 --- a/docs/README.md +++ b/docs/README.md @@ -66,6 +66,9 @@ procedure is required. For storage dashboards, see [Storage metrics and observer selection](operations/storage-metrics.md): drive ownership, snapshot freshness, counter queries, and rolling upgrades. +For per-bucket capacity and object counts, see [Bucket usage metrics](operations/bucket-usage-metrics.md): +OTLP export, cluster deduplication, accounting semantics, and scanner freshness. + For optional shard commitments, see [Independent shard integrity rollout](operations/shard-integrity-rollout.md): activation, legacy repair results, multipart mode changes, and rollback limits. diff --git a/docs/operations/bucket-usage-metrics.md b/docs/operations/bucket-usage-metrics.md new file mode 100644 index 000000000..ed65d05a7 --- /dev/null +++ b/docs/operations/bucket-usage-metrics.md @@ -0,0 +1,122 @@ +# Bucket usage metrics + +Use these metrics to collect every bucket's logical usage through the existing +OpenTelemetry pipeline. RustFS reads the cached scanner usage snapshot once per +collection; exporting metrics does not enumerate objects or run an inline scan. + +## Export and query + +Configure the RustFS nodes with an OTLP/HTTP destination and a stable cluster ID: + +```bash +RUSTFS_OBS_ENDPOINT=http://otel-collector:4318 +RUSTFS_OBS_METRICS_EXPORT_ENABLED=true +OTEL_RESOURCE_ATTRIBUTES=rustfs.cluster.id=production-a +``` + +Append the cluster ID to any existing resource attributes. The shipped +[Collector configuration](../../.docker/observability/otel-collector-config.yaml) +receives OTLP and exposes Prometheus samples on port 8889; its +`resource_to_telemetry_conversion` promotes the cluster ID to +`rustfs_cluster_id`. Prometheus scrapes the Collector. See +[storage metrics](storage-metrics.md#configure-the-pipeline) for pipeline setup +and resource identity. + +The following existing gauges carry a `bucket` label: + +| Metric | Meaning | +|---|---| +| `rustfs_cluster_usage_buckets_total_bytes` | Logical scanner bytes across retained object versions, including noncurrent versions; excludes delete-marker bytes. | +| `rustfs_cluster_usage_buckets_objects_count` | Scanner object-entry/key count, separate from version and delete-marker counts. | + +Logical usage differs from physical drive usage, which includes erasure-coding +overhead and can differ because of compression or remote tiering. Object-entry +count can include keys with only noncurrent versions or delete markers, so it +need not equal the number of currently visible objects from `ListObjectsV2`. + +For monitoring, query all buckets in one cluster: + +```promql +max by (rustfs_cluster_id, job, bucket) ( + rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="production-a"} +) +``` + +```promql +max by (rustfs_cluster_id, job, bucket) ( + rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="production-a"} +) +``` + +Each reporting node exports the whole cluster's logical usage. Summing those +copies would multiply the result by the node count. Keep cluster identity when +grouping: independent clusters can share both a job and a bucket name. The +bundled Bucket Size and Bucket Objects panels use the selected **Storage +cluster** and deduplicate that cluster's reporting nodes. + +Nodes can briefly observe different snapshots. `max` is useful for dashboards +but can retain an older, larger value after deletes. For a consistent accounting +observation, select one reporting node for both bytes and object count: + +```promql +rustfs_cluster_usage_buckets_total_bytes{ + rustfs_cluster_id="production-a", service_instance_id="node1:9000" +} +``` + +Use the same selector with `rustfs_cluster_usage_buckets_objects_count`, and +check the promoted `service_instance_id` in your Collector output. These +floating-point, periodic gauges are usage observations, not an exact billing +ledger or a transactional view of writes and deletes. + +## Freshness and missing values + +Usage collection runs on `RUSTFS_METRICS_CLUSTER_INTERVAL_SEC` (60 seconds by +default), independently of the OTLP export interval +`RUSTFS_OBS_METER_INTERVAL` (30 seconds by default). The separate bucket +collector defaults to 300 seconds. Neither interval forces a scanner cycle. + +Check these gauges on the same reporting node as the bucket values: + +- `rustfs_cluster_usage_objects_since_last_update_seconds`: age of the + persisted usage snapshot when it was collected. A bucket cleanup can advance + the timestamp; it is not necessarily the age of a full object rescan. +- `rustfs_cluster_usage_objects_snapshot_converged`: `1` for a converged + snapshot, `0` when convergence is not confirmed, including observational or + unknown state. Complete observations can still supply usage during sustained writes. + +Before a complete snapshot covers the current bucket namespace, the usage +collector omits its values. A missing series is unknown; do not replace it with +zero for accounting. A bucket confirmed empty in a complete snapshot exports +zero bytes and zero objects. + +A temporary collection failure preserves the last valid export snapshot. +RustFS stops exporting that snapshot once collection stalls beyond three times +the larger collection/export interval. Deleted bucket series receive bounded +zero tombstones and then stop being exported after a valid updated snapshot. +The Collector can retain the last sample after export stops; the shipped +configuration uses a five-minute `metric_expiration`. Prometheus history also +retains past samples. + +If metrics are absent, first check OTLP connectivity and metrics export enablement, +then the scanner leader's `GET /rustfs/admin/v3/scanner/status` and the admin +usage snapshot. A shorter scrape interval cannot repair an incomplete scanner +snapshot. The scanner publication and cache path is shared with the Console's +usage display. + +## Regression checks + +The SDK test uses the production bucket collector and an actual OpenTelemetry +reader to verify two populated buckets, a confirmed-empty bucket, decreasing +usage, and removal from subsequent exports: + +```bash +RUSTUP_TOOLCHAIN=stable cargo test -p rustfs-obs actual_sdk_exports_bucket_size_and_object_count_as_current_gauges --lib +``` + +The PromQL fixture checks same-name buckets in separate clusters and multiple +reporting nodes. Run it from `.docker/observability/tests`: + +```bash +promtool test rules bucket-usage.test.yml +``` From 8a94c6493484b3b9fcda28e1aba4502aa5103578 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 18:56:44 +0800 Subject: [PATCH 013/143] fix(audit): preserve request identity and verified client address (#8513) Capture the authenticated request access key when building the audit entry and preserve it through completion instead of replacing it with root credentials. Resolve remotehost from validated client context or the socket peer, and ignore untrusted forwarding headers. Cover request identities, completion outcomes, client address precedence, and audit serialization without credential secrets. Co-authored-by: heihutu Co-authored-by: zhi22915 --- rustfs/src/storage/helper.rs | 176 ++++++++++++++++++++++++++++++----- 1 file changed, 153 insertions(+), 23 deletions(-) diff --git a/rustfs/src/storage/helper.rs b/rustfs/src/storage/helper.rs index b8eb3c15d..805b5a870 100644 --- a/rustfs/src/storage/helper.rs +++ b/rustfs/src/storage/helper.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::module_switches::{is_audit_module_enabled, is_notify_module_enabled}; -use crate::shared_types::convert_ecstore_object_info; +use crate::shared_types::{RemoteAddr, convert_ecstore_object_info}; use crate::storage::access::{ReqInfo, request_context_from_req}; use crate::storage::request_context::RequestContext; use crate::storage::sse::KmsRequestAuditScope; @@ -33,6 +33,7 @@ use rustfs_s3_types::EventName; use rustfs_targets::{ extract_params_header, extract_req_params, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent, }; +use rustfs_trusted_proxies::ClientInfo; use rustfs_utils::http::headers::AMZ_REQUEST_ID; use s3s::{S3Request, S3Response, S3Result}; use serde_json::Value; @@ -150,15 +151,6 @@ impl OperationHelper { .filter(|value| !value.is_empty()) .unwrap_or(path_object_key); - // Infer remote address - let remote_host = req - .headers - .get(rustfs_utils::http::X_FORWARDED_FOR) - .and_then(|v| v.to_str().ok()) - .or_else(|| req.headers.get(rustfs_utils::http::X_REAL_IP).and_then(|v| v.to_str().ok())) - .unwrap_or("") - .to_string(); - let trigger = op.as_str(); // Initialize audit builder @@ -179,15 +171,30 @@ impl OperationHelper { let request_id = request_context.request_id.clone(); let audit_builder = if audit_enabled { - Some( - AuditEntryBuilder::new("1.0", event, trigger, api_builder.clone().build()) - .remote_host(remote_host) - .user_agent(get_request_user_agent(&req.headers)) - .req_host(get_request_host(&req.headers)) - .req_path(req.uri.path().to_string()) - .req_query(extract_req_params(req)) - .request_id(&request_id), - ) + let remote_host = req + .extensions + .get::() + .map(|info| info.real_ip.to_string()) + .or_else(|| { + req.extensions + .get::>() + .and_then(|addr| addr.map(|addr| addr.0.ip().to_string())) + }) + .unwrap_or_default(); + let mut builder = AuditEntryBuilder::new("1.0", event, trigger, api_builder.clone().build()) + .remote_host(remote_host) + .user_agent(get_request_user_agent(&req.headers)) + .req_host(get_request_host(&req.headers)) + .req_path(req.uri.path().to_string()) + .req_query(extract_req_params(req)) + .request_id(&request_id); + if let Some(cred) = req_info + .and_then(|info| info.cred.as_ref()) + .filter(|cred| !cred.access_key.is_empty()) + { + builder = builder.access_key(&cred.access_key); + } + Some(builder) } else { None }; @@ -346,10 +353,6 @@ impl OperationHelper { final_builder = final_builder.error(err); } - if let Some(cred) = runtime_sources::current_action_credentials() { - final_builder = final_builder.access_key(&cred.access_key); - } - // Inject OpenTelemetry trace context into audit tags for distributed tracing correlation let mut tags = HashMap::new(); if let Some(ref tid) = state.request_context.trace_id { @@ -572,6 +575,133 @@ mod tests { ); } + #[test] + fn operation_helper_audits_the_authenticated_request_identity() { + with_vars( + [ + (rustfs_config::ENV_NOTIFY_ENABLE, Some("false")), + (rustfs_config::ENV_AUDIT_ENABLE, Some("true")), + ], + || { + refresh_notify_module_enabled(); + refresh_audit_module_enabled(); + + for (access_key, parent_user) in [ + (Some("testuser"), ""), + (Some("rustfsadmin"), ""), + (Some("service-account"), "testuser"), + (Some("sts-session"), "testuser"), + (None, ""), + (Some(""), ""), + ] { + let expected_access_key = access_key.filter(|key| !key.is_empty()); + for outcome in [None, Some(true), Some(false)] { + let input = GetObjectInput::builder() + .bucket("audit-bucket".to_string()) + .key("object.txt".to_string()) + .build() + .expect("get object input should build"); + let mut req = build_request(input, Method::GET, Uri::from_static("/audit-bucket/object.txt")); + req.extensions.insert(ReqInfo { + cred: access_key.map(|key| Credentials { + access_key: key.to_string(), + secret_key: "audit-secret-must-not-leak".to_string(), + session_token: "audit-token-must-not-leak".to_string(), + parent_user: parent_user.to_string(), + ..Default::default() + }), + ..Default::default() + }); + let mut helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject); + if let Some(success) = outcome { + let result = if success { + Ok(S3Response::new(GetObjectOutput::default())) + } else { + Err(S3Error::new(S3ErrorCode::NoSuchKey)) + }; + helper = helper.complete(&result); + } + let OperationHelper::Enabled(state) = &mut helper else { + panic!("helper should be enabled when the audit switch is on"); + }; + let entry = state.audit_builder.take().expect("audit builder should exist").build(); + assert_eq!(entry.access_key.as_deref(), expected_access_key); + let json = serde_json::to_value(&entry).expect("audit entry should serialize"); + assert_eq!(json.get("accessKey").and_then(|key| key.as_str()), expected_access_key); + let rendered = serde_json::to_string(&entry).expect("audit entry should serialize"); + assert!(!rendered.contains("audit-secret-must-not-leak")); + assert!(!rendered.contains("audit-token-must-not-leak")); + } + } + }, + ); + } + + #[test] + fn operation_helper_audits_verified_client_addresses() { + use crate::shared_types::RemoteAddr; + use rustfs_trusted_proxies::{ClientInfo, ValidationMode}; + use std::net::SocketAddr; + + with_vars( + [ + (rustfs_config::ENV_NOTIFY_ENABLE, Some("false")), + (rustfs_config::ENV_AUDIT_ENABLE, Some("true")), + ], + || { + refresh_notify_module_enabled(); + refresh_audit_module_enabled(); + + let peer: SocketAddr = "192.0.2.10:54321".parse().unwrap(); + let ipv6_peer: SocketAddr = "[2001:db8::10]:54321".parse().unwrap(); + let proxied = ClientInfo::from_trusted_proxy( + "198.51.100.20".parse().unwrap(), + None, + None, + peer.ip(), + 1, + ValidationMode::Strict, + Vec::new(), + ); + for (remote_addr, client_info, expected) in [ + (Some(RemoteAddr(peer)), None, "192.0.2.10"), + (Some(RemoteAddr(ipv6_peer)), None, "2001:db8::10"), + (Some(RemoteAddr(peer)), Some(proxied), "198.51.100.20"), + (Some(RemoteAddr(peer)), Some(ClientInfo::direct(peer)), "192.0.2.10"), + (None, None, ""), + ] { + for spoofed in [false, true] { + let input = GetObjectInput::builder() + .bucket("audit-bucket".to_string()) + .key("object.txt".to_string()) + .build() + .expect("get object input should build"); + let mut req = build_request(input, Method::GET, Uri::from_static("/audit-bucket/object.txt")); + req.extensions.insert(remote_addr); + if let Some(client_info) = client_info.as_ref() { + req.extensions.insert(client_info.clone()); + } + if spoofed { + req.headers + .insert("x-forwarded-for", HeaderValue::from_static("203.0.113.99")); + req.headers.insert("x-real-ip", HeaderValue::from_static("203.0.113.98")); + } + let result = Ok(S3Response::new(GetObjectOutput::default())); + let mut helper = + OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject).complete(&result); + let OperationHelper::Enabled(state) = &mut helper else { + panic!("helper should be enabled when the audit switch is on"); + }; + let entry = state.audit_builder.take().expect("audit builder should exist").build(); + assert_eq!(entry.remote_host.as_deref(), Some(expected)); + let rendered = serde_json::to_value(&entry).expect("audit entry should serialize"); + assert_eq!(rendered["remotehost"], expected); + } + } + }, + ); + } + #[test] fn operation_helper_adds_objects_to_audit_details() { with_vars( From fb5c002a30a57cdadad46b49021dbc70fbce6000 Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Tue, 13 Oct 2026 18:59:29 +0800 Subject: [PATCH 014/143] refactor(iam): encapsulate federated authorization models (#8514) --- crates/iam/src/federation/mapper.rs | 245 +++++++++++++++++- crates/iam/src/federation/mod.rs | 10 +- crates/iam/src/federation/model.rs | 153 +---------- crates/iam/src/federation/test_support.rs | 54 ++++ crates/iam/src/federation/transaction.rs | 51 ++-- crates/iam/src/oidc/recovery_tests.rs | 6 +- crates/iam/src/oidc/runtime_tests.rs | 10 +- docs/architecture/crate-boundaries.md | 2 + rustfs/src/admin/handlers/oidc.rs | 10 +- rustfs/src/admin/handlers/sts.rs | 2 +- .../src/admin/service/federated_identity.rs | 157 +++++------ 11 files changed, 440 insertions(+), 260 deletions(-) create mode 100644 crates/iam/src/federation/test_support.rs diff --git a/crates/iam/src/federation/mapper.rs b/crates/iam/src/federation/mapper.rs index 1dca51e5c..6680a2471 100644 --- a/crates/iam/src/federation/mapper.rs +++ b/crates/iam/src/federation/mapper.rs @@ -12,12 +12,123 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{FederatedAuthorization, VerifiedFederatedIdentity}; +use super::{FederatedClaims, VerifiedFederatedIdentity}; use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC}; use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive}; +use rustfs_utils::HashAlgorithm; use std::{collections::HashMap, sync::Arc}; use tracing::debug; +#[cfg(any(test, feature = "test-util"))] +#[path = "test_support.rs"] +pub(super) mod test_support; + +/// Read-only authorization result produced by core federation mapping. +/// +/// The claims representation stays inside IAM: +/// +/// ```compile_fail +/// use rustfs_iam::federation::FederatedClaims; +/// ``` +/// +/// Callers cannot mutate mapped fields: +/// +/// ```compile_fail +/// use rustfs_iam::federation::FederatedAuthorization; +/// +/// fn mutate(authorization: &mut FederatedAuthorization) { +/// authorization.groups.clear(); +/// } +/// ``` +/// +/// Callers cannot construct authorization results: +/// +/// ```compile_fail +/// use rustfs_iam::federation::FederatedAuthorization; +/// +/// fn construct(authorization: FederatedAuthorization) -> FederatedAuthorization { +/// FederatedAuthorization { ..authorization } +/// } +/// ``` +#[derive(Debug, Clone)] +pub struct FederatedAuthorization { + provider_id: String, + claims: FederatedClaims, + policies: Vec, + group_claim_policies: Vec, + groups: Vec, + roles_claim_key: Option, + roles: Vec, +} + +impl FederatedAuthorization { + pub fn provider_id(&self) -> &str { + &self.provider_id + } + + pub fn subject(&self) -> &str { + &self.claims.sub + } + + pub fn email(&self) -> &str { + &self.claims.email + } + + pub fn username(&self) -> &str { + &self.claims.username + } + + pub fn session_identity(&self) -> String { + self.claims.session_identity() + } + + pub fn policies(&self) -> &[String] { + &self.policies + } + + /// Subset of `policies` derived only from the groups claim (including merged roles claim + /// values); these may be ignored when no policy of that name exists. Everything else in + /// `policies` must resolve. + pub fn group_claim_policies(&self) -> &[String] { + &self.group_claim_policies + } + + pub fn mapped_iam_groups(&self) -> &[String] { + &self.groups + } + + pub fn roles_claim_key(&self) -> Option<&str> { + self.roles_claim_key.as_deref() + } + + pub fn roles(&self) -> &[String] { + &self.roles + } + + pub fn has_authorization_context(&self) -> bool { + !self.policies.is_empty() || !self.groups.is_empty() + } + + pub fn oidc_virtual_parent(&self) -> Option { + let issuer = self.claims.raw.get("iss")?.as_str()?; + let subject = self.claims.sub.as_str(); + if issuer.is_empty() || subject.is_empty() { + return None; + } + + let subject_len = u64::try_from(subject.len()).ok()?; + let issuer_len = u64::try_from(issuer.len()).ok()?; + let mut source = Vec::with_capacity(23 + subject.len() + issuer.len()); + source.extend_from_slice(b"openid:"); + source.extend_from_slice(&subject_len.to_be_bytes()); + source.extend_from_slice(subject.as_bytes()); + source.extend_from_slice(&issuer_len.to_be_bytes()); + source.extend_from_slice(issuer.as_bytes()); + let digest = HashAlgorithm::SHA256.hash_encode(&source); + Some(base64_simd::URL_SAFE_NO_PAD.encode_to_string(digest.as_ref())) + } +} + #[derive(Clone)] pub(crate) struct FederatedAuthorizationRule { provider_id: String, @@ -308,8 +419,136 @@ fn claim_value_type(value: &serde_json::Value) -> &'static str { #[cfg(test)] mod tests { use super::*; - use crate::federation::{FederatedClaims, FederatedProviderRef, VerifiedFederatedIdentity}; - use serde_json::json; + use crate::federation::{FederatedAuthorizationFixture, FederatedClaims, FederatedProviderRef, VerifiedFederatedIdentity}; + use serde_json::{Value, json}; + + fn authorization(policies: Vec, groups: Vec) -> FederatedAuthorization { + FederatedAuthorizationFixture { + provider_id: "standard_oidc".to_string(), + sub: "subject".to_string(), + attributes: HashMap::from([("iss".to_string(), Value::String("https://idp.example.test".to_string()))]), + policies, + mapped_iam_groups: groups, + ..Default::default() + } + .into_authorization() + } + + #[test] + fn authorization_context_accepts_policy_or_group() { + assert!(!authorization(Vec::new(), Vec::new()).has_authorization_context()); + assert!(authorization(vec!["consoleAdmin".to_string()], Vec::new()).has_authorization_context()); + assert!(authorization(Vec::new(), vec!["RustFS.ConsoleAdmin".to_string()]).has_authorization_context()); + } + + #[test] + fn oidc_virtual_parent_is_issuer_scoped() { + let first = authorization(Vec::new(), Vec::new()); + let mut second = first.clone(); + second + .claims + .raw + .insert("iss".to_string(), Value::String("https://other-idp.example.test".to_string())); + + assert_eq!( + first.oidc_virtual_parent().as_deref(), + Some("HwDfWftzOy4jiuS3WjKytC_Sg_A2hKhrRAFtBDhoBr0") + ); + assert!(!rustfs_policy::auth::contains_reserved_chars( + first.oidc_virtual_parent().as_deref().expect("virtual parent") + )); + assert_ne!(first.oidc_virtual_parent(), second.oidc_virtual_parent()); + } + + #[test] + fn oidc_virtual_parent_length_delimits_identity_parts() { + let mut first = authorization(Vec::new(), Vec::new()); + first.claims.sub = "subject".to_string(); + first.claims.raw.insert( + "iss".to_string(), + Value::String("https://issuer.example/path:https://other.example".to_string()), + ); + let mut second = authorization(Vec::new(), Vec::new()); + second.claims.sub = "subject:https://issuer.example/path".to_string(); + second + .claims + .raw + .insert("iss".to_string(), Value::String("https://other.example".to_string())); + + assert_ne!(first.oidc_virtual_parent(), second.oidc_virtual_parent()); + } + + #[test] + fn oidc_virtual_parent_requires_verified_identity_parts() { + let mut missing_issuer = authorization(Vec::new(), Vec::new()); + missing_issuer.claims.raw.clear(); + let mut missing_subject = authorization(Vec::new(), Vec::new()); + missing_subject.claims.sub.clear(); + + assert!(missing_issuer.oidc_virtual_parent().is_none()); + assert!(missing_subject.oidc_virtual_parent().is_none()); + } + + #[test] + fn oidc_virtual_parent_preserves_exact_subject() { + let plain = authorization(Vec::new(), Vec::new()); + let mut padded = plain.clone(); + padded.claims.sub = format!(" {} ", plain.claims.sub); + + assert_ne!(plain.oidc_virtual_parent(), padded.oidc_virtual_parent()); + } + + #[test] + fn authorization_accessors_preserve_exact_values_and_distinct_groups() { + let attributes = HashMap::from([("custom".to_string(), json!(["value", null, 7]))]); + let authorization = FederatedAuthorizationFixture { + provider_id: " Corp Provider ".to_string(), + sub: " subject ".to_string(), + email: " email@example.test ".to_string(), + username: " username ".to_string(), + source_groups: vec!["different-source-group".to_string()], + attributes: attributes.clone(), + policies: vec![" policy-b ".to_string(), "policy-a".to_string(), " policy-b ".to_string()], + group_claim_policies: vec![" policy-b ".to_string(), " policy-b ".to_string()], + mapped_iam_groups: vec!["mapped-a".to_string(), "mapped-b".to_string()], + roles_claim_key: Some(" Roles ".to_string()), + roles: vec!["role-b".to_string(), " role-a ".to_string(), "role-b".to_string()], + } + .into_authorization(); + + assert_eq!(authorization.provider_id(), " Corp Provider "); + assert_eq!(authorization.subject(), " subject "); + assert_eq!(authorization.email(), " email@example.test "); + assert_eq!(authorization.username(), " username "); + assert_eq!(authorization.session_identity(), " username "); + assert_eq!(authorization.policies(), [" policy-b ", "policy-a", " policy-b "]); + assert_eq!(authorization.group_claim_policies(), [" policy-b ", " policy-b "]); + assert_eq!(authorization.claims.groups, ["different-source-group"]); + assert_eq!(authorization.claims.raw, attributes); + assert_eq!(authorization.mapped_iam_groups(), ["mapped-a", "mapped-b"]); + assert_eq!(authorization.roles_claim_key(), Some(" Roles ")); + assert_eq!(authorization.roles(), ["role-b", " role-a ", "role-b"]); + } + + #[test] + fn default_authorization_fixture_has_no_identity_or_mapping() { + let authorization = FederatedAuthorizationFixture::default().into_authorization(); + + assert_eq!(authorization.provider_id(), ""); + assert_eq!(authorization.subject(), ""); + assert_eq!(authorization.email(), ""); + assert_eq!(authorization.username(), ""); + assert_eq!(authorization.session_identity(), "oidc-user-unknown"); + assert!(authorization.claims.groups.is_empty()); + assert!(authorization.claims.raw.is_empty()); + assert!(authorization.policies().is_empty()); + assert!(authorization.group_claim_policies().is_empty()); + assert!(authorization.mapped_iam_groups().is_empty()); + assert!(authorization.roles_claim_key().is_none()); + assert!(authorization.roles().is_empty()); + assert!(!authorization.has_authorization_context()); + assert!(authorization.oidc_virtual_parent().is_none()); + } fn mapper(rule: FederatedAuthorizationRule) -> CoreFederatedAuthorizationMapper { CoreFederatedAuthorizationMapper::new(FederatedAuthorizationRules::new([rule])) diff --git a/crates/iam/src/federation/mod.rs b/crates/iam/src/federation/mod.rs index b7498cf0a..6b54f62e4 100644 --- a/crates/iam/src/federation/mod.rs +++ b/crates/iam/src/federation/mod.rs @@ -24,11 +24,13 @@ pub use error::{FederatedSessionBindingError, FederationError, Result}; pub(crate) use mapper::FederatedAuthorizationRule; #[cfg(test)] pub(crate) use mapper::FederatedAuthorizationRuleRef; -pub use mapper::{CoreFederatedAuthorizationMapper, FederatedAuthorizationRules}; -pub(crate) use model::OpaqueLogoutContinuation; +#[cfg(any(test, feature = "test-util"))] +pub use mapper::test_support::FederatedAuthorizationFixture; +pub use mapper::{CoreFederatedAuthorizationMapper, FederatedAuthorization, FederatedAuthorizationRules}; +pub(crate) use model::{FederatedClaims, OpaqueLogoutContinuation}; pub use model::{ - FederatedAuthorization, FederatedClaims, FederatedLoginSession, FederatedProviderRef, FederatedSession, - FederatedSessionTransaction, OIDC_VIRTUAL_PARENT_CLAIM, VerifiedFederatedIdentity, + FederatedLoginSession, FederatedProviderRef, FederatedSession, FederatedSessionTransaction, OIDC_VIRTUAL_PARENT_CLAIM, + VerifiedFederatedIdentity, }; pub(crate) use provider::{FederatedProviderQuery, StandardOidcAuthentication, VerifiedFederatedCodeExchange}; pub use provider::{FederatedProviderView, FederatedRedirectPolicy}; diff --git a/crates/iam/src/federation/model.rs b/crates/iam/src/federation/model.rs index 76b8a9816..54b6725b9 100644 --- a/crates/iam/src/federation/model.rs +++ b/crates/iam/src/federation/model.rs @@ -13,23 +13,24 @@ // limitations under the License. use rustfs_credentials::Credentials; -use rustfs_utils::HashAlgorithm; use serde_json::Value; use std::{collections::HashMap, fmt}; +use super::FederatedAuthorization; + pub const OIDC_VIRTUAL_PARENT_CLAIM: &str = "x-rustfs-internal-oidc-parent"; #[derive(Debug, Clone)] -pub struct FederatedClaims { - pub sub: String, - pub email: String, - pub username: String, - pub groups: Vec, - pub raw: HashMap, +pub(crate) struct FederatedClaims { + pub(crate) sub: String, + pub(crate) email: String, + pub(crate) username: String, + pub(crate) groups: Vec, + pub(crate) raw: HashMap, } impl FederatedClaims { - pub fn session_identity(&self) -> String { + pub(crate) fn session_identity(&self) -> String { if !self.username.is_empty() { self.username.clone() } else if !self.email.is_empty() { @@ -125,45 +126,6 @@ impl VerifiedFederatedIdentity { } } -#[derive(Debug, Clone)] -pub struct FederatedAuthorization { - pub provider_id: String, - pub claims: FederatedClaims, - pub policies: Vec, - /// Subset of `policies` derived only from the groups claim (including merged roles claim - /// values); these may be ignored when no policy of that name exists. Everything else in - /// `policies` must resolve. - pub group_claim_policies: Vec, - pub groups: Vec, - pub roles_claim_key: Option, - pub roles: Vec, -} - -impl FederatedAuthorization { - pub fn has_authorization_context(&self) -> bool { - !self.policies.is_empty() || !self.groups.is_empty() - } - - pub fn oidc_virtual_parent(&self) -> Option { - let issuer = self.claims.raw.get("iss")?.as_str()?; - let subject = self.claims.sub.as_str(); - if issuer.is_empty() || subject.is_empty() { - return None; - } - - let subject_len = u64::try_from(subject.len()).ok()?; - let issuer_len = u64::try_from(issuer.len()).ok()?; - let mut source = Vec::with_capacity(23 + subject.len() + issuer.len()); - source.extend_from_slice(b"openid:"); - source.extend_from_slice(&subject_len.to_be_bytes()); - source.extend_from_slice(subject.as_bytes()); - source.extend_from_slice(&issuer_len.to_be_bytes()); - source.extend_from_slice(issuer.as_bytes()); - let digest = HashAlgorithm::SHA256.hash_encode(&source); - Some(base64_simd::URL_SAFE_NO_PAD.encode_to_string(digest.as_ref())) - } -} - /// Carries the provider-bound ID token between code exchange and logout token creation. pub(crate) struct OpaqueLogoutContinuation { provider: FederatedProviderRef, @@ -224,21 +186,6 @@ mod tests { } } - fn authorization(policies: Vec, groups: Vec) -> FederatedAuthorization { - FederatedAuthorization { - provider_id: "standard_oidc".to_string(), - claims: FederatedClaims { - raw: HashMap::from([("iss".to_string(), Value::String("https://idp.example.test".to_string()))]), - ..claims("", "", "subject") - }, - policies, - group_claim_policies: Vec::new(), - groups, - roles_claim_key: None, - roles: Vec::new(), - } - } - #[test] fn session_identity_preserves_existing_fallback_order() { assert_eq!(claims("john", "john@example.com", "sub-1").session_identity(), "john"); @@ -247,70 +194,6 @@ mod tests { assert_eq!(claims("", "", "").session_identity(), "oidc-user-unknown"); } - #[test] - fn authorization_context_accepts_policy_or_group() { - assert!(!authorization(Vec::new(), Vec::new()).has_authorization_context()); - assert!(authorization(vec!["consoleAdmin".to_string()], Vec::new()).has_authorization_context()); - assert!(authorization(Vec::new(), vec!["RustFS.ConsoleAdmin".to_string()]).has_authorization_context()); - } - - #[test] - fn oidc_virtual_parent_is_issuer_scoped() { - let first = authorization(Vec::new(), Vec::new()); - let mut second = first.clone(); - second - .claims - .raw - .insert("iss".to_string(), Value::String("https://other-idp.example.test".to_string())); - - assert_eq!( - first.oidc_virtual_parent().as_deref(), - Some("HwDfWftzOy4jiuS3WjKytC_Sg_A2hKhrRAFtBDhoBr0") - ); - assert!(!rustfs_policy::auth::contains_reserved_chars( - first.oidc_virtual_parent().as_deref().expect("virtual parent") - )); - assert_ne!(first.oidc_virtual_parent(), second.oidc_virtual_parent()); - } - - #[test] - fn oidc_virtual_parent_length_delimits_identity_parts() { - let mut first = authorization(Vec::new(), Vec::new()); - first.claims.sub = "subject".to_string(); - first.claims.raw.insert( - "iss".to_string(), - Value::String("https://issuer.example/path:https://other.example".to_string()), - ); - let mut second = authorization(Vec::new(), Vec::new()); - second.claims.sub = "subject:https://issuer.example/path".to_string(); - second - .claims - .raw - .insert("iss".to_string(), Value::String("https://other.example".to_string())); - - assert_ne!(first.oidc_virtual_parent(), second.oidc_virtual_parent()); - } - - #[test] - fn oidc_virtual_parent_requires_verified_identity_parts() { - let mut missing_issuer = authorization(Vec::new(), Vec::new()); - missing_issuer.claims.raw.clear(); - let mut missing_subject = authorization(Vec::new(), Vec::new()); - missing_subject.claims.sub.clear(); - - assert!(missing_issuer.oidc_virtual_parent().is_none()); - assert!(missing_subject.oidc_virtual_parent().is_none()); - } - - #[test] - fn oidc_virtual_parent_preserves_exact_subject() { - let plain = authorization(Vec::new(), Vec::new()); - let mut padded = plain.clone(); - padded.claims.sub = format!(" {} ", plain.claims.sub); - - assert_ne!(plain.oidc_virtual_parent(), padded.oidc_virtual_parent()); - } - #[test] fn verified_identity_preserves_exact_values_and_distinct_groups() { let attributes = HashMap::from([ @@ -341,24 +224,6 @@ mod tests { assert!(!debug.contains(" username ")); assert!(!debug.contains(" source-b ")); assert!(!debug.contains("identity-attribute-poison")); - - let authorization = FederatedAuthorization { - provider_id: " Corp Provider ".to_string(), - claims: FederatedClaims { - sub: "subject".to_string(), - email: String::new(), - username: String::new(), - groups: vec!["different-source-group".to_string()], - raw: HashMap::new(), - }, - policies: Vec::new(), - group_claim_policies: Vec::new(), - groups: vec!["mapped-a".to_string(), "mapped-b".to_string()], - roles_claim_key: None, - roles: Vec::new(), - }; - assert_eq!(authorization.claims.groups, ["different-source-group"]); - assert_eq!(authorization.groups, ["mapped-a", "mapped-b"]); } #[test] diff --git a/crates/iam/src/federation/test_support.rs b/crates/iam/src/federation/test_support.rs new file mode 100644 index 000000000..3933ae910 --- /dev/null +++ b/crates/iam/src/federation/test_support.rs @@ -0,0 +1,54 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{FederatedAuthorization, FederatedClaims}; +use serde_json::Value; +use std::collections::HashMap; + +/// Exact authorization inputs for tests that exercise the session binding boundary. +#[derive(Default)] +pub struct FederatedAuthorizationFixture { + pub provider_id: String, + pub sub: String, + pub email: String, + pub username: String, + pub source_groups: Vec, + pub attributes: HashMap, + pub policies: Vec, + pub group_claim_policies: Vec, + pub mapped_iam_groups: Vec, + pub roles_claim_key: Option, + pub roles: Vec, +} + +impl FederatedAuthorizationFixture { + /// Preserves supplied values without authentication, validation, or policy mapping. + pub fn into_authorization(self) -> FederatedAuthorization { + FederatedAuthorization { + provider_id: self.provider_id, + claims: FederatedClaims { + sub: self.sub, + email: self.email, + username: self.username, + groups: self.source_groups, + raw: self.attributes, + }, + policies: self.policies, + group_claim_policies: self.group_claim_policies, + groups: self.mapped_iam_groups, + roles_claim_key: self.roles_claim_key, + roles: self.roles, + } + } +} diff --git a/crates/iam/src/federation/transaction.rs b/crates/iam/src/federation/transaction.rs index 3f7ec15dc..84f4c2969 100644 --- a/crates/iam/src/federation/transaction.rs +++ b/crates/iam/src/federation/transaction.rs @@ -17,6 +17,7 @@ use super::{ FederatedProviderView, FederatedRedirectPolicy, FederatedSession, FederatedSessionBinding, FederatedSessionTransaction, FederationError, OpaqueLogoutContinuation, Result, StandardOidcAuthentication, }; +use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC}; use std::sync::Arc; const DEFAULT_OIDC_PROVIDER_ID: &str = "default"; @@ -133,22 +134,28 @@ impl FederatedIdentityService { let authorization = self.verify_identity(jwt).await?; if !authorization.has_authorization_context() { tracing::warn!( - provider_id = %authorization.provider_id, - username = %authorization.claims.username, - sub = %authorization.claims.sub, - policy_count = authorization.policies.len(), - group_count = authorization.groups.len(), + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + provider_id = %authorization.provider_id(), + username = %authorization.username(), + sub = %authorization.subject(), + policy_count = authorization.policies().len(), + group_count = authorization.mapped_iam_groups().len(), "AssumeRoleWithWebIdentity has no mapped policies or groups" ); return Err(FederationError::NoAuthorizationContext); } tracing::debug!( - provider_id = %authorization.provider_id, - username = %authorization.claims.username, - policy_count = authorization.policies.len(), - group_count = authorization.groups.len(), - policies = ?authorization.policies, - groups = ?authorization.groups, + event = EVENT_OIDC_DIAGNOSTICS, + component = LOG_COMPONENT_IAM, + subsystem = LOG_SUBSYSTEM_OIDC, + provider_id = %authorization.provider_id(), + username = %authorization.username(), + policy_count = authorization.policies().len(), + group_count = authorization.mapped_iam_groups().len(), + policies = ?authorization.policies(), + groups = ?authorization.mapped_iam_groups(), "AssumeRoleWithWebIdentity mapped OIDC policies and groups" ); @@ -379,19 +386,19 @@ mod tests { ) -> core::result::Result { self.events.lock().expect("event log should not be poisoned").push("bind"); self.transactions.lock().expect("transactions should not be poisoned").push(( - transaction.authorization.provider_id.clone(), + transaction.authorization.provider_id().to_string(), transaction.duration_seconds, transaction.session_policy.clone(), )); self.mapped_policies .lock() .expect("mapped policies should not be poisoned") - .push(transaction.authorization.policies.clone()); + .push(transaction.authorization.policies().to_vec()); if self.fail { return Err(FederatedSessionBindingError::Internal("binding failed".to_string())); } Ok(Credentials { - access_key: transaction.authorization.claims.session_identity(), + access_key: transaction.authorization.session_identity(), ..Default::default() }) } @@ -487,7 +494,7 @@ mod tests { .await .expect("callback flow should complete"); assert_eq!(login.session.credentials.access_key, "user"); - assert_eq!(login.session.authorization.provider_id, "corp"); + assert_eq!(login.session.authorization.provider_id(), "corp"); assert_eq!(login.redirect_after.as_deref(), Some("/browser")); assert_eq!(login.logout_token, "logout-token"); assert_eq!( @@ -501,7 +508,7 @@ mod tests { .await .expect("web identity flow should complete"); assert_eq!(web_identity.credentials.access_key, "user"); - assert_eq!(web_identity.authorization.provider_id, "partner"); + assert_eq!(web_identity.authorization.provider_id(), "partner"); assert_eq!(events.lock().expect("event log should not be poisoned").as_slice(), ["verify", "bind"]); assert_eq!( binding @@ -530,8 +537,8 @@ mod tests { .complete_authorization_code("state", "code", "https://console.example/callback", 3600, binding.as_ref()) .await .expect("standard callback flow should complete"); - assert_eq!(login.session.authorization.provider_id, "corp"); - assert_eq!(login.session.authorization.policies, ["readwrite"]); + assert_eq!(login.session.authorization.provider_id(), "corp"); + assert_eq!(login.session.authorization.policies(), ["readwrite"]); assert_eq!( events.lock().expect("event log should not be poisoned").as_slice(), ["exchange", "bind", "logout"] @@ -542,8 +549,8 @@ mod tests { .assume_role_with_web_identity("jwt", 7200, Some("session-policy".to_string()), binding.as_ref()) .await .expect("standard web identity flow should complete"); - assert_eq!(web_identity.authorization.provider_id, "partner"); - assert_eq!(web_identity.authorization.policies, ["readwrite"]); + assert_eq!(web_identity.authorization.provider_id(), "partner"); + assert_eq!(web_identity.authorization.policies(), ["readwrite"]); assert_eq!(events.lock().expect("event log should not be poisoned").as_slice(), ["verify", "bind"]); assert_eq!( binding @@ -586,8 +593,8 @@ mod tests { .await .expect("a mapped group is an authorization context"); - assert_eq!(session.authorization.policies, ["developers"]); - assert_eq!(session.authorization.groups, ["developers"]); + assert_eq!(session.authorization.policies(), ["developers"]); + assert_eq!(session.authorization.mapped_iam_groups(), ["developers"]); assert_eq!(events.lock().expect("event log should not be poisoned").as_slice(), ["verify", "bind"]); } diff --git a/crates/iam/src/oidc/recovery_tests.rs b/crates/iam/src/oidc/recovery_tests.rs index f1b500b85..96fece5bc 100644 --- a/crates/iam/src/oidc/recovery_tests.rs +++ b/crates/iam/src/oidc/recovery_tests.rs @@ -177,9 +177,9 @@ async fn web_identity_recovers_transport_failure_and_preserves_verification() { .await .expect("verify recovered workload token"); let authorization = mapper.map(identity); - assert_eq!(authorization.provider_id, "workload"); - assert_eq!(authorization.claims.sub, "recovered-user"); - assert_eq!(authorization.policies, vec!["team/readonly"]); + assert_eq!(authorization.provider_id(), "workload"); + assert_eq!(authorization.subject(), "recovered-user"); + assert_eq!(authorization.policies(), ["team/readonly"]); assert!( sys.authorize_url("workload", "https://console.example.com/callback", None) .await diff --git a/crates/iam/src/oidc/runtime_tests.rs b/crates/iam/src/oidc/runtime_tests.rs index 84dd95943..52181e530 100644 --- a/crates/iam/src/oidc/runtime_tests.rs +++ b/crates/iam/src/oidc/runtime_tests.rs @@ -371,12 +371,12 @@ async fn assert_complete_console_login(initially_unavailable: bool) { .await .unwrap_or_else(|err| panic!("hidden={hidden}, explicit={explicit}: {err}")); let (identity, redirect_after, _) = exchange.into_parts(); + assert_eq!(identity.source_groups(), ["readwrite"]); let authorization = mapper.map(identity); - assert_eq!(authorization.provider_id, "console"); - assert_eq!(authorization.claims.sub, "existing-user"); - assert_eq!(authorization.claims.email, "user@example.com"); - assert_eq!(authorization.claims.groups, vec!["readwrite"]); - assert_eq!(authorization.policies, vec!["team/readwrite"]); + assert_eq!(authorization.provider_id(), "console"); + assert_eq!(authorization.subject(), "existing-user"); + assert_eq!(authorization.email(), "user@example.com"); + assert_eq!(authorization.policies(), ["team/readwrite"]); assert_eq!(redirect_after.as_deref(), Some("/buckets")); assert!(matches!(sys.exchange_code(&state, "test-authorization-code", redirect).await, Err(error) if error == "invalid or expired OIDC state")); diff --git a/docs/architecture/crate-boundaries.md b/docs/architecture/crate-boundaries.md index 7a37c707f..9d940e37b 100644 --- a/docs/architecture/crate-boundaries.md +++ b/docs/architecture/crate-boundaries.md @@ -85,6 +85,8 @@ The server-config model (`Config`, `KV`, `KVS`) and the global server-config sna `crates/iam/src/oidc/` owns provider configuration, discovery, HTTP transport, login state, and the OIDC runtime. `crates/iam/src/federation/` owns verified identity transactions and authorization mapping; its production code does not import concrete OIDC runtime or provider configuration types. Production `FederatedAuthorization` values are constructed by `CoreFederatedAuthorizationMapper` in `crates/iam/src/federation/mapper.rs`. +`FederatedClaims` is crate-internal. `FederatedAuthorization` has private fields owned by the mapper module and exposes read-only accessors to session binding consumers, including `mapped_iam_groups()` for the IAM groups selected by authorization mapping. Test fixtures are exposed only through the `test-util` feature or unit-test configuration. + `rustfs/src/startup_auth.rs` constructs `StandardOidcAdapter` and publishes the federation service with its OIDC query as one runtime pair. `AppContext` stores and publishes that pair through `FederatedIdentityRuntimeSnapshot`; consumers receive interfaces rather than constructing a second OIDC runtime. `scripts/check_oidc_architecture_boundaries.sh` enforces these boundaries through the existing architecture guard, including the Keycloak workflow path for `crates/iam/src/oidc/`. `crates/iam/src/oidc/config.rs` owns OIDC configuration input, validation, persisted KVS conversion, and `OidcSiteReplicationSnapshot`. `rustfs/src/admin/service/oidc_config.rs` owns stored configuration reads, compare-and-save updates, restart comparison, and provider discovery validation; the OIDC Admin handler translates HTTP requests and errors. `OidcConfigQuery::site_replication_snapshot()` returns only the provider ID, claim name, role policy, client ID, and hashed client secret needed by site replication. `rustfs/src/admin/handlers/site_replication.rs` maps this narrow view to its existing response format without reading raw OIDC credentials. The Keycloak workflow tracks the Admin configuration service path along with IAM OIDC code. diff --git a/rustfs/src/admin/handlers/oidc.rs b/rustfs/src/admin/handlers/oidc.rs index 8a095d2e1..383ae9d72 100644 --- a/rustfs/src/admin/handlers/oidc.rs +++ b/rustfs/src/admin/handlers/oidc.rs @@ -477,7 +477,7 @@ impl Operation for OidcCallbackHandler { callback_federation_error(error) })?; let authorization = &login.session.authorization; - let actual_provider_id = &authorization.provider_id; + let actual_provider_id = authorization.provider_id(); debug!( event = EVENT_ADMIN_OIDC_STATE, @@ -493,10 +493,10 @@ impl Operation for OidcCallbackHandler { component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_OIDC, provider_id = %actual_provider_id, - policy_count = authorization.policies.len(), - group_count = authorization.groups.len(), - policies = ?authorization.policies, - groups = ?authorization.groups, + policy_count = authorization.policies().len(), + group_count = authorization.mapped_iam_groups().len(), + policies = ?authorization.policies(), + groups = ?authorization.mapped_iam_groups(), state = "claims_mapped", "admin oidc state" ); diff --git a/rustfs/src/admin/handlers/sts.rs b/rustfs/src/admin/handlers/sts.rs index 054a1d11c..75211130c 100644 --- a/rustfs/src/admin/handlers/sts.rs +++ b/rustfs/src/admin/handlers/sts.rs @@ -432,7 +432,7 @@ async fn handle_assume_role_with_web_identity(body: AssumeRoleRequest) -> S3Resu })?; let authorization = &session.authorization; let new_cred = &session.credentials; - let subject = authorization.claims.session_identity(); + let subject = authorization.session_identity(); let xml = assume_role_with_web_identity_response_xml(new_cred, &subject); diff --git a/rustfs/src/admin/service/federated_identity.rs b/rustfs/src/admin/service/federated_identity.rs index cdc197633..679539079 100644 --- a/rustfs/src/admin/service/federated_identity.rs +++ b/rustfs/src/admin/service/federated_identity.rs @@ -108,34 +108,33 @@ fn select_oidc_policies( fn build_oidc_token_claims(transaction: &FederatedSessionTransaction) -> HashMap { let authorization = &transaction.authorization; - let claims = &authorization.claims; let mut token_claims = HashMap::new(); - token_claims.insert("sub".to_string(), Value::String(claims.sub.clone())); + token_claims.insert("sub".to_string(), Value::String(authorization.subject().to_string())); token_claims.insert("iss".to_string(), Value::String("rustfs-oidc".to_string())); - token_claims.insert("oidc_provider".to_string(), Value::String(authorization.provider_id.clone())); + token_claims.insert("oidc_provider".to_string(), Value::String(authorization.provider_id().to_string())); - if !claims.email.is_empty() { - token_claims.insert("email".to_string(), Value::String(claims.email.clone())); + if !authorization.email().is_empty() { + token_claims.insert("email".to_string(), Value::String(authorization.email().to_string())); } - if !claims.username.is_empty() { - token_claims.insert("preferred_username".to_string(), Value::String(claims.username.clone())); + if !authorization.username().is_empty() { + token_claims.insert("preferred_username".to_string(), Value::String(authorization.username().to_string())); } - if !authorization.groups.is_empty() { + if !authorization.mapped_iam_groups().is_empty() { token_claims.insert( "groups".to_string(), Value::Array( authorization - .groups + .mapped_iam_groups() .iter() .map(|group| Value::String(group.clone())) .collect(), ), ); } - if !authorization.roles.is_empty() { + if !authorization.roles().is_empty() { token_claims.insert( "roles".to_string(), - Value::Array(authorization.roles.iter().cloned().map(Value::String).collect()), + Value::Array(authorization.roles().iter().cloned().map(Value::String).collect()), ); } @@ -246,7 +245,6 @@ fn issue_credentials( secret: Option<&str>, ) -> Result { let authorization = &transaction.authorization; - let claims = &authorization.claims; let parent_user = authorization.oidc_virtual_parent().ok_or_else(|| { FederatedSessionBindingError::InvalidRequest("verified OIDC identity is missing issuer or subject".to_string()) })?; @@ -266,16 +264,19 @@ fn issue_credentials( )); } debug!( - provider_id = %authorization.provider_id, + event = EVENT_ADMIN_OIDC_POLICY, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_OIDC, + provider_id = %authorization.provider_id(), parent_user = %parent_user, - email = %claims.email, - username = %claims.username, - sub = %claims.sub, - policy_count = authorization.policies.len(), - group_count = authorization.groups.len(), - policies = ?authorization.policies, - groups = ?authorization.groups, - roles_claim_key = ?authorization.roles_claim_key, + email = %authorization.email(), + username = %authorization.username(), + sub = %authorization.subject(), + policy_count = authorization.policies().len(), + group_count = authorization.mapped_iam_groups().len(), + policies = ?authorization.policies(), + groups = ?authorization.mapped_iam_groups(), + roles_claim_key = ?authorization.roles_claim_key(), has_session_policy = transaction.session_policy.is_some(), "OIDC STS credential claims prepared" ); @@ -295,7 +296,7 @@ fn issue_credentials( let mut credentials = get_new_credentials_with_metadata(&token_claims, secret) .map_err(|error| FederatedSessionBindingError::Internal(format!("credential generation failed: {error}")))?; credentials.parent_user = parent_user; - credentials.groups = Some(authorization.groups.clone()); + credentials.groups = Some(authorization.mapped_iam_groups().to_vec()); Ok(credentials) } @@ -329,12 +330,12 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { FederatedSessionBindingError::InvalidRequest("verified OIDC identity is missing issuer or subject".to_string()) })?; let (selected_policy_names, ignorable_policy_names) = match iam_store - .sts_policy_db_get(&parent_user, &Some(authorization.groups.clone())) + .sts_policy_db_get(&parent_user, &Some(authorization.mapped_iam_groups().to_vec())) .await .map_err(|_| FederatedSessionBindingError::Internal("failed to resolve OIDC policy mapping".to_string()))? { mapped_policy_names if !mapped_policy_names.is_empty() => (mapped_policy_names, &[][..]), - _ => (authorization.policies.clone(), authorization.group_claim_policies.as_slice()), + _ => (authorization.policies().to_vec(), authorization.group_claim_policies()), }; // A name may be skipped only after storage confirms the policy is absent. A failed lookup // must not be mistaken for absence, or an unreadable Deny policy would be dropped from the @@ -346,7 +347,7 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { event = EVENT_ADMIN_OIDC_POLICY, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_OIDC, - provider_id = %authorization.provider_id, + provider_id = %authorization.provider_id(), parent_user = %parent_user, policy = %policy_name, error = %err, @@ -377,7 +378,7 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { event = EVENT_ADMIN_OIDC_POLICY, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_OIDC, - provider_id = %authorization.provider_id, + provider_id = %authorization.provider_id(), parent_user = %parent_user, unsupported_policies = ?policy_names, "OIDC STS rejected existing policies whose names are not allowed in session claims" @@ -389,7 +390,7 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { event = EVENT_ADMIN_OIDC_POLICY, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_OIDC, - provider_id = %authorization.provider_id, + provider_id = %authorization.provider_id(), parent_user = %parent_user, unresolved_policies = ?policy_names, "OIDC STS rejected required policies without a current policy" @@ -405,7 +406,7 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { event = EVENT_ADMIN_OIDC_POLICY, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_OIDC, - provider_id = %authorization.provider_id, + provider_id = %authorization.provider_id(), parent_user = %parent_user, ignored_group_policies = ?selection.ignored, "OIDC STS ignoring group claim values without a matching policy" @@ -418,10 +419,10 @@ impl FederatedSessionBinding for DefaultFederatedSessionBinding { if tracing::enabled!(tracing::Level::DEBUG) { log_oidc_policy_diagnostics( &iam_store, - &authorization.provider_id, + authorization.provider_id(), &credentials.parent_user, &selected_policy_names, - &authorization.groups, + authorization.mapped_iam_groups(), ) .await; } @@ -444,7 +445,7 @@ mod tests { use super::*; use crate::admin::runtime_sources::{AppContext, current_object_store_handle, publish_test_app_context}; use hmac::{Hmac, KeyInit, Mac}; - use rustfs_iam::federation::{FederatedAuthorization, FederatedClaims}; + use rustfs_iam::federation::FederatedAuthorizationFixture; use rustfs_iam::store::{ Store, UserType, object::{IAM_CONFIG_POLICIES_PREFIX, IAM_CONFIG_PREFIX}, @@ -459,31 +460,37 @@ mod tests { use sha2::Sha512; use std::sync::Arc; - fn transaction() -> FederatedSessionTransaction { + fn authorization_fixture() -> FederatedAuthorizationFixture { + FederatedAuthorizationFixture { + provider_id: "default".to_string(), + sub: "subject".to_string(), + email: "user@example.com".to_string(), + username: "user".to_string(), + source_groups: vec!["source-group".to_string()], + attributes: HashMap::from([ + ("iss".to_string(), serde_json::json!("https://idp.example.test")), + ("raw_poison".to_string(), serde_json::json!("must-not-leak")), + ]), + policies: vec!["readwrite".to_string()], + group_claim_policies: Vec::new(), + mapped_iam_groups: vec!["devs".to_string()], + roles_claim_key: Some("roles".to_string()), + roles: vec!["admin".to_string(), "reader".to_string()], + } + } + + fn transaction_from_fixture(authorization: FederatedAuthorizationFixture) -> FederatedSessionTransaction { FederatedSessionTransaction { - authorization: FederatedAuthorization { - provider_id: "default".to_string(), - claims: FederatedClaims { - sub: "subject".to_string(), - email: "user@example.com".to_string(), - username: "user".to_string(), - groups: vec!["source-group".to_string()], - raw: HashMap::from([ - ("iss".to_string(), serde_json::json!("https://idp.example.test")), - ("raw_poison".to_string(), serde_json::json!("must-not-leak")), - ]), - }, - policies: vec!["readwrite".to_string()], - group_claim_policies: Vec::new(), - groups: vec!["devs".to_string()], - roles_claim_key: Some("roles".to_string()), - roles: vec!["admin".to_string(), "reader".to_string()], - }, + authorization: authorization.into_authorization(), duration_seconds: 3600, session_policy: None, } } + fn transaction() -> FederatedSessionTransaction { + transaction_from_fixture(authorization_fixture()) + } + async fn ready_test_iam() -> Arc> { let _ = rustfs_credentials::init_global_action_credentials( Some("TESTROOTACCESSKEY".to_string()), @@ -547,11 +554,12 @@ mod tests { #[test] fn token_claims_omit_empty_optional_oidc_fields() { - let mut transaction = transaction(); - transaction.authorization.claims.email.clear(); - transaction.authorization.claims.username.clear(); - transaction.authorization.groups.clear(); - transaction.authorization.roles.clear(); + let mut authorization = authorization_fixture(); + authorization.email.clear(); + authorization.username.clear(); + authorization.mapped_iam_groups.clear(); + authorization.roles.clear(); + let transaction = transaction_from_fixture(authorization); assert_eq!( build_oidc_token_claims(&transaction), @@ -650,9 +658,10 @@ mod tests { #[serial] async fn binding_uses_sts_policy_when_regular_mapping_collides() { let iam = ready_test_iam().await; - let mut transaction = transaction(); - transaction.authorization.claims.sub = "binding-sts-policy-subject".to_string(); - transaction.authorization.policies = vec!["readwrite".to_string()]; + let mut authorization = authorization_fixture(); + authorization.sub = "binding-sts-policy-subject".to_string(); + authorization.policies = vec!["readwrite".to_string()]; + let transaction = transaction_from_fixture(authorization); let parent = transaction .authorization .oidc_virtual_parent() @@ -766,7 +775,7 @@ mod tests { let mut transaction = transaction(); transaction.session_policy = Some("not-json".to_string()); - let error = issue_credentials(&transaction, &transaction.authorization.policies, None) + let error = issue_credentials(&transaction, transaction.authorization.policies(), None) .expect_err("invalid policy should fail first"); assert!(matches!(error, FederatedSessionBindingError::InvalidRequest(_))); } @@ -798,10 +807,11 @@ mod tests { #[test] fn missing_oidc_issuer_fails_closed_before_credential_issuance() { - let mut transaction = transaction(); - transaction.authorization.claims.raw.clear(); + let mut authorization = authorization_fixture(); + authorization.attributes.clear(); + let transaction = transaction_from_fixture(authorization); - let error = issue_credentials(&transaction, &transaction.authorization.policies, Some("signing-secret")) + let error = issue_credentials(&transaction, transaction.authorization.policies(), Some("signing-secret")) .expect_err("credential issuance should reject a missing issuer"); assert!(matches!(error, FederatedSessionBindingError::InvalidRequest(_))); } @@ -881,12 +891,12 @@ mod tests { } fn group_claim_transaction(subject: &str, groups: &[&str]) -> FederatedSessionTransaction { - let mut transaction = transaction(); - transaction.authorization.claims.sub = subject.to_string(); - transaction.authorization.groups = names(groups); - transaction.authorization.policies = names(groups); - transaction.authorization.group_claim_policies = names(groups); - transaction + let mut authorization = authorization_fixture(); + authorization.sub = subject.to_string(); + authorization.mapped_iam_groups = names(groups); + authorization.policies = names(groups); + authorization.group_claim_policies = names(groups); + transaction_from_fixture(authorization) } fn assert_binding_rejected(error: FederatedSessionBindingError, expected_message: &str) { @@ -914,7 +924,7 @@ mod tests { .expect("issued session token should verify"); assert_eq!(claims.get("policy"), Some(&serde_json::json!("readonly"))); - assert_eq!(credentials.groups.as_deref(), Some(transaction.authorization.groups.as_slice())); + assert_eq!(credentials.groups.as_deref(), Some(transaction.authorization.mapped_iam_groups())); let groups = credentials.groups.clone(); let conditions = HashMap::new(); @@ -1003,10 +1013,11 @@ mod tests { #[serial] async fn binding_keeps_role_policy_strict() { ready_test_iam().await; - let mut transaction = transaction(); - transaction.authorization.claims.sub = "binding-strict-role-policy-subject".to_string(); - transaction.authorization.policies = names(&["readonly", "missing-role-policy"]); - transaction.authorization.group_claim_policies = Vec::new(); + let mut authorization = authorization_fixture(); + authorization.sub = "binding-strict-role-policy-subject".to_string(); + authorization.policies = names(&["readonly", "missing-role-policy"]); + authorization.group_claim_policies = Vec::new(); + let transaction = transaction_from_fixture(authorization); let error = DefaultFederatedSessionBinding .bind(&transaction) From adf17bbfbaed0fb0f6c16a592ecd80be4642e70d Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 13 Oct 2026 19:12:28 +0800 Subject: [PATCH 015/143] fix(heal): reject incomplete administrator bucket discovery (#8512) --- .config/e2e-full-selection.txt | 4 +- .config/e2e-nightly-selection.txt | 4 +- .../src/heal_erasure_disk_rebuild_test.rs | 152 ++++++++++++++++++ crates/ecstore/src/store/bucket.rs | 91 ++++++++++- crates/heal/src/error.rs | 5 +- crates/heal/src/heal/channel.rs | 7 + crates/heal/src/heal/erasure_healer.rs | 13 ++ crates/heal/src/heal/erasure_healer/admin.rs | 3 +- .../resume_loop_tests/admin_outcome.rs | 47 ++++++ crates/heal/src/heal/manager/tests.rs | 29 ++++ .../heal/manager/tests/running_mainline.rs | 7 + crates/heal/src/heal/storage.rs | 12 ++ crates/heal/src/heal/task/heal_bucket.rs | 7 +- crates/heal/src/heal/task/tests.rs | 64 ++++++++ crates/heal/src/lib.rs | 7 + crates/heal/tests/heal_bug_fixes_test.rs | 14 ++ .../heal/tests/heal_concurrent_delete_test.rs | 7 + crates/heal/tests/storage_api.rs | 2 +- .../src/scanner_folder/tests/mrf_ownership.rs | 7 + crates/scanner/src/storage_api.rs | 2 + docs/operations/bucket-heal-recovery.md | 23 +++ rustfs/src/storage/rpc/node_service.rs | 7 + rustfs/src/storage/storage_api.rs | 2 + 23 files changed, 504 insertions(+), 12 deletions(-) diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index 2040896ee..8b2d39b00 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=b8937d6de585895a837b1505c7a9f91aa45394fbdc47cd179287a35a6f1de8be -sha256-linux=e803ee5fbd29bd45b8542b4256b75d5a2196ccc7ccfd1ccac6e14553730e9047 +sha256-darwin=a58501e7f6f1407201170c300f5fe91032a43addd4cb215306cf321ab59a0805 +sha256-linux=57791d182063049c5f0ca91ae0a6bf64006af6f991a5094e0cb7c8d8491187ed diff --git a/.config/e2e-nightly-selection.txt b/.config/e2e-nightly-selection.txt index 9a0869842..10bd6dfeb 100644 --- a/.config/e2e-nightly-selection.txt +++ b/.config/e2e-nightly-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=ce7b419faf1ff79840dfc6325d31ee8d1ca5eab85125b94d0bcd58cdf07f0f76 -sha256-linux=5e9e3d701ee873c13fb9094575f6889779c6078b1107624bb57663bef9b9d21f +sha256-darwin=8ef80b7353d1e0c4f863a2aca7e996d23858b8c32b9edf8fd272076fa63a83d8 +sha256-linux=99fe1cd128ade2df69b9e3ecf94b91021185a9769293420b1ba5d24760f2ea99 diff --git a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs index c5d2e20b7..b02fc304f 100644 --- a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs +++ b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs @@ -2633,4 +2633,156 @@ mod tests { }; assert!(!outage_candidate_targets_replacement(&insufficient_peer_shards, 12, replacement_set_slot)); } + async fn two_node_heal_status( + cluster: &RustFSTestClusterEnvironment, + bucket: Option<&str>, + recreate: bool, + scan_mode: u8, + ) -> Result> { + let url = format!("{}/rustfs/admin/v3/heal/{}", cluster.nodes[0].url, bucket.unwrap_or_default()); + let options = serde_json::json!({ + "recursive": true, "dryRun": false, "remove": false, + "recreate": recreate, "scanMode": scan_mode, "updateParity": false, "nolock": false, + }); + let started = + start_root_heal_when_control_ready(&url, &options.to_string(), &cluster.access_key, &cluster.secret_key).await?; + let started: serde_json::Value = serde_json::from_str(&started)?; + let token = started["clientToken"] + .as_str() + .ok_or_else(|| format!("missing heal token: {started}"))?; + let status_url = format!("{url}?clientToken={token}"); + let deadline = Instant::now() + Duration::from_secs(240); + loop { + let body = timeout( + Duration::from_secs(20), + signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key), + ) + .await??; + let status: serde_json::Value = serde_json::from_str(&body)?; + match status["summary"].as_str() { + Some("finished" | "stopped") => return Ok(status), + Some("running") if Instant::now() < deadline => sleep(Duration::from_millis(250)).await, + _ => return Err(format!("heal did not reach a terminal state: {status}").into()), + } + } + } + + async fn verify_two_node_directory_recovery(remove_bucket: bool, scan_mode: u8) -> Result<(), Box> { + init_logging(); + let mut cluster = RustFSTestClusterEnvironment::new(2).await?; + // Isolate explicit administrator recovery from scanner, MRF, and disk healing. + for key in [ + "RUSTFS_HEAL_AUTO_HEAL_ENABLE", + "RUSTFS_HEAL_MRF_ENABLE", + "RUSTFS_SCANNER_ENABLED", + "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", + ] { + cluster.set_env(key, "false"); + } + cluster.start().await?; + let source = cluster.create_s3_client(0)?; + let target = cluster.create_s3_client(1)?; + let bucket = "two-node-directory-recovery"; + source.create_bucket().bucket(bucket).send().await?; + let objects = [ + ("empty", Vec::new()), + ("nested/inline", vec![0x3c; 1024]), + ("nested/external", (0_u8..=255).cycle().take(1024 * 1024).collect::>()), + ]; + for (key, bytes) in &objects { + source + .put_object() + .bucket(bucket) + .key(*key) + .body(ByteStream::from(bytes.clone())) + .send() + .await?; + } + let target_bucket = Path::new(&cluster.nodes[1].data_dir).join(bucket); + let original = objects + .iter() + .map(|(key, _)| census_object_version_on_disk(Path::new(&cluster.nodes[1].data_dir), bucket, key, None)) + .collect::, _>>()?; + for census in &original { + assert!(census.has_xl_meta, "fixture must have a target metadata copy: {census:?}"); + } + assert!( + !original[2].present_part_fingerprints.is_empty(), + "large fixture must exercise external parts" + ); + let removed = Path::new(&cluster.temp_dir).join("removed"); + tokio::fs::create_dir(&removed).await?; + if remove_bucket { + tokio::fs::rename(&target_bucket, removed.join(bucket)).await?; + let status = two_node_heal_status(&cluster, None, false, scan_mode).await?; + assert_eq!( + status["summary"], "stopped", + "missing bucket must not look like an empty successful scan: {status}" + ); + assert_eq!(status["outcome"]["execution"]["state"], "aborted"); + assert_eq!(status["outcome"]["coverage"], "partial"); + assert_eq!(status["outcome"]["counters"]["processed"], 0); + assert!(!target_bucket.exists(), "default discovery must not silently recreate a bucket"); + let scoped_default = two_node_heal_status(&cluster, Some(bucket), false, scan_mode).await?; + assert_eq!( + scoped_default["summary"], "stopped", + "default namespace quorum protection must remain: {scoped_default}" + ); + } else { + for (index, (key, _)) in objects.iter().enumerate() { + tokio::fs::rename(target_bucket.join(key), removed.join(index.to_string())).await?; + } + } + let status = two_node_heal_status(&cluster, remove_bucket.then_some(bucket), remove_bucket, scan_mode).await?; + assert_eq!(status["summary"], "finished", "explicit recovery should complete: {status}"); + assert_eq!(status["outcome"]["execution"]["state"], "completed"); + assert_eq!(status["outcome"]["coverage"], "complete"); + for ((key, _), before) in objects.iter().zip(&original) { + let after = census_object_version_on_disk(Path::new(&cluster.nodes[1].data_dir), bucket, key, None)?; + assert!(after.has_xl_meta, "metadata must be restored before GET: {key}"); + assert_eq!( + after.present_part_fingerprints, before.present_part_fingerprints, + "external shard bytes must match: {key}" + ); + assert_eq!( + after.inline_data_fingerprint, before.inline_data_fingerprint, + "inline shard bytes must match: {key}" + ); + } + cluster.stop_node(0)?; + for (key, expected) in &objects { + let actual = timeout(Duration::from_secs(30), async { + target + .get_object() + .bucket(bucket) + .key(*key) + .send() + .await? + .body + .collect() + .await + .map(|body| body.into_bytes()) + .map_err(Box::::from) + }) + .await??; + assert_eq!(actual.len(), expected.len(), "target-only read length: {key}"); + assert_eq!(sha256_hex(&actual), sha256_hex(expected), "target-only content: {key}"); + } + Ok(()) + } + + #[tokio::test] + async fn test_two_node_bucket_discovery_and_explicit_recreate() -> Result<(), Box> { + verify_two_node_directory_recovery(true, 2).await + } + + #[tokio::test] + async fn test_two_node_object_directory_recovery_normal() -> Result<(), Box> { + verify_two_node_directory_recovery(false, 1).await + } + + #[tokio::test] + async fn test_two_node_object_directory_recovery_deep() -> Result<(), Box> { + verify_two_node_directory_recovery(false, 2).await + } } diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 57ccd3581..1d02140d8 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -20,6 +20,7 @@ use crate::bucket::{ use crate::error::is_err_bucket_not_found; use crate::runtime::sources as runtime_sources; use crate::set_disk::{BucketInfoQuorum, get_lock_acquire_timeout}; +use crate::storage_api_contracts::admin::DiskSetSelector; use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp}; use crate::storage_api_contracts::namespace::NamespaceLocking as _; use futures::stream::{self, StreamExt}; @@ -1232,13 +1233,13 @@ impl ECStore { pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result> { // TODO(backlog): support cached bucket listing via opts.cached Ok(self - .list_bucket_from_sets(opts, usize::MAX, BucketInfoQuorum::Write) + .list_bucket_from_sets(opts, usize::MAX, BucketInfoQuorum::Write, None) .await? .buckets) } pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result { - self.list_bucket_from_sets(opts, SCANNER_BUCKET_LIST_SET_CONCURRENCY, BucketInfoQuorum::Write) + self.list_bucket_from_sets(opts, SCANNER_BUCKET_LIST_SET_CONCURRENCY, BucketInfoQuorum::Write, None) .await } @@ -1254,20 +1255,44 @@ impl ECStore { }, SCANNER_BUCKET_LIST_SET_CONCURRENCY, BucketInfoQuorum::Read, + None, ) .await? .buckets) } + /// Preserve namespace completeness for administrator traversal. A scoped + /// request must not depend on unrelated sets being available. + pub async fn list_buckets_for_admin_heal( + &self, + selector: Option, + ) -> Result { + self.list_bucket_from_sets( + &BucketOptions { + no_metadata: true, + ..Default::default() + }, + SCANNER_BUCKET_LIST_SET_CONCURRENCY, + BucketInfoQuorum::Write, + selector, + ) + .await + } + async fn list_bucket_from_sets( &self, opts: &BucketOptions, max_concurrency: usize, quorum: BucketInfoQuorum, + selector: Option, ) -> Result { - let set_count = self.pools.iter().map(|pool| pool.disk_set.len()).sum(); + let sets = self + .bucket_sets() + .filter(|(pool, set, _)| selector.is_none_or(|scope| scope.pool_idx == *pool && scope.set_idx == *set)) + .collect::>(); + let set_count = sets.len(); let concurrency = bucket_list_set_concurrency(set_count, max_concurrency); - let mut set_listings = stream::iter(self.bucket_sets().map(move |(pool_index, set_index, set)| async move { + let mut set_listings = stream::iter(sets.into_iter().map(move |(pool_index, set_index, set)| async move { set.list_bucket_with_quorum(quorum) .await .map(|(buckets, complete)| (pool_index, set_index, buckets, complete)) @@ -2236,6 +2261,64 @@ mod tests { } } + #[tokio::test] + async fn admin_bucket_discovery_preserves_completeness_and_scope() { + use crate::storage_api_contracts::admin::DiskSetSelector; + let (temp, store) = setup_bucket_quorum_test_env(&[2, 2], None).await; + let empty = store + .list_buckets_for_admin_heal(None) + .await + .expect("empty namespace enumeration"); + assert!(empty.buckets.is_empty()); + assert!(empty.topology_complete); + for disk in 0..2 { + tokio::fs::create_dir(temp.path().join(format!("pool0-disk{disk}/healthy-bucket"))) + .await + .expect("healthy bucket"); + } + tokio::fs::create_dir(temp.path().join("pool1-disk0/partial-bucket")) + .await + .expect("single surviving bucket directory"); + let global = store + .list_buckets_for_admin_heal(None) + .await + .expect("online disk enumeration"); + assert!(!global.topology_complete, "a filtered sub-quorum bucket must invalidate coverage"); + assert_eq!(global.buckets.iter().map(|b| b.name.as_str()).collect::>(), ["healthy-bucket"]); + let healthy = store + .list_buckets_for_admin_heal(Some(DiskSetSelector::new(0, 0))) + .await + .expect("healthy scope"); + assert!(healthy.topology_complete); + assert_eq!(healthy.buckets.len(), 1); + assert_eq!(healthy.set_buckets.len(), 1); + let partial = store + .list_buckets_for_admin_heal(Some(DiskSetSelector::new(1, 0))) + .await + .expect("partial scope"); + assert!(!partial.topology_complete); + assert!( + partial.buckets.is_empty(), + "missing buckets cannot be mistaken for a complete empty namespace" + ); + let invalid = store + .list_buckets_for_admin_heal(Some(DiskSetSelector::new(0, 1))) + .await + .expect("unconfigured scope"); + assert!(!invalid.topology_complete); + assert!(invalid.buckets.is_empty()); + let offline = take_set_disks_offline(&store, &store.pools[1].disk_set[0], &[0]).await; + assert!(store.list_buckets_for_admin_heal(None).await.is_err(), "quorum errors must remain errors"); + assert!( + store + .list_buckets_for_admin_heal(Some(DiskSetSelector::new(0, 0))) + .await + .expect("unrelated offline pool") + .topology_complete + ); + restore_set_disks(&store, &store.pools[1].disk_set[0], offline).await; + } + #[test] fn should_not_override_when_metadata_created_is_unix_epoch() { assert!(!should_override_created_from_metadata(OffsetDateTime::UNIX_EPOCH)); diff --git a/crates/heal/src/error.rs b/crates/heal/src/error.rs index d1db7292a..ac901e921 100644 --- a/crates/heal/src/error.rs +++ b/crates/heal/src/error.rs @@ -90,6 +90,9 @@ pub enum Error { source: Box, }, + #[error("Bucket discovery incomplete for the requested heal scope; not all buckets could be traversed")] + BucketDiscoveryIncomplete, + #[error("Invalid heal type: {heal_type}")] InvalidHealType { heal_type: String }, @@ -127,7 +130,7 @@ impl Error { match self { Error::TaskCancelled | Error::TaskTimeout | Error::StaleBucketIncarnation { .. } => false, Error::ReplacementTargetNotReady(_) | Error::DanglingDeleteDeferred { .. } => true, - Error::TransientSkip { .. } => true, + Error::TransientSkip { .. } | Error::BucketDiscoveryIncomplete => true, // Lock failures classify by LockError's own taxonomy: only the // fatal variants (ResourceNotFound / PermissionDenied / // Configuration) are terminal - retrying cannot fix them - while diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index 20b37eff2..5e78e43d8 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -853,6 +853,13 @@ mod tests { async fn get_bucket_info(&self, _bucket: &str) -> crate::Result> { Ok(None) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> crate::Result> { + self.list_buckets().await + } + async fn list_buckets(&self) -> crate::Result> { Ok(vec![]) } diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index 5a02978ba..fdc45fcf4 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -2390,6 +2390,8 @@ mod resume_loop_tests { #[derive(Default)] struct FakeStorage { admin_disk: Mutex>, + bucket_discovery_incomplete: AtomicBool, + bucket_discovery_scopes: Mutex>>, bucket_pages: Mutex>, incarnations: Mutex>, receipts: Mutex>>, @@ -2469,6 +2471,17 @@ mod resume_loop_tests { ..Default::default() })) } + async fn list_buckets_for_admin_heal( + &self, + selector: Option, + ) -> Result> { + self.bucket_discovery_scopes.lock().expect("discovery scopes").push(selector); + if self.bucket_discovery_incomplete.load(Ordering::SeqCst) { + return Err(Error::BucketDiscoveryIncomplete); + } + self.list_buckets().await + } + async fn list_buckets(&self) -> Result> { Ok(self .bucket_pages diff --git a/crates/heal/src/heal/erasure_healer/admin.rs b/crates/heal/src/heal/erasure_healer/admin.rs index 5c9a686cd..bb88cdded 100644 --- a/crates/heal/src/heal/erasure_healer/admin.rs +++ b/crates/heal/src/heal/erasure_healer/admin.rs @@ -51,9 +51,10 @@ impl ErasureSetHealer { } let resolved; let buckets = if buckets.is_empty() { + let (pool_idx, set_idx) = crate::heal::utils::parse_set_disk_id(set_disk_id)?; resolved = self .storage - .list_buckets() + .list_buckets_for_admin_heal(Some(crate::heal::storage_api::storage::DiskSetSelector::new(pool_idx, set_idx))) .await? .into_iter() .map(|bucket| bucket.name) diff --git a/crates/heal/src/heal/erasure_healer/resume_loop_tests/admin_outcome.rs b/crates/heal/src/heal/erasure_healer/resume_loop_tests/admin_outcome.rs index e893f8eca..30a51fbe7 100644 --- a/crates/heal/src/heal/erasure_healer/resume_loop_tests/admin_outcome.rs +++ b/crates/heal/src/heal/erasure_healer/resume_loop_tests/admin_outcome.rs @@ -483,3 +483,50 @@ async fn admin_erasure_checkpoint_publication_cannot_undo_acknowledged_cancel() assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); assert_eq!((outcome.counters.processed, outcome.counters.healed), (1, 1)); } + +#[tokio::test] +async fn admin_bucket_discovery_rejects_incomplete_checkpoint_and_freezes_complete_empty_scope() { + let env = make_env().await; + env.storage.bucket_discovery_incomplete.store(true, Ordering::SeqCst); + let result = env + .healer + .prepare_admin_checkpoint(&env.checkpoint, &[], "pool_0_set_0") + .await; + assert!(matches!(result, Err(Error::BucketDiscoveryIncomplete))); + assert!(env.checkpoint.get_checkpoint().await.admin.is_none()); + let reloaded = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id) + .await + .expect("reload checkpoint"); + assert!(reloaded.get_checkpoint().await.admin.is_none(), "incomplete scope must not be persisted"); + assert_eq!( + env.storage.bucket_discovery_scopes.lock().expect("scopes").as_slice(), + &[Some(crate::heal::storage_api::storage::DiskSetSelector::new(0, 0))] + ); + + env.storage.bucket_discovery_incomplete.store(false, Ordering::SeqCst); + assert!( + env.healer + .prepare_admin_checkpoint(&env.checkpoint, &[], "pool_0_set_0") + .await + .expect("complete empty scope") + .is_empty() + ); + env.storage.bucket_discovery_incomplete.store(true, Ordering::SeqCst); + objects(&env.storage, "created-after-admission", &["object"]); + let reloaded = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id) + .await + .expect("reload frozen scope"); + assert!( + env.healer + .prepare_admin_checkpoint(&reloaded, &[], "pool_0_set_0") + .await + .expect("resume frozen scope") + .is_empty() + ); + assert_eq!( + env.storage.bucket_discovery_scopes.lock().expect("scopes").len(), + 2, + "resume must not rediscover or expand scope" + ); + assert!(env.storage.calls().is_empty()); +} diff --git a/crates/heal/src/heal/manager/tests.rs b/crates/heal/src/heal/manager/tests.rs index 997c85e64..4829b7c0b 100644 --- a/crates/heal/src/heal/manager/tests.rs +++ b/crates/heal/src/heal/manager/tests.rs @@ -601,6 +601,13 @@ impl HealStorageAPI for MockStorage { Ok(None) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> Result> { + self.list_buckets().await + } + async fn list_buckets(&self) -> Result> { if let Some(hook) = manager_recovery_test_hook() { *hook.listed.lock().expect("manager recovery listed lock should not poison") = true; @@ -5902,3 +5909,25 @@ fn test_heal_config_respects_feature_flags() { }, ); } + +#[test] +fn admin_bucket_discovery_retries_keep_the_existing_budget() { + let storage: Arc = Arc::new(MockStorage); + let mut request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal); + request.source = HealRequestSource::Admin; + let id = request.id.clone(); + let error = Err(Error::BucketDiscoveryIncomplete); + for attempt in 1..=MAX_RECOVERABLE_HEAL_RETRIES { + let task = HealTask::from_request(request, storage.clone()); + let (retry, delay, _) = retry_request_for_result(&task, &error).expect("discovery can recover within budget"); + assert_eq!(retry.id, id); + assert_eq!(retry.retry_attempts, attempt); + assert!(delay > Duration::ZERO); + request = retry; + } + let task = HealTask::from_request(request, storage); + assert!( + retry_request_for_result(&task, &error).is_none(), + "persistent incompleteness must terminate" + ); +} diff --git a/crates/heal/src/heal/manager/tests/running_mainline.rs b/crates/heal/src/heal/manager/tests/running_mainline.rs index d7b827ce6..4cc153d28 100644 --- a/crates/heal/src/heal/manager/tests/running_mainline.rs +++ b/crates/heal/src/heal/manager/tests/running_mainline.rs @@ -80,6 +80,13 @@ impl HealStorageAPI for RunningStorage { ..Default::default() })) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> Result> { + self.list_buckets().await + } + async fn list_buckets(&self) -> Result> { Ok(Vec::new()) } diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index 35facd16b..d67519ff2 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -491,6 +491,10 @@ pub trait HealStorageAPI: Send + Sync { /// Get all buckets async fn list_buckets(&self) -> Result>; + /// Return only a complete bucket inventory for an administrator task. + /// Incomplete discovery must fail before traversal or scope checkpointing. + async fn list_buckets_for_admin_heal(&self, selector: Option) -> Result>; + /// Check object exists async fn object_exists(&self, bucket: &str, object: &str) -> Result; @@ -1335,6 +1339,14 @@ impl HealStorageAPI for ECStoreHealStorage { } } + async fn list_buckets_for_admin_heal(&self, selector: Option) -> Result> { + let listing = self.ecstore.list_buckets_for_admin_heal(selector).await?; + if !listing.topology_complete { + return Err(Error::BucketDiscoveryIncomplete); + } + Ok(listing.buckets) + } + async fn list_buckets(&self) -> Result> { debug!( target: "rustfs::heal::storage", diff --git a/crates/heal/src/heal/task/heal_bucket.rs b/crates/heal/src/heal/task/heal_bucket.rs index cf6b4786f..9c95c8c03 100644 --- a/crates/heal/src/heal/task/heal_bucket.rs +++ b/crates/heal/src/heal/task/heal_bucket.rs @@ -308,7 +308,12 @@ impl HealTask { "Heal cluster started" ); - let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?; + let bucket_infos = if self.source == HealRequestSource::Admin { + self.await_with_control(self.storage.list_buckets_for_admin_heal(None)) + .await? + } else { + self.await_with_control(self.storage.list_buckets()).await? + }; let mut failed = 0_u64; let mut retryable = 0_u64; let mut permanent = 0_u64; diff --git a/crates/heal/src/heal/task/tests.rs b/crates/heal/src/heal/task/tests.rs index 9919fb522..4058335bc 100644 --- a/crates/heal/src/heal/task/tests.rs +++ b/crates/heal/src/heal/task/tests.rs @@ -24,6 +24,59 @@ mod canonical_outcome { use super::*; use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage}; + #[tokio::test] + async fn admin_bucket_discovery_incomplete_cannot_finish_an_empty_cluster() { + for dry_run in [false, true] { + let storage = Arc::new(MockStorage { + listed_buckets: Mutex::new(Some(Vec::new())), + bucket_discovery_incomplete: true, + ..Default::default() + }); + let mut request = HealRequest::new( + HealType::Cluster, + HealOptions { + dry_run, + recursive: true, + ..Default::default() + }, + HealPriority::Normal, + ); + request.source = HealRequestSource::Admin; + let task = HealTask::from_request(request, storage.clone()); + assert!(matches!(task.execute().await, Err(Error::BucketDiscoveryIncomplete))); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable)); + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + assert_eq!(outcome.counters.processed, 0); + assert_eq!(outcome.counters.failed, 0); + assert!(storage.bucket_heal_calls.lock().expect("bucket calls").is_empty()); + assert!(storage.heal_object_calls.lock().expect("object calls").is_empty()); + let wire = serde_json::to_value(&outcome).expect("outcome serializes"); + assert_eq!( + crate::heal::outcome::legacy_wire_status("finished", &wire, false) + .expect("legacy status") + .0, + "stopped" + ); + } + } + + #[tokio::test] + async fn admin_bucket_discovery_complete_empty_cluster_finishes() { + let storage = Arc::new(MockStorage { + listed_buckets: Mutex::new(Some(Vec::new())), + ..Default::default() + }); + let mut request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal); + request.source = HealRequestSource::Admin; + let task = HealTask::from_request(request, storage); + task.execute().await.expect("complete empty namespace is valid"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Completed); + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + assert_eq!(outcome.counters.processed, 0); + } + fn bucket_task(storage: Arc) -> HealTask { HealTask::from_request( HealRequest::new( @@ -1481,6 +1534,7 @@ struct MockStorage { truncate_without_token: Mutex, include_object_dir_candidate: Mutex, listed_buckets: Mutex>>, + bucket_discovery_incomplete: bool, bucket_heal_errors: Mutex>>, bucket_heal_calls: Mutex>, block_heal_object: Mutex, @@ -1980,6 +2034,16 @@ impl HealStorageAPI for MockStorage { Ok(*self.usage_baseline.lock().unwrap()) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> Result> { + if self.bucket_discovery_incomplete { + return Err(Error::BucketDiscoveryIncomplete); + } + self.list_buckets().await + } + async fn list_buckets(&self) -> Result> { let buckets = self .listed_buckets diff --git a/crates/heal/src/lib.rs b/crates/heal/src/lib.rs index 8a4fdf8b5..d792aa877 100644 --- a/crates/heal/src/lib.rs +++ b/crates/heal/src/lib.rs @@ -398,6 +398,13 @@ mod tests { Ok(None) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> Result, Error> { + self.list_buckets().await + } + async fn list_buckets(&self) -> Result, Error> { Ok(Vec::new()) } diff --git a/crates/heal/tests/heal_bug_fixes_test.rs b/crates/heal/tests/heal_bug_fixes_test.rs index d2690b6be..832c71cf7 100644 --- a/crates/heal/tests/heal_bug_fixes_test.rs +++ b/crates/heal/tests/heal_bug_fixes_test.rs @@ -110,6 +110,13 @@ fn test_heal_task_status_atomic_update() { async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result> { Ok(None) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> rustfs_heal::Result> { + self.list_buckets().await + } + async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(vec![]) } @@ -205,6 +212,13 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { Ok(None) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> rustfs_heal::Result> { + self.list_buckets().await + } + async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(Vec::new()) } diff --git a/crates/heal/tests/heal_concurrent_delete_test.rs b/crates/heal/tests/heal_concurrent_delete_test.rs index 2770295dc..4a8630a4f 100644 --- a/crates/heal/tests/heal_concurrent_delete_test.rs +++ b/crates/heal/tests/heal_concurrent_delete_test.rs @@ -79,6 +79,13 @@ impl HealStorageAPI for PausedTraversal { self.inner.get_bucket_info(bucket).await } + async fn list_buckets_for_admin_heal( + &self, + selector: Option, + ) -> Result> { + self.inner.list_buckets_for_admin_heal(selector).await + } + async fn list_buckets(&self) -> Result> { self.inner.list_buckets().await } diff --git a/crates/heal/tests/storage_api.rs b/crates/heal/tests/storage_api.rs index b47737203..f9bae2473 100644 --- a/crates/heal/tests/storage_api.rs +++ b/crates/heal/tests/storage_api.rs @@ -2,7 +2,7 @@ pub(crate) mod bug_fixes { pub(crate) use rustfs_ecstore::api::disk::DiskStore; pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint; - pub(crate) use rustfs_storage_api::BucketInfo; + pub(crate) use rustfs_storage_api::{BucketInfo, DiskSetSelector}; } #[allow(unused_imports)] diff --git a/crates/scanner/src/scanner_folder/tests/mrf_ownership.rs b/crates/scanner/src/scanner_folder/tests/mrf_ownership.rs index 3e7f9f570..a166a20ef 100644 --- a/crates/scanner/src/scanner_folder/tests/mrf_ownership.rs +++ b/crates/scanner/src/scanner_folder/tests/mrf_ownership.rs @@ -243,6 +243,13 @@ impl HealStorageAPI for NoticeStorage { async fn bucket_incarnation_id(&self, _: &str) -> rustfs_heal::Result> { Ok(Some(self.bucket_incarnation_id)) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> rustfs_heal::Result> { + self.list_buckets().await + } + async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(Vec::new()) } diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index ad88e6ff0..4523fa02c 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -145,6 +145,8 @@ use rustfs_storage_api as storage_contracts; pub(crate) type EcstoreHealResultItem = ::HealResultItem; pub(crate) mod owner { + #[cfg(test)] + pub(crate) use super::storage_contracts::DiskSetSelector; pub(crate) use super::{ MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementError, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner, diff --git a/docs/operations/bucket-heal-recovery.md b/docs/operations/bucket-heal-recovery.md index 60d10b3ca..ed6453644 100644 --- a/docs/operations/bucket-heal-recovery.md +++ b/docs/operations/bucket-heal-recovery.md @@ -1,10 +1,33 @@ # Bucket heal recovery and bucket recreation +**Use this when:** diagnosing incomplete administrator heal coverage, restoring a missing bucket directory, or replaying a bucket heal after restart. + Explicit administrator bucket heals bind the bucket name to its authoritative, non-nil incarnation UUID at admission. The same identity follows the task through queueing, retries, shutdown persistence, and startup replay. Deleting a bucket and creating another with the same name does not transfer the old task to the new bucket. +## Incomplete bucket discovery + +An administrator heal of the cluster or a selected pool/set must enumerate a complete bucket inventory before traversal. A bucket directory present on fewer disks than the namespace write quorum can be omitted from the ordinary bucket list even when every disk answers. Administrator discovery preserves that incomplete result instead of reporting an empty successful traversal. A pool/set task checks only its selected set; unavailable unrelated pools do not invalidate that scope. + +Incomplete discovery uses the existing bounded task retry budget. Once exhausted, the canonical outcome is `execution.state=aborted`, `execution.reason=untraversable`, and `coverage=partial`; the legacy summary is `stopped`. Object counters can remain zero because traversal never started. A genuinely empty, completely enumerated namespace still finishes successfully. Dry runs use the same coverage checks. + +A pool/set task persists its resolved bucket scope only after complete discovery. Resumed checkpoints keep their original frozen scope, including a valid empty scope. This change does not reinterpret historical checkpoints or terminal reports; start a fresh task to verify coverage after upgrading. + +## Restoring a missing bucket directory + +For a bucket that is still intended to exist but has lost its directory on one drive: + +1. Check that the peer drives and authoritative bucket metadata remain available. Identify the affected bucket explicitly. +2. Start an authenticated administrator heal for that bucket with `recursive=true` and `recreate=true`. A deep scan (`scanMode=2`) additionally checks data integrity. For the v3 API, the start endpoint is `POST /rustfs/admin/v3/heal/`; retain the returned `clientToken` and poll the same endpoint with that query parameter. +3. Inspect both execution and coverage in the terminal outcome. A readable object on a healthy peer alone does not establish that the missing copy was restored. +4. Verify recovered object contents and lengths against known hashes. In an isolated recovery test, stop the original healthy source and read through the repaired target to demonstrate independent recovery. Do not take a production source offline merely to inspect a heal result. + +Default bucket heal still enforces its existing namespace quorum checks. `recreate` is an explicit recovery option, not permission to revive a bucket deleted through S3. Bucket incarnation checks continue to reject stale tasks after deletion and same-name recreation. A cluster-wide `recreate` request also requires complete discovery; when discovery is incomplete, use the explicit affected-bucket path after resolving its authority. + +Legacy objects without independent storage commitments can be physically repaired while their canonical dispositions remain `unknown`. Do not infer a verified repair count from progress counters; see [heal terminal reports](../architecture/heal-terminal-reports.md). + ## Recovery records and stale tokens Pending records use schema 3 in `.rustfs.sys/root-heal-.json` on the existing diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 255bf1044..588bc26f6 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -3209,6 +3209,13 @@ mod tests { Ok(None) } + async fn list_buckets_for_admin_heal( + &self, + _selector: Option, + ) -> rustfs_heal::Result> { + self.list_buckets().await + } + async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(Vec::new()) } diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index c0a9608d5..ce971c878 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -59,6 +59,8 @@ fn bucket_targets_metadata_lock_shard(bucket: &str) -> usize { pub(crate) mod contract { pub(crate) mod admin { + #[cfg(test)] + pub(crate) use super::super::storage_contracts::DiskSetSelector; pub(crate) use super::super::storage_contracts::StorageAdminApi; } From ab512768d5ac9de30cfd3608316fb44858aeb405 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 20:05:36 +0800 Subject: [PATCH 016/143] fix(ecstore): group rename data commit observations (#8516) Group disk observations into a borrowed struct to keep the commit helper within Clippy's argument limit without changing quorum snapshot logic. Co-authored-by: heihutu Co-authored-by: zhi22915 --- .../src/set_disk/core/io_primitives.rs | 48 ++++++++++++------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 2b99bd9c1..6ade60cf4 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -2451,6 +2451,17 @@ impl RenameConvergence { } } +struct RenameDataObservations<'a> { + disks: &'a [Option], + file_infos: &'a [FileInfo], + disk_versions: &'a [Option>], + errs: &'a [Option], + cleanup_data_dirs: &'a [Option], + old_current_sizes: &'a [Option], + old_current_source_checked: &'a [bool], + old_current_sources: &'a [Option], +} + pub(in crate::set_disk) struct RenameDataCommit { pub(in crate::set_disk) online_disks: Vec>, /// Disks whose capacity must be marked for this commit. Under early ACK, @@ -3608,16 +3619,19 @@ impl SetDisks { } fn rename_data_commit_from_observations( - disks: &[Option], - file_infos: &[FileInfo], - disk_versions: &[Option>], - errs: &[Option], - cleanup_data_dirs: &[Option], - old_current_sizes: &[Option], - old_current_source_checked: &[bool], - old_current_sources: &[Option], + observations: RenameDataObservations<'_>, write_quorum: usize, ) -> disk::error::Result { + let RenameDataObservations { + disks, + file_infos, + disk_versions, + errs, + cleanup_data_dirs, + old_current_sizes, + old_current_source_checked, + old_current_sources, + } = observations; let data_dir = Self::reduce_common_data_dir(cleanup_data_dirs, write_quorum); let convergence = Self::classify_rename_convergence(disk_versions, errs); let old_current_size = Self::reduce_common_old_current_size(old_current_sizes, write_quorum); @@ -3944,14 +3958,16 @@ impl SetDisks { if !sent_commit && success_count >= write_quorum { let snapshot_commit = Self::rename_data_commit_from_observations( - &coordinator_disks, - &file_infos, - &disk_versions, - &errs, - &cleanup_data_dirs, - &old_current_sizes, - &old_current_source_checked, - &old_current_sources, + RenameDataObservations { + disks: &coordinator_disks, + file_infos: &file_infos, + disk_versions: &disk_versions, + errs: &errs, + cleanup_data_dirs: &cleanup_data_dirs, + old_current_sizes: &old_current_sizes, + old_current_source_checked: &old_current_source_checked, + old_current_sources: &old_current_sources, + }, write_quorum, ); if let Ok(commit) = snapshot_commit.as_ref() { From a79c6072f1b32d55994906158e2278fe23d58d0b Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 13 Oct 2026 05:31:23 -0700 Subject: [PATCH 017/143] fix: suppress clippy too_many_arguments on rename_data_commit_from_observations (#8515) fix(ecstore): suppress clippy too_many_arguments on rename_data_commit_from_observations The function takes 9 slice parameters for the renamedata commit phase. Annotate with #[allow(clippy::too_many_arguments)] to match the existing codebase pattern for internal multi-slice commit helpers. --- crates/ecstore/src/set_disk/core/io_primitives.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 6ade60cf4..de172a855 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -3618,6 +3618,7 @@ impl SetDisks { } } + #[allow(clippy::too_many_arguments)] fn rename_data_commit_from_observations( observations: RenameDataObservations<'_>, write_quorum: usize, From da52303f2a05ef1bcff1295715dddcfc12e901ee Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 13 Oct 2026 20:32:27 +0800 Subject: [PATCH 018/143] fix(server): restore socket peer fallback for client rate limits (#8519) Read the optional socket peer extension supplied by external ingress so API and Console client limits remain effective without trusted proxies. Keep validated client precedence and the existing missing-peer policy. Exercise the full external stack across HTTP request versions, direct IPv4/IPv6 peers, forged headers, and distinct trusted proxy clients. Align API and Console fixtures with the production extension contract. Co-authored-by: heihutu Co-authored-by: zhi22915 --- rustfs/src/admin/console.rs | 10 +- rustfs/src/server/http.rs | 157 ++++++++++++++++++++++++++++++++ rustfs/src/server/rate_limit.rs | 18 ++-- 3 files changed, 175 insertions(+), 10 deletions(-) diff --git a/rustfs/src/admin/console.rs b/rustfs/src/admin/console.rs index 4019c9876..f03a3f3ee 100644 --- a/rustfs/src/admin/console.rs +++ b/rustfs/src/admin/console.rs @@ -1051,10 +1051,12 @@ mod tests { .uri(format!("{CONSOLE_PREFIX}/index.html")) .body(Body::empty()) .expect("failed to build request"); - request.extensions_mut().insert(crate::server::RemoteAddr(SocketAddr::new( - IpAddr::from([198, 51, 100, last_octet]), - 50000, - ))); + request + .extensions_mut() + .insert(Some(crate::server::RemoteAddr(SocketAddr::new( + IpAddr::from([198, 51, 100, last_octet]), + 50000, + )))); request }; diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index d5a90e6bc..e0615835a 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -2818,6 +2818,163 @@ mod tests { use tokio::sync::{Notify, mpsc}; use tower::{Layer, Service, ServiceBuilder}; + type RateLimitTestCore = tower::util::BoxCloneService< + HttpRequest, + Response>>, + Box, + >; + type RateLimitTestStack = tower::util::BoxCloneService< + HttpRequest, + Response>>, + Box, + >; + + fn rate_limited_external_stack( + peer: SocketAddr, + proxy_layer: Option, + inner: RateLimitTestCore, + ) -> RateLimitTestStack { + use crate::server::rate_limit::RateLimitQuota; + + let readiness = Arc::new(GlobalReadiness::new()); + readiness.mark_stage(rustfs_common::SystemStage::FullReady); + RateLimitTestStack::new(external_service_stack!( + inner, + Some(RemoteAddr(peer)), + proxy_layer, + false, + Some(RateLimitLayer::new( + Some(RateLimitQuota { + requests_per_minute: 1, + burst: 1 + }), + None, + vec![] + )), + false, + readiness, + None::>, + HttpCompressionConfig::default(), + ServerContextSlot::new(), + false + )) + } + + #[tokio::test] + async fn external_stack_limits_socket_peers_without_trusted_proxies() { + use crate::server::hybrid::HybridBody; + use tower::ServiceExt; + + for peer in ["192.0.2.10:51000", "[2001:db8::10]:51000"] { + let peer: SocketAddr = peer.parse().expect("socket peer"); + for version in [Version::HTTP_11, Version::HTTP_2, Version::HTTP_3] { + let calls = Arc::new(AtomicUsize::new(0)); + let inner_calls = calls.clone(); + let inner = tower::service_fn(move |request: HttpRequest| { + assert_eq!( + request + .extensions() + .get::>() + .copied() + .flatten() + .map(|addr| addr.0), + Some(peer) + ); + inner_calls.fetch_add(1, Ordering::SeqCst); + async { + Ok::<_, Box>(Response::new( + HybridBody::>::Rest { + rest_body: s3s::Body::empty(), + }, + )) + } + }); + let mut service = rate_limited_external_stack(peer, None, RateLimitTestCore::new(inner)); + + for (forwarded, expected) in [ + ("203.0.113.1", StatusCode::OK), + ("203.0.113.2", StatusCode::TOO_MANY_REQUESTS), + ] { + let request = HttpRequest::builder() + .uri("/bucket/object") + .version(version) + .header("x-forwarded-for", forwarded) + .header("x-real-ip", forwarded) + .body(s3s::Body::empty()) + .expect("request"); + let response = service + .ready() + .await + .expect("service ready") + .call(request) + .await + .expect("response"); + assert_eq!(response.status(), expected, "peer={peer}, version={version:?}"); + if expected == StatusCode::TOO_MANY_REQUESTS { + assert!(response.headers().contains_key(http::header::RETRY_AFTER)); + let body = response.into_body().collect().await.expect("body").to_bytes(); + assert!(String::from_utf8_lossy(&body).contains("TooManyRequests")); + } + } + assert_eq!(calls.load(Ordering::SeqCst), 1, "throttled requests must not reach the inner service"); + } + } + } + + #[tokio::test] + async fn external_stack_limits_validated_proxy_clients_independently() { + use crate::server::hybrid::HybridBody; + use rustfs_trusted_proxies::{ + LegacyTrustedProxyLayer, TrustedProxy, TrustedProxyConfig, TrustedProxyLayer, ValidationMode, + }; + use tower::ServiceExt; + + let peer: SocketAddr = "192.0.2.100:51000".parse().expect("proxy peer"); + for version in [Version::HTTP_11, Version::HTTP_2, Version::HTTP_3] { + let proxy_config = + TrustedProxyConfig::new(vec![TrustedProxy::Single(peer.ip())], ValidationMode::HopByHop, false, 3, false, vec![]); + let proxy_layer = TrustedProxyLayer::legacy(LegacyTrustedProxyLayer::new(proxy_config, None, true)); + let calls = Arc::new(AtomicUsize::new(0)); + let inner_calls = calls.clone(); + let inner = tower::service_fn(move |request: HttpRequest| { + let info = request.extensions().get::().expect("validated client context"); + assert!(info.is_from_trusted_proxy); + assert_ne!(info.real_ip, peer.ip()); + inner_calls.fetch_add(1, Ordering::SeqCst); + async { + Ok::<_, Box>(Response::new( + HybridBody::>::Rest { + rest_body: s3s::Body::empty(), + }, + )) + } + }); + let mut service = rate_limited_external_stack(peer, Some(proxy_layer), RateLimitTestCore::new(inner)); + + for (client, expected) in [ + ("203.0.113.1", StatusCode::OK), + ("203.0.113.2", StatusCode::OK), + ("203.0.113.1", StatusCode::TOO_MANY_REQUESTS), + ] { + let request = HttpRequest::builder() + .uri("/bucket/object") + .version(version) + .header("x-forwarded-for", client) + .body(s3s::Body::empty()) + .expect("request"); + let response = service + .ready() + .await + .expect("service ready") + .call(request) + .await + .expect("response"); + assert_eq!(response.status(), expected, "client={client}, version={version:?}"); + } + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + } + #[test] fn peer_transport_close_kind_uses_io_error_sources() { #[derive(Debug)] diff --git a/rustfs/src/server/rate_limit.rs b/rustfs/src/server/rate_limit.rs index 483b805a0..ca39b8d28 100644 --- a/rustfs/src/server/rate_limit.rs +++ b/rustfs/src/server/rate_limit.rs @@ -301,14 +301,16 @@ impl RateLimiter { /// Resolve the client identity for rate limiting. /// /// Only trusted sources are consulted: `ClientInfo` is inserted by the -/// trusted-proxy layer after validating the proxy chain, and `RemoteAddr` is -/// the raw socket peer. Spoofable headers (`X-Forwarded-For`, `X-Real-IP`) -/// are deliberately never read here. +/// trusted-proxy layer after validating the proxy chain, and `Option` +/// carries the socket peer injected by the external HTTP stack. Spoofable +/// headers (`X-Forwarded-For`, `X-Real-IP`) are deliberately never read here. pub(crate) fn client_ip(req: &Request) -> Option { if let Some(info) = req.extensions().get::() { return Some(info.real_ip); } - req.extensions().get::().map(|remote| remote.0.ip()) + req.extensions() + .get::>() + .and_then(|remote| remote.map(|remote| remote.0.ip())) } /// Resolve the host a request addresses, for virtual-hosted-style bucket @@ -929,7 +931,7 @@ mod tests { fn request_from(peer: IpAddr, path: &str) -> Request<()> { let mut req = Request::builder().uri(path).body(()).expect("request"); - req.extensions_mut().insert(RemoteAddr(SocketAddr::new(peer, 51000))); + req.extensions_mut().insert(Some(RemoteAddr(SocketAddr::new(peer, 51000)))); req } @@ -1047,7 +1049,11 @@ mod tests { async fn missing_client_identity_fails_open() { let mut service = service_with_quota(60, 1); for _ in 0..3 { - let req = Request::builder().uri("/bucket/object").body(()).expect("request"); + let mut req = Request::builder().uri("/bucket/object").body(()).expect("request"); + req.extensions_mut().insert(None::); + req.headers_mut() + .insert("x-forwarded-for", HeaderValue::from_static("203.0.113.1")); + req.headers_mut().insert("x-real-ip", HeaderValue::from_static("203.0.113.2")); assert_eq!(service.call(req).await.expect("ok").status(), StatusCode::OK); } } From 3b428bffeffd23a58f4c77f7e207ef7e05c945f1 Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:18:58 +0800 Subject: [PATCH 019/143] fix(heal): merge disjoint replacement generations in cluster status (#8521) --- rustfs/src/admin/handlers/heal.rs | 96 ++++++++++++++++++++++++++----- 1 file changed, 82 insertions(+), 14 deletions(-) diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index f6c02aa0c..147bf275e 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -732,6 +732,21 @@ fn merge_replacement_recovery_records( records.into_values().collect() } +/// Distinct replacement generations must not claim the same target slot: two +/// completed histories for one replaced disk cannot be reconciled into a +/// definitive cluster view. Identical copies of one task are deduplicated by +/// the merge, and disagreeing copies already surface as `Unknown` records. +fn replacement_generations_claim_overlapping_targets(records: &[rustfs_heal::ReplacementRecoveryRecord]) -> bool { + records.iter().any(|left| { + !left.target_slots.is_empty() + && records.iter().any(|right| { + right.task_id != left.task_id + && !right.target_slots.is_empty() + && right.target_slots.iter().any(|slot| left.target_slots.contains(slot)) + }) + }) +} + fn aggregate_replacement_recovery_cluster_status( mut snapshots: Vec, peer_statuses: Vec, String>>, @@ -762,23 +777,18 @@ fn aggregate_replacement_recovery_cluster_status( reason.get_or_insert("peer_replacement_recovery_status_not_definitive"); } - // Replacement recovery state is anchored on one surviving disk. Nodes - // without that disk legitimately report a definitive empty snapshot, so - // only non-empty snapshots must agree with each other. - let mut non_empty_records = snapshots - .iter() - .filter(|snapshot| !snapshot.records.is_empty()) - .map(|snapshot| canonical_replacement_records(&snapshot.records)); - if let Some(first_records) = non_empty_records.next() - && non_empty_records.any(|records| records != first_records) - { - reason.get_or_insert("peer_replacement_recovery_status_conflict"); - } - + // Replacement completion proofs are anchored per generation on one + // surviving disk, so sequential automatic generations legitimately leave + // different completed tasks on different peers, and peers without an + // anchor report definitive empty snapshots. Peers conflict only when they + // disagree about one task (the merge marks those copies `Unknown`) or + // when distinct generations claim the same target slot; disjoint + // generations merge into one definitive cluster history. let records = merge_replacement_recovery_records(&snapshots); if records .iter() .any(|record| matches!(record.state, rustfs_heal::ReplacementRecoveryState::Unknown)) + || replacement_generations_claim_overlapping_targets(&records) { reason.get_or_insert("peer_replacement_recovery_status_conflict"); } @@ -2369,7 +2379,10 @@ mod tests { } #[test] - fn replacement_recovery_cluster_requires_matching_peer_records() { + fn replacement_recovery_cluster_rejects_overlapping_targets_across_generations() { + // Both helper records claim the same target slot, so two completed + // generations over one replaced disk stay non-definitive even though + // the task ids differ. let local = replacement_snapshot("11111111-1111-4111-8111-111111111111"); let peer = replacement_snapshot("22222222-2222-4222-8222-222222222222"); let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); @@ -2379,6 +2392,61 @@ mod tests { assert_eq!(cluster.records.len(), 2); } + #[test] + fn replacement_recovery_cluster_merges_disjoint_generations_across_anchors() { + // rustfs/rustfs#8366: sequential automatic generations anchor their + // completion proofs on different surviving disks, so peers hold + // disjoint completed tasks and the remaining peers hold none at all. + let mut rebuilt_record = replacement_record("11111111-1111-4111-8111-111111111111"); + rebuilt_record.target_slots = vec!["http://node-a:9000/mnt/disk2".to_string()]; + let mut anchor_record = replacement_record("22222222-2222-4222-8222-222222222222"); + anchor_record.target_slots = vec!["http://node-a:9000/mnt/disk1".to_string()]; + let rebuilt = rustfs_heal::ReplacementRecoverySnapshot { + records: vec![rebuilt_record], + definitive: true, + reason: None, + }; + let anchor = rustfs_heal::ReplacementRecoverySnapshot { + records: vec![anchor_record], + definitive: true, + reason: None, + }; + let empty = rustfs_heal::ReplacementRecoverySnapshot { + records: Vec::new(), + definitive: true, + reason: None, + }; + let cluster = + aggregate_replacement_recovery_cluster_status(vec![rebuilt], vec![Ok(Some(anchor)), Ok(Some(empty))], 3, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records.len(), 2); + assert!( + cluster + .records + .iter() + .all(|record| matches!(record.state, rustfs_heal::ReplacementRecoveryState::Completed)) + ); + } + + #[test] + fn replacement_recovery_cluster_rejects_peer_disagreement_about_one_task() { + let local = replacement_snapshot("11111111-1111-4111-8111-111111111111"); + let mut disagreeing = replacement_record("11111111-1111-4111-8111-111111111111"); + disagreeing.set_disk_id = Some("pool_0_set_1".to_string()); + let peer = rustfs_heal::ReplacementRecoverySnapshot { + records: vec![disagreeing], + definitive: true, + reason: None, + }; + let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); + + assert!(!cluster.definitive); + assert_eq!(cluster.reason.as_deref(), Some("peer_replacement_recovery_status_conflict")); + assert_eq!(cluster.records[0].state, rustfs_heal::ReplacementRecoveryState::Unknown); + } + #[test] fn replacement_recovery_cluster_accepts_definitive_empty_peers() { let local = rustfs_heal::ReplacementRecoverySnapshot { From 025ed4c6d16bd42b31a9507674a239eb94514f83 Mon Sep 17 00:00:00 2001 From: Hauser Date: Wed, 14 Oct 2026 00:36:33 +0800 Subject: [PATCH 020/143] fix(heal): preserve retained replacement histories (#8525) Keep definitive local histories while rejecting ambiguous overlapping peer records. Co-authored-by: heihutu Co-authored-by: zhi22915 --- docs/operations/scanner-runtime-controls.md | 2 +- rustfs/src/admin/handlers/heal.rs | 140 ++++++++++++++++++-- 2 files changed, 129 insertions(+), 13 deletions(-) diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index faf999409..eb0add3bd 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -493,7 +493,7 @@ Stale-upload cleanup differs in one crash-recovery detail. RustFS's stale multip Treat replacement recovery as verified only after the repair task has completed for the exact replacement instance and an operator has confirmed the target disk contains the expected `xl.meta` and data parts for every relevant object version. A replacement that is not mounted, is unsafe to format, loses its marker, or returns a partial target outcome must be treated as deferred or incomplete. Do not automate destructive replacement actions from an `idle` observation alone. A new node must not infer replacement completion from an old or unavailable peer; regard that information as unknown or degraded until every required peer can report the same replacement instance and verified completion. -`GET /rustfs/admin/v4/heal/replacement-recovery` reports durable automatic replacement records from survivor disks. `local.records[]` entries distinguish `waiting_for_replacement`, `running`, `incomplete`, `unrecoverable`, `cleanup_pending`, `completed`, and `unknown`; `local.definitive=false` or any `unknown` record means the node could not prove a local replacement state. The `cluster` section queries the replacement-recovery peer RPC and sets `cluster.definitive=true` only when the expected peer topology is complete, every peer supports the RPC, every peer snapshot is locally definitive, and all peers report the same replacement records. Old peers, unavailable peers, malformed peer payloads, topology gaps, and generation disagreements are reported as degraded or unknown rather than complete. +`GET /rustfs/admin/v4/heal/replacement-recovery` reports durable automatic replacement records from survivor disks. `local.records[]` entries distinguish `waiting_for_replacement`, `running`, `incomplete`, `unrecoverable`, `cleanup_pending`, `completed`, and `unknown`; `local.definitive=false` or any `unknown` record means the node could not prove a local replacement state. The `cluster` section queries the replacement-recovery peer RPC and sets `cluster.definitive=true` only when the expected peer topology is complete, every peer supports the RPC, every peer snapshot is locally definitive, and copies of each task agree. Different anchors can report different tasks, and a definitive node may report an empty list or a subset of another node's history. Disjoint target generations merge directly. Generations sharing a target slot must have been reported together in at least one definitive local snapshot; this preserves retained completion proofs and committed handoff histories while keeping overlapping records found only on separate peers non-definitive. A definitive history can include retired or unfinished generations, so consumers must inspect each record's state rather than treating `cluster.definitive` as proof that every generation completed. Old peers, unavailable peers, malformed peer payloads, topology gaps, and task disagreements are reported as degraded or unknown rather than complete. Replacement resume and checkpoint files use an independent on-disk schema. A newer reader rejects a future schema rather than continuing with data it cannot interpret, while an older binary cannot safely enforce the new generation fence. Do not roll a cluster back after a replacement generation has started; complete that recovery with the current-or-newer release, and if it cannot complete, keep that version for diagnosis rather than deleting its durable records or continuing with an older binary. diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 147bf275e..13449b763 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -732,17 +732,35 @@ fn merge_replacement_recovery_records( records.into_values().collect() } -/// Distinct replacement generations must not claim the same target slot: two -/// completed histories for one replaced disk cannot be reconciled into a -/// definitive cluster view. Identical copies of one task are deduplicated by -/// the merge, and disagreeing copies already surface as `Unknown` records. -fn replacement_generations_claim_overlapping_targets(records: &[rustfs_heal::ReplacementRecoveryRecord]) -> bool { +/// A definitive local snapshot can retain multiple generations for one slot, +/// including completed replacements and committed handoff predecessors. Only +/// overlapping generations never observed together in a definitive snapshot +/// leave the merged peer histories ambiguous. Disagreeing copies of one task +/// are already rejected by the merge. +fn replacement_generations_claim_conflicting_targets( + records: &[rustfs_heal::ReplacementRecoveryRecord], + snapshots: &[rustfs_heal::ReplacementRecoverySnapshot], +) -> bool { + let local_histories = snapshots + .iter() + .filter(|snapshot| snapshot.definitive) + .map(|snapshot| { + snapshot + .records + .iter() + .map(|record| record.task_id.as_str()) + .collect::>() + }) + .collect::>(); records.iter().any(|left| { !left.target_slots.is_empty() && records.iter().any(|right| { right.task_id != left.task_id && !right.target_slots.is_empty() && right.target_slots.iter().any(|slot| left.target_slots.contains(slot)) + && !local_histories + .iter() + .any(|history| history.contains(left.task_id.as_str()) && history.contains(right.task_id.as_str())) }) }) } @@ -782,13 +800,13 @@ fn aggregate_replacement_recovery_cluster_status( // different completed tasks on different peers, and peers without an // anchor report definitive empty snapshots. Peers conflict only when they // disagree about one task (the merge marks those copies `Unknown`) or - // when distinct generations claim the same target slot; disjoint - // generations merge into one definitive cluster history. + // claim overlapping targets without a shared definitive local history. + // Retained histories and disjoint generations can be merged. let records = merge_replacement_recovery_records(&snapshots); if records .iter() .any(|record| matches!(record.state, rustfs_heal::ReplacementRecoveryState::Unknown)) - || replacement_generations_claim_overlapping_targets(&records) + || replacement_generations_claim_conflicting_targets(&records, &snapshots) { reason.get_or_insert("peer_replacement_recovery_status_conflict"); } @@ -2205,6 +2223,14 @@ mod tests { } } + fn replacement_snapshot_with_retained_history() -> rustfs_heal::ReplacementRecoverySnapshot { + let mut snapshot = replacement_snapshot("11111111-1111-4111-8111-111111111111"); + snapshot + .records + .push(replacement_record("22222222-2222-4222-8222-222222222222")); + snapshot + } + #[tokio::test] async fn heal_selector_preflight_rejects_before_probe_and_token_creation() { use crate::admin::storage_api::runtime::PoolEndpoints; @@ -2379,10 +2405,9 @@ mod tests { } #[test] - fn replacement_recovery_cluster_rejects_overlapping_targets_across_generations() { - // Both helper records claim the same target slot, so two completed - // generations over one replaced disk stay non-definitive even though - // the task ids differ. + fn replacement_recovery_cluster_rejects_overlapping_targets_without_shared_history() { + // Separate anchors cannot establish that these overlapping records + // describe compatible historical replacement instances. let local = replacement_snapshot("11111111-1111-4111-8111-111111111111"); let peer = replacement_snapshot("22222222-2222-4222-8222-222222222222"); let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); @@ -2447,6 +2472,97 @@ mod tests { assert_eq!(cluster.records[0].state, rustfs_heal::ReplacementRecoveryState::Unknown); } + #[test] + fn replacement_recovery_cluster_accepts_retained_history_on_one_anchor() { + let local = replacement_snapshot_with_retained_history(); + let empty = rustfs_heal::ReplacementRecoverySnapshot { + records: Vec::new(), + definitive: true, + reason: None, + }; + let cluster = aggregate_replacement_recovery_cluster_status(vec![local.clone()], vec![Ok(Some(empty))], 2, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records, local.records); + } + + #[test] + fn replacement_recovery_cluster_accepts_matching_retained_histories() { + let local = replacement_snapshot_with_retained_history(); + let cluster = aggregate_replacement_recovery_cluster_status(vec![local.clone()], vec![Ok(Some(local.clone()))], 2, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records, local.records); + } + + #[test] + fn replacement_recovery_cluster_accepts_committed_handoff_history() { + let mut local = replacement_snapshot_with_retained_history(); + let successor = local.records[1].task_id.clone(); + let predecessor = &mut local.records[0]; + predecessor.state = rustfs_heal::ReplacementRecoveryState::Unrecoverable; + predecessor.reason = Some(format!("replacement responsibility transferred to {successor}")); + predecessor.verified_at = None; + let cluster = aggregate_replacement_recovery_cluster_status(vec![local.clone()], Vec::new(), 1, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records, local.records); + } + + #[test] + fn replacement_recovery_cluster_merges_retained_history_with_disjoint_peer_generation() { + let local = replacement_snapshot_with_retained_history(); + let mut peer = replacement_snapshot("33333333-3333-4333-8333-333333333333"); + peer.records[0].target_slots = vec!["http://node-a:9000/mnt/disk2".to_string()]; + let mut expected = local.records.clone(); + expected.extend(peer.records.clone()); + let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records, expected); + } + + #[test] + fn replacement_recovery_cluster_accepts_retained_history_reported_only_by_peer() { + let local = replacement_snapshot("11111111-1111-4111-8111-111111111111"); + let peer = replacement_snapshot_with_retained_history(); + let expected = peer.records.clone(); + let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records, expected); + } + + #[test] + fn replacement_recovery_cluster_rejects_overlap_outside_retained_history() { + let local = replacement_snapshot_with_retained_history(); + let peer = replacement_snapshot("33333333-3333-4333-8333-333333333333"); + let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); + + assert!(!cluster.definitive); + assert_eq!(cluster.reason.as_deref(), Some("peer_replacement_recovery_status_conflict")); + assert_eq!(cluster.records.len(), 3); + } + + #[test] + fn replacement_recovery_cluster_rejects_disagreeing_copies_in_retained_history() { + let local = replacement_snapshot_with_retained_history(); + let mut peer = local.clone(); + peer.records[0].verified_at = Some(43); + let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); + + assert!(!cluster.definitive); + assert_eq!(cluster.reason.as_deref(), Some("peer_replacement_recovery_status_conflict")); + assert_eq!(cluster.records.len(), 2); + assert_eq!(cluster.records[0].state, rustfs_heal::ReplacementRecoveryState::Unknown); + assert_eq!(cluster.records[1].state, rustfs_heal::ReplacementRecoveryState::Completed); + } + #[test] fn replacement_recovery_cluster_accepts_definitive_empty_peers() { let local = rustfs_heal::ReplacementRecoverySnapshot { From b80590f1a73819012b9674d46c45154c9d1b775e Mon Sep 17 00:00:00 2001 From: Hauser Date: Wed, 14 Oct 2026 02:06:21 +0800 Subject: [PATCH 021/143] perf(heal): index replacement history reconciliation (#8527) refactor(heal): index replacement generation reconciliation Reconcile owned records and their definitive sources in one pass. Group claims by target slot and use a shared-anchor fast path while retaining pairwise ambiguity checks. Co-authored-by: heihutu Co-authored-by: zhi22915 --- docs/operations/scanner-runtime-controls.md | 2 +- rustfs/src/admin/handlers/heal.rs | 250 +++++++++++++------- 2 files changed, 165 insertions(+), 87 deletions(-) diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index eb0add3bd..372cce791 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -493,7 +493,7 @@ Stale-upload cleanup differs in one crash-recovery detail. RustFS's stale multip Treat replacement recovery as verified only after the repair task has completed for the exact replacement instance and an operator has confirmed the target disk contains the expected `xl.meta` and data parts for every relevant object version. A replacement that is not mounted, is unsafe to format, loses its marker, or returns a partial target outcome must be treated as deferred or incomplete. Do not automate destructive replacement actions from an `idle` observation alone. A new node must not infer replacement completion from an old or unavailable peer; regard that information as unknown or degraded until every required peer can report the same replacement instance and verified completion. -`GET /rustfs/admin/v4/heal/replacement-recovery` reports durable automatic replacement records from survivor disks. `local.records[]` entries distinguish `waiting_for_replacement`, `running`, `incomplete`, `unrecoverable`, `cleanup_pending`, `completed`, and `unknown`; `local.definitive=false` or any `unknown` record means the node could not prove a local replacement state. The `cluster` section queries the replacement-recovery peer RPC and sets `cluster.definitive=true` only when the expected peer topology is complete, every peer supports the RPC, every peer snapshot is locally definitive, and copies of each task agree. Different anchors can report different tasks, and a definitive node may report an empty list or a subset of another node's history. Disjoint target generations merge directly. Generations sharing a target slot must have been reported together in at least one definitive local snapshot; this preserves retained completion proofs and committed handoff histories while keeping overlapping records found only on separate peers non-definitive. A definitive history can include retired or unfinished generations, so consumers must inspect each record's state rather than treating `cluster.definitive` as proof that every generation completed. Old peers, unavailable peers, malformed peer payloads, topology gaps, and task disagreements are reported as degraded or unknown rather than complete. +`GET /rustfs/admin/v4/heal/replacement-recovery` reports durable automatic replacement records from survivor disks. `local.records[]` entries distinguish `waiting_for_replacement`, `running`, `incomplete`, `unrecoverable`, `cleanup_pending`, `completed`, and `unknown`; `local.definitive=false` or any `unknown` record means the node could not prove a local replacement state. The `cluster` section queries the replacement-recovery peer RPC and sets `cluster.definitive=true` only when the expected peer topology is complete, every peer supports the RPC, every peer snapshot is locally definitive, and copies of each task agree. Different anchors can report different tasks, and a definitive node may report an empty list or a subset of another node's history. Disjoint target generations merge directly. Every pair of generations sharing a target slot must appear together in at least one definitive local snapshot; this preserves retained completion proofs and committed handoff histories while keeping overlapping records found only on separate peers non-definitive. Shared history is not transitive: observing A with B and B with C does not establish that A and C are compatible. A definitive history can include retired or unfinished generations, so consumers must inspect each record's state rather than treating `cluster.definitive` as proof that every generation completed. Old peers, unavailable peers, malformed peer payloads, topology gaps, and task disagreements are reported as degraded or unknown rather than complete. Replacement resume and checkpoint files use an independent on-disk schema. A newer reader rejects a future schema rather than continuing with data it cannot interpret, while an older binary cannot safely enforce the new generation fence. Do not roll a cluster back after a replacement generation has started; complete that recovery with the current-or-newer release, and if it cannot complete, keep that version for diagnosis rather than deleting its durable records or continuing with an older binary. diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 13449b763..48f647d77 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -673,96 +673,80 @@ where } } -fn canonical_replacement_records( - records: &[rustfs_heal::ReplacementRecoveryRecord], -) -> Vec { - let mut records = records.to_vec(); - for record in &mut records { - record.target_slots.sort(); +fn reconcile_replacement_recovery_records( + snapshots: Vec, +) -> (Vec, bool) { + struct Observation { + record: rustfs_heal::ReplacementRecoveryRecord, + sources: Vec, } - records.sort_by(|left, right| { - ( - &left.task_id, - &left.generation, - &left.set_disk_id, - left.state, - &left.target_slots, - &left.verified_at, - &left.reason, - ) - .cmp(&( - &right.task_id, - &right.generation, - &right.set_disk_id, - right.state, - &right.target_slots, - &right.verified_at, - &right.reason, - )) - }); - records -} -fn merge_replacement_recovery_records( - snapshots: &[rustfs_heal::ReplacementRecoverySnapshot], -) -> Vec { - let mut records = BTreeMap::::new(); - for snapshot in snapshots { - for record in canonical_replacement_records(&snapshot.records) { - match records.entry(record.task_id.clone()) { - std::collections::btree_map::Entry::Vacant(entry) => { - entry.insert(record); - } - std::collections::btree_map::Entry::Occupied(entry) if entry.get() == &record => {} + let mut observations = BTreeMap::::new(); + for (source, snapshot) in snapshots.into_iter().enumerate() { + for mut record in snapshot.records { + record.target_slots.sort(); + let observation = match observations.entry(record.task_id.clone()) { + std::collections::btree_map::Entry::Vacant(entry) => entry.insert(Observation { + record, + sources: Vec::new(), + }), std::collections::btree_map::Entry::Occupied(mut entry) => { - let task_id = entry.key().clone(); - entry.insert(rustfs_heal::ReplacementRecoveryRecord { - task_id, - state: rustfs_heal::ReplacementRecoveryState::Unknown, - generation: None, - set_disk_id: None, - target_slots: Vec::new(), - reason: Some("peer replacement recovery records disagree".to_string()), - verified_at: None, - }); + if entry.get().record != record { + entry.get_mut().record = rustfs_heal::ReplacementRecoveryRecord { + task_id: entry.key().clone(), + state: rustfs_heal::ReplacementRecoveryState::Unknown, + generation: None, + set_disk_id: None, + target_slots: Vec::new(), + reason: Some("peer replacement recovery records disagree".to_string()), + verified_at: None, + }; + } + entry.into_mut() } + }; + // Snapshot order keeps sources sorted; repeated copies within one + // snapshot must not duplicate a generation's provenance. + if snapshot.definitive && observation.sources.last() != Some(&source) { + observation.sources.push(source); } } } - records.into_values().collect() -} -/// A definitive local snapshot can retain multiple generations for one slot, -/// including completed replacements and committed handoff predecessors. Only -/// overlapping generations never observed together in a definitive snapshot -/// leave the merged peer histories ambiguous. Disagreeing copies of one task -/// are already rejected by the merge. -fn replacement_generations_claim_conflicting_targets( - records: &[rustfs_heal::ReplacementRecoveryRecord], - snapshots: &[rustfs_heal::ReplacementRecoverySnapshot], -) -> bool { - let local_histories = snapshots - .iter() - .filter(|snapshot| snapshot.definitive) - .map(|snapshot| { - snapshot - .records + let conflict = if observations + .values() + .any(|observation| observation.record.state == rustfs_heal::ReplacementRecoveryState::Unknown) + { + true + } else { + let mut target_sources = BTreeMap::<&str, Vec<&[usize]>>::new(); + for observation in observations.values() { + for target in &observation.record.target_slots { + target_sources.entry(target).or_default().push(&observation.sources); + } + } + target_sources.into_values().any(|sources| { + if sources.len() < 2 { + return false; + } + // A complete retained history normally has one common anchor. + // Checking that witness avoids comparing every historical pair. + if sources[0] .iter() - .map(|record| record.task_id.as_str()) - .collect::>() - }) - .collect::>(); - records.iter().any(|left| { - !left.target_slots.is_empty() - && records.iter().any(|right| { - right.task_id != left.task_id - && !right.target_slots.is_empty() - && right.target_slots.iter().any(|slot| left.target_slots.contains(slot)) - && !local_histories - .iter() - .any(|history| history.contains(left.task_id.as_str()) && history.contains(right.task_id.as_str())) + .any(|source| sources[1..].iter().all(|other| other.binary_search(source).is_ok())) + { + return false; + } + // Pairwise shared history is sufficient even without one anchor + // containing the whole group. A transitive chain alone is not. + sources.iter().enumerate().any(|(index, left)| { + sources[index + 1..] + .iter() + .any(|right| !left.iter().any(|source| right.binary_search(source).is_ok())) }) - }) + }) + }; + (observations.into_values().map(|observation| observation.record).collect(), conflict) } fn aggregate_replacement_recovery_cluster_status( @@ -802,12 +786,8 @@ fn aggregate_replacement_recovery_cluster_status( // disagree about one task (the merge marks those copies `Unknown`) or // claim overlapping targets without a shared definitive local history. // Retained histories and disjoint generations can be merged. - let records = merge_replacement_recovery_records(&snapshots); - if records - .iter() - .any(|record| matches!(record.state, rustfs_heal::ReplacementRecoveryState::Unknown)) - || replacement_generations_claim_conflicting_targets(&records, &snapshots) - { + let (records, conflict) = reconcile_replacement_recovery_records(snapshots); + if conflict { reason.get_or_insert("peer_replacement_recovery_status_conflict"); } @@ -2563,6 +2543,104 @@ mod tests { assert_eq!(cluster.records[1].state, rustfs_heal::ReplacementRecoveryState::Completed); } + #[test] + fn replacement_recovery_cluster_reconciles_target_and_source_matrix() { + let task_ids = [ + "11111111-1111-4111-8111-111111111111", + "22222222-2222-4222-8222-222222222222", + "33333333-3333-4333-8333-333333333333", + ]; + for targets in 0u8..8 { + let mut records = task_ids.map(replacement_record); + for (task, record) in records.iter_mut().enumerate() { + record.target_slots = vec![format!("http://node-a:9000/mnt/disk{}", (targets >> task) & 1)]; + } + // All non-empty source subsets cover pairwise histories without + // one common anchor, transitive-only histories, and empty peers. + for a in 1u8..8 { + for b in 1u8..8 { + for c in 1u8..8 { + let masks = [a, b, c]; + let snapshots = (0..3) + .map(|source| rustfs_heal::ReplacementRecoverySnapshot { + records: records + .iter() + .enumerate() + .filter(|(task, _)| masks[*task] & (1 << source) != 0) + .map(|(_, record)| record.clone()) + .collect(), + definitive: true, + reason: None, + }) + .collect::>(); + let expected = (0..3).all(|left| { + (left + 1..3).all(|right| { + records[left].target_slots != records[right].target_slots || masks[left] & masks[right] != 0 + }) + }); + let mut snapshots = snapshots.into_iter(); + let local = snapshots.next().expect("three source snapshots"); + let cluster = aggregate_replacement_recovery_cluster_status( + vec![local], + snapshots.map(|snapshot| Ok(Some(snapshot))).collect(), + 3, + true, + ); + + assert_eq!(cluster.definitive, expected, "targets={targets}, sources={masks:?}"); + assert_eq!(cluster.reason.is_none(), expected); + assert_eq!(cluster.records, records); + } + } + } + } + } + + #[test] + fn replacement_recovery_cluster_rejects_overlap_on_a_secondary_target() { + let mut local = replacement_snapshot_with_retained_history(); + local.records[0].target_slots.push("http://node-a:9000/mnt/disk2".to_string()); + let mut peer = replacement_snapshot("33333333-3333-4333-8333-333333333333"); + peer.records[0].target_slots = vec!["http://node-a:9000/mnt/disk2".to_string()]; + let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); + + assert!(!cluster.definitive); + assert_eq!(cluster.reason.as_deref(), Some("peer_replacement_recovery_status_conflict")); + assert_eq!(cluster.records.len(), 3); + } + + #[test] + fn replacement_recovery_cluster_deduplicates_reordered_targets() { + let mut local = replacement_snapshot("11111111-1111-4111-8111-111111111111"); + local.records[0].target_slots.push("http://node-a:9000/mnt/disk2".to_string()); + let mut peer = local.clone(); + peer.records[0].target_slots.reverse(); + let expected = local.records.clone(); + let cluster = aggregate_replacement_recovery_cluster_status(vec![local], vec![Ok(Some(peer))], 2, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records, expected); + } + + #[test] + fn replacement_recovery_cluster_accepts_history_from_a_high_index_peer() { + let empty = rustfs_heal::ReplacementRecoverySnapshot { + records: Vec::new(), + definitive: true, + reason: None, + }; + let mut peers = vec![Ok(Some(empty.clone())); 80]; + let history = replacement_snapshot_with_retained_history(); + let expected = history.records.clone(); + peers.push(Ok(Some(history))); + let cluster = aggregate_replacement_recovery_cluster_status(vec![empty], peers, 82, true); + + assert!(cluster.definitive); + assert!(cluster.reason.is_none()); + assert_eq!(cluster.records, expected); + } + #[test] fn replacement_recovery_cluster_accepts_definitive_empty_peers() { let local = rustfs_heal::ReplacementRecoverySnapshot { From 93d51d172f51711799d91b1932abce02513d40e3 Mon Sep 17 00:00:00 2001 From: Hauser Date: Wed, 14 Oct 2026 02:19:22 +0800 Subject: [PATCH 022/143] fix(server): cancel request watchdogs with response futures (#8528) Tie watchdog cancellation to response future ownership before the first poll, and prefer cancellation when the watchdog deadline is also ready. Preserve completion logging and genuine slow-request warnings. Add paused-clock regression coverage for dropped, completed, panicking, and still-pending request futures. Co-authored-by: heihutu Co-authored-by: zhi22915 --- rustfs/src/server/layer.rs | 155 ++++++++++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 3 deletions(-) diff --git a/rustfs/src/server/layer.rs b/rustfs/src/server/layer.rs index 1cbbeefbc..bcc21fb37 100644 --- a/rustfs/src/server/layer.rs +++ b/rustfs/src/server/layer.rs @@ -587,6 +587,7 @@ where let watchdog_context = context.clone(); async move { tokio::select! { + biased; _ = watchdog.cancelled() => {} _ = tokio::time::sleep(HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD) => { watchdog_context.log_slow_inflight(); @@ -596,11 +597,11 @@ where }); } + // An unpolled response future must also cancel its watchdog on drop. + let watchdog_guard = watchdog.map(CancellationToken::drop_guard); Box::pin(async move { let result = inner.call(req).await; - if let Some(watchdog) = watchdog { - watchdog.cancel(); - } + drop(watchdog_guard); match &result { Ok(response) => context.log_response(response), Err(error) => context.log_failure(error), @@ -5747,6 +5748,154 @@ mod tests { assert_eq!(redact_sensitive_uri_query(&uri), "/rustfs/admin/v3/users?token=not-a-download-token"); } + #[tokio::test(start_paused = true)] + async fn request_logging_watchdog_cancels_dropped_futures() { + for poll_request in [false, true] { + for deadline_ready in [false, true] { + let attempts = if deadline_ready { 32 } else { 1 }; + for attempt in 0..attempts { + let writer = SharedWriter::default(); + let captured = writer.buffer.clone(); + let _subscriber = tracing::subscriber::set_default( + tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_writer(writer) + .finish(), + ); + assert!(tracing::enabled!(target: HTTP_SERVER_LOG_TARGET, Level::WARN)); + let inner = + tower::service_fn(|_: HttpRequest<()>| std::future::pending::, Infallible>>()); + let mut service = RequestLoggingLayer.layer(inner); + let mut response = service.call(HttpRequest::new(())); + if poll_request { + assert!(futures::poll!(&mut response).is_pending()); + } + tokio::task::yield_now().await; + if deadline_ready { + tokio::time::advance(HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD).await; + } + drop(response); + tokio::time::advance(HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD + Duration::from_millis(1)).await; + tokio::task::yield_now().await; + + let output = String::from_utf8(captured.lock().expect("logs").clone()).expect("UTF-8 logs"); + assert!( + !output.contains(HTTP_REQUEST_INFLIGHT_SLOW_EVENT), + "polled={poll_request}, deadline_ready={deadline_ready}, attempt={attempt}: {output}" + ); + } + } + } + } + + #[tokio::test(start_paused = true)] + async fn request_logging_watchdog_cancels_panicking_future() { + let writer = SharedWriter::default(); + let captured = writer.buffer.clone(); + let _subscriber = tracing::subscriber::set_default( + tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_writer(writer) + .finish(), + ); + assert!(tracing::enabled!(target: HTTP_SERVER_LOG_TARGET, Level::WARN)); + let inner = tower::service_fn(|_: HttpRequest<()>| { + std::future::poll_fn(|_| -> Poll, Infallible>> { panic!("test service panic") }) + }); + let mut service = RequestLoggingLayer.layer(inner); + let mut response = service.call(HttpRequest::new(())); + tokio::task::yield_now().await; + let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + response.as_mut().poll(&mut Context::from_waker(std::task::Waker::noop())) + })); + assert!(panic.is_err()); + tokio::time::advance(HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD + Duration::from_millis(1)).await; + tokio::task::yield_now().await; + + let output = String::from_utf8(captured.lock().expect("logs").clone()).expect("UTF-8 logs"); + assert!(!output.contains(HTTP_REQUEST_INFLIGHT_SLOW_EVENT), "{output}"); + drop(response); + } + + #[tokio::test(start_paused = true)] + async fn request_logging_watchdog_cancels_completed_futures() { + for fail_request in [false, true] { + let writer = SharedWriter::default(); + let captured = writer.buffer.clone(); + let _subscriber = tracing::subscriber::set_default( + tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_writer(writer) + .finish(), + ); + assert!(tracing::enabled!(target: HTTP_SERVER_LOG_TARGET, Level::WARN)); + let inner = tower::service_fn(move |_: HttpRequest<()>| { + std::future::ready(if fail_request { + Err(io::Error::other("test service failure")) + } else { + Ok(Response::new(())) + }) + }); + let mut service = RequestLoggingLayer.layer(inner); + let mut response = service.call(HttpRequest::new(())); + tokio::task::yield_now().await; + let Poll::Ready(result) = futures::poll!(&mut response) else { + panic!("service should complete immediately"); + }; + assert_eq!(result.is_err(), fail_request); + // Retaining a completed future must not retain an active watchdog. + tokio::time::advance(HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD + Duration::from_millis(1)).await; + tokio::task::yield_now().await; + + let output = String::from_utf8(captured.lock().expect("logs").clone()).expect("UTF-8 logs"); + assert!(!output.contains(HTTP_REQUEST_INFLIGHT_SLOW_EVENT), "{output}"); + let expected_event = if fail_request { + HTTP_REQUEST_FAILED_EVENT + } else { + HTTP_REQUEST_COMPLETED_EVENT + }; + assert_eq!(output.matches(expected_event).count(), 1, "{output}"); + drop(response); + } + } + + #[tokio::test(start_paused = true)] + async fn request_logging_watchdog_warns_once_for_pending_request() { + let writer = SharedWriter::default(); + let captured = writer.buffer.clone(); + let _subscriber = tracing::subscriber::set_default( + tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_writer(writer) + .finish(), + ); + assert!(tracing::enabled!(target: HTTP_SERVER_LOG_TARGET, Level::WARN)); + let inner = tower::service_fn(|_: HttpRequest<()>| std::future::pending::, Infallible>>()); + let mut service = RequestLoggingLayer.layer(inner); + let mut response = service.call(HttpRequest::new(())); + assert!(futures::poll!(&mut response).is_pending()); + tokio::task::yield_now().await; + tokio::time::advance(HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD - Duration::from_millis(1)).await; + tokio::task::yield_now().await; + assert!(captured.lock().expect("logs").is_empty()); + + tokio::time::advance(Duration::from_millis(1)).await; + tokio::task::yield_now().await; + let output = String::from_utf8(captured.lock().expect("logs").clone()).expect("UTF-8 logs"); + assert_eq!(output.matches(HTTP_REQUEST_INFLIGHT_SLOW_EVENT).count(), 1, "{output}"); + assert!(output.contains("response_pending"), "{output}"); + + tokio::time::advance(HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD).await; + tokio::task::yield_now().await; + let output = String::from_utf8(captured.lock().expect("logs").clone()).expect("UTF-8 logs"); + assert_eq!(output.matches(HTTP_REQUEST_INFLIGHT_SLOW_EVENT).count(), 1, "{output}"); + drop(response); + } + #[tokio::test] async fn request_logging_bounds_s3_failure_bursts_without_losing_counts_or_leaking_queries() { use rustfs_io_metrics::s3_http_metrics::s3_http_metrics_snapshot; From 4502a1f9b0b6ba44553470fa71381e4076cd110e Mon Sep 17 00:00:00 2001 From: Hauser Date: Wed, 14 Oct 2026 02:59:59 +0800 Subject: [PATCH 023/143] fix(ci): repair profiling and compact transition test gates (#8526) * fix(deps): patch profiling dependency for RUSTSEC-2026-0344 Vendor the published Pyroscope package with an object 0.40 dependency and lock ruzstd 0.9.1. Preserve runtime sources and profiling features, add bounded decoder regressions, and run the patched package in audit CI. Co-Authored-By: heihutu Co-Authored-By: zhi22915 Signed-off-by: Hauser * fix(deps): correct vendored pyroscope documentation typos Correct copied comments and changelog spelling for the repository typo check, and describe these comment-only changes in the vendor provenance. Co-Authored-By: heihutu Co-Authored-By: zhi22915 Signed-off-by: Hauser * fix(ecstore): refresh compact transition proof per test case Install the short-lived homogeneous fleet proof immediately before each kill-point transaction. Earlier cancellation and recovery work must not age the next case's writer capability fixture. Report an early transition result instead of waiting for an unreachable kill point. Preserve cancellation, remote-owner and payload assertions. Co-Authored-By: heihutu Co-Authored-By: zhi22915 Signed-off-by: Hauser --------- Signed-off-by: Hauser Co-authored-by: heihutu Co-authored-by: zhi22915 --- .github/workflows/audit.yml | 9 + Cargo.lock | 12 +- Cargo.toml | 6 + crates/ecstore/src/store/init.rs | 14 +- third-party/pyroscope/.gitignore | 2 + third-party/pyroscope/CHANGELOG.md | 265 +++++++ third-party/pyroscope/Cargo.toml | 233 ++++++ third-party/pyroscope/Cargo.toml.orig | 119 +++ third-party/pyroscope/LICENSE | 202 +++++ third-party/pyroscope/README.md | 22 + third-party/pyroscope/RUSTFS.md | 9 + third-party/pyroscope/build.rs | 19 + third-party/pyroscope/examples/jemalloc.rs | 67 ++ third-party/pyroscope/rustfmt.toml | 0 third-party/pyroscope/src/backend/backend.rs | 134 ++++ third-party/pyroscope/src/backend/jemalloc.rs | 87 +++ third-party/pyroscope/src/backend/mod.rs | 71 ++ third-party/pyroscope/src/backend/pprof.rs | 240 ++++++ .../src/backend/pprofrs/addr_validate.rs | 140 ++++ .../pprofrs/backtrace/framehop_unwinder.rs | 189 +++++ .../backtrace/framehop_unwinder/shlib.rs | 78 ++ .../src/backend/pprofrs/backtrace/mod.rs | 51 ++ .../src/backend/pprofrs/collector.rs | 558 ++++++++++++++ .../pyroscope/src/backend/pprofrs/error.rs | 17 + .../pyroscope/src/backend/pprofrs/frames.rs | 236 ++++++ .../pyroscope/src/backend/pprofrs/mod.rs | 32 + .../pyroscope/src/backend/pprofrs/profiler.rs | 540 +++++++++++++ .../pyroscope/src/backend/pprofrs/report.rs | 103 +++ .../pyroscope/src/backend/pprofrs/timer.rs | 70 ++ third-party/pyroscope/src/backend/ruleset.rs | 82 ++ third-party/pyroscope/src/backend/tests.rs | 378 +++++++++ third-party/pyroscope/src/backend/types.rs | 303 ++++++++ .../pyroscope/src/encode/gen/google.rs | 233 ++++++ third-party/pyroscope/src/encode/gen/mod.rs | 3 + third-party/pyroscope/src/encode/gen/push.rs | 31 + third-party/pyroscope/src/encode/gen/types.rs | 11 + third-party/pyroscope/src/encode/mod.rs | 2 + third-party/pyroscope/src/encode/pprof.rs | 159 ++++ third-party/pyroscope/src/error.rs | 67 ++ third-party/pyroscope/src/ffikit.rs | 74 ++ third-party/pyroscope/src/lib.rs | 67 ++ third-party/pyroscope/src/pyroscope.rs | 723 ++++++++++++++++++ third-party/pyroscope/src/session.rs | 270 +++++++ third-party/pyroscope/src/timer/epoll.rs | 272 +++++++ third-party/pyroscope/src/timer/kqueue.rs | 195 +++++ third-party/pyroscope/src/timer/mod.rs | 38 + third-party/pyroscope/src/timer/sleep.rs | 118 +++ third-party/pyroscope/src/utils.rs | 167 ++++ third-party/pyroscope/tests/agent.rs | 41 + third-party/pyroscope/tests/pprof_backend.rs | 49 ++ third-party/pyroscope/tests/rustfs_zstd.rs | 41 + third-party/pyroscope/tests/session.rs | 61 ++ third-party/pyroscope/tests/timer-epoll.rs | 86 +++ third-party/pyroscope/tests/timer.rs | 29 + 54 files changed, 7015 insertions(+), 10 deletions(-) create mode 100644 third-party/pyroscope/.gitignore create mode 100644 third-party/pyroscope/CHANGELOG.md create mode 100644 third-party/pyroscope/Cargo.toml create mode 100644 third-party/pyroscope/Cargo.toml.orig create mode 100644 third-party/pyroscope/LICENSE create mode 100644 third-party/pyroscope/README.md create mode 100644 third-party/pyroscope/RUSTFS.md create mode 100644 third-party/pyroscope/build.rs create mode 100644 third-party/pyroscope/examples/jemalloc.rs create mode 100644 third-party/pyroscope/rustfmt.toml create mode 100644 third-party/pyroscope/src/backend/backend.rs create mode 100644 third-party/pyroscope/src/backend/jemalloc.rs create mode 100644 third-party/pyroscope/src/backend/mod.rs create mode 100644 third-party/pyroscope/src/backend/pprof.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/addr_validate.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder/shlib.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/backtrace/mod.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/collector.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/error.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/frames.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/mod.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/profiler.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/report.rs create mode 100644 third-party/pyroscope/src/backend/pprofrs/timer.rs create mode 100644 third-party/pyroscope/src/backend/ruleset.rs create mode 100644 third-party/pyroscope/src/backend/tests.rs create mode 100644 third-party/pyroscope/src/backend/types.rs create mode 100644 third-party/pyroscope/src/encode/gen/google.rs create mode 100644 third-party/pyroscope/src/encode/gen/mod.rs create mode 100644 third-party/pyroscope/src/encode/gen/push.rs create mode 100644 third-party/pyroscope/src/encode/gen/types.rs create mode 100644 third-party/pyroscope/src/encode/mod.rs create mode 100644 third-party/pyroscope/src/encode/pprof.rs create mode 100644 third-party/pyroscope/src/error.rs create mode 100644 third-party/pyroscope/src/ffikit.rs create mode 100644 third-party/pyroscope/src/lib.rs create mode 100644 third-party/pyroscope/src/pyroscope.rs create mode 100644 third-party/pyroscope/src/session.rs create mode 100644 third-party/pyroscope/src/timer/epoll.rs create mode 100644 third-party/pyroscope/src/timer/kqueue.rs create mode 100644 third-party/pyroscope/src/timer/mod.rs create mode 100644 third-party/pyroscope/src/timer/sleep.rs create mode 100644 third-party/pyroscope/src/utils.rs create mode 100644 third-party/pyroscope/tests/agent.rs create mode 100644 third-party/pyroscope/tests/pprof_backend.rs create mode 100644 third-party/pyroscope/tests/rustfs_zstd.rs create mode 100644 third-party/pyroscope/tests/session.rs create mode 100644 third-party/pyroscope/tests/timer-epoll.rs create mode 100644 third-party/pyroscope/tests/timer.rs diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index c1406bec2..ea0e3efdf 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -20,6 +20,7 @@ on: paths: - '**/Cargo.toml' - '**/Cargo.lock' + - 'third-party/pyroscope/**' - 'deny.toml' - '.github/actions/**' - '.github/workflows/**' @@ -39,6 +40,7 @@ on: paths: - '**/Cargo.toml' - '**/Cargo.lock' + - 'third-party/pyroscope/**' - 'deny.toml' - '.github/actions/**' - '.github/workflows/**' @@ -129,6 +131,13 @@ jobs: - name: Run cargo-deny run: cargo deny check --hide-inclusion-graph advisories sources bans licenses + - name: Test patched profiling dependency + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target/pyroscope-dependency + run: >- + cargo test --manifest-path third-party/pyroscope/Cargo.toml + --features backend-pprof-rs --lib --test rustfs_zstd + workflow-pin-report: name: Workflow Pin Report if: github.repository == 'rustfs/rustfs' && (github.event_name != 'pull_request' || github.event.action != 'closed') diff --git a/Cargo.lock b/Cargo.lock index 3d4cf7f87..afba0c499 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7257,9 +7257,7 @@ version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" dependencies = [ - "flate2", "memchr", - "ruzstd", ] [[package]] @@ -7269,9 +7267,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd229a0361b9d0d4396176e02d65897f487eebeab7caa6d443855ee152ca0b9c" dependencies = [ "crc32fast", + "flate2", "hashbrown 0.17.1", "indexmap 2.14.2", "memchr", + "ruzstd", ] [[package]] @@ -8530,8 +8530,6 @@ dependencies = [ [[package]] name = "pyroscope" version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b1989a61e856eba4ef903eadad8313f8376e88e7c9cc5d5db8217d832a64d9" dependencies = [ "aligned-vec", "backtrace", @@ -8542,7 +8540,7 @@ dependencies = [ "log", "memmap2", "nix 0.31.3", - "object 0.39.1", + "object 0.40.0", "once_cell", "prost 0.14.4", "reqwest", @@ -11288,9 +11286,9 @@ dependencies = [ [[package]] name = "ruzstd" -version = "0.8.3" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7c1c839d570d835527c9a5e4db7cb2198683a988cb9d7293fc8674e6bd58fc8" +checksum = "69e4ce4a039f42bf25920b95667e4d789e98749ab11bbb8d2c2936f788421bb5" dependencies = [ "twox-hash", ] diff --git a/Cargo.toml b/Cargo.toml index a0b447ac7..fd410f859 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ members = [ "crates/io-core", # Buffered Bytes, mmap-then-copy, and aligned pread I/O helpers "crates/zip", # ZIP file handling and compression ] +exclude = ["third-party/pyroscope"] resolver = "3" [workspace.package] @@ -421,3 +422,8 @@ inherits = "release" inherits = "release" debug = true strip = "none" + +# Remove this patch when Pyroscope releases support for object 0.40 or later. +# object 0.39 requires ruzstd 0.8, affected by RUSTSEC-2026-0344. +[patch.crates-io] +pyroscope = { path = "third-party/pyroscope" } diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 86cf16be7..6086f80e6 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -22143,7 +22143,6 @@ mod tests { .make_bucket(bucket, &MakeBucketOptions::default()) .await .expect("kill-point bucket should be created"); - let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test"); for (index, point, expected_state, expected_revision, expected_committed, expected_recovered) in [ ( @@ -22187,10 +22186,14 @@ mod tests { .await .expect("kill-point source should be written"); let remote_before = backend.object_count().await; + // Earlier cancellation/recovery cases must not consume this + // transaction's 30-second homogeneous-fleet proof window. + let _compaction_proof = + install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test"); let barrier = TransitionTransactionKillPointBarrier::install(bucket, &object, point); let transition_store = store.clone(); let transition_object = object.clone(); - let transition = tokio::spawn(async move { + let mut transition = tokio::spawn(async move { transition_store .transition_object( bucket, @@ -22209,7 +22212,12 @@ mod tests { ) .await }); - barrier.wait_until_paused().await; + tokio::select! { + _ = barrier.wait_until_paused() => {} + result = &mut transition => { + panic!("transition exited before reaching {point:?}: {result:?}"); + } + } transition.abort(); assert!( transition diff --git a/third-party/pyroscope/.gitignore b/third-party/pyroscope/.gitignore new file mode 100644 index 000000000..042776aad --- /dev/null +++ b/third-party/pyroscope/.gitignore @@ -0,0 +1,2 @@ +/Cargo.lock +/target/ diff --git a/third-party/pyroscope/CHANGELOG.md b/third-party/pyroscope/CHANGELOG.md new file mode 100644 index 000000000..eca61685b --- /dev/null +++ b/third-party/pyroscope/CHANGELOG.md @@ -0,0 +1,265 @@ +# Changelog + +## [2.1.1](https://github.com/grafana/pyroscope-rs/compare/lib-2.1.0...lib-2.1.1) (2026-07-21) + + +### Bug Fixes + +* avoid slice construction in address validation ([#569](https://github.com/grafana/pyroscope-rs/issues/569)) ([d6f45fe](https://github.com/grafana/pyroscope-rs/commit/d6f45fef1443d26c64a40721013ecfa7b1d3eccd)) + + +### Miscellaneous Chores + +* **deps:** lock file maintenance ([#570](https://github.com/grafana/pyroscope-rs/issues/570)) ([c37f52e](https://github.com/grafana/pyroscope-rs/commit/c37f52e71719da9ddb4a32ac8386a4c94534c805)) + +## [2.1.0](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.6...lib-2.1.0) (2026-07-07) + + +### Features + +* add a method to set the timer's ticker in PyroscopeAgent ([#557](https://github.com/grafana/pyroscope-rs/issues/557)) ([cc13d38](https://github.com/grafana/pyroscope-rs/commit/cc13d38cb5e84017a4f5e229648457b790bd6a1a)) +* add otel.scope.name, otel.scope.version, process.runtime.name, process.runtime.version labels ([#562](https://github.com/grafana/pyroscope-rs/issues/562)) ([32f076e](https://github.com/grafana/pyroscope-rs/commit/32f076efd45aa2f3f203eb0bf8b03f222cb40648)) + + +### Bug Fixes + +* **security/unknown/:** update security memmap2 to v0.9.11 [security] ([#558](https://github.com/grafana/pyroscope-rs/issues/558)) ([d4d83f8](https://github.com/grafana/pyroscope-rs/commit/d4d83f8d350d927739c79a53bcfd7cd0115c77a4)) + + +### Miscellaneous Chores + +* bump nix ([#566](https://github.com/grafana/pyroscope-rs/issues/566)) ([1c02f30](https://github.com/grafana/pyroscope-rs/commit/1c02f3058135d1db06262ca57ead33fdc1593e6f)) +* **deps:** lock file maintenance ([#555](https://github.com/grafana/pyroscope-rs/issues/555)) ([3917b99](https://github.com/grafana/pyroscope-rs/commit/3917b99d6de4f4d6cec078aba2ec7b6f4e7044c7)) + +## [2.0.6](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.5...lib-2.0.6) (2026-05-28) + + +### Miscellaneous Chores + +* **deps:** vendor pprofrs ([#552](https://github.com/grafana/pyroscope-rs/issues/552)) ([6684026](https://github.com/grafana/pyroscope-rs/commit/6684026ba0ea4fc997e4869703d671f23ee377a7)) + + +### Continuous Integration + +* remove pr-title-check workflow ([#549](https://github.com/grafana/pyroscope-rs/issues/549)) ([e4739e4](https://github.com/grafana/pyroscope-rs/commit/e4739e4bfa280e771fbd932f39d53faa599718ba)) + +## [2.0.5](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.4...lib-2.0.5) (2026-05-27) + + +### Miscellaneous Chores + +* **deps:** Remove unnecessary framehop dep ([#550](https://github.com/grafana/pyroscope-rs/issues/550)) ([d9c0e02](https://github.com/grafana/pyroscope-rs/commit/d9c0e02c7386b3a55e0270083c1135cac0e3ce0e)) + +## [2.0.4](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.3...lib-2.0.4) (2026-05-20) + + +### Bug Fixes + +* do not recreate profiling guard, just reset the data ([#518](https://github.com/grafana/pyroscope-rs/issues/518)) ([b1fe114](https://github.com/grafana/pyroscope-rs/commit/b1fe1141ee0d86caa59db383298c818763f3fa95)) +* remove kindasafe CI leftovers ([#521](https://github.com/grafana/pyroscope-rs/issues/521)) ([f8bc0f3](https://github.com/grafana/pyroscope-rs/commit/f8bc0f31c3b775b9d0e97fb412af76059f0819ec)) + + +### Miscellaneous Chores + +* **deps:** bump openssl from 0.10.79 to 0.10.80 ([#548](https://github.com/grafana/pyroscope-rs/issues/548)) ([f3ed911](https://github.com/grafana/pyroscope-rs/commit/f3ed911e8b179d6fd63eacc2a793e884f126dfa0)) +* **deps:** lock file maintenance ([#535](https://github.com/grafana/pyroscope-rs/issues/535)) ([9554564](https://github.com/grafana/pyroscope-rs/commit/955456438ea768adfe0a89cd1ef7da22e7974bcd)) +* **deps:** lock file maintenance ([#541](https://github.com/grafana/pyroscope-rs/issues/541)) ([3abc7b6](https://github.com/grafana/pyroscope-rs/commit/3abc7b6d0c01f420a16cd2d08f44f10424c3ef3c)) +* **deps:** lock file maintenance ([#547](https://github.com/grafana/pyroscope-rs/issues/547)) ([cd30ebf](https://github.com/grafana/pyroscope-rs/commit/cd30ebfb07bfbf243125df53f051a2f2d1dd5d63)) +* **deps:** pin rust docker tag to a9cfb75 ([#544](https://github.com/grafana/pyroscope-rs/issues/544)) ([ea14091](https://github.com/grafana/pyroscope-rs/commit/ea140911e4b8bbabc8c42fbb2c7d43e0384ec4fe)) +* **deps:** remove names crate dependency ([#527](https://github.com/grafana/pyroscope-rs/issues/527)) ([5893aac](https://github.com/grafana/pyroscope-rs/commit/5893aacb5bc3660dcd269b2fee33847f1fe1ab82)) +* **deps:** update rust crate libc to v0.2.186 ([#533](https://github.com/grafana/pyroscope-rs/issues/533)) ([a7468c1](https://github.com/grafana/pyroscope-rs/commit/a7468c1db5d7955dfa0eb702a8b0371775ff2350)) +* **deps:** update rust crate reqwest to v0.13.3 ([#539](https://github.com/grafana/pyroscope-rs/issues/539)) ([b8c6fbb](https://github.com/grafana/pyroscope-rs/commit/b8c6fbb97cc5f3b22185be35c9e19e72237b912e)) +* **deps:** update rust:trixie docker digest to a9cfb75 ([#538](https://github.com/grafana/pyroscope-rs/issues/538)) ([bff1d5d](https://github.com/grafana/pyroscope-rs/commit/bff1d5d98f2a81d7a291c257bfb8291b41ffc8e5)) +* **renovate:** vulnerabilities-only with profiling deps allowlist ([#540](https://github.com/grafana/pyroscope-rs/issues/540)) ([4aa8e56](https://github.com/grafana/pyroscope-rs/commit/4aa8e5656472a08ddea21cb2128b85203920243b)) + + +### Continuous Integration + +* add ARM cross-compilation test job ([#536](https://github.com/grafana/pyroscope-rs/issues/536)) ([54bb03f](https://github.com/grafana/pyroscope-rs/commit/54bb03f6996a74ad98cc75bdd08d790bce61efa3)) +* auto-publish to crates.io after release-please PR merge ([#529](https://github.com/grafana/pyroscope-rs/issues/529)) ([a4f76b6](https://github.com/grafana/pyroscope-rs/commit/a4f76b6f9de6b6c35e4d947aba4dbfe49d8c019a)) +* Potential fix for code scanning alert no. 14: Workflow does not contain permissions ([#542](https://github.com/grafana/pyroscope-rs/issues/542)) ([51ac4b7](https://github.com/grafana/pyroscope-rs/commit/51ac4b78e4a8cc0c8130984b52181fce12e3589f)) + +## [2.0.3](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.2...lib-2.0.3) (2026-04-26) + + +### Bug Fixes + +* **epoll:** use libc::c_long for itimerspec fields ([#532](https://github.com/grafana/pyroscope-rs/issues/532)) ([093406e](https://github.com/grafana/pyroscope-rs/commit/093406e667175e5bc28ca7a1c5efe62be6b9b4a0)) + + +### Miscellaneous Chores + +* **deps:** update googleapis/release-please-action action to v5 ([#530](https://github.com/grafana/pyroscope-rs/issues/530)) ([6c97b9d](https://github.com/grafana/pyroscope-rs/commit/6c97b9d414c7a1e26b07334a1cf0922e1c2cd2c9)) + +## [2.0.2](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.1...lib-2.0.2) (2026-04-24) + + +### Miscellaneous Chores + +* **deps:** bump rustls-webpki from 0.103.12 to 0.103.13 ([#524](https://github.com/grafana/pyroscope-rs/issues/524)) ([935384d](https://github.com/grafana/pyroscope-rs/commit/935384dc4d3479a1a12df2502898a8813b775884)) +* **deps:** lock file maintenance ([#522](https://github.com/grafana/pyroscope-rs/issues/522)) ([4bd95bb](https://github.com/grafana/pyroscope-rs/commit/4bd95bb47f84f3aa6db6aa22173a7d74f774ab45)) +* **deps:** update dependency ruby to v4.0.3 ([#523](https://github.com/grafana/pyroscope-rs/issues/523)) ([e126aa2](https://github.com/grafana/pyroscope-rs/commit/e126aa24517ba6efea62302e465ea736e4cef54c)) +* **deps:** update rust:trixie docker digest to e4f09e8 ([#515](https://github.com/grafana/pyroscope-rs/issues/515)) ([f450327](https://github.com/grafana/pyroscope-rs/commit/f450327b1901f43d97d96eee5e0d73ccb64aa768)) + +## [2.0.1](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.0...lib-2.0.1) (2026-04-19) + + +### Bug Fixes + +* revert 0976d999e4a50c14459b8a0b39e72434be6d4bfc ([#483](https://github.com/grafana/pyroscope-rs/issues/483)) ([ad3d032](https://github.com/grafana/pyroscope-rs/commit/ad3d03217a14a18868b8ba21f3dbf4ed844d759a)) + + +### Miscellaneous Chores + +* **deps:** lock file maintenance ([#467](https://github.com/grafana/pyroscope-rs/issues/467)) ([09411da](https://github.com/grafana/pyroscope-rs/commit/09411daca91120c4863983273863c55b02b4dd89)) +* **deps:** lock file maintenance ([#478](https://github.com/grafana/pyroscope-rs/issues/478)) ([447d8ee](https://github.com/grafana/pyroscope-rs/commit/447d8eef3621be5b9b09b0b9eca3968bca081c2a)) +* **deps:** lock file maintenance ([#498](https://github.com/grafana/pyroscope-rs/issues/498)) ([4072a04](https://github.com/grafana/pyroscope-rs/commit/4072a04626abd98aa18bc6cda37fb0ccc0803f59)) +* **deps:** lock file maintenance ([#502](https://github.com/grafana/pyroscope-rs/issues/502)) ([bef3bc3](https://github.com/grafana/pyroscope-rs/commit/bef3bc3ec08b57b74d5d177204118492cdaa50b2)) +* **deps:** pin dependencies ([#427](https://github.com/grafana/pyroscope-rs/issues/427)) ([a46830a](https://github.com/grafana/pyroscope-rs/commit/a46830ac9a75a7634d9cc1ecc514a21b5c0f0603)) +* **deps:** pin rust docker tag to e8e2bb5 ([#501](https://github.com/grafana/pyroscope-rs/issues/501)) ([c177d5b](https://github.com/grafana/pyroscope-rs/commit/c177d5b8f28041d3956851d93e61be3aebc6d975)) +* **deps:** update dtolnay/rust-toolchain digest to 3c5f7ea ([#480](https://github.com/grafana/pyroscope-rs/issues/480)) ([c4f2e7d](https://github.com/grafana/pyroscope-rs/commit/c4f2e7dd1b1234f8870ce0f858f92542aef37c2e)) +* **deps:** update dtolnay/rust-toolchain digest to 3c5f7ea ([#500](https://github.com/grafana/pyroscope-rs/issues/500)) ([fc21bda](https://github.com/grafana/pyroscope-rs/commit/fc21bda959bdf5d5d5a44f559a2b9de8072d2fba)) +* **deps:** update googleapis/release-please-action digest to 5c625bf ([#509](https://github.com/grafana/pyroscope-rs/issues/509)) ([c347e17](https://github.com/grafana/pyroscope-rs/commit/c347e1775487456b996b87ce4672a20d9b9f4541)) +* **deps:** update rust crate env_logger to v0.11.10 ([#490](https://github.com/grafana/pyroscope-rs/issues/490)) ([24fb12d](https://github.com/grafana/pyroscope-rs/commit/24fb12d4c26c9cc167585f3fd93172015864a0e4)) +* **deps:** update rust crate libc to v0.2.183 ([#489](https://github.com/grafana/pyroscope-rs/issues/489)) ([b44bac8](https://github.com/grafana/pyroscope-rs/commit/b44bac8f4274882d793e9e9f92ab0ad9fc4692f0)) +* **deps:** update rust crate libc to v0.2.185 ([#514](https://github.com/grafana/pyroscope-rs/issues/514)) ([e72f5c6](https://github.com/grafana/pyroscope-rs/commit/e72f5c6e4944e7b2ac949d6dbd3201dc426704d5)) +* **deps:** update rust crate libflate to v2.3.0 ([#516](https://github.com/grafana/pyroscope-rs/issues/516)) ([da2835f](https://github.com/grafana/pyroscope-rs/commit/da2835f536771e089635e9fd8b97a860b09b9479)) +* **deps:** update rust crate rbspy to 0.43 ([#471](https://github.com/grafana/pyroscope-rs/issues/471)) ([1e524d3](https://github.com/grafana/pyroscope-rs/commit/1e524d38d84dfedc799515cb70ee434ef91a28e6)) +* **deps:** update rust crate rbspy to 0.44 ([#477](https://github.com/grafana/pyroscope-rs/issues/477)) ([c7cfe39](https://github.com/grafana/pyroscope-rs/commit/c7cfe39fecd8a0624df5c5d22b709e73747ddb4d)) +* **deps:** update rust crate uuid to v1.23.0 ([#481](https://github.com/grafana/pyroscope-rs/issues/481)) ([9c4b53a](https://github.com/grafana/pyroscope-rs/commit/9c4b53aef55b13dc6da99396119e33ec9dc02aa0)) +* **deps:** update rust crate uuid to v1.23.1 ([#517](https://github.com/grafana/pyroscope-rs/issues/517)) ([6a89238](https://github.com/grafana/pyroscope-rs/commit/6a89238e7127cb5f3fa0f480ba7447369a21fc12)) +* **deps:** update rust-lang/crates-io-auth-action action to v1.0.4 ([#479](https://github.com/grafana/pyroscope-rs/issues/479)) ([f4e08f3](https://github.com/grafana/pyroscope-rs/commit/f4e08f34aa2b14a1e417c005d37f75981940d675)) +* **deps:** update rust:trixie docker digest to 652612f ([#510](https://github.com/grafana/pyroscope-rs/issues/510)) ([7f76541](https://github.com/grafana/pyroscope-rs/commit/7f76541afc2a03cd783b0a0627b7a876f18c4918)) +* **deps:** update softprops/action-gh-release action to v2.5.2 ([#473](https://github.com/grafana/pyroscope-rs/issues/473)) ([1700be3](https://github.com/grafana/pyroscope-rs/commit/1700be3e34e7351a2f5f030187a2fac1ce1082e9)) +* **deps:** update softprops/action-gh-release action to v2.5.3 ([#474](https://github.com/grafana/pyroscope-rs/issues/474)) ([12e34c1](https://github.com/grafana/pyroscope-rs/commit/12e34c109bd0e408dc0401430f4da4b5245a09da)) +* **deps:** update softprops/action-gh-release action to v2.6.1 ([#475](https://github.com/grafana/pyroscope-rs/issues/475)) ([209c4c0](https://github.com/grafana/pyroscope-rs/commit/209c4c03ce08fe411046316dc333654fb64ab75b)) +* remove FFI code and kindasafe (moved to separate repos) ([#488](https://github.com/grafana/pyroscope-rs/issues/488)) ([35a4c6a](https://github.com/grafana/pyroscope-rs/commit/35a4c6aa0808f0392e39e752d95a91536068ab74)) + + +### Documentation + +* fix broken Rust documentation and add doc-test CI ([#484](https://github.com/grafana/pyroscope-rs/issues/484)) ([98ae8a5](https://github.com/grafana/pyroscope-rs/commit/98ae8a595f18946f2edf27544ea6e2bb29724cbf)) + + +### Continuous Integration + +* add musl/Alpine to Rust test matrix ([#495](https://github.com/grafana/pyroscope-rs/issues/495)) ([a4b7ad4](https://github.com/grafana/pyroscope-rs/commit/a4b7ad4c9e530576adbbf278b104d65c970d43b0)) +* add release-please configuration ([#491](https://github.com/grafana/pyroscope-rs/issues/491)) ([fbb2d8e](https://github.com/grafana/pyroscope-rs/commit/fbb2d8edc7cf400343650e77580464d32369d93d)) + +## v2.0.0 + +## Breaking Changes +- `Backend::report()` now returns `ReportBatch` instead of `Vec` ([#437](https://github.com/grafana/pyroscope-rs/pull/437), [#447](https://github.com/grafana/pyroscope-rs/pull/447)) +- New `ReportData` enum replaces the plain `Vec` in `ReportBatch`, supporting both structured reports (`ReportData::Reports`) and pre-encoded pprof bytes (`ReportData::RawPprof`) ([#447](https://github.com/grafana/pyroscope-rs/pull/447)) +- Minimum supported Rust version bumped from 1.64 to 1.66 + +## New Features +- **Jemalloc memory profiling backend** — new `backend-jemalloc` feature flag enables heap profiling via `jemalloc_pprof`. Use `pyroscope::backend::jemalloc::jemalloc_backend()` to get started ([#378](https://github.com/grafana/pyroscope-rs/pull/378)) +- **`ReportBatch` type** — backends now return a `ReportBatch` with a `profile_type` field (e.g. `"process_cpu"`, `"memory"`), enabling multi-profile support ([#437](https://github.com/grafana/pyroscope-rs/pull/437)) + +## Dependencies +- Updated `pprof` (pprof-pyroscope-fork) to v0.1500.3 ([#407](https://github.com/grafana/pyroscope-rs/pull/407)) +- Updated `object` crate to 0.38 ([#430](https://github.com/grafana/pyroscope-rs/pull/430)) +- Disabled py-spy default features to exclude CLI dependencies ([#418](https://github.com/grafana/pyroscope-rs/pull/418)) +- Added `jemalloc_pprof` 0.8 and `tokio` 1 as workspace dependencies + +## v1.0.0 +## Breaking Changes +- Removed `auth_token` from Python and Ruby FFI bindings and related code +- Removed `detect_subprocesses` from Python and Ruby configs +- Config constructor now requires app/spy identity and sample rate inputs +- Removed support for collapsed format +- Removed global tags from ruleset + +## New Features +- Integrated pprof-rs backend into main crate behind optional `backend-pprof-rs` feature +- Switched to push API (from `/ingest` to `/push`) +- Generated push API protos +- Added `ThreadId` type +- Added `rustls-no-provider` TLS feature + +## Bug Fixes / Improvements +- Unified signal logic +- Report cleanup functions +- Optimized ruleset +- Removed obscure thread id hash check +- Ruby: inline thread_id crate; remove detect_subprocess +- Dependency updates (reqwest 0.13, prost 0.14, thiserror 2.0, serde_json 1.0.115, uuid 1.20, libflate 2.1) + +## v0.5.4 +## New Features +- Add report transformation function which allows changing the report before + sending it to the Pyroscope Server. +- Add Gzip support. + +## Bug Fixes +- Use URL Builder. ([786c89b](https://github.com/pyroscope-io/pyroscope-rs/commit/786c89bb99839c45778410012a6da60267d395df)) + +## v0.5.3 +## New Features +- Add BackendConfig to make reporting of pid, thread_id and thread_name + optional. +- Backends can add a suffix to the "application_name" + +## Bug Fixes +- **main**: fixed an obsecure bug when counting stacktraces ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/bdecaa13aeae3ce7d4c3d97f88bdd104ec35e7c5)) + +## v0.5.2 +## New features +- Authentication Token support + +## API Changes +- use rust-tls instead of openssl + +## v0.5.1 +## API Changes +- Backend shutdown bug +- Docs update + +## v0.5.0 +## API Changes +- New API for building, starting and stopping the profiling agent. +- Backend supports reporting multiple threads. +- Tagging within local thread-scope + +## v0.4.0 +## API Changes +- Backend now support passing a configuration struct. +- TimerSignal enum +- pprof-rs backend is split into a different package. It has to be imported manually. + +## What's Changed +* fix: avoid binding two unrelated vars to the same type by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/18 +* avoid almost all unwraps by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/14 +* use more features of `thiserror` by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/11 +* introduce LOG_TAGs, avoid repetitive prefixes by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/10 +* allow configurable accumulation_cycle by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/21 +* Add CI Targets by @omarabid in https://github.com/pyroscope-io/pyroscope-rs/pull/22 +* 0.4.0 release by @omarabid in https://github.com/pyroscope-io/pyroscope-rs/pull/23 + +## New Contributors +* @drahnr made their first contribution in https://github.com/pyroscope-io/pyroscope-rs/pull/18 + +**Full Changelog**: https://github.com/pyroscope-io/pyroscope-rs/compare/0.3.1...lib-0.4.0 + +## v0.3.1 +Minor release with bug fixes. + +## Bug Fixes +- **session**: avoid breaking SessionManager if request fails ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/1254bcc9a3b0d76b7adbeb82ba21f33b875c0b39)) +- **typo**: variable name typo ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/0d8caffbe7855bec8158333dba2923cd07286a5f)) +- **pprof**: fix #12 ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/936d3d99a2cc82812bc8251fd2fbf8152a87d4cb)) +- **copyright**: fix #13 ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/b8eaf13137810df932e1b70e33b3ad3e25b65ece)) + +## Code Refactoring +- **option**: replace unwrap for various Options ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/3fd4e794d74523855c66f65c0b7fc8ff9dfe4604)) +- **time**: add get_time_range ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/a6d4dbcef494b2bfe8016a817201499937cf3528)) + +## v0.3.0 +First stable release + +## v0.0.2-alpha +Second beta release + +## v0.0.1-alpha +Initial beta release diff --git a/third-party/pyroscope/Cargo.toml b/third-party/pyroscope/Cargo.toml new file mode 100644 index 000000000..d4deaa8d5 --- /dev/null +++ b/third-party/pyroscope/Cargo.toml @@ -0,0 +1,233 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.66" +name = "pyroscope" +version = "2.1.1" +authors = ["Abid Omar "] +build = "build.rs" +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = """ +Pyroscope Profiler Agent for continuous profiling of Rust, Python and Ruby applications. +""" +homepage = "https://grafana.com/docs/pyroscope/latest/configure-client/language-sdks/rust/" +documentation = "https://docs.rs/pyroscope" +readme = "README.md" +keywords = [ + "pyroscope", + "profiler", + "profiling", + "pprof", +] +license = "Apache-2.0" +repository = "https://github.com/grafana/pyroscope-rs" + +[features] +backend-jemalloc = ["dep:jemalloc_pprof"] +backend-pprof-rs = [ + "dep:backtrace", + "dep:backtrace", + "dep:once_cell", + "dep:nix", + "dep:spin", + "dep:tempfile", + "dep:findshlibs", + "dep:smallvec", + "dep:framehop", + "dep:memmap2", + "dep:object", + "dep:aligned-vec", + "dep:symbolic-demangle", +] +default = ["rustls-tls"] +native-tls = ["reqwest/native-tls"] +native-tls-vendored = ["reqwest/native-tls-vendored"] +rustls-no-provider = ["reqwest/rustls-no-provider"] +rustls-tls = ["reqwest/rustls"] + +[lib] +name = "pyroscope" +path = "src/lib.rs" + +[[example]] +name = "jemalloc" +path = "examples/jemalloc.rs" +required-features = ["backend-jemalloc"] + +[[test]] +name = "agent" +path = "tests/agent.rs" + +[[test]] +name = "pprof_backend" +path = "tests/pprof_backend.rs" + +[[test]] +name = "session" +path = "tests/session.rs" + +[[test]] +name = "timer" +path = "tests/timer.rs" + +[[test]] +name = "timer-epoll" +path = "tests/timer-epoll.rs" + +[dependencies.aligned-vec] +version = "0.6" +optional = true + +[dependencies.backtrace] +version = "0.3" +optional = true + +[dependencies.findshlibs] +version = "0.10" +optional = true + +[dependencies.framehop] +version = "0.16" +optional = true + +[dependencies.jemalloc_pprof] +version = "0.9" +features = ["symbolize"] +optional = true + +[dependencies.libc] +version = "^0.2.124" + +[dependencies.libflate] +version = "2.1.0" + +[dependencies.log] +version = "0.4" + +[dependencies.memmap2] +version = "0.9" +optional = true + +[dependencies.nix] +version = "0.31" +features = [ + "signal", + "fs", +] +optional = true +default-features = false + +[dependencies.object] +version = "0.40" +optional = true + +[dependencies.once_cell] +version = "1.9" +optional = true + +[dependencies.prost] +version = "0.14" + +[dependencies.reqwest] +version = "0.13" +features = [ + "blocking", + "query", +] +default-features = false + +[dependencies.serde_json] +version = "1.0.115" + +[dependencies.smallvec] +version = "1.7" +optional = true + +[dependencies.spin] +version = "0.12" +optional = true + +[dependencies.symbolic-demangle] +version = "13.1.0" +features = ["rust"] +optional = true +default-features = false + +[dependencies.tempfile] +version = "3.1" +optional = true + +[dependencies.thiserror] +version = "2.0" + +[dependencies.url] +version = "2.2.2" + +[dependencies.uuid] +version = "1.20.0" +features = ["v4"] + +[dev-dependencies.assert_matches] +version = "=1.5.0" + +[dev-dependencies.claims] +version = "=0.8.0" + +[dev-dependencies.env_logger] +version = "=0.11.10" + +[dev-dependencies.tikv-jemallocator] +version = "0.7" +features = ["profiling"] + +[profile.bench] +opt-level = 3 +lto = true +codegen-units = 1 +debug = 0 +debug-assertions = false +rpath = false + +[profile.dev] +opt-level = 0 +lto = false +codegen-units = 4 +debug = 2 +debug-assertions = true +rpath = false + +[profile.release] +opt-level = 3 +lto = true +codegen-units = 1 +debug = 0 +debug-assertions = false +rpath = false +strip = "symbols" + +[profile.test] +opt-level = 1 +lto = false +codegen-units = 4 +debug = 2 +debug-assertions = true +rpath = false + +[[test]] +name = "rustfs_zstd" +path = "tests/rustfs_zstd.rs" +required-features = ["backend-pprof-rs"] diff --git a/third-party/pyroscope/Cargo.toml.orig b/third-party/pyroscope/Cargo.toml.orig new file mode 100644 index 000000000..cfeb84fc8 --- /dev/null +++ b/third-party/pyroscope/Cargo.toml.orig @@ -0,0 +1,119 @@ +[package] +name = "pyroscope" +description = """ +Pyroscope Profiler Agent for continuous profiling of Rust, Python and Ruby applications. +""" +keywords = ["pyroscope", "profiler", "profiling", "pprof"] +authors = ["Abid Omar "] +version = "2.1.1" +edition = "2021" +license = "Apache-2.0" +homepage = "https://grafana.com/docs/pyroscope/latest/configure-client/language-sdks/rust/" +documentation = "https://docs.rs/pyroscope" +repository = "https://github.com/grafana/pyroscope-rs" +readme = "README.md" +autobins = false +autoexamples = true +autotests = true +autobenches = true +rust-version = "1.66" + +[dependencies] +thiserror = "2.0" +log = "0.4" +reqwest = { version = "0.13", features = ["blocking", "query"], default-features = false } +uuid = { version = "1.20.0", features = ["v4"] } +url = "2.2.2" +libflate = "2.1.0" +libc = "^0.2.124" +prost = "0.14" +serde_json = "1.0.115" +jemalloc_pprof = { version = "0.9", features = ["symbolize"], optional = true } + + +# pprofrs dependencies +backtrace = { version = "0.3", optional = true } +once_cell = { version = "1.9", optional = true } +nix = { version = "0.31", default-features = false, features = ["signal", "fs"], optional = true } +spin = { version = "0.12", optional = true } +tempfile = { version = "3.1", optional = true } +findshlibs = { version = "0.10", optional = true } +smallvec = { version = "1.7", optional = true } +framehop = { version = "0.16", optional = true } +memmap2 = { version = "0.9", optional = true } +object = { version = "0.40", optional = true } +aligned-vec = { version = "0.6", optional = true } +symbolic-demangle = { version = "13.1.0", default-features = false, features = ["rust"], optional = true } + + + +[dev-dependencies] +assert_matches = "=1.5.0" +claims = "=0.8.0" +tikv-jemallocator = { version = "0.7", features = ["profiling"] } +env_logger = "=0.11.10" + +[[example]] +name = "jemalloc" +required-features = ["backend-jemalloc"] + +[features] +default = ["rustls-tls"] +rustls-tls = ["reqwest/rustls"] +native-tls = ["reqwest/native-tls"] +native-tls-vendored = ["reqwest/native-tls-vendored"] +rustls-no-provider = ["reqwest/rustls-no-provider"] +backend-pprof-rs = [ + "dep:backtrace", + "dep:backtrace", + "dep:once_cell", + "dep:nix", + "dep:spin", + "dep:tempfile", + "dep:findshlibs", + "dep:smallvec", + "dep:framehop", + "dep:memmap2", + "dep:object", + "dep:aligned-vec", + "dep:symbolic-demangle", +] +backend-jemalloc = ["dep:jemalloc_pprof"] + +[profile.dev] +opt-level = 0 +debug = true +rpath = false +lto = false +debug-assertions = true +codegen-units = 4 + +[profile.release] +opt-level = 3 +debug = false +strip = "symbols" +rpath = false +lto = true +debug-assertions = false +codegen-units = 1 + +[profile.test] +opt-level = 1 +debug = true +rpath = false +lto = false +debug-assertions = true +codegen-units = 4 + +[profile.bench] +opt-level = 3 +debug = false +rpath = false +lto = true +debug-assertions = false +codegen-units = 1 + +[[test]] +name = "rustfs_zstd" +path = "tests/rustfs_zstd.rs" +required-features = ["backend-pprof-rs"] diff --git a/third-party/pyroscope/LICENSE b/third-party/pyroscope/LICENSE new file mode 100644 index 000000000..d403710af --- /dev/null +++ b/third-party/pyroscope/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2022 Pyroscope + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third-party/pyroscope/README.md b/third-party/pyroscope/README.md new file mode 100644 index 000000000..6775f68b7 --- /dev/null +++ b/third-party/pyroscope/README.md @@ -0,0 +1,22 @@ +## Pyroscope Profiler + +**Pyroscope Profiler for Rust. Profile your Rust applications.** + +[![license](https://img.shields.io/badge/license-Apache2.0-blue.svg)](LICENSE) +[![Crate](https://img.shields.io/crates/v/pyroscope.svg)](https://crates.io/crates/pyroscope) + + +### Major Contributors + +We'd like to give a big thank you to the following contributors who have made significant contributions to this project: + +* [Abid Omar](https://github.com/omarabid) +* [Anatoly Korniltsev](https://github.com/korniltsev) +* [Bernhard Schuster](https://github.com/drahnr) + + +### License + +Pyroscope is distributed under the Apache License (Version 2.0). + +See [LICENSE](LICENSE) for details. diff --git a/third-party/pyroscope/RUSTFS.md b/third-party/pyroscope/RUSTFS.md new file mode 100644 index 000000000..d65c20cf3 --- /dev/null +++ b/third-party/pyroscope/RUSTFS.md @@ -0,0 +1,9 @@ +# Temporary Pyroscope dependency patch + +This is the published [pyroscope 2.1.1](https://crates.io/crates/pyroscope/2.1.1) source, from upstream commit `bc958d9e9d934ab97bc3c6887ed62c5880467d6b`, with its original license retained. Repository automation is omitted and documentation trailing whitespace is normalized. Copied comments and documentation also have spelling corrections required by the repository typo check. + +The only runtime dependency change is the `object` dependency requirement, from `0.39` to `0.40`, in both manifests. Runtime behavior, features, build script, examples, and original tests are unchanged; source edits beyond the dependency requirement only correct comments. `tests/rustfs_zstd.rs` adds bounded valid and oversized-block regression coverage through `object`'s decompression API, registered in both manifests. `object 0.39` requires vulnerable `ruzstd 0.8`; `object 0.40` permits the patched `ruzstd 0.9.1` required by [RUSTSEC-2026-0344](https://rustsec.org/advisories/RUSTSEC-2026-0344). Keep the patched version in the workspace lockfile and retain the existing Cargo Deny gate. + +Remove this directory, the workspace exclusion, and the `[patch.crates-io]` entry once an upstream Pyroscope release accepts `object 0.40` or later. Track validation and removal in [rustfs/backlog#2825](https://github.com/rustfs/backlog/issues/2825). + +Run the retained upstream unit tests and the RustFS regressions with `cargo test --manifest-path third-party/pyroscope/Cargo.toml --features backend-pprof-rs --lib --test rustfs_zstd`. diff --git a/third-party/pyroscope/build.rs b/third-party/pyroscope/build.rs new file mode 100644 index 000000000..c178ad379 --- /dev/null +++ b/third-party/pyroscope/build.rs @@ -0,0 +1,19 @@ +use std::env; +use std::process::Command; + +fn main() { + let rustc = env::var("RUSTC").unwrap(); + + let output = Command::new(rustc) + .arg("--version") + .output() + .expect("Failed to run rustc"); + + let version_string = + String::from_utf8(output.stdout).expect("rustc --version stdout is not utf8"); + + println!( + "cargo:rustc-env=PYROSCOPE__RUSTC_VERSION={}", + version_string.trim() + ); +} diff --git a/third-party/pyroscope/examples/jemalloc.rs b/third-party/pyroscope/examples/jemalloc.rs new file mode 100644 index 000000000..af10ab894 --- /dev/null +++ b/third-party/pyroscope/examples/jemalloc.rs @@ -0,0 +1,67 @@ +//! Jemalloc memory profiling example. +//! +//! This example demonstrates how to use the jemalloc backend for memory profiling. +//! It requires jemalloc to be configured as the global allocator with profiling enabled. +//! +//! # Requirements +//! +//! Add these dependencies to your Cargo.toml: +//! +//! ```toml +//! [dependencies] +//! pyroscope = { version = "2.0.0", features = ["backend-jemalloc"] } +//! tikv-jemallocator = { version = "0.6", features = ["profiling"] } +//! ``` +//! +//! # Running +//! +//! ```sh +//! # Enable jemalloc profiling via environment variable +//! _RJEM_MALLOC_CONF=prof:true,prof_active:true,lg_prof_sample:19 \ +//! cargo run --example jemalloc --features backend-jemalloc +//! ``` + +use pyroscope::backend::jemalloc::jemalloc_backend; +use pyroscope::pyroscope::PyroscopeAgentBuilder; +use std::time::Duration; + +// Configure jemalloc as the global allocator. +// Profiling must also be enabled at runtime via MALLOC_CONF or _RJEM_MALLOC_CONF. +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +fn main() -> Result<(), Box> { + env_logger::init(); + + let agent = PyroscopeAgentBuilder::new( + "http://localhost:4040", + "example.jemalloc", + 100, + "pyroscope-rs", + env!("CARGO_PKG_VERSION"), + jemalloc_backend(), + ) + .tags(vec![("env", "dev")]) + .build()?; + + let agent_running = agent.start()?; + + // Simulate heavy allocations for 30 seconds. + let start = std::time::Instant::now(); + let mut iteration = 0u64; + while start.elapsed() < Duration::from_secs(30) { + // Allocate vectors of varying sizes (1KB to 1MB). + for i in 0..100 { + let size = 1024 * (1 + (iteration as usize + i) % 1024); + let v: Vec = vec![0u8; size]; + std::hint::black_box(&v); + } + iteration += 1; + } + eprintln!("Completed {iteration} iterations"); + + let agent_ready = agent_running.stop()?; + agent_ready.shutdown(); + + Ok(()) +} diff --git a/third-party/pyroscope/rustfmt.toml b/third-party/pyroscope/rustfmt.toml new file mode 100644 index 000000000..e69de29bb diff --git a/third-party/pyroscope/src/backend/backend.rs b/third-party/pyroscope/src/backend/backend.rs new file mode 100644 index 000000000..4274a9f54 --- /dev/null +++ b/third-party/pyroscope/src/backend/backend.rs @@ -0,0 +1,134 @@ +#![allow(clippy::module_inception)] + +use crate::error::{PyroscopeError, Result}; +use std::{ + fmt::Debug, + sync::{Arc, Mutex}, +}; + +use super::{ReportBatch, ThreadTag}; + +/// Backend Config +#[derive(Debug, Copy, Clone, Default)] +pub struct BackendConfig { + pub report_thread_id: bool, + pub report_thread_name: bool, + pub report_pid: bool, +} + +/// Backend Trait +pub trait Backend: Send { + /// Initialize the backend. + fn initialize(&mut self) -> Result<()>; + /// Drop the backend. + fn shutdown(self: Box) -> Result<()>; + /// Generate profiling report + fn report(&mut self) -> Result; + fn add_tag(&self, tag: ThreadTag) -> Result<()>; + fn remove_tag(&self, tag: ThreadTag) -> Result<()>; +} + +/// Marker struct for Empty BackendImpl +#[derive(Debug)] +pub struct BackendBare; + +/// Marker struct for Uninitialized Backend +#[derive(Debug)] +pub struct BackendUninitialized; + +/// Marker struct for Initialized Backend +#[derive(Debug)] +pub struct BackendReady; + +/// Backend State Trait +pub trait BackendState {} +impl BackendState for BackendBare {} +impl BackendState for BackendUninitialized {} +impl BackendState for BackendReady {} + +/// Backend Accessibility Trait +pub trait BackendAccessible: BackendState {} +impl BackendAccessible for BackendUninitialized {} +impl BackendAccessible for BackendReady {} + +/// Precursor Backend Implementation +/// This struct is used to implement the Backend trait. It serves two purposes: +/// 1. It enforces state transitions using the Type System. +/// 2. It manages the lifetime of the backend through an Arc>. +pub struct BackendImpl { + /// Backend + pub backend: Arc>>>, + + /// Backend State + _state: std::marker::PhantomData, +} + +impl BackendImpl { + /// Create a new BackendImpl instance + pub fn new(backend_box: Box) -> BackendImpl { + BackendImpl { + backend: Arc::new(Mutex::new(Some(backend_box))), + _state: std::marker::PhantomData, + } + } +} + +impl BackendImpl { + /// Initialize the backend + pub fn initialize(self) -> Result> { + let backend = self.backend.clone(); + + // Initialize the backend + backend + .lock()? + .as_mut() + .ok_or(PyroscopeError::BackendImpl)? + .initialize()?; + + // Transition to BackendReady + Ok(BackendImpl { + backend, + _state: std::marker::PhantomData, + }) + } +} + +impl BackendImpl { + pub fn add_tag(&self, tag: ThreadTag) -> Result<()> { + self.backend + .lock()? + .as_ref() + .ok_or(PyroscopeError::BackendImpl)? + .add_tag(tag) + } + + pub fn remove_tag(&self, rule: ThreadTag) -> Result<()> { + self.backend + .lock()? + .as_ref() + .ok_or(PyroscopeError::BackendImpl)? + .remove_tag(rule) + } +} + +impl BackendImpl { + /// Shutdown the backend and destroy BackendImpl + pub fn shutdown(self) -> Result<()> { + self.backend + .lock()? + .take() + .ok_or(PyroscopeError::BackendImpl)? + .shutdown()?; + + Ok(()) + } + + /// Generate profiling report + pub fn report(&mut self) -> Result { + self.backend + .lock()? + .as_mut() + .ok_or(PyroscopeError::BackendImpl)? + .report() + } +} diff --git a/third-party/pyroscope/src/backend/jemalloc.rs b/third-party/pyroscope/src/backend/jemalloc.rs new file mode 100644 index 000000000..b4b3de042 --- /dev/null +++ b/third-party/pyroscope/src/backend/jemalloc.rs @@ -0,0 +1,87 @@ +use crate::backend::{ + Backend, BackendImpl, BackendUninitialized, ReportBatch, ReportData, ThreadTag, +}; +use crate::error::{PyroscopeError, Result}; + +const LOG_TAG: &str = "Pyroscope::Jemalloc"; + +/// Create a jemalloc memory profiling backend. +/// +/// # Example +/// +/// ```no_run +/// use pyroscope::pyroscope::PyroscopeAgentBuilder; +/// use pyroscope::backend::jemalloc::jemalloc_backend; +/// +/// # fn main() -> Result<(), Box> { +/// let agent = PyroscopeAgentBuilder::new( +/// "http://localhost:4040", "my-app", 100, +/// "pyroscope-rs", env!("CARGO_PKG_VERSION"), +/// jemalloc_backend(), +/// ) +/// .build()?; +/// agent.start()?; +/// # Ok(()) +/// # } +/// ``` +pub fn jemalloc_backend() -> BackendImpl { + BackendImpl::new(Box::new(Jemalloc)) +} + +#[derive(Debug)] +struct Jemalloc; + +impl Backend for Jemalloc { + fn initialize(&mut self) -> Result<()> { + let prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().ok_or_else(|| { + PyroscopeError::new( + "jemalloc: PROF_CTL not available. Ensure jemalloc is configured with prof:true", + ) + })?; + let guard = prof_ctl.try_lock().map_err(|_| { + PyroscopeError::new( + "jemalloc: failed to acquire PROF_CTL lock during initialization. \ + This is unexpected at startup; ensure no other code holds the lock.", + ) + })?; + if !guard.activated() { + return Err(PyroscopeError::new( + "jemalloc: profiling is not activated. Ensure malloc_conf includes prof:true,prof_active:true", + )); + } + + log::info!(target: LOG_TAG, "Jemalloc profiling backend initialized"); + + Ok(()) + } + + fn shutdown(self: Box) -> Result<()> { + log::trace!(target: LOG_TAG, "Shutting down jemalloc backend"); + Ok(()) + } + + fn report(&mut self) -> Result { + let prof_ctl = jemalloc_pprof::PROF_CTL + .as_ref() + .ok_or_else(|| PyroscopeError::new("jemalloc: PROF_CTL not available"))?; + let mut guard = prof_ctl.try_lock().map_err(|_| { + PyroscopeError::new("jemalloc: failed to acquire PROF_CTL lock for report") + })?; + let pprof_data = guard + .dump_pprof() + .map_err(|e| PyroscopeError::new(&format!("jemalloc: dump_pprof failed: {e}")))?; + + Ok(ReportBatch { + profile_type: "memory".into(), + data: ReportData::RawPprof(pprof_data), + }) + } + + fn add_tag(&self, _tag: ThreadTag) -> Result<()> { + Ok(()) + } + + fn remove_tag(&self, _tag: ThreadTag) -> Result<()> { + Ok(()) + } +} diff --git a/third-party/pyroscope/src/backend/mod.rs b/third-party/pyroscope/src/backend/mod.rs new file mode 100644 index 000000000..8dae70c9b --- /dev/null +++ b/third-party/pyroscope/src/backend/mod.rs @@ -0,0 +1,71 @@ +pub mod backend; +#[cfg(feature = "backend-jemalloc")] +pub mod jemalloc; +#[cfg(all( + feature = "backend-pprof-rs", + any( + all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") + ), + all( + target_os = "macos", + any(target_arch = "x86_64", target_arch = "aarch64") + ) + ) +))] +pub mod pprof; +#[cfg(all( + feature = "backend-pprof-rs", + any( + all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") + ), + all( + target_os = "macos", + any(target_arch = "x86_64", target_arch = "aarch64") + ) + ) +))] +mod pprofrs; + +#[cfg(all( + feature = "backend-pprof-rs", + any( + all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") + ), + all( + target_os = "macos", + any(target_arch = "x86_64", target_arch = "aarch64") + ) + ) +))] +pub use pprof::*; + +#[cfg(all( + feature = "backend-pprof-rs", + not(any( + all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") + ), + all( + target_os = "macos", + any(target_arch = "x86_64", target_arch = "aarch64") + ) + )) +))] +compile_error!("feature `backend-pprof-rs` is only supported on Linux/macOS x86_64/aarch64"); + +pub mod ruleset; +pub mod tests; +pub mod types; + +pub use backend::*; +#[cfg(feature = "backend-jemalloc")] +pub use jemalloc::*; +pub use ruleset::*; +pub use types::*; diff --git a/third-party/pyroscope/src/backend/pprof.rs b/third-party/pyroscope/src/backend/pprof.rs new file mode 100644 index 000000000..12d85c958 --- /dev/null +++ b/third-party/pyroscope/src/backend/pprof.rs @@ -0,0 +1,240 @@ +use crate::backend::pprofrs::{ProfilerGuard, ProfilerGuardBuilder}; +use crate::backend::{ + Backend, BackendConfig, BackendImpl, BackendUninitialized, Report, ReportBatch, ReportData, + StackBuffer, StackFrame, StackTrace, ThreadTag, ThreadTagsSet, +}; +use crate::error::{PyroscopeError, Result}; +use std::{ + collections::HashMap, + ffi::OsStr, + ops::Deref, + sync::{Arc, Mutex}, +}; + +const LOG_TAG: &str = "Pyroscope::Pprofrs"; + +pub fn pprof_backend( + config: PprofConfig, + backend_config: BackendConfig, +) -> BackendImpl { + BackendImpl::new(Box::new(Pprof::new(config, backend_config))) +} + +#[derive(Debug)] +pub struct PprofConfig { + pub sample_rate: u32, +} + +impl Default for PprofConfig { + fn default() -> Self { + PprofConfig { sample_rate: 100 } + } +} + +#[derive(Default)] +pub struct Pprof<'a> { + buffer: Arc>, + config: PprofConfig, + backend_config: BackendConfig, + guard: Arc>>>, + ruleset: ThreadTagsSet, +} + +impl std::fmt::Debug for Pprof<'_> { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { + write!(fmt, "Pprof Backend") + } +} + +impl<'a> Pprof<'a> { + pub fn new(config: PprofConfig, backend_config: BackendConfig) -> Self { + Pprof { + buffer: Arc::new(Mutex::new(StackBuffer::default())), + config, + backend_config, + guard: Arc::new(Mutex::new(None)), + ruleset: ThreadTagsSet::default(), + } + } +} + +impl Backend for Pprof<'_> { + fn shutdown(self: Box) -> Result<()> { + log::trace!(target: LOG_TAG, "Shutting down sampler thread"); + Ok(()) + } + + fn initialize(&mut self) -> Result<()> { + let profiler = ProfilerGuardBuilder::default() + .frequency(self.config.sample_rate as i32) + .build() + .map_err(|e| PyroscopeError::new(e.to_string().as_str()))?; + + *self.guard.lock()? = Some(profiler); + + Ok(()) + } + + fn report(&mut self) -> Result { + self.dump_report()?; + + let buffer = self.buffer.clone(); + + let report: StackBuffer = buffer.lock()?.deref().to_owned(); + + let reports: Vec = report.into(); + + buffer.lock()?.clear(); + + Ok(ReportBatch { + profile_type: "process_cpu".into(), + data: ReportData::Reports(reports), + }) + } + + fn add_tag(&self, tag: ThreadTag) -> Result<()> { + if self.guard.lock()?.as_ref().is_some() { + self.dump_report()?; + } + + self.ruleset.add(tag)?; + + Ok(()) + } + + fn remove_tag(&self, tag: ThreadTag) -> Result<()> { + if self.guard.lock()?.as_ref().is_some() { + self.dump_report()?; + } + + self.ruleset.remove(tag)?; + + Ok(()) + } +} + +impl Pprof<'_> { + /// Workaround for pprof-rs to interrupt the profiler. + pub fn dump_report(&self) -> Result<()> { + let report = self + .guard + .lock()? + .as_ref() + .ok_or_else(|| PyroscopeError::new("pprof-rs: ProfilerGuard report error"))? + .report() + .build_and_clear(true) + .map_err(|e| PyroscopeError::new(e.to_string().as_str()))?; + + let stack_buffer = Into::::into(Into::::into(( + report, + &self.backend_config, + ))); + + let data: HashMap = stack_buffer + .data + .iter() + .map(|(stacktrace, ss)| { + let stacktrace = stacktrace.to_owned().add_tag_rules(&self.ruleset); + (stacktrace, ss.to_owned()) + }) + .collect(); + + let buffer = self.buffer.clone(); + + for (stacktrace, count) in data { + buffer.lock()?.record_with_count(stacktrace, count)?; + } + + Ok(()) + } +} + +struct StackBufferWrapper(StackBuffer); + +impl From for StackBuffer { + fn from(stackbuffer: StackBufferWrapper) -> Self { + stackbuffer.0 + } +} + +impl From<(crate::backend::pprofrs::Report, &BackendConfig)> for StackBufferWrapper { + fn from(arg: (crate::backend::pprofrs::Report, &BackendConfig)) -> Self { + let (report, config) = arg; + let buffer_data: HashMap = report + .data + .iter() + .map(|(key, value)| { + ( + Into::::into((key.to_owned(), config)).into(), + value.to_owned() as usize, + ) + }) + .collect(); + StackBufferWrapper(StackBuffer::new(buffer_data)) + } +} + +struct StackTraceWrapper(StackTrace); + +impl From for StackTrace { + fn from(stack_trace: StackTraceWrapper) -> Self { + stack_trace.0 + } +} + +impl From<(crate::backend::pprofrs::Frames, &BackendConfig)> for StackTraceWrapper { + fn from(arg: (crate::backend::pprofrs::Frames, &BackendConfig)) -> Self { + let (frames, config) = arg; + let thread_id = frames.thread_id as libc::pthread_t; + StackTraceWrapper(StackTrace::new( + config, + None, + Some(thread_id.into()), + Some(frames.thread_name), + frames + .frames + .concat() + .iter() + .map(|frame| Into::::into(frame.to_owned()).into()) + .collect(), + )) + } +} + +struct StackFrameWrapper(StackFrame); + +impl From for StackFrame { + fn from(stack_frame: StackFrameWrapper) -> Self { + stack_frame.0 + } +} + +impl From for StackFrameWrapper { + fn from(symbol: crate::backend::pprofrs::Symbol) -> Self { + StackFrameWrapper(StackFrame::new( + None, + Some(symbol.name()), + Some( + symbol + .filename + .clone() + .unwrap_or_default() + .file_name() + .unwrap_or_else(|| OsStr::new("")) + .to_str() + .unwrap_or("") + .to_string(), + ), + Some( + symbol + .filename + .unwrap_or_default() + .to_str() + .unwrap_or("") + .to_string(), + ), + None, + symbol.lineno, + )) + } +} diff --git a/third-party/pyroscope/src/backend/pprofrs/addr_validate.rs b/third-party/pyroscope/src/backend/pprofrs/addr_validate.rs new file mode 100644 index 000000000..c7dd803be --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/addr_validate.rs @@ -0,0 +1,140 @@ +use nix::{errno::Errno, unistd::read}; +use std::mem::size_of; +use std::os::fd::{AsRawFd, OwnedFd}; + +#[derive(Default)] +pub(crate) struct Pipes { + read_fd: Option, + write_fd: Option, +} +#[inline] +#[cfg(any(target_os = "android", target_os = "linux"))] +fn create_pipe() -> nix::Result<(OwnedFd, OwnedFd)> { + use nix::fcntl::OFlag; + use nix::unistd::pipe2; + + pipe2(OFlag::O_CLOEXEC | OFlag::O_NONBLOCK) +} + +#[inline] +#[cfg(any(target_os = "macos", target_os = "freebsd"))] +fn create_pipe() -> nix::Result<(OwnedFd, OwnedFd)> { + use nix::fcntl::{fcntl, FcntlArg, FdFlag, OFlag}; + use nix::unistd::pipe; + + fn set_flags(fd: &OwnedFd) -> nix::Result<()> { + let mut flags = FdFlag::from_bits(fcntl(fd, FcntlArg::F_GETFD)?).unwrap(); + flags |= FdFlag::FD_CLOEXEC; + fcntl(fd, FcntlArg::F_SETFD(flags))?; + let mut flags = OFlag::from_bits(fcntl(fd, FcntlArg::F_GETFL)?).unwrap(); + flags |= OFlag::O_NONBLOCK; + fcntl(fd, FcntlArg::F_SETFL(flags))?; + Ok(()) + } + + let (read_fd, write_fd) = pipe()?; + set_flags(&read_fd)?; + set_flags(&write_fd)?; + Ok((read_fd, write_fd)) +} + +fn open_pipe(pipes: &mut Pipes) -> nix::Result<()> { + pipes.read_fd = None; + pipes.write_fd = None; + + let (read_fd, write_fd) = create_pipe()?; + + pipes.read_fd = Some(read_fd); + pipes.write_fd = Some(write_fd); + + Ok(()) +} + +// validate whether the address `addr` is readable through `write()` to a pipe +// +// if the second argument of `write(fd, buf, count)` is not a valid address, the +// `write()` will return an error the error number should be `EFAULT` in most +// cases, but we regard all errors (except EINTR) as a failure of validation +pub fn validate(pipes: &mut Pipes, addr: *const libc::c_void) -> bool { + // Short-circuit null pointers without asking the kernel to validate them. + if addr.is_null() { + return false; + } + + const CHECK_LENGTH: usize = 2 * size_of::<*const libc::c_void>() / size_of::(); + + // read data in the pipe + let valid_read = loop { + match &pipes.read_fd { + None => break false, + Some(read_fd) => { + let mut buf = [0u8; CHECK_LENGTH]; + + match read(read_fd, &mut buf) { + Ok(bytes) => break bytes > 0, + Err(_err @ Errno::EINTR) => continue, + Err(_err @ Errno::EAGAIN) => break true, + Err(_) => break false, + } + } + } + }; + + if !valid_read && open_pipe(pipes).is_err() { + return false; + } + + let Some(write_fd) = &pipes.write_fd else { + return false; // impossible + }; + loop { + let res = unsafe { libc::write(write_fd.as_raw_fd(), addr, CHECK_LENGTH) }; + let res = if res == -1 { + Err(Errno::last()) + } else { + Ok(res) + }; + match res { + Ok(bytes) => break bytes > 0, + Err(_err @ Errno::EINTR) => continue, + Err(_) => break false, + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use claims::assert_some; + + #[test] + fn validate_stack() { + let mut p = Pipes::default(); + let i = 0; + + assert!(validate(&mut p, &i as *const _ as *const libc::c_void)); + assert_some!(p.read_fd); + assert_some!(p.write_fd); + } + + #[test] + fn validate_heap() { + let mut p = Pipes::default(); + let vec = vec![0; 1000]; + + for i in vec.iter() { + assert!(validate(&mut p, i as *const _ as *const libc::c_void)); + } + assert_some!(p.read_fd); + assert_some!(p.write_fd); + } + + #[test] + fn failed_validate() { + let mut p = Pipes::default(); + assert!(!validate(&mut p, std::ptr::null::())); + assert!(!validate(&mut p, -1_i32 as usize as *const libc::c_void)); + assert_some!(p.read_fd); + assert_some!(p.write_fd); + } +} diff --git a/third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder.rs b/third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder.rs new file mode 100644 index 000000000..f9797a75b --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder.rs @@ -0,0 +1,189 @@ +use crate::backend::pprofrs::addr_validate::Pipes; +use framehop::{ + CacheNative, MustNotAllocateDuringUnwind, UnwindRegsNative, Unwinder, UnwinderNative, +}; +use libc::{c_void, ucontext_t}; +use once_cell::sync::Lazy; +use spin::RwLock; + +mod shlib; + +#[cfg(all(target_arch = "aarch64", target_os = "macos"))] +fn get_regs_from_context(ucontext: *mut c_void) -> Option<(UnwindRegsNative, u64)> { + let ucontext: *mut ucontext_t = ucontext as *mut ucontext_t; + if ucontext.is_null() { + return None; + } + + let thread_state = unsafe { + let mcontext = (*ucontext).uc_mcontext; + if mcontext.is_null() { + return None; + } else { + (*mcontext).__ss + } + }; + + Some(( + UnwindRegsNative::new(thread_state.__lr, thread_state.__sp, thread_state.__fp), + thread_state.__pc, + )) +} + +#[cfg(all(target_arch = "x86_64", target_os = "macos"))] +fn get_regs_from_context(ucontext: *mut c_void) -> Option<(UnwindRegsNative, u64)> { + let ucontext: *mut ucontext_t = ucontext as *mut ucontext_t; + if ucontext.is_null() { + return None; + } + + let thread_state = unsafe { + let mcontext = (*ucontext).uc_mcontext; + if mcontext.is_null() { + return None; + } else { + (*mcontext).__ss + } + }; + + Some(( + UnwindRegsNative::new(thread_state.__rip, thread_state.__rsp, thread_state.__rbp), + thread_state.__rip, + )) +} + +#[cfg(all(target_arch = "aarch64", target_os = "linux"))] +fn get_regs_from_context(ucontext: *mut c_void) -> Option<(UnwindRegsNative, u64)> { + let ucontext: *mut ucontext_t = ucontext as *mut ucontext_t; + if ucontext.is_null() { + return None; + } + + let regs = unsafe { &(*ucontext).uc_mcontext.regs }; + let sp = unsafe { (*ucontext).uc_mcontext.sp }; + Some((UnwindRegsNative::new(regs[30], sp, regs[29]), regs[30])) +} + +#[cfg(all(target_arch = "x86_64", target_os = "linux"))] +fn get_regs_from_context(ucontext: *mut c_void) -> Option<(UnwindRegsNative, u64)> { + let ucontext: *mut ucontext_t = ucontext as *mut ucontext_t; + if ucontext.is_null() { + return None; + } + let regs = unsafe { &(*ucontext).uc_mcontext.gregs }; + + Some(( + UnwindRegsNative::new( + regs[libc::REG_RIP as usize] as u64, + regs[libc::REG_RSP as usize] as u64, + regs[libc::REG_RBP as usize] as u64, + ), + regs[libc::REG_RIP as usize] as u64, + )) +} + +struct FramehopUnwinder { + unwinder: UnwinderNative, MustNotAllocateDuringUnwind>, + cache: CacheNative, + pipes: Pipes, +} + +impl FramehopUnwinder { + pub fn new() -> Self { + let mut unwinder = UnwinderNative::new(); + for obj in shlib::get_objects() { + unwinder.add_module(obj.clone()); + } + let cache = CacheNative::default(); + let pipes = Pipes::default(); + FramehopUnwinder { + unwinder, + cache, + pipes, + } + } + + pub fn iter_frames bool>(&mut self, ctx: *mut c_void, mut cb: F) { + let (regs, pc) = match get_regs_from_context(ctx) { + Some(fp) => fp, + None => return, + }; + + let mut closure = |addr| read_stack(&mut self.pipes, addr); + let mut iter = self + .unwinder + .iter_frames(pc, regs, &mut self.cache, &mut closure); + while let Ok(Some(frame)) = iter.next() { + if !cb(&Frame { + ip: frame.address() as usize, + }) { + break; + } + } + } +} + +fn read_stack(pipes: &mut Pipes, addr: u64) -> Result { + let aligned_addr = addr & !0b111; + if crate::backend::pprofrs::addr_validate::validate(pipes, aligned_addr as _) { + Ok(unsafe { (aligned_addr as *const u64).read() }) + } else { + Err(()) + } +} + +static UNWINDER: Lazy> = + Lazy::new(|| RwLock::new(FramehopUnwinder::new())); +#[derive(Clone, Debug)] +pub struct Frame { + pub ip: usize, +} + +extern "C" { + fn _Unwind_FindEnclosingFunction(pc: *mut c_void) -> *mut c_void; + +} + +impl super::Frame for Frame { + type S = backtrace::Symbol; + fn ip(&self) -> usize { + self.ip + } + + fn symbol_address(&self) -> *mut c_void { + if cfg!(target_os = "macos") || cfg!(target_os = "ios") { + self.ip as *mut c_void + } else { + unsafe { _Unwind_FindEnclosingFunction(self.ip as *mut c_void) } + } + } + + fn resolve_symbol(&self, cb: F) { + backtrace::resolve(self.ip as *mut c_void, cb); + } +} + +pub struct Trace; + +impl super::Trace for Trace { + type Frame = Frame; + + fn init() { + let _ = UNWINDER.read(); + } + + fn trace bool>(ctx: *mut c_void, cb: F) + where + Self: Sized, + { + // For Linux, this `try_write` should always succeed, because `SIGPROF` will never be delivered to + // another thread while the signal handler is running. However, I'm not sure about other OSes, so + // we use `try_write` to be safe instead of using `static mut` and `unsafe` directly. + match UNWINDER.try_write() { + None => (), + Some(mut unwinder) => { + unwinder.iter_frames(ctx, cb); + } + } + } +} diff --git a/third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder/shlib.rs b/third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder/shlib.rs new file mode 100644 index 000000000..0ad59089e --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/backtrace/framehop_unwinder/shlib.rs @@ -0,0 +1,78 @@ +use std::path::PathBuf; + +use findshlibs::{SharedLibrary, TargetSharedLibrary}; +use framehop::{Module, ModuleSectionInfo}; +use memmap2::Mmap; +use object::{Object, ObjectSection}; +use once_cell::sync::Lazy; + +static OBJECTS: Lazy>>> = Lazy::new(find_objects); + +pub fn get_objects() -> &'static [Module>] { + &OBJECTS +} + +pub struct ObjectInfo<'f>(object::File<'f>); + +impl<'file, D> ModuleSectionInfo for ObjectInfo<'file> +where + D: From<&'file [u8]>, +{ + fn base_svma(&self) -> u64 { + if let Some(section) = self.0.section_by_name("__TEXT") { + // in mach-o addresses are relative to __TEXT + section.address() + } else { + self.0.relative_address_base() + } + } + + fn section_svma_range(&mut self, name: &[u8]) -> Option> { + if let Some(section) = self.0.section_by_name_bytes(name) { + let start = section.address(); + let end = start + section.size(); + Some(start..end) + } else { + None + } + } + + fn section_data(&mut self, name: &[u8]) -> Option { + if let Some(section) = self.0.section_by_name_bytes(name) { + let data = section.data().ok()?; + Some(D::from(data)) + } else { + None + } + } +} + +fn open_mmap(path: &PathBuf) -> Option { + let file = std::fs::File::open(path).ok()?; + let mmap = unsafe { Mmap::map(&file) }.ok()?; + Some(mmap) +} + +fn find_objects() -> Vec>> { + let mut objects = Vec::new(); + // objects + TargetSharedLibrary::each(|shlib| { + let path = PathBuf::from(shlib.name()); + let base_avma = shlib.actual_load_addr().0 as u64; + let avma_range = base_avma..base_avma + shlib.len() as u64; + if let Some(mmap) = open_mmap(&path) { + if let Ok(obj) = object::File::parse(&*mmap) { + let section_info = ObjectInfo(obj); + + objects.push(Module::new( + path.to_string_lossy().to_string(), + avma_range, + base_avma, + section_info, + )); + } + } + }); + + objects +} diff --git a/third-party/pyroscope/src/backend/pprofrs/backtrace/mod.rs b/third-party/pyroscope/src/backend/pprofrs/backtrace/mod.rs new file mode 100644 index 000000000..5d5843359 --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/backtrace/mod.rs @@ -0,0 +1,51 @@ +// Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. + +use libc::c_void; +use std::path::PathBuf; + +pub trait Symbol: Sized { + fn name(&self) -> Option>; + fn lineno(&self) -> Option; + fn filename(&self) -> Option; +} + +impl Symbol for backtrace::Symbol { + fn name(&self) -> Option> { + self.name().map(|name| name.as_bytes().to_vec()) + } + + fn lineno(&self) -> Option { + self.lineno() + } + + fn filename(&self) -> Option { + self.filename().map(|filename| filename.to_owned()) + } +} + +pub trait Frame: Sized + Clone { + type S: Symbol; + + fn resolve_symbol(&self, cb: F); + + #[allow(dead_code)] + fn symbol_address(&self) -> *mut c_void; + + #[allow(dead_code)] + fn ip(&self) -> usize; +} + +pub trait Trace { + type Frame; + + // init will be called before running the first trace in signal handler + fn init() {} + + fn trace bool>(_: *mut libc::c_void, cb: F) + where + Self: Sized; +} + +pub mod framehop_unwinder; + +pub use framehop_unwinder::Trace as TraceImpl; diff --git a/third-party/pyroscope/src/backend/pprofrs/collector.rs b/third-party/pyroscope/src/backend/pprofrs/collector.rs new file mode 100644 index 000000000..57cfd9db2 --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/collector.rs @@ -0,0 +1,558 @@ +// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. + +use std::collections::hash_map::DefaultHasher; +use std::convert::TryInto; +use std::fmt::Debug; +use std::hash::{Hash, Hasher}; +use std::io::{Read, Seek, SeekFrom, Write}; + +use crate::backend::pprofrs::frames::UnresolvedFrames; + +use aligned_vec::AVec; +use tempfile::NamedTempFile; + +pub const BUCKETS: usize = 1 << 12; +pub const BUCKETS_ASSOCIATIVITY: usize = 4; +pub const BUFFER_LENGTH: usize = (1 << 18) / std::mem::size_of::>(); + +#[derive(Debug)] +pub struct Entry { + pub item: T, + pub count: isize, +} + +impl Default for Entry { + fn default() -> Self { + Entry { + item: Default::default(), + count: 0, + } + } +} + +#[derive(Debug)] +pub struct Bucket { + pub length: usize, + entries: Box<[Entry; BUCKETS_ASSOCIATIVITY]>, +} + +impl Default for Bucket { + fn default() -> Bucket { + let entries = Box::default(); + + Self { length: 0, entries } + } +} + +impl Bucket { + pub fn clear(&mut self) { + self.length = 0; + } + + pub fn add(&mut self, key: T, count: isize) -> Option> { + let mut done = false; + self.entries[0..self.length].iter_mut().for_each(|ele| { + if ele.item == key { + ele.count += count; + done = true; + } + }); + + if done { + None + } else if self.length < BUCKETS_ASSOCIATIVITY { + let ele = &mut self.entries[self.length]; + ele.item = key; + ele.count = count; + + self.length += 1; + None + } else { + let mut min_index = 0; + let mut min_count = self.entries[0].count; + for index in 0..self.length { + let count = self.entries[index].count; + if count < min_count { + min_index = index; + min_count = count; + } + } + + let mut new_entry = Entry { item: key, count }; + std::mem::swap(&mut self.entries[min_index], &mut new_entry); + Some(new_entry) + } + } + + pub fn iter(&self) -> BucketIterator<'_, T> { + BucketIterator:: { + related_bucket: self, + index: 0, + } + } +} + +pub struct BucketIterator<'a, T: 'static> { + related_bucket: &'a Bucket, + index: usize, +} + +impl<'a, T> Iterator for BucketIterator<'a, T> { + type Item = &'a Entry; + + fn next(&mut self) -> Option { + if self.index < self.related_bucket.length { + self.index += 1; + Some(&self.related_bucket.entries[self.index - 1]) + } else { + None + } + } +} + +pub struct HashCounter { + buckets: Box<[Bucket; BUCKETS]>, +} + +impl Default for HashCounter { + fn default() -> Self { + let mut v: Vec> = Vec::with_capacity(BUCKETS); + v.resize_with(BUCKETS, Default::default); + let buckets = v.into_boxed_slice().try_into().unwrap(); + + Self { buckets } + } +} + +impl HashCounter { + pub fn clear(&mut self) { + for bucket in self.buckets.iter_mut() { + bucket.clear(); + } + } + + fn hash(key: &T) -> u64 { + let mut s = DefaultHasher::new(); + key.hash(&mut s); + s.finish() + } + + pub fn add(&mut self, key: T, count: isize) -> Option> { + let hash_value = Self::hash(&key); + let bucket = &mut self.buckets[(hash_value % BUCKETS as u64) as usize]; + + bucket.add(key, count) + } + + pub fn iter(&self) -> impl Iterator> { + self.buckets.iter().flat_map(|bucket| bucket.iter()) + } +} + +pub struct TempFdArray { + file: NamedTempFile, + buffer: Box<[T; BUFFER_LENGTH]>, + buffer_index: usize, + flush_n: usize, +} + +impl TempFdArray { + fn new() -> std::io::Result> { + let file = NamedTempFile::new()?; + + let mut v: Vec = Vec::with_capacity(BUFFER_LENGTH); + v.resize_with(BUFFER_LENGTH, Default::default); + let buffer = v.into_boxed_slice().try_into().unwrap(); + + Ok(Self { + file, + buffer, + buffer_index: 0, + flush_n: 0, + }) + } +} + +impl TempFdArray { + fn clear(&mut self) -> std::io::Result<()> { + self.buffer_index = 0; + self.flush_n = 0; + self.file.as_file().set_len(0)?; + self.file.seek(SeekFrom::Start(0))?; + Ok(()) + } + + fn flush_buffer(&mut self) -> std::io::Result<()> { + self.buffer_index = 0; + let buf = unsafe { + std::slice::from_raw_parts( + self.buffer.as_ptr() as *const u8, + BUFFER_LENGTH * std::mem::size_of::(), + ) + }; + self.flush_n += 1; + self.file.write_all(buf)?; + + Ok(()) + } + + fn push(&mut self, entry: T) -> std::io::Result<()> { + if self.buffer_index >= BUFFER_LENGTH { + self.flush_buffer()?; + } + + self.buffer[self.buffer_index] = entry; + self.buffer_index += 1; + + Ok(()) + } + + fn try_iter(&self) -> std::io::Result> { + let size = BUFFER_LENGTH * self.flush_n * std::mem::size_of::(); + + let mut file_vec = AVec::with_capacity(std::mem::align_of::(), size); + let mut file = self.file.reopen()?; + + unsafe { + // it's safe as the capacity is initialized to `size`, and it'll be filled with `size` bytes + file_vec.set_len(size); + } + file.read_exact(&mut file_vec[0..size])?; + file.seek(SeekFrom::End(0))?; + + Ok(TempFdArrayIterator { + buffer: &self.buffer[0..self.buffer_index], + file_vec, + index: 0, + }) + } +} + +pub struct TempFdArrayIterator<'a, T> { + pub buffer: &'a [T], + pub file_vec: AVec, + pub index: usize, +} + +impl<'a, T> Iterator for TempFdArrayIterator<'a, T> { + type Item = &'a T; + + fn next(&mut self) -> Option { + if self.index < self.buffer.len() { + self.index += 1; + Some(&self.buffer[self.index - 1]) + } else { + let length = self.file_vec.len() / std::mem::size_of::(); + let ts = + unsafe { std::slice::from_raw_parts(self.file_vec.as_ptr() as *const T, length) }; + if self.index - self.buffer.len() < ts.len() { + self.index += 1; + Some(&ts[self.index - self.buffer.len() - 1]) + } else { + None + } + } + } +} + +pub struct Collector { + map: HashCounter, + temp_array: TempFdArray>, +} + +impl Collector { + pub fn new() -> std::io::Result { + Ok(Self { + map: HashCounter::::default(), + temp_array: TempFdArray::>::new()?, + }) + } +} + +impl Collector { + pub fn clear(&mut self) -> std::io::Result<()> { + self.map.clear(); + self.temp_array.clear()?; + Ok(()) + } + + pub fn add(&mut self, key: T, count: isize) -> std::io::Result<()> { + if let Some(evict) = self.map.add(key, count) { + self.temp_array.push(evict)?; + } + + Ok(()) + } + + pub fn try_iter(&self) -> std::io::Result>> { + Ok(self.map.iter().chain(self.temp_array.try_iter()?)) + } + + #[cfg(test)] + pub fn flushed_to_disk(&self) -> usize { + self.temp_array.flush_n + } +} + +#[cfg(test)] +mod test_utils { + use super::*; + use std::collections::BTreeMap; + + pub fn add_map(hashmap: &mut BTreeMap, entry: &Entry) { + match hashmap.get_mut(&entry.item) { + None => { + hashmap.insert(entry.item, entry.count); + } + Some(count) => *count += entry.count, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeMap; + + #[test] + fn stack_hash_counter() { + let mut stack_hash_counter = HashCounter::::default(); + stack_hash_counter.add(0, 1); + stack_hash_counter.add(1, 1); + stack_hash_counter.add(1, 1); + + stack_hash_counter.iter().for_each(|item| { + if item.item == 0 { + assert_eq!(item.count, 1); + } else if item.item == 1 { + assert_eq!(item.count, 2); + } else { + unreachable!(); + } + }); + } + + #[test] + fn evict_test() { + let mut stack_hash_counter = HashCounter::::default(); + let mut real_map = BTreeMap::new(); + + for item in 0..(1 << 10) * 4 { + for _ in 0..(item % 4) { + match stack_hash_counter.add(item, 1) { + None => {} + Some(evict) => { + test_utils::add_map(&mut real_map, &evict); + } + } + } + } + + stack_hash_counter.iter().for_each(|entry| { + test_utils::add_map(&mut real_map, entry); + }); + + for item in 0..(1 << 10) * 4 { + let count = (item % 4) as isize; + match real_map.get(&item) { + Some(item) => { + assert_eq!(*item, count); + } + None => { + assert_eq!(count, 0); + } + } + } + } + + #[test] + fn collector_test() { + let mut collector = Collector::new().unwrap(); + let mut real_map = BTreeMap::new(); + + for item in 0..(1 << 12) * 4 { + for _ in 0..(item % 4) { + collector.add(item, 1).unwrap(); + } + } + + collector.try_iter().unwrap().for_each(|entry| { + test_utils::add_map(&mut real_map, entry); + }); + + for item in 0..(1 << 12) * 4 { + let count = (item % 4) as isize; + match real_map.get(&item) { + Some(value) => { + assert_eq!(count, *value); + } + None => { + assert_eq!(count, 0); + } + } + } + } + + #[derive(Debug, Hash, Eq, PartialEq, PartialOrd, Ord, Default, Clone, Copy)] + struct AlignTest { + a: u16, + b: u32, + c: u64, + d: u64, + } + + // collector_align_test uses a bigger item to test the alignment of the collector + #[test] + fn collector_align_test() { + let mut collector = Collector::new().unwrap(); + let mut real_map = BTreeMap::new(); + + for item in 0..(1 << 12) * 4 { + for _ in 0..(item % 4) { + collector + .add( + AlignTest { + a: item as u16, + b: item as u32, + c: item as u64, + d: item as u64, + }, + 1, + ) + .unwrap(); + } + } + + collector.try_iter().unwrap().for_each(|entry| { + test_utils::add_map(&mut real_map, entry); + }); + + for item in 0..(1 << 12) * 4 { + let count = (item % 4) as isize; + let align_item = AlignTest { + a: item as u16, + b: item as u32, + c: item as u64, + d: item as u64, + }; + match real_map.get(&align_item) { + Some(value) => { + assert_eq!(count, *value); + } + None => { + assert_eq!(count, 0); + } + } + } + } + + #[cfg(test)] + fn assert_entries<'a, T: Ord + Copy + Debug + 'a>( + iter: std::io::Result>>, + expected: Vec>, + ) { + #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] + struct EntryCmp { + item: T, + count: isize, + } + + let to_cmp = |e: &Entry| EntryCmp { + item: e.item, + count: e.count, + }; + + assert!(iter.is_ok(), "iterator error: {:?}", iter.err()); + let Ok(iter) = iter else { return }; + let mut actual: Vec> = iter.map(to_cmp).collect(); + let mut expected: Vec> = expected.iter().map(to_cmp).collect(); + actual.sort(); + expected.sort(); + assert_eq!(actual, expected); + } + + #[test] + fn hash_counter_clear() { + let mut counter = HashCounter::::default(); + counter.add(1, 1); + counter.add(2, 3); + + assert_entries( + Ok(counter.iter()), + vec![ + Entry { + item: 1usize, + count: 1, + }, + Entry { item: 2, count: 3 }, + ], + ); + + counter.clear(); + + assert_entries(Ok(counter.iter()), vec![]); + + counter.add(42, 7); + assert_entries( + Ok(counter.iter()), + vec![Entry { + item: 42usize, + count: 7, + }], + ); + } + + #[test] + fn temp_fd_array_clear() { + let mut arr = TempFdArray::>::new().unwrap(); + let mut expected: Vec> = Vec::new(); + + for i in 0..=(BUFFER_LENGTH + 10) { + arr.push(Entry { item: i, count: 1 }).unwrap(); + expected.push(Entry { item: i, count: 1 }); + } + assert!(arr.flush_n > 0); + + assert_entries(arr.try_iter(), expected); + + arr.clear().unwrap(); + + assert_eq!(arr.buffer_index, 0); + assert_eq!(arr.flush_n, 0); + assert_entries(arr.try_iter(), vec![]); + + let mut expected_reuse: Vec> = Vec::new(); + for i in 0..=(BUFFER_LENGTH + 10) { + arr.push(Entry { item: i, count: 2 }).unwrap(); + expected_reuse.push(Entry { item: i, count: 2 }); + } + assert!(arr.flush_n > 0); + assert_entries(arr.try_iter(), expected_reuse); + } + + #[test] + fn collector_clear_with_disk_eviction() { + let mut collector = Collector::::new().unwrap(); + + let n = BUCKETS * BUCKETS_ASSOCIATIVITY * 4; + let mut expected_before: Vec> = Vec::new(); + for item in 0..n { + collector.add(item, 1).unwrap(); + expected_before.push(Entry { item, count: 1 }); + } + + assert!(collector.flushed_to_disk() > 0); + assert_entries(collector.try_iter(), expected_before); + + collector.clear().unwrap(); + + assert_entries(collector.try_iter(), vec![]); + + let mut expected_reuse: Vec> = Vec::new(); + for item in 0..10 { + collector.add(item, 2).unwrap(); + expected_reuse.push(Entry { item, count: 2 }); + } + assert_entries(collector.try_iter(), expected_reuse); + } +} diff --git a/third-party/pyroscope/src/backend/pprofrs/error.rs b/third-party/pyroscope/src/backend/pprofrs/error.rs new file mode 100644 index 000000000..a868b4f69 --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/error.rs @@ -0,0 +1,17 @@ +// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("{0}")] + Nix(#[from] nix::Error), + #[error("{0}")] + Io(#[from] std::io::Error), + #[error("create profiler error")] + Creating, + #[error("start running cpu profiler error")] + Running, + #[error("stop running cpu profiler error")] + NotRunning, +} + +pub type Result = std::result::Result; diff --git a/third-party/pyroscope/src/backend/pprofrs/frames.rs b/third-party/pyroscope/src/backend/pprofrs/frames.rs new file mode 100644 index 000000000..0f3224c11 --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/frames.rs @@ -0,0 +1,236 @@ +// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. + +use std::fmt::{self, Debug, Display, Formatter}; +use std::hash::{Hash, Hasher}; +use std::path::PathBuf; +use std::time::SystemTime; + +use smallvec::SmallVec; +use symbolic_demangle::demangle; + +use crate::backend::pprofrs::backtrace::{Frame, Trace, TraceImpl}; +use crate::backend::pprofrs::{MAX_DEPTH, MAX_THREAD_NAME}; + +// #[cfg(not(feature = "perfmaps"))] +fn resolve_in_perfmap(_ip: usize) -> Option { + None +} + +#[derive(Clone)] +pub struct UnresolvedFrames { + pub frames: SmallVec<[::Frame; MAX_DEPTH]>, + pub thread_name: [u8; MAX_THREAD_NAME], + pub thread_name_length: usize, + pub thread_id: u64, + pub sample_timestamp: SystemTime, +} + +impl Default for UnresolvedFrames { + fn default() -> Self { + let frames = SmallVec::with_capacity(MAX_DEPTH); + Self { + frames, + thread_name: [0; MAX_THREAD_NAME], + thread_name_length: 0, + thread_id: 0, + sample_timestamp: SystemTime::now(), + } + } +} + +impl Debug for UnresolvedFrames { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + self.frames.fmt(f) + } +} + +impl UnresolvedFrames { + pub fn new( + frames: SmallVec<[::Frame; MAX_DEPTH]>, + tn: &[u8], + thread_id: u64, + sample_timestamp: SystemTime, + ) -> Self { + let thread_name_length = tn.len(); + let mut thread_name = [0; MAX_THREAD_NAME]; + thread_name[0..thread_name_length].clone_from_slice(tn); + + Self { + frames, + thread_name, + thread_name_length, + thread_id, + sample_timestamp, + } + } +} + +impl PartialEq for UnresolvedFrames { + fn eq(&self, other: &Self) -> bool { + let (frames1, frames2) = (&self.frames, &other.frames); + if self.thread_id != other.thread_id || frames1.len() != frames2.len() { + false + } else { + Iterator::zip(frames1.iter(), frames2.iter()) + .all(|(s1, s2)| s1.symbol_address() == s2.symbol_address()) + } + } +} + +impl Eq for UnresolvedFrames {} + +impl Hash for UnresolvedFrames { + fn hash(&self, state: &mut H) { + self.frames + .iter() + .for_each(|frame| frame.symbol_address().hash(state)); + self.thread_id.hash(state); + } +} + +/// Symbol is a representation of a function symbol. It contains name and addr of it. If built with +/// debug message, it can also provide line number and filename. The name in it is not demangled. +#[derive(Debug, Clone)] +pub struct Symbol { + /// This name is raw name of a symbol (which hasn't been demangled). + pub name: Option>, + + /// Line number of this symbol. If compiled with debug message, you can get it. + pub lineno: Option, + + /// Filename of this symbol. If compiled with debug message, you can get it. + pub filename: Option, +} + +impl Symbol { + pub fn raw_name(&self) -> &[u8] { + self.name.as_deref().unwrap_or(b"Unknown") + } + + pub fn name(&self) -> String { + demangle(&String::from_utf8_lossy(self.raw_name())).into_owned() + } +} + +unsafe impl Send for Symbol {} + +impl From<&T> for Symbol +where + T: crate::backend::pprofrs::backtrace::Symbol, +{ + fn from(symbol: &T) -> Self { + Symbol { + name: symbol.name(), + lineno: symbol.lineno(), + filename: symbol.filename(), + } + } +} + +impl Display for Symbol { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + f.write_str(&self.name()) + } +} + +impl PartialEq for Symbol { + fn eq(&self, other: &Self) -> bool { + self.raw_name() == other.raw_name() + } +} + +impl Hash for Symbol { + fn hash(&self, state: &mut H) { + self.raw_name().hash(state) + } +} + +/// A representation of a backtrace. `thread_name` and `thread_id` was got from `pthread_getname_np` +/// and `pthread_self`. frames is a vector of symbols. +#[derive(Clone, PartialEq, Hash)] +pub struct Frames { + pub frames: Vec>, + pub thread_name: String, + pub thread_id: u64, + pub sample_timestamp: SystemTime, +} + +impl From for Frames { + fn from(frames: UnresolvedFrames) -> Self { + let mut fs = Vec::new(); + + let mut frame_iter = frames.frames.iter(); + + while let Some(frame) = frame_iter.next() { + let mut symbols: Vec = Vec::new(); + + if let Some(perfmap_symbol) = resolve_in_perfmap(frame.ip()) { + symbols.push(perfmap_symbol); + } else { + frame.resolve_symbol(|symbol| { + let symbol = Symbol::from(symbol); + symbols.push(symbol); + }); + } + + if symbols.iter().any(|symbol| { + // macOS prepends an underscore even with `#[no_mangle]` + matches!( + &*symbol.name(), + "perf_signal_handler" | "_perf_signal_handler" + ) + }) { + // ignore frame itself and its next one + frame_iter.next(); + continue; + } + + if !symbols.is_empty() { + fs.push(symbols); + } + } + + Self { + frames: fs, + thread_name: String::from_utf8_lossy(&frames.thread_name[0..frames.thread_name_length]) + .into_owned(), + thread_id: frames.thread_id, + sample_timestamp: frames.sample_timestamp, + } + } +} + +impl Eq for Frames {} + +impl Debug for Frames { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + for frame in self.frames.iter() { + write!(f, "FRAME: ")?; + for symbol in frame.iter() { + write!(f, "{symbol} -> ")?; + } + } + write!(f, "THREAD: ")?; + if !self.thread_name.is_empty() { + write!(f, "{}", self.thread_name) + } else { + write!(f, "ThreadId({})", self.thread_id) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn demangle_rust() { + let symbol = Symbol { + name: Some(b"_ZN3foo3barE".to_vec()), + lineno: None, + filename: None, + }; + + assert_eq!(&symbol.name(), "foo::bar") + } +} diff --git a/third-party/pyroscope/src/backend/pprofrs/mod.rs b/third-party/pyroscope/src/backend/pprofrs/mod.rs new file mode 100644 index 000000000..9b8349163 --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/mod.rs @@ -0,0 +1,32 @@ +// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. + +//! pprof-rs is an integrated profiler for rust program. + +/// Define the MAX supported stack depth. TODO: make this variable mutable. +// #[cfg(feature = "large-depth")] +// pub const MAX_DEPTH: usize = 1024; +// +// #[cfg(all(feature = "huge-depth", not(feature = "large-depth")))] +// pub const MAX_DEPTH: usize = 512; +// +// #[cfg(not(any(feature = "large-depth", feature = "huge-depth")))] +pub const MAX_DEPTH: usize = 128; + +/// Define the MAX supported thread name length. TODO: make this variable mutable. +pub const MAX_THREAD_NAME: usize = 16; + +mod addr_validate; + +mod backtrace; +mod collector; +mod error; +mod frames; +mod profiler; +mod report; +mod timer; + +// pub use self::collector::{Collector, HashCounter}; +pub use self::error::{Error, Result}; +pub use self::frames::{Frames, Symbol}; +pub use self::profiler::{ProfilerGuard, ProfilerGuardBuilder}; +pub use self::report::Report; diff --git a/third-party/pyroscope/src/backend/pprofrs/profiler.rs b/third-party/pyroscope/src/backend/pprofrs/profiler.rs new file mode 100644 index 000000000..e0ea37ab7 --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/profiler.rs @@ -0,0 +1,540 @@ +// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. + +use std::convert::TryInto; +use std::os::raw::c_int; +use std::time::SystemTime; + +use nix::sys::signal; +use once_cell::sync::Lazy; +use smallvec::SmallVec; +use spin::RwLock; + +use crate::backend::pprofrs::backtrace::{Trace, TraceImpl}; +use crate::backend::pprofrs::collector::Collector; +use crate::backend::pprofrs::error::{Error, Result}; +use crate::backend::pprofrs::frames::UnresolvedFrames; +use crate::backend::pprofrs::report::ReportBuilder; +use crate::backend::pprofrs::timer::Timer; +use crate::backend::pprofrs::{MAX_DEPTH, MAX_THREAD_NAME}; + +pub(crate) static PROFILER: Lazy>> = + Lazy::new(|| RwLock::new(Profiler::new())); + +pub struct Profiler { + pub(crate) data: Collector, + sample_counter: i32, + + running: bool, + + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))] + blocklist_segments: Vec<(usize, usize)>, +} + +#[derive(Clone)] +pub struct ProfilerGuardBuilder { + frequency: c_int, + + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))] + blocklist_segments: Vec<(usize, usize)>, +} + +impl Default for ProfilerGuardBuilder { + fn default() -> ProfilerGuardBuilder { + ProfilerGuardBuilder { + frequency: 99, + + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))] + blocklist_segments: Vec::new(), + } + } +} + +impl ProfilerGuardBuilder { + pub fn frequency(self, frequency: c_int) -> Self { + Self { frequency, ..self } + } + + pub fn build(self) -> Result> { + trigger_lazy(); + + match PROFILER.write().as_mut() { + Err(err) => { + log::error!("Error in creating profiler: {}", err); + Err(Error::Creating) + } + Ok(profiler) => { + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))] + { + profiler.blocklist_segments = self.blocklist_segments; + } + + match profiler.start() { + Ok(()) => Ok(ProfilerGuard::<'static> { + profiler: &PROFILER, + timer: Some(Timer::new(self.frequency)), + }), + Err(err) => Err(err), + } + } + } + } +} + +/// RAII structure used to stop profiling when dropped. It is the only interface to access profiler. +pub struct ProfilerGuard<'a> { + profiler: &'a RwLock>, + timer: Option, +} + +fn trigger_lazy() { + let _ = backtrace::Backtrace::new(); + let _profiler = PROFILER.read(); + TraceImpl::init(); +} + +impl ProfilerGuard<'_> { + /// Generate a report + pub fn report(&self) -> ReportBuilder<'_> { + ReportBuilder::new(self.profiler) + } +} + +impl<'a> Drop for ProfilerGuard<'a> { + fn drop(&mut self) { + drop(self.timer.take()); + + match self.profiler.write().as_mut() { + Err(_) => {} + Ok(profiler) => match profiler.stop() { + Ok(()) => {} + Err(err) => log::error!("error while stopping profiler {}", err), + }, + } + } +} + +fn write_thread_name_fallback(current_thread: libc::pthread_t, name: &mut [libc::c_char]) { + let mut len = 0; + let mut base = 1; + + while current_thread as u128 > base && len < MAX_THREAD_NAME { + base *= 10; + len += 1; + } + + let mut index = 0; + while index < len && base > 1 { + base /= 10; + + name[index] = match (48 + (current_thread as u128 / base) % 10).try_into() { + Ok(digit) => digit, + Err(_) => { + log::error!("fail to convert thread_id to string"); + 0 + } + }; + + index += 1; + } +} + +#[cfg(not(all(any(target_os = "linux", target_os = "macos"), target_env = "gnu")))] +fn write_thread_name(current_thread: libc::pthread_t, name: &mut [libc::c_char]) { + write_thread_name_fallback(current_thread, name); +} + +#[cfg(all(any(target_os = "linux", target_os = "macos"), target_env = "gnu"))] +fn write_thread_name(current_thread: libc::pthread_t, name: &mut [libc::c_char]) { + let name_ptr = name as *mut [libc::c_char] as *mut libc::c_char; + let ret = unsafe { libc::pthread_getname_np(current_thread, name_ptr, MAX_THREAD_NAME) }; + + if ret != 0 { + write_thread_name_fallback(current_thread, name); + } +} + +struct ErrnoProtector(libc::c_int); + +impl ErrnoProtector { + fn new() -> Self { + unsafe { + #[cfg(target_os = "android")] + { + let errno = *libc::__errno(); + Self(errno) + } + #[cfg(target_os = "linux")] + { + let errno = *libc::__errno_location(); + Self(errno) + } + #[cfg(any(target_os = "macos", target_os = "freebsd"))] + { + let errno = *libc::__error(); + Self(errno) + } + } + } +} + +impl Drop for ErrnoProtector { + fn drop(&mut self) { + unsafe { + #[cfg(target_os = "android")] + { + *libc::__errno() = self.0; + } + #[cfg(target_os = "linux")] + { + *libc::__errno_location() = self.0; + } + #[cfg(any(target_os = "macos", target_os = "freebsd"))] + { + *libc::__error() = self.0; + } + } + } +} + +#[no_mangle] +#[cfg_attr( + not(all(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))), + allow(unused_variables) +)] +#[allow(clippy::unnecessary_cast)] +extern "C" fn perf_signal_handler( + _signal: c_int, + _siginfo: *mut libc::siginfo_t, + ucontext: *mut libc::c_void, +) { + let _errno = ErrnoProtector::new(); + + if let Some(mut guard) = PROFILER.try_write() { + if let Ok(profiler) = guard.as_mut() { + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))] + if !ucontext.is_null() { + let ucontext: *mut libc::ucontext_t = ucontext as *mut libc::ucontext_t; + + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + let addr = + unsafe { (*ucontext).uc_mcontext.gregs[libc::REG_RIP as usize] as usize }; + + #[cfg(all(target_arch = "x86_64", target_os = "freebsd"))] + let addr = unsafe { (*ucontext).uc_mcontext.mc_rip as usize }; + + #[cfg(all(target_arch = "x86_64", target_os = "macos"))] + let addr = unsafe { + let mcontext = (*ucontext).uc_mcontext; + if mcontext.is_null() { + 0 + } else { + (*mcontext).__ss.__rip as usize + } + }; + + #[cfg(all( + target_arch = "aarch64", + any(target_os = "android", target_os = "linux") + ))] + let addr = unsafe { (*ucontext).uc_mcontext.pc as usize }; + + #[cfg(all(target_arch = "aarch64", target_os = "freebsd"))] + let addr = unsafe { (*ucontext).mc_gpregs.gp_elr as usize }; + + #[cfg(all(target_arch = "aarch64", target_os = "macos"))] + let addr = unsafe { + let mcontext = (*ucontext).uc_mcontext; + if mcontext.is_null() { + 0 + } else { + (*mcontext).__ss.__pc as usize + } + }; + + #[cfg(all(target_arch = "riscv64", target_os = "linux"))] + let addr = unsafe { (*ucontext).uc_mcontext.__gregs[libc::REG_PC] as usize }; + + #[cfg(all(target_arch = "loongarch64", target_os = "linux"))] + let addr = unsafe { (*ucontext).uc_mcontext.__pc as usize }; + + if profiler.is_blocklisted(addr) { + return; + } + } + + let mut bt: SmallVec<[::Frame; MAX_DEPTH]> = + SmallVec::with_capacity(MAX_DEPTH); + let mut index = 0; + + let sample_timestamp: SystemTime = SystemTime::now(); + TraceImpl::trace(ucontext, |frame| { + if index < MAX_DEPTH { + bt.push(frame.clone()); + index += 1; + true + } else { + false + } + }); + + let current_thread = unsafe { libc::pthread_self() }; + let mut name = [0; MAX_THREAD_NAME]; + let name_ptr = &mut name as *mut [libc::c_char] as *mut libc::c_char; + + write_thread_name(current_thread, &mut name); + + let name = unsafe { std::ffi::CStr::from_ptr(name_ptr) }; + profiler.sample(bt, name.to_bytes(), current_thread as u64, sample_timestamp); + } + } +} + +impl Profiler { + fn new() -> Result { + Ok(Profiler { + data: Collector::new()?, + sample_counter: 0, + running: false, + + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))] + blocklist_segments: Vec::new(), + }) + } + + #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "loongarch64" + ))] + fn is_blocklisted(&self, addr: usize) -> bool { + for libs in &self.blocklist_segments { + if addr > libs.0 && addr < libs.1 { + return true; + } + } + false + } +} + +impl Profiler { + pub fn start(&mut self) -> Result<()> { + log::info!("starting cpu profiler"); + if self.running { + Err(Error::Running) + } else { + self.register_signal_handler()?; + self.running = true; + + Ok(()) + } + } + + fn init(&mut self) -> Result<()> { + self.sample_counter = 0; + self.data = Collector::new()?; + self.running = false; + + Ok(()) + } + + /// Clear the sample data collector without stopping profiling. + /// Signal handler and timer remain active — only the accumulated + /// samples are discarded. + /// + /// # Errors + /// + /// Returns [`Error::NotRunning`] if the profiler is not currently active. + /// Returns an I/O error if truncating the overflow backing file fails. + /// + /// NOTE: pyroscope patch — added to support periodic report collection + /// without recreating the ProfilerGuard. See https://github.com/grafana/pyroscope-rs/issues/399 + pub fn clear(&mut self) -> Result<()> { + if self.running { + self.sample_counter = 0; + self.data.clear()?; + Ok(()) + } else { + Err(Error::NotRunning) + } + } + + pub fn stop(&mut self) -> Result<()> { + log::info!("stopping cpu profiler"); + if self.running { + self.unregister_signal_handler()?; + self.init()?; + + Ok(()) + } else { + Err(Error::NotRunning) + } + } + + fn register_signal_handler(&mut self) -> Result<()> { + let handler = signal::SigHandler::SigAction(perf_signal_handler); + // SA_RESTART will only restart a syscall when it's safe to do so, + // e.g. when it's a blocking read(2) or write(2). See man 7 signal. + let flags = signal::SaFlags::SA_SIGINFO | signal::SaFlags::SA_RESTART; + let sigaction = signal::SigAction::new(handler, flags, signal::SigSet::empty()); + _ = unsafe { signal::sigaction(signal::SIGPROF, &sigaction) }?; + Ok(()) + } + + // On macOS 26 ARM64 the kernel passes the handler address directly to + // _sigtramp even when the disposition is SIG_IGN (value 1). _sigtramp + // branches to it unconditionally, so address 0x1 is not instruction-aligned + // → SIGBUS (PC alignment fault). A real function pointer is always a valid + // aligned address. + // https://github.com/grafana/pyroscope-pprof-rs/pull/40/changes + #[cfg(target_os = "macos")] + fn unregister_signal_handler(&mut self) -> Result<()> { + extern "C" fn noop(_: c_int, _: *mut libc::siginfo_t, _: *mut libc::c_void) {} + let noop = signal::SigAction::new( + signal::SigHandler::SigAction(noop), + signal::SaFlags::SA_SIGINFO | signal::SaFlags::SA_RESTART, + signal::SigSet::empty(), + ); + unsafe { signal::sigaction(signal::SIGPROF, &noop) }?; + Ok(()) + } + + // Never restore SIG_DFL: SIGPROF's default action terminates the process, + // so a pending signal delivered between unregister and re-register would be + // fatal. SIG_IGN safely discards those signals on non-macOS platforms. + // See https://github.com/tikv/pprof-rs/issues/288 + // https://github.com/grafana/pprof-rs/pull/8 + #[cfg(not(target_os = "macos"))] + fn unregister_signal_handler(&mut self) -> Result<()> { + let ignore = signal::SigAction::new( + signal::SigHandler::SigIgn, + signal::SaFlags::empty(), + signal::SigSet::empty(), + ); + unsafe { signal::sigaction(signal::SIGPROF, &ignore) }?; + Ok(()) + } + + // This function has to be AS-safe + pub fn sample( + &mut self, + backtrace: SmallVec<[::Frame; MAX_DEPTH]>, + thread_name: &[u8], + thread_id: u64, + sample_timestamp: SystemTime, + ) { + let frames = UnresolvedFrames::new(backtrace, thread_name, thread_id, sample_timestamp); + self.sample_counter += 1; + + if let Ok(()) = self.data.add(frames, 1) {} + } +} + +#[cfg(test)] +pub mod tests { + use super::*; + + struct AllocDetector { + should_count_alloc: std::sync::atomic::AtomicBool, + alloc_count: std::sync::atomic::AtomicUsize, + } + + unsafe impl std::alloc::GlobalAlloc for AllocDetector { + unsafe fn alloc(&self, layout: std::alloc::Layout) -> *mut u8 { + if self + .should_count_alloc + .load(std::sync::atomic::Ordering::SeqCst) + { + self.alloc_count + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + + unsafe { std::alloc::System.alloc(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: std::alloc::Layout) { + unsafe { std::alloc::System.dealloc(ptr, layout) } + } + } + impl AllocDetector { + fn enable_count_alloc(&self) { + self.should_count_alloc + .store(true, std::sync::atomic::Ordering::SeqCst); + } + + fn disable_count_alloc(&self) { + self.should_count_alloc + .store(false, std::sync::atomic::Ordering::SeqCst); + } + + fn alloc_count(&self) -> usize { + self.alloc_count.load(std::sync::atomic::Ordering::SeqCst) + } + } + + #[global_allocator] + static ALLOC: AllocDetector = AllocDetector { + should_count_alloc: std::sync::atomic::AtomicBool::new(false), + alloc_count: std::sync::atomic::AtomicUsize::new(0), + }; + + #[test] + fn test_no_alloc_during_unwind() { + // This test cannot run parallelly because it requires the global allocator to + // record the allocation count. + + trigger_lazy(); + PROFILER.write().as_mut().unwrap().start().unwrap(); + let timer = Timer::new(999); + let start = std::time::Instant::now(); + ALLOC.enable_count_alloc(); + + // alloc something to make sure the ALLOC works fine. + let _alloc = Box::new(1usize); + // busy loop for a while to trigger some samples + while start.elapsed().as_millis() < 500 { + std::hint::black_box(()); + } + ALLOC.disable_count_alloc(); + + assert_eq!(ALLOC.alloc_count(), 1); + + drop(timer); + PROFILER.write().as_mut().unwrap().stop().unwrap(); + } +} diff --git a/third-party/pyroscope/src/backend/pprofrs/report.rs b/third-party/pyroscope/src/backend/pprofrs/report.rs new file mode 100644 index 000000000..f946ace4f --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/report.rs @@ -0,0 +1,103 @@ +// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. + +use std::collections::HashMap; +use std::fmt::{Debug, Formatter}; + +use spin::RwLock; + +use crate::backend::pprofrs::frames::Frames; +use crate::backend::pprofrs::profiler::Profiler; + +use crate::backend::pprofrs::{Error, Result}; + +/// The final presentation of a report which is actually an `HashMap` from `Frames` to isize (count). +pub struct Report { + /// Key is a backtrace captured by profiler and value is count of it. + pub data: HashMap, +} + +type FramesPostProcessor = Box; + +/// A builder of `Report` and `UnresolvedReport`. It builds report from a running `Profiler`. +pub struct ReportBuilder<'a> { + frames_post_processor: Option, + profiler: &'a RwLock>, +} + +impl<'a> ReportBuilder<'a> { + pub(crate) fn new(profiler: &'a RwLock>) -> Self { + Self { + frames_post_processor: None, + profiler, + } + } + + /// Build a `Report`. If `clear` is true, atomically clears the + /// profiler's sample data under the same write lock. + /// + /// # Errors + /// + /// Returns [`Error::Creating`] if the profiler lock is poisoned. + /// Returns an I/O error if reading the overflow backing file or clearing the collector fails. + /// + /// NOTE: pyroscope patch — added to support periodic report collection + /// without recreating the ProfilerGuard. See https://github.com/grafana/pyroscope-rs/issues/399 + pub fn build_and_clear(&self, clear: bool) -> Result { + let mut hash_map = HashMap::new(); + + match self.profiler.write().as_mut() { + Err(err) => { + log::error!("Error in creating profiler: {}", err); + Err(Error::Creating) + } + Ok(profiler) => { + profiler.data.try_iter()?.for_each(|entry| { + let count = entry.count; + if count > 0 { + let mut key = Frames::from(entry.item.clone()); + if let Some(processor) = &self.frames_post_processor { + processor(&mut key); + } + + match hash_map.get_mut(&key) { + Some(value) => { + *value += count; + } + None => { + match hash_map.insert(key, count) { + None => {} + Some(_) => { + unreachable!(); + } + }; + } + } + } + }); + + if clear { + profiler.clear()?; + } + + Ok(Report { data: hash_map }) + } + } + } +} + +/// This will generate Report in a human-readable format: +/// +/// ```shell +/// FRAME: pprof::profiler::perf_signal_handler::h7b995c4ab2e66493 -> FRAME: Unknown -> FRAME: {func1} -> +/// FRAME: {func2} -> FRAME: {func3} -> THREAD: {thread_name} {count} +/// ``` +impl Debug for Report { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + for (key, val) in self.data.iter() { + write!(f, "{key:?} {val}")?; + writeln!(f)?; + } + + Ok(()) + } +} diff --git a/third-party/pyroscope/src/backend/pprofrs/timer.rs b/third-party/pyroscope/src/backend/pprofrs/timer.rs new file mode 100644 index 000000000..27013766b --- /dev/null +++ b/third-party/pyroscope/src/backend/pprofrs/timer.rs @@ -0,0 +1,70 @@ +// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. + +use std::os::raw::c_int; +use std::ptr::null_mut; + +#[repr(C)] +#[derive(Clone)] +struct Timeval { + pub tv_sec: i64, + pub tv_usec: i64, +} + +#[repr(C)] +#[derive(Clone)] +struct Itimerval { + pub it_interval: Timeval, + pub it_value: Timeval, +} + +extern "C" { + fn setitimer(which: c_int, new_value: *mut Itimerval, old_value: *mut Itimerval) -> c_int; +} + +const ITIMER_PROF: c_int = 2; + +pub struct Timer {} + +impl Timer { + pub fn new(frequency: c_int) -> Timer { + let interval = 1e6 as i64 / i64::from(frequency); + let it_interval = Timeval { + tv_sec: interval / 1e6 as i64, + tv_usec: interval % 1e6 as i64, + }; + let it_value = it_interval.clone(); + + unsafe { + setitimer( + ITIMER_PROF, + &mut Itimerval { + it_interval, + it_value, + }, + null_mut(), + ) + }; + + Timer {} + } +} + +impl Drop for Timer { + fn drop(&mut self) { + let it_interval = Timeval { + tv_sec: 0, + tv_usec: 0, + }; + let it_value = it_interval.clone(); + unsafe { + setitimer( + ITIMER_PROF, + &mut Itimerval { + it_interval, + it_value, + }, + null_mut(), + ) + }; + } +} diff --git a/third-party/pyroscope/src/backend/ruleset.rs b/third-party/pyroscope/src/backend/ruleset.rs new file mode 100644 index 000000000..ce4f7de12 --- /dev/null +++ b/third-party/pyroscope/src/backend/ruleset.rs @@ -0,0 +1,82 @@ +use super::{StackTrace, Tag}; +use crate::error::Result; +use std::collections::HashSet; +use std::sync::{Arc, Mutex}; + +#[derive(Debug, Eq, PartialEq, Hash, Clone)] +pub struct ThreadTag { + tid: crate::utils::ThreadId, + tag: Tag, +} + +impl ThreadTag { + pub fn new(tid: crate::ThreadId, tag: Tag) -> Self { + Self { tid, tag } + } +} + +#[derive(Debug, Default, Clone)] +pub struct ThreadTagsSet { + pub rules: Arc>>, +} + +impl ThreadTagsSet { + pub fn new() -> Self { + Self { + rules: Arc::new(Mutex::new(HashSet::new())), + } + } + + pub fn add(&self, rule: ThreadTag) -> Result { + let rules = self.rules.clone(); + + let insert = rules.lock()?.insert(rule); + + Ok(insert) + } + + pub fn remove(&self, rule: ThreadTag) -> Result { + let rules = self.rules.clone(); + + let remove = rules.lock()?.remove(&rule); + + Ok(remove) + } + + #[cfg(test)] + pub fn thread_tags(&self, tid: crate::ThreadId) -> Vec { + let s = StackTrace { + pid: None, + thread_id: Some(tid.clone()), + thread_name: None, + frames: vec![], + metadata: Default::default(), + }; + let tags: Vec = s.add_tag_rules(self).metadata.tags.into_iter().collect(); + tags + } +} + +impl StackTrace { + pub fn add_tag_rules(self, other: &ThreadTagsSet) -> Self { + let mut metadata = self.metadata; + + if let Ok(rules) = other.rules.lock() { + rules.iter().for_each(|rule| { + if let Some(stack_thread_id) = &self.thread_id { + if rule.tid == *stack_thread_id { + metadata.add_tag(rule.tag.clone()); + } + } + }) + } + + Self { + pid: self.pid, + thread_id: self.thread_id, + thread_name: self.thread_name, + frames: self.frames, + metadata, + } + } +} diff --git a/third-party/pyroscope/src/backend/tests.rs b/third-party/pyroscope/src/backend/tests.rs new file mode 100644 index 000000000..e58834028 --- /dev/null +++ b/third-party/pyroscope/src/backend/tests.rs @@ -0,0 +1,378 @@ +#[cfg(test)] +use crate::backend::{ + BackendConfig, Report, StackBuffer, StackFrame, StackTrace, Tag, ThreadTag, ThreadTagsSet, +}; +#[cfg(test)] +use std::collections::{HashMap, HashSet}; + +#[test] +fn test_stack_frame_display() { + let frame = StackFrame::new( + Some("module".to_string()), + Some("name".to_string()), + Some("filename".to_string()), + Some("relative_path".to_string()), + Some("absolute_path".to_string()), + Some(1), + ); + + assert_eq!(format!("{frame}"), "filename:1 - name"); +} + +#[test] +fn test_stack_trace_display() { + let frames = vec![ + StackFrame::new( + Some("module".to_string()), + Some("name".to_string()), + Some("filename".to_string()), + Some("relative_path".to_string()), + Some("absolute_path".to_string()), + Some(1), + ), + StackFrame::new( + Some("module".to_string()), + Some("name".to_string()), + Some("filename".to_string()), + Some("relative_path".to_string()), + Some("absolute_path".to_string()), + Some(2), + ), + ]; + + let stack_trace = StackTrace::new(&BackendConfig::default(), None, None, None, frames); + + assert_eq!( + format!("{stack_trace}"), + "filename:2 - name;filename:1 - name" + ); +} + +#[test] +fn test_report_record() { + let mut report = Report::new(HashMap::new()); + + let stack_trace = StackTrace::new(&BackendConfig::default(), None, None, None, vec![]); + + report.record(stack_trace); + assert_eq!(report.data.len(), 1); +} + +#[test] +fn test_tag_new() { + let tag = Tag::new("key".to_string(), "value".to_string()); + + assert_eq!(tag.key, "key"); + assert_eq!(tag.value, "value"); +} + +#[test] +fn test_rule_new() { + let tid = crate::ThreadId::pthread_self(); + let rule = ThreadTag::new( + tid.clone(), + Tag::new("key".to_string(), "value".to_string()), + ); + + assert_eq!( + rule, + ThreadTag::new(tid, Tag::new("key".to_string(), "value".to_string())) + ); +} + +#[cfg(test)] +fn test_thread_id(v: u64) -> crate::utils::ThreadId { + (v as libc::pthread_t).into() +} + +#[test] +fn test_ruleset_new() { + let ruleset = ThreadTagsSet::new(); + + assert_eq!(ruleset.rules.lock().unwrap().len(), 0); +} + +#[test] +fn test_ruleset_add_rule() { + let tid = crate::ThreadId::pthread_self(); + let ruleset = ThreadTagsSet::new(); + + let rule = ThreadTag::new(tid, Tag::new("key".to_string(), "value".to_string())); + + ruleset.add(rule).unwrap(); + + assert_eq!(ruleset.rules.lock().unwrap().len(), 1); +} + +#[test] +fn test_ruleset_remove_rule() { + let tid = crate::ThreadId::pthread_self(); + let ruleset = ThreadTagsSet::new(); + + let add_rule = ThreadTag::new( + tid.clone(), + Tag::new("key".to_string(), "value".to_string()), + ); + + ruleset.add(add_rule).unwrap(); + + assert_eq!(ruleset.rules.lock().unwrap().len(), 1); + + let remove_rule = ThreadTag::new(tid, Tag::new("key".to_string(), "value".to_string())); + + ruleset.remove(remove_rule).unwrap(); + + assert_eq!(ruleset.rules.lock().unwrap().len(), 0); +} + +#[test] +fn test_ruleset() { + let ruleset = ThreadTagsSet::new(); + + ruleset + .add(ThreadTag::new( + test_thread_id(1), + Tag::new("key1".to_string(), "value".to_string()), + )) + .unwrap(); + + ruleset + .add(ThreadTag::new( + test_thread_id(2), + Tag::new("key1".to_string(), "value".to_string()), + )) + .unwrap(); + + ruleset + .add(ThreadTag::new( + test_thread_id(3), + Tag::new("key1".to_string(), "value".to_string()), + )) + .unwrap(); + + // Remove ThreadTag number 2 + ruleset + .remove(ThreadTag::new( + test_thread_id(2), + Tag::new("key1".to_string(), "value".to_string()), + )) + .unwrap(); + + // Verify ThreadTag number 2 is removed from the ruleset Vector + assert_eq!( + ruleset.rules.lock().unwrap().clone(), + HashSet::from([ + ThreadTag::new( + test_thread_id(1), + Tag::new("key1".to_string(), "value".to_string(),) + ), + ThreadTag::new( + test_thread_id(3), + Tag::new("key1".to_string(), "value".to_string(),) + ) + ]) + ); +} + +#[test] +fn test_ruleset_duplicates() { + let ruleset = ThreadTagsSet::new(); + + let tid = crate::ThreadId::pthread_self(); + ruleset + .add(ThreadTag::new( + tid.clone(), + Tag::new("key1".to_string(), "value".to_string()), + )) + .unwrap(); + + ruleset + .add(ThreadTag::new( + tid.clone(), + Tag::new("key1".to_string(), "value".to_string()), + )) + .unwrap(); + + assert_eq!( + ruleset.thread_tags(tid), + vec![Tag::new("key1".to_string(), "value".to_string())] + ); +} + +#[test] +fn test_ruleset_remove_nonexistent() { + let ruleset = ThreadTagsSet::new(); + + let tid = crate::ThreadId::pthread_self(); + ruleset + .add(ThreadTag::new( + tid.clone(), + Tag::new("key1".to_string(), "value".to_string()), + )) + .unwrap(); + + ruleset + .remove(ThreadTag::new( + tid.clone(), + Tag::new("key2".to_string(), "value".to_string()), + )) + .unwrap(); + + assert_eq!( + ruleset.thread_tags(tid), + vec![Tag::new("key1".to_string(), "value".to_string())] + ); +} + +#[test] +fn test_stacktrace_add() { + let ruleset = ThreadTagsSet::new(); + + ruleset + .add(ThreadTag::new( + test_thread_id(55), + Tag::new("keyA".to_string(), "valueA".to_string()), + )) + .unwrap(); + + ruleset + .add(ThreadTag::new( + test_thread_id(100), + Tag::new("keyB".to_string(), "valueB".to_string()), + )) + .unwrap(); + + let backend_config = BackendConfig { + report_pid: true, + report_thread_id: true, + report_thread_name: true, + }; + + let stacktrace = StackTrace::new( + &backend_config, + Some(1), + Some(test_thread_id(55)), + Some("thread_name".to_string()), + vec![crate::backend::StackFrame::new( + Some("file1".to_string()), + Some("function1".to_string()), + Some("file1".to_string()), + Some("file1".to_string()), + Some("file1".to_string()), + Some(1), + )], + ); + + // assert initial metadata of the stacktrace + let mut initial_metadata = crate::backend::Metadata::default(); + initial_metadata.add_tag(Tag::new("pid".to_string(), "1".to_string())); + initial_metadata.add_tag(Tag::new("thread_id".to_string(), "55".to_string())); + initial_metadata.add_tag(Tag::new( + "thread_name".to_string(), + "thread_name".to_string(), + )); + + assert_eq!(stacktrace.metadata, initial_metadata); + + // Add the Stacktrace to the Ruleset + let applied_stacktrace = stacktrace.add_tag_rules(&ruleset); + + initial_metadata.add_tag(Tag::new("keyA".to_string(), "valueA".to_string())); + + // assert that the metadata of the stacktrace is updated + assert_eq!(applied_stacktrace.metadata, initial_metadata); + + // Re-apply the Ruleset + let re_applied_stacktrace = applied_stacktrace.add_tag_rules(&ruleset); + + // assert that the metadata of the stacktrace is the same + assert_eq!(re_applied_stacktrace.metadata, initial_metadata); +} + +#[test] +fn test_stackbuffer_record() { + let mut buffer = StackBuffer::new(HashMap::new()); + let stack_trace = StackTrace::new( + &BackendConfig::default(), + None, + None, + None, + vec![StackFrame::new( + None, + Some("test_record".to_string()), + None, + None, + None, + None, + )], + ); + // First record + buffer.record(stack_trace.clone()).unwrap(); + assert_eq!(buffer.data.len(), 1); + assert_eq!(buffer.data[&stack_trace], 1); + + // Second record + buffer.record(stack_trace.clone()).unwrap(); + assert_eq!(buffer.data.len(), 1); + assert_eq!(buffer.data[&stack_trace], 2); +} + +#[test] +fn test_stackbuffer_record_with_count() { + let mut buffer = StackBuffer::new(HashMap::new()); + let stack_trace = StackTrace::new( + &BackendConfig::default(), + None, + None, + None, + vec![StackFrame::new( + None, + Some("test_record".to_string()), + None, + None, + None, + None, + )], + ); + // First record + buffer.record_with_count(stack_trace.clone(), 1).unwrap(); + assert_eq!(buffer.data.len(), 1); + assert_eq!(buffer.data[&stack_trace], 1); + + // Second record + buffer.record_with_count(stack_trace.clone(), 2).unwrap(); + assert_eq!(buffer.data.len(), 1); + assert_eq!(buffer.data[&stack_trace], 3); +} + +#[test] +fn test_stackbuffer_clear() { + let mut buffer = StackBuffer::new(HashMap::new()); + let stack_trace = StackTrace::new( + &BackendConfig::default(), + None, + None, + None, + vec![StackFrame::new( + None, + Some("test_record".to_string()), + None, + None, + None, + None, + )], + ); + // First record + buffer.record(stack_trace.clone()).unwrap(); + assert_eq!(buffer.data.len(), 1); + assert_eq!(buffer.data[&stack_trace], 1); + + // Second record + buffer.record(stack_trace.clone()).unwrap(); + assert_eq!(buffer.data.len(), 1); + assert_eq!(buffer.data[&stack_trace], 2); + + // Clear + buffer.clear(); + assert_eq!(buffer.data.len(), 0); +} diff --git a/third-party/pyroscope/src/backend/types.rs b/third-party/pyroscope/src/backend/types.rs new file mode 100644 index 000000000..5f5bf0589 --- /dev/null +++ b/third-party/pyroscope/src/backend/types.rs @@ -0,0 +1,303 @@ +use super::BackendConfig; +use crate::error::Result; +use std::{ + collections::{hash_map::DefaultHasher, BTreeSet, HashMap}, + hash::{Hash, Hasher}, +}; + +/// Pyroscope Tag +#[derive(Debug, PartialOrd, Ord, Eq, PartialEq, Hash, Clone)] +pub struct Tag { + /// Tag key + pub key: String, + /// Tag value + pub value: String, +} + +impl Tag { + /// Create a new Tag + pub fn new(key: String, value: String) -> Self { + Self { key, value } + } +} + +impl std::fmt::Display for Tag { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}={}", self.key, self.value) + } +} + +/// Stack buffer +#[derive(Debug, Default, Clone)] +pub struct StackBuffer { + /// Buffer data bucket + pub data: HashMap, +} + +impl StackBuffer { + /// Create a new StackBuffer with the given data + pub fn new(data: HashMap) -> Self { + Self { data } + } + + /// Record a new stack trace + pub fn record(&mut self, stack_trace: StackTrace) -> Result<()> { + *self.data.entry(stack_trace).or_insert(0) += 1; + + Ok(()) + } + + /// Record a new stack trace with count + pub fn record_with_count(&mut self, stack_trace: StackTrace, count: usize) -> Result<()> { + *self.data.entry(stack_trace).or_insert(0) += count; + + Ok(()) + } + + /// Clear the buffer + pub fn clear(&mut self) { + self.data.clear(); + } +} + +impl From for Vec { + fn from(stack_buffer: StackBuffer) -> Self { + stack_buffer + .data + .into_iter() + .fold( + HashMap::new(), + |acc: HashMap, (stacktrace, count): (StackTrace, usize)| { + let mut acc = acc; + if let Some(report) = acc.get_mut(&stacktrace.metadata.get_id()) { + report.record_with_count(stacktrace, count); + } else { + let report = Report::new(HashMap::new()); + let report_id = stacktrace.metadata.get_id(); + let mut report = report.metadata(stacktrace.metadata.clone()); + report.record_with_count(stacktrace, count); + acc.insert(report_id, report); + } + acc + }, + ) + .into_values() + .collect() + } +} + +/// Metadata +/// Metadata attached to a StackTrace or a Report. For now, this is just tags. +#[derive(Debug, Default, Clone, Hash, PartialEq, Eq)] +pub struct Metadata { + /// Tags + pub tags: BTreeSet, +} + +impl Metadata { + /// Add a tag to the metadata + pub fn add_tag(&mut self, tag: Tag) { + self.tags.insert(tag); + } + + /// Get the id of the metadata. This uses the hash of the Metadata type. + pub fn get_id(&self) -> usize { + let mut hasher = DefaultHasher::new(); + self.hash(&mut hasher); + hasher.finish() as usize + } +} + +/// The payload of a report batch: either structured stack-trace reports +/// (which will be encoded into pprof by the session layer) or pre-encoded +/// pprof bytes produced directly by a backend (e.g. jemalloc). +pub enum ReportData { + /// Structured stack-trace reports that must be pprof-encoded before sending. + Reports(Vec), + /// Pre-encoded pprof bytes (may already be gzipped). Used by backends + /// like jemalloc that produce a complete pprof profile directly. + RawPprof(Vec), +} + +/// A batch of reports with a shared profile type. +pub struct ReportBatch { + /// Profile type name (e.g. "process_cpu", "memory") + pub profile_type: String, + /// Report data in this batch + pub data: ReportData, +} + +/// Report +#[derive(Debug, Default, Clone)] +pub struct Report { + /// Report StackTraces + pub data: HashMap, + /// Metadata + pub metadata: Metadata, +} + +/// Custom implementation of the Hash trait for Report. +/// Only the metadata is hashed. +impl Hash for Report { + fn hash(&self, state: &mut H) { + self.metadata.hash(state); + } +} + +impl Report { + /// Create a new Report. + pub fn new(data: HashMap) -> Self { + Self { + data, + metadata: Metadata::default(), + } + } + + /// Return an iterator over the StackTraces of the Report. + pub fn iter(&self) -> impl Iterator { + self.data.iter() + } + + /// Set the metadata of the report. + pub fn metadata(self, metadata: Metadata) -> Self { + Self { + data: self.data, + metadata, + } + } + + pub fn record(&mut self, stack_trace: StackTrace) { + *self.data.entry(stack_trace).or_insert(0) += 1; + } + + pub fn record_with_count(&mut self, stack_trace: StackTrace, count: usize) { + *self.data.entry(stack_trace).or_insert(0) += count; + } +} + +/// StackTrace +/// A representation of a stack trace. +#[derive(Debug, Default, PartialEq, Eq, Hash, Clone)] +pub struct StackTrace { + /// Process ID + pub pid: Option, + /// Thread ID + pub thread_id: Option, + /// Thread Name + pub thread_name: Option, + /// Stack Trace + pub frames: Vec, + /// Metadata + pub metadata: Metadata, +} + +impl std::fmt::Display for StackTrace { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + &self + .frames + .iter() + .rev() + .map(|frame| format!("{frame}")) + .collect::>() + .join(";") + ) + } +} + +impl StackTrace { + /// Create a new StackTrace + pub fn new( + config: &BackendConfig, + pid: Option, + thread_id: Option, + thread_name: Option, + frames: Vec, + ) -> Self { + let mut metadata = Metadata::default(); + + if config.report_pid { + if let Some(pid) = pid { + metadata.add_tag(Tag::new("pid".to_owned(), pid.to_string())); + } + } + + if config.report_thread_id { + if let Some(thread_id) = &thread_id { + metadata.add_tag(Tag::new("thread_id".to_owned(), thread_id.to_string())); + } + } + + if config.report_thread_name { + if let Some(thread_name) = thread_name.clone() { + metadata.add_tag(Tag::new("thread_name".to_owned(), thread_name)); + } + } + + Self { + pid, + thread_id, + thread_name, + frames, + metadata, + } + } + + /// Return an iterator over the frames of the stacktrace. + pub fn iter(&self) -> impl Iterator { + self.frames.iter() + } +} + +/// StackFrame +/// A representation of a stack frame. +#[derive(Debug, Default, PartialEq, Eq, Hash, Clone)] +pub struct StackFrame { + /// Module name + pub module: Option, + /// Function name + pub name: Option, + /// File name + pub filename: Option, + /// File relative path + pub relative_path: Option, + /// File absolute path + pub absolute_path: Option, + /// Line number + pub line: Option, +} + +impl StackFrame { + /// Create a new StackFrame. + pub fn new( + module: Option, + name: Option, + filename: Option, + relative_path: Option, + absolute_path: Option, + line: Option, + ) -> Self { + Self { + module, + name, + filename, + relative_path, + absolute_path, + line, + } + } +} + +impl std::fmt::Display for StackFrame { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}:{} - {}", + self.filename.as_ref().unwrap_or(&"".to_string()), + self.line.unwrap_or(0), + self.name.as_ref().unwrap_or(&"".to_string()) + ) + } +} diff --git a/third-party/pyroscope/src/encode/gen/google.rs b/third-party/pyroscope/src/encode/gen/google.rs new file mode 100644 index 000000000..6858b9683 --- /dev/null +++ b/third-party/pyroscope/src/encode/gen/google.rs @@ -0,0 +1,233 @@ +// @generated +// This file is @generated by prost-build. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Profile { + /// A description of the samples associated with each Sample.value. + /// For a cpu profile this might be: + /// \[["cpu","nanoseconds"]\] or \[["wall","seconds"]\] or \[["syscall","count"]\] + /// For a heap profile, this might be: + /// \[["allocations","count"\], \["space","bytes"]\], + /// If one of the values represents the number of events represented + /// by the sample, by convention it should be at index 0 and use + /// sample_type.unit == "count". + #[prost(message, repeated, tag = "1")] + pub sample_type: ::prost::alloc::vec::Vec, + /// The set of samples recorded in this profile. + #[prost(message, repeated, tag = "2")] + pub sample: ::prost::alloc::vec::Vec, + /// Mapping from address ranges to the image/binary/library mapped + /// into that address range. mapping\[0\] will be the main binary. + #[prost(message, repeated, tag = "3")] + pub mapping: ::prost::alloc::vec::Vec, + /// Useful program location + #[prost(message, repeated, tag = "4")] + pub location: ::prost::alloc::vec::Vec, + /// Functions referenced by locations + #[prost(message, repeated, tag = "5")] + pub function: ::prost::alloc::vec::Vec, + /// A common table for strings referenced by various messages. + /// string_table\[0\] must always be "". + #[prost(string, repeated, tag = "6")] + pub string_table: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// frames with Function.function_name fully matching the following + /// regexp will be dropped from the samples, along with their successors. + /// + /// Index into string table. + #[prost(int64, tag = "7")] + pub drop_frames: i64, + /// frames with Function.function_name fully matching the following + /// regexp will be kept, even if it matches drop_frames. + /// + /// Index into string table. + #[prost(int64, tag = "8")] + pub keep_frames: i64, + // The following fields are informational, do not affect + // interpretation of results. + /// Time of collection (UTC) represented as nanoseconds past the epoch. + #[prost(int64, tag = "9")] + pub time_nanos: i64, + /// Duration of the profile, if a duration makes sense. + #[prost(int64, tag = "10")] + pub duration_nanos: i64, + /// The kind of events between sampled occurrences. + /// e.g \[ "cpu","cycles" \] or \[ "heap","bytes" \] + #[prost(message, optional, tag = "11")] + pub period_type: ::core::option::Option, + /// The number of events between sampled occurrences. + #[prost(int64, tag = "12")] + pub period: i64, + /// Freeform text associated to the profile. + /// + /// Indices into string table. + #[prost(int64, repeated, tag = "13")] + pub comment: ::prost::alloc::vec::Vec, + /// Index into the string table of the type of the preferred sample + /// value. If unset, clients should default to the last sample value. + #[prost(int64, tag = "14")] + pub default_sample_type: i64, +} +/// ValueType describes the semantics and measurement units of a value. +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ValueType { + /// Index into string table. + #[prost(int64, tag = "1")] + pub r#type: i64, + /// Index into string table. + #[prost(int64, tag = "2")] + pub unit: i64, +} +/// Each Sample records values encountered in some program +/// context. The program context is typically a stack trace, perhaps +/// augmented with auxiliary information like the thread-id, some +/// indicator of a higher level request being handled etc. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Sample { + /// The ids recorded here correspond to a Profile.location.id. + /// The leaf is at location_id\[0\]. + #[prost(uint64, repeated, tag = "1")] + pub location_id: ::prost::alloc::vec::Vec, + /// The type and unit of each value is defined by the corresponding + /// entry in Profile.sample_type. All samples must have the same + /// number of values, the same as the length of Profile.sample_type. + /// When aggregating multiple samples into a single sample, the + /// result has a list of values that is the element-wise sum of the + /// lists of the originals. + #[prost(int64, repeated, tag = "2")] + pub value: ::prost::alloc::vec::Vec, + /// label includes additional context for this sample. It can include + /// things like a thread id, allocation size, etc + #[prost(message, repeated, tag = "3")] + pub label: ::prost::alloc::vec::Vec