diff --git a/.github/workflows/cache-warm.yml b/.github/workflows/cache-warm.yml index 9c02ca4e5..a9902fc4d 100644 --- a/.github/workflows/cache-warm.yml +++ b/.github/workflows/cache-warm.yml @@ -182,7 +182,12 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" - # Readers: test-and-lint-rio-v2, build-rustfs-debug-binary-rio-v2. + # Readers: test-and-lint-rio-v2 (per-PR), build-rustfs-debug-binary-rio-v2 + # (weekly schedule / manual dispatch only — dormant rio-v2 variant, see + # rustfs/backlog#1835 and docs/architecture/minio-file-format-compat.md). + # The second build below stays despite the reduced cadence: it warms the + # rio-v2,e2e-test-hooks feature resolution the scheduled build restores, + # which keeps that lane inside its 30-minute timeout. warm-ci-feat-rio: name: Warm ci-feat-rio runs-on: sm-standard-4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 981eda359..1988cb54d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -533,7 +533,12 @@ jobs: build-rustfs-debug-binary-rio-v2: name: Build RustFS Debug Binary (rio-v2) - if: github.event_name != 'pull_request' || github.event.action != 'closed' + # Dormant rio-v2 variant (rustfs/backlog#1835): the feature ships in no + # default build, so this full-suite lane runs only on the weekly schedule + # and manual dispatch. Per-PR cfg-seam coverage stays with + # test-and-lint-rio-v2. Lifecycle and the promote-or-delete condition: + # docs/architecture/minio-file-format-compat.md ("rio-v2 variant lifecycle"). + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' needs: [ quick-checks ] runs-on: sm-standard-4 timeout-minutes: 30 @@ -824,6 +829,9 @@ jobs: e2e-tests-rio-v2: name: End-to-End Tests (rio-v2) + # Inherits the schedule/dispatch-only gate through needs: on every other + # event build-rustfs-debug-binary-rio-v2 is skipped, so this job skips + # with it (see the dormant-variant comment on that job). needs: [ build-rustfs-debug-binary-rio-v2 ] runs-on: sm-standard-2 timeout-minutes: 30 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f19668bc8..809d0a3f5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -101,7 +101,10 @@ refactors. The `rustfs` binary crate composes these libraries into the running server. `ecstore` remains the storage engine at the architectural center; its internal -module split is tracked under `docs/architecture/`. +module split is tracked under `docs/architecture/`. `rio-v2` is the +feature-gated MinIO on-disk format compatibility I/O layer; it ships in no +default build (lifecycle: +[docs/architecture/minio-file-format-compat.md](docs/architecture/minio-file-format-compat.md)). ## Architecture Invariants diff --git a/Cargo.toml b/Cargo.toml index 8aa1cd67b..3993e54aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,7 +41,7 @@ members = [ "crates/protocols", # Protocol implementations (FTPS, SFTP, etc.) "crates/protos", # Protocol buffer definitions "crates/rio", # Rust I/O utilities and abstractions - "crates/rio-v2", # Next-generation Rust I/O compatibility layer + "crates/rio-v2", # MinIO on-disk format compatibility I/O layer (feature-gated, ships in no default build) "crates/replication", # Replication contracts and wire formats "crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling "crates/s3-types", # S3 event type definitions diff --git a/crates/config/src/constants/object.rs b/crates/config/src/constants/object.rs index 279789719..081308754 100644 --- a/crates/config/src/constants/object.rs +++ b/crates/config/src/constants/object.rs @@ -137,6 +137,22 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false; const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE); const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED); +/// Request the object-transaction fencing contract used by storage-owned +/// cleanup receipts and lock-window optimizations. +/// +/// This is fail-closed: enabling the writer without a live fleet proof rejects +/// the commit rather than silently using a legacy-safe path. +pub const ENV_OBJECT_TRANSACTION_FENCING_WRITE: &str = "RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE"; +pub const DEFAULT_OBJECT_TRANSACTION_FENCING_WRITE: bool = false; + +/// Operator-attested confirmation that every serving node understands the +/// object transaction fencing contract. +pub const ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED: &str = "RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED"; +pub const DEFAULT_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED: bool = false; + +const _: () = assert!(!DEFAULT_OBJECT_TRANSACTION_FENCING_WRITE); +const _: () = assert!(!DEFAULT_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED); + /// Request preserving legacy per-part checksum metadata during data movement. /// /// This remains ineffective until @@ -673,4 +689,13 @@ mod remote_version_state_tests { "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED" ); } + + #[test] + fn object_transaction_fencing_gate_uses_stable_environment_names() { + assert_eq!(super::ENV_OBJECT_TRANSACTION_FENCING_WRITE, "RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE"); + assert_eq!( + super::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, + "RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED" + ); + } } diff --git a/crates/ecstore/src/cluster/mod.rs b/crates/ecstore/src/cluster/mod.rs index 0cd8f36b3..28e947f60 100644 --- a/crates/ecstore/src/cluster/mod.rs +++ b/crates/ecstore/src/cluster/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: cluster/RPC migration leaves transport capabilities staged for upcoming owners. -#![allow(dead_code)] mod control_plane; pub(crate) mod rpc; diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index 0e1395fff..66c500b2c 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -256,6 +256,7 @@ impl ReplayScopeChannel { } } +#[allow(dead_code, reason = "replay-state probe asserted by this file's tests (backlog#1823)")] fn peer_replay_state(audience: &str) -> PeerReplayState { PEER_REPLAY_STATES .lock() diff --git a/crates/ecstore/src/cluster/rpc/internode_data_transport.rs b/crates/ecstore/src/cluster/rpc/internode_data_transport.rs index a9be2f95e..9c33d191a 100644 --- a/crates/ecstore/src/cluster/rpc/internode_data_transport.rs +++ b/crates/ecstore/src/cluster/rpc/internode_data_transport.rs @@ -43,6 +43,10 @@ use tokio::io::{AsyncReadExt, AsyncWrite}; use tokio::sync::OnceCell; use uuid::Uuid; +#[allow( + dead_code, + reason = "live in the cfg(not(test)) half of build_internode_data_transport_from_env (backlog#1823)" +)] static INTERNODE_DATA_TRANSPORT: OnceLock, String>> = OnceLock::new(); const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream"; @@ -134,6 +138,10 @@ fn put_file_capability_status_is_legacy(status: u16) -> bool { } #[derive(Debug, Clone, Copy, Eq, PartialEq)] +#[allow( + dead_code, + reason = "capability-negotiation seam; constructed only by transport test doubles (backlog#1823)" +)] pub struct InternodeDataTransportCapabilities { /// Backend can open a streaming remote disk reader. pub streaming_read: bool, @@ -150,6 +158,10 @@ pub struct InternodeDataTransportCapabilities { } impl InternodeDataTransportCapabilities { + #[allow( + dead_code, + reason = "capability-negotiation seam; used by transport test doubles (backlog#1823)" + )] pub const fn tcp_http() -> Self { Self { streaming_read: true, @@ -234,7 +246,12 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug { async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result { Err(Error::MethodNotAllowed) } + // Interface facet nobody calls yet: every transport implements both, but no + // caller negotiates on them. Kept for the internode transport split + // (backlog#1350); deleting them would delete the seam and six impls. + #[allow(dead_code, reason = "unused capability-negotiation facet (backlog#1823)")] fn name(&self) -> &'static str; + #[allow(dead_code, reason = "unused capability-negotiation facet (backlog#1823)")] fn capabilities(&self) -> InternodeDataTransportCapabilities; } @@ -670,6 +687,10 @@ fn build_internode_data_transport_result( } } +#[allow( + dead_code, + reason = "live in the cfg(test) half of build_internode_data_transport_from_env, which bypasses the process static (backlog#1823)" +)] pub fn build_internode_data_transport(configured_transport: Option<&str>) -> Result> { build_internode_data_transport_result(configured_transport).map_err(Error::other) } diff --git a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs index dd1822548..de7f0e595 100644 --- a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs @@ -854,7 +854,6 @@ impl PeerS3Client for LocalPeerS3Client { #[derive(Debug)] pub struct RemotePeerS3Client { - pub node: Option, pub pools: Option>, addr: String, /// Health tracker for connection monitoring @@ -886,7 +885,6 @@ impl RemotePeerS3Client { pub fn new(node: Option, pools: Option>) -> Self { let addr = node.as_ref().map(|v| v.url.to_string()).unwrap_or_default(); let client = Self { - node, pools, addr, health: Arc::new(DiskHealthTracker::new()), @@ -905,10 +903,6 @@ impl RemotePeerS3Client { .map_err(|err| Error::other(format!("can not get client, err: {err}"))) } - pub fn get_addr(&self) -> String { - self.addr.clone() - } - /// Start health monitoring for the remote peer fn start_health_monitoring(&self) { let health = Arc::clone(&self.health); @@ -1208,6 +1202,10 @@ impl PeerS3Client for RemotePeerS3Client { } } +#[allow( + dead_code, + reason = "local bucket-heal path reached only by this file's tests (backlog#1823)" +)] pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result { let disks = clone_drives().await; heal_bucket_local_on_disks(bucket, opts, disks).await @@ -1404,6 +1402,10 @@ pub(crate) async fn heal_bucket_local_on_disks( } } +#[allow( + dead_code, + reason = "reached only through heal_bucket_local, which only tests call (backlog#1823)" +)] async fn clone_drives() -> Vec> { runtime_sources::local_disk_entries().await } @@ -1585,15 +1587,7 @@ mod tests { } fn test_remote_peer(addr: &str) -> RemotePeerS3Client { - let node = Node { - url: url::Url::parse(addr).expect("test peer URL should parse"), - pools: vec![0], - is_local: false, - grid_host: addr.to_string(), - }; - RemotePeerS3Client { - node: Some(node), pools: Some(vec![0]), addr: addr.to_string(), health: Arc::new(DiskHealthTracker::new()), diff --git a/crates/ecstore/src/cluster/rpc/remote_locker.rs b/crates/ecstore/src/cluster/rpc/remote_locker.rs index c5994ac14..d17033e6b 100644 --- a/crates/ecstore/src/cluster/rpc/remote_locker.rs +++ b/crates/ecstore/src/cluster/rpc/remote_locker.rs @@ -48,10 +48,6 @@ impl RemoteClient { Self { addr: endpoint } } - pub fn from_url(url: url::Url) -> Self { - Self { addr: url.to_string() } - } - fn build_ping_request() -> PingRequest { let mut fbb = flatbuffers::FlatBufferBuilder::new(); let payload = fbb.create_vector(b"health-check"); diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 259c34f65..3b2729b8b 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -46,7 +46,6 @@ use rustfs_config::{ SCANNER_SUB_SYS, }; use rustfs_filemeta::FileInfo; -use rustfs_utils::path::SLASH_SEPARATOR; use serde_json::{Map, Value}; use std::collections::{HashMap, HashSet}; use std::sync::LazyLock; @@ -200,8 +199,6 @@ pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class"; pub const COMMA_SEPARATED_LISTS: &[&str] = &[rustfs_config::oidc::OIDC_SCOPES, rustfs_config::oidc::OIDC_OTHER_AUDIENCES]; -static CONFIG_BUCKET: LazyLock = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{CONFIG_PREFIX}")); - type ServerConfigDecryptFn = crate::bucket::migration::LegacyBlobDecryptFn; static SERVER_CONFIG_DECRYPT_FN: LazyLock>> = LazyLock::new(|| RwLock::new(None)); diff --git a/crates/ecstore/src/config/mod.rs b/crates/ecstore/src/config/mod.rs index f1fe35811..5e6e3a3e3 100644 --- a/crates/ecstore/src/config/mod.rs +++ b/crates/ecstore/src/config/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: configuration migration keeps legacy subsystem definitions available behind this module. -#![allow(dead_code)] mod audit; pub mod com; diff --git a/crates/ecstore/src/config/storageclass.rs b/crates/ecstore/src/config/storageclass.rs index 11af87adf..043bf508e 100644 --- a/crates/ecstore/src/config/storageclass.rs +++ b/crates/ecstore/src/config/storageclass.rs @@ -101,6 +101,7 @@ const DEFAULT_RRS_STORAGE_CLASS: &str = "EC:1"; const ZERO_SET_DRIVE_COUNT_ERROR: &str = "set drive count must be greater than zero"; pub static DEFAULT_INLINE_BLOCK: usize = 128 * 1024; +const DEFAULT_INLINE_OBJECT_BUDGET: usize = 2 * DEFAULT_INLINE_BLOCK; pub static DEFAULT_KVS: LazyLock = LazyLock::new(|| { let kvs = vec![ @@ -150,6 +151,8 @@ pub struct Config { optimize: Option, inline_block: usize, initialized: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + inline_block_explicit: bool, #[serde(skip)] standard_parities: Vec, #[serde(skip)] @@ -186,6 +189,10 @@ impl Config { /// A topology-bound lookup fails closed for unknown drive counts and for /// deserialized legacy configurations that have no pool topology. Legacy /// callers retain scalar compatibility through [`Self::get_parity_for_sc`]. + #[allow( + dead_code, + reason = "per-set parity resolution asserted by this file's tests (backlog#1823)" + )] pub(crate) fn parity_for_sc(&self, sc: &str, drives_per_set: usize) -> Option { if !self.initialized { return None; @@ -233,17 +240,19 @@ impl Config { .map(|(pool_index, pool)| (pool_index, pool.drives_per_set)) } - pub fn should_inline(&self, shard_size: i64, versioned: bool) -> bool { - if shard_size < 0 { + pub fn should_inline(&self, shard_size: i64, data_shards: usize, versioned: bool) -> bool { + if shard_size < 0 || data_shards == 0 { return false; } let shard_size = shard_size as usize; - - let mut inline_block = DEFAULT_INLINE_BLOCK; - if self.initialized { - inline_block = self.inline_block; - } + // Keep the historical two-data-shard object budget while preventing + // wider EC layouts from multiplying the maximum inline object size. + let inline_block = if self.initialized && self.inline_block_explicit { + self.inline_block + } else { + (DEFAULT_INLINE_OBJECT_BUDGET / data_shards).min(DEFAULT_INLINE_BLOCK) + }; if versioned { shard_size <= inline_block / 8 @@ -392,6 +401,7 @@ fn lookup_config_for_pools_with_env( } let optimize = overrides.optimize; + let inline_block_explicit = overrides.inline_block.is_some(); let inline_block = if let Some(value) = overrides.inline_block { let block = value .parse::() @@ -424,6 +434,7 @@ fn lookup_config_for_pools_with_env( optimize, inline_block, initialized: true, + inline_block_explicit, standard_parities, rrs_parities, }) @@ -541,22 +552,26 @@ mod tests { } #[test] - fn should_inline_preserves_exact_default_shard_boundaries() { - let config = Config::default(); + fn should_inline_scales_default_threshold_by_data_shards() { + let config = lookup_config_for_pools_with_env(&KVS::new(), &[3, 12], no_env_overrides()) + .expect("default inline policy should resolve for EC2+1 and EC8+4"); - for (case, shard_size, versioned, expected) in [ - ("unversioned below", 128 * 1024 - 1, false, true), - ("unversioned exact", 128 * 1024, false, true), - ("unversioned above", 128 * 1024 + 1, false, false), - ("versioned below", 16 * 1024 - 1, true, true), - ("versioned exact", 16 * 1024, true, true), - ("versioned above", 16 * 1024 + 1, true, false), - ("negative", -1, false, false), + for (case, shard_size, data_shards, versioned, expected) in [ + ("EC2+1 unversioned exact", 128 * 1024, 2, false, true), + ("EC2+1 unversioned above", 128 * 1024 + 1, 2, false, false), + ("EC2+1 versioned exact", 16 * 1024, 2, true, true), + ("EC2+1 versioned above", 16 * 1024 + 1, 2, true, false), + ("EC8+4 unversioned exact", 32 * 1024, 8, false, true), + ("EC8+4 unversioned above", 32 * 1024 + 1, 8, false, false), + ("EC8+4 versioned exact", 4 * 1024, 8, true, true), + ("EC8+4 versioned above", 4 * 1024 + 1, 8, true, false), + ("negative", -1, 2, false, false), + ("zero data shards", 0, 0, false, false), ] { assert_eq!( - config.should_inline(shard_size, versioned), + config.should_inline(shard_size, data_shards, versioned), expected, - "{case}: shard_size={shard_size}, versioned={versioned}" + "{case}: shard_size={shard_size}, data_shards={data_shards}, versioned={versioned}" ); } } @@ -577,13 +592,28 @@ mod tests { let shard_size = erasure.shard_file_size(object_size); assert_eq!(shard_size, expected_shard_size, "{case}: object_size={object_size}"); assert_eq!( - config.should_inline(shard_size, versioned), + config.should_inline(shard_size, erasure.data_shards, versioned), expected, "{case}: object_size={object_size}, shard_size={shard_size}, versioned={versioned}" ); } } + #[test] + fn explicit_inline_block_preserves_fixed_per_shard_rollback() { + let overrides = StorageClassEnvOverrides { + inline_block: Some("128KiB".to_string()), + ..Default::default() + }; + let config = lookup_config_for_pools_with_env(&KVS::new(), &[12], overrides) + .expect("explicit inline block should resolve for EC8+4"); + + assert!(config.should_inline(128 * 1024, 8, false)); + assert!(!config.should_inline(128 * 1024 + 1, 8, false)); + assert!(config.should_inline(16 * 1024, 8, true)); + assert!(!config.should_inline(16 * 1024 + 1, 8, true)); + } + #[test] fn write_capability_contract_only_accepts_implemented_layouts() { assert_eq!(SUPPORTED_WRITE_CLASSES, [STANDARD, RRS]); @@ -777,6 +807,7 @@ mod tests { let encoded = serde_json::to_string(&cfg).expect("config should serialize"); assert!(!encoded.contains("standard_parities")); assert!(!encoded.contains("rrs_parities")); + assert!(!encoded.contains("inline_block_explicit")); let decoded: Config = serde_json::from_str(&encoded).expect("legacy scalar config should deserialize"); assert_eq!(decoded.get_parity_for_sc(STANDARD), Some(2)); @@ -786,6 +817,25 @@ mod tests { assert!(validate_parity(0, 0).is_err()); } + #[test] + fn explicit_inline_block_survives_config_round_trip() { + let cfg = lookup_config_for_pools_with_env( + &KVS::new(), + &[12], + StorageClassEnvOverrides { + inline_block: Some("128KiB".to_string()), + ..Default::default() + }, + ) + .expect("explicit inline block should resolve"); + assert!(cfg.should_inline(100 * 1024, 8, false)); + + let encoded = serde_json::to_string(&cfg).expect("config should serialize"); + assert!(encoded.contains("\"inline_block_explicit\":true")); + let decoded: Config = serde_json::from_str(&encoded).expect("explicit inline config should deserialize"); + assert!(decoded.should_inline(100 * 1024, 8, false)); + } + #[test] fn lookup_config_reads_rrs_from_class_rrs_key() { // Regression: kvs.get(RRS) used RRS="REDUCED_REDUNDANCY" instead of diff --git a/crates/ecstore/src/core/mod.rs b/crates/ecstore/src/core/mod.rs index c232ed96f..9b9e52f69 100644 --- a/crates/ecstore/src/core/mod.rs +++ b/crates/ecstore/src/core/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: pool coordination helpers are being migrated behind runtime owners. -#![allow(dead_code)] pub(crate) mod pools; pub(crate) mod sets; diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 70aa73bde..f35017760 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -226,6 +226,7 @@ fn ensure_decommission_start_rebalance_meta_allowed(meta: Option<&RebalanceMeta> ensure_decommission_not_rebalancing(meta.is_some_and(is_rebalance_conflicting_with_decommission)) } +#[allow(dead_code, reason = "leader precondition asserted by this file's tests (backlog#1823)")] fn ensure_local_decommission_pool_leaders(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<()> { for idx in indices { ensure_local_decommission_pool_leader(endpoints, *idx)?; @@ -1058,11 +1059,19 @@ fn should_cleanup_decommission_source_entry(decommissioned: usize, total_version } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[allow( + dead_code, + reason = "terminal-state classification asserted by this file's tests (backlog#1823)" +)] enum DecommissionTerminalState { Completed, Failed, } +#[allow( + dead_code, + reason = "terminal-state classification asserted by this file's tests (backlog#1823)" +)] fn classify_decommission_terminal_state(failed_items_present: bool) -> DecommissionTerminalState { if failed_items_present { DecommissionTerminalState::Failed diff --git a/crates/ecstore/src/data_movement/mod.rs b/crates/ecstore/src/data_movement/mod.rs index a5f631579..8f7d63580 100644 --- a/crates/ecstore/src/data_movement/mod.rs +++ b/crates/ecstore/src/data_movement/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: data-movement migration keeps staged cleanup helpers until copy paths converge. -#![allow(dead_code)] pub(crate) mod backpressure; @@ -1019,6 +1018,10 @@ struct SourceCleanupDeleteBarrierState { } #[cfg(test)] +#[allow( + dead_code, + reason = "installed by set_disk object tests behind `--features test-util` (backlog#1823)" +)] pub(crate) struct SourceCleanupDeleteBarrier { state: Arc, } @@ -1028,6 +1031,10 @@ static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock Self { let state = Arc::new(SourceCleanupDeleteBarrierState { @@ -1166,6 +1173,7 @@ async fn find_data_movement_target_info( } } +#[allow(dead_code, reason = "resume adjudication asserted by this file's tests (backlog#1823)")] fn resolve_data_movement_overwrite_resume_result( err: &Error, target_result: Result>, diff --git a/crates/ecstore/src/diagnostics/admin_server_info.rs b/crates/ecstore/src/diagnostics/admin_server_info.rs index b03b576ea..1ffbac27a 100644 --- a/crates/ecstore/src/diagnostics/admin_server_info.rs +++ b/crates/ecstore/src/diagnostics/admin_server_info.rs @@ -653,6 +653,7 @@ fn reconcile_servers_with_endpoint_topology( (added, report) } +#[allow(dead_code, reason = "exercised by this file's topology tests (backlog#1823)")] fn server_topology_completeness_report( servers: &[ServerProperties], endpoints: &EndpointServerPools, diff --git a/crates/ecstore/src/diagnostics/get.rs b/crates/ecstore/src/diagnostics/get.rs index 4523ba42b..c02e41161 100644 --- a/crates/ecstore/src/diagnostics/get.rs +++ b/crates/ecstore/src/diagnostics/get.rs @@ -46,21 +46,49 @@ pub(crate) const GET_CODEC_STREAMING_OBJECT_CLASS_MULTIPART: &str = "multipart"; pub(crate) const GET_STAGE_DECODE: &str = "decode"; pub(crate) const GET_STAGE_EMIT: &str = "emit"; pub(crate) const GET_STAGE_FILL: &str = "fill"; +#[allow( + dead_code, + reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)" +)] pub(crate) const GET_STAGE_FIRST_BYTE: &str = "first_byte"; +#[allow( + dead_code, + reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)" +)] pub(crate) const GET_STAGE_FIRST_METADATA_RESPONSE: &str = "first_metadata_response"; +#[allow( + dead_code, + reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)" +)] pub(crate) const GET_STAGE_FIRST_VALID_METADATA_RESPONSE: &str = "first_valid_metadata_response"; +#[allow( + dead_code, + reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)" +)] pub(crate) const GET_STAGE_FIRST_SHARD_READ: &str = "first_shard_read"; +#[allow( + dead_code, + reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)" +)] pub(crate) const GET_STAGE_FULL_BODY: &str = "full_body"; pub(crate) const GET_STAGE_INLINE_PREPARE: &str = "inline_prepare"; pub(crate) const GET_STAGE_LOCK_ACQUIRE: &str = "lock_acquire"; pub(crate) const GET_STAGE_METADATA: &str = "metadata"; pub(crate) const GET_STAGE_METADATA_CACHE_LOOKUP: &str = "metadata_cache_lookup"; +#[allow( + dead_code, + reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)" +)] pub(crate) const GET_STAGE_METADATA_FANOUT: &str = "metadata_fanout"; pub(crate) const GET_STAGE_METADATA_RESOLVE: &str = "metadata_resolve"; pub(crate) const GET_STAGE_OBJECT_INFO: &str = "object_info"; pub(crate) const GET_STAGE_OUTPUT_LOCK_WAIT: &str = "output_lock_wait"; pub(crate) const GET_STAGE_OUTPUT_POLL: &str = "output_poll"; pub(crate) const GET_STAGE_PATH_DECISION: &str = "path_decision"; +#[allow( + dead_code, + reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)" +)] pub(crate) const GET_STAGE_QUORUM_REACHED: &str = "quorum_reached"; pub(crate) const GET_STAGE_RANGE: &str = "range"; pub(crate) const GET_STAGE_READER_SETUP: &str = "reader_setup"; @@ -84,12 +112,28 @@ pub(crate) const GET_STAGE_READER_STREAM_FIRST_READ: &str = "reader_stream_first pub(crate) const GET_STAGE_READER_TASK_BITROT_READER_INIT: &str = "reader_task_bitrot_reader_init"; pub(crate) const GET_STAGE_READER_TASK_FILE_OPEN: &str = "reader_task_file_open"; pub(crate) const GET_STAGE_READER_TASK_READER_CONSTRUCTION: &str = "reader_task_reader_construction"; +pub(crate) const GET_STAGE_READ_VERSION_DECODE: &str = "read_version_decode"; +pub(crate) const GET_STAGE_READ_VERSION_PATH_CHECK: &str = "read_version_path_check"; +pub(crate) const GET_STAGE_READ_VERSION_PATH_RESOLVE: &str = "read_version_path_resolve"; +pub(crate) const GET_STAGE_READ_VERSION_XLMETA_READ: &str = "read_version_xlmeta_read"; pub(crate) const GET_STAGE_RECONSTRUCT: &str = "reconstruct"; +#[allow( + dead_code, + reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)" +)] pub(crate) const GET_STAGE_RESPONSE_HANDOFF: &str = "response_handoff"; +#[allow( + dead_code, + reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)" +)] pub(crate) const GET_STAGE_SLOWEST_METADATA_RESPONSE: &str = "slowest_metadata_response"; pub(crate) const GET_STAGE_STRIPE_READ: &str = "stripe_read"; pub(crate) const GET_STAGE_STRIPE_READ_FIRST_SHARD: &str = "stripe_read_first_shard"; pub(crate) const GET_STAGE_STRIPE_READ_QUORUM: &str = "stripe_read_quorum"; +#[allow( + dead_code, + reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)" +)] pub(crate) const GET_STAGE_BITROT_VERIFY: &str = "bitrot_verify"; pub(crate) const GET_READER_BUFFER_OUTPUT: &str = "output"; @@ -155,8 +199,20 @@ pub(crate) const GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND: &str = "versi pub(crate) const GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM: &str = "version_match_quorum"; /// Early-stop active state labels +#[allow( + dead_code, + reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)" +)] pub(crate) const EARLY_STOP_ACTIVE_HIT: &str = "hit"; +#[allow( + dead_code, + reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)" +)] pub(crate) const EARLY_STOP_ACTIVE_MISS: &str = "miss"; +#[allow( + dead_code, + reason = "GET stage vocabulary; value pinned by this file's tests, no writer yet (backlog#1823)" +)] pub(crate) const EARLY_STOP_ACTIVE_DISABLED: &str = "disabled"; #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -442,6 +498,10 @@ mod tests { assert_eq!(GET_STAGE_QUORUM_REACHED, "quorum_reached"); assert_eq!(GET_STAGE_RANGE, "range"); assert_eq!(GET_STAGE_READER_SETUP, "reader_setup"); + assert_eq!(GET_STAGE_READ_VERSION_DECODE, "read_version_decode"); + assert_eq!(GET_STAGE_READ_VERSION_PATH_CHECK, "read_version_path_check"); + assert_eq!(GET_STAGE_READ_VERSION_PATH_RESOLVE, "read_version_path_resolve"); + assert_eq!(GET_STAGE_READ_VERSION_XLMETA_READ, "read_version_xlmeta_read"); assert_eq!(GET_STAGE_RECONSTRUCT, "reconstruct"); assert_eq!(GET_STAGE_RESPONSE_HANDOFF, "response_handoff"); assert_eq!(GET_STAGE_SLOWEST_METADATA_RESPONSE, "slowest_metadata_response"); diff --git a/crates/ecstore/src/diagnostics/mod.rs b/crates/ecstore/src/diagnostics/mod.rs index c0f7673a5..45c50e928 100644 --- a/crates/ecstore/src/diagnostics/mod.rs +++ b/crates/ecstore/src/diagnostics/mod.rs @@ -13,8 +13,6 @@ // limitations under the License. // #730: diagnostics constants are staged for request-path telemetry migration. -#![allow(dead_code)] pub(crate) mod admin_server_info; pub(crate) mod get; -pub(crate) mod pool; diff --git a/crates/ecstore/src/diagnostics/pool.rs b/crates/ecstore/src/diagnostics/pool.rs deleted file mode 100644 index 2a4d7d88d..000000000 --- a/crates/ecstore/src/diagnostics/pool.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! BytesPool metric label constants. -//! -//! These constants are used when recording pool acquisition and return -//! metrics to avoid string allocations and ensure label consistency. - -/// BytesPool tier labels -pub const POOL_TIER_SMALL: &str = "small"; -pub const POOL_TIER_MEDIUM: &str = "medium"; -pub const POOL_TIER_LARGE: &str = "large"; -pub const POOL_TIER_XLARGE: &str = "xlarge"; - -/// BytesPool outcome labels -pub const POOL_OUTCOME_HIT: &str = "hit"; -pub const POOL_OUTCOME_MISS: &str = "miss"; -pub const POOL_OUTCOME_RECYCLED: &str = "recycled"; -pub const POOL_OUTCOME_DROPPED: &str = "dropped"; diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 258904f9d..64e945d1d 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -15,6 +15,11 @@ use crate::config::storageclass::DEFAULT_INLINE_BLOCK; use crate::crash_inject::{self, CrashPoint}; use crate::data_usage::local_snapshot::ensure_data_usage_layout; +use crate::diagnostics::get::{ + GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READ_VERSION_DECODE, + GET_STAGE_READ_VERSION_PATH_CHECK, GET_STAGE_READ_VERSION_PATH_RESOLVE, GET_STAGE_READ_VERSION_XLMETA_READ, + get_stage_timer_if_enabled, record_get_stage_duration_if_enabled, +}; #[cfg(test)] use crate::disk::HEALING_MARKER_PATH; use crate::disk::disk_store::{get_drive_walkdir_stall_timeout, get_object_disk_read_timeout}; @@ -9840,6 +9845,12 @@ impl DiskAPI for LocalDisk { opts: &ReadOptions, ) -> Result { crate::hp_guard!("LocalDisk::read_version"); + let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + let metrics_path = if stage_metrics_enabled && crate::bucket::utils::is_meta_bucketname(volume) { + GET_OBJECT_PATH_INTERNAL_META + } else { + GET_OBJECT_PATH_LEGACY_DUPLEX + }; if !org_volume.is_empty() { let org_volume_path = self.io_get_bucket_path(org_volume)?; if !skip_access_checks(org_volume) { @@ -9849,37 +9860,46 @@ impl DiskAPI for LocalDisk { } } + let path_resolve_start = get_stage_timer_if_enabled(stage_metrics_enabled); let file_path = self.io_get_object_path(volume, path)?; let volume_dir = self.io_get_bucket_path(volume)?; + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READ_VERSION_PATH_RESOLVE, path_resolve_start); + let path_check_start = get_stage_timer_if_enabled(stage_metrics_enabled); check_path_length(file_path.to_string_lossy().as_ref())?; + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READ_VERSION_PATH_CHECK, path_check_start); let read_data = opts.read_data; - let (data, _) = self - .read_raw(volume, volume_dir.clone(), file_path, read_data) - .await - .map_err(|e| { - if e == DiskError::FileNotFound && !version_id.is_empty() { - DiskError::FileVersionNotFound - } else { - e - } - })?; + let xlmeta_read_start = get_stage_timer_if_enabled(stage_metrics_enabled); + let raw_read_result = self.read_raw(volume, volume_dir.clone(), file_path, read_data).await; + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READ_VERSION_XLMETA_READ, xlmeta_read_start); + let (data, _) = raw_read_result.map_err(|e| { + if e == DiskError::FileNotFound && !version_id.is_empty() { + DiskError::FileVersionNotFound + } else { + e + } + })?; - let mut fi = get_file_info( - &data, - volume, - path, - version_id, - FileInfoOpts { - data: read_data, - include_free_versions: opts.incl_free_versions, - include_part_checksums: false, - }, - )?; - - fi.validate_for_metadata_read()?; + let decode_start = get_stage_timer_if_enabled(stage_metrics_enabled); + let file_info_result: Result = (|| { + let fi = get_file_info( + &data, + volume, + path, + version_id, + FileInfoOpts { + data: read_data, + include_free_versions: opts.incl_free_versions, + include_part_checksums: false, + }, + )?; + fi.validate_for_metadata_read()?; + Ok(fi) + })(); + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READ_VERSION_DECODE, decode_start); + let mut fi = file_info_result?; if fi.is_canonical_delete_marker() { return Ok(fi); } @@ -10562,6 +10582,108 @@ mod test { meta.marshal_msg().expect("test metadata should encode") } + #[test] + #[serial_test::serial] + fn read_version_records_local_metadata_stage_breakdown() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime should be created"); + let recorder = crate::test_metrics::CapturingRecorder::default(); + let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled(); + rustfs_io_metrics::set_get_stage_metrics_enabled(true); + + metrics::with_local_recorder(&recorder, || { + runtime.block_on(async { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = + Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + let bucket = "bucket"; + let object = "stage-breakdown"; + ensure_test_volume(&disk, bucket).await; + + let object_dir = dir.path().join(bucket).join(object); + fs::create_dir_all(&object_dir) + .await + .expect("object directory should be created"); + fs::write( + object_dir.join(STORAGE_FORMAT_FILE), + test_meta(test_file_info(object, Uuid::new_v4(), None, Some(Bytes::from_static(b"inline")))), + ) + .await + .expect("object metadata should be written"); + + disk.read_version( + "", + bucket, + object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read_version should succeed"); + + let meta_object = "stage-breakdown-meta"; + let meta_object_dir = dir.path().join(RUSTFS_META_BUCKET).join(meta_object); + fs::create_dir_all(&meta_object_dir) + .await + .expect("internal metadata object directory should be created"); + fs::write( + meta_object_dir.join(STORAGE_FORMAT_FILE), + test_meta(test_file_info(meta_object, Uuid::new_v4(), None, Some(Bytes::from_static(b"meta")))), + ) + .await + .expect("internal metadata should be written"); + + disk.read_version( + "", + RUSTFS_META_BUCKET, + meta_object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("internal metadata read_version should succeed"); + }); + }); + rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate); + + for stage in [ + GET_STAGE_READ_VERSION_PATH_RESOLVE, + GET_STAGE_READ_VERSION_PATH_CHECK, + GET_STAGE_READ_VERSION_XLMETA_READ, + GET_STAGE_READ_VERSION_DECODE, + ] { + assert_eq!( + recorder + .histogram_values( + "rustfs_io_get_object_stage_duration_seconds", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX), ("stage", stage)] + ) + .len(), + 1, + "{stage} should be recorded once for user-bucket LocalDisk::read_version" + ); + assert_eq!( + recorder + .histogram_values( + "rustfs_io_get_object_stage_duration_seconds", + &[("path", GET_OBJECT_PATH_INTERNAL_META), ("stage", stage)] + ) + .len(), + 1, + "{stage} should be recorded once for internal-meta LocalDisk::read_version" + ); + } + } + #[test] fn inline_metadata_rollback_dir_avoids_real_data_dir_collision() { let target_version = Uuid::parse_str("11111111-2222-3333-4444-555555555555").expect("version id should parse"); diff --git a/crates/ecstore/src/erasure/codec/bridge.rs b/crates/ecstore/src/erasure/codec/bridge.rs index 05433cd79..75388ac87 100644 --- a/crates/ecstore/src/erasure/codec/bridge.rs +++ b/crates/ecstore/src/erasure/codec/bridge.rs @@ -26,6 +26,7 @@ pub(crate) const GET_RECONSTRUCT_OUTCOME_SKIP_DATA_COMPLETE: &str = "skip_data_c pub(crate) const GET_RECONSTRUCT_OUTCOME_SKIP_EMPTY_PAYLOAD: &str = "skip_empty_payload"; pub(crate) trait DecodeWorkspace: Send + Sync + 'static { + #[allow(dead_code, reason = "workspace width asserted by decode_reader tests (backlog#1823)")] fn shard_len(&self) -> usize; } @@ -33,11 +34,14 @@ pub(crate) trait ErasureDecodeEngine: Send + Sync + 'static { type Workspace: DecodeWorkspace; fn data_shards(&self) -> usize; + #[allow(dead_code, reason = "engine trait facet asserted by decode_reader tests (backlog#1823)")] fn parity_shards(&self) -> usize; fn block_size(&self) -> usize; fn engine_name(&self) -> &'static str; + #[allow(dead_code, reason = "engine trait facet asserted by decode_reader tests (backlog#1823)")] fn supports_progressive_decode(&self) -> bool; + #[allow(dead_code, reason = "engine trait facet asserted by decode_reader tests (backlog#1823)")] fn supports_aligned_shards(&self) -> bool; fn prepare_workspace(&self, shard_len: usize) -> io::Result; diff --git a/crates/ecstore/src/erasure/codec/workspace.rs b/crates/ecstore/src/erasure/codec/workspace.rs index 9cd1a54cc..d019e98ef 100644 --- a/crates/ecstore/src/erasure/codec/workspace.rs +++ b/crates/ecstore/src/erasure/codec/workspace.rs @@ -24,6 +24,7 @@ impl RustfsCodecDecodeWorkspace { } #[inline] + #[allow(dead_code, reason = "workspace width asserted by decode_reader tests (backlog#1823)")] pub(crate) fn shard_len(&self) -> usize { self.shard_len } diff --git a/crates/ecstore/src/erasure/coding/decode.rs b/crates/ecstore/src/erasure/coding/decode.rs index f20bbce2e..8b616c058 100644 --- a/crates/ecstore/src/erasure/coding/decode.rs +++ b/crates/ecstore/src/erasure/coding/decode.rs @@ -213,6 +213,7 @@ fn shard_read_launch_rank(cost: ShardReadCost) -> u8 { } } +#[allow(dead_code, reason = "launch ordering asserted by this file's tests (backlog#1823)")] fn shard_read_launch_order(read_costs: &[ShardReadCost], num_readers: usize, locality_preference_enabled: bool) -> Vec { let mut order: Vec = (0..num_readers).collect(); if locality_preference_enabled { @@ -408,6 +409,10 @@ where R: crate::erasure::coding::ShardSource, { // Readers should handle disk errors before being passed in, ensuring each reader reaches the available number of BitrotReaders + #[allow( + dead_code, + reason = "ParallelReader constructor used only by this file's tests (backlog#1823)" + )] pub fn new(readers: Vec>>, e: Erasure, offset: usize, total_length: usize) -> Self { Self::new_with_metrics_path_read_timeout_and_reconstruction_verification( readers, @@ -420,6 +425,7 @@ where ) } + #[allow(dead_code, reason = "constructor used only by this file's tests (backlog#1823)")] pub fn new_with_metrics_path( readers: Vec>>, e: Erasure, @@ -438,6 +444,7 @@ where ) } + #[allow(dead_code, reason = "constructor used only by this file's tests (backlog#1823)")] pub fn new_with_metrics_path_and_read_costs( readers: Vec>>, e: Erasure, @@ -514,6 +521,7 @@ where ) } + #[allow(dead_code, reason = "constructor used only by this file's tests (backlog#1823)")] fn new_with_read_timeout( readers: Vec>>, e: Erasure, @@ -1330,10 +1338,6 @@ where } } } - - pub fn can_decode(&self, shards: &[Option>]) -> bool { - shards.iter().filter(|s| s.is_some()).count() >= self.data_shards - } } #[async_trait::async_trait] @@ -1539,6 +1543,7 @@ impl Erasure { .await } + #[allow(dead_code, reason = "read-cost decode path asserted by this file's tests (backlog#1823)")] pub(crate) async fn decode_with_read_costs( &self, writer: &mut W, @@ -1609,9 +1614,9 @@ impl Erasure { *ret_err = Some(err.into()); } - // Equivalent to `ParallelReader::can_decode`; inlined so this helper does - // not need to borrow the reader, leaving the reader free for the - // concurrent next-stripe read under prefetch. + // Shard-availability check, written out here rather than called on the + // reader so this helper does not need to borrow it, leaving the reader + // free for the concurrent next-stripe read under prefetch. let available_shards = shards.iter().filter(|shard| shard.is_some()).count(); if available_shards < self.data_shards { let reason = GetObjectFailureReason::ReadQuorum; diff --git a/crates/ecstore/src/erasure/coding/decode_reader.rs b/crates/ecstore/src/erasure/coding/decode_reader.rs index 8fb7a7cfc..21415021e 100644 --- a/crates/ecstore/src/erasure/coding/decode_reader.rs +++ b/crates/ecstore/src/erasure/coding/decode_reader.rs @@ -138,6 +138,10 @@ where S: ShardStripeSource + Send + 'static, E: ErasureDecodeEngine + Clone + Send + Sync + 'static, { + #[allow( + dead_code, + reason = "default-metrics-path constructor used only by this file's tests (backlog#1823)" + )] pub(crate) fn new(source: S, engine: E, total_length: usize) -> io::Result { Self::new_with_metrics_path(source, engine, total_length, GET_OBJECT_PATH_CODEC_STREAMING) } @@ -679,6 +683,10 @@ pub(crate) struct SyncErasureDecodeReader { } impl SyncErasureDecodeReader { + #[allow( + dead_code, + reason = "default-metrics-path constructor used only by this file's tests (backlog#1823)" + )] pub(crate) fn new(inner: R) -> Self { Self::new_with_metrics_path(inner, GET_OBJECT_PATH_CODEC_STREAMING) } @@ -805,6 +813,7 @@ where Ok(true) } +#[allow(dead_code, reason = "shard emission asserted by this file's tests (backlog#1823)")] fn emit_data_shards(state: &StripeReadState, data_shards: usize, block_size: usize, remaining: usize) -> io::Result> { let mut output = Vec::new(); emit_data_shards_into(state, data_shards, block_size, remaining, &mut output)?; diff --git a/crates/ecstore/src/erasure/coding/encode.rs b/crates/ecstore/src/erasure/coding/encode.rs index 100cb3a1d..97fdc9525 100644 --- a/crates/ecstore/src/erasure/coding/encode.rs +++ b/crates/ecstore/src/erasure/coding/encode.rs @@ -166,6 +166,7 @@ where if total == 0 { Ok(None) } else { Ok(Some(total)) } } +#[allow(dead_code, reason = "byte accounting asserted by this file's tests (backlog#1823)")] fn queued_block_bytes(block: &[Bytes]) -> usize { block.iter().map(Bytes::len).sum() } diff --git a/crates/ecstore/src/erasure/coding/erasure.rs b/crates/ecstore/src/erasure/coding/erasure.rs index 8b7ff1c10..a9e4c8bc3 100644 --- a/crates/ecstore/src/erasure/coding/erasure.rs +++ b/crates/ecstore/src/erasure/coding/erasure.rs @@ -71,10 +71,16 @@ impl EncodedBlock { const MODERN_MAX_TOTAL_SHARDS: usize = ::ORDER; const MODERN_REED_SOLOMON_CACHE_MAX_ENTRIES: usize = 64; +const LEGACY_REED_SOLOMON_CACHE_MAX_ENTRIES: usize = 16; +// Vec growth may retain twice the requested logical length. Keeping the logical +// workspace at half the budget bounds each cached workspace's shard allocation to 1 MiB. +const LEGACY_REED_SOLOMON_CACHE_MAX_LOGICAL_SHARD_BYTES_PER_WORKSPACE: usize = 512 * 1024; type ModernReedSolomonCache = RwLock>>; +type LegacyReedSolomonCache = RwLock>>; static MODERN_REED_SOLOMON_CACHE: OnceLock = OnceLock::new(); +static LEGACY_REED_SOLOMON_CACHE: OnceLock = OnceLock::new(); /// Errors returned when constructing an [`Erasure`] codec. #[derive(Debug, thiserror::Error)] @@ -141,43 +147,61 @@ pub fn calc_shard_size_legacy(block_size: usize, data_shards: usize) -> usize { struct LegacyReedSolomonEncoder { data_shards: usize, parity_shards: usize, - encoder_cache: std::sync::RwLock>, - decoder_cache: std::sync::RwLock>, -} - -impl Clone for LegacyReedSolomonEncoder { - fn clone(&self) -> Self { - Self { - data_shards: self.data_shards, - parity_shards: self.parity_shards, - encoder_cache: std::sync::RwLock::new(None), - decoder_cache: std::sync::RwLock::new(None), - } - } + cache_workspaces: bool, + encoder_cache: RwLock>, + decoder_cache: RwLock>, } impl LegacyReedSolomonEncoder { - fn new(_data_shards: usize, _parity_shards: usize) -> io::Result { + fn new(data_shards: usize, parity_shards: usize) -> io::Result { + Self::with_workspace_cache(data_shards, parity_shards, false) + } + + fn with_workspace_cache(data_shards: usize, parity_shards: usize, cache_workspaces: bool) -> io::Result { Ok(Self { - data_shards: _data_shards, - parity_shards: _parity_shards, - encoder_cache: std::sync::RwLock::new(None), - decoder_cache: std::sync::RwLock::new(None), + data_shards, + parity_shards, + cache_workspaces, + encoder_cache: RwLock::new(None), + decoder_cache: RwLock::new(None), }) } + fn logical_shard_bytes_upper_bound(&self, shard_len: usize) -> Option { + let aligned_shard_len = shard_len.checked_add(63)?.checked_div(64)?.checked_mul(64)?; + let high_rate_decoder_work_count = self + .parity_shards + .checked_next_power_of_two()? + .checked_add(self.data_shards)? + .checked_next_power_of_two()?; + let low_rate_decoder_work_count = self + .data_shards + .checked_next_power_of_two()? + .checked_add(self.parity_shards)? + .checked_next_power_of_two()?; + aligned_shard_len.checked_mul(high_rate_decoder_work_count.max(low_rate_decoder_work_count)) + } + + fn should_cache_workspace(&self, shard_len: usize) -> bool { + self.cache_workspaces + && self + .logical_shard_bytes_upper_bound(shard_len) + .is_some_and(|bytes| bytes <= LEGACY_REED_SOLOMON_CACHE_MAX_LOGICAL_SHARD_BYTES_PER_WORKSPACE) + } + fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> { let mut shards_vec: Vec<&mut [u8]> = shards.into_vec(); if shards_vec.is_empty() { return Ok(()); } let shard_len = shards_vec[0].len(); + let cached_encoder = self + .encoder_cache + .write() + .map_err(|_| io::Error::other("Failed to acquire encoder cache lock"))? + .take(); let mut encoder = { - let mut cache_guard = self - .encoder_cache - .write() - .map_err(|_| io::Error::other("Failed to acquire encoder cache lock"))?; - match cache_guard.take() { + match cached_encoder { Some(mut cached) => { if cached.reset(self.data_shards, self.parity_shards, shard_len).is_err() { reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len) @@ -204,10 +228,15 @@ impl LegacyReedSolomonEncoder { } } drop(result); - *self - .encoder_cache - .write() - .map_err(|_| io::Error::other("Failed to return encoder to cache"))? = Some(encoder); + if self.should_cache_workspace(shard_len) { + let mut cache = self + .encoder_cache + .write() + .map_err(|_| io::Error::other("Failed to return encoder to cache"))?; + if cache.is_none() { + *cache = Some(encoder); + } + } Ok(()) } @@ -221,13 +250,13 @@ impl LegacyReedSolomonEncoder { .find_map(|s| s.as_ref().map(|v| v.len())) .ok_or_else(|| io::Error::other("No valid shards found for reconstruction"))?; + let cached_decoder = self + .decoder_cache + .write() + .map_err(|_| io::Error::other("Failed to acquire decoder cache lock"))? + .take(); let mut decoder = { - let mut cache_guard = self - .decoder_cache - .write() - .map_err(|_| io::Error::other("Failed to acquire decoder cache lock"))?; - - match cache_guard.take() { + match cached_decoder { Some(mut cached_decoder) => { if let Err(e) = cached_decoder.reset(self.data_shards, self.parity_shards, shard_len) { warn!("Failed to reset SIMD decoder: {:?}, creating new one", e); @@ -274,10 +303,15 @@ impl LegacyReedSolomonEncoder { drop(result); - *self - .decoder_cache - .write() - .map_err(|_| io::Error::other("Failed to return decoder to cache"))? = Some(decoder); + if self.should_cache_workspace(shard_len) { + let mut cache = self + .decoder_cache + .write() + .map_err(|_| io::Error::other("Failed to return decoder to cache"))?; + if cache.is_none() { + *cache = Some(decoder); + } + } Ok(()) } @@ -435,6 +469,39 @@ fn cached_modern_reed_solomon(data_shards: usize, parity_shards: usize) -> Resul Ok(encoder) } +fn cached_legacy_reed_solomon(data_shards: usize, parity_shards: usize) -> io::Result> { + let cache = LEGACY_REED_SOLOMON_CACHE.get_or_init(|| RwLock::new(HashMap::new())); + cached_legacy_reed_solomon_in(cache, data_shards, parity_shards) +} + +fn cached_legacy_reed_solomon_in( + cache: &LegacyReedSolomonCache, + data_shards: usize, + parity_shards: usize, +) -> io::Result> { + let key = (data_shards, parity_shards); + if let Some(encoder) = cache + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .get(&key) + .cloned() + { + return Ok(encoder); + } + + let mut cache = cache.write().unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(existing) = cache.get(&key) { + return Ok(Arc::clone(existing)); + } + if cache.len() < LEGACY_REED_SOLOMON_CACHE_MAX_ENTRIES { + let encoder = Arc::new(LegacyReedSolomonEncoder::with_workspace_cache(data_shards, parity_shards, true)?); + cache.insert(key, Arc::clone(&encoder)); + return Ok(encoder); + } + drop(cache); + Ok(Arc::new(LegacyReedSolomonEncoder::new(data_shards, parity_shards)?)) +} + fn encode_parity_shards(shards: &mut [Option>], data_shards: usize, parity_shards: usize, encode: F) -> io::Result<()> where F: FnOnce(SmallVec<[&mut [u8]; 16]>) -> io::Result<()>, @@ -551,7 +618,7 @@ pub struct Erasure { pub data_shards: usize, pub parity_shards: usize, encoder: Option, - legacy_encoder: Option, + legacy_encoder: Option>, pub block_size: usize, uses_legacy: bool, _id: Uuid, @@ -687,7 +754,7 @@ impl Erasure { let legacy_encoder = if uses_legacy && parity_shards > 0 { Some( - LegacyReedSolomonEncoder::new(data_shards, parity_shards) + cached_legacy_reed_solomon(data_shards, parity_shards) .map_err(|source| ErasureConstructionError::LegacyEncoder { source })?, ) } else { @@ -1043,6 +1110,10 @@ impl Erasure { /// /// # Errors /// Returns error if reading from reader fails or if callback returns error + #[allow( + dead_code, + reason = "callback encode path exercised only by this file's tests (backlog#1823)" + )] pub(crate) async fn encode_stream_callback_async( self: std::sync::Arc, reader: &mut R, @@ -1405,7 +1476,7 @@ mod tests { assert_eq!(cloned.block_size, legacy.block_size); assert!(cloned.uses_legacy); - let data = b"legacy clone should keep independent SIMD caches"; + let data = b"legacy clone should preserve SIMD codec behavior"; let encoded = cloned.encode_data(data).expect("legacy clone should encode"); let mut shards = optional_shards(&encoded); shards[0] = None; @@ -1413,6 +1484,93 @@ mod tests { assert_eq!(recover_data(&shards, cloned.data_shards, data.len()), data); } + #[test] + fn legacy_codecs_share_process_cache_across_erasure_instances() { + let first = Erasure::new_with_options(6, 3, 64, true) + .legacy_encoder + .expect("legacy codec should be initialized"); + let second = Erasure::new_with_options(6, 3, 128, true) + .legacy_encoder + .expect("same legacy shard layout should be initialized"); + + assert!(Arc::ptr_eq(&first, &second)); + } + + #[test] + fn legacy_workspace_cache_rejects_oversize_buffers_and_isolates_layouts() { + let four_plus_two = Erasure::new_with_options(4, 2, 64, true) + .legacy_encoder + .expect("legacy codec should be initialized"); + let four_plus_one = Erasure::new_with_options(4, 1, 64, true) + .legacy_encoder + .expect("distinct parity layout should be initialized"); + let three_plus_two = Erasure::new_with_options(3, 2, 64, true) + .legacy_encoder + .expect("distinct data layout should be initialized"); + + assert!(!Arc::ptr_eq(&four_plus_two, &four_plus_one)); + assert!(!Arc::ptr_eq(&four_plus_two, &three_plus_two)); + assert_eq!(four_plus_two.logical_shard_bytes_upper_bound(64 * 1024), Some(512 * 1024)); + assert!(four_plus_two.should_cache_workspace(64 * 1024)); + assert!(!four_plus_two.should_cache_workspace(64 * 1024 + 1)); + + let nine_plus_seven = + LegacyReedSolomonEncoder::with_workspace_cache(9, 7, true).expect("9+7 legacy codec should construct"); + assert_eq!(nine_plus_seven.logical_shard_bytes_upper_bound(16 * 1024), Some(512 * 1024)); + assert!(nine_plus_seven.should_cache_workspace(16 * 1024)); + assert!(!nine_plus_seven.should_cache_workspace(16 * 1024 + 1)); + + let uncached = LegacyReedSolomonEncoder::new(4, 2).expect("uncached legacy codec should construct"); + assert!(!uncached.should_cache_workspace(64)); + } + + #[test] + fn saturated_legacy_codec_cache_does_not_retain_more_workspaces() { + let cache = RwLock::new(HashMap::new()); + for parity_shards in 1..=LEGACY_REED_SOLOMON_CACHE_MAX_ENTRIES { + let cached = + cached_legacy_reed_solomon_in(&cache, 32, parity_shards).expect("cacheable legacy codec should construct"); + assert!(cached.cache_workspaces); + } + + let uncached = + cached_legacy_reed_solomon_in(&cache, 31, 1).expect("uncached legacy codec should construct after saturation"); + assert!(!uncached.cache_workspaces); + assert_eq!( + cache.read().expect("cache lock should remain healthy").len(), + LEGACY_REED_SOLOMON_CACHE_MAX_ENTRIES + ); + } + + #[test] + fn concurrent_legacy_codecs_preserve_byte_exact_results() { + let barrier = Arc::new(std::sync::Barrier::new(2)); + let payloads = [vec![0x35; 257], vec![0xca; 1025]]; + + std::thread::scope(|scope| { + let handles = payloads.each_ref().map(|payload| { + let barrier = Arc::clone(&barrier); + scope.spawn(move || { + let erasure = Erasure::new_with_options(6, 3, 2048, true); + barrier.wait(); + let encoded = erasure.encode_data(payload).expect("concurrent legacy encode should succeed"); + barrier.wait(); + + let mut shards = optional_shards(&encoded); + shards[0] = None; + erasure + .decode_data(&mut shards) + .expect("concurrent legacy decode should reconstruct the missing shard"); + recover_data(&shards, erasure.data_shards, payload.len()) + }) + }); + + for (handle, payload) in handles.into_iter().zip(payloads.iter()) { + assert_eq!(handle.join().expect("concurrent legacy codec worker should not panic"), *payload); + } + }); + } + #[test] fn legacy_verify_reports_invalid_empty_valid_and_corrupt_parity_sets() { let legacy = LegacyReedSolomonEncoder::new(2, 2).expect("legacy encoder should construct"); diff --git a/crates/ecstore/src/erasure/mod.rs b/crates/ecstore/src/erasure/mod.rs index 09b0c957e..cf151adaf 100644 --- a/crates/ecstore/src/erasure/mod.rs +++ b/crates/ecstore/src/erasure/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: erasure codec migration keeps staged streaming decode paths in this module. -#![allow(dead_code)] pub(crate) mod codec; pub(crate) mod coding; diff --git a/crates/ecstore/src/error/mod.rs b/crates/ecstore/src/error/mod.rs index 2208f359e..4e8ff791f 100644 --- a/crates/ecstore/src/error/mod.rs +++ b/crates/ecstore/src/error/mod.rs @@ -13,13 +13,12 @@ // limitations under the License. // #730: error taxonomy still exposes compatibility variants while callers move to contracts. -#![allow(dead_code)] use crate::bucket::error::BucketMetadataError; use crate::disk::error::DiskError; use crate::storage_api_contracts::{error::StorageErrorCode, range::HTTPRangeError}; use rustfs_utils::path::decode_dir_object; -use s3s::{S3Error, S3ErrorCode}; +use s3s::S3ErrorCode; pub type Error = StorageError; pub type Result = core::result::Result; @@ -902,6 +901,7 @@ pub fn is_err_decommission_running(err: &Error) -> bool { matches!(err, &StorageError::DecommissionAlreadyRunning) } +#[allow(dead_code, reason = "predicate asserted by this file's tests (backlog#1823)")] pub fn is_err_rebalance_running(err: &Error) -> bool { matches!(err, &StorageError::RebalanceAlreadyRunning) } @@ -910,14 +910,11 @@ pub fn is_err_operation_canceled(err: &Error) -> bool { matches!(err, &StorageError::OperationCanceled) } +#[allow(dead_code, reason = "predicate asserted by this file's tests (backlog#1823)")] pub fn is_err_not_initialized(err: &Error) -> bool { err.to_string().contains("errServerNotInitialized") || err.to_string().contains("ServerNotInitialized") } -pub fn is_err_io(err: &Error) -> bool { - matches!(err, &StorageError::Io(_)) -} - /// Strict "not found" predicate that only matches genuine object/version/volume /// absence errors: `FileNotFound`/`VolumeNotFound`/`FileVersionNotFound`/ /// `ObjectNotFound`/`VersionNotFound`. @@ -1078,21 +1075,9 @@ pub struct GenericError { #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum ObjectApiError { - #[error("Operation timed out")] - OperationTimedOut, - - #[error("etag of the object has changed")] - InvalidETag, - #[error("BackendDown")] BackendDown(String), - #[error("Unsupported headers in Metadata")] - UnsupportedMetadata, - - #[error("Method not allowed: {}/{}", .0.bucket, .0.object)] - MethodNotAllowed(GenericError), - #[error("The operation is not valid for the current state of the object {}/{}({})", .0.bucket, .0.object, .0.version_id)] InvalidObjectState(GenericError), } @@ -1175,30 +1160,6 @@ pub fn error_resp_to_object_err(err: ErrorResponse, params: Vec<&str>) -> std::i err } -pub fn storage_to_object_err(err: Error, params: Vec<&str>) -> S3Error { - let storage_err = &err; - let mut bucket: String = "".to_string(); - let mut object: String = "".to_string(); - if !params.is_empty() { - bucket = params[0].to_string(); - } - if params.len() >= 2 { - object = decode_dir_object(params[1]); - } - match storage_err { - StorageError::MethodNotAllowed => S3Error::with_message( - S3ErrorCode::MethodNotAllowed, - ObjectApiError::MethodNotAllowed(GenericError { - bucket, - object, - ..Default::default() - }) - .to_string(), - ), - _ => s3s::S3Error::with_message(S3ErrorCode::Custom("err".into()), err.to_string()), - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/ecstore/src/event/mod.rs b/crates/ecstore/src/event/mod.rs index ff60c662e..8a8e576ef 100644 --- a/crates/ecstore/src/event/mod.rs +++ b/crates/ecstore/src/event/mod.rs @@ -13,8 +13,6 @@ // limitations under the License. // #730: event target types are retained for notification owner migration. -#![allow(dead_code)] pub mod name; -pub mod targetid; pub mod targetlist; diff --git a/crates/ecstore/src/event/targetid.rs b/crates/ecstore/src/event/targetid.rs deleted file mode 100644 index d41f1e956..000000000 --- a/crates/ecstore/src/event/targetid.rs +++ /dev/null @@ -1,25 +0,0 @@ -#![allow(clippy::all)] -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub struct TargetID { - id: String, - name: String, -} - -impl TargetID { - fn to_string(&self) -> String { - format!("{}:{}", self.id, self.name) - } -} diff --git a/crates/ecstore/src/event/targetlist.rs b/crates/ecstore/src/event/targetlist.rs index 29927b068..b63b786ef 100644 --- a/crates/ecstore/src/event/targetlist.rs +++ b/crates/ecstore/src/event/targetlist.rs @@ -12,18 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::event::targetid::TargetID; use std::sync::atomic::AtomicI64; +/// Placeholder notification target list held by `EventNotifier`. +/// +/// The working notification stack lives in `rustfs-notify` / `rustfs-targets`; +/// this type never grew past its counter. `total_events` is read by the +/// notifier's log line but nothing increments it, so that field reports zero. #[derive(Default)] pub struct TargetList { - pub current_send_calls: AtomicI64, pub total_events: AtomicI64, - pub events_skipped: AtomicI64, - pub events_errors_total: AtomicI64, - //pub targets: HashMap, - //pub queue: AsyncEvent, - //pub targetStats: HashMap, } impl TargetList { @@ -31,14 +29,3 @@ impl TargetList { TargetList::default() } } - -struct TargetStat { - current_send_calls: i64, - total_events: i64, - failed_events: i64, -} - -struct TargetIDResult { - id: TargetID, - err: std::io::Error, -} diff --git a/crates/ecstore/src/io_support/mod.rs b/crates/ecstore/src/io_support/mod.rs index ddea43226..040371161 100644 --- a/crates/ecstore/src/io_support/mod.rs +++ b/crates/ecstore/src/io_support/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: I/O backend selection keeps test-only and staged rio helpers scoped here. -#![allow(dead_code)] pub(crate) mod bitrot; pub(crate) mod compress; diff --git a/crates/ecstore/src/io_support/rio.rs b/crates/ecstore/src/io_support/rio.rs index e62d1d1e3..138a4be53 100644 --- a/crates/ecstore/src/io_support/rio.rs +++ b/crates/ecstore/src/io_support/rio.rs @@ -25,9 +25,20 @@ use tokio::io::AsyncRead; #[cfg(feature = "rio-v2")] const MINIO_S2_COMPRESSION_SCHEME: &str = "klauspost/compress/s2"; +// The S2 padding multiple rio-v2 pads compressed streams to before +// encryption. Only the padding test asserts it today, so the lib target sees +// it as unused (backlog#1823). #[cfg(feature = "rio-v2")] +#[allow(dead_code, reason = "on-disk contract asserted by the rio-v2 padding test (backlog#1823)")] const ENCRYPTED_S2_PADDING_MULTIPLE: usize = 256; +/// Which rio implementation this build compiled in. Only the feature-seam +/// guard test in lib.rs reads it, so the lib target sees it as unused +/// (backlog#1823). +#[allow( + dead_code, + reason = "asserted by the rio backend feature-seam test in lib.rs (backlog#1823)" +)] pub const fn backend_name() -> &'static str { #[cfg(feature = "rio-v2")] { @@ -53,17 +64,6 @@ pub fn compression_metadata_value(algorithm: CompressionAlgorithm) -> String { } } -pub fn compression_scheme_to_algorithm(scheme: &str) -> std::io::Result { - #[cfg(feature = "rio-v2")] - if scheme.eq_ignore_ascii_case(MINIO_S2_COMPRESSION_SCHEME) { - // rio_v2 currently routes all compressed-object handling through the S2 - // reader implementation, so the enum is only a placeholder token here. - return Ok(CompressionAlgorithm::default()); - } - - CompressionAlgorithm::from_str(scheme) -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ReadCompressionBackend { Legacy, @@ -82,6 +82,11 @@ pub fn compression_scheme_to_read_plan(scheme: &str) -> std::io::Result<(Compres #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ReadEncryptionBackend { Legacy, + // Never constructed today — every read still selects Legacy — but the + // decrypt paths below carry live match arms for it. This is the rio-v2 + // read seam (backlog#1638 / #1835), not dead code: deleting the variant + // would delete those arms with it. + #[allow(dead_code, reason = "rio-v2 read seam; match arms below are live (backlog#1823)")] V2, } diff --git a/crates/ecstore/src/layout/endpoints.rs b/crates/ecstore/src/layout/endpoints.rs index 979525059..1f1a9bdd1 100644 --- a/crates/ecstore/src/layout/endpoints.rs +++ b/crates/ecstore/src/layout/endpoints.rs @@ -209,15 +209,12 @@ impl AsMut> for PoolEndpointList { } impl PoolEndpointList { - /// creates a list of endpoints per pool, resolves their relevant - /// hostnames and discovers those are local or remote. - async fn create_pool_endpoints(server_addr: &str, disks_layout: &DisksLayout) -> Result { - Self::create_pool_endpoints_with(server_addr, disks_layout, None, None).await - } - - /// Same as [`create_pool_endpoints`] but lets tests inject an explicit - /// startup topology convergence policy and local endpoint host instead of - /// resolving them from the environment. + /// Creates a list of endpoints per pool, resolves their relevant hostnames + /// and discovers whether those are local or remote. + /// + /// The policy and host overrides let tests inject an explicit startup + /// topology convergence policy and local endpoint host instead of + /// resolving them from the environment; production passes `None` for both. async fn create_pool_endpoints_with( server_addr: &str, disks_layout: &DisksLayout, @@ -594,6 +591,10 @@ impl PoolEndpointList { } const DNS_RETRY_BASE_DELAY: Duration = Duration::from_millis(500); +#[allow( + dead_code, + reason = "retry-cap bound asserted by this file's dns_retry_delay tests (backlog#1823)" +)] const DNS_RETRY_MAX_DELAY: Duration = Duration::from_secs(8); const DNS_RETRY_JITTER_PERCENT: u64 = 20; /// Minimum spacing between "still retrying" warnings so a long orchestrated diff --git a/crates/ecstore/src/layout/mod.rs b/crates/ecstore/src/layout/mod.rs index e58d006dc..bf1b60012 100644 --- a/crates/ecstore/src/layout/mod.rs +++ b/crates/ecstore/src/layout/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: set-layout contracts are staged while ECStore ownership boundaries shrink. -#![allow(dead_code)] //! Static ECStore layout boundaries. //! diff --git a/crates/ecstore/src/layout/set_layout.rs b/crates/ecstore/src/layout/set_layout.rs index cc73252a4..c0441e944 100644 --- a/crates/ecstore/src/layout/set_layout.rs +++ b/crates/ecstore/src/layout/set_layout.rs @@ -4,6 +4,7 @@ use std::io::{Error, Result}; use uuid::Uuid; #[derive(Debug, Clone, PartialEq, Eq)] +#[allow(dead_code, reason = "ESET-001 layout model; exercised by this file's tests (backlog#1823)")] pub(crate) struct StaticSetLayoutSnapshot { pub(crate) deployment_id: Uuid, pub(crate) set_count: usize, @@ -12,6 +13,7 @@ pub(crate) struct StaticSetLayoutSnapshot { pub(crate) distribution_algo: DistributionAlgoVersion, } +#[allow(dead_code, reason = "ESET-001 layout model; exercised by this file's tests (backlog#1823)")] impl StaticSetLayoutSnapshot { pub(crate) fn from_format(format: &FormatV3) -> Self { let disk_ids = format.erasure.sets.clone(); @@ -39,17 +41,20 @@ impl StaticSetLayoutSnapshot { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[allow(dead_code, reason = "ESET-001 layout model; exercised by this file's tests (backlog#1823)")] pub(crate) struct SetDiskPosition { pub(crate) set_index: usize, pub(crate) disk_index: usize, } #[derive(Debug, Clone, PartialEq, Eq)] +#[allow(dead_code, reason = "ESET-001 layout model; exercised by this file's tests (backlog#1823)")] pub(crate) struct RuntimeSetLayoutPlan { pub(crate) sets: Vec>, lock_hosts_by_set: Vec>, } +#[allow(dead_code, reason = "ESET-001 layout model; exercised by this file's tests (backlog#1823)")] impl RuntimeSetLayoutPlan { pub(crate) fn from_endpoint_hosts(set_count: usize, drives_per_set: usize, endpoint_hosts: &[S]) -> Result where @@ -108,6 +113,7 @@ impl RuntimeSetLayoutPlan { } #[derive(Debug, Clone, PartialEq, Eq)] +#[allow(dead_code, reason = "ESET-001 layout model; exercised by this file's tests (backlog#1823)")] pub(crate) struct RuntimeSetDrivePlan { pub(crate) set_index: usize, pub(crate) disk_index: usize, diff --git a/crates/ecstore/src/object_api/mod.rs b/crates/ecstore/src/object_api/mod.rs index e3ed89d93..41c4ff403 100644 --- a/crates/ecstore/src/object_api/mod.rs +++ b/crates/ecstore/src/object_api/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: object API readers keep staged compatibility paths during facade migration. -#![allow(dead_code)] use crate::bucket::metadata_sys::get_versioning_config; use crate::bucket::replication::{ diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index 35aefe053..cc3a1986a 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -15,6 +15,7 @@ use super::*; use crate::io_support::rio::Index; +use std::mem::MaybeUninit; #[cfg(feature = "rio-v2")] const DARE_PAYLOAD_SIZE: i64 = 64 * 1024; @@ -448,10 +449,16 @@ impl GetObjectReader { } enum ReadTransform { - Plain { - visible_offset: usize, - visible_length: i64, - }, + // Written but never read by production code: the enclosing struct already + // carries the same pair as `storage_offset`/`storage_length`. They survive + // as the read plan's test-visible record — four tests assert them by + // literal pattern (`Plain { visible_offset: 6, visible_length: 4 }`), which + // rustc does not count as a read. + #[allow( + dead_code, + reason = "asserted by literal pattern in this file's read-plan tests (backlog#1823)" + )] + Plain { visible_offset: usize, visible_length: i64 }, Compressed { algorithm: CompressionAlgorithm, backend: crate::io_support::rio::ReadCompressionBackend, @@ -922,7 +929,7 @@ struct SkipReader { inner: R, bytes_to_skip: usize, bytes_skipped: usize, - scratch: Vec, + scratch: Box<[MaybeUninit]>, } impl SkipReader { @@ -931,7 +938,7 @@ impl SkipReader { inner, bytes_to_skip, bytes_skipped: 0, - scratch: vec![0u8; 8192], + scratch: Box::<[u8]>::new_uninit_slice(8192), } } } @@ -943,7 +950,7 @@ impl AsyncRead for SkipReader { while this.bytes_skipped < this.bytes_to_skip { let remaining = this.bytes_to_skip - this.bytes_skipped; let scratch_len = remaining.min(this.scratch.len()); - let mut scratch_buf = ReadBuf::new(&mut this.scratch[..scratch_len]); + let mut scratch_buf = ReadBuf::uninit(&mut this.scratch[..scratch_len]); match Pin::new(&mut this.inner).poll_read(cx, &mut scratch_buf) { Poll::Pending => return Poll::Pending, Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), @@ -974,7 +981,7 @@ pub struct RangedDecompressReader target_length: usize, current_offset: usize, bytes_returned: usize, - scratch: Vec, + scratch: Box<[MaybeUninit]>, drain_on_done: bool, drain_task: Option>, } @@ -1012,7 +1019,7 @@ impl RangedDecompressReader { target_length: actual_length, current_offset: 0, bytes_returned: 0, - scratch: vec![0u8; 8192], + scratch: Box::<[u8]>::new_uninit_slice(8192), drain_on_done, drain_task: None, }) @@ -1062,7 +1069,7 @@ impl AsyncRead for RangedDecompres } let scratch_len = std::cmp::min(this.scratch.len(), std::cmp::max(buf_capacity, 1)); - let mut temp_read_buf = ReadBuf::new(&mut this.scratch[..scratch_len]); + let mut temp_read_buf = ReadBuf::uninit(&mut this.scratch[..scratch_len]); let Some(inner) = this.inner.as_mut() else { return Poll::Ready(Ok(())); @@ -1114,7 +1121,8 @@ impl AsyncRead for RangedDecompres ); if bytes_to_return > 0 { - let data_slice = &this.scratch[data_start_in_buffer..data_start_in_buffer + bytes_to_return]; + let data_slice = + &temp_read_buf.filled()[data_start_in_buffer..data_start_in_buffer + bytes_to_return]; buf.put_slice(data_slice); this.bytes_returned += bytes_to_return; @@ -1133,7 +1141,7 @@ impl AsyncRead for RangedDecompres std::cmp::min(n, std::cmp::min(buf.remaining(), this.target_length - this.bytes_returned)); if bytes_to_return > 0 { - buf.put_slice(&this.scratch[..bytes_to_return]); + buf.put_slice(&temp_read_buf.filled()[..bytes_to_return]); this.bytes_returned += bytes_to_return; tracing::trace!("Returned {} bytes at offset {}", bytes_to_return, old_offset); @@ -1203,20 +1211,7 @@ impl AsyncRead for StreamConsumer { impl Drop for StreamConsumer { fn drop(&mut self) { - if self.consumer_task.is_none() && self.inner.is_some() { - let mut inner = self.inner.take().unwrap(); - let task = tokio::spawn(async move { - let mut buf = [0u8; 8192]; - loop { - match inner.read(&mut buf).await { - Ok(0) => break, // EOF - Ok(_) => continue, // Keep consuming - Err(_) => break, // Error, stop consuming - } - } - }); - self.consumer_task = Some(task); - } + self.ensure_consumer_started(); } } @@ -1263,6 +1258,43 @@ mod tests { use temp_env::async_with_vars; use tokio::io::AsyncReadExt; + #[derive(Debug)] + struct PendingPartialReader { + data: &'static [u8], + position: usize, + pending: bool, + } + + impl PendingPartialReader { + fn new(data: &'static [u8]) -> Self { + Self { + data, + position: 0, + pending: true, + } + } + } + + impl AsyncRead for PendingPartialReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if self.pending { + self.pending = false; + cx.waker().wake_by_ref(); + return Poll::Pending; + } + if self.position == self.data.len() { + return Poll::Ready(Ok(())); + } + + let length = buf.remaining().min(3).min(self.data.len() - self.position); + let end = self.position + length; + buf.put_slice(&self.data[self.position..end]); + self.position = end; + self.pending = true; + Poll::Ready(Ok(())) + } + } + const TEST_DIRECT_KEY_HEADER: &str = "x-rustfs-test-direct-key"; const TEST_OBJECT_KEY_HEADER: &str = "x-rustfs-test-object-key"; const TEST_NONCE_HEADER: &str = "x-rustfs-test-nonce"; @@ -1400,6 +1432,36 @@ mod tests { assert_eq!(result, b"World"); } + #[tokio::test] + async fn uninitialized_scratch_preserves_partial_pending_and_eof_reads() { + let mut skipped = SkipReader::new(PendingPartialReader::new(b"0123456789abcdef"), 5); + let mut skipped_output = Vec::new(); + skipped + .read_to_end(&mut skipped_output) + .await + .expect("skip reader should survive partial pending reads through EOF"); + assert_eq!(skipped_output, b"56789abcdef"); + + let mut ranged = RangedDecompressReader::new(PendingPartialReader::new(b"0123456789abcdef"), 5, 7, 16) + .expect("valid range should construct"); + let mut ranged_output = Vec::new(); + ranged + .read_to_end(&mut ranged_output) + .await + .expect("range reader should survive partial pending reads through EOF"); + assert_eq!(ranged_output, b"56789ab"); + } + + #[tokio::test] + async fn uninitialized_skip_scratch_reports_early_eof() { + let mut reader = SkipReader::new(PendingPartialReader::new(b"short"), 6); + let error = reader + .read_to_end(&mut Vec::new()) + .await + .expect_err("EOF before the skip boundary must remain visible"); + assert_eq!(error.kind(), std::io::ErrorKind::UnexpectedEof); + } + #[tokio::test] async fn test_ranged_decompress_reader_from_start() { let original_data = b"Hello, World! This is a test."; diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index ec79567c8..291b9039a 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -172,6 +172,7 @@ impl ObjectLockConfigSnapshot { } } + #[allow(dead_code, reason = "snapshot-scope predicate asserted by this file's tests (backlog#1823)")] pub(crate) fn is_for_store_bucket( &self, store_id: Uuid, diff --git a/crates/ecstore/src/runtime/global.rs b/crates/ecstore/src/runtime/global.rs index f8c0c0100..fcc4411f0 100644 --- a/crates/ecstore/src/runtime/global.rs +++ b/crates/ecstore/src/runtime/global.rs @@ -31,7 +31,6 @@ use std::{ use tokio::sync::{OnceCell, RwLock}; use tokio_util::sync::CancellationToken; use tracing::warn; -use uuid::Uuid; pub const DISK_ASSUME_UNKNOWN_SIZE: u64 = 1 << 30; pub const DISK_MIN_INODES: u64 = 1000; @@ -109,18 +108,6 @@ pub fn set_global_rustfs_port(value: u16) { } } -/// Set the global deployment id -/// -/// # Arguments -/// * `id` - The Uuid to set as the global deployment id -/// -/// # Returns -/// * None -/// -pub fn set_global_deployment_id(id: Uuid) { - current_ctx().set_deployment_id(id); -} - /// Get the global deployment id /// /// # Returns @@ -288,19 +275,6 @@ pub fn get_global_region() -> Option { current_ctx().region() } -/// Initialize the global background services cancellation token -/// -/// # Arguments -/// * `cancel_token` - The CancellationToken instance to set globally -/// -/// # Returns -/// * `Ok(())` if successful -/// * `Err(CancellationToken)` if setting fails -/// -pub fn init_background_services_cancel_token(cancel_token: CancellationToken) -> Result<(), CancellationToken> { - current_ctx().init_background_cancel_token(cancel_token) -} - /// Get the global background services cancellation token /// /// # Returns @@ -310,18 +284,6 @@ pub fn get_background_services_cancel_token() -> Option { current_ctx().background_cancel_token() } -/// Create and initialize the global background services cancellation token -/// -/// # Returns -/// * `CancellationToken` - The newly created global cancellation token -/// -pub fn create_background_services_cancel_token() -> CancellationToken { - let cancel_token = CancellationToken::new(); - init_background_services_cancel_token(cancel_token.clone()) - .expect("background services cancel token should be initialized once during startup"); - cancel_token -} - /// Shutdown all background services gracefully /// /// # Returns diff --git a/crates/ecstore/src/runtime/instance.rs b/crates/ecstore/src/runtime/instance.rs index 95ff16ffe..71a1898cf 100644 --- a/crates/ecstore/src/runtime/instance.rs +++ b/crates/ecstore/src/runtime/instance.rs @@ -402,6 +402,10 @@ impl InstanceContext { } #[cfg(test)] + #[allow( + dead_code, + reason = "driven by the tier-delete-journal recovery test behind `--features test-util` (backlog#1823)" + )] pub(crate) fn wake_tier_delete_journal_recovery(&self) { self.tier_delete_journal_recovery_wakeup.notify_one(); } diff --git a/crates/ecstore/src/runtime/mod.rs b/crates/ecstore/src/runtime/mod.rs index 81812cac3..9dd84d401 100644 --- a/crates/ecstore/src/runtime/mod.rs +++ b/crates/ecstore/src/runtime/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: runtime source migration keeps fallback handles until all owners inject state. -#![allow(dead_code)] pub(crate) mod global; pub(crate) mod instance; diff --git a/crates/ecstore/src/runtime/sources.rs b/crates/ecstore/src/runtime/sources.rs index ed35c14d3..26a3e296c 100644 --- a/crates/ecstore/src/runtime/sources.rs +++ b/crates/ecstore/src/runtime/sources.rs @@ -38,7 +38,6 @@ use crate::{ set_object_layer, update_erasure_type, }, services::batch_processor::{GlobalBatchProcessors, get_global_processors}, - services::event_notification::EventNotifier, services::notification_sys::{NotificationSys, get_global_notification_sys}, services::tier::tier::TierConfigMgr, store::ECStore, @@ -143,6 +142,10 @@ pub async fn setup_is_erasure_sd() -> bool { is_erasure_sd().await } +#[allow( + dead_code, + reason = "setup-type override used only by tests across this crate (backlog#1823)" +)] pub(crate) async fn current_setup_type() -> SetupType { if setup_is_dist_erasure().await { SetupType::DistErasure @@ -155,6 +158,10 @@ pub(crate) async fn current_setup_type() -> SetupType { } } +#[allow( + dead_code, + reason = "setup-type override used only by tests across this crate (backlog#1823)" +)] pub(crate) async fn set_setup_type(setup_type: SetupType) { update_erasure_type(setup_type).await; } @@ -232,14 +239,6 @@ pub(crate) fn ensure_test_rpc_secret() { let _ = rustfs_credentials::set_global_rpc_secret(TEST_RPC_SECRET.to_owned()); } -pub(crate) fn storage_class_parity(storage_class: Option<&str>) -> Option { - get_global_storage_class_snapshot().get_parity_for_sc(storage_class.unwrap_or_default()) -} - -pub(crate) fn storage_class_should_inline(shard_size: i64, versioned: bool) -> bool { - get_global_storage_class_snapshot().should_inline(shard_size, versioned) -} - pub(crate) fn deployment_upload_id(upload_id: &str) -> String { base64_simd::URL_SAFE_NO_PAD .encode_to_string(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_id).as_bytes()) @@ -332,21 +331,6 @@ pub(crate) fn storage_class_config_snapshot() -> Arc { get_global_storage_class_snapshot() } -/// Scalar STANDARD / RRS parity for backend-info reporting. -/// -/// Retained for the rebalance/backend-info path. `get_parity_for_sc` returns -/// `None` when the runtime config is uninitialized or (post per-pool support) -/// when pools disagree, so STANDARD falls back to the caller's default and RRS -/// stays `None` — matching the pre-per-pool scalar reporting. -pub(crate) fn backend_storage_class_parities(default_standard_parity: usize) -> (Option, Option) { - let sc = get_global_storage_class_snapshot(); - let standard = sc - .get_parity_for_sc(storageclass::CLASS_STANDARD) - .or(Some(default_standard_parity)); - let reduced_redundancy = sc.get_parity_for_sc(storageclass::RRS); - (standard, reduced_redundancy) -} - pub(crate) fn set_storage_class_config(config: storageclass::Config) { set_global_storage_class(config); } @@ -414,10 +398,6 @@ pub fn transition_state_handle() -> Arc { crate::runtime::global::current_ctx().transition_state() } -pub(crate) fn event_notifier_handle() -> Arc> { - crate::runtime::global::current_ctx().event_notifier() -} - pub(crate) async fn local_disk_by_path(path: &str) -> Option { local_disk_map_handle().read().await.get(path).cloned().flatten() } @@ -511,30 +491,6 @@ pub(crate) async fn local_disk_set_drive( instance_ctx.local_disk_set_drives().read().await[pool_idx][set_idx][disk_idx].clone() } -pub(crate) async fn local_disk_for_endpoint(endpoint: &Endpoint) -> Option { - let set_drives = local_disk_set_drives_handle(); - let global_set_drives = set_drives.read().await; - if global_set_drives.is_empty() { - return local_disk_map_handle() - .read() - .await - .get(&endpoint.to_string()) - .cloned() - .unwrap_or(None); - } - - let pool_idx = usize::try_from(endpoint.pool_idx).ok()?; - let set_idx = usize::try_from(endpoint.set_idx).ok()?; - let disk_idx = usize::try_from(endpoint.disk_idx).ok()?; - - global_set_drives - .get(pool_idx) - .and_then(|sets| sets.get(set_idx)) - .and_then(|disks| disks.get(disk_idx)) - .cloned() - .unwrap_or(None) -} - pub(crate) async fn local_disk_paths() -> Vec { local_disk_map_handle().read().await.keys().cloned().collect() } diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index b08211853..c63dbc401 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -206,6 +206,38 @@ pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStat }) } +#[cfg(test)] +pub(crate) struct RemoteVersionStateFleetProofGuard; + +#[cfg(test)] +impl Drop for RemoteVersionStateFleetProofGuard { + fn drop(&mut self) { + replace_remote_version_state_fleet_proof(None); + } +} + +#[cfg(test)] +pub(crate) fn install_remote_version_state_fleet_proof_for_test(topology_fingerprint: &str) -> RemoteVersionStateFleetProofGuard { + match REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.to_string()) { + Ok(()) => {} + Err(_) + if REMOTE_VERSION_STATE_PROBE_TOPOLOGY + .get() + .is_some_and(|current| current == topology_fingerprint) => {} + Err(_) => panic!("remote version state test topology is already bound to another fingerprint"), + } + let peer_epochs = BTreeMap::new(); + if let Some(err) = publish_remote_version_state_probe_result( + remote_version_state_fleet_proof_slot(), + topology_fingerprint, + Ok(peer_epochs), + Instant::now(), + ) { + panic!("test proof installation must not fail: {err}"); + } + RemoteVersionStateFleetProofGuard +} + fn remote_version_state_fleet_proof_valid_at( proof: Option<&RemoteVersionStateFleetProof>, expected_topology: &str, diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 944c3858e..1fd1b02d5 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -39,8 +39,8 @@ use crate::diagnostics::get::{ GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, - GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, - GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, + GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, + GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure, @@ -70,6 +70,57 @@ use std::{ task::{Context, Poll}, time::{Duration, Instant}, }; + +fn metadata_metrics_path(bucket: &str) -> &'static str { + if crate::bucket::utils::is_meta_bucketname(bucket) { + GET_OBJECT_PATH_INTERNAL_META + } else { + GET_OBJECT_PATH_LEGACY_DUPLEX + } +} + +fn metadata_distribution_key(bucket: &str, object: &str) -> String { + [bucket, object].join("/") +} + +pub(in crate::set_disk) fn bounded_metadata_fanout_order( + bucket: &str, + object: &str, + total_disks: usize, + default_parity_count: usize, +) -> Vec { + let fallback_order = || (0..total_disks).collect::>(); + if default_parity_count == 0 || default_parity_count >= total_disks { + return fallback_order(); + } + + let data_blocks = total_disks - default_parity_count; + let distribution_key = metadata_distribution_key(bucket, object); + let distribution = FileInfo::new(&distribution_key, data_blocks, default_parity_count) + .erasure + .distribution; + if distribution.len() != total_disks { + return fallback_order(); + } + + let mut order = Vec::with_capacity(total_disks); + for block_index in 1..=data_blocks { + let Some(disk_index) = distribution + .iter() + .position(|distributed_block| *distributed_block == block_index) + else { + return fallback_order(); + }; + order.push(disk_index); + } + order.extend( + distribution + .iter() + .enumerate() + .filter_map(|(disk_index, block_index)| (*block_index > data_blocks).then_some(disk_index)), + ); + order +} use tokio::io::{AsyncRead, ReadBuf}; use tokio::sync::RwLock; use tokio::task::JoinSet; @@ -600,6 +651,83 @@ pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo && left.erasure.distribution == right.erasure.distribution } +pub(in crate::set_disk) async fn data_read_early_stop_inline_body_verified( + bucket: &str, + object: &str, + candidate: &FileInfo, + parts_metadata: &[FileInfo], + disks: &[Option], +) -> bool { + if !candidate.inline_data() + || candidate.is_compressed() + || candidate + .metadata + .keys() + .any(|key| rustfs_utils::http::is_object_encryption_marker(key)) + || candidate.is_remote() + || candidate.deleted + || candidate.size <= 0 + || candidate.parts.len() != 1 + || !candidate.has_valid_erasure_geometry() + { + return false; + } + + let Ok(object_size) = usize::try_from(candidate.size) else { + return false; + }; + if candidate.parts.first().is_none_or(|part| part.size != object_size) { + return false; + } + if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) { + return false; + } + + let Ok(erasure) = coding::Erasure::try_new_with_options( + candidate.erasure.data_blocks, + candidate.erasure.parity_blocks, + candidate.erasure.block_size, + candidate.uses_legacy_checksum, + ) else { + return false; + }; + let Some(data_files) = + collect_inline_data_shard_fileinfos_by_index(parts_metadata, candidate, erasure.data_shards, |index| { + disks.get(index).is_some_and(Option::is_some) + }) + else { + return false; + }; + + let Some(part) = candidate.parts.first() else { + return false; + }; + let checksum_info = candidate.erasure.get_checksum_info(part.number); + let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S { + HashAlgorithm::HighwayHash256SLegacy + } else { + checksum_info.algorithm + }; + let read_length = inline_erasure_shard_file_offset( + 0, + object_size, + object_size, + candidate.erasure.block_size, + erasure.data_shards, + candidate.uses_legacy_checksum, + ); + let shard_size = inline_erasure_shard_size(candidate.erasure.block_size, erasure.data_shards, candidate.uses_legacy_checksum); + let Ok(mut readers) = + build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await + else { + return false; + }; + + try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size) + .await + .is_some_and(|body| body.len() == object_size) +} + pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str { match err { DiskError::FileNotFound | DiskError::VolumeNotFound => GET_METADATA_RESPONSE_NOT_FOUND, @@ -2187,11 +2315,12 @@ impl SetDisks { .await; } if early_stop_enabled { + let metrics_path = metadata_metrics_path(bucket); rustfs_io_metrics::record_get_object_metadata_early_stop_miss( - GET_OBJECT_PATH_LEGACY_DUPLEX, + metrics_path, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, ); - rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(GET_OBJECT_PATH_LEGACY_DUPLEX, 0); + rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0); } Self::read_all_fileinfo_full_wait( @@ -2224,6 +2353,7 @@ impl SetDisks { let mut ress = Vec::with_capacity(disks.len()); let mut errors = Vec::with_capacity(disks.len()); let mut observations = observe.then(|| Vec::with_capacity(disks.len())); + let scheduled_count = disks.len(); let opts = ReadOptions { incl_free_versions, read_data, @@ -2289,6 +2419,14 @@ impl SetDisks { (Some(fanout_start), Some(observations)) => MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations), _ => MetadataFanoutDiagnostics::default(), }; + if observe { + rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle( + metadata_metrics_path(bucket.as_ref()), + scheduled_count, + scheduled_count, + 0, + ); + } Ok((ress, errors, diagnostics)) } @@ -2319,9 +2457,17 @@ impl SetDisks { let bucket: Arc = Arc::from(bucket); let object: Arc = Arc::from(object); let version_id: Arc = Arc::from(version_id); + let metrics_path = metadata_metrics_path(bucket.as_ref()); let mut join_set = JoinSet::new(); let bounded_fanout = is_get_metadata_early_stop_bounded_fanout_enabled(); - let mut next_disk_index = 0usize; + let fanout_order = if bounded_fanout { + bounded_metadata_fanout_order(bucket.as_ref(), object.as_ref(), disks.len(), default_parity_count) + } else { + Vec::new() + }; + let mut next_fanout_index = 0usize; + let mut scheduled_count = 0usize; + let mut force_full_wait = false; let spawn_read_version = |join_set: &mut JoinSet<(usize, disk::error::Result, Duration)>, index: usize, disk: Option| { let task_opts = opts; @@ -2332,6 +2478,8 @@ impl SetDisks { join_set.spawn(async move { let response_start = Instant::now(); let result = if let Some(disk) = disk { + #[allow(clippy::let_unit_value)] + let _fanout_task_guard = Self::rename_fanout_task_guard(&object); Self::record_read_version_call(&object, index); #[cfg(test)] Self::read_version_fanout_barrier(&object, index).await; @@ -2346,15 +2494,18 @@ impl SetDisks { if bounded_fanout { let initial_target = accumulator.default_write_quorum().min(disks.len()); - while next_disk_index < initial_target { - if let Some(disk) = disks.get(next_disk_index).cloned() { - spawn_read_version(&mut join_set, next_disk_index, disk); + while next_fanout_index < initial_target { + let disk_index = fanout_order[next_fanout_index]; + if let Some(disk) = disks.get(disk_index).cloned() { + spawn_read_version(&mut join_set, disk_index, disk); + scheduled_count = scheduled_count.saturating_add(1); } - next_disk_index = next_disk_index.saturating_add(1); + next_fanout_index = next_fanout_index.saturating_add(1); } } else { for (index, disk) in disks.iter().cloned().enumerate() { spawn_read_version(&mut join_set, index, disk); + scheduled_count = scheduled_count.saturating_add(1); } } @@ -2383,46 +2534,87 @@ impl SetDisks { } } - if let Some(decision) = accumulator - .early_stop_decision() - .or_else(|| accumulator.version_early_stop_decision()) + if !force_full_wait + && let Some(decision) = accumulator + .early_stop_decision() + .or_else(|| accumulator.version_early_stop_decision()) { - let saved_responses = if bounded_fanout { - disks.len().saturating_sub(observations.len()) + let should_return_early = if read_data { + let allow_data_read_early_stop = match accumulator.candidate.as_ref() { + Some(candidate) => { + data_read_early_stop_inline_body_verified(bucket.as_ref(), object.as_ref(), candidate, &ress, disks) + .await + } + None => false, + }; + if !allow_data_read_early_stop { + force_full_wait = true; + } + allow_data_read_early_stop } else { - join_set.len() + true }; - join_set.abort_all(); - rustfs_io_metrics::record_get_object_metadata_early_stop_hit(GET_OBJECT_PATH_LEGACY_DUPLEX, decision.reason); - rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses( - GET_OBJECT_PATH_LEGACY_DUPLEX, - saved_responses, - ); - while join_set.join_next().await.is_some() {} - let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations); - return Ok((ress, errors, diagnostics)); + + if should_return_early { + let saved_responses = if bounded_fanout { + disks.len().saturating_sub(observations.len()) + } else { + join_set.len() + }; + join_set.abort_all(); + rustfs_io_metrics::record_get_object_metadata_early_stop_hit(metrics_path, decision.reason); + rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, saved_responses); + let mut cancelled_count = 0usize; + while let Some(join_result) = join_set.join_next().await { + match join_result { + Err(join_error) if join_error.is_cancelled() => { + cancelled_count = cancelled_count.saturating_add(1); + } + _ => {} + } + } + rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle( + metrics_path, + scheduled_count, + scheduled_count.saturating_sub(cancelled_count), + cancelled_count, + ); + let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations); + return Ok((ress, errors, diagnostics)); + } } let pending_responses = join_set.len(); - let should_hedge_single_pending_data_read = - read_data && pending_responses == 1 && accumulator.can_still_reach_early_stop_with_pending(pending_responses); - if bounded_fanout - && next_disk_index < disks.len() + let should_hedge_single_pending_data_read = read_data + && !force_full_wait + && pending_responses == 1 + && accumulator.can_still_reach_early_stop_with_pending(pending_responses); + if bounded_fanout && force_full_wait { + while next_fanout_index < disks.len() { + let disk_index = fanout_order[next_fanout_index]; + if let Some(disk) = disks.get(disk_index).cloned() { + spawn_read_version(&mut join_set, disk_index, disk); + scheduled_count = scheduled_count.saturating_add(1); + } + next_fanout_index = next_fanout_index.saturating_add(1); + } + } else if bounded_fanout + && next_fanout_index < disks.len() && (!accumulator.can_still_reach_early_stop_with_pending(pending_responses) || should_hedge_single_pending_data_read) { - if let Some(disk) = disks.get(next_disk_index).cloned() { - spawn_read_version(&mut join_set, next_disk_index, disk); + let disk_index = fanout_order[next_fanout_index]; + if let Some(disk) = disks.get(disk_index).cloned() { + spawn_read_version(&mut join_set, disk_index, disk); + scheduled_count = scheduled_count.saturating_add(1); } - next_disk_index = next_disk_index.saturating_add(1); + next_fanout_index = next_fanout_index.saturating_add(1); } } - rustfs_io_metrics::record_get_object_metadata_early_stop_miss( - GET_OBJECT_PATH_LEGACY_DUPLEX, - accumulator.final_miss_reason(), - ); - rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(GET_OBJECT_PATH_LEGACY_DUPLEX, 0); + rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, accumulator.final_miss_reason()); + rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0); + rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0); let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations); Ok((ress, errors, diagnostics)) } @@ -2933,17 +3125,20 @@ impl SetDisks { let results = fanout.await.map_err(|_| DiskError::Unexpected)?; for (idx, result) in results.iter().enumerate() { - match result.as_ref().map_err(|_| DiskError::Unexpected)? { - Ok(res) => { + match result { + Ok(Ok(res)) => { data_dirs[idx] = res.rollback_data_dir.or(res.old_data_dir); cleanup_data_dirs[idx] = res.cleanup_data_dir; disk_versions[idx].clone_from(&res.sign); old_current_sizes[idx] = res.old_current_size; errs.push(None); } - Err(e) => { + Ok(Err(e)) => { errs.push(Some(e.clone())); } + Err(_) => { + errs.push(Some(DiskError::Unexpected)); + } } } @@ -5078,7 +5273,7 @@ mod tests { use super::*; use std::io::Cursor; use tempfile::TempDir; - use tokio::io::AsyncReadExt; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; #[test] fn write_precondition_lookup_errors_fail_closed_unless_absence_is_known() { @@ -5408,39 +5603,598 @@ mod tests { } } + async fn inline_metadata_fanout_fileinfos_with_mode( + bucket: &str, + object: &str, + payload: &[u8], + uses_legacy_checksum: bool, + ) -> Vec { + let distribution_key = metadata_distribution_key(bucket, object); + let mut base = FileInfo::new(&distribution_key, 2, 2); + base.volume = bucket.to_string(); + base.name = object.to_string(); + base.size = i64::try_from(payload.len()).expect("test payload should fit i64"); + base.is_latest = true; + base.version_id = Some(Uuid::new_v4()); + base.data_dir = Some(Uuid::new_v4()); + base.mod_time = Some(OffsetDateTime::now_utc()); + base.metadata.insert("etag".to_string(), "etag-inline".to_string()); + base.add_object_part(1, "part-etag-inline".to_string(), payload.len(), base.mod_time, base.size, None, None); + base.set_inline_data(); + base.uses_legacy_checksum = uses_legacy_checksum; + + let erasure = coding::Erasure::new_with_options( + base.erasure.data_blocks, + base.erasure.parity_blocks, + base.erasure.block_size, + base.uses_legacy_checksum, + ); + let shards = erasure.encode_data(payload).expect("inline payload should encode"); + let checksum_algo = if base.uses_legacy_checksum { + HashAlgorithm::HighwayHash256SLegacy + } else { + HashAlgorithm::HighwayHash256S + }; + + let mut files = Vec::with_capacity(shards.len()); + for (index, shard) in shards.into_iter().enumerate() { + let mut writer = coding::BitrotWriterWrapper::new( + coding::CustomWriter::new_inline_buffer(), + erasure.shard_size(), + checksum_algo.clone(), + ); + writer.write(&shard).await.expect("inline shard should write"); + writer.shutdown().await.expect("inline writer should shutdown"); + let mut fi = base.clone(); + fi.erasure.index = index + 1; + fi.data = Some(Bytes::from( + writer + .into_inline_data() + .expect("inline bitrot writer should retain encoded data"), + )); + files.push(fi); + } + files + } + + async fn inline_metadata_fanout_fileinfos(bucket: &str, object: &str, payload: &[u8]) -> Vec { + inline_metadata_fanout_fileinfos_with_mode(bucket, object, payload, false).await + } + + async fn install_inline_metadata_fanout_fileinfo( + disks: &[Option], + bucket: &str, + object: &str, + payload: &[u8], + mutate: impl FnOnce(&mut [FileInfo]), + ) { + let mut files = inline_metadata_fanout_fileinfos(bucket, object, payload).await; + mutate(&mut files); + install_inline_metadata_fanout_files(disks, bucket, object, files).await; + } + + async fn install_inline_metadata_fanout_files(disks: &[Option], bucket: &str, object: &str, files: Vec) { + let distribution = files + .first() + .map(|file| file.erasure.distribution.clone()) + .expect("inline metadata fixture should include shards"); + for (disk_index, disk) in disks + .iter() + .enumerate() + .filter_map(|(disk_index, disk)| disk.as_ref().map(|disk| (disk_index, disk))) + { + let block_index = distribution + .get(disk_index) + .copied() + .expect("inline metadata fixture should cover every disk"); + let file_info = files + .get(block_index.checked_sub(1).expect("erasure block indexes are one-based")) + .expect("inline metadata fixture should include every distributed shard") + .clone(); + disk.write_metadata(bucket, bucket, object, file_info) + .await + .expect("inline metadata should be installed on every disk"); + } + } + + fn object_with_initial_data_shards(bucket: &str, prefix: &str, data_shards: usize, initial_fanout: usize) -> String { + (0..1000) + .map(|index| format!("{prefix}-{index}")) + .find(|name| { + let order = bounded_metadata_fanout_order(bucket, name, data_shards + 2, 2); + let distribution_key = metadata_distribution_key(bucket, name); + let distribution = FileInfo::new(&distribution_key, data_shards, 2).erasure.distribution; + order + .iter() + .take(initial_fanout) + .filter_map(|disk_index| distribution.get(*disk_index).copied()) + .filter(|block_index| (1..=data_shards).contains(block_index)) + .collect::>() + .len() + == data_shards + }) + .expect("test should find an object whose initial fanout covers every data shard") + } + + fn initial_data_shard_indexes(bucket: &str, object: &str, data_shards: usize, initial_fanout: usize) -> Vec { + let order = bounded_metadata_fanout_order(bucket, object, data_shards + 2, 2); + let distribution_key = metadata_distribution_key(bucket, object); + let distribution = FileInfo::new(&distribution_key, data_shards, 2).erasure.distribution; + order + .iter() + .take(initial_fanout) + .filter_map(|disk_index| distribution.get(*disk_index).copied()) + .filter(|block_index| (1..=data_shards).contains(block_index)) + .collect() + } + + fn bounded_spare_disk_index(bucket: &str, object: &str, data_shards: usize, parity_shards: usize) -> usize { + let total_disks = data_shards + parity_shards; + let initial_fanout = if data_shards == parity_shards { + data_shards + 1 + } else { + data_shards + }; + *bounded_metadata_fanout_order(bucket, object, total_disks, parity_shards) + .get(initial_fanout) + .expect("test geometry should leave one bounded spare disk") + } + + #[test] + fn bounded_metadata_fanout_order_prioritizes_default_data_shards() { + let bucket = "bounded-order-bucket"; + let object = "bounded-order-data-shards-first"; + let order = bounded_metadata_fanout_order(bucket, object, 16, 4); + let distribution_key = [bucket, object].join("/"); + let distribution = FileInfo::new(&distribution_key, 12, 4).erasure.distribution; + let initial_blocks: HashSet<_> = order + .iter() + .take(12) + .filter_map(|disk_index| distribution.get(*disk_index).copied()) + .collect(); + + assert_eq!(order.len(), 16); + assert_eq!(order.iter().copied().collect::>().len(), 16); + assert_eq!(initial_blocks, (1..=12).collect()); + } + + #[test] + fn bounded_metadata_fanout_order_uses_written_bucket_object_key() { + let bucket = "bounded-order-rotation-bucket"; + let (object, bare_distribution, stored_distribution) = (0..1000) + .map(|index| format!("bounded-order-rotation-object-{index}")) + .find_map(|object| { + let bare_distribution = FileInfo::new(&object, 12, 4).erasure.distribution; + let stored_key = [bucket, object.as_str()].join("/"); + let stored_distribution = FileInfo::new(&stored_key, 12, 4).erasure.distribution; + (bare_distribution != stored_distribution).then_some((object, bare_distribution, stored_distribution)) + }) + .expect("test should find a bucket/object pair with a different distribution rotation"); + + let order = bounded_metadata_fanout_order(bucket, &object, 16, 4); + let initial_stored_blocks: HashSet<_> = order + .iter() + .take(12) + .filter_map(|disk_index| stored_distribution.get(*disk_index).copied()) + .collect(); + let initial_bare_blocks: HashSet<_> = order + .iter() + .take(12) + .filter_map(|disk_index| bare_distribution.get(*disk_index).copied()) + .collect(); + + assert_eq!(initial_stored_blocks, (1..=12).collect()); + assert_ne!( + initial_bare_blocks, + (1..=12).collect(), + "test fixture must prove the bare object distribution would pick the wrong initial data-shard set" + ); + } + #[tokio::test] - async fn bounded_metadata_early_stop_ab_hedges_data_get_read_version_fanout() { - const DISKS: usize = 4; - let bucket = "bounded-data-get-fanout-bucket"; - let control_object = "bounded-data-get-control-object"; - let treatment_object = "bounded-data-get-treatment-object"; + async fn bounded_metadata_early_stop_non_inline_fallback_schedules_beyond_initial_quorum() { + const DISKS: usize = 6; + let bucket = "bounded-data-get-six-disk-fanout-bucket"; + let object = "bounded-data-get-six-disk-object"; let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; - install_metadata_fanout_fileinfo(&disks, bucket, control_object, None).await; - install_metadata_fanout_fileinfo(&disks, bucket, treatment_object, None).await; + install_metadata_fanout_fileinfo(&disks, bucket, object, None).await; temp_env::async_with_vars( [ ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), - ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("false")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), ], async { - let calls = disk_call_counters::observe(control_object); - let (_, _, diagnostics) = - SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, control_object, "", true, false, false, true, 2) + let calls = disk_call_counters::observe(object); + let (parts_metadata, errs, diagnostics) = + SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 3) .await - .expect("control metadata should resolve"); + .expect("non-inline metadata should resolve after force-full-wait fallback"); assert_eq!( calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64, - "control path should keep full fanout when data-read early stop is explicitly disabled" + "force-full-wait fallback must schedule disks beyond the initial write quorum" ); assert_eq!(diagnostics.total_responses(), DISKS); + assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); + assert!(errs.iter().all(Option::is_none)); }, ) .await; + drop(dirs); + } + + #[tokio::test] + async fn bounded_metadata_early_stop_allows_verified_inline_data_get_quorum() { + const DISKS: usize = 4; + let bucket = "bounded-inline-data-get-fanout-bucket"; + let object = object_with_initial_data_shards(bucket, "bounded-inline-data-get-object", 2, 3); + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"verified inline payload", |_| {}).await; + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let spare_disk = bounded_spare_disk_index(bucket, &object, 2, 2); + let barrier = rename_fanout_barrier::arm(&object, spare_disk, rename_fanout_barrier::PHASE_READ_VERSION); + let tracker = rename_fanout_barrier::observe_tasks(&object); + let calls = disk_call_counters::observe(&object); + let disks_for_read = disks.clone(); + let object_for_read = object.clone(); + let mut read = tokio::spawn(async move { + SetDisks::read_all_fileinfo_observed( + &disks_for_read, + bucket, + bucket, + &object_for_read, + "", + true, + false, + false, + true, + 2, + ) + .await + }); + + tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused()) + .await + .expect("bounded fanout should hedge and pause the spare disk"); + let (parts_metadata, errs, diagnostics) = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read) + .await + .expect("verified inline quorum should return without the paused spare") + .expect("read task should join") + .expect("verified inline metadata should reach early-stop quorum"); + + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + DISKS as u64, + "bounded fanout may schedule one spare before the verified inline quorum returns" + ); + assert_eq!( + tracker.running(), + 0, + "early-stop should drain spawned read_version tasks before returning" + ); + assert_eq!(diagnostics.total_responses(), 3); + assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), 3); + assert!(errs.iter().all(Option::is_none)); + }, + ) + .await; + + drop(dirs); + } + + #[tokio::test] + async fn data_read_early_stop_verifies_legacy_inline_checksum_payload() { + let bucket = "legacy-inline-data-get-fanout-bucket"; + let object = "legacy-inline-data-get-object"; + let payload = b"legacy inline payload whose size is not divisible by the data shard count"; + let (_dirs, disks) = call_counter_local_disks(bucket, 4).await; + let files = inline_metadata_fanout_fileinfos_with_mode(bucket, object, payload, true).await; + let distribution = files + .first() + .map(|file| file.erasure.distribution.clone()) + .expect("legacy fixture should include metadata"); + let order = bounded_metadata_fanout_order(bucket, object, 4, 2); + let mut parts_metadata = vec![FileInfo::default(); 4]; + for disk_index in order.into_iter().take(3) { + let block_index = distribution + .get(disk_index) + .copied() + .expect("legacy fixture distribution should cover every disk"); + parts_metadata[disk_index] = files + .get(block_index.checked_sub(1).expect("erasure block indexes are one-based")) + .expect("legacy fixture should include every distributed shard") + .clone(); + } + let candidate = parts_metadata + .iter() + .find(|file| file.name == object) + .expect("legacy fixture should include observed metadata") + .clone(); + + assert!( + data_read_early_stop_inline_body_verified(bucket, object, &candidate, &parts_metadata, &disks).await, + "legacy inline metadata must use the legacy bitrot shard sizing and checksum algorithm" + ); + } + + #[test] + #[serial_test::serial] + fn metadata_fanout_lifecycle_records_real_early_stop_abort() { + assert_metadata_fanout_lifecycle_records_real_early_stop_abort( + "lifecycle-inline-data-get-fanout-bucket", + "lifecycle-inline-data-get-object", + GET_OBJECT_PATH_LEGACY_DUPLEX, + None, + ); + } + + #[test] + #[serial_test::serial] + fn metadata_fanout_lifecycle_records_bounded_early_stop_abort() { + assert_metadata_fanout_lifecycle_records_real_early_stop_abort( + "lifecycle-bounded-inline-data-get-fanout-bucket", + "lifecycle-bounded-inline-data-get-object", + GET_OBJECT_PATH_LEGACY_DUPLEX, + Some("true"), + ); + } + + #[test] + #[serial_test::serial] + fn metadata_fanout_lifecycle_records_internal_meta_early_stop_abort_path() { + assert_metadata_fanout_lifecycle_records_real_early_stop_abort( + RUSTFS_META_BUCKET, + "buckets/.usage-cache/lifecycle-inline-data-get-object", + GET_OBJECT_PATH_INTERNAL_META, + None, + ); + } + + #[test] + #[serial_test::serial] + fn metadata_fanout_records_internal_meta_final_miss_path() { + const DISKS: usize = 4; + let bucket = RUSTFS_META_BUCKET; + let object = object_with_initial_data_shards(bucket, "buckets/.usage-cache/final-miss-inline-data-get-object", 2, 3); + let corrupt_shard = initial_data_shard_indexes(bucket, &object, 2, 3)[0]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime should build"); + let recorder = crate::test_metrics::CapturingRecorder::default(); + let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled(); + rustfs_io_metrics::set_get_stage_metrics_enabled(true); + + metrics::with_local_recorder(&recorder, || { + runtime.block_on(async { + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"verified inline payload", |files| { + if let Some(data) = files.get_mut(corrupt_shard - 1).and_then(|file| file.data.as_mut()) { + let mut corrupt = data.to_vec(); + corrupt[0] ^= 0x01; + *data = Bytes::from(corrupt); + } + }) + .await; + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let (parts_metadata, errs, diagnostics) = SetDisks::read_all_fileinfo_observed( + &disks, bucket, bucket, &object, "", true, false, false, true, 2, + ) + .await + .expect("corrupt internal inline metadata should fall back to full fanout"); + + assert_eq!(diagnostics.total_responses(), DISKS); + assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); + assert!(errs.iter().all(Option::is_none)); + }, + ) + .await; + drop(dirs); + }); + }); + rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate); + + assert_eq!( + recorder.counter_value( + "rustfs_io_get_object_metadata_early_stop_total", + &[ + ("path", GET_OBJECT_PATH_INTERNAL_META), + ("decision", "miss"), + ("reason", GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM), + ], + ), + 1, + "internal metadata final early-stop miss must retain its path label" + ); + assert_eq!( + recorder.counter_value( + "rustfs_io_get_object_metadata_early_stop_total", + &[ + ("path", GET_OBJECT_PATH_LEGACY_DUPLEX), + ("decision", "miss"), + ("reason", GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM), + ], + ), + 0, + "internal metadata final early-stop miss must not leak into legacy_duplex" + ); + assert_eq!( + recorder.histogram_values( + "rustfs_io_get_object_metadata_early_stop_saved_responses", + &[("path", GET_OBJECT_PATH_INTERNAL_META)] + ), + vec![0.0], + "internal metadata final miss must record zero saved responses on internal_meta" + ); + assert!( + recorder + .histogram_values( + "rustfs_io_get_object_metadata_early_stop_saved_responses", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] + ) + .is_empty(), + "internal metadata final miss saved responses must not leak into legacy_duplex" + ); + } + + fn assert_metadata_fanout_lifecycle_records_real_early_stop_abort( + bucket: &'static str, + object_prefix: &str, + expected_path: &'static str, + bounded_fanout_env: Option<&'static str>, + ) { + const DISKS: usize = 4; + let object = object_with_initial_data_shards(bucket, object_prefix, 2, 3); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime should build"); + let recorder = crate::test_metrics::CapturingRecorder::default(); + let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled(); + rustfs_io_metrics::set_get_stage_metrics_enabled(true); + + metrics::with_local_recorder(&recorder, || { + runtime.block_on(async { + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"verified inline payload", |_| {}).await; + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", bounded_fanout_env), + ], + async { + let barrier_disk = if bounded_fanout_env.is_some() { + bounded_spare_disk_index(bucket, &object, 2, 2) + } else { + 3 + }; + let barrier = + rename_fanout_barrier::arm(&object, barrier_disk, rename_fanout_barrier::PHASE_READ_VERSION); + let disks_for_read = disks.clone(); + let object_for_read = object.clone(); + let mut read = tokio::spawn(async move { + SetDisks::read_all_fileinfo_observed( + &disks_for_read, + bucket, + bucket, + &object_for_read, + "", + true, + false, + false, + true, + 2, + ) + .await + }); + + tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused()) + .await + .expect("metadata fanout should pause the spare metadata task"); + let (parts_metadata, errs, diagnostics) = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read) + .await + .expect("verified inline quorum should abort the paused eager fanout") + .expect("read task should join") + .expect("verified inline metadata should reach early-stop quorum"); + + assert_eq!(diagnostics.total_responses(), 3); + assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), 3); + assert!(errs.iter().all(Option::is_none)); + }, + ) + .await; + drop(dirs); + }); + }); + rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate); + + assert_eq!( + recorder.histogram_values("rustfs_io_get_object_metadata_fanout_scheduled", &[("path", expected_path)]), + vec![4.0] + ); + assert_eq!( + recorder.histogram_values("rustfs_io_get_object_metadata_fanout_completed", &[("path", expected_path)]), + vec![3.0] + ); + assert_eq!( + recorder.histogram_values("rustfs_io_get_object_metadata_fanout_cancelled", &[("path", expected_path)]), + vec![1.0] + ); + assert_eq!( + recorder.counter_value( + "rustfs_io_get_object_metadata_early_stop_total", + &[ + ("path", expected_path), + ("decision", "hit"), + ("reason", GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM), + ], + ), + 1, + "early-stop hit must retain its path label" + ); + let unexpected_path = if expected_path == GET_OBJECT_PATH_INTERNAL_META { + GET_OBJECT_PATH_LEGACY_DUPLEX + } else { + GET_OBJECT_PATH_INTERNAL_META + }; + assert_eq!( + recorder.counter_value( + "rustfs_io_get_object_metadata_early_stop_total", + &[ + ("path", unexpected_path), + ("decision", "hit"), + ("reason", GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM), + ], + ), + 0, + "early-stop hit must not leak into the other metadata path" + ); + assert_eq!( + recorder.histogram_values("rustfs_io_get_object_metadata_early_stop_saved_responses", &[("path", expected_path)]), + vec![1.0] + ); + assert!( + recorder + .histogram_values("rustfs_io_get_object_metadata_early_stop_saved_responses", &[("path", unexpected_path)]) + .is_empty(), + "early-stop saved responses must not leak into the other metadata path" + ); + } + + #[tokio::test] + async fn bounded_metadata_early_stop_full_waits_when_inline_data_is_corrupt() { + const DISKS: usize = 4; + let bucket = "bounded-inline-data-get-corrupt-bucket"; + let object = object_with_initial_data_shards(bucket, "bounded-inline-data-get-corrupt-object", 2, 3); + let corrupt_shard = initial_data_shard_indexes(bucket, &object, 2, 3)[0]; + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"verified inline payload", |files| { + if let Some(data) = files.get_mut(corrupt_shard - 1).and_then(|file| file.data.as_mut()) { + let mut corrupt = data.to_vec(); + corrupt[0] ^= 0x01; + *data = Bytes::from(corrupt); + } + }) + .await; + temp_env::async_with_vars( [ ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), @@ -5448,31 +6202,359 @@ mod tests { ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), ], async { - let calls = disk_call_counters::observe(treatment_object); - let (parts_metadata, errs, diagnostics) = SetDisks::read_all_fileinfo_observed( - &disks, - bucket, - bucket, - treatment_object, - "", - true, - false, - false, - true, - 2, - ) - .await - .expect("healthy object metadata should reach early-stop quorum"); + let calls = disk_call_counters::observe(&object); + let (parts_metadata, errs, diagnostics) = + SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, &object, "", true, false, false, true, 2) + .await + .expect("corrupt inline metadata should fall back to full fanout"); - assert!( - (3..=DISKS as u64).contains(&calls.total(disk_call_counters::KIND_READ_VERSION)), - "healthy 2+2 bounded data-read fanout may finish at quorum before a spare hedge is needed" + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + DISKS as u64, + "inline bitrot failure must keep metadata fanout open instead of aborting spares" ); - assert!( - (3..=DISKS).contains(&diagnostics.total_responses()), - "treatment path should return after reaching quorum, with at most the spare hedge response observed" + assert_eq!(diagnostics.total_responses(), DISKS); + assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); + assert!(errs.iter().all(Option::is_none)); + }, + ) + .await; + + drop(dirs); + } + + #[tokio::test] + async fn bounded_metadata_early_stop_full_waits_when_inline_generation_differs() { + const DISKS: usize = 4; + let bucket = "bounded-inline-data-get-generation-bucket"; + let object = object_with_initial_data_shards(bucket, "bounded-inline-data-get-generation-object", 2, 3); + let stale_shard = initial_data_shard_indexes(bucket, &object, 2, 3)[0]; + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"verified inline payload", |files| { + if let Some(file) = files.get_mut(stale_shard - 1) { + file.version_id = Some(Uuid::new_v4()); + } + }) + .await; + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let calls = disk_call_counters::observe(&object); + let (parts_metadata, errs, diagnostics) = + SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, &object, "", true, false, false, true, 2) + .await + .expect("mixed-generation inline metadata should fall back to full fanout"); + + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + DISKS as u64, + "inline data from a different metadata generation must not satisfy the data-read gate" ); - assert!(parts_metadata.iter().filter(|fi| fi.name == treatment_object).count() >= 3); + assert_eq!(diagnostics.total_responses(), DISKS); + assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); + assert!(errs.iter().all(Option::is_none)); + }, + ) + .await; + + drop(dirs); + } + + #[tokio::test] + async fn bounded_metadata_early_stop_full_waits_when_inline_shard_identity_is_copied() { + const DISKS: usize = 4; + let bucket = "bounded-inline-data-get-copied-shard-bucket"; + let object = object_with_initial_data_shards(bucket, "bounded-inline-data-get-copied-shard-object", 2, 3); + let data_shards = initial_data_shard_indexes(bucket, &object, 2, 3); + let source_shard = data_shards[0]; + let target_shard = data_shards[1]; + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"verified inline payload", |files| { + let copied = files[source_shard - 1].clone(); + files[target_shard - 1].erasure.index = copied.erasure.index; + files[target_shard - 1].data = copied.data; + }) + .await; + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let spare_disk = bounded_spare_disk_index(bucket, &object, 2, 2); + let barrier = rename_fanout_barrier::arm(&object, spare_disk, rename_fanout_barrier::PHASE_READ_VERSION); + let calls = disk_call_counters::observe(&object); + let disks_for_read = disks.clone(); + let object_for_read = object.clone(); + let mut read = tokio::spawn(async move { + SetDisks::read_all_fileinfo_observed( + &disks_for_read, + bucket, + bucket, + &object_for_read, + "", + true, + false, + false, + true, + 2, + ) + .await + }); + + tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused()) + .await + .expect("full-wait fallback should schedule the spare disk"); + assert!( + tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await.is_err(), + "copied inline shard identity must not satisfy the early-stop data gate" + ); + barrier.release(); + + let (parts_metadata, errs, diagnostics) = read + .await + .expect("metadata read task should not panic") + .expect("copied inline shard identity should fall back to full fanout"); + + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + DISKS as u64, + "copied shard identity must keep metadata fanout open instead of aborting spares" + ); + assert_eq!(diagnostics.total_responses(), DISKS); + assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); + assert!(errs.iter().all(Option::is_none)); + }, + ) + .await; + + drop(dirs); + } + + #[tokio::test] + async fn bounded_metadata_early_stop_full_waits_for_transformed_inline_metadata() { + const DISKS: usize = 4; + let bucket = "bounded-inline-data-get-transform-bucket"; + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + + async fn assert_full_wait(disks: &[Option], bucket: &str, object: &str, mutate: impl FnOnce(&mut [FileInfo])) { + const DISKS: usize = 4; + install_inline_metadata_fanout_fileinfo(disks, bucket, object, b"verified inline payload", mutate).await; + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let calls = disk_call_counters::observe(object); + let (parts_metadata, errs, diagnostics) = + SetDisks::read_all_fileinfo_observed(disks, bucket, bucket, object, "", true, false, false, true, 2) + .await + .expect("transformed inline metadata should fall back to full fanout"); + + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + DISKS as u64, + "transformed inline metadata must not pass the plaintext data-read gate" + ); + assert_eq!(diagnostics.total_responses(), DISKS); + assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); + assert!(errs.iter().all(Option::is_none)); + }, + ) + .await; + } + + assert_full_wait(&disks, bucket, "bounded-inline-data-get-compressed-object", |files| { + for file in files { + rustfs_utils::http::insert_str(&mut file.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string()); + } + }) + .await; + + assert_full_wait(&disks, bucket, "bounded-inline-data-get-encrypted-object", |files| { + for file in files { + file.metadata + .insert(rustfs_utils::http::headers::SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string()); + } + }) + .await; + + drop(dirs); + } + + #[tokio::test] + async fn bounded_metadata_early_stop_full_waits_for_unsafe_inline_metadata_shapes() { + const DISKS: usize = 4; + let bucket = "bounded-inline-data-get-unsafe-shape-bucket"; + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + + async fn assert_full_wait(disks: &[Option], bucket: &str, object: &str, mutate: impl FnOnce(&mut [FileInfo])) { + const DISKS: usize = 4; + install_inline_metadata_fanout_fileinfo(disks, bucket, object, b"verified inline payload", mutate).await; + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let calls = disk_call_counters::observe(object); + let (parts_metadata, errs, diagnostics) = + SetDisks::read_all_fileinfo_observed(disks, bucket, bucket, object, "", true, false, false, true, 2) + .await + .expect("unsafe inline metadata shape should fall back to full fanout"); + + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + DISKS as u64, + "unsafe inline metadata shape {object} must not abort remaining metadata responses" + ); + assert_eq!(diagnostics.total_responses(), DISKS); + assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); + assert!(errs.iter().all(Option::is_none)); + }, + ) + .await; + } + + assert_full_wait(&disks, bucket, "bounded-inline-data-get-remote-object", |files| { + for file in files { + file.transition_status = TRANSITION_COMPLETE.to_string(); + } + }) + .await; + + assert_full_wait(&disks, bucket, "bounded-inline-data-get-zero-size-object", |files| { + for file in files { + file.size = 0; + if let Some(part) = file.parts.first_mut() { + part.size = 0; + } + } + }) + .await; + + assert_full_wait(&disks, bucket, "bounded-inline-data-get-multipart-object", |files| { + for file in files { + let mut second_part = file.parts[0].clone(); + second_part.number = 2; + file.parts.push(second_part); + } + }) + .await; + + assert_full_wait(&disks, bucket, "bounded-inline-data-get-part-size-mismatch-object", |files| { + for file in files { + if let Some(part) = file.parts.first_mut() { + part.size = part.size.saturating_add(1); + } + } + }) + .await; + + assert_full_wait(&disks, bucket, "bounded-inline-data-get-oversize-object", |files| { + for file in files { + let oversize = file.erasure.block_size.saturating_add(1); + file.size = i64::try_from(oversize).expect("test block size should fit i64"); + if let Some(part) = file.parts.first_mut() { + part.size = oversize; + } + } + }) + .await; + + drop(dirs); + } + + #[tokio::test] + async fn bounded_metadata_early_stop_full_waits_for_purge_pending_inline_payload() { + const DISKS: usize = 4; + let bucket = "bounded-inline-data-get-purge-pending-bucket"; + let object = object_with_initial_data_shards(bucket, "bounded-inline-data-get-purge-pending-object", 2, 3); + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"verified inline payload", |files| { + for file in files { + rustfs_utils::http::insert_str( + &mut file.metadata, + rustfs_utils::http::SUFFIX_PURGESTATUS, + "target=PENDING;".to_string(), + ); + let replication_state = crate::bucket::replication::ReplicationState { + version_purge_status_internal: Some("target=PENDING;".to_string()), + purge_targets: crate::bucket::replication::version_purge_statuses_map("target=PENDING;"), + ..Default::default() + }; + file.replication_state_internal = + Some(crate::bucket::replication::replication_state_to_filemeta(&replication_state)); + file.deleted = true; + assert!( + !file.is_canonical_delete_marker(), + "test fixture must remain an erasure-backed purge-pending payload" + ); + } + }) + .await; + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let spare_disk = bounded_spare_disk_index(bucket, &object, 2, 2); + let barrier = rename_fanout_barrier::arm(&object, spare_disk, rename_fanout_barrier::PHASE_READ_VERSION); + let calls = disk_call_counters::observe(&object); + let disks_for_read = disks.clone(); + let object_for_read = object.clone(); + let mut read = tokio::spawn(async move { + SetDisks::read_all_fileinfo_observed( + &disks_for_read, + bucket, + bucket, + &object_for_read, + "", + true, + false, + false, + true, + 2, + ) + .await + }); + + tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused()) + .await + .expect("purge-pending fallback should schedule the spare disk"); + assert!( + tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await.is_err(), + "purge-pending payload metadata must not satisfy the inline data-read gate" + ); + barrier.release(); + + let (parts_metadata, errs, diagnostics) = read + .await + .expect("metadata read task should not panic") + .expect("purge-pending inline payload should fall back to full fanout"); + + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + DISKS as u64, + "purge-pending payload must keep metadata fanout open instead of aborting spares" + ); + assert_eq!(diagnostics.total_responses(), DISKS); + assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); assert!(errs.iter().all(Option::is_none)); }, ) @@ -5482,7 +6564,7 @@ mod tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn bounded_data_get_hedges_single_pending_read_version() { + async fn bounded_non_inline_data_get_hedges_then_waits_for_full_fanout() { const DISKS: usize = 4; let bucket = "bounded-data-get-hedge-bucket"; let object = "bounded-data-get-hedge-object"; @@ -5515,12 +6597,14 @@ mod tests { .await .expect("bounded data-read fanout should hedge by starting the spare disk"); - let completed = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await; - if completed.is_err() { - barrier.release(); - } - let (parts_metadata, errs, diagnostics) = completed - .expect("spare metadata should allow early-stop without waiting for the paused disk") + let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await; + assert!( + pending.is_err(), + "non-inline data reads must not return before the paused metadata response" + ); + barrier.release(); + let (parts_metadata, errs, diagnostics) = read + .await .expect("metadata read task should not panic") .expect("healthy spare metadata should resolve"); @@ -5529,8 +6613,8 @@ mod tests { DISKS as u64, "bounded data-read fanout should issue the paused disk plus one spare hedge" ); - assert_eq!(diagnostics.total_responses(), 3); - assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), 3); + assert_eq!(diagnostics.total_responses(), DISKS); + assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); assert!(errs.iter().all(Option::is_none)); }, ) diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 20e481c2d..27e382a3e 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -880,12 +880,44 @@ mod prepared_get_object_metadata_tests { use super::*; use crate::ecstore_validation_blackbox::make_local_set_disks; use crate::object_api::{BLOCK_SIZE_V2, PutObjReader}; - use crate::set_disk::core::io_primitives::disk_call_counters; + use crate::set_disk::core::io_primitives::{bounded_metadata_fanout_order, disk_call_counters, rename_fanout_barrier}; use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions}; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + use crate::test_metrics::CapturingRecorder; use http::HeaderMap; use tokio::io::AsyncReadExt; + const READ_VERSION_BARRIER_GUARD: std::time::Duration = std::time::Duration::from_secs(10); + + fn object_with_initial_data_shards(bucket: &str, prefix: &str) -> String { + (0..1000) + .map(|index| format!("{prefix}-{index}.bin")) + .find(|name| { + let order = bounded_metadata_fanout_order(bucket, name, 4, 2); + let distribution = FileInfo::new(&[bucket, name].join("/"), 2, 2).erasure.distribution; + let mut seen = [false; 2]; + for disk_index in order.into_iter().take(3) { + if let Some(block_index @ 1..=2) = distribution.get(disk_index).copied() { + seen[block_index - 1] = true; + } + } + seen.into_iter().all(|seen| seen) + }) + .expect("test should find an object whose initial fanout covers both data shards") + } + + fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize { + *bounded_metadata_fanout_order(bucket, object, 4, 2) + .get(3) + .expect("4-disk test geometry should leave one bounded spare disk") + } + + fn bounded_slow_initial_disk_index(bucket: &str, object: &str) -> usize { + *bounded_metadata_fanout_order(bucket, object, 4, 2) + .get(2) + .expect("4-disk test geometry should include a third initial metadata disk") + } + #[tokio::test] async fn prepared_metadata_is_consumed_exactly_once() { let snapshot = GetObjectFileInfo::owned(FileInfo::default(), Vec::new(), Vec::new()); @@ -1002,6 +1034,307 @@ mod prepared_get_object_metadata_tests { ); } + #[test] + #[serial_test::serial(body_cache_hook)] + fn inline_data_read_early_stop_reader_returns_exact_body() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime should build"); + let bucket = "inline-data-read-early-stop-reader"; + let object = object_with_initial_data_shards(bucket, "inline-data-read-early-stop-reader-object"); + let payload = b"inline early-stop reader payload".repeat(256); + let recorder = CapturingRecorder::default(); + let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled(); + rustfs_io_metrics::set_get_stage_metrics_enabled(true); + + let (restored, object_size, calls_total) = metrics::with_local_recorder(&recorder, || { + runtime.block_on(async { + let (_dirs, set_disks) = make_local_set_disks(4, 2).await; + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, &object, &mut put_reader, &opts) + .await + .expect("inline object should be written"); + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let slow_initial_disk = bounded_slow_initial_disk_index(bucket, &object); + let barrier = + rename_fanout_barrier::arm(&object, slow_initial_disk, rename_fanout_barrier::PHASE_READ_VERSION); + let calls = disk_call_counters::observe(&object); + let set_disks_for_read = Arc::clone(&set_disks); + let opts_for_read = opts.clone(); + let object_for_read = object.clone(); + let mut open_reader = tokio::spawn(async move { + set_disks_for_read + .get_object_reader(bucket, &object_for_read, None, HeaderMap::new(), &opts_for_read) + .await + }); + + tokio::time::timeout(READ_VERSION_BARRIER_GUARD, barrier.wait_until_paused()) + .await + .expect("bounded inline GET should pause a slow initial metadata read"); + let mut reader = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut open_reader) + .await + .expect("production inline GET should return before the paused metadata response") + .expect("inline GET reader task should not panic") + .expect("inline GET reader should open"); + let object_size = reader.object_info.size; + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("inline GET body should stream"); + + (restored, object_size, calls.total(disk_call_counters::KIND_READ_VERSION)) + }, + ) + .await + }) + }); + rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate); + + assert_eq!(object_size, payload.len() as i64); + assert_eq!(restored, payload); + assert_eq!(calls_total, 4, "bounded production GET should schedule the initial quorum plus one spare"); + assert_eq!( + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_scheduled", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] + ), + vec![4.0], + "bounded production GET should record all scheduled metadata tasks" + ); + assert_eq!( + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_completed", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] + ), + vec![3.0], + "bounded production GET should record only observed metadata responses as completed" + ); + assert_eq!( + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_cancelled", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] + ), + vec![1.0], + "bounded production GET should record the aborted slow metadata task" + ); + } + + #[test] + #[serial_test::serial(body_cache_hook)] + fn prepared_metadata_uses_full_fanout_even_when_data_read_early_stop_is_enabled() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime should build"); + let bucket = "prepared-metadata-early-stop-enabled"; + let object = object_with_initial_data_shards(bucket, "prepared-metadata-early-stop-enabled-object"); + let payload = b"prepared metadata early-stop enabled payload".repeat(16); + let recorder = CapturingRecorder::default(); + let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled(); + rustfs_io_metrics::set_get_stage_metrics_enabled(true); + + let (restored, calls_total) = metrics::with_local_recorder(&recorder, || { + runtime.block_on(async { + let (_dirs, set_disks) = make_local_set_disks(4, 2).await; + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, &object, &mut put_reader, &opts) + .await + .expect("object should be written"); + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let calls = disk_call_counters::observe(&object); + let metadata = set_disks + .prepare_get_object_metadata(bucket, &object, &opts) + .await + .expect("prepared metadata should resolve"); + let calls_total = calls.total(disk_call_counters::KIND_READ_VERSION); + + let mut reader = set_disks + .get_object_reader_with_prepared_metadata(bucket, &object, None, HeaderMap::new(), &opts, metadata) + .await + .expect("prepared body reader should open"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("prepared body should stream"); + (restored, calls_total) + }, + ) + .await + }) + }); + rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate); + + assert_eq!(restored, payload); + assert_eq!( + calls_total, 4, + "prepared metadata must opt out of data-read early-stop until the read shape is known" + ); + assert_eq!( + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_scheduled", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] + ), + vec![4.0], + "prepared metadata should schedule the full metadata fanout" + ); + assert_eq!( + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_completed", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] + ), + vec![4.0], + "prepared metadata must wait for every scheduled metadata response" + ); + assert_eq!( + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_cancelled", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] + ), + vec![0.0], + "prepared metadata must not cancel metadata responses" + ); + } + + #[test] + #[serial_test::serial(body_cache_hook)] + fn data_read_early_stop_request_shapes_full_wait_in_production_reader() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime should build"); + let bucket = "data-read-early-stop-shape-reader"; + let payload = b"shape-gated inline reader payload".repeat(256); + + for (object_prefix, range, configure_opts, expected_body) in [ + ( + "data-read-early-stop-range-reader-object", + Some(HTTPRangeSpec { + start: 0, + end: 3, + is_suffix_length: false, + }), + None, + payload[..4].to_vec(), + ), + ("data-read-early-stop-part-reader-object", None, Some(1), payload.clone()), + ] { + let recorder = CapturingRecorder::default(); + let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled(); + rustfs_io_metrics::set_get_stage_metrics_enabled(true); + let (restored, calls_total) = metrics::with_local_recorder(&recorder, || { + runtime.block_on(async { + let (_dirs, set_disks) = make_local_set_disks(4, 2).await; + let object = object_with_initial_data_shards(bucket, object_prefix); + let mut opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + opts.part_number = configure_opts; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, &object, &mut put_reader, &opts) + .await + .expect("inline object should be written"); + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let calls = disk_call_counters::observe(&object); + let mut reader = set_disks + .get_object_reader(bucket, &object, range, HeaderMap::new(), &opts) + .await + .expect("shape-gated GET reader should open"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("shape-gated GET body should stream"); + (restored, calls.total(disk_call_counters::KIND_READ_VERSION)) + }, + ) + .await + }) + }); + rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate); + + assert_eq!(restored, expected_body); + assert_eq!(calls_total, 4, "shape-gated production GET should keep full metadata fanout"); + assert_eq!( + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_scheduled", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] + ), + vec![4.0], + "shape-gated production GET should schedule the full metadata fanout" + ); + assert_eq!( + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_completed", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] + ), + vec![4.0], + "shape-gated production GET must wait for every scheduled metadata response" + ); + assert_eq!( + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_cancelled", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] + ), + vec![0.0], + "shape-gated production GET must not cancel metadata responses" + ); + } + } + #[tokio::test] #[serial_test::serial(body_cache_hook)] async fn prepared_reader_rebuilds_object_info_when_precomputed_value_is_absent() { @@ -1105,7 +1438,7 @@ impl SetDisks { object: &str, opts: &ObjectOptions, ) -> Result { - let snapshot = self.get_object_fileinfo(bucket, object, opts, true, true).await?; + let snapshot = self.get_object_fileinfo(bucket, object, opts, true, false).await?; let object_info = build_get_object_info(snapshot.fi(), bucket, object, opts.versioned || opts.version_suspended); Ok(PreparedGetObjectMetadata { snapshot, @@ -2408,12 +2741,12 @@ mod write_layout_tests { let held_layout = resolve_write_layout(&held, 0, 4, 2, None, false).expect("held snapshot should remain valid"); assert_eq!(held_layout.parity_drives, 2); - assert!(held.should_inline(512, false)); + assert!(held.should_inline(512, held_layout.data_drives, false)); let current = published.load_full(); let current_layout = resolve_write_layout(¤t, 0, 4, 2, None, false).expect("new snapshot should resolve"); assert_eq!(current_layout.parity_drives, 1); - assert!(!current.should_inline(512, false)); + assert!(!current.should_inline(512, current_layout.data_drives, false)); } } @@ -2459,6 +2792,9 @@ pub struct SetDisks { get_object_metadata_cache: moka::future::Cache>, get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState, get_object_metadata_cache_generations: Arc<[AtomicU64]>, + /// GET codecs keyed by every persisted layout dimension that affects + /// decoding. Clones of a set share the memoized shells. + erasure_cache: Arc, pub lockers: Vec>, shared_lockers: Arc<[Arc]>, local_lock_manager: Arc, @@ -2481,6 +2817,137 @@ pub struct SetDisks { storage_class_config_override: Arc>>>, } +const ERASURE_CACHE_MAX_ENTRIES: usize = 32; + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +struct ErasureCacheKey { + data_shards: usize, + parity_shards: usize, + block_size: usize, + uses_legacy: bool, +} + +struct ErasureCache { + entries: parking_lot::RwLock>>, +} + +impl std::fmt::Debug for ErasureCache { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ErasureCache") + .field("entries", &self.entries.read().len()) + .finish() + } +} + +impl ErasureCache { + fn new() -> Self { + Self { + entries: parking_lot::RwLock::new(HashMap::new()), + } + } + + fn get_or_try_insert( + &self, + key: ErasureCacheKey, + ) -> std::result::Result, coding::ErasureConstructionError> { + if let Some(erasure) = self.entries.read().get(&key) { + return Ok(Arc::clone(erasure)); + } + + // Serialize first construction for a key so concurrent cold GETs still + // create exactly one shell. Codec construction never awaits. + let mut entries = self.entries.write(); + if let Some(erasure) = entries.get(&key) { + return Ok(Arc::clone(erasure)); + } + let erasure = Arc::new(coding::Erasure::try_new_with_options( + key.data_shards, + key.parity_shards, + key.block_size, + key.uses_legacy, + )?); + if entries.len() < ERASURE_CACHE_MAX_ENTRIES { + entries.insert(key, Arc::clone(&erasure)); + } + Ok(erasure) + } + + fn get_for_file_info(&self, fi: &FileInfo) -> Result> { + self.get_or_try_insert(ErasureCacheKey { + data_shards: fi.erasure.data_blocks, + parity_shards: fi.erasure.parity_blocks, + block_size: fi.erasure.block_size, + uses_legacy: fi.uses_legacy_checksum, + }) + .map_err(Error::from) + } +} + +#[cfg(test)] +mod erasure_cache_tests { + use super::*; + + #[test] + fn reuses_shells_and_keeps_every_layout_dimension_in_the_key() { + let cache = ErasureCache::new(); + let base = ErasureCacheKey { + data_shards: 4, + parity_shards: 2, + block_size: 1_048_576, + uses_legacy: false, + }; + let first = cache.get_or_try_insert(base).expect("modern shell should construct"); + let reused = cache.get_or_try_insert(base).expect("same modern shell should be cached"); + assert!(Arc::ptr_eq(&first, &reused)); + + for distinct in [ + ErasureCacheKey { data_shards: 3, ..base }, + ErasureCacheKey { + parity_shards: 1, + ..base + }, + ErasureCacheKey { + block_size: 524_288, + ..base + }, + ErasureCacheKey { + uses_legacy: true, + ..base + }, + ] { + let shell = cache.get_or_try_insert(distinct).expect("distinct shell should construct"); + assert!(!Arc::ptr_eq(&first, &shell)); + } + assert_eq!(cache.entries.read().len(), 5); + } + + #[test] + fn does_not_cache_invalid_layouts_or_grow_past_the_bound() { + let cache = ErasureCache::new(); + let invalid = ErasureCacheKey { + data_shards: 4, + parity_shards: 2, + block_size: 0, + uses_legacy: false, + }; + assert!(cache.get_or_try_insert(invalid).is_err()); + assert!(cache.entries.read().is_empty()); + + for block_size in 1..=(ERASURE_CACHE_MAX_ENTRIES + 1) { + cache + .get_or_try_insert(ErasureCacheKey { + data_shards: 4, + parity_shards: 2, + block_size, + uses_legacy: false, + }) + .expect("bounded cache fixture should construct"); + } + assert_eq!(cache.entries.read().len(), ERASURE_CACHE_MAX_ENTRIES); + } +} + #[derive(Clone, Debug, Eq, PartialEq)] struct GetObjectMetadataCacheKey { bucket: Arc, @@ -2879,6 +3346,7 @@ impl SetDisks { .map(|_| AtomicU64::new(0)) .collect::>(), ), + erasure_cache: Arc::new(ErasureCache::new()), lockers, shared_lockers, // Sourced from the instance context so each instance owns its lock @@ -3411,9 +3879,15 @@ fn collect_inline_data_shard_fileinfos_by_index<'a>( if block_index == 0 || block_index > data_shards { continue; } + if file_info.erasure.index != block_index { + continue; + } if !file_info.has_valid_erasure_geometry() { continue; } + if !core::io_primitives::metadata_early_stop_candidate_matches(file_info, fi) { + continue; + } if file_info.data.as_ref().is_none_or(|data| data.is_empty()) { continue; } @@ -9221,6 +9695,9 @@ mod tests { HashAlgorithm::HighwayHash256S }; let shards = erasure.encode_data(payload).expect("payload should encode"); + let version_id = Some(Uuid::new_v4()); + let data_dir = Some(Uuid::new_v4()); + let mod_time = Some(OffsetDateTime::now_utc()); let mut files = Vec::with_capacity(shards.len()); for shard in shards { @@ -9233,6 +9710,16 @@ mod tests { writer.shutdown().await.expect("inline writer should shutdown"); let data = writer.into_inline_data().expect("inline data should be retained"); let mut file = FileInfo::new("bucket/object", erasure.data_shards, erasure.parity_shards); + file.volume = "bucket".to_string(); + file.name = "object".to_string(); + file.size = i64::try_from(payload.len()).expect("test payload should fit i64"); + file.is_latest = true; + file.version_id = version_id; + file.data_dir = data_dir; + file.mod_time = mod_time; + file.metadata.insert("etag".to_string(), "etag-inline".to_string()); + file.add_object_part(1, "part-etag-inline".to_string(), payload.len(), file.mod_time, file.size, None, None); + file.set_inline_data(); file.erasure.index = files.len() + 1; file.data = Some(Bytes::from(data)); files.push(file); @@ -9245,16 +9732,40 @@ mod tests { inline_bitrot_files_for_payload_with_mode(payload, false).await } + fn disk_ordered_fileinfos(files: &[FileInfo]) -> Vec { + let distribution = &files + .first() + .expect("inline data shard fixture should include metadata") + .erasure + .distribution; + distribution + .iter() + .map(|block_index| { + files + .get(block_index.checked_sub(1).expect("erasure block indexes are one-based")) + .expect("inline data shard fixture should include every distributed shard") + .clone() + }) + .collect() + } + fn inline_data_shard_fileinfo( - name: &str, data_blocks: usize, parity_blocks: usize, erasure_index: usize, distribution: &[usize], data: Option<&'static [u8]>, ) -> FileInfo { - let mut fi = FileInfo::new(name, data_blocks, parity_blocks); - fi.name = name.to_string(); + let mut fi = FileInfo::new("object", data_blocks, parity_blocks); + fi.name = "object".to_string(); + fi.volume = "bucket".to_string(); + fi.size = 4; + fi.is_latest = true; + fi.data_dir = Some(Uuid::nil()); + fi.mod_time = Some(OffsetDateTime::UNIX_EPOCH); + fi.metadata.insert("etag".to_string(), "etag-inline".to_string()); + fi.add_object_part(1, "part-etag-inline".to_string(), 4, fi.mod_time, 4, None, None); + fi.set_inline_data(); fi.erasure.index = erasure_index; fi.erasure.distribution = distribution.to_vec(); fi.data = data.map(Bytes::from_static); @@ -9264,36 +9775,41 @@ mod tests { #[test] fn collect_inline_data_shards_by_index_uses_distribution_order() { let distribution = vec![3, 1, 5, 2, 4, 6]; - let mut fi = FileInfo::new("object", 4, 2); + let mut fi = inline_data_shard_fileinfo(4, 2, 1, &distribution, Some(b"x")); + fi.erasure.index = 1; fi.erasure.distribution = distribution.clone(); let files = vec![ - inline_data_shard_fileinfo("block-3", 4, 2, 3, &distribution, Some(b"c")), - inline_data_shard_fileinfo("block-1", 4, 2, 1, &distribution, Some(b"a")), - inline_data_shard_fileinfo("parity-5", 4, 2, 5, &distribution, Some(b"p")), - inline_data_shard_fileinfo("block-2", 4, 2, 2, &distribution, Some(b"b")), - inline_data_shard_fileinfo("block-4", 4, 2, 4, &distribution, Some(b"d")), - inline_data_shard_fileinfo("parity-6", 4, 2, 6, &distribution, Some(b"q")), + inline_data_shard_fileinfo(4, 2, 3, &distribution, Some(b"c")), + inline_data_shard_fileinfo(4, 2, 1, &distribution, Some(b"a")), + inline_data_shard_fileinfo(4, 2, 5, &distribution, Some(b"p")), + inline_data_shard_fileinfo(4, 2, 2, &distribution, Some(b"b")), + inline_data_shard_fileinfo(4, 2, 4, &distribution, Some(b"d")), + inline_data_shard_fileinfo(4, 2, 6, &distribution, Some(b"q")), ]; let data_files = collect_inline_data_shard_fileinfos_by_index(&files, &fi, 4, |_| true).expect("all data shards should be collected"); assert_eq!( - data_files.iter().map(|file| file.name.as_str()).collect::>(), - ["block-1", "block-2", "block-3", "block-4"] + data_files + .iter() + .map(|file| file.data.as_deref().expect("fixture carries inline bytes")) + .collect::>(), + [b"a".as_slice(), b"b".as_slice(), b"c".as_slice(), b"d".as_slice()] ); } #[test] fn collect_inline_data_shards_by_index_rejects_missing_data_shard() { let distribution = vec![1, 2, 3, 4]; - let mut fi = FileInfo::new("object", 2, 2); + let mut fi = inline_data_shard_fileinfo(2, 2, 1, &distribution, Some(b"x")); + fi.erasure.index = 1; fi.erasure.distribution = distribution.clone(); let files = vec![ - inline_data_shard_fileinfo("block-1", 2, 2, 1, &distribution, Some(b"a")), - inline_data_shard_fileinfo("block-2", 2, 2, 2, &distribution, None), - inline_data_shard_fileinfo("parity-3", 2, 2, 3, &distribution, Some(b"p")), - inline_data_shard_fileinfo("parity-4", 2, 2, 4, &distribution, Some(b"q")), + inline_data_shard_fileinfo(2, 2, 1, &distribution, Some(b"a")), + inline_data_shard_fileinfo(2, 2, 2, &distribution, None), + inline_data_shard_fileinfo(2, 2, 3, &distribution, Some(b"p")), + inline_data_shard_fileinfo(2, 2, 4, &distribution, Some(b"q")), ]; assert!(collect_inline_data_shard_fileinfos_by_index(&files, &fi, 2, |_| true).is_none()); @@ -9426,10 +9942,8 @@ mod tests { let payload = vec![b'i'; 192 * 1024]; let (erasure, files, _read_length, _checksum_algo) = inline_bitrot_files_for_payload(&payload).await; - let mut fi = FileInfo::new("bucket/object", erasure.data_shards, erasure.parity_shards); - fi.size = payload.len() as i64; - fi.data = files[0].data.clone(); - fi.add_object_part(1, String::new(), payload.len(), None, payload.len() as i64, None, None); + let fi = files[0].clone(); + let disk_files = disk_ordered_fileinfos(&files); let disks = vec![Some(disk); erasure.total_shard_count()]; let metrics_size_bucket = rustfs_io_metrics::get_object_size_bucket(fi.size); @@ -9437,8 +9951,9 @@ mod tests { let body = SetDisks::try_get_object_direct_data_shards_with_fileinfo( "bucket", "object", + Arc::new(ErasureCache::new()), &fi, - &files, + &disk_files, &disks, true, GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, @@ -9469,10 +9984,8 @@ mod tests { let payload = vec![b'v'; 64 * 1024]; let payload_size = i64::try_from(payload.len()).expect("test payload size should fit i64"); let (erasure, files, _read_length, _checksum_algo) = inline_bitrot_files_for_payload(&payload).await; - let mut fi = FileInfo::new("bucket/object", erasure.data_shards, erasure.parity_shards); - fi.size = payload_size; - fi.data = files[0].data.clone(); - fi.add_object_part(1, String::new(), payload.len(), None, payload_size, None, None); + let fi = files[0].clone(); + let disk_files = disk_ordered_fileinfos(&files); let mut object_info = ObjectInfo { size: payload_size, @@ -9502,8 +10015,9 @@ mod tests { let body = SetDisks::try_get_object_direct_data_shards_with_fileinfo( "bucket", "object", + Arc::new(ErasureCache::new()), &fi, - &files, + &disk_files, &vec![Some(disk); erasure.total_shard_count()], true, GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, @@ -9582,6 +10096,7 @@ mod tests { let body = SetDisks::try_get_object_direct_data_shards_with_fileinfo( bucket, object, + Arc::new(ErasureCache::new()), &fi, &files, &disks, @@ -9667,6 +10182,7 @@ mod tests { SetDisks::get_object_with_fileinfo( bucket, object, + Arc::new(ErasureCache::new()), range_offset, range_length as i64, &mut writer, @@ -9778,6 +10294,7 @@ mod tests { SetDisks::get_object_with_fileinfo( bucket, object, + Arc::new(ErasureCache::new()), 0, total_size as i64, &mut writer, diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index e5ca4fbec..089515ec6 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -1425,6 +1425,33 @@ impl SetDisks { /// post-heal tail — reclaim identically. Never fails the heal: delete errors /// are logged and swallowed. Callers must gate this on `!opts.dry_run`. async fn reclaim_orphan_data_dirs_best_effort(&self, bucket: &str, object: &str) { + match self.reconcile_old_data_cleanup_receipts(bucket, object).await { + Ok(removed) if removed > 0 => { + debug!( + event = EVENT_SET_DISK_HEAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket, + object, + removed, + state = "old_data_cleanup_receipt_reconciled", + "Set disk old-data cleanup receipts reconciled" + ); + } + Ok(_) => {} + Err(e) => { + warn!( + event = EVENT_SET_DISK_HEAL, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket, + object, + error = %e, + state = "old_data_cleanup_receipt_reconcile_failed", + "Set disk old-data cleanup receipt reconcile failed" + ); + } + } match self.reclaim_orphan_data_dirs(bucket, object).await { Ok(removed) if removed > 0 => { debug!( diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 1cbad8cc1..236c8af3b 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -22,6 +22,11 @@ use super::super::*; use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards}; +use super::object::{ + assign_object_transaction_epoch, object_transaction_fencing_fleet_proof, object_transaction_fencing_fleet_proof_matches, + object_transaction_fencing_requested, old_data_cleanup_receipt_path, read_object_transaction_epoch_fence, + verify_object_transaction_epoch_fence, +}; use crate::crash_inject::{self, CrashPoint}; use crate::multipart_listing::paginate_multipart_listing; use futures::{StreamExt, stream}; @@ -63,6 +68,9 @@ pub(crate) enum MultipartCommitPause { PutPartBeforeLockLost, PutPartAfterRename, BeforeLockLost, + BeforeTransactionEpochVerify, + BeforeObjectPublication, + AfterObjectPublication, AfterRename, } @@ -153,13 +161,24 @@ impl Drop for MultipartCommitBarrier { #[cfg(test)] async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) { - let barrier = MULTIPART_COMMIT_BARRIER - .get_or_init(|| std::sync::Mutex::new(None)) - .lock() - .expect("multipart commit barrier mutex should not poison") - .as_ref() - .filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause) - .cloned(); + let barrier = { + let mut slot = MULTIPART_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("multipart commit barrier mutex should not poison"); + if slot + .as_ref() + .is_some_and(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause) + { + if pause == MultipartCommitPause::BeforeTransactionEpochVerify { + slot.take() + } else { + slot.clone() + } + } else { + None + } + }; if let Some(barrier) = barrier && let Ok(previous) = barrier.arrivals.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { (current < barrier.expected_arrivals).then_some(current + 1) @@ -2296,140 +2315,196 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; - let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); - - // Crash-consistency injection: hard power loss after the upload is fully - // staged and locked but before the authoritative rename_data commit. No - // disk has moved the staged data, so a crash here must leave any prior - // committed version byte-for-byte intact (rustfs/backlog#864) and the - // upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`. - if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, object) { - return Err(StorageError::Unexpected); + let transaction_fencing_proof = object_transaction_fencing_fleet_proof(); + if object_transaction_fencing_requested() && transaction_fencing_proof.is_none() { + return Err(Error::other("object transaction fencing requires a live fleet capability proof")); } + let transaction_epoch_fence = if transaction_fencing_proof.is_some() { + Some(read_object_transaction_epoch_fence(self.as_ref(), bucket, object).await?) + } else { + None + }; + let transaction_epoch = + transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&shuffle_disks, &mut parts_metadatas)); - // The trailing `_` drops the rename_data old-size backfill - // (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit - // `get_object_info` lookup, so the backfill has no consumer here yet. - let (online_disks, convergence, op_old_dir, cleanup_disks, _) = Self::rename_data( - &shuffle_disks, - RUSTFS_META_MULTIPART_BUCKET, - &upload_id_path, - &parts_metadatas, - bucket, - object, - write_quorum, - ) - .await?; - - // Detach admission before any post-commit await: client cancellation - // must not couple durable convergence repair to cleanup work. - if convergence.needs_heal() { - let mut request = rustfs_common::heal_channel::create_heal_request_with_options( - bucket.to_string(), - Some(object.to_string()), - false, - Some(HealChannelPriority::Normal), - Some(self.pool_index), - Some(self.set_index), - ); - request.object_version_id = fi - .version_id - .or_else(|| opts.version_suspended.then(Uuid::nil)) - .map(|version_id| version_id.to_string()); - tokio::spawn(async move { - let _ = rustfs_common::heal_channel::send_heal_request(request).await; - }); - } - - // Crash-consistency injection: hard power loss after the authoritative - // rename_data commit succeeded but before the stale part.N.meta cleanup. - // The new version is durably committed and visible, so a crash here must - // leave the object readable as the new version; the un-reclaimed staging - // parts are swept by a retried completion or upload GC (rustfs/backlog#946). - // Compiles to a no-op outside `#[cfg(test)]`. - if crash_inject::should_crash_at(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object) { - return Err(StorageError::Unexpected); - } - - // backlog#946: reclaim the stale per-part metadata (and any superfluous - // part.N data files no longer in the completed set) only *after* the - // authoritative rename_data commit above has succeeded. If rename_data - // fails write quorum and returns via `?`, the upload directory must keep - // its part.N.meta so a retried CompleteMultipartUpload can still read the - // parts; deleting them before the commit would strand the upload - // permanently. This mirrors the "clean up only after commit" pattern - // already used for the old data-dir GC and the upload-dir delete_all below. - self.cleanup_multipart_path(&parts).await; - - if let Some(old_dir) = op_old_dir { - let committed_dir = fi.data_dir.unwrap_or_default().to_string(); - // backlog#898: best-effort reclaim of the dereferenced old data dir. - // Returns a receipt (never `Err`); a failed GC must not turn an - // already-committed multipart completion into a 503. - let cleanup = self - .commit_rename_data_dir(&cleanup_disks, bucket, object, &old_dir.to_string(), &committed_dir, write_quorum) - .await; - self.report_old_data_dir_cleanup(bucket, object, &old_dir.to_string(), &cleanup) - .await; - } - - if let Some(stage_start) = complete_tail_stage_start { - rustfs_io_metrics::record_put_object_stage_duration( - "multipart_complete_tail", - stage_start.elapsed().as_secs_f64() * 1000.0, - ); - } - - #[cfg(test)] - pause_multipart_commit(bucket, object, MultipartCommitPause::AfterRename).await; - - let cleanup_store = self.clone(); - let cleanup_upload_id_path = upload_id_path.clone(); - let cleanup_bucket = bucket.to_owned(); - let cleanup_object = object.to_owned(); - let cleanup_upload_id = upload_id.to_owned(); - let cleanup_handle = tokio::spawn(async move { + let commit_set = self.clone(); + let commit_bucket = bucket.to_owned(); + let commit_object = object.to_owned(); + let commit_upload_id = upload_id.to_owned(); + let commit_upload_id_path = upload_id_path.clone(); + let commit_version_suspended = opts.version_suspended; + let commit_is_versioned = opts.versioned || opts.version_suspended; + let commit_capacity_scope_token = opts.capacity_scope_token; + let commit_object_lock_guard = object_lock_guard.take(); + let detach_commit_owner = commit_object_lock_guard.is_some() || upload_guard.is_some(); + let commit = async move { + let _object_lock_guard = commit_object_lock_guard; let _upload_guard = upload_guard; - if let Err(err) = cleanup_store - .delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &cleanup_upload_id_path, write_quorum) + let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); + + // Crash-consistency injection: hard power loss after the upload is fully + // staged and locked but before the authoritative rename_data commit. No + // disk has moved the staged data, so a crash here must leave any prior + // committed version byte-for-byte intact (rustfs/backlog#864) and the + // upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`. + if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, &commit_object) { + return Err(StorageError::Unexpected); + } + + // The trailing `_` drops the rename_data old-size backfill + // (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit + // `get_object_info` lookup, so the backfill has no consumer here yet. + if let Some(proof) = transaction_fencing_proof.as_ref() + && !object_transaction_fencing_fleet_proof_matches(proof) + { + return Err(Error::other( + "object transaction fencing fleet capability changed during complete_multipart_upload", + )); + } + if let Some(expected) = transaction_epoch_fence { + #[cfg(test)] + pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeTransactionEpochVerify).await; + verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?; + } + let (online_disks, convergence, op_old_dir, cleanup_disks, _) = SetDisks::rename_data( + &shuffle_disks, + RUSTFS_META_MULTIPART_BUCKET, + &commit_upload_id_path, + &parts_metadatas, + &commit_bucket, + &commit_object, + write_quorum, + ) + .await?; + + // Detach admission before any post-commit await: client cancellation + // must not couple durable convergence repair to cleanup work. + if convergence.needs_heal() { + let mut request = rustfs_common::heal_channel::create_heal_request_with_options( + commit_bucket.clone(), + Some(commit_object.clone()), + false, + Some(HealChannelPriority::Normal), + Some(commit_set.pool_index), + Some(commit_set.set_index), + ); + request.object_version_id = fi + .version_id + .or_else(|| commit_version_suspended.then(Uuid::nil)) + .map(|version_id| version_id.to_string()); + tokio::spawn(async move { + let _ = rustfs_common::heal_channel::send_heal_request(request).await; + }); + } + + if let Some(old_dir) = op_old_dir { + commit_set + .persist_old_data_cleanup_receipts( + &cleanup_disks, + &commit_bucket, + &commit_object, + old_dir, + fi.data_dir, + transaction_epoch, + ) + .await; + } + + // Crash-consistency injection: hard power loss after the authoritative + // rename_data commit succeeded but before the stale part.N.meta cleanup. + // The new version is durably committed and visible, so a crash here must + // leave the object readable as the new version; the un-reclaimed staging + // parts are swept by a retried completion or upload GC (rustfs/backlog#946). + // Compiles to a no-op outside `#[cfg(test)]`. + if crash_inject::should_crash_at(CrashPoint::MultipartAfterCommitBeforePartsCleanup, &commit_object) { + return Err(StorageError::Unexpected); + } + + if let Some(committed_slot) = online_disks.iter().position(Option::is_some) { + fi = parts_metadatas[committed_slot].clone(); + } + let committed_dir = fi.data_dir.unwrap_or_default().to_string(); + + commit_set.record_capacity_scope_if_needed(commit_capacity_scope_token, &online_disks); + + fi.is_latest = true; + + #[cfg(test)] + pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeObjectPublication).await; + + commit_set + .invalidate_get_object_metadata_cache(&commit_bucket, &commit_object) + .await; + + drop(_object_lock_guard); // release the object lock before multipart cleanup tail IO. + + #[cfg(test)] + pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::AfterObjectPublication).await; + + // backlog#946: reclaim the stale per-part metadata (and any superfluous + // part.N data files no longer in the completed set) only *after* the + // authoritative rename_data commit above has succeeded. If rename_data + // fails write quorum and returns via `?`, the upload directory must keep + // its part.N.meta so a retried CompleteMultipartUpload can still read the + // parts; deleting them before the commit would strand the upload + // permanently. This mirrors the "clean up only after commit" pattern + // already used for the old data-dir GC and the upload-dir delete_all below. + commit_set.cleanup_multipart_path(&parts).await; + + if let Some(old_dir) = op_old_dir { + // backlog#898: best-effort reclaim of the dereferenced old data dir. + // Returns a receipt (never `Err`); a failed GC must not turn an + // already-committed multipart completion into a 503. + let cleanup = commit_set + .commit_rename_data_dir( + &cleanup_disks, + &commit_bucket, + &commit_object, + &old_dir.to_string(), + &committed_dir, + write_quorum, + ) + .await; + commit_set + .report_old_data_dir_cleanup(&commit_bucket, &commit_object, &old_dir.to_string(), &cleanup) + .await; + } + + if let Some(stage_start) = complete_tail_stage_start { + rustfs_io_metrics::record_put_object_stage_duration( + "multipart_complete_tail", + stage_start.elapsed().as_secs_f64() * 1000.0, + ); + } + + #[cfg(test)] + pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::AfterRename).await; + + if let Err(err) = commit_set + .delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &commit_upload_id_path, write_quorum) .await { warn!( - bucket = %cleanup_bucket, - object = %cleanup_object, - upload_id = %cleanup_upload_id, + bucket = %commit_bucket, + object = %commit_object, + upload_id = %commit_upload_id, error = ?err, "completed multipart upload staging cleanup did not reach write quorum" ); } - }); - if let Err(err) = cleanup_handle.await { - warn!( - bucket = %bucket, - object = %object, - upload_id = %upload_id, - error = ?err, - "completed multipart upload staging cleanup task failed" - ); + + drop(_upload_guard); + + Ok(ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned)) + }; + + if detach_commit_owner { + tokio::spawn(commit) + .await + .map_err(|err| Error::other(format!("complete_multipart_upload commit task failed: {err}")))? + } else { + commit.await } - drop(object_lock_guard); // drop object lock guard to release the lock - - for (i, op_disk) in online_disks.iter().enumerate() { - if let Some(disk) = op_disk - && disk.is_online().await - { - fi = parts_metadatas[i].clone(); - break; - } - } - - self.record_capacity_scope_if_needed(opts.capacity_scope_token, &online_disks); - - fi.is_latest = true; - - self.invalidate_get_object_metadata_cache(bucket, object).await; - - Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended)) } } @@ -2452,9 +2527,10 @@ fn resolve_complete_etag(opts: &ObjectOptions, uploaded_parts: &[CompletePart]) mod tests { use super::*; use crate::config::storageclass::lookup_config_for_pools_without_env; - use crate::disk::DiskAPI as _; + use crate::disk::{DiskAPI as _, ReadOptions}; use crate::disk::{endpoint::Endpoint, format::FormatV3}; use crate::layout::endpoints::SetupType; + use crate::services::notification_sys::install_remote_version_state_fleet_proof_for_test; // No-locker helpers resolve to the isolated-context variants (see // `hermetic_set_disks_isolated`); the guard-based tests build through // `hermetic_set_disks_with_lockers`, which stays on the bootstrap context @@ -2463,6 +2539,7 @@ mod tests { hermetic_set_disks_for_pool_with_default_parity_isolated as hermetic_set_disks_for_pool_with_default_parity, hermetic_set_disks_isolated as hermetic_set_disks, hermetic_set_disks_with_lockers, }; + use crate::set_disk::ops::object::{PutObjectCommitBarrier, PutObjectCommitPause}; use crate::storage_api_contracts::namespace::NamespaceLocking as _; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; use rustfs_config::server_config::KVS; @@ -2865,6 +2942,208 @@ mod tests { } } + async fn object_transaction_epochs(disks: &[DiskStore], bucket: &str, object: &str) -> Vec> { + let mut epochs = Vec::with_capacity(disks.len()); + for (disk_index, disk) in disks.iter().enumerate() { + let file_info = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .unwrap_or_else(|err| panic!("disk {disk_index} should persist object metadata: {err}")); + epochs.push( + file_info + .object_transaction_epoch() + .unwrap_or_else(|err| panic!("disk {disk_index} transaction epoch should decode: {err}")), + ); + } + epochs + } + + #[tokio::test] + #[serial(storage_class_env)] + async fn object_transaction_fencing_requires_live_fleet_proof_before_multipart_commit() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-transaction-fencing-no-proof"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + let (upload_id, parts) = stage_upload_with_create_opts( + &set_disks, + bucket, + object, + b"must-not-complete-without-proof", + &ObjectOptions::default(), + ) + .await; + + let err = temp_env::async_with_vars( + [ + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")), + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")), + ], + async { + set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default()) + .await + }, + ) + .await + .expect_err("multipart completion must fail closed without a live fleet proof"); + + assert!( + err.to_string() + .contains("object transaction fencing requires a live fleet capability proof"), + "unexpected error: {err:?}" + ); + set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect_err("failed fenced completion must not publish object metadata"); + } + + #[tokio::test] + #[serial(storage_class_env)] + async fn object_transaction_fencing_persists_epoch_on_multipart_commit() { + let _proof = install_remote_version_state_fleet_proof_for_test("object-transaction-fencing-test"); + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-object-transaction-epoch"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + let (upload_id, parts) = + stage_upload_with_create_opts(&set_disks, bucket, object, b"multipart fenced epoch", &ObjectOptions::default()).await; + + temp_env::async_with_vars( + [ + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")), + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")), + ], + async { + set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default()) + .await + .expect("fenced multipart completion should commit with a live proof"); + }, + ) + .await; + + let epochs = object_transaction_epochs(&disk_stores, bucket, object).await; + let first = epochs[0].expect("fenced multipart completion should persist an epoch"); + assert!(!first.is_nil()); + assert!(epochs.into_iter().all(|epoch| epoch == Some(first))); + } + + #[tokio::test] + #[serial(storage_class_env)] + async fn object_transaction_fencing_rejects_stale_multipart_epoch() { + let _proof = install_remote_version_state_fleet_proof_for_test("object-transaction-fencing-test"); + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-object-transaction-stale-epoch"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + + temp_env::async_with_vars( + [ + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")), + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")), + ], + async { + let mut initial_reader = PutObjReader::from_vec(b"initial fenced object".to_vec()); + set_disks + .put_object( + bucket, + object, + &mut initial_reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("initial fenced PUT should commit"); + let initial_epoch = object_transaction_epochs(&disk_stores, bucket, object) + .await + .into_iter() + .next() + .flatten() + .expect("initial fenced PUT should persist an epoch"); + + let (upload_id, parts) = + stage_upload_with_create_opts(&set_disks, bucket, object, b"stale multipart body", &ObjectOptions::default()) + .await; + let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::BeforeTransactionEpochVerify); + let stale_set = Arc::clone(&set_disks); + let stale = tokio::spawn(async move { + stale_set + .clone() + .complete_multipart_upload( + bucket, + object, + &upload_id, + parts, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + }); + barrier.wait_until_paused().await; + + let mut winner_reader = PutObjReader::from_vec(b"winning put body".to_vec()); + set_disks + .put_object( + bucket, + object, + &mut winner_reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("concurrent fenced PUT should advance the epoch"); + let winning_epoch = object_transaction_epochs(&disk_stores, bucket, object) + .await + .into_iter() + .next() + .flatten() + .expect("winning fenced PUT should persist an epoch"); + assert_ne!(winning_epoch, initial_epoch); + + barrier.release(); + let err = stale + .await + .expect("stale multipart task should not panic") + .expect_err("stale epoch multipart completion must be rejected"); + assert_eq!(err, StorageError::PreconditionFailed); + + let final_epochs = object_transaction_epochs(&disk_stores, bucket, object).await; + assert!(final_epochs.into_iter().all(|epoch| epoch == Some(winning_epoch))); + let mut reader = set_disks + .get_object_reader( + bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("winning object should remain readable"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("winning body should stream"); + assert_eq!(restored, b"winning put body"); + }, + ) + .await; + } + #[tokio::test] async fn complete_multipart_quota_rejection_preserves_destination_and_upload() { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; @@ -4883,6 +5162,250 @@ mod tests { .await; } + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn cancelled_complete_keeps_upload_lock_through_tail_cleanup() { + temp_env::async_with_vars( + [ + (crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true")), + (rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60")), + ], + async { + let manager = Arc::new(rustfs_lock::GlobalLockManager::new()); + let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager)))); + let lockers: Vec> = vec![signaling.clone()]; + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await; + let bucket = "multipart-cancelled-tail-lock-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + let (upload_id, parts) = + stage_upload_with_create_opts(&set_disks, bucket, object, &[0x53; 4096], &ObjectOptions::default()).await; + let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id); + signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path)); + signaling.clear_observed(); + let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await; + let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename); + + let complete_store = set_disks.clone(); + let complete_upload_id = upload_id.clone(); + let complete = tokio::spawn(async move { + complete_store + .complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + + let abort_store = set_disks.clone(); + let abort_upload_id = upload_id.clone(); + let abort = tokio::spawn(async move { + abort_store + .abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default()) + .await + }); + signaling.wait_for_attempts(2).await; + tokio::task::yield_now().await; + assert!(!abort.is_finished(), "abort must wait while completion tail owns the upload lock"); + + complete.abort(); + assert!( + complete + .await + .expect_err("the completion request should be cancellable while the tail is paused") + .is_cancelled() + ); + tokio::task::yield_now().await; + assert!(!abort.is_finished(), "cancelling the completion waiter must not release the upload lock"); + + barrier.release(); + let abort_err = abort + .await + .expect("abort task should not panic") + .expect_err("the committed upload should no longer exist when abort acquires the lock"); + assert!( + matches!(abort_err, StorageError::InvalidUploadID(..)), + "abort should return InvalidUploadID after the detached completion tail, got {abort_err:?}" + ); + }, + ) + .await; + } + + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn complete_releases_object_lock_before_cleanup_and_keeps_upload_lock() { + temp_env::async_with_vars( + [ + (crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true")), + (rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60")), + ], + async { + let manager = Arc::new(rustfs_lock::GlobalLockManager::new()); + let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager)))); + let lockers: Vec> = vec![signaling.clone()]; + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await; + let bucket = "multipart-object-lock-short-tail-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + let completed_body = vec![0x63; 4096]; + let replacement_body = vec![0x64; 4096]; + let (upload_id, parts) = + stage_upload_with_create_opts(&set_disks, bucket, object, &completed_body, &ObjectOptions::default()).await; + let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id); + signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path)); + signaling.clear_observed(); + let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await; + let completion_barrier = + MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterObjectPublication); + + let complete_store = set_disks.clone(); + let complete_upload_id = upload_id.clone(); + let complete = tokio::spawn(async move { + complete_store + .complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default()) + .await + }); + completion_barrier.wait_until_paused().await; + + let mut reader = tokio::time::timeout( + Duration::from_secs(10), + set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()), + ) + .await + .expect("GET should not wait for multipart cleanup after object publication") + .expect("completed object should be readable while the upload tail is paused"); + let mut observed_body = Vec::new(); + tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut observed_body)) + .await + .expect("completed object body should stream while the upload tail is paused") + .expect("completed object body should read successfully"); + assert_eq!(observed_body, completed_body); + + let put_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace); + let put_store = set_disks.clone(); + let put_payload = replacement_body.clone(); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(put_payload); + put_store + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + }); + put_barrier.wait_until_paused().await; + + let abort_store = set_disks.clone(); + let abort_upload_id = upload_id.clone(); + let abort = tokio::spawn(async move { + abort_store + .abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default()) + .await + }); + signaling.wait_for_attempts(2).await; + tokio::task::yield_now().await; + assert!( + !abort.is_finished(), + "abort must still wait while the completion tail owns the upload lock" + ); + + complete.abort(); + assert!( + complete + .await + .expect_err("the completion waiter should remain cancellable after object publication") + .is_cancelled() + ); + tokio::task::yield_now().await; + assert!(!abort.is_finished(), "cancelling the waiter must not release the upload lock"); + + completion_barrier.release(); + let abort_err = abort + .await + .expect("abort task should not panic") + .expect_err("the committed upload should no longer exist after the detached tail drains"); + assert!( + matches!(abort_err, StorageError::InvalidUploadID(..)), + "abort should return InvalidUploadID after the completion tail, got {abort_err:?}" + ); + + put_barrier.release(); + put.await + .expect("same-key PUT task should not panic") + .expect("same-key PUT should commit after the object lock is released early"); + + let mut reader = set_disks + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("final object should be readable"); + let mut final_body = Vec::new(); + reader + .stream + .read_to_end(&mut final_body) + .await + .expect("final object should stream fully"); + assert_eq!(final_body, replacement_body); + }, + ) + .await; + } + + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn complete_keeps_object_lock_until_publication_fence() { + temp_env::async_with_vars( + [ + (crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true")), + (rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60")), + ], + async { + let manager = Arc::new(rustfs_lock::GlobalLockManager::new()); + let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager)))); + let lockers: Vec> = vec![signaling.clone()]; + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await; + let bucket = "multipart-publication-fence-lock-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + let (upload_id, parts) = + stage_upload_with_create_opts(&set_disks, bucket, object, &[0x65; 4096], &ObjectOptions::default()).await; + let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await; + let completion_barrier = + MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::BeforeObjectPublication); + + let complete_store = set_disks.clone(); + let complete_upload_id = upload_id.clone(); + let complete = tokio::spawn(async move { + complete_store + .complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default()) + .await + }); + completion_barrier.wait_until_paused().await; + + let before_namespace_barrier = + PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace); + let after_namespace_barrier = + PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace); + let put_store = set_disks.clone(); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![0x66; 4096]); + put_store + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + }); + before_namespace_barrier.wait_until_paused().await; + before_namespace_barrier.release_and_wait_until_namespace_pending().await; + + completion_barrier.release(); + after_namespace_barrier.wait_until_paused().await; + after_namespace_barrier.release(); + complete + .await + .expect("completion task should not panic") + .expect("completion should commit after publication fence"); + put.await + .expect("same-key PUT task should not panic") + .expect("same-key PUT should commit after completion publishes and releases the object lock"); + }, + ) + .await; + } + #[tokio::test(flavor = "multi_thread")] #[serial] async fn complete_validates_parts_after_an_inflight_upload_part_commit() { @@ -6108,6 +6631,24 @@ mod tests { (body, etag) } + async fn current_data_dir(disk: &DiskStore, bucket: &str, object: &str) -> Uuid { + disk.read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("current object metadata should read") + .data_dir + .expect("test object should be stored out-of-line") + } + + async fn data_dir_exists(disk: &DiskStore, bucket: &str, object: &str, data_dir: Uuid) -> bool { + disk.read_all(bucket, &format!("{object}/{data_dir}/part.1")).await.is_ok() + } + + async fn cleanup_receipt_exists(disk: &DiskStore, bucket: &str, object: &str, data_dir: Uuid) -> bool { + disk.read_all(bucket, &old_data_cleanup_receipt_path(object, data_dir)) + .await + .is_ok() + } + async fn upload_is_listed(set_disks: &Arc, bucket: &str, object: &str, upload_id: &str) -> bool { let page = set_disks .list_multipart_uploads_for_incarnation(bucket, object, None, None, None, 1000, None) @@ -6228,6 +6769,106 @@ mod tests { let (body_after, _) = read_object(&set_disks, bucket, object).await; assert_eq!(body_after, new, "reclaiming the leftover upload must not disturb the committed object"); } + + #[tokio::test] + #[serial(storage_class_env)] + async fn post_commit_crash_receipt_reclaims_old_data_after_restart() { + let _proof = install_remote_version_state_fleet_proof_for_test("object-transaction-fencing-test"); + let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-crash-old-data-receipt"; + let object = "crash-old-data-object"; + make_bucket_on_all(&disk_stores, bucket).await; + + temp_env::async_with_vars( + [ + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")), + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")), + ], + async { + let old = payload(0x51); + let (u_old, parts_old) = stage_upload(&set_disks, bucket, object, &old).await; + complete(&set_disks, bucket, object, &u_old, parts_old) + .await + .expect("the old version should commit"); + let old_dir = current_data_dir(&disk_stores[0], bucket, object).await; + + let new = payload(0x52); + let (u_new, parts_new) = stage_upload(&set_disks, bucket, object, &new).await; + crash_inject::arm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object); + let crashed = complete(&set_disks, bucket, object, &u_new, parts_new).await; + assert!( + matches!(crashed, Err(StorageError::Unexpected)), + "the post-commit crash point must surface as unexpected, got {crashed:?}" + ); + crash_inject::disarm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object); + + let (body, _) = read_object(&set_disks, bucket, object).await; + assert_eq!(body, new, "the committed replacement must remain readable after the crash"); + for disk in &disk_stores { + assert!( + cleanup_receipt_exists(disk, bucket, object, old_dir).await, + "post-commit crash must leave a durable old-data cleanup receipt" + ); + assert!( + data_dir_exists(disk, bucket, object, old_dir).await, + "post-commit crash must leave old data for restart reconciliation" + ); + } + + let restarted_endpoints = temp_dirs + .iter() + .enumerate() + .map(|(disk_idx, dir)| { + let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")) + .expect("endpoint should parse"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(disk_idx); + endpoint + }) + .collect::>(); + let mut reloaded = Vec::with_capacity(restarted_endpoints.len()); + for endpoint in &restarted_endpoints { + reloaded.push( + new_disk( + endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("disk should restart"), + ); + } + let restarted_set = SetDisks::new_with_instance_ctx( + "restart-cleanup-receipt-test-owner".to_string(), + Arc::new(RwLock::new(reloaded.iter().cloned().map(Some).collect())), + 4, + 2, + 0, + 0, + restarted_endpoints, + set_disks.format.clone(), + Vec::new(), + Arc::new(crate::runtime::instance::InstanceContext::new()), + ) + .await; + let removed = restarted_set + .reconcile_old_data_cleanup_receipts(bucket, object) + .await + .expect("restart receipt reconciliation should succeed"); + assert_eq!(removed, 4, "restart reconciler should delete all receipt targets"); + for disk in &reloaded { + assert!( + !data_dir_exists(disk, bucket, object, old_dir).await, + "restart reconciler must reclaim the old data dir" + ); + } + }, + ) + .await; + } } #[test] diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index caa16bf04..a5f7c953f 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -44,10 +44,11 @@ use crate::bucket::replication::{ DeleteReplicationConfigSnapshot, VersionPurgeStatusType, replication_state_to_filemeta, version_purge_status_to_filemeta, }; use crate::diagnostics::get::GetObjectFailureReason; -use crate::disk::OldCurrentSize; +use crate::disk::{DataDirDeleteStatus, OldCurrentSize}; use crate::error::is_err_invalid_upload_id; use crate::object_api::NamespaceLockFence; use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed}; +use crate::services::notification_sys::RemoteVersionStateFleetProofToken; use crate::services::tier::tier::{TierConfigMgr, TierOperationLease}; use crate::store::ECStore; use crate::store::utils::clean_metadata; @@ -55,6 +56,9 @@ use futures::FutureExt as _; use http::HeaderValue; use rustfs_utils::path::decode_dir_object; use std::future::Future; +use std::sync::OnceLock; + +const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json"; #[inline] fn duration_millis_f64(duration: std::time::Duration) -> f64 { @@ -65,6 +69,133 @@ fn committed_response_metadata_slot(committed_disks: &[Option], fallback_s committed_disks.iter().position(Option::is_some).unwrap_or(fallback_slot) } +pub(in crate::set_disk::ops) fn assign_object_transaction_epoch( + shuffle_disks: &[Option], + parts_metadatas: &mut [FileInfo], +) -> Uuid { + let epoch = Uuid::new_v4(); + for (disk, file_info) in shuffle_disks.iter().zip(parts_metadatas.iter_mut()) { + if disk.is_some() { + file_info.set_object_transaction_epoch(epoch); + } + } + epoch +} + +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct OldDataCleanupReceiptRecord { + epoch: String, + old_data_dir: String, + committed_data_dir: Option, +} + +#[derive(Clone, Copy)] +struct OldDataCleanupReceipt { + epoch: Uuid, + old_data_dir: Uuid, + committed_data_dir: Option, +} + +impl OldDataCleanupReceipt { + fn new(epoch: Uuid, old_data_dir: Uuid, committed_data_dir: Option) -> Self { + Self { + epoch, + old_data_dir, + committed_data_dir, + } + } + + fn encode(self) -> disk::error::Result { + let record = OldDataCleanupReceiptRecord { + epoch: self.epoch.to_string(), + old_data_dir: self.old_data_dir.to_string(), + committed_data_dir: self.committed_data_dir.map(|dir| dir.to_string()), + }; + Ok(Bytes::from(serde_json::to_vec(&record)?)) + } + + fn decode(data: &[u8]) -> disk::error::Result { + let record: OldDataCleanupReceiptRecord = serde_json::from_slice(data)?; + let epoch = Uuid::parse_str(&record.epoch).map_err(DiskError::other)?; + let old_data_dir = Uuid::parse_str(&record.old_data_dir).map_err(DiskError::other)?; + let committed_data_dir = record + .committed_data_dir + .as_deref() + .map(Uuid::parse_str) + .transpose() + .map_err(DiskError::other)?; + if epoch.is_nil() || old_data_dir.is_nil() || committed_data_dir.is_some_and(|dir| dir.is_nil()) { + return Err(DiskError::FileCorrupt); + } + Ok(Self::new(epoch, old_data_dir, committed_data_dir)) + } +} + +pub(in crate::set_disk::ops) fn old_data_cleanup_receipt_path(object: &str, old_data_dir: Uuid) -> String { + path_join_buf(&[object, &old_data_dir.to_string(), OLD_DATA_CLEANUP_RECEIPT_FILE]) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(in crate::set_disk::ops) enum ObjectTransactionEpochFence { + Absent, + Present(Uuid), +} + +impl ObjectTransactionEpochFence { + fn from_file_info(file_info: &FileInfo) -> Result { + match file_info.object_transaction_epoch() { + Ok(Some(epoch)) => Ok(Self::Present(epoch)), + Ok(None) => Ok(Self::Absent), + Err(_) => Err(StorageError::FileCorrupt), + } + } +} + +pub(in crate::set_disk::ops) async fn read_object_transaction_epoch_fence( + set: &SetDisks, + bucket: &str, + object: &str, +) -> Result { + let current = set + .get_object_fileinfo( + bucket, + object, + &ObjectOptions { + no_lock: true, + metadata_cache_safe: false, + versioned: true, + ..Default::default() + }, + false, + false, + ) + .await; + match current { + Ok(snapshot) => ObjectTransactionEpochFence::from_file_info(snapshot.fi()), + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(ObjectTransactionEpochFence::Absent), + Err(err) => Err(err), + } +} + +pub(in crate::set_disk::ops) async fn verify_object_transaction_epoch_fence( + set: &SetDisks, + bucket: &str, + object: &str, + expected: ObjectTransactionEpochFence, +) -> Result<()> { + let current = read_object_transaction_epoch_fence(set, bucket, object).await?; + if current == expected { + Ok(()) + } else { + Err(StorageError::PreconditionFailed) + } +} + +fn old_data_cleanup_receipt_epoch_matches_current(receipt: OldDataCleanupReceipt, current: ObjectTransactionEpochFence) -> bool { + matches!(current, ObjectTransactionEpochFence::Present(epoch) if epoch == receipt.epoch) +} + #[cfg(test)] mod duration_metrics_tests { use super::duration_millis_f64; @@ -188,6 +319,74 @@ async fn get_object_reader_with_context( GetObjectReader::new_with_resolver(reader, range, object_info, opts, headers, ctx.object_encryption_resolver()).await } +fn data_read_metadata_early_stop_request_shape_allowed(range: &Option, opts: &ObjectOptions) -> bool { + range.is_none() + && opts.part_number.is_none() + && opts.version_id.is_none() + && !opts.incl_free_versions + && !opts.skip_free_version + && !opts.raw_data_movement_read + && !opts.data_movement + && !crate::object_api::restore_request_active(opts) +} + +#[cfg(test)] +mod data_read_metadata_early_stop_request_shape_tests { + use super::*; + + #[test] + fn data_read_metadata_early_stop_only_allows_whole_latest_plain_get_shape() { + assert!(data_read_metadata_early_stop_request_shape_allowed(&None, &ObjectOptions::default())); + + let range = Some(HTTPRangeSpec { + is_suffix_length: false, + start: 0, + end: 0, + }); + assert!(!data_read_metadata_early_stop_request_shape_allowed(&range, &ObjectOptions::default())); + + let part_opts = ObjectOptions { + part_number: Some(1), + ..Default::default() + }; + assert!(!data_read_metadata_early_stop_request_shape_allowed(&None, &part_opts)); + + let version_opts = ObjectOptions { + version_id: Some(Uuid::new_v4().to_string()), + ..Default::default() + }; + assert!(!data_read_metadata_early_stop_request_shape_allowed(&None, &version_opts)); + + let incl_free_opts = ObjectOptions { + incl_free_versions: true, + ..Default::default() + }; + assert!(!data_read_metadata_early_stop_request_shape_allowed(&None, &incl_free_opts)); + + let skip_free_opts = ObjectOptions { + skip_free_version: true, + ..Default::default() + }; + assert!(!data_read_metadata_early_stop_request_shape_allowed(&None, &skip_free_opts)); + + let data_movement_opts = ObjectOptions { + data_movement: true, + ..Default::default() + }; + assert!(!data_read_metadata_early_stop_request_shape_allowed(&None, &data_movement_opts)); + + let raw_data_movement_opts = ObjectOptions { + raw_data_movement_read: true, + ..Default::default() + }; + assert!(!data_read_metadata_early_stop_request_shape_allowed(&None, &raw_data_movement_opts)); + + let mut restore_opts = ObjectOptions::default(); + restore_opts.transition.restore_request.days = Some(1); + assert!(!data_read_metadata_early_stop_request_shape_allowed(&None, &restore_opts)); + } +} + /// Length of the full plaintext body when — and only when — this read's output /// is exactly the object's complete plaintext, so the app-layer body cache may /// serve it in place of the erasure read. @@ -431,7 +630,16 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { let (snapshot, prepared_object_info) = if let Some(prepared) = take_prepared_get_object_metadata() { (prepared.snapshot, prepared.object_info) } else { - match self.get_object_fileinfo(bucket, object, opts, true, true).await { + match self + .get_object_fileinfo( + bucket, + object, + opts, + true, + data_read_metadata_early_stop_request_shape_allowed(&range, opts), + ) + .await + { Ok(snapshot) => (snapshot, None), Err(err) => { rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_stage_start.elapsed().as_secs_f64()); @@ -583,7 +791,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { } } - let erasure = erasure_from_file_info(fi, fi.uses_legacy_checksum)?; + let erasure = self.erasure_cache.get_for_file_info(fi)?; let read_length = erasure.shard_file_offset(0, object_size, object_size); let total_shards = data_shards + fi.erasure.parity_blocks; let (_disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, files, fi); @@ -752,6 +960,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { if let Some(body) = Self::try_get_object_direct_data_shards_with_fileinfo( bucket, object, + Arc::clone(&self.erasure_cache), fi, files, disks, @@ -787,6 +996,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { Self::get_object_with_fileinfo( bucket, object, + Arc::clone(&self.erasure_cache), 0, object_info.size, &mut output, @@ -830,6 +1040,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { match Self::get_object_decode_reader_with_fileinfo( bucket, object, + Arc::clone(&self.erasure_cache), fi, files, disks, @@ -896,6 +1107,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { let set_index = self.set_index; let pool_index = self.pool_index; let skip_verify = opts.skip_verify_bitrot; + let erasure_cache = Arc::clone(&self.erasure_cache); let (fi, files, disks) = snapshot.into_owned(); tokio::spawn(async move { let _guard = read_lock_guard; @@ -907,6 +1119,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { let producer_result = Self::get_object_with_fileinfo( &bucket, &object, + erasure_cache, offset, length, &mut writer, @@ -1001,6 +1214,114 @@ fn delete_file_info_with_replication_transport_metadata(fi: &FileInfo) -> FileIn } impl SetDisks { + pub(in crate::set_disk) async fn persist_old_data_cleanup_receipts( + &self, + disks: &[Option], + bucket: &str, + object: &str, + old_data_dir: Uuid, + committed_data_dir: Option, + epoch: Option, + ) { + let Some(epoch) = epoch else { return }; + if committed_data_dir == Some(old_data_dir) { + return; + } + let receipt = OldDataCleanupReceipt::new(epoch, old_data_dir, committed_data_dir); + let Ok(encoded) = receipt.encode() else { + return; + }; + let path = old_data_cleanup_receipt_path(object, old_data_dir); + let futures = disks.iter().filter_map(|disk| { + disk.as_ref().map(|disk| { + let disk = disk.clone(); + let encoded = encoded.clone(); + let bucket = bucket.to_owned(); + let path = path.clone(); + async move { disk.write_all(&bucket, &path, encoded).await } + }) + }); + for result in join_all(futures).await { + if let Err(err) = result { + debug!( + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket, + object, + old_dir = %old_data_dir, + error = %err, + state = "cleanup_receipt_persist_failed", + "SetDisk old-data cleanup receipt persist failed" + ); + } + } + } + + pub(in crate::set_disk) async fn reconcile_old_data_cleanup_receipts( + &self, + bucket: &str, + object: &str, + ) -> disk::error::Result { + if object_transaction_fencing_fleet_proof().is_none() { + return Ok(0); + } + let current = read_object_transaction_epoch_fence(self, bucket, object) + .await + .map_err(DiskError::from)?; + let disks = self.get_disks_internal().await; + let mut removed = 0usize; + + for disk in disks.iter().flatten() { + let entries = match disk.list_dir("", bucket, object, 0).await { + Ok(entries) => entries, + Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => continue, + Err(err) => return Err(err), + }; + for entry in entries { + let Some(name) = entry.strip_suffix(SLASH_SEPARATOR) else { continue }; + let Ok(data_dir) = Uuid::parse_str(name) else { continue }; + if data_dir.is_nil() { + continue; + } + + let receipt_path = old_data_cleanup_receipt_path(object, data_dir); + let receipt_bytes = match disk.read_all(bucket, &receipt_path).await { + Ok(bytes) => bytes, + Err(DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound) => continue, + Err(err) => return Err(err), + }; + let receipt = OldDataCleanupReceipt::decode(&receipt_bytes)?; + if receipt.old_data_dir != data_dir + || receipt.committed_data_dir == Some(receipt.old_data_dir) + || !old_data_cleanup_receipt_epoch_matches_current(receipt, current) + { + continue; + } + + let old_path = format!("{object}/{}", receipt.old_data_dir); + match disk + .delete_data_dir( + bucket, + &old_path, + DeleteOptions { + recursive: true, + immediate: true, + ..Default::default() + }, + ) + .await + { + Ok(DataDirDeleteStatus::Deleted) => removed += 1, + Ok(DataDirDeleteStatus::Deferred) => {} + Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => {} + Err(err) => return Err(err), + } + } + } + + Ok(removed) + } + async fn validate_bucket_incarnation(&self, bucket: &str, expected: Uuid) -> Result<()> { let current = metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await?; if current != expected { @@ -1109,11 +1430,13 @@ impl SetDisks { let tmp_object = format!("{}/{}/part.1", tmp_dir, fi.data_dir.unwrap()); + let mut tmp_cleanup_owned = false; let result: Result<(ObjectInfo, Option)> = async { let erasure = Arc::new(erasure_from_file_info(&fi, false)?); let put_object_size = known_put_object_storage_size(data.size()); - let is_inline_buffer = storage_class_config.should_inline(erasure.shard_file_size(put_object_size), opts.versioned); + let is_inline_buffer = + storage_class_config.should_inline(erasure.shard_file_size(put_object_size), erasure.data_shards, opts.versioned); let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled(); let shard_file_size = erasure.shard_file_size(put_object_size); @@ -1600,169 +1923,294 @@ impl SetDisks { }); } - let rename_stage_start = Instant::now(); - let (online_disks, convergence, op_old_dir, cleanup_disks, old_current_size) = Self::rename_data( - &shuffle_disks, - RUSTFS_META_TMP_BUCKET, - tmp_dir.as_str(), - &parts_metadatas, - bucket, - object, - write_quorum, - ) - .await?; - // Do this before any post-commit await so request cancellation cannot - // bypass best-effort admission. A process crash before admission - // remains subject to the existing scanner reconciliation path. - if convergence.needs_heal() { - let mut request = rustfs_common::heal_channel::create_heal_request_with_options( - bucket.to_string(), - Some(object.to_string()), - false, - Some(HealChannelPriority::Normal), - Some(self.pool_index), - Some(self.set_index), - ); - request.object_version_id = committed_version_id.map(|version_id| version_id.to_string()); - tokio::spawn(async move { - let _ = rustfs_common::heal_channel::send_heal_request(request).await; - }); + let transaction_fencing_proof = object_transaction_fencing_fleet_proof(); + if object_transaction_fencing_requested() && transaction_fencing_proof.is_none() { + return Err(Error::other("object transaction fencing requires a live fleet capability proof")); } + let transaction_epoch_fence = if transaction_fencing_proof.is_some() { + Some(read_object_transaction_epoch_fence(self, bucket, object).await?) + } else { + None + }; + let transaction_epoch = + transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&shuffle_disks, &mut parts_metadatas)); - let rename_stage_elapsed = rename_stage_start.elapsed(); - let rename_stage_ms = rename_stage_elapsed.as_millis() as u64; + let commit_set = self.clone(); + let commit_bucket = bucket.to_owned(); + let commit_object = object.to_owned(); + let commit_tmp_dir = tmp_dir.clone(); + let commit_object_lock_guard = object_lock_guard.take(); + let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take(); + let detach_commit_owner = commit_object_lock_guard.is_some() || commit_bucket_lifecycle_guard.is_some(); + let commit_write_path_label = write_path.metric_label(); + let commit_is_versioned = opts.versioned || opts.version_suspended; + let commit_capacity_scope_token = opts.capacity_scope_token; + let commit_replication_state = replication_state_to_filemeta(&opts.put_replication_state()); + tmp_cleanup_owned = true; - self.invalidate_get_object_metadata_cache(bucket, object).await; - - // `rename_data` has completed the authoritative quorum commit. The - // exact old-data-dir reclamation below is best-effort space cleanup; - // it must not serialize the next operation on this object. - drop(object_lock_guard); - - rustfs_io_metrics::record_put_object_stage_duration("set_disk_rename", duration_millis_f64(rename_stage_elapsed)); - if (rename_stage_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS { - warn!( - event = EVENT_SET_DISK_COMMIT_TAIL_SLOW, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_SET_DISK, - stage = "rename_data", - bucket = %bucket, - object = %object, - tmp_dir = %tmp_dir, - duration_ms = { rename_stage_ms }, + let commit = async move { + let _object_lock_guard = commit_object_lock_guard; + let _bucket_lifecycle_guard = commit_bucket_lifecycle_guard; + let rename_stage_start = Instant::now(); + let pre_rename_result: Result<()> = async { + if let Some(proof) = transaction_fencing_proof.as_ref() + && !object_transaction_fencing_fleet_proof_matches(proof) + { + return Err(Error::other("object transaction fencing fleet capability changed during put_object")); + } + if let Some(expected) = transaction_epoch_fence { + #[cfg(any(test, feature = "test-util"))] + pause_put_object_commit( + &commit_bucket, + &commit_object, + PutObjectCommitPause::BeforeTransactionEpochVerify, + ) + .await; + verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?; + } + Ok(()) + } + .await; + if let Err(err) = pre_rename_result { + if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await { + warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data"); + } else if issue3031_diag_enabled() { + warn!( + target: "rustfs_ecstore::set_disk", + bucket = %commit_bucket, + object = %commit_object, + tmp_dir = %commit_tmp_dir, + "issue3031_put_object_tmp_cleanup_done" + ); + } + return Err(err); + } + let rename_result = SetDisks::rename_data( + &shuffle_disks, + RUSTFS_META_TMP_BUCKET, + commit_tmp_dir.as_str(), + &parts_metadatas, + &commit_bucket, + &commit_object, write_quorum, - state = "slow", - "SetDisk commit tail stage is slow" - ); - } + ) + .await; + let (online_disks, convergence, op_old_dir, cleanup_disks, old_current_size) = match rename_result { + Ok(commit) => commit, + Err(err) => { + if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await { + warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data"); + } else if issue3031_diag_enabled() { + warn!( + target: "rustfs_ecstore::set_disk", + bucket = %commit_bucket, + object = %commit_object, + tmp_dir = %commit_tmp_dir, + "issue3031_put_object_tmp_cleanup_done" + ); + } + return Err(err.into()); + } + }; + // Do this before any post-commit await so request cancellation cannot + // bypass best-effort admission. A process crash before admission + // remains subject to the existing scanner reconciliation path. + if convergence.needs_heal() { + let mut request = rustfs_common::heal_channel::create_heal_request_with_options( + commit_bucket.clone(), + Some(commit_object.clone()), + false, + Some(HealChannelPriority::Normal), + Some(commit_set.pool_index), + Some(commit_set.set_index), + ); + request.object_version_id = committed_version_id.map(|version_id| version_id.to_string()); + tokio::spawn(async move { + let _ = rustfs_common::heal_channel::send_heal_request(request).await; + }); + } - let mut cleanup_stage_ms: Option = None; - if let Some(old_dir) = op_old_dir { - let committed_dir = committed_data_dir.unwrap_or_default().to_string(); - let cleanup_stage_start = Instant::now(); - // backlog#898: reclaiming the dereferenced old data dir is - // best-effort and returns a receipt (never `Err`). A failed GC - // here must not negate an already-committed, durable write, so we - // deliberately do NOT `?`-propagate it into a 503. On residue the - // report path emits the leak metric and enqueues a heal. - let cleanup = self - .commit_rename_data_dir(&cleanup_disks, bucket, object, &old_dir.to_string(), &committed_dir, write_quorum) + let rename_stage_elapsed = rename_stage_start.elapsed(); + let rename_stage_ms = rename_stage_elapsed.as_millis() as u64; + + if let Some(old_dir) = op_old_dir { + commit_set + .persist_old_data_cleanup_receipts( + &cleanup_disks, + &commit_bucket, + &commit_object, + old_dir, + committed_data_dir, + transaction_epoch, + ) + .await; + } + + commit_set + .invalidate_get_object_metadata_cache(&commit_bucket, &commit_object) .await; - let cleanup_elapsed = cleanup_stage_start.elapsed(); - let cleanup_ms = cleanup_elapsed.as_millis() as u64; - cleanup_stage_ms = Some(cleanup_ms); - rustfs_io_metrics::record_put_object_stage_duration( - "set_disk_old_data_cleanup", - duration_millis_f64(cleanup_elapsed), - ); - self.report_old_data_dir_cleanup(bucket, object, &old_dir.to_string(), &cleanup) - .await; - if (cleanup_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS { + + // `rename_data` has completed the authoritative quorum commit. The + // exact old-data-dir reclamation below is best-effort space cleanup; + // it must not serialize the next operation on this object. + drop(_object_lock_guard); + drop(_bucket_lifecycle_guard); + + rustfs_io_metrics::record_put_object_stage_duration("set_disk_rename", duration_millis_f64(rename_stage_elapsed)); + if (rename_stage_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS { warn!( event = EVENT_SET_DISK_COMMIT_TAIL_SLOW, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_SET_DISK, - stage = "commit_rename_data_dir", - bucket = %bucket, - object = %object, - tmp_dir = %tmp_dir, - old_dir = %old_dir, - duration_ms = cleanup_ms, + stage = "rename_data", + bucket = %commit_bucket, + object = %commit_object, + tmp_dir = %commit_tmp_dir, + duration_ms = { rename_stage_ms }, write_quorum, state = "slow", "SetDisk commit tail stage is slow" ); } + + let mut cleanup_stage_ms: Option = None; + if let Some(old_dir) = op_old_dir { + let committed_dir = committed_data_dir.unwrap_or_default().to_string(); + let cleanup_stage_start = Instant::now(); + // backlog#898: reclaiming the dereferenced old data dir is + // best-effort and returns a receipt (never `Err`). A failed GC + // here must not negate an already-committed, durable write, so we + // deliberately do NOT `?`-propagate it into a 503. On residue the + // report path emits the leak metric and enqueues a heal. + let cleanup = commit_set + .commit_rename_data_dir( + &cleanup_disks, + &commit_bucket, + &commit_object, + &old_dir.to_string(), + &committed_dir, + write_quorum, + ) + .await; + let cleanup_elapsed = cleanup_stage_start.elapsed(); + let cleanup_ms = cleanup_elapsed.as_millis() as u64; + cleanup_stage_ms = Some(cleanup_ms); + rustfs_io_metrics::record_put_object_stage_duration( + "set_disk_old_data_cleanup", + duration_millis_f64(cleanup_elapsed), + ); + commit_set + .report_old_data_dir_cleanup(&commit_bucket, &commit_object, &old_dir.to_string(), &cleanup) + .await; + if (cleanup_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS { + warn!( + event = EVENT_SET_DISK_COMMIT_TAIL_SLOW, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + stage = "commit_rename_data_dir", + bucket = %commit_bucket, + object = %commit_object, + tmp_dir = %commit_tmp_dir, + old_dir = %old_dir, + duration_ms = cleanup_ms, + write_quorum, + state = "slow", + "SetDisk commit tail stage is slow" + ); + } + } + + let committed_metadata_slot = committed_response_metadata_slot(&online_disks, response_metadata_slot); + let mut fi = std::mem::take(&mut parts_metadatas[committed_metadata_slot]); + + if is_compressed { + record_compression_total_memory(actual_size as u64, w_size as u64).await; + } + commit_set.record_capacity_scope_if_needed(commit_capacity_scope_token, &online_disks); + + fi.replication_state_internal = Some(commit_replication_state); + + fi.is_latest = true; + + if issue3031_diag_enabled() { + let online_success_count = online_disks.iter().filter(|disk| disk.is_some()).count(); + warn!( + target: "rustfs_ecstore::set_disk", + bucket = %commit_bucket, + object = %commit_object, + tmp_dir = %commit_tmp_dir, + data_dir = ?fi.data_dir, + write_quorum, + online_success_count, + op_old_dir = ?op_old_dir, + "issue3031_put_object_commit_succeeded" + ); + } + + let total_commit_tail_ms = rename_stage_start.elapsed().as_millis(); + if total_commit_tail_ms >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS { + warn!( + event = EVENT_SET_DISK_COMMIT_TAIL_SLOW, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + stage = "put_object_commit_tail", + bucket = %commit_bucket, + object = %commit_object, + tmp_dir = %commit_tmp_dir, + duration_ms = total_commit_tail_ms as u64, + write_quorum, + state = "slow", + "SetDisk commit tail is slow" + ); + } + + if issue3031_diag_enabled() { + warn!( + event = EVENT_SET_DISK_PUT_OBJECT_STAGE_SUMMARY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_SET_DISK, + bucket = %commit_bucket, + object = %commit_object, + write_quorum, + write_path = commit_write_path_label, + writer_setup_ms, + encode_ms, + rename_ms = rename_stage_ms, + cleanup_ms = cleanup_stage_ms.unwrap_or_default(), + cleanup_present = cleanup_stage_ms.is_some(), + commit_tail_ms = total_commit_tail_ms as u64, + result = "success", + "SetDisk put_object stage summary" + ); + } + + let cleanup_set = commit_set.clone(); + let cleanup_tmp_dir = commit_tmp_dir.clone(); + tokio::spawn(async move { + if let Err(err) = cleanup_set.delete_all(RUSTFS_META_TMP_BUCKET, &cleanup_tmp_dir).await { + warn!(tmp_dir = %cleanup_tmp_dir, error = ?err, "failed to cleanup put_object temporary data"); + } else if issue3031_diag_enabled() { + warn!( + target: "rustfs_ecstore::set_disk", + tmp_dir = %cleanup_tmp_dir, + "issue3031_put_object_tmp_cleanup_done" + ); + } + }); + + Ok(( + ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned), + old_current_size, + )) + }; + + if detach_commit_owner { + tokio::spawn(commit) + .await + .map_err(|err| Error::other(format!("put_object commit task failed: {err}")))? + } else { + commit.await } - - let committed_metadata_slot = committed_response_metadata_slot(&online_disks, response_metadata_slot); - let mut fi = std::mem::take(&mut parts_metadatas[committed_metadata_slot]); - - if is_compressed { - record_compression_total_memory(actual_size as u64, w_size as u64).await; - } - self.record_capacity_scope_if_needed(opts.capacity_scope_token, &online_disks); - - fi.replication_state_internal = Some(replication_state_to_filemeta(&opts.put_replication_state())); - - fi.is_latest = true; - - if issue3031_diag_enabled() { - let online_success_count = online_disks.iter().filter(|disk| disk.is_some()).count(); - warn!( - target: "rustfs_ecstore::set_disk", - bucket = %bucket, - object = %object, - tmp_dir = %tmp_dir, - data_dir = ?fi.data_dir, - write_quorum, - online_success_count, - op_old_dir = ?op_old_dir, - "issue3031_put_object_commit_succeeded" - ); - } - - let total_commit_tail_ms = rename_stage_start.elapsed().as_millis(); - if total_commit_tail_ms >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS { - warn!( - event = EVENT_SET_DISK_COMMIT_TAIL_SLOW, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_SET_DISK, - stage = "put_object_commit_tail", - bucket = %bucket, - object = %object, - tmp_dir = %tmp_dir, - duration_ms = total_commit_tail_ms as u64, - write_quorum, - state = "slow", - "SetDisk commit tail is slow" - ); - } - - if issue3031_diag_enabled() { - warn!( - event = EVENT_SET_DISK_PUT_OBJECT_STAGE_SUMMARY, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_SET_DISK, - bucket = %bucket, - object = %object, - write_quorum, - write_path = write_path.metric_label(), - writer_setup_ms, - encode_ms, - rename_ms = rename_stage_ms, - cleanup_ms = cleanup_stage_ms.unwrap_or_default(), - cleanup_present = cleanup_stage_ms.is_some(), - commit_tail_ms = total_commit_tail_ms as u64, - result = "success", - "SetDisk put_object stage summary" - ); - } - - Ok(( - ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended), - old_current_size, - )) } .await; @@ -1798,7 +2246,8 @@ impl SetDisks { ); } - if result.is_ok() { + if tmp_cleanup_owned && result.is_ok() { + } else if result.is_ok() { // Success path: `rename_data` has already moved the data dir out of // the tmp workspace and removed the (empty) tmp dir where it could, // so this delete_all is a speculative safety net that normally hits @@ -2846,14 +3295,12 @@ fn remote_version_state_writer_enabled() -> bool { remote_version_state_writer_fleet_proof().is_some() } -fn remote_version_state_writer_fleet_proof() -> Option { - remote_version_state_writer_requested() - .then(crate::services::notification_sys::acquire_remote_version_state_fleet_proof) - .flatten() +fn remote_version_state_writer_fleet_proof() -> Option { + transaction_fencing_fleet_proof(remote_version_state_writer_requested()) } fn remote_version_state_writer_requested() -> bool { - remote_version_state_writer_enabled_for( + transaction_fencing_gate_requested_for( rustfs_utils::get_env_bool( rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE, @@ -2866,20 +3313,66 @@ fn remote_version_state_writer_requested() -> bool { ) } -fn remote_version_state_writer_fleet_proof_matches( - proof: &crate::services::notification_sys::RemoteVersionStateFleetProofToken, -) -> bool { - remote_version_state_writer_fleet_proof_matches_for( +fn remote_version_state_writer_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool { + transaction_fencing_fleet_proof_matches_for( remote_version_state_writer_requested(), crate::services::notification_sys::remote_version_state_fleet_proof_matches(proof), ) } -fn remote_version_state_writer_fleet_proof_matches_for(requested: bool, fleet_proof_matches: bool) -> bool { +pub(in crate::set_disk::ops) fn object_transaction_fencing_fleet_proof() -> Option { + transaction_fencing_fleet_proof(object_transaction_fencing_requested()) +} + +pub(in crate::set_disk::ops) fn object_transaction_fencing_requested() -> bool { + object_transaction_fencing_requested_cached() +} + +#[cfg(not(test))] +fn object_transaction_fencing_requested_cached() -> bool { + static REQUESTED: OnceLock = OnceLock::new(); + *REQUESTED.get_or_init(load_object_transaction_fencing_requested) +} + +#[cfg(test)] +fn object_transaction_fencing_requested_cached() -> bool { + load_object_transaction_fencing_requested() +} + +fn load_object_transaction_fencing_requested() -> bool { + transaction_fencing_gate_requested_for( + rustfs_utils::get_env_bool( + rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, + rustfs_config::DEFAULT_OBJECT_TRANSACTION_FENCING_WRITE, + ), + rustfs_utils::get_env_bool( + rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, + rustfs_config::DEFAULT_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, + ), + true, + ) +} + +pub(in crate::set_disk::ops) fn object_transaction_fencing_fleet_proof_matches( + proof: &RemoteVersionStateFleetProofToken, +) -> bool { + transaction_fencing_fleet_proof_matches_for( + object_transaction_fencing_requested(), + crate::services::notification_sys::remote_version_state_fleet_proof_matches(proof), + ) +} + +fn transaction_fencing_fleet_proof(requested: bool) -> Option { + requested + .then(crate::services::notification_sys::acquire_remote_version_state_fleet_proof) + .flatten() +} + +fn transaction_fencing_fleet_proof_matches_for(requested: bool, fleet_proof_matches: bool) -> bool { requested && fleet_proof_matches } -fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool, fleet_proof_valid: bool) -> bool { +fn transaction_fencing_gate_requested_for(requested: bool, fleet_confirmed: bool, fleet_proof_valid: bool) -> bool { requested && fleet_confirmed && fleet_proof_valid } @@ -2889,6 +3382,7 @@ pub enum PutObjectCommitPause { BeforeNamespace, AfterNamespace, BeforeMetadata, + BeforeTransactionEpochVerify, } #[cfg(any(test, feature = "test-util"))] @@ -2970,13 +3464,24 @@ impl Drop for PutObjectCommitBarrier { #[cfg(any(test, feature = "test-util"))] async fn pause_put_object_commit(bucket: &str, object: &str, pause: PutObjectCommitPause) { - let barrier = PUT_OBJECT_COMMIT_BARRIER - .get_or_init(|| std::sync::Mutex::new(Vec::new())) - .lock() - .expect("put object commit barrier mutex should not poison") - .iter() - .find(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause) - .cloned(); + let barrier = { + let mut slot = PUT_OBJECT_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(Vec::new())) + .lock() + .expect("put object commit barrier mutex should not poison"); + if let Some(index) = slot + .iter() + .position(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause) + { + if pause == PutObjectCommitPause::BeforeTransactionEpochVerify { + Some(slot.remove(index)) + } else { + Some(Arc::clone(&slot[index])) + } + } else { + None + } + }; if let Some(barrier) = barrier { barrier.arrived.notify_one(); barrier.release.notified().await; @@ -3336,7 +3841,7 @@ mod transition_upload_completion_tests { mod transition_version_id_tests { use super::{ TransitionUploadCandidate, persisted_transition_version, persisted_transition_version_with_gate, - remote_version_state_writer_enabled_for, remote_version_state_writer_fleet_proof_matches_for, + transaction_fencing_fleet_proof_matches_for, transaction_fencing_gate_requested_for, }; use rustfs_filemeta::TransitionVersionState; use uuid::Uuid; @@ -3389,7 +3894,24 @@ mod transition_version_id_tests { ("fully upgraded fleet", true, true, true, true), ] { assert_eq!( - remote_version_state_writer_enabled_for(requested, fleet_confirmed, fleet_proof_valid), + transaction_fencing_gate_requested_for(requested, fleet_confirmed, fleet_proof_valid), + expected, + "{case}" + ); + } + } + + #[test] + fn object_transaction_fencing_gate_requires_request_confirmation_and_live_proof() { + for (case, requested, fleet_confirmed, fleet_proof_valid, expected) in [ + ("old defaults", false, false, false, false), + ("missing fleet confirmation", true, false, true, false), + ("missing local opt-in", false, true, true, false), + ("missing fleet proof", true, true, false, false), + ("fully upgraded fleet", true, true, true, true), + ] { + assert_eq!( + transaction_fencing_gate_requested_for(requested, fleet_confirmed, fleet_proof_valid), expected, "{case}" ); @@ -3404,7 +3926,7 @@ mod transition_version_id_tests { ("current authorization", true, true, true), ] { assert_eq!( - remote_version_state_writer_fleet_proof_matches_for(requested, fleet_proof_matches), + transaction_fencing_fleet_proof_matches_for(requested, fleet_proof_matches), expected, "{case}" ); @@ -5018,11 +5540,13 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let pool_index = self.pool_index; let skip_verify = opts.skip_verify_bitrot; let metrics_size_bucket = rustfs_io_metrics::get_object_size_bucket(cloned_fi.size); + let erasure_cache = Arc::clone(&self.erasure_cache); let producer = async move { let mut writer = TransitionUploadWriter::new(pw); Self::get_object_with_fileinfo( &cloned_bucket, &cloned_object, + erasure_cache, 0, cloned_fi.size, &mut writer, @@ -5931,8 +6455,11 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support { mod inline_put_commit_path_tests { use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; use super::*; + use crate::config::storageclass::lookup_config_for_pools_without_env; use crate::disk::{DiskAPI as _, ReadOptions}; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + use rustfs_config::server_config::KVS; + use serial_test::serial; use tokio::io::AsyncReadExt; async fn make_bucket(disks: &[DiskStore], bucket: &str) { @@ -5996,6 +6523,122 @@ mod inline_put_commit_path_tests { assert_eq!(restored, payload); } + #[tokio::test] + async fn repeated_gets_reuse_the_set_erasure_shell() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "get-erasure-shell-cache"; + let object = "object.bin"; + let payload = vec![0x4d; 1024 * 1024]; + make_bucket(&disk_stores, bucket).await; + + let mut reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("non-inline object should commit"); + assert!(set_disks.erasure_cache.entries.read().is_empty()); + + for _ in 0..2 { + let mut object_reader = set_disks + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("cached-shell GET should succeed"); + let mut restored = Vec::new(); + object_reader + .stream + .read_to_end(&mut restored) + .await + .expect("cached-shell GET should stream"); + assert_eq!(restored, payload); + assert_eq!(set_disks.erasure_cache.entries.read().len(), 1); + } + } + + #[tokio::test] + async fn ec_8_4_default_budget_keeps_large_inline_candidate_out_of_xl_meta() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(12).await; + set_disks.set_test_storage_class_config( + lookup_config_for_pools_without_env(&KVS::new(), &[12]).expect("EC8+4 storage class should resolve"), + ); + let bucket = "ec-8-4-inline-budget"; + let object = "object.bin"; + let payload = vec![0x5c; 300 * 1024]; + make_bucket(&disk_stores, bucket).await; + + let mut reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("EC8+4 PUT should commit through the non-inline path"); + + for (disk_index, disk) in disk_stores.iter().enumerate() { + let file_info = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .unwrap_or_else(|err| panic!("disk {disk_index} should persist EC8+4 metadata: {err}")); + assert_eq!(file_info.erasure.data_blocks, 8); + assert_eq!(file_info.erasure.parity_blocks, 4); + assert!(!file_info.inline_data(), "disk {disk_index} must keep the shard outside xl.meta"); + } + + let mut object_reader = set_disks + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("non-inline EC8+4 object should remain readable"); + let mut restored = Vec::new(); + object_reader + .stream + .read_to_end(&mut restored) + .await + .expect("non-inline EC8+4 object should stream"); + assert_eq!(restored, payload); + } + + #[tokio::test] + async fn ec_8_4_versioned_budget_reaches_put_placement_decision() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(12).await; + set_disks.set_test_storage_class_config( + lookup_config_for_pools_without_env(&KVS::new(), &[12]).expect("EC8+4 storage class should resolve"), + ); + let bucket = "ec-8-4-versioned-inline-budget"; + let object = "object.bin"; + let payload = vec![0x73; 64 * 1024]; + make_bucket(&disk_stores, bucket).await; + + let options = ObjectOptions { + versioned: true, + ..Default::default() + }; + let mut reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, object, &mut reader, &options) + .await + .expect("versioned EC8+4 PUT should use the reduced inline budget"); + + for (disk_index, disk) in disk_stores.iter().enumerate() { + let file_info = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .unwrap_or_else(|err| panic!("disk {disk_index} should persist versioned EC8+4 metadata: {err}")); + assert!( + !file_info.inline_data(), + "disk {disk_index} must keep the versioned shard outside xl.meta" + ); + } + + let mut object_reader = set_disks + .get_object_reader(bucket, object, None, HeaderMap::new(), &options) + .await + .expect("versioned non-inline EC8+4 object should remain readable"); + let mut restored = Vec::new(); + object_reader + .stream + .read_to_end(&mut restored) + .await + .expect("versioned non-inline EC8+4 object should stream"); + assert_eq!(restored, payload); + } + #[tokio::test] async fn inline_put_direct_commit_accepts_exact_quorum_and_rejects_quorum_minus_one() { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; @@ -6189,7 +6832,10 @@ mod inline_put_commit_path_tests { mod get_object_downstream_close_accounting_tests { use super::hermetic_set_disks_support::hermetic_set_disks; use super::*; - use crate::diagnostics::get::{GET_OBJECT_PATH_INTERNAL_META, GET_STAGE_DECODE, GET_STAGE_EMIT, GetObjectFailureReason}; + use crate::diagnostics::get::{ + GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_OBJECT_PATH_INTERNAL_META, GET_STAGE_DECODE, GET_STAGE_EMIT, + GetObjectFailureReason, + }; use crate::disk::RUSTFS_META_BUCKET; use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions}; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; @@ -6308,7 +6954,22 @@ mod get_object_downstream_close_accounting_tests { let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled(); rustfs_io_metrics::set_get_stage_metrics_enabled(true); - let (internal_missing, legacy_unknown, internal_fanout, legacy_fanout) = metrics::with_local_recorder(&recorder, || { + let ( + internal_missing, + legacy_unknown, + internal_fanout, + legacy_fanout, + internal_scheduled, + legacy_scheduled, + internal_completed, + legacy_completed, + internal_cancelled, + legacy_cancelled, + internal_unsafe_miss, + legacy_unsafe_miss, + internal_saved, + legacy_saved, + ) = metrics::with_local_recorder(&recorder, || { runtime.block_on(async { let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await; let options = ObjectOptions { @@ -6352,6 +7013,54 @@ mod get_object_downstream_close_accounting_tests { "rustfs_io_get_object_metadata_fanout_error_responses", &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)], ), + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_scheduled", + &[("path", GET_OBJECT_PATH_INTERNAL_META)], + ), + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_scheduled", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)], + ), + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_completed", + &[("path", GET_OBJECT_PATH_INTERNAL_META)], + ), + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_completed", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)], + ), + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_cancelled", + &[("path", GET_OBJECT_PATH_INTERNAL_META)], + ), + recorder.histogram_values( + "rustfs_io_get_object_metadata_fanout_cancelled", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)], + ), + recorder.counter_value( + "rustfs_io_get_object_metadata_early_stop_total", + &[ + ("path", GET_OBJECT_PATH_INTERNAL_META), + ("decision", "miss"), + ("reason", GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST), + ], + ), + recorder.counter_value( + "rustfs_io_get_object_metadata_early_stop_total", + &[ + ("path", GET_OBJECT_PATH_LEGACY_DUPLEX), + ("decision", "miss"), + ("reason", GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST), + ], + ), + recorder.histogram_values( + "rustfs_io_get_object_metadata_early_stop_saved_responses", + &[("path", GET_OBJECT_PATH_INTERNAL_META)], + ), + recorder.histogram_values( + "rustfs_io_get_object_metadata_early_stop_saved_responses", + &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)], + ), ) }) }); @@ -6364,6 +7073,50 @@ mod get_object_downstream_close_accounting_tests { ); assert_eq!(internal_fanout, vec![4.0], "internal metadata fanout must retain its path label"); assert!(legacy_fanout.is_empty(), "internal metadata fanout must not leak into legacy_duplex"); + assert_eq!( + internal_scheduled, + vec![4.0], + "internal metadata lifecycle scheduled count must retain its path label" + ); + assert!( + legacy_scheduled.is_empty(), + "internal metadata lifecycle scheduled count must not leak into legacy_duplex" + ); + assert_eq!( + internal_completed, + vec![4.0], + "internal metadata lifecycle completed count must retain its path label" + ); + assert!( + legacy_completed.is_empty(), + "internal metadata lifecycle completed count must not leak into legacy_duplex" + ); + assert_eq!( + internal_cancelled, + vec![0.0], + "internal metadata full-wait lifecycle must record zero cancellations" + ); + assert!( + legacy_cancelled.is_empty(), + "internal metadata lifecycle cancelled count must not leak into legacy_duplex" + ); + assert_eq!( + internal_unsafe_miss, 1, + "internal metadata unsafe early-stop miss must retain its path label" + ); + assert_eq!( + legacy_unsafe_miss, 0, + "internal metadata unsafe early-stop miss must not leak into legacy_duplex" + ); + assert_eq!( + internal_saved, + vec![0.0], + "internal metadata unsafe miss must record zero saved responses on internal_meta" + ); + assert!( + legacy_saved.is_empty(), + "internal metadata unsafe miss saved responses must not leak into legacy_duplex" + ); } } @@ -7612,6 +8365,139 @@ mod transition_commit_failure_tests { ); } + #[tokio::test] + #[serial_test::serial] + async fn no_lock_restore_finalize_requires_live_namespace_fence() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "restore-finalize-fence-bucket"; + let object = "object.bin"; + let payload = b"restore finalize no_lock fence must fail closed".repeat(1024); + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let operation_id = Uuid::new_v4(); + let mut reader = PutObjReader::from_vec(payload); + set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + eval_metadata: Some(restore_metadata(operation_id, true)), + ..Default::default() + }, + ) + .await + .expect("restore metadata should be installed"); + let restoring = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("restore metadata should be readable"); + + let err = set_disks + .finalize_restore_metadata( + bucket, + object, + &restoring, + &ObjectOptions { + no_lock: true, + namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()), + user_defined: restore_operation_id_metadata(operation_id), + ..Default::default() + }, + ) + .await + .expect_err("lost outer namespace fence must reject no_lock restore finalization"); + assert!(matches!(err, Error::NamespaceLockQuorumUnavailable { .. })); + + let current = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("restore metadata should remain readable"); + let restore_status = parse_restore_obj_status( + current + .user_defined + .get(s3s::header::X_AMZ_RESTORE.as_str()) + .expect("restore header must remain pending"), + ) + .expect("restore header should remain parseable"); + assert!( + restore_status.on_going(), + "lost no_lock finalization must not publish restored completion metadata" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn no_lock_restore_cleanup_requires_live_namespace_fence() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "restore-cleanup-fence-bucket"; + let object = "object.bin"; + let payload = b"restore cleanup no_lock fence must fail closed".repeat(1024); + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let operation_id = Uuid::new_v4(); + let mut reader = PutObjReader::from_vec(payload); + set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + eval_metadata: Some(restore_metadata(operation_id, true)), + ..Default::default() + }, + ) + .await + .expect("restore metadata should be installed"); + let restoring = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("restore metadata should be readable"); + + let err = set_disks + .update_restore_metadata( + bucket, + object, + &restoring, + &ObjectOptions { + no_lock: true, + namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()), + user_defined: restore_operation_id_metadata(operation_id), + ..Default::default() + }, + ) + .await + .expect_err("lost outer namespace fence must reject no_lock restore cleanup"); + assert!(matches!(err, Error::NamespaceLockQuorumUnavailable { .. })); + + let current = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("restore metadata should remain readable"); + assert!( + current.user_defined.contains_key(s3s::header::X_AMZ_RESTORE.as_str()), + "lost no_lock cleanup must not remove the restore header" + ); + assert_eq!( + rustfs_utils::http::metadata_compat::get_consistent_str( + current.user_defined.as_ref(), + rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID, + ), + Some(operation_id.to_string().as_str()), + "lost no_lock cleanup must not remove the restore operation id" + ); + } + #[tokio::test] #[serial_test::serial] async fn restore_worker_propagates_operation_id_to_final_put_commit() { @@ -8139,6 +9025,48 @@ mod transition_commit_failure_tests { .expect("the transitioned object body should drain"); assert_eq!(restored, payload); } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn object_transaction_fencing_requires_live_fleet_proof_before_put_commit() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "transaction-fencing-no-proof"; + let object = "object.bin"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let err = temp_env::async_with_vars( + [ + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")), + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")), + ], + async { + let mut reader = PutObjReader::from_vec(b"must-not-commit-without-proof".to_vec()); + set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + }, + ) + .await + .expect_err("object fencing must fail closed without a live fleet proof"); + + assert!( + err.to_string() + .contains("object transaction fencing requires a live fleet capability proof"), + "unexpected error: {err:?}" + ); + for disk in &disk_stores { + let missing = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect_err("failed fenced PUT must not publish object metadata"); + assert!( + matches!(missing, DiskError::FileNotFound | DiskError::FileVersionNotFound), + "failed fenced PUT left unexpected disk state: {missing:?}" + ); + } + } } #[cfg(all(test, feature = "test-util"))] @@ -9605,13 +10533,62 @@ mod transition_source_identity_matrix_tests { #[cfg(test)] mod heterogeneous_pool_put_tests { - use super::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated as hermetic_set_disks_for_pool_with_default_parity; + use super::hermetic_set_disks_support::{ + hermetic_set_disks_for_pool_with_default_parity_isolated as hermetic_set_disks_for_pool_with_default_parity, + hermetic_set_disks_isolated as hermetic_set_disks, + }; use super::*; use crate::config::storageclass::lookup_config_for_pools_without_env; use crate::disk::{DiskAPI as _, ReadOptions}; + use crate::services::notification_sys::install_remote_version_state_fleet_proof_for_test; use rustfs_config::server_config::KVS; + use serial_test::serial; use tokio::io::AsyncReadExt; + async fn make_bucket(disks: &[DiskStore], bucket: &str) { + for disk in disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + } + + async fn object_transaction_epochs(disks: &[DiskStore], bucket: &str, object: &str) -> Vec> { + let mut epochs = Vec::with_capacity(disks.len()); + for (disk_index, disk) in disks.iter().enumerate() { + let file_info = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .unwrap_or_else(|err| panic!("disk {disk_index} should persist object metadata: {err}")); + epochs.push( + file_info + .object_transaction_epoch() + .unwrap_or_else(|err| panic!("disk {disk_index} transaction epoch should decode: {err}")), + ); + } + epochs + } + + async fn current_data_dir(disk: &DiskStore, bucket: &str, object: &str) -> Uuid { + disk.read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("current object metadata should read") + .data_dir + .expect("test object should be stored out-of-line") + } + + async fn data_dir_exists(disk: &DiskStore, bucket: &str, object: &str, data_dir: Uuid) -> bool { + disk.read_all(bucket, &format!("{object}/{data_dir}/part.1")).await.is_ok() + } + + async fn cleanup_receipt_exists(disk: &DiskStore, bucket: &str, object: &str, data_dir: Uuid) -> bool { + disk.read_all(bucket, &old_data_cleanup_receipt_path(object, data_dir)) + .await + .is_ok() + } + + fn large_payload(fill: u8) -> Vec { + vec![fill; 1024 * 1024] + } + #[tokio::test] async fn second_pool_regular_put_uses_its_own_layout_and_round_trips() { // Deliberately inject the first pool's invalid scalar fallback. The @@ -9656,6 +10633,317 @@ mod heterogeneous_pool_put_tests { .expect("second-pool regular PUT should stream"); assert_eq!(restored, payload); } + + #[tokio::test] + #[serial(storage_class_env)] + async fn object_transaction_fencing_persists_epoch_only_when_gate_is_enabled() { + let _proof = install_remote_version_state_fleet_proof_for_test("object-transaction-fencing-test"); + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "put-object-transaction-epoch"; + make_bucket(&disk_stores, bucket).await; + + let mut default_reader = PutObjReader::from_vec(b"default gate stays epoch-free".to_vec()); + set_disks + .put_object(bucket, "default.bin", &mut default_reader, &ObjectOptions::default()) + .await + .expect("default PUT should commit"); + assert_eq!( + object_transaction_epochs(&disk_stores, bucket, "default.bin").await, + vec![None, None, None, None], + "live proof alone must not write epoch metadata while the opt-in gate is disabled" + ); + + temp_env::async_with_vars( + [ + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")), + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")), + ], + async { + let mut fenced_reader = PutObjReader::from_vec(b"fenced epoch commit".to_vec()); + set_disks + .put_object(bucket, "fenced.bin", &mut fenced_reader, &ObjectOptions::default()) + .await + .expect("fenced PUT should commit with a live proof"); + }, + ) + .await; + + let epochs = object_transaction_epochs(&disk_stores, bucket, "fenced.bin").await; + let first = epochs[0].expect("fenced PUT should persist an epoch"); + assert!(!first.is_nil()); + assert!(epochs.into_iter().all(|epoch| epoch == Some(first))); + } + + #[tokio::test] + #[serial(storage_class_env)] + async fn old_data_cleanup_receipt_reconciles_failed_put_cleanup_idempotently() { + use crate::set_disk::core::io_primitives::cleanup_fault_injection; + + let _proof = install_remote_version_state_fleet_proof_for_test("object-transaction-fencing-test"); + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "put-cleanup-receipt-reconcile"; + let object = "object.bin"; + make_bucket(&disk_stores, bucket).await; + + temp_env::async_with_vars( + [ + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")), + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")), + ], + async { + let mut first_reader = PutObjReader::from_vec(large_payload(0x11)); + set_disks + .put_object(bucket, object, &mut first_reader, &ObjectOptions::default()) + .await + .expect("first fenced PUT should commit"); + let old_dir = current_data_dir(&disk_stores[0], bucket, object).await; + + let fault = cleanup_fault_injection::fail_cleanup_on(object, &[0, 1, 2, 3]); + let mut overwrite_reader = PutObjReader::from_vec(large_payload(0x22)); + set_disks + .put_object(bucket, object, &mut overwrite_reader, &ObjectOptions::default()) + .await + .expect("overwrite should commit even when old-data cleanup fails"); + for disk in &disk_stores { + assert!( + cleanup_receipt_exists(disk, bucket, object, old_dir).await, + "fenced failed cleanup should leave a durable receipt" + ); + assert!( + data_dir_exists(disk, bucket, object, old_dir).await, + "injected cleanup failure should leave the old data dir for reconciliation" + ); + } + drop(fault); + + let removed = set_disks + .reconcile_old_data_cleanup_receipts(bucket, object) + .await + .expect("receipt reconciliation should succeed"); + assert_eq!(removed, 4, "all receipt-targeted old dirs should be reclaimed"); + let repeated = set_disks + .reconcile_old_data_cleanup_receipts(bucket, object) + .await + .expect("receipt reconciliation should be idempotent"); + assert_eq!(repeated, 0, "a drained receipt must not be counted again"); + for disk in &disk_stores { + assert!( + !data_dir_exists(disk, bucket, object, old_dir).await, + "reconciled old data dir must be gone" + ); + } + }, + ) + .await; + } + + #[tokio::test] + #[serial(storage_class_env)] + async fn old_data_cleanup_receipt_noops_after_epoch_mismatch() { + use crate::set_disk::core::io_primitives::cleanup_fault_injection; + + let _proof = install_remote_version_state_fleet_proof_for_test("object-transaction-fencing-test"); + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "put-cleanup-receipt-epoch-mismatch"; + let object = "object.bin"; + make_bucket(&disk_stores, bucket).await; + + temp_env::async_with_vars( + [ + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")), + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")), + ], + async { + let mut first_reader = PutObjReader::from_vec(large_payload(0x31)); + set_disks + .put_object(bucket, object, &mut first_reader, &ObjectOptions::default()) + .await + .expect("first fenced PUT should commit"); + let old_dir = current_data_dir(&disk_stores[0], bucket, object).await; + + let fault = cleanup_fault_injection::fail_cleanup_on(object, &[0, 1, 2, 3]); + let mut second_reader = PutObjReader::from_vec(large_payload(0x32)); + set_disks + .put_object(bucket, object, &mut second_reader, &ObjectOptions::default()) + .await + .expect("second fenced PUT should commit and leave a receipt"); + drop(fault); + + let mut third_reader = PutObjReader::from_vec(large_payload(0x33)); + set_disks + .put_object(bucket, object, &mut third_reader, &ObjectOptions::default()) + .await + .expect("third fenced PUT should advance the current epoch"); + + let removed = set_disks + .reconcile_old_data_cleanup_receipts(bucket, object) + .await + .expect("stale receipt reconciliation should succeed"); + assert_eq!(removed, 0, "stale receipt epoch must not reclaim after a newer commit"); + for disk in &disk_stores { + assert!( + cleanup_receipt_exists(disk, bucket, object, old_dir).await, + "stale receipt should remain as no-op evidence" + ); + assert!( + data_dir_exists(disk, bucket, object, old_dir).await, + "epoch mismatch must preserve the old receipt target" + ); + } + }, + ) + .await; + } + + #[tokio::test] + #[serial(storage_class_env)] + async fn old_data_cleanup_receipt_is_not_persisted_without_epoch_gate() { + use crate::set_disk::core::io_primitives::cleanup_fault_injection; + + let _proof = install_remote_version_state_fleet_proof_for_test("object-transaction-fencing-test"); + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "put-cleanup-receipt-gate-off"; + let object = "object.bin"; + make_bucket(&disk_stores, bucket).await; + + let mut first_reader = PutObjReader::from_vec(large_payload(0x41)); + set_disks + .put_object(bucket, object, &mut first_reader, &ObjectOptions::default()) + .await + .expect("default-gate first PUT should commit"); + let old_dir = current_data_dir(&disk_stores[0], bucket, object).await; + + let _fault = cleanup_fault_injection::fail_cleanup_on(object, &[0, 1, 2, 3]); + let mut second_reader = PutObjReader::from_vec(large_payload(0x42)); + set_disks + .put_object(bucket, object, &mut second_reader, &ObjectOptions::default()) + .await + .expect("default-gate overwrite should commit"); + + let removed = set_disks + .reconcile_old_data_cleanup_receipts(bucket, object) + .await + .expect("gate-off receipt reconciliation should be a no-op"); + assert_eq!(removed, 0, "mixed-version/default gate path must not consume epoch-dependent receipts"); + for disk in &disk_stores { + assert!( + !cleanup_receipt_exists(disk, bucket, object, old_dir).await, + "mixed-version/default gate path must not write epoch-dependent receipts" + ); + assert!( + data_dir_exists(disk, bucket, object, old_dir).await, + "cleanup fault should leave old dir without receipt" + ); + } + } + + #[tokio::test] + #[serial(storage_class_env)] + async fn object_transaction_fencing_rejects_stale_no_lock_put_epoch() { + let _proof = install_remote_version_state_fleet_proof_for_test("object-transaction-fencing-test"); + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "put-object-transaction-stale-epoch"; + let object = "object.bin"; + make_bucket(&disk_stores, bucket).await; + + temp_env::async_with_vars( + [ + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")), + (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")), + ], + async { + let mut initial_reader = PutObjReader::from_vec(b"initial fenced body".to_vec()); + set_disks + .put_object( + bucket, + object, + &mut initial_reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("initial fenced PUT should commit"); + let initial_epoch = object_transaction_epochs(&disk_stores, bucket, object) + .await + .into_iter() + .next() + .flatten() + .expect("initial fenced PUT should persist an epoch"); + + let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeTransactionEpochVerify); + let stale_set = Arc::clone(&set_disks); + let stale = tokio::spawn(async move { + let mut stale_reader = PutObjReader::from_vec(b"stale writer body".to_vec()); + stale_set + .put_object( + bucket, + object, + &mut stale_reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + }); + barrier.wait_until_paused().await; + + let mut winner_reader = PutObjReader::from_vec(b"winning writer body".to_vec()); + set_disks + .put_object( + bucket, + object, + &mut winner_reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("concurrent fenced PUT should advance the epoch"); + let winning_epoch = object_transaction_epochs(&disk_stores, bucket, object) + .await + .into_iter() + .next() + .flatten() + .expect("winning fenced PUT should persist an epoch"); + assert_ne!(winning_epoch, initial_epoch); + + barrier.release(); + let err = stale + .await + .expect("stale PUT task should not panic") + .expect_err("stale epoch PUT must be rejected"); + assert_eq!(err, StorageError::PreconditionFailed); + + let final_epochs = object_transaction_epochs(&disk_stores, bucket, object).await; + assert!(final_epochs.into_iter().all(|epoch| epoch == Some(winning_epoch))); + let mut reader = set_disks + .get_object_reader( + bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("winning object should remain readable"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("winning body should stream"); + assert_eq!(restored, b"winning writer body"); + }, + ) + .await; + } } #[cfg(test)] @@ -9918,6 +11206,74 @@ mod put_object_tmp_cleanup_tests { assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]); } + #[tokio::test] + async fn cancelled_rename_keeps_namespace_lock_until_publication() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "put-commit-lock-cancelled-rename"; + let object = "commit-lock-cancelled-rename-object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let rename_tasks = rename_fanout_barrier::observe_tasks(object); + let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let first_store = Arc::clone(&set_disks); + let first = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); + first_store + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + }); + tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused()) + .await + .expect("first PUT should pause during the authoritative rename"); + + let second_namespace_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace); + let second_store = Arc::clone(&set_disks); + let second = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]); + second_store + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + }); + second_namespace_barrier.release_and_wait_until_namespace_pending().await; + + first.abort(); + assert!( + first + .await + .expect_err("the first request should be cancelled while rename is parked") + .is_cancelled() + ); + tokio::task::yield_now().await; + assert!( + !second.is_finished(), + "the second writer must remain blocked by the cancelled commit owner" + ); + + rename_barrier.release(); + drop(rename_barrier); + tokio::time::timeout(Duration::from_secs(30), async { + while rename_tasks.running() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("the cancelled owner's rename fanout should drain"); + second + .await + .expect("second overwrite task should join") + .expect("second overwrite should commit after the cancelled owner reaches publication"); + + let mut reader = set_disks + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("the latest overwrite should be readable"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("latest body should drain"); + assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]); + } + #[tokio::test] async fn put_object_no_lock_aborts_after_outer_namespace_lock_loss() { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 3fd19b151..d0011cd11 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -482,6 +482,7 @@ impl SetDisks { pub(super) async fn try_get_object_direct_data_shards_with_fileinfo( bucket: &str, object: &str, + erasure_cache: Arc, fi: &FileInfo, files: &[FileInfo], disks: &[Option], @@ -502,13 +503,7 @@ impl SetDisks { return Ok(None); } - let erasure = coding::Erasure::try_new_with_options( - fi.erasure.data_blocks, - fi.erasure.parity_blocks, - fi.erasure.block_size, - fi.uses_legacy_checksum, - ) - .map_err(Error::from)?; + let erasure = erasure_cache.get_for_file_info(fi)?; let checksum_info = fi.erasure.get_checksum_info(part.number); let checksum_algo = if fi.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S { @@ -636,6 +631,7 @@ impl SetDisks { // &self, bucket: &str, object: &str, + erasure_cache: Arc, offset: usize, length: i64, writer: &mut W, @@ -730,13 +726,7 @@ impl SetDisks { object, offset, length, end_offset, part_index, last_part_index, last_part_relative_offset, "Multipart read bounds" ); - let erasure = coding::Erasure::try_new_with_options( - fi.erasure.data_blocks, - fi.erasure.parity_blocks, - fi.erasure.block_size, - fi.uses_legacy_checksum, - ) - .map_err(Error::from)?; + let erasure = erasure_cache.get_for_file_info(&fi)?; let part_indices: Vec = (part_index..=last_part_index).collect(); debug!(bucket, object, ?part_indices, "Multipart part indices to stream"); @@ -1170,6 +1160,7 @@ impl SetDisks { pub(super) async fn get_object_decode_reader_with_fileinfo( bucket: &str, object: &str, + erasure_cache: Arc, fi: &FileInfo, files: &[FileInfo], disks: &[Option], @@ -1180,14 +1171,7 @@ impl SetDisks { metrics_size_bucket: &'static str, prefer_data_blocks_first_reader_setup: bool, ) -> Result { - let erasure = coding::Erasure::try_new_with_options( - fi.erasure.data_blocks, - fi.erasure.parity_blocks, - fi.erasure.block_size, - fi.uses_legacy_checksum, - ) - .map_err(Error::from)?; - + let erasure = erasure_cache.get_for_file_info(fi)?; let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, files, fi); if fi.parts.len() == 1 { @@ -1574,7 +1558,7 @@ struct LazyCodecPartContext { fi: FileInfo, files: Vec, disks: Vec>, - erasure: coding::Erasure, + erasure: Arc, skip_verify_bitrot: bool, metrics_object_class: &'static str, metrics_size_bucket: &'static str, @@ -2058,6 +2042,7 @@ mod metadata_cache_tests { let err = SetDisks::get_object_with_fileinfo( "bucket", "object", + Arc::new(ErasureCache::new()), 0, 1, &mut output, @@ -2088,6 +2073,7 @@ mod metadata_cache_tests { let err = SetDisks::get_object_with_fileinfo( bucket, object, + Arc::new(ErasureCache::new()), 2, 1, &mut output, @@ -2111,6 +2097,7 @@ mod metadata_cache_tests { let err = SetDisks::get_object_with_fileinfo( bucket, object, + Arc::new(ErasureCache::new()), usize::MAX, 1, &mut output, @@ -2132,6 +2119,7 @@ mod metadata_cache_tests { let err = SetDisks::get_object_with_fileinfo( bucket, object, + Arc::new(ErasureCache::new()), 1, 1, &mut output, @@ -2155,6 +2143,7 @@ mod metadata_cache_tests { let err = SetDisks::get_object_with_fileinfo( bucket, object, + Arc::new(ErasureCache::new()), 0, 1, &mut output, @@ -2192,6 +2181,7 @@ mod metadata_cache_tests { SetDisks::get_object_with_fileinfo( bucket, object, + Arc::new(ErasureCache::new()), 0, 0, &mut output, @@ -2224,6 +2214,7 @@ mod metadata_cache_tests { let err = SetDisks::get_object_with_fileinfo( bucket, object, + Arc::new(ErasureCache::new()), 0, 1, &mut output, @@ -4128,6 +4119,7 @@ mod tests { let result = SetDisks::get_object_decode_reader_with_fileinfo( CODEC_STREAMING_TEST_BUCKET, CODEC_STREAMING_TEST_OBJECT, + Arc::new(ErasureCache::new()), &fi, &[], &[], @@ -4150,6 +4142,7 @@ mod tests { let invalid_size = SetDisks::get_object_decode_reader_with_fileinfo( CODEC_STREAMING_TEST_BUCKET, CODEC_STREAMING_TEST_OBJECT, + Arc::new(ErasureCache::new()), &single_part, &[], &[], @@ -4170,6 +4163,7 @@ mod tests { SetDisks::get_object_decode_reader_with_fileinfo( CODEC_STREAMING_TEST_BUCKET, CODEC_STREAMING_TEST_OBJECT, + Arc::new(ErasureCache::new()), &multipart, &[], &[], @@ -4194,6 +4188,7 @@ mod tests { SetDisks::get_object_decode_reader_with_fileinfo( CODEC_STREAMING_TEST_BUCKET, CODEC_STREAMING_TEST_OBJECT, + Arc::new(ErasureCache::new()), &multipart, &[], &[], @@ -4222,6 +4217,7 @@ mod tests { SetDisks::get_object_decode_reader_with_fileinfo( CODEC_STREAMING_TEST_BUCKET, CODEC_STREAMING_TEST_OBJECT, + Arc::new(ErasureCache::new()), &multipart, &[], &[], @@ -4275,6 +4271,7 @@ mod tests { SetDisks::get_object_decode_reader_with_fileinfo( CODEC_STREAMING_TEST_BUCKET, CODEC_STREAMING_TEST_OBJECT, + Arc::new(ErasureCache::new()), &fi, &files, &disks, @@ -4328,6 +4325,7 @@ mod tests { SetDisks::get_object_decode_reader_with_fileinfo( CODEC_STREAMING_TEST_BUCKET, CODEC_STREAMING_TEST_OBJECT, + Arc::new(ErasureCache::new()), &fi, &files, &disks, @@ -4372,6 +4370,7 @@ mod tests { SetDisks::get_object_with_fileinfo( CODEC_STREAMING_TEST_BUCKET, CODEC_STREAMING_TEST_OBJECT, + Arc::new(ErasureCache::new()), 0, part_data.len() as i64, &mut output, diff --git a/crates/ecstore/src/set_disk/replication.rs b/crates/ecstore/src/set_disk/replication.rs index d968cd487..86126c535 100644 --- a/crates/ecstore/src/set_disk/replication.rs +++ b/crates/ecstore/src/set_disk/replication.rs @@ -48,6 +48,23 @@ impl RestoreCleanupIdentity { } } +fn ensure_restore_metadata_lock_held(bucket: &str, object: &str, opts: &ObjectOptions, mode: &'static str) -> Result<()> { + if opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode, + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + Ok(()) +} + impl SetDisks { pub(super) async fn finalize_restore_metadata( &self, @@ -88,6 +105,7 @@ impl SetDisks { if !expected.matches_file_info(&fi, &expected_etag) { return Err(Error::other("restored object changed before restore metadata finalization")); } + ensure_restore_metadata_lock_held(bucket, object, opts, "restore_finalize_metadata")?; let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), opts.transition.restore_request.days.unwrap_or(1)); fi.metadata.insert( @@ -159,6 +177,7 @@ impl SetDisks { if !expected.matches_file_info(&fi, &expected_etag) { return Ok(()); } + ensure_restore_metadata_lock_held(bucket, object, opts, "restore_cleanup_metadata")?; fi.metadata.remove(X_AMZ_RESTORE.as_str()); fi.metadata.remove(AMZ_RESTORE_EXPIRY_DAYS); fi.metadata.remove(AMZ_RESTORE_REQUEST_DATE); diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 2d35b551b..077caf714 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::*; -use crate::core::pools::local_decommission_queue_prefix; +use crate::core::pools::{local_decommission_queue_prefix, pool_meta_has_active_decommission}; use crate::error::is_err_decommission_running; use crate::runtime::instance::InstanceContext; use crate::runtime::sources as runtime_sources; @@ -109,14 +109,6 @@ fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_ rebalance_meta_loaded && !decommission_running } -fn pool_meta_has_active_decommission(meta: &PoolMeta) -> bool { - meta.pools.iter().any(|pool| { - pool.decommission - .as_ref() - .is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled) - }) -} - async fn wait_for_local_decommission_resume_delay(rx: &CancellationToken, delay: Duration) -> bool { tokio::select! { _ = rx.cancelled() => false, diff --git a/crates/filemeta/src/fileinfo.rs b/crates/filemeta/src/fileinfo.rs index 0f799a40e..393975c18 100644 --- a/crates/filemeta/src/fileinfo.rs +++ b/crates/filemeta/src/fileinfo.rs @@ -18,8 +18,9 @@ use rmp_serde::Serializer; use rustfs_utils::HashAlgorithm; use rustfs_utils::http::{ AMZ_OBJECT_TAGGING, SUFFIX_COMPRESSION, SUFFIX_DATA_MOVED, SUFFIX_DATA_MOVED_TAGS, SUFFIX_FREE_VERSION, SUFFIX_HEALING, - SUFFIX_INLINE_DATA, SUFFIX_TIER_FV_ID, SUFFIX_TIER_FV_MARKER, SUFFIX_TIER_SKIP_FV_ID, contains_key_str, get_str, - has_internal_suffix, insert_str, is_encryption_metadata_key, starts_with_ignore_ascii_case, + SUFFIX_INLINE_DATA, SUFFIX_OBJECT_TRANSACTION_EPOCH, SUFFIX_TIER_FV_ID, SUFFIX_TIER_FV_MARKER, SUFFIX_TIER_SKIP_FV_ID, + contains_key_str, get_consistent_str, get_str, has_internal_suffix, insert_str, is_encryption_metadata_key, + starts_with_ignore_ascii_case, }; use s3s::dto::{RestoreStatus, Timestamp}; use s3s::header::X_AMZ_RESTORE; @@ -1172,6 +1173,22 @@ impl FileInfo { insert_str(&mut self.metadata, SUFFIX_DATA_MOVED, String::new()); } + pub fn set_object_transaction_epoch(&mut self, epoch: Uuid) { + insert_str(&mut self.metadata, SUFFIX_OBJECT_TRANSACTION_EPOCH, epoch.to_string()); + } + + pub fn object_transaction_epoch(&self) -> Result> { + if !contains_key_str(&self.metadata, SUFFIX_OBJECT_TRANSACTION_EPOCH) { + return Ok(None); + } + let value = get_consistent_str(&self.metadata, SUFFIX_OBJECT_TRANSACTION_EPOCH).ok_or(Error::FileCorrupt)?; + let epoch = Uuid::parse_str(value).map_err(|_| Error::FileCorrupt)?; + if epoch.is_nil() { + return Err(Error::FileCorrupt); + } + Ok(Some(epoch)) + } + pub fn inline_data(&self) -> bool { contains_key_str(&self.metadata, SUFFIX_INLINE_DATA) && !self.is_remote() } @@ -1484,6 +1501,46 @@ mod tests { assert_eq!(ei.get_checksum_info(99).algorithm, HashAlgorithm::HighwayHash256S); } + #[test] + fn object_transaction_epoch_uses_consistent_dual_internal_metadata() { + let mut fi = validation_test_fileinfo(); + assert_eq!(fi.object_transaction_epoch().expect("absent epoch should decode"), None); + + let epoch = Uuid::new_v4(); + let epoch_text = epoch.to_string(); + fi.set_object_transaction_epoch(epoch); + assert_eq!(fi.object_transaction_epoch().expect("written epoch should decode"), Some(epoch)); + assert_eq!(fi.metadata.get("x-rustfs-internal-object-transaction-epoch"), Some(&epoch_text)); + assert_eq!(fi.metadata.get("x-minio-internal-object-transaction-epoch"), Some(&epoch_text)); + + let mut rustfs_only = validation_test_fileinfo(); + rustfs_only + .metadata + .insert("x-rustfs-internal-object-transaction-epoch".to_string(), epoch_text); + assert_eq!( + rustfs_only + .object_transaction_epoch() + .expect("single compatibility key should decode"), + Some(epoch) + ); + + let mut conflicting = fi.clone(); + conflicting + .metadata + .insert("x-minio-internal-object-transaction-epoch".to_string(), Uuid::new_v4().to_string()); + assert_eq!(conflicting.object_transaction_epoch(), Err(Error::FileCorrupt)); + + let mut malformed = validation_test_fileinfo(); + malformed + .metadata + .insert("x-rustfs-internal-object-transaction-epoch".to_string(), "not-a-uuid".to_string()); + assert_eq!(malformed.object_transaction_epoch(), Err(Error::FileCorrupt)); + + let mut nil = validation_test_fileinfo(); + nil.set_object_transaction_epoch(Uuid::nil()); + assert_eq!(nil.object_transaction_epoch(), Err(Error::FileCorrupt)); + } + // backlog#949: distribution range/permutation validation. #[test] fn is_valid_distribution_accepts_permutation() { diff --git a/crates/heal/src/error.rs b/crates/heal/src/error.rs index 10da452ae..68ca7eef4 100644 --- a/crates/heal/src/error.rs +++ b/crates/heal/src/error.rs @@ -34,36 +34,21 @@ pub enum Error { #[error("Configuration error: {0}")] Config(String), - #[error("Heal configuration error: {message}")] - ConfigurationError { message: String }, - #[error("Other error: {0}")] Other(String), #[error("Serialization error: {0}")] Serialization(String), - #[error("IO error: {0}")] - IO(String), - - #[error("Not found: {0}")] - NotFound(String), - #[error("Invalid checkpoint: {0}")] InvalidCheckpoint(String), #[error("Heal task not found: {task_id}")] TaskNotFound { task_id: String }, - #[error("Heal task already exists: {task_id}")] - TaskAlreadyExists { task_id: String }, - #[error("Invalid heal client token")] InvalidClientToken, - #[error("Heal manager is not running")] - ManagerNotRunning, - #[error("Heal task execution failed: {message}")] TaskExecutionFailed { message: String }, @@ -78,12 +63,6 @@ pub enum Error { #[error("Heal task timeout")] TaskTimeout, - - #[error("Heal event processing failed: {message}")] - EventProcessingFailed { message: String }, - - #[error("Heal progress tracking failed: {message}")] - ProgressTrackingFailed { message: String }, } /// A specialized Result type for heal operations @@ -129,9 +108,7 @@ impl Error { | DiskError::FaultyDisk ) || is_recoverable_heal_error_message(&err.to_string()) } - Error::TaskExecutionFailed { message } | Error::IO(message) | Error::Other(message) => { - is_recoverable_heal_error_message(message) - } + Error::TaskExecutionFailed { message } | Error::Other(message) => is_recoverable_heal_error_message(message), Error::Io(err) => is_recoverable_heal_error_message(&err.to_string()), _ => false, } diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index 6937f69e7..916b51822 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -597,7 +597,7 @@ impl HealTask { | EcstoreError::ObjectNotFound(_, _) | EcstoreError::VersionNotFound(_, _, _), ) => true, - Error::Other(message) | Error::IO(message) => { + Error::Other(message) => { message.contains("File not found") || message.contains("file not found") || message.contains("File version not found") diff --git a/crates/heal/tests/endpoint_index_test.rs b/crates/heal/tests/endpoint_index_test.rs index 52684b9af..0df8ab5a6 100644 --- a/crates/heal/tests/endpoint_index_test.rs +++ b/crates/heal/tests/endpoint_index_test.rs @@ -14,6 +14,8 @@ //! test endpoint index settings +#![recursion_limit = "256"] + use std::net::SocketAddr; use tempfile::TempDir; use tokio_util::sync::CancellationToken; diff --git a/crates/heal/tests/heal_b5_versioned_regression_test.rs b/crates/heal/tests/heal_b5_versioned_regression_test.rs index 2b4a0c604..61a542955 100644 --- a/crates/heal/tests/heal_b5_versioned_regression_test.rs +++ b/crates/heal/tests/heal_b5_versioned_regression_test.rs @@ -22,6 +22,8 @@ //! bucket-metadata-sys OnceCell) — under `cargo nextest` each test runs //! in its own process so the OnceCell never collides. +#![recursion_limit = "256"] + use http::HeaderMap; use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_heal::heal::{ diff --git a/crates/heal/tests/heal_b920_subquorum_union_test.rs b/crates/heal/tests/heal_b920_subquorum_union_test.rs index f3121aaa3..6d188b3f3 100644 --- a/crates/heal/tests/heal_b920_subquorum_union_test.rs +++ b/crates/heal/tests/heal_b920_subquorum_union_test.rs @@ -21,6 +21,8 @@ //! These drive the REAL `ECStoreHealStorage` + `ECStore` against real disks. //! Every test is `#[serial]`; under `cargo nextest` each runs in its own process. +#![recursion_limit = "256"] + use http::HeaderMap; use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_heal::heal::storage::{ diff --git a/crates/heal/tests/heal_integration_test.rs b/crates/heal/tests/heal_integration_test.rs index 7094614db..21a0ab78b 100644 --- a/crates/heal/tests/heal_integration_test.rs +++ b/crates/heal/tests/heal_integration_test.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![recursion_limit = "256"] + use http::HeaderMap; use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_heal::heal::{ diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index 658dee400..1d99f50e5 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -812,6 +812,17 @@ pub fn record_get_object_metadata_fanout_shape(path: &'static str, total: usize, .record(metadata_fanout_count_to_f64(non_valid)); } +/// Record task lifecycle shape for one GetObject metadata fanout. +#[inline(always)] +pub fn record_get_object_metadata_fanout_lifecycle(path: &'static str, scheduled: usize, completed: usize, cancelled: usize) { + if !get_stage_metrics_enabled() { + return; + } + histogram!("rustfs_io_get_object_metadata_fanout_scheduled", "path" => path).record(metadata_fanout_count_to_f64(scheduled)); + histogram!("rustfs_io_get_object_metadata_fanout_completed", "path" => path).record(metadata_fanout_count_to_f64(completed)); + histogram!("rustfs_io_get_object_metadata_fanout_cancelled", "path" => path).record(metadata_fanout_count_to_f64(cancelled)); +} + /// Record a guarded metadata early-stop hit for GetObject. #[inline(always)] pub fn record_get_object_metadata_early_stop_hit(path: &'static str, reason: &'static str) { @@ -2692,12 +2703,17 @@ mod tests { record_get_object_reader_prefetch_wait("codec_streaming", 0.0002); record_get_object_response_handoff("standard", "selected", 8192, 1024, 0.0001); record_get_object_metadata_fanout_duration("legacy_duplex", 0.001); + record_get_object_stage_duration("legacy_duplex", "read_version_path_resolve", 0.0001); + record_get_object_stage_duration("legacy_duplex", "read_version_path_check", 0.0001); + record_get_object_stage_duration("legacy_duplex", "read_version_xlmeta_read", 0.0005); + record_get_object_stage_duration("legacy_duplex", "read_version_decode", 0.0002); record_get_object_first_metadata_response_latency("legacy_duplex", 0.001); record_get_object_first_valid_metadata_response_latency("legacy_duplex", 0.001); record_get_object_slowest_metadata_response_latency("legacy_duplex", 0.003); record_get_object_quorum_reached_latency("legacy_duplex", 0.002); record_get_object_metadata_response("legacy_duplex", "valid"); record_get_object_metadata_fanout_shape("legacy_duplex", 4, 3, 1, 1); + record_get_object_metadata_fanout_lifecycle("legacy_duplex", 4, 3, 1); record_get_object_metadata_early_stop_hit("legacy_duplex", "valid_quorum"); record_get_object_metadata_early_stop_miss("legacy_duplex", "insufficient_quorum"); record_get_object_metadata_early_stop_saved_responses("legacy_duplex", 1); @@ -2768,6 +2784,38 @@ mod tests { assert!(remote_scheduled >= remote_avoid_potential); } + #[test] + fn metadata_fanout_lifecycle_records_named_histograms() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + metrics::with_local_recorder(&recorder, || { + set_get_stage_metrics_enabled(true); + record_get_object_metadata_fanout_lifecycle("legacy_duplex", 4, 3, 1); + set_get_stage_metrics_enabled(false); + }); + + let metrics = snapshotter.snapshot().into_vec(); + for (name, expected) in [ + ("rustfs_io_get_object_metadata_fanout_scheduled", 4.0), + ("rustfs_io_get_object_metadata_fanout_completed", 3.0), + ("rustfs_io_get_object_metadata_fanout_cancelled", 1.0), + ] { + let value = metrics.iter().find_map(|(composite, _, _, value)| { + let has_path = composite + .key() + .labels() + .any(|label| label.key() == "path" && label.value() == "legacy_duplex"); + (composite.kind() == MetricKind::Histogram && composite.key().name() == name && has_path).then_some(value) + }); + assert!( + matches!(value, Some(DebugValue::Histogram(values)) if values.len() == 1 && values[0].0 == expected), + "{name} must record the exact fanout lifecycle sample" + ); + } + } + #[test] fn test_record_get_object_fill_metrics() { record_get_object_fill_queued("codec_streaming", "single_inflight", 1); diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index d80cdeb78..c33d1ae0a 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -41,6 +41,7 @@ pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL"; /// unset leaves rotation readiness unreported. Read once when the manager is /// built, by [`crate::manager::KmsManager`]. pub const ENV_KMS_ROTATION_MAX_AGE_SECS: &str = "RUSTFS_KMS_ROTATION_MAX_AGE_SECS"; +pub const ENV_KMS_ROTATION_MAX_WRAPS: &str = "RUSTFS_KMS_ROTATION_MAX_WRAPS"; pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret"; pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata"; pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle"; diff --git a/crates/kms/src/manager.rs b/crates/kms/src/manager.rs index c9ba1d6e3..f2cc936c5 100644 --- a/crates/kms/src/manager.rs +++ b/crates/kms/src/manager.rs @@ -17,7 +17,7 @@ use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink}; use crate::backends::KmsBackend; use crate::cache::{KmsCache, KmsCacheStats}; -use crate::config::{ENV_KMS_ALLOW_IMMEDIATE_DELETION, ENV_KMS_ROTATION_MAX_AGE_SECS, KmsConfig}; +use crate::config::{ENV_KMS_ALLOW_IMMEDIATE_DELETION, ENV_KMS_ROTATION_MAX_AGE_SECS, ENV_KMS_ROTATION_MAX_WRAPS, KmsConfig}; use crate::deletion_worker::DeletionReferenceChecker; use crate::error::{KmsError, Result}; use crate::types::{ @@ -42,6 +42,13 @@ use tracing::warn; /// after it was rotated, which trains operators to ignore the signal. const MIN_ROTATION_MAX_AGE: Duration = Duration::from_secs(3600); +/// Smallest wrap budget that can be configured. +/// +/// Wraps are accounted in reserved blocks, so any threshold below one block +/// would be crossed by a single reservation and report a key that has barely +/// wrapped anything as overdue. +const MIN_ROTATION_MAX_WRAPS: u64 = 1_000_000; + /// Rotation age from the environment, or `None` when the signal is off. /// /// Unset leaves it off rather than guessing a policy: how often a deployment @@ -68,6 +75,33 @@ fn parse_rotation_max_age(value: Option<&str>) -> Option { Some(Duration::from_secs(seconds).max(MIN_ROTATION_MAX_AGE)) } +/// Wrap budget from the environment, or `None` when the signal is off. +/// +/// Same discipline as the age threshold: unset means unreported rather than a +/// guessed policy, and an unparsable value is refused loudly instead of +/// falling back to a number the operator did not write. Clamped to +/// [`MIN_ROTATION_MAX_WRAPS`] because the backend accounts for wraps in +/// reserved blocks, so a threshold below one block would trip on the first +/// reservation regardless of how many wraps actually happened. +fn configured_rotation_max_wraps() -> Option { + parse_rotation_max_wraps(std::env::var(ENV_KMS_ROTATION_MAX_WRAPS).ok().as_deref()) +} + +fn parse_rotation_max_wraps(value: Option<&str>) -> Option { + let value = value?; + let Ok(wraps) = value.trim().parse::() else { + warn!( + variable = ENV_KMS_ROTATION_MAX_WRAPS, + "ignoring unparsable KMS rotation wrap budget; rotation readiness stays unreported" + ); + return None; + }; + if wraps == 0 { + return None; + } + Some(wraps.max(MIN_ROTATION_MAX_WRAPS)) +} + #[derive(Clone)] pub struct KmsManager { backend: Arc, @@ -82,6 +116,7 @@ pub struct KmsManager { /// the verdict unreported. Read once at construction so a listing cannot /// change its answer halfway through. rotation_max_age: Option, + rotation_max_wraps: Option, } impl KmsManager { @@ -103,6 +138,7 @@ impl KmsManager { allow_immediate_deletion: config.allow_immediate_deletion, reference_checker: None, rotation_max_age: configured_rotation_max_age(), + rotation_max_wraps: configured_rotation_max_wraps(), } } @@ -314,9 +350,22 @@ impl KmsManager { key.rotation_due_reason = Some(RotationDueReason::Unsupported); return; } + key.rotation_due = false; + key.rotation_due_reason = None; + + // The wrap budget is checked first: it is the cryptographic bound (the + // AES-GCM random-nonce ceiling), whereas the age threshold is a policy + // choice, so when both are crossed the reason an operator most needs to + // see is the one they cannot negotiate. + if let (Some(max_wraps), Some(wraps)) = (self.rotation_max_wraps, key.wrap_budget_reserved) + && wraps >= max_wraps + { + key.rotation_due = true; + key.rotation_due_reason = Some(RotationDueReason::Wraps); + return; + } + let Some(max_age) = self.rotation_max_age else { - key.rotation_due = false; - key.rotation_due_reason = None; return; }; @@ -333,9 +382,6 @@ impl KmsManager { if age >= max_age { key.rotation_due = true; key.rotation_due_reason = Some(reason); - } else { - key.rotation_due = false; - key.rotation_due_reason = None; } } @@ -1685,10 +1731,15 @@ mod tests { } fn readiness_manager(rotation_max_age: Option) -> KmsManager { + readiness_manager_with(rotation_max_age, None) + } + + fn readiness_manager_with(rotation_max_age: Option, rotation_max_wraps: Option) -> KmsManager { let temp_dir = tempfile::tempdir().expect("temp dir"); let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults(); let mut manager = KmsManager::new(Arc::new(ScriptedBackend::succeeding()), config); manager.rotation_max_age = rotation_max_age; + manager.rotation_max_wraps = rotation_max_wraps; manager } @@ -1765,6 +1816,85 @@ mod tests { assert!(!key.rotation_due, "clock skew must not manufacture an overdue key"); } + /// The wrap-budget half of the verdict: the cryptographic bound, checked + /// independently of the age policy and reported under its own reason. + #[test] + fn rotation_readiness_reports_an_exhausted_wrap_budget() { + let now = Zoned::now(); + let recently = &now - jiff::Span::new().hours(1); + let long_ago = &now - jiff::Span::new().days(400); + let day = Duration::from_secs(86_400); + let budget = 2_000_000; + + let with_wraps = |manager: &KmsManager, wraps: Option, rotated_at: Option| { + let mut key = aged_key(rotated_at, recently.clone()); + key.wrap_budget_reserved = wraps; + manager.apply_rotation_readiness(&mut key, true, &now); + (key.rotation_due, key.rotation_due_reason) + }; + + // Budget configured and exceeded on a freshly rotated key: due, and the + // reason names the wrap budget rather than an age nobody crossed. + let manager = readiness_manager_with(Some(day), Some(budget)); + assert_eq!( + with_wraps(&manager, Some(budget), Some(recently.clone())), + (true, Some(RotationDueReason::Wraps)) + ); + // At the threshold exactly, not only past it: the bound is a ceiling. + assert_eq!( + with_wraps(&manager, Some(budget + 1), Some(recently.clone())), + (true, Some(RotationDueReason::Wraps)) + ); + // Under the threshold: no verdict from the wrap half. + assert_eq!(with_wraps(&manager, Some(budget - 1), Some(recently.clone())), (false, None)); + + // The cryptographic bound outranks the policy one when both are crossed. + let mut key = aged_key(Some(long_ago.clone()), long_ago); + key.wrap_budget_reserved = Some(budget); + manager.apply_rotation_readiness(&mut key, true, &now); + assert_eq!(key.rotation_due_reason, Some(RotationDueReason::Wraps)); + + // No wrap threshold configured: an enormous count reports nothing, the + // same way an unset age threshold does. + let age_only = readiness_manager_with(Some(day), None); + assert_eq!(with_wraps(&age_only, Some(u64::MAX), Some(recently.clone())), (false, None)); + + // Backend reports no count (Transit, AWS, or a pre-accounting record): + // the wrap half stays silent instead of guessing, and the age half + // still decides. + let wraps_only = readiness_manager_with(None, Some(budget)); + assert_eq!(with_wraps(&wraps_only, None, Some(recently.clone())), (false, None)); + assert_eq!( + with_wraps(&wraps_only, Some(budget), Some(recently.clone())), + (true, Some(RotationDueReason::Wraps)) + ); + + // A backend that cannot rotate is never told to, whatever it wrapped. + let mut key = aged_key(None, recently); + key.wrap_budget_reserved = Some(u64::MAX); + wraps_only.apply_rotation_readiness(&mut key, false, &now); + assert!(!key.rotation_due); + assert_eq!(key.rotation_due_reason, Some(RotationDueReason::Unsupported)); + } + + /// Threshold parsing matches the age threshold's discipline: unset and + /// unparsable both disable the signal rather than inventing a policy. + #[test] + fn rotation_wrap_threshold_parsing_refuses_to_guess() { + assert_eq!(parse_rotation_max_wraps(None), None); + assert_eq!(parse_rotation_max_wraps(Some("not-a-number")), None); + assert_eq!(parse_rotation_max_wraps(Some("")), None); + assert_eq!(parse_rotation_max_wraps(Some("-1")), None); + assert_eq!(parse_rotation_max_wraps(Some("0")), None); + // Clamped: below one reservation block the first reservation would trip it. + assert_eq!(parse_rotation_max_wraps(Some("1")), Some(MIN_ROTATION_MAX_WRAPS)); + assert_eq!( + parse_rotation_max_wraps(Some(" 5000000 ")), + Some(5_000_000), + "a configured budget above the floor is honored verbatim" + ); + } + /// The two fields are additive on the wire: a payload written before they /// existed still deserializes, and a key with no verdict serializes exactly /// as it did before. diff --git a/crates/kms/src/types.rs b/crates/kms/src/types.rs index ac41d0c98..b87d26549 100644 --- a/crates/kms/src/types.rs +++ b/crates/kms/src/types.rs @@ -217,6 +217,12 @@ pub enum RotationDueReason { /// The key has never been rotated and has existed longer than the /// configured maximum age. NeverRotated, + /// The key has wrapped more data keys than the configured maximum. + /// + /// Counted per key-material version, so a rotation restarts the budget. + /// The count is an over-estimate by construction (see the backend's + /// reservation accounting), so this verdict errs toward rotating early. + Wraps, /// The backend cannot rotate keys at all, so no age makes one due. Unsupported, } diff --git a/crates/obs/src/metrics/collectors/audit.rs b/crates/obs/src/metrics/collectors/audit.rs index eb6d75f30..506c2390d 100644 --- a/crates/obs/src/metrics/collectors/audit.rs +++ b/crates/obs/src/metrics/collectors/audit.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! Audit metrics collector. //! //! Collects audit log metrics including failed messages, queue length, diff --git a/crates/obs/src/metrics/collectors/cluster_config.rs b/crates/obs/src/metrics/collectors/cluster_config.rs index e079be17e..5985b6017 100644 --- a/crates/obs/src/metrics/collectors/cluster_config.rs +++ b/crates/obs/src/metrics/collectors/cluster_config.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! Cluster config metrics collector. //! //! Collects cluster configuration metrics including storage class diff --git a/crates/obs/src/metrics/collectors/cluster_erasure_set.rs b/crates/obs/src/metrics/collectors/cluster_erasure_set.rs index 0a94b10a3..b919b17e0 100644 --- a/crates/obs/src/metrics/collectors/cluster_erasure_set.rs +++ b/crates/obs/src/metrics/collectors/cluster_erasure_set.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! Cluster erasure set metrics collector. //! //! Collects erasure coding set metrics including parity, quorum, diff --git a/crates/obs/src/metrics/collectors/cluster_health.rs b/crates/obs/src/metrics/collectors/cluster_health.rs index 2f579c163..d84c428d4 100644 --- a/crates/obs/src/metrics/collectors/cluster_health.rs +++ b/crates/obs/src/metrics/collectors/cluster_health.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! Cluster health metrics collector. //! //! Collects cluster-wide health metrics including drive counts diff --git a/crates/obs/src/metrics/collectors/cluster_iam.rs b/crates/obs/src/metrics/collectors/cluster_iam.rs index 39a5e187f..435c5b39d 100644 --- a/crates/obs/src/metrics/collectors/cluster_iam.rs +++ b/crates/obs/src/metrics/collectors/cluster_iam.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! Cluster IAM metrics collector. //! //! Collects IAM (Identity and Access Management) metrics including diff --git a/crates/obs/src/metrics/collectors/cluster_usage.rs b/crates/obs/src/metrics/collectors/cluster_usage.rs index c493145ec..76ccd6cf5 100644 --- a/crates/obs/src/metrics/collectors/cluster_usage.rs +++ b/crates/obs/src/metrics/collectors/cluster_usage.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! Cluster usage metrics collector. //! //! Collects cluster-wide and per-bucket usage metrics including diff --git a/crates/obs/src/metrics/collectors/ilm.rs b/crates/obs/src/metrics/collectors/ilm.rs index 664f3a36b..d9a3d55a3 100644 --- a/crates/obs/src/metrics/collectors/ilm.rs +++ b/crates/obs/src/metrics/collectors/ilm.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! ILM (Information Lifecycle Management) metrics collector. //! //! Collects ILM metrics including pending tasks, active tasks, diff --git a/crates/obs/src/metrics/collectors/notification.rs b/crates/obs/src/metrics/collectors/notification.rs index 2c7d97f61..7a10e0efd 100644 --- a/crates/obs/src/metrics/collectors/notification.rs +++ b/crates/obs/src/metrics/collectors/notification.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! Notification metrics collector. //! //! Collects notification system metrics including events sent, diff --git a/crates/obs/src/metrics/collectors/notification_target.rs b/crates/obs/src/metrics/collectors/notification_target.rs index 797f00ff4..7f6095285 100644 --- a/crates/obs/src/metrics/collectors/notification_target.rs +++ b/crates/obs/src/metrics/collectors/notification_target.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::metrics::report::PrometheusMetric; use crate::metrics::schema::notification_target::{ NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD, NOTIFICATION_TARGET_FAILED_MESSAGES_MD, diff --git a/crates/obs/src/metrics/collectors/replication.rs b/crates/obs/src/metrics/collectors/replication.rs index c1311696b..4dc347058 100644 --- a/crates/obs/src/metrics/collectors/replication.rs +++ b/crates/obs/src/metrics/collectors/replication.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! Replication metrics collector. //! //! Collects cluster-wide replication metrics including queue stats, diff --git a/crates/obs/src/metrics/collectors/request.rs b/crates/obs/src/metrics/collectors/request.rs index 2aae53666..fe5037102 100644 --- a/crates/obs/src/metrics/collectors/request.rs +++ b/crates/obs/src/metrics/collectors/request.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! API request metrics collector. //! //! Collects API request metrics including request counts, errors, diff --git a/crates/obs/src/metrics/collectors/scanner.rs b/crates/obs/src/metrics/collectors/scanner.rs index 7ce05c23d..db220ca96 100644 --- a/crates/obs/src/metrics/collectors/scanner.rs +++ b/crates/obs/src/metrics/collectors/scanner.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! Scanner metrics collector. //! //! Collects background scanner metrics including bucket-drive scans, diff --git a/crates/obs/src/metrics/collectors/system_cpu.rs b/crates/obs/src/metrics/collectors/system_cpu.rs index 5e829a8b4..88fde08ef 100644 --- a/crates/obs/src/metrics/collectors/system_cpu.rs +++ b/crates/obs/src/metrics/collectors/system_cpu.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! System CPU metrics collector. //! //! Collects CPU metrics including load average, CPU time distribution, diff --git a/crates/obs/src/metrics/collectors/system_drive.rs b/crates/obs/src/metrics/collectors/system_drive.rs index 936a88c4e..8306ef175 100644 --- a/crates/obs/src/metrics/collectors/system_drive.rs +++ b/crates/obs/src/metrics/collectors/system_drive.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! System drive metrics collector. //! //! Collects detailed drive/disk metrics including capacity, I/O statistics, diff --git a/crates/obs/src/metrics/collectors/system_gpu.rs b/crates/obs/src/metrics/collectors/system_gpu.rs index beb0aca3c..72719c0ad 100644 --- a/crates/obs/src/metrics/collectors/system_gpu.rs +++ b/crates/obs/src/metrics/collectors/system_gpu.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! System GPU metrics collector. //! //! Collects GPU memory usage metrics using NVML library. diff --git a/crates/obs/src/metrics/collectors/system_memory.rs b/crates/obs/src/metrics/collectors/system_memory.rs index c9e319365..acd4ad4e9 100644 --- a/crates/obs/src/metrics/collectors/system_memory.rs +++ b/crates/obs/src/metrics/collectors/system_memory.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! System memory metrics collector. //! //! Collects memory-related metrics including total, used, free, diff --git a/crates/obs/src/metrics/collectors/system_network.rs b/crates/obs/src/metrics/collectors/system_network.rs index 8bb660de0..4bd4949c0 100644 --- a/crates/obs/src/metrics/collectors/system_network.rs +++ b/crates/obs/src/metrics/collectors/system_network.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! System network metrics collector. //! //! Collects internode network metrics including errors, dial times, diff --git a/crates/obs/src/metrics/collectors/system_process.rs b/crates/obs/src/metrics/collectors/system_process.rs index 708c1e975..99dfd83b9 100644 --- a/crates/obs/src/metrics/collectors/system_process.rs +++ b/crates/obs/src/metrics/collectors/system_process.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! System process metrics collector. //! //! Collects process-level metrics including file descriptors, memory, diff --git a/crates/obs/src/metrics/schema/audit.rs b/crates/obs/src/metrics/schema/audit.rs index b08e95269..a4021c323 100644 --- a/crates/obs/src/metrics/schema/audit.rs +++ b/crates/obs/src/metrics/schema/audit.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/bucket.rs b/crates/obs/src/metrics/schema/bucket.rs index e4af34615..d5d8d802d 100644 --- a/crates/obs/src/metrics/schema/bucket.rs +++ b/crates/obs/src/metrics/schema/bucket.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/bucket_replication.rs b/crates/obs/src/metrics/schema/bucket_replication.rs index 1d7b1c924..2131c7412 100644 --- a/crates/obs/src/metrics/schema/bucket_replication.rs +++ b/crates/obs/src/metrics/schema/bucket_replication.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/cluster.rs b/crates/obs/src/metrics/schema/cluster.rs index 58637fc0f..ee4793ced 100644 --- a/crates/obs/src/metrics/schema/cluster.rs +++ b/crates/obs/src/metrics/schema/cluster.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/cluster_config.rs b/crates/obs/src/metrics/schema/cluster_config.rs index 2b10d3fb2..eb35a9036 100644 --- a/crates/obs/src/metrics/schema/cluster_config.rs +++ b/crates/obs/src/metrics/schema/cluster_config.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/cluster_erasure_set.rs b/crates/obs/src/metrics/schema/cluster_erasure_set.rs index 3c496f92f..23f852d28 100644 --- a/crates/obs/src/metrics/schema/cluster_erasure_set.rs +++ b/crates/obs/src/metrics/schema/cluster_erasure_set.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/cluster_health.rs b/crates/obs/src/metrics/schema/cluster_health.rs index 444bfa927..1c8b05614 100644 --- a/crates/obs/src/metrics/schema/cluster_health.rs +++ b/crates/obs/src/metrics/schema/cluster_health.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/cluster_iam.rs b/crates/obs/src/metrics/schema/cluster_iam.rs index 7c0ff00a3..4c0bae72c 100644 --- a/crates/obs/src/metrics/schema/cluster_iam.rs +++ b/crates/obs/src/metrics/schema/cluster_iam.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/cluster_notification.rs b/crates/obs/src/metrics/schema/cluster_notification.rs index 5fbedc67c..8be9a9807 100644 --- a/crates/obs/src/metrics/schema/cluster_notification.rs +++ b/crates/obs/src/metrics/schema/cluster_notification.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/cluster_usage.rs b/crates/obs/src/metrics/schema/cluster_usage.rs index 24dab6187..4b44104ed 100644 --- a/crates/obs/src/metrics/schema/cluster_usage.rs +++ b/crates/obs/src/metrics/schema/cluster_usage.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/ilm.rs b/crates/obs/src/metrics/schema/ilm.rs index 01316de37..ac0538654 100644 --- a/crates/obs/src/metrics/schema/ilm.rs +++ b/crates/obs/src/metrics/schema/ilm.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/node_bucket.rs b/crates/obs/src/metrics/schema/node_bucket.rs index c8e2393da..77561ee92 100644 --- a/crates/obs/src/metrics/schema/node_bucket.rs +++ b/crates/obs/src/metrics/schema/node_bucket.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_gauge_md}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/node_disk.rs b/crates/obs/src/metrics/schema/node_disk.rs index fad52b6c7..3a521e3fd 100644 --- a/crates/obs/src/metrics/schema/node_disk.rs +++ b/crates/obs/src/metrics/schema/node_disk.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_gauge_md}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/notification_target.rs b/crates/obs/src/metrics/schema/notification_target.rs index 32c4c4502..cec0ca9cf 100644 --- a/crates/obs/src/metrics/schema/notification_target.rs +++ b/crates/obs/src/metrics/schema/notification_target.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/process_resource.rs b/crates/obs/src/metrics/schema/process_resource.rs index 5bac84f5a..d0a243a3a 100644 --- a/crates/obs/src/metrics/schema/process_resource.rs +++ b/crates/obs/src/metrics/schema/process_resource.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::node_identity::SERVER_LABEL; use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_gauge_md}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/replication.rs b/crates/obs/src/metrics/schema/replication.rs index 93cd0cbd7..1290a770e 100644 --- a/crates/obs/src/metrics/schema/replication.rs +++ b/crates/obs/src/metrics/schema/replication.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/request.rs b/crates/obs/src/metrics/schema/request.rs index 9f90a64c9..8009742ab 100644 --- a/crates/obs/src/metrics/schema/request.rs +++ b/crates/obs/src/metrics/schema/request.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; /// name label @@ -32,6 +30,13 @@ const API_SERVER_NAME_TYPE_LE_LABELS: [&str; 4] = [SERVER_LABEL, NAME_LABEL, TYP const API_TYPE_LABELS: [&str; 1] = [TYPE_LABEL]; const API_SERVER_TYPE_LABELS: [&str; 2] = [SERVER_LABEL, TYPE_LABEL]; +// Declared for MinIO metric parity but never emitted: no collector passes these +// descriptors to `PrometheusMetric::from_descriptor`, so the wire names +// (`rejected_auth_total`, `rejected_header_total`, `rejected_timestamp_total`, +// `rejected_invalid_total`, `waiting_total`, `incoming_total`) never appear in a +// scrape. Kept so the gap stays greppable rather than silently disappearing with +// their `MetricName` variants; wiring an emitter is what retires these allows. +#[allow(dead_code, reason = "declared metric with no emitter; see note above (backlog#1823)")] pub static API_REJECTED_AUTH_TOTAL_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::ApiRejectedAuthTotal, @@ -41,6 +46,7 @@ pub static API_REJECTED_AUTH_TOTAL_MD: LazyLock = LazyLock::ne ) }); +#[allow(dead_code, reason = "declared metric with no emitter; see note above (backlog#1823)")] pub static API_REJECTED_HEADER_TOTAL_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::ApiRejectedHeaderTotal, @@ -50,6 +56,7 @@ pub static API_REJECTED_HEADER_TOTAL_MD: LazyLock = LazyLock:: ) }); +#[allow(dead_code, reason = "declared metric with no emitter; see note above (backlog#1823)")] pub static API_REJECTED_TIMESTAMP_TOTAL_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::ApiRejectedTimestampTotal, @@ -59,6 +66,7 @@ pub static API_REJECTED_TIMESTAMP_TOTAL_MD: LazyLock = LazyLoc ) }); +#[allow(dead_code, reason = "declared metric with no emitter; see note above (backlog#1823)")] pub static API_REJECTED_INVALID_TOTAL_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::ApiRejectedInvalidTotal, @@ -68,6 +76,7 @@ pub static API_REJECTED_INVALID_TOTAL_MD: LazyLock = LazyLock: ) }); +#[allow(dead_code, reason = "declared metric with no emitter; see note above (backlog#1823)")] pub static API_REQUESTS_WAITING_TOTAL_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ApiRequestsWaitingTotal, @@ -77,6 +86,7 @@ pub static API_REQUESTS_WAITING_TOTAL_MD: LazyLock = LazyLock: ) }); +#[allow(dead_code, reason = "declared metric with no emitter; see note above (backlog#1823)")] pub static API_REQUESTS_INCOMING_TOTAL_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ApiRequestsIncomingTotal, diff --git a/crates/obs/src/metrics/schema/scanner.rs b/crates/obs/src/metrics/schema/scanner.rs index a9027bf07..53cb9548b 100644 --- a/crates/obs/src/metrics/schema/scanner.rs +++ b/crates/obs/src/metrics/schema/scanner.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/system_cpu.rs b/crates/obs/src/metrics/schema/system_cpu.rs index f0a3ea784..8ef132128 100644 --- a/crates/obs/src/metrics/schema/system_cpu.rs +++ b/crates/obs/src/metrics/schema/system_cpu.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::node_identity::SERVER_LABEL; use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; /// CPU system-related metric descriptors diff --git a/crates/obs/src/metrics/schema/system_drive.rs b/crates/obs/src/metrics/schema/system_drive.rs index e32fab738..e64ee0b9d 100644 --- a/crates/obs/src/metrics/schema/system_drive.rs +++ b/crates/obs/src/metrics/schema/system_drive.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/system_gpu.rs b/crates/obs/src/metrics/schema/system_gpu.rs index 958e6cace..02c0db587 100644 --- a/crates/obs/src/metrics/schema/system_gpu.rs +++ b/crates/obs/src/metrics/schema/system_gpu.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! GPU-related metric descriptors. //! //! This module defines metric descriptors for GPU monitoring, diff --git a/crates/obs/src/metrics/schema/system_memory.rs b/crates/obs/src/metrics/schema/system_memory.rs index 0ffa9f193..7204bd984 100644 --- a/crates/obs/src/metrics/schema/system_memory.rs +++ b/crates/obs/src/metrics/schema/system_memory.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::node_identity::SERVER_LABEL; use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/system_network.rs b/crates/obs/src/metrics/schema/system_network.rs index 5571951fd..425fc1655 100644 --- a/crates/obs/src/metrics/schema/system_network.rs +++ b/crates/obs/src/metrics/schema/system_network.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::node_identity::SERVER_LABEL; use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/system_network_host.rs b/crates/obs/src/metrics/schema/system_network_host.rs index 45cc89ffe..26d4cc98f 100644 --- a/crates/obs/src/metrics/schema/system_network_host.rs +++ b/crates/obs/src/metrics/schema/system_network_host.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::node_identity::SERVER_LABEL; use crate::{MetricDescriptor, MetricName, new_counter_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/schema/system_process.rs b/crates/obs/src/metrics/schema/system_process.rs index ecddbc9c3..c6b45453b 100644 --- a/crates/obs/src/metrics/schema/system_process.rs +++ b/crates/obs/src/metrics/schema/system_process.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use crate::node_identity::SERVER_LABEL; use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; diff --git a/crates/obs/src/metrics/stats_collector.rs b/crates/obs/src/metrics/stats_collector.rs index c312f7eff..cf5e77025 100644 --- a/crates/obs/src/metrics/stats_collector.rs +++ b/crates/obs/src/metrics/stats_collector.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! Statistics collection functions for metrics. //! //! This module contains functions that collect statistics from various diff --git a/crates/protocols/tests/swift_metadata_persistence.rs b/crates/protocols/tests/swift_metadata_persistence.rs index 86f205a45..6b0b2df8d 100644 --- a/crates/protocols/tests/swift_metadata_persistence.rs +++ b/crates/protocols/tests/swift_metadata_persistence.rs @@ -23,6 +23,7 @@ //! two are tested together because a reload is the only way to tell a real //! merge from one that happened to look right in the cache. +#![recursion_limit = "256"] #![cfg(feature = "swift")] use std::collections::HashMap; diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index 6b6deb29d..4a5cc7543 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![recursion_limit = "256"] #![cfg_attr(docsrs, feature(doc_auto_cfg))] #![warn( // missing_docs, diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index 9fef7a372..0a43a8060 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![recursion_limit = "256"] + use futures::FutureExt; use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT; use rustfs_scanner::scanner_folder::ScannerItem; diff --git a/crates/utils/src/http/metadata_compat.rs b/crates/utils/src/http/metadata_compat.rs index bc71f81ec..a3cc07b6d 100644 --- a/crates/utils/src/http/metadata_compat.rs +++ b/crates/utils/src/http/metadata_compat.rs @@ -59,6 +59,7 @@ pub const SUFFIX_TRANSITION_TIER_DESTINATION_ID: &str = "transition-tier-destina pub const SUFFIX_TRANSITION_TRANSACTION_ID: &str = "transition-transaction-id"; pub const SUFFIX_RESTORE_OPERATION_ID: &str = "restore-operation-id"; pub const SUFFIX_BUCKET_INCARNATION_ID: &str = "bucket-incarnation-id"; +pub const SUFFIX_OBJECT_TRANSACTION_EPOCH: &str = "object-transaction-epoch"; pub const SUFFIX_FREE_VERSION: &str = "free-version"; pub const SUFFIX_PURGESTATUS: &str = "purgestatus"; pub const SUFFIX_REPLICA_STATUS: &str = "replica-status"; diff --git a/crates/utils/src/http/object_encryption_keys.rs b/crates/utils/src/http/object_encryption_keys.rs index 7b71faeae..661d5bd98 100644 --- a/crates/utils/src/http/object_encryption_keys.rs +++ b/crates/utils/src/http/object_encryption_keys.rs @@ -25,7 +25,8 @@ // The lowercase stored forms, matching exactly what encryption_material_to_metadata // persists. The read-path SSE-C check is case-sensitive, so restoring under any // other casing would classify the replica as managed-SSE and reject SSE-C GETs. -use super::headers::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER}; +use super::headers::{AMZ_ENCRYPTION_KMS, SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER}; +use std::collections::HashMap; pub const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id"; pub const INTERNAL_ENCRYPTION_KEY_HEADER: &str = "x-rustfs-encryption-key"; @@ -165,6 +166,143 @@ pub fn is_replication_stripped_encryption_key(key: &str) -> bool { || super::starts_with_ignore_ascii_case(key, RUSTFS_INTERNAL_ENCRYPTION_PREFIX) } +// ============================================================================ +// Managed-SSE attribution (shared classifier) +// ============================================================================ +// +// Single source of truth for classifying stored managed-SSE (SSE-S3 / SSE-KMS) +// object metadata. These live here — rather than in the `rustfs` binary +// crate's SSE module — so lower-layer consumers such as the scanner can +// attribute encrypted objects without growing a second copy of the +// normalization/classification logic (backlog#1643 PR-B0). The binary crate +// re-exports them from `rustfs::storage::sse`, and a source-scan test there +// pins that no second definition reappears. +// +// Every metadata lookup below is a case-SENSITIVE exact match on the stored +// `HashMap` keys, mirroring the SSE read path. Do not +// "harmonize" these with the lowercase-normalizing helpers in +// `header_compat.rs`: the lowercase `x-amz-*` stored forms and the TitleCase +// MinIO-internal names are load-bearing exactly as written. + +/// Type of encryption used +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SSEType { + /// SSE-S3 (AES256) + SseS3, + /// SSE-KMS (aws:kms) + SseKms, + /// SSE-C (customer-provided key) + SseC, +} + +impl SSEType { + /// Stable scheme name for audit consumers. + pub fn audit_label(self) -> &'static str { + match self { + SSEType::SseS3 => "SSE-S3", + SSEType::SseKms => "SSE-KMS", + SSEType::SseC => "SSE-C", + } + } +} + +/// Recodes a stored MinIO KMS context value — base64-wrapped JSON under +/// [`MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER`] — into the plain-JSON form +/// RustFS stores under [`INTERNAL_ENCRYPTION_CONTEXT_HEADER`]. +/// +/// Injected by callers because this crate deliberately carries no JSON codec. +/// Returning `None` skips the context mapping, matching the historical +/// silent-skip on a value that fails to decode. +pub type KmsContextRecoder = fn(&str) -> Option; + +/// True when the stored metadata carries a managed-SSE (SSE-S3 / SSE-KMS) +/// encryption envelope, under either the RustFS-branded or the MinIO-branded +/// internal keys. +pub fn contains_managed_encryption_metadata(metadata: &HashMap) -> bool { + metadata.contains_key(INTERNAL_ENCRYPTION_KEY_HEADER) + || metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER) + || metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER) + || metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER) + || metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER) +} + +/// Maps the MinIO-branded internal SSE keys onto the RustFS-branded stored +/// keys (the dual internal metadata keys invariant). RustFS-branded keys +/// already present always win; every source lookup is a case-sensitive exact +/// match on the specific TitleCase MinIO names. +pub fn normalize_managed_metadata( + metadata: &HashMap, + recode_kms_context: Option, +) -> HashMap { + let mut normalized = metadata.clone(); + + if !normalized.contains_key(INTERNAL_ENCRYPTION_KEY_HEADER) + && let Some(value) = metadata + .get(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER) + .or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)) + .or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)) + .or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)) + { + normalized.insert(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), value.clone()); + } + + if !normalized.contains_key(INTERNAL_ENCRYPTION_IV_HEADER) + && let Some(value) = metadata.get(MINIO_INTERNAL_ENCRYPTION_IV_HEADER) + { + normalized.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), value.clone()); + } + + if !normalized.contains_key(INTERNAL_ENCRYPTION_ALGORITHM_HEADER) + && let Some(value) = metadata.get(MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER) + { + normalized.insert(INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), value.clone()); + } + + if !normalized.contains_key(INTERNAL_ENCRYPTION_KEY_ID_HEADER) + && let Some(value) = metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER) + { + normalized.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), value.clone()); + } + + if !normalized.contains_key(INTERNAL_ENCRYPTION_CONTEXT_HEADER) + && let Some(value) = metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER) + && let Some(recode) = recode_kms_context + && let Some(encoded) = recode(value) + { + normalized.insert(INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(), encoded); + } + + normalized +} + +/// Resolve the scheme and KMS key a stored managed-SSE object was wrapped with. +/// +/// Mirrors the lookup `apply_managed_decryption_material` performs, so both agree on +/// which key a read is authorized against. +/// +/// No [`KmsContextRecoder`] is taken: the context mapping only ever inserts +/// [`INTERNAL_ENCRYPTION_CONTEXT_HEADER`], which this lookup never reads, so +/// the result is identical with or without it. +pub fn stored_managed_encryption_key(metadata: &HashMap) -> Option<(SSEType, String)> { + if !contains_managed_encryption_metadata(metadata) { + return None; + } + + // Case-sensitive: the SSE writer stores the scheme under the lowercase + // `x-amz-server-side-encryption` key; other casings are not stored forms. + let sse_type = match metadata.get("x-amz-server-side-encryption")?.as_str() { + AMZ_ENCRYPTION_KMS => SSEType::SseKms, + _ => SSEType::SseS3, + }; + let key_id = normalize_managed_metadata(metadata, None) + .get(INTERNAL_ENCRYPTION_KEY_ID_HEADER) + .or_else(|| metadata.get("x-amz-server-side-encryption-aws-kms-key-id")) + .cloned() + .unwrap_or_else(|| "default".to_string()); + + Some((sse_type, key_id)) +} + #[cfg(test)] mod tests { use super::*; @@ -273,6 +411,134 @@ mod tests { assert!(!is_replication_stripped_encryption_key("content-type")); } + #[test] + fn managed_envelope_predicate_matches_both_key_families() { + assert!(!contains_managed_encryption_metadata(&HashMap::new())); + + for key in [ + INTERNAL_ENCRYPTION_KEY_HEADER, + MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, + ] { + let single = HashMap::from([(key.to_string(), "value".to_string())]); + assert!(contains_managed_encryption_metadata(&single), "{key} must classify as managed SSE"); + } + + // SSE-C material alone is not a managed envelope. + let ssec_only = HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]); + assert!(!contains_managed_encryption_metadata(&ssec_only)); + } + + #[test] + fn normalize_maps_minio_keys_onto_missing_rustfs_keys_only() { + let metadata = HashMap::from([ + (MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(), "minio-dek".to_string()), + (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), "minio-iv".to_string()), + (MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), "DAREv2-HMAC-SHA256".to_string()), + (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "minio-key".to_string()), + ]); + + let normalized = normalize_managed_metadata(&metadata, None); + assert_eq!(normalized.get(INTERNAL_ENCRYPTION_KEY_HEADER).map(String::as_str), Some("minio-dek")); + assert_eq!(normalized.get(INTERNAL_ENCRYPTION_IV_HEADER).map(String::as_str), Some("minio-iv")); + assert_eq!( + normalized.get(INTERNAL_ENCRYPTION_ALGORITHM_HEADER).map(String::as_str), + Some("DAREv2-HMAC-SHA256") + ); + assert_eq!(normalized.get(INTERNAL_ENCRYPTION_KEY_ID_HEADER).map(String::as_str), Some("minio-key")); + + // Existing RustFS-branded keys always win over the MinIO twins. + let mut both = metadata; + both.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "rustfs-key".to_string()); + assert_eq!( + normalize_managed_metadata(&both, None) + .get(INTERNAL_ENCRYPTION_KEY_ID_HEADER) + .map(String::as_str), + Some("rustfs-key") + ); + + // The mapping is a case-sensitive exact match on the TitleCase MinIO + // names; a lowercased twin must not normalize. + let lowercased = HashMap::from([(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_lowercase(), "minio-key".to_string())]); + assert!(!normalize_managed_metadata(&lowercased, None).contains_key(INTERNAL_ENCRYPTION_KEY_ID_HEADER)); + } + + #[test] + fn normalize_recodes_kms_context_only_through_the_injected_codec() { + let metadata = HashMap::from([(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), "encoded-context".to_string())]); + + // Without a codec the context stays unnormalized. + assert!(!normalize_managed_metadata(&metadata, None).contains_key(INTERNAL_ENCRYPTION_CONTEXT_HEADER)); + + // A codec that fails to decode also leaves it unnormalized. + fn reject(_value: &str) -> Option { + None + } + assert!(!normalize_managed_metadata(&metadata, Some(reject)).contains_key(INTERNAL_ENCRYPTION_CONTEXT_HEADER)); + + fn recode(value: &str) -> Option { + Some(format!("recoded:{value}")) + } + assert_eq!( + normalize_managed_metadata(&metadata, Some(recode)) + .get(INTERNAL_ENCRYPTION_CONTEXT_HEADER) + .map(String::as_str), + Some("recoded:encoded-context") + ); + + // A stored RustFS context wins without invoking the codec. + let mut both = metadata; + both.insert(INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(), "stored-context".to_string()); + assert_eq!( + normalize_managed_metadata(&both, Some(recode)) + .get(INTERNAL_ENCRYPTION_CONTEXT_HEADER) + .map(String::as_str), + Some("stored-context") + ); + } + + #[test] + fn stored_managed_encryption_key_attributes_scheme_and_key() { + // Plaintext metadata carries no managed envelope. + assert!(stored_managed_encryption_key(&HashMap::new()).is_none()); + + // A managed envelope without the stored SSE marker cannot be attributed. + let envelope_only = HashMap::from([(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), "dek".to_string())]); + assert!(stored_managed_encryption_key(&envelope_only).is_none()); + + // The stored SSE marker is the lowercase form; a TitleCase key is not + // a stored form and must not be recognized. + let mut titlecase = envelope_only.clone(); + titlecase.insert("X-Amz-Server-Side-Encryption".to_string(), "aws:kms".to_string()); + assert!(stored_managed_encryption_key(&titlecase).is_none()); + + let mut sse_s3 = envelope_only.clone(); + sse_s3.insert("x-amz-server-side-encryption".to_string(), "AES256".to_string()); + assert_eq!(stored_managed_encryption_key(&sse_s3), Some((SSEType::SseS3, "default".to_string()))); + + let mut sse_kms = envelope_only; + sse_kms.insert("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()); + assert_eq!(stored_managed_encryption_key(&sse_kms), Some((SSEType::SseKms, "default".to_string()))); + + // Key-id precedence: RustFS stored key id, then the MinIO twin, then + // the lowercase amz key id, then "default". + sse_kms.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), "amz-key".to_string()); + assert_eq!(stored_managed_encryption_key(&sse_kms), Some((SSEType::SseKms, "amz-key".to_string()))); + sse_kms.insert(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "minio-key".to_string()); + assert_eq!(stored_managed_encryption_key(&sse_kms), Some((SSEType::SseKms, "minio-key".to_string()))); + sse_kms.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "rustfs-key".to_string()); + assert_eq!(stored_managed_encryption_key(&sse_kms), Some((SSEType::SseKms, "rustfs-key".to_string()))); + } + + #[test] + fn sse_type_audit_labels_are_stable() { + assert_eq!(SSEType::SseS3.audit_label(), "SSE-S3"); + assert_eq!(SSEType::SseKms.audit_label(), "SSE-KMS"); + assert_eq!(SSEType::SseC.audit_label(), "SSE-C"); + } + #[test] fn transport_prefixes_cover_every_transport_value_key() { // Every transport key that carries material must match a redaction diff --git a/docs/architecture/background-services-inventory.md b/docs/architecture/background-services-inventory.md index 587934670..d484a1004 100644 --- a/docs/architecture/background-services-inventory.md +++ b/docs/architecture/background-services-inventory.md @@ -22,7 +22,7 @@ not define a new scheduler, controller framework, or shutdown contract. | Scanner | `rustfs/src/main.rs::run` calls `init_data_scanner(ctx.clone(), store.clone())` after successful startup log and global init time. | Main shutdown calls `ctx.cancel()`; if scanner was enabled it also calls `shutdown_background_services()`. | Scanner loop receives the main runtime token. | | Heal/AHM | Main creates `create_ahm_services_cancel_token()` before scanner/heal feature checks and calls `init_heal_manager(...)` when heal or scanner is enabled. | Main shutdown calls `shutdown_ahm_services()` when heal or scanner was enabled. | Global AHM token plus channel/worker-local state. | | Replication pool | Main calls `init_background_replication(store.clone())` after global config init, then `pool.init_resync(ctx.clone(), buckets.clone())` after bucket listing. | No direct main shutdown call for the replication pool; resync receives the main runtime token. | Resync routine uses the main runtime token; per-bucket resync uses registered cancel tokens. | -| Lifecycle expiry/transition | `ECStore::init` calls `init_background_expiry(self.clone())` and `init_background_stale_multipart_upload_cleanup(self.clone())`. | Expiry workers read `get_background_services_cancel_token()` and fall back to a private token if none exists. Stale multipart cleanup exits when the weak ECStore reference cannot upgrade. | Inventory search found no current startup caller for `create_background_services_cancel_token()`. | +| Lifecycle expiry/transition | `ECStore::init` calls `init_background_expiry(self.clone())` and `init_background_stale_multipart_upload_cleanup(self.clone())`. | Expiry workers read `get_background_services_cancel_token()` and fall back to a private token if none exists. Stale multipart cleanup exits when the weak ECStore reference cannot upgrade. | `ECStore::init` binds the main runtime token into the instance context with `bind_background_cancel_token(ctx)` before expiry starts, so the private-token fallback is a defensive path rather than the normal one. | | Notification runtime | Main calls `init_event_notifier()` after buffer profile init. | Main shutdown calls `shutdown_event_notifier().await`. | Notification runtime owns target/replay shutdown internally. | | Audit runtime | Main calls `start_audit_system().await`. | Main shutdown calls `stop_audit_system().await`. | Audit runtime owns target/replay shutdown internally. | | Metrics and memory loops | Main calls `init_metrics_runtime(ctx.clone())`, `init_memory_observability(ctx.clone())`, and `init_auto_tuner(ctx.clone())` when observability metrics are enabled. | Main shutdown only cancels the shared runtime token. | Shared runtime token. | diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index e1a0b8040..880396b5f 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -34,6 +34,7 @@ for later deletion. - `tonic-013-status-render` peer RPC failure classification: internode failures that reach a node only as text (a peer's error_info payload, a status flattened through format!) are classified by matching the rendering of an Unavailable gRPC status. Releases up to 1.0.0-alpha.38 shipped tonic 0.13, which rendered that status as "status: Unavailable, message: ..."; tonic 0.14 renders it as "code: 'The service is currently unavailable', message: ...". Both forms are matched so an older peer's relayed text still marks an unreachable peer offline. Remove the tonic 0.13 form after the minimum supported RustFS peer version ships tonic 0.14 or later. - `rustfs-5063` pre-beta.9 Local KMS recovery: persisted Local KMS configs from beta.8 and earlier predate the explicit insecure-development flag, and encrypted key files use the legacy SHA-256 KDF. Remove the config fallback after supported upgrades have rewritten or explicitly resaved all pre-beta.9 configs with the development-default field, and remove the legacy KDF after supported upgrades have rewritten all pre-beta.9 Local KMS key files with explicit at-rest protection. - `sse-local-dek-json-v1` legacy local SSE DEK decoding: releases before the JSON envelope wrote wrapped DEKs as `base64(nonce):base64(ciphertext)`, so readers retain that decoder while all new writes use the versioned JSON envelope. Remove the colon decoder after the minimum supported direct-upgrade release writes JSON envelopes and migration tooling has rewritten every retained legacy object. +- `rio-v2-dormant-variant` dormant `rio-v2` build variant: `crates/rio-v2` and the `rio-v2` feature ship in no default or release build and exist only as the candidate MinIO stream-format implementation for the rustfs/backlog#1638 SSE-interop adjudication. Per-PR CI keeps only `test-and-lint-rio-v2` to guard the `#[cfg(feature = "rio-v2")]` seam; the full-suite lanes (`build-rustfs-debug-binary-rio-v2`, `e2e-tests-rio-v2` in `.github/workflows/ci.yml`) run on schedule/workflow_dispatch only — see the "`rio-v2` variant lifecycle" section in [minio-file-format-compat.md](minio-file-format-compat.md). While both implementations exist, DARE/S2 stream fixes must land in both `crates/rio` and `crates/rio-v2`. No `RUSTFS_COMPAT_TODO` source marker applies: the temporary surface is CI workflow YAML plus an entire feature-gated candidate crate, not a compatibility code path inside shipping code, and workflow files are outside the marker convention's Rust scope. Remove after the #1638 adjudication lands and converges on one implementation: delete the losing implementation, its feature seam, and the gating CI jobs. ## Review Checklist diff --git a/docs/architecture/minio-file-format-compat.md b/docs/architecture/minio-file-format-compat.md index ec17791cc..c6343c1bc 100644 --- a/docs/architecture/minio-file-format-compat.md +++ b/docs/architecture/minio-file-format-compat.md @@ -271,6 +271,32 @@ Seam 2 surfaces its own error, but only for objects that got past seam 1. The interop harness reflects this. The reader tests are `#[ignore]` (`rustfs/src/storage/minio_generated_read_test.rs:244`, `:250`), the workflow that would run them is disabled at the GitHub Actions level and states in its own header that end-to-end MinIO-to-RustFS SSE interop is not implemented (`.github/workflows/minio-interop.yml:24-29`, `:34-39`), and the fixture suite's scope note says the tests "do not yet validate full plaintext reconstruction from MinIO-written encrypted data" (`crates/rio-v2/tests/README.md:55`). +### `rio-v2` variant lifecycle + +The variant is deliberately **dormant** until rustfs/backlog#1638 is +adjudicated. Dormant means: + +- **Per-PR CI keeps one guard job.** Only `test-and-lint-rio-v2` in + `.github/workflows/ci.yml` runs per PR; its job is to keep the + `#[cfg(feature = "rio-v2")]` seam compiling and its unit tests green so the + variant does not bit-rot. The full-suite lanes — + `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` — run only on the + weekly `schedule` and on `workflow_dispatch`, not per PR, per main push, or + in the merge queue. +- **Post-1.0 the variant is promoted or deleted.** The #1638 adjudication + converges on one implementation: either `rio-v2` becomes a shipped + configuration, or the losing side is removed together with its feature seam + and its gating CI jobs. Tracked as `rio-v2-dormant-variant` in + [compat-cleanup-register.md](compat-cleanup-register.md). +- **DARE/S2 fixes land in both crates.** While both implementations exist, + any fix to the DARE V2 stream format or the S2 compression framing/index + must be applied to `crates/rio` **and** `crates/rio-v2` (each has its own + `encrypt_reader.rs` and `compress_reader.rs`). Both implement the same + stream primitives; a single-sided fix forks on-disk behavior between + default and `rio-v2` builds and invalidates the dormant variant as an + interop baseline — and with full-suite CI now weekly-only, the divergence + could go unnoticed for up to a week. + ### Reverse direction Migrating back is also unsupported. Under `rio-v2` RustFS writes its own DEK envelope into MinIO's sealed-key metadata slots and labels it with MinIO's seal algorithm (`rustfs/src/storage/sse.rs:1830-1852`), so the metadata is MinIO-shaped while the key bytes are not MinIO-openable. Default builds do not populate those slots at all (`rustfs/src/storage/sse.rs:1796-1798`). Treat RustFS-written SSE objects as readable only by RustFS. diff --git a/docs/operations/kms-backend-security.md b/docs/operations/kms-backend-security.md index 2ab33f02a..4fe4aad07 100644 --- a/docs/operations/kms-backend-security.md +++ b/docs/operations/kms-backend-security.md @@ -112,6 +112,8 @@ RustFS does not rotate keys on a schedule. There is no built-in rotation worker, Set `RUSTFS_KMS_ROTATION_MAX_AGE_SECS` to that period in whole seconds. Unset — the default — leaves the verdict unreported rather than assuming a policy: how often keys must be rotated is a compliance decision, and a built-in default would report keys as overdue against a rule nobody wrote. An unparsable value is treated the same way, with a warning, instead of silently falling back to a number the operator did not choose. Values below one hour are raised to one hour, because a threshold of seconds reports every key as overdue moments after it was rotated and teaches operators to ignore the signal. +A second, independent threshold covers the cryptographic bound rather than the policy one. `RUSTFS_KMS_ROTATION_MAX_WRAPS` is the number of data keys one key's material may wrap before the verdict reports `rotation_due` with reason `wraps`. It follows the same discipline — unset or unparsable leaves the verdict unreported, and values below one million are raised to one million because wraps are accounted in reserved blocks of that size, so a smaller threshold would trip on the first reservation. Only backends where RustFS wraps locally and can rotate report a count (Vault KV2 today); Transit and AWS wrap externally and report none, so the wrap half stays silent there rather than guessing. When both thresholds are crossed the reported reason is `wraps`: the AES-GCM random-nonce ceiling is not negotiable, while the age period is a policy an operator chose. + `GET /rustfs/admin/v3/kms/keys` then carries two additional fields per key: - `rotation_due` — whether the key has outlived the configured period. diff --git a/docs/operations/kms-observability-runbook.md b/docs/operations/kms-observability-runbook.md index a9c75e88b..4a76fdbd4 100644 --- a/docs/operations/kms-observability-runbook.md +++ b/docs/operations/kms-observability-runbook.md @@ -204,7 +204,7 @@ Meaning: `rustfs_kms_oldest_key_rotation_age_seconds` — seconds since the leas Investigation: -1. Find which keys are due. The gauge deliberately names no key — a per-key label would carry key identifiers into the metric stream — so read the per-key verdict from the listing: `GET /rustfs/admin/v3/kms/keys` carries `rotation_due` and `rotation_due_reason` (`age`, `never_rotated`, or `unsupported`) per key, computed against `RUSTFS_KMS_ROTATION_MAX_AGE_SECS`. The verdict appears only on the listing, not on single-key describe. If `RUSTFS_KMS_ROTATION_MAX_AGE_SECS` is unset, set it to your policy's rotation period so the per-key verdict and this alert agree on what "overdue" means. +1. Find which keys are due. The gauge deliberately names no key — a per-key label would carry key identifiers into the metric stream — so read the per-key verdict from the listing: `GET /rustfs/admin/v3/kms/keys` carries `rotation_due` and `rotation_due_reason` (`age`, `never_rotated`, `wraps`, or `unsupported`) per key, computed against `RUSTFS_KMS_ROTATION_MAX_AGE_SECS` and `RUSTFS_KMS_ROTATION_MAX_WRAPS`. A `wraps` reason means the key's material has wrapped more data keys than the configured budget — the AES-GCM random-nonce ceiling rather than an age policy, so it is not satisfied by relaxing the age threshold. The verdict appears only on the listing, not on single-key describe. If `RUSTFS_KMS_ROTATION_MAX_AGE_SECS` is unset, set it to your policy's rotation period so the per-key verdict and this alert agree on what "overdue" means. 2. If the reason is `unsupported`, the backend cannot rotate at all (Local, Static). There is no key-level response; the decision is a backend migration, and the wrap ceiling above is the reason it cannot be deferred forever. See the [rotation drivers and scheduling matrix](kms-backend-security.md#rotation-drivers-and-scheduling-per-backend). 3. On a backend that can rotate, act per the driver matrix: on **Vault KV2**, check why your external rotation scheduler did not run (or set one up — RustFS deliberately ships none) and satisfy the [pre-rotation checklist](kms-backend-security.md#rotation-drivers-and-scheduling-per-backend) before rotating, above all the [upgrade-ordering hard constraint](kms-backend-security.md#upgrade-before-first-rotation-hard-constraint) — never respond to this alert by rotating in the middle of a rolling upgrade. On **Vault Transit**, check `auto_rotate_period` on the key in Vault. On **AWS KMS**, check the key's automatic rotation status in AWS — and do not schedule rotation through the RustFS endpoint, which maps to quota-limited `RotateKeyOnDemand`. 4. Know the gauge's blind spot on Transit and AWS before chasing a rotation that already happened: only KV2 persists a rotation timestamp, so Transit and AWS keys age from creation permanently and this alert will not clear after a rotation there. Confirm the real cadence at the owning system — the Transit key's version history in Vault, or the key's rotation status in AWS — and treat a confirmed-healthy cadence as a known overstatement of this gauge rather than an overdue key. diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index 72443d051..0290ebbcb 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -672,7 +672,6 @@ impl Operation for RemoveTier { } } -#[allow(dead_code)] pub struct VerifyTier {} #[async_trait::async_trait] impl Operation for VerifyTier { @@ -776,7 +775,6 @@ fn filter_tier_stats(daily_stats: DailyAllTierStats, tier_name: Option<&str>) -> .collect() } -#[allow(dead_code)] fn map_tier_verify_error(err: std::io::Error) -> S3Error { if let Some(admin_err) = err.get_ref().and_then(|inner| inner.downcast_ref::()) { return match admin_err.code.as_str() { diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index cc4d0b798..8dd561da1 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -46,6 +46,7 @@ use super::storage_api::multipart_usecase::options::{ get_content_sha256_with_query, get_opts, namespace_reserved_user_metadata, parse_copy_source_range, put_opts_with_replication_authorization, validate_archive_content_encoding, }; +use super::storage_api::multipart_usecase::request_context::spawn_traced_join; use super::storage_api::multipart_usecase::s3_api::multipart::{ ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output, parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number, @@ -588,56 +589,94 @@ impl DefaultMultipartUsecase { None => None, }; - let obj_info = store - .clone() - .complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, &opts) - .await - .map_err(ApiError::from)?; - let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await; - record_capacity_write(Some(capacity_scope_token)).await; - - if let Some(metadata_sys) = quota_metadata_sys.as_ref() { - if opts.replication_request { - let quota_checker = QuotaChecker::new(metadata_sys.clone()); - match quota_checker - .check_quota(&bucket, QuotaOperation::PutObject, obj_info.size.max(0) as u64) + let complete_commit = spawn_traced_join({ + let store = Arc::clone(&store); + let bucket = bucket.clone(); + let key = key.clone(); + let upload_id = upload_id.clone(); + let opts = opts.clone(); + let quota_metadata_sys = quota_metadata_sys.clone(); + async move { + let obj_info = store + .clone() + .complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, &opts) .await - { - Ok(check_result) if !check_result.allowed => { - let _ = store.delete_object(&bucket, &key, ObjectOptions::default()).await; - let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await; - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - format!( - "Bucket quota exceeded. Current usage: {} bytes, limit: {} bytes", - check_result.current_usage.unwrap_or(0), - check_result.quota_limit.unwrap_or(0) - ), - )); + .map_err(ApiError::from)?; + let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await; + record_capacity_write(Some(capacity_scope_token)).await; + + if let Some(metadata_sys) = quota_metadata_sys.as_ref() { + if opts.replication_request { + let quota_checker = QuotaChecker::new(metadata_sys.clone()); + match quota_checker + .check_quota(&bucket, QuotaOperation::PutObject, obj_info.size.max(0) as u64) + .await + { + Ok(check_result) if !check_result.allowed => { + let _ = store.delete_object(&bucket, &key, ObjectOptions::default()).await; + let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await; + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!( + "Bucket quota exceeded. Current usage: {} bytes, limit: {} bytes", + check_result.current_usage.unwrap_or(0), + check_result.quota_limit.unwrap_or(0) + ), + )); + } + Err(err) => { + warn!("Quota check failed for bucket {} after multipart completion: {}", bucket, err); + } + Ok(_) => {} + } } - Err(err) => { - warn!("Quota check failed for bucket {} after multipart completion: {}", bucket, err); + + let committed_size = if opts.replication_request { + obj_info.size.max(0) as u64 + } else { + quota_accounting_object_size(&obj_info, opts.quota_admission.is_some())? + }; + if versioned { + record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; + } else { + record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await; } - Ok(_) => {} } + + enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await; + + let mt2 = obj_info.user_defined.clone(); + let dsc = must_replicate_object( + &bucket, + &key, + &mt2, + "".to_string(), + opts.delete_marker_replication_status(), + opts.clone(), + ) + .await; + + if dsc.replicate_any() { + warn!("need multipart replication"); + schedule_object_replication(obj_info.clone(), store, dsc).await; + } + + rustfs_scanner::record_dirty_usage_bucket(&bucket); + Ok::<_, S3Error>(obj_info) } + }); + let obj_info = complete_commit.await.map_err(|err| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("complete multipart upload commit owner task failed: {err}"), + ) + })??; - let committed_size = if opts.replication_request { - obj_info.size.max(0) as u64 - } else { - quota_accounting_object_size(&obj_info, opts.quota_admission.is_some())? - }; - if versioned { - record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; - } else { - record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await; - } - } - - enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await; - - let raw_mpu_version = obj_info.version_id.map(|v| v.to_string()); - let mpu_version = if versioned { raw_mpu_version.clone() } else { None }; + let mpu_version = if versioned { + obj_info.version_id.map(|v| v.to_string()) + } else { + None + }; let mpu_version_for_event = mpu_version.clone(); // checksum: stored (decrypted) values take precedence over the request input; // additional algorithms (XXHash3/64/128, SHA-512, MD5), which have no typed @@ -660,28 +699,18 @@ impl DefaultMultipartUsecase { bucket: Some(bucket.clone()), key: Some(key.clone()), e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)), - location: Some(location.clone()), + location: Some(location), server_side_encryption: server_side_encryption.clone(), ssekms_key_id: ssekms_key_id.clone(), - checksum_crc32: checksum_crc32.clone(), - checksum_crc32c: checksum_crc32c.clone(), - checksum_sha1: checksum_sha1.clone(), - checksum_sha256: checksum_sha256.clone(), - checksum_crc64nvme: checksum_crc64nvme.clone(), - checksum_type: checksum_type.clone(), + checksum_crc32, + checksum_crc32c, + checksum_sha1, + checksum_sha256, + checksum_crc64nvme, + checksum_type, version_id: mpu_version, ..Default::default() }; - let mt2 = obj_info.user_defined.clone(); - let dsc = - must_replicate_object(&bucket, &key, &mt2, "".to_string(), opts.delete_marker_replication_status(), opts.clone()) - .await; - - if dsc.replicate_any() { - warn!("need multipart replication"); - schedule_object_replication(obj_info.clone(), store, dsc).await; - } - // Set object info for event notification helper = helper.object(obj_info); if let Some(version_id) = &mpu_version_for_event { @@ -712,7 +741,6 @@ impl DefaultMultipartUsecase { } let result = Ok(response); let _ = helper.complete(&result); - rustfs_scanner::record_dirty_usage_bucket(&bucket); result } diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 5c4e1e4c6..b48430d28 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -86,7 +86,7 @@ use super::storage_api::object_usecase::options::{ namespace_reserved_user_metadata, normalize_content_encoding_for_storage, preserve_unclassified_user_metadata, put_opts_with_replication_authorization, validate_archive_content_encoding, }; -use super::storage_api::object_usecase::request_context::{self, spawn_traced}; +use super::storage_api::object_usecase::request_context::{self, spawn_traced, spawn_traced_join}; use super::storage_api::object_usecase::s3_api::multipart::parse_list_parts_params; use super::storage_api::object_usecase::set_disk::{ get_lock_acquire_timeout, get_object_disk_read_timeout, is_valid_storage_class, @@ -2989,6 +2989,11 @@ struct PutObjectChecksums { crc64nvme: Option, } +struct PutObjectCommitResult { + obj_info: ObjectInfo, + put_versioned: bool, +} + fn normalize_delete_objects_version_id( version_id: Option, ) -> std::result::Result<(Option, Option), String> { @@ -5932,7 +5937,7 @@ impl DefaultObjectUsecase { reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?; rustfs_io_metrics::record_put_object_stage_duration_from("app_encryption_prepare", encryption_stage_start); - let mut reader = PutObjReader::new(reader); + let reader = PutObjReader::new(reader); let mt2 = metadata.clone(); opts.user_defined.extend(metadata); @@ -6005,97 +6010,145 @@ impl DefaultObjectUsecase { } else { None }; - let object_traffic_progress = object_traffic_health - .as_deref() - .and_then(ObjectTrafficHealth::track_write_storage); - let store_put_stage_start = put_stage_metrics_enabled.then(Instant::now); - let (obj_info, backfilled_old_current_size) = match store - .put_object_with_old_current_size(&bucket, &key, &mut reader, &opts) - .await - .map_err(ApiError::from) - { - Ok(obj_info) => { - store_put_watchdog.cancel(); - debug!( - target: "rustfs::app::object_usecase", - event = EVENT_PUT_OBJECT_STORE_RETURNED, - component = LOG_COMPONENT_APP, - subsystem = LOG_SUBSYSTEM_OBJECT, - request_id = %request_id, - bucket = %bucket, - key = %key, - put_path = put_path, - object_size = actual_size, - duration_ms = start_time.elapsed().as_millis() as u64, - result = "success", - "PutObject store write returned" - ); - obj_info + let put_commit = spawn_traced_join({ + let store = Arc::clone(&store); + let bucket = bucket.clone(); + let key = key.clone(); + let opts = opts.clone(); + let cache_adapter = cache_adapter.clone(); + let request_id = request_id.clone(); + let put_path = put_path.to_string(); + async move { + let object_traffic_progress = object_traffic_health + .as_deref() + .and_then(ObjectTrafficHealth::track_write_storage); + let mut reader = reader; + let store_put_stage_start = put_stage_metrics_enabled.then(Instant::now); + let (obj_info, backfilled_old_current_size) = match store + .put_object_with_old_current_size(&bucket, &key, &mut reader, &opts) + .await + .map_err(ApiError::from) + { + Ok(obj_info) => { + store_put_watchdog.cancel(); + debug!( + target: "rustfs::app::object_usecase", + event = EVENT_PUT_OBJECT_STORE_RETURNED, + component = LOG_COMPONENT_APP, + subsystem = LOG_SUBSYSTEM_OBJECT, + request_id = %request_id, + bucket = %bucket, + key = %key, + put_path = %put_path, + object_size = actual_size, + duration_ms = start_time.elapsed().as_millis() as u64, + result = "success", + "PutObject store write returned" + ); + obj_info + } + Err(err) => { + store_put_watchdog.cancel(); + rustfs_io_metrics::record_put_object_stage_duration_from("app_store_put", store_put_stage_start); + warn!( + target: "rustfs::app::object_usecase", + event = EVENT_PUT_OBJECT_STORE_RETURNED, + component = LOG_COMPONENT_APP, + subsystem = LOG_SUBSYSTEM_OBJECT, + request_id = %request_id, + bucket = %bucket, + key = %key, + put_path = %put_path, + object_size = actual_size, + duration_ms = start_time.elapsed().as_millis() as u64, + result = "error", + error = %err, + "PutObject store write returned" + ); + return Err(err.into()); + } + }; + rustfs_io_metrics::record_put_object_stage_duration_from("app_store_put", store_put_stage_start); + drop(object_traffic_progress); + #[cfg(test)] + wait_for_put_post_store_test_hook(&bucket).await; + + let post_store_stage_start = put_stage_metrics_enabled.then(Instant::now); + maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await; + let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await; + + let put_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; + // Fast in-memory update for immediate quota and admin usage consistency. + // The previous current size comes from the prelookup when it ran, + // otherwise from the rename_data backfill (rustfs/backlog#1009); the + // backfill reproduces the lookup's observation bit for bit (latest + // version's ObjectInfo.size — 0 for a delete-marker latest — or + // not-found → None). + match prelookup_previous_current_size.or_else(|| previous_current_size_from_backfill(backfilled_old_current_size)) + { + Some(previous_current_size) => { + if put_versioned { + record_bucket_object_version_write_memory( + &bucket, + previous_current_size, + obj_info.size.max(0) as u64, + ) + .await; + } else { + record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await; + } + } + None => { + // Neither source could determine the previous state (peers + // predating the backfill field during a rolling upgrade, or + // sub-quorum metadata divergence). Record the components that + // are correct regardless; the next authoritative scanner + // refresh replaces the in-memory numbers. + debug!( + target: "rustfs::app::object_usecase", + bucket = %bucket, + key = %key, + put_versioned, + "put_object old-size backfill unknown; recording degraded usage delta" + ); + record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, put_versioned) + .await; + } + } + + if dsc.replicate_any() { + schedule_object_replication(obj_info.clone(), store, dsc).await; + } + + rustfs_scanner::record_dirty_usage_bucket(&bucket); + rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_bookkeeping", post_store_stage_start); + + let capacity_update_stage_start = put_stage_metrics_enabled.then(Instant::now); + let manager = get_capacity_manager(); + manager.record_write_operation().await; + rustfs_io_metrics::record_put_object_stage_duration_from("app_capacity_update", capacity_update_stage_start); + + Ok::<_, S3Error>(PutObjectCommitResult { obj_info, put_versioned }) + } + }); + let PutObjectCommitResult { obj_info, put_versioned } = match put_commit.await { + Ok(Ok(result)) => result, + Ok(Err(err)) => { + let result: S3Result> = Err(err); + put_request_guard.finish_err(); + let _ = helper.complete(&result); + return result; } Err(err) => { - store_put_watchdog.cancel(); - rustfs_io_metrics::record_put_object_stage_duration_from("app_store_put", store_put_stage_start); - warn!( - target: "rustfs::app::object_usecase", - event = EVENT_PUT_OBJECT_STORE_RETURNED, - component = LOG_COMPONENT_APP, - subsystem = LOG_SUBSYSTEM_OBJECT, - request_id = %request_id, - bucket = %bucket, - key = %key, - put_path = put_path, - object_size = actual_size, - duration_ms = start_time.elapsed().as_millis() as u64, - result = "error", - error = %err, - "PutObject store write returned" - ); - let result: S3Result> = Err(err.into()); + let result: S3Result> = Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("put object commit owner task failed: {err}"), + )); put_request_guard.finish_err(); let _ = helper.complete(&result); return result; } }; - rustfs_io_metrics::record_put_object_stage_duration_from("app_store_put", store_put_stage_start); - drop(object_traffic_progress); - #[cfg(test)] - wait_for_put_post_store_test_hook(&bucket).await; - - let post_store_stage_start = put_stage_metrics_enabled.then(Instant::now); - maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await; - let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await; - - let put_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; - // Fast in-memory update for immediate quota and admin usage consistency. - // The previous current size comes from the prelookup when it ran, - // otherwise from the rename_data backfill (rustfs/backlog#1009); the - // backfill reproduces the lookup's observation bit for bit (latest - // version's ObjectInfo.size — 0 for a delete-marker latest — or - // not-found → None). - match prelookup_previous_current_size.or_else(|| previous_current_size_from_backfill(backfilled_old_current_size)) { - Some(previous_current_size) => { - if put_versioned { - record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await; - } else { - record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await; - } - } - None => { - // Neither source could determine the previous state (peers - // predating the backfill field during a rolling upgrade, or - // sub-quorum metadata divergence). Record the components that - // are correct regardless; the next authoritative scanner - // refresh replaces the in-memory numbers. - debug!( - target: "rustfs::app::object_usecase", - bucket = %bucket, - key = %key, - put_versioned, - "put_object old-size backfill unknown; recording degraded usage delta" - ); - record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, put_versioned).await; - } - } let raw_version = obj_info.version_id.map(|v| v.to_string()); @@ -6110,17 +6163,6 @@ impl DefaultObjectUsecase { let expiration = resolve_put_object_expiration(&bucket, &obj_info).await; - // Reuse the single replication decision computed before commit (see `dsc` - // above) so the pending metadata persisted with the object and the - // post-commit schedule always derive from the same immutable decision. - // Recomputing here would repeat the versioning/config/target traversal and, - // worse, allow a replication-config hot update between the two phases to - // produce a pending-without-schedule or schedule-without-pending divergence - // (https://github.com/rustfs/backlog/issues/1320). - if dsc.replicate_any() { - schedule_object_replication(obj_info.clone(), store, dsc).await; - } - let mut checksums = PutObjectChecksums { crc32: input.checksum_crc32, crc32c: input.checksum_crc32c, @@ -6159,14 +6201,6 @@ impl DefaultObjectUsecase { inject_additional_checksum_headers(&mut response.headers, &put_extra_checksum_headers); let result = Ok(response); let _ = helper.complete(&result); - rustfs_scanner::record_dirty_usage_bucket(&bucket); - rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_bookkeeping", post_store_stage_start); - - // Record write operation for capacity management (inline to avoid per-request tokio::spawn overhead) - let capacity_update_stage_start = put_stage_metrics_enabled.then(Instant::now); - let manager = get_capacity_manager(); - manager.record_write_operation().await; - rustfs_io_metrics::record_put_object_stage_duration_from("app_capacity_update", capacity_update_stage_start); // Record PutObject metrics via zero-copy-metrics { @@ -7506,31 +7540,50 @@ impl DefaultObjectUsecase { let cache_adapter = self.object_data_cache(); let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await; - let oi = store - .copy_object(&src_bucket, &src_key, &bucket, &key, &mut src_info, &src_opts, &dst_opts) - .await - .map_err(ApiError::from)?; - drop(_self_copy_lock_guard); + let copy_commit = spawn_traced_join({ + let store = Arc::clone(&store); + let src_bucket = src_bucket.clone(); + let src_key = src_key.clone(); + let bucket = bucket.clone(); + let key = key.clone(); + let src_opts = src_opts.clone(); + let dst_opts = dst_opts.clone(); + async move { + let _source_bucket_lifecycle_guard = source_bucket_lifecycle_guard; + let _destination_bucket_lifecycle_guard_storage = destination_bucket_lifecycle_guard_storage; + let _self_copy_lock_guard = _self_copy_lock_guard; - // Reuse the single pre-commit replication decision (see `dsc` above) so - // the persisted pending marker and the schedule always agree, mirroring - // the PUT path. - if dsc.replicate_any() { - schedule_object_replication(oi.clone(), store.clone(), dsc).await; - } + let oi = store + .copy_object(&src_bucket, &src_key, &bucket, &key, &mut src_info, &src_opts, &dst_opts) + .await + .map_err(ApiError::from)?; - maybe_enqueue_transition_immediate(&oi, LcEventSrc::S3CopyObject).await; - let _ = invalidate_object_data_cache_after_copy_success(&cache_adapter, &bucket, &key).await; + // Reuse the single pre-commit replication decision (see `dsc` above) so + // the persisted pending marker and the schedule always agree, mirroring + // the PUT path. + if dsc.replicate_any() { + schedule_object_replication(oi.clone(), Arc::clone(&store), dsc).await; + } - let dest_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; - // Update quota tracking after successful copy - if has_bucket_metadata { - if dest_versioned { - record_bucket_object_version_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await; - } else { - record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await; + maybe_enqueue_transition_immediate(&oi, LcEventSrc::S3CopyObject).await; + let _ = invalidate_object_data_cache_after_copy_success(&cache_adapter, &bucket, &key).await; + + let dest_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; + if has_bucket_metadata { + if dest_versioned { + record_bucket_object_version_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await; + } else { + record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await; + } + } + + rustfs_scanner::record_dirty_usage_bucket(&bucket); + Ok::<_, S3Error>((oi, dest_versioned)) } - } + }); + let (oi, dest_versioned) = copy_commit.await.map_err(|err| { + S3Error::with_message(S3ErrorCode::InternalError, format!("copy object commit owner task failed: {err}")) + })??; let raw_dest_version = oi.version_id.map(|v| v.to_string()); let dest_version = if dest_versioned { raw_dest_version } else { None }; @@ -7578,7 +7631,7 @@ impl DefaultObjectUsecase { } } let copy_object_result = CopyObjectResult { - e_tag: oi.etag.map(|etag| to_s3s_etag(&etag)), + e_tag: oi.etag.as_ref().map(|etag| to_s3s_etag(etag)), last_modified: oi.mod_time.map(Timestamp::from), checksum_crc32: response_checksums.crc32, checksum_crc32c: response_checksums.crc32c, @@ -7609,7 +7662,6 @@ impl DefaultObjectUsecase { let result = Ok(S3Response::new(output)); let _ = helper.complete(&result); - rustfs_scanner::record_dirty_usage_bucket(&bucket); result } @@ -11489,6 +11541,76 @@ mod tests { assert!(!recovered.write_stalled); } + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn cancelled_put_request_completes_post_commit_publication() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let (store, context) = real_cold_fill_test_context().await; + let bucket = format!("put-owner-tail-{}", Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("PUT owner-tail bucket must be created"); + + let old_body = Bytes::from_static(b"old body that must be invalidated"); + let old_info = put_real_cold_fill_object(&store, &bucket, object, &old_body).await; + let adapter = context.object_data_cache(); + let old_plan = real_cold_fill_plan(&adapter, &bucket, object, &old_info); + + let post_store_entered = Arc::new(tokio::sync::Barrier::new(2)); + let post_store_resume = Arc::new(tokio::sync::Barrier::new(2)); + install_put_post_store_test_hook(bucket.clone(), Arc::clone(&post_store_entered), Arc::clone(&post_store_resume)); + + let payload = Bytes::from_static(b"published despite caller cancellation"); + let put_input = PutObjectInput::builder() + .bucket(bucket.clone()) + .key(object.to_string()) + .body(Some(StreamingBlob::from(s3s::Body::from(payload.clone())))) + .content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64"))) + .build() + .expect("PUT input must build"); + let put_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context))); + let put = tokio::spawn(async move { + put_usecase + .execute_put_object(&FS::new(), build_request(put_input, Method::PUT)) + .await + }); + + tokio::time::timeout(Duration::from_secs(10), post_store_entered.wait()) + .await + .expect("PUT must reach the post-store owner-tail hook"); + assert_eq!( + adapter.fill_body(&old_plan, old_body.clone()).await, + rustfs_object_data_cache::ObjectDataCacheFillResult::Inserted, + "test must republish the old body while the owner tail is paused" + ); + put.abort(); + post_store_resume.wait().await; + let _ = put.await.expect_err("outer request task must be cancelled"); + + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if matches!( + adapter.lookup_body(&old_plan).await, + rustfs_object_data_cache::ObjectDataCacheLookup::Miss + ) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("post-commit owner tail must invalidate stale body cache after caller cancellation"); + + let recovered = store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("cancelled request's owned commit must still publish the object"); + assert_eq!(recovered.size, i64::try_from(payload.len()).expect("test payload length must fit i64")); + } + #[tokio::test] async fn object_progress_tracks_zero_byte_and_zero_copy_put_lock_waits() { use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index cfb5c5493..39d7d7685 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -998,7 +998,7 @@ pub(crate) mod options { } pub(crate) mod request_context { - pub(crate) use crate::storage::storage_api::request_context_consumer::{RequestContext, spawn_traced}; + pub(crate) use crate::storage::storage_api::request_context_consumer::{RequestContext, spawn_traced, spawn_traced_join}; } pub(crate) mod sse { @@ -1150,7 +1150,9 @@ pub(crate) mod multipart_usecase { } } - pub(crate) use super::{access, bucket, data_usage, error, helper, io, object_utils, options, s3_api, set_disk, sse}; + pub(crate) use super::{ + access, bucket, data_usage, error, helper, io, object_utils, options, request_context, s3_api, set_disk, sse, + }; pub(crate) use crate::storage::storage_api::{ECStore, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader}; } diff --git a/rustfs/src/storage/request_context.rs b/rustfs/src/storage/request_context.rs index d8bf66074..dddf5de92 100644 --- a/rustfs/src/storage/request_context.rs +++ b/rustfs/src/storage/request_context.rs @@ -257,6 +257,15 @@ where tokio::spawn(tracing::Instrument::instrument(fut, tracing::Span::current())); } +/// Spawn a request-internal task and return its join handle to the caller. +pub fn spawn_traced_join(fut: F) -> tokio::task::JoinHandle +where + F: std::future::Future + Send + 'static, + F::Output: Send + 'static, +{ + tokio::spawn(tracing::Instrument::instrument(fut, tracing::Span::current())) +} + #[cfg(test)] #[allow(unused_imports)] mod tests { diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index 66dd2095a..920d6b6ca 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -119,8 +119,14 @@ use rustfs_utils::http::object_encryption_keys::{ MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, - MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, SSEC_ORIGINAL_SIZE_HEADER, + MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, SSEC_ORIGINAL_SIZE_HEADER, normalize_managed_metadata, + stored_managed_encryption_key, }; +// The managed-SSE classifier lives in the shared encryption-keys module so the +// scanner can reuse it (backlog#1643 PR-B0); these re-exports keep the +// historical `crate::storage::sse` paths compiling. +pub use rustfs_utils::http::object_encryption_keys::SSEType; +pub(crate) use rustfs_utils::http::object_encryption_keys::contains_managed_encryption_metadata; #[cfg(feature = "rio-v2")] const MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM: &str = "DAREv2-HMAC-SHA256"; #[cfg(feature = "rio-v2")] @@ -783,28 +789,6 @@ pub struct DecryptionMaterial { pub key_kind: EncryptionKeyKind, } -/// Type of encryption used -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SSEType { - /// SSE-S3 (AES256) - SseS3, - /// SSE-KMS (aws:kms) - SseKms, - /// SSE-C (customer-provided key) - SseC, -} - -impl SSEType { - /// Stable scheme name for audit consumers. - fn audit_label(self) -> &'static str { - match self { - SSEType::SseS3 => "SSE-S3", - SSEType::SseKms => "SSE-KMS", - SSEType::SseC => "SSE-C", - } - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum EncryptionKeyKind { Direct, @@ -1064,28 +1048,6 @@ pub async fn authorize_sse_kms_object_read( result } -/// Resolve the scheme and KMS key a stored managed-SSE object was wrapped with. -/// -/// Mirrors the lookup `apply_managed_decryption_material` performs, so both agree on -/// which key a read is authorized against. -fn stored_managed_encryption_key(metadata: &HashMap) -> Option<(SSEType, String)> { - if !contains_managed_encryption_metadata(metadata) { - return None; - } - - let sse_type = match metadata.get("x-amz-server-side-encryption")?.as_str() { - ServerSideEncryption::AWS_KMS => SSEType::SseKms, - _ => SSEType::SseS3, - }; - let key_id = normalize_managed_metadata(metadata) - .get(INTERNAL_ENCRYPTION_KEY_ID_HEADER) - .or_else(|| metadata.get("x-amz-server-side-encryption-aws-kms-key-id")) - .cloned() - .unwrap_or_else(|| "default".to_string()); - - Some((sse_type, key_id)) -} - // ============================================================================ // Data-plane KMS audit attachment (SSE-S3 / SSE-KMS) // ============================================================================ @@ -1339,7 +1301,9 @@ fn envelope_master_key_version(envelope_bytes: &[u8]) -> Option { /// Master-key version of the envelope stored on an object, for the audit /// summary of a read against that object. fn stored_envelope_master_key_version(metadata: &HashMap) -> Option { - let encoded = normalize_managed_metadata(metadata); + // No context recoder: the recode only ever inserts the context key, which + // this lookup never reads, so the normalized result is identical without it. + let encoded = normalize_managed_metadata(metadata, None); let encoded = encoded.get(INTERNAL_ENCRYPTION_KEY_HEADER)?; let envelope = BASE64_STANDARD.decode(encoded).ok()?; envelope_master_key_version(&envelope) @@ -2487,7 +2451,7 @@ async fn apply_managed_decryption_material_inner( // Safe: presence is guaranteed by the contains_key check above. let server_side_encryption = metadata.get("x-amz-server-side-encryption").cloned().unwrap_or_default(); - let normalized_metadata = normalize_managed_metadata(metadata); + let normalized_metadata = normalize_managed_metadata(metadata, Some(recode_minio_kms_context)); let encryption_type = match server_side_encryption.as_str() { ServerSideEncryption::AES256 => SSEType::SseS3, @@ -3229,14 +3193,6 @@ pub fn mark_encrypted_multipart_metadata(metadata: &mut HashMap) metadata.insert(MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER.to_string(), String::new()); } -pub(crate) fn contains_managed_encryption_metadata(metadata: &HashMap) -> bool { - metadata.contains_key(INTERNAL_ENCRYPTION_KEY_HEADER) - || metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER) - || metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER) - || metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER) - || metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER) -} - #[cfg(feature = "rio-v2")] fn is_legacy_rustfs_managed_metadata(metadata: &HashMap) -> bool { metadata.contains_key(INTERNAL_ENCRYPTION_KEY_HEADER) @@ -3282,47 +3238,16 @@ fn parse_minio_managed_sealed_key( Ok(Some(ManagedSealedKey { iv, sealed_key })) } -fn normalize_managed_metadata(metadata: &HashMap) -> HashMap { - let mut normalized = metadata.clone(); - - if !normalized.contains_key(INTERNAL_ENCRYPTION_KEY_HEADER) - && let Some(value) = metadata - .get(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER) - .or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)) - .or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)) - .or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)) - { - normalized.insert(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), value.clone()); - } - - if !normalized.contains_key(INTERNAL_ENCRYPTION_IV_HEADER) - && let Some(value) = metadata.get(MINIO_INTERNAL_ENCRYPTION_IV_HEADER) - { - normalized.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), value.clone()); - } - - if !normalized.contains_key(INTERNAL_ENCRYPTION_ALGORITHM_HEADER) - && let Some(value) = metadata.get(MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER) - { - normalized.insert(INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), value.clone()); - } - - if !normalized.contains_key(INTERNAL_ENCRYPTION_KEY_ID_HEADER) - && let Some(value) = metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER) - { - normalized.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), value.clone()); - } - - if !normalized.contains_key(INTERNAL_ENCRYPTION_CONTEXT_HEADER) - && let Some(value) = metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER) - && let Ok(decoded) = BASE64_STANDARD.decode(value) - && let Ok(context) = serde_json::from_slice::>(&decoded) - && let Ok(encoded) = serde_json::to_string(&context) - { - normalized.insert(INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(), encoded); - } - - normalized +/// Recodes a stored MinIO KMS context value (base64-wrapped JSON) into the +/// plain-JSON form RustFS stores under [`INTERNAL_ENCRYPTION_CONTEXT_HEADER`]. +/// +/// Injected into the shared [`normalize_managed_metadata`] because the shared +/// crate carries no JSON codec; any decode failure returns `None`, which skips +/// the context mapping exactly like the historical inline `if let Ok` chain. +fn recode_minio_kms_context(value: &str) -> Option { + let decoded = BASE64_STANDARD.decode(value).ok()?; + let context = serde_json::from_slice::>(&decoded).ok()?; + serde_json::to_string(&context).ok() } // ============================================================================ @@ -3473,9 +3398,9 @@ mod tests { encryption_material_to_metadata, extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, generate_ssec_nonce, is_managed_sse, kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata, md5_base64, normalize_managed_metadata, - reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, sse_encryption, sse_prepare_encryption, - strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, - validate_ssec_params, verify_ssec_key_match, + recode_minio_kms_context, reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, sse_encryption, + sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, + validate_ssec_for_read, validate_ssec_params, verify_ssec_key_match, }; #[cfg(feature = "rio-v2")] use super::{ @@ -3484,6 +3409,38 @@ mod tests { }; use rustfs_utils::http::headers::SSEC_ALGORITHM_HEADER; + /// backlog#1643 PR-B0 acceptance guard: the managed-SSE classifier must + /// have exactly one definition — in the shared encryption-keys module — + /// so the scanner and the S3 layer can never disagree on attribution. + /// This module may only re-export or call it. + #[test] + fn managed_sse_classifier_has_exactly_one_definition() { + let classifier_fns = [ + "contains_managed_encryption_metadata", + "normalize_managed_metadata", + "stored_managed_encryption_key", + ]; + + let sse_src = include_str!("sse.rs"); + let shared_src = + std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/../crates/utils/src/http/object_encryption_keys.rs")) + .expect("shared encryption-keys module should be readable"); + + for name in classifier_fns { + // Built at runtime so this test's own source cannot satisfy the scan. + let definition = format!("fn {name}("); + assert!( + !sse_src.contains(&definition), + "{name} must not be redefined in storage/sse.rs; call the shared rustfs_utils::http::object_encryption_keys implementation instead" + ); + assert_eq!( + shared_src.matches(&definition).count(), + 1, + "{name} must be defined exactly once, in the shared encryption-keys module" + ); + } + } + #[test] fn ssec_read_headers_are_sensitive() { let headers = super::build_ssec_read_headers( @@ -4869,7 +4826,7 @@ mod tests { (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()), ]); - let normalized = normalize_managed_metadata(&metadata); + let normalized = normalize_managed_metadata(&metadata, Some(recode_minio_kms_context)); assert_eq!( normalized.get(INTERNAL_ENCRYPTION_KEY_HEADER), diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index d5dfc3af5..b12169c71 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -203,7 +203,9 @@ pub(crate) mod options_consumer { } pub(crate) mod request_context_consumer { - pub(crate) use super::super::request_context::{RequestContext, extract_request_id_from_headers, spawn_traced}; + pub(crate) use super::super::request_context::{ + RequestContext, extract_request_id_from_headers, spawn_traced, spawn_traced_join, + }; } pub(crate) mod rpc_consumer {