diff --git a/.config/make/tests.mak b/.config/make/tests.mak index d0424f80c..c83dcd653 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -26,6 +26,9 @@ script-tests: ## Run shell script tests @echo "Running script tests..." ./scripts/test_build_rustfs_options.sh ./scripts/test_entrypoint_credentials.sh + bash -n ./scripts/validate_object_data_cache_cold_stampede.sh + python3 ./scripts/check_object_data_cache_follower_samples.py --self-test + ./scripts/validate_object_data_cache_cold_stampede.sh --self-test .PHONY: test test: core-deps script-tests ## Run all tests (needs cargo-nextest; RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1 to override) diff --git a/Cargo.lock b/Cargo.lock index 039cd8273..e2253570a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8840,6 +8840,7 @@ dependencies = [ "matchit 0.9.2", "md5", "metrics", + "metrics-util", "mimalloc", "mime_guess", "opentelemetry", diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index c72758f5a..8a5f6446e 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -353,9 +353,12 @@ pub mod notification { pub mod object { pub use crate::object_api::{ - BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectReader, ObjectInfo, ObjectMutationHook, ObjectOptions, - PutObjReader, RangedDecompressReader, StreamConsumer, register_get_object_body_cache_hook, register_object_mutation_hook, + BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource, + GetObjectReader, ObjectInfo, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, StreamConsumer, + get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, + register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook, }; + pub use crate::store::PreparedGetObjectReader; } pub mod rebalance { diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index 425f05ec5..8920df6bc 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -35,7 +35,7 @@ use crate::{ object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}, runtime::instance::{InstanceContext, bootstrap_ctx}, runtime::sources as runtime_sources, - set_disk::SetDisks, + set_disk::{PreparedGetObjectMetadata, SetDisks}, store::init_format::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file}, }; use futures::{ @@ -403,6 +403,31 @@ impl crate::storage_api_contracts::object::ObjectIO for Sets { } impl Sets { + pub(crate) async fn prepare_get_object_reader_metadata( + &self, + bucket: &str, + object: &str, + opts: &ObjectOptions, + ) -> Result { + self.get_disks_by_key(object) + .prepare_get_object_metadata(bucket, object, opts) + .await + } + + pub(crate) async fn get_object_reader_with_prepared_metadata( + &self, + bucket: &str, + object: &str, + range: Option, + headers: HeaderMap, + opts: &ObjectOptions, + metadata: PreparedGetObjectMetadata, + ) -> Result { + self.get_disks_by_key(object) + .get_object_reader_with_prepared_metadata(bucket, object, range, headers, opts, metadata) + .await + } + /// `put_object` plus the rename_data old-size backfill /// (rustfs/backlog#1009); see `SetDisks::put_object_with_old_current_size`. pub async fn put_object_with_old_current_size( diff --git a/crates/ecstore/src/diagnostics/get.rs b/crates/ecstore/src/diagnostics/get.rs index 6c57016ee..b72b16cfb 100644 --- a/crates/ecstore/src/diagnostics/get.rs +++ b/crates/ecstore/src/diagnostics/get.rs @@ -137,6 +137,7 @@ pub(crate) const GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED: &str = "not_fou pub(crate) const GET_METADATA_CACHE_REASON_NOT_READ_DATA: &str = "not_read_data"; pub(crate) const GET_METADATA_CACHE_REASON_PART_NUMBER: &str = "part_number"; pub(crate) const GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ: &str = "raw_data_movement_read"; +pub(crate) const GET_METADATA_CACHE_REASON_STALE_PUBLICATION: &str = "stale_publication"; pub(crate) const GET_METADATA_CACHE_REASON_USABLE: &str = "usable"; pub(crate) const GET_METADATA_CACHE_REASON_VERSION_ID: &str = "version_id"; pub(crate) const GET_METADATA_CACHE_REASON_VERSION_SUSPENDED: &str = "version_suspended"; @@ -386,6 +387,7 @@ mod tests { assert_eq!(GET_METADATA_CACHE_REASON_NOT_READ_DATA, "not_read_data"); assert_eq!(GET_METADATA_CACHE_REASON_PART_NUMBER, "part_number"); assert_eq!(GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, "raw_data_movement_read"); + assert_eq!(GET_METADATA_CACHE_REASON_STALE_PUBLICATION, "stale_publication"); assert_eq!(GET_METADATA_CACHE_REASON_USABLE, "usable"); assert_eq!(GET_METADATA_CACHE_REASON_VERSION_ID, "version_id"); assert_eq!(GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, "version_suspended"); diff --git a/crates/ecstore/src/object_api/body_cache_hook.rs b/crates/ecstore/src/object_api/body_cache_hook.rs index a56ec614a..27485762b 100644 --- a/crates/ecstore/src/object_api/body_cache_hook.rs +++ b/crates/ecstore/src/object_api/body_cache_hook.rs @@ -20,11 +20,17 @@ //! probing earlier would require a second metadata fan-out, and probing later //! (after the reader is built) means a hit no longer saves any disk I/O. -use crate::object_api::ObjectInfo; use crate::object_api::hook_slot::HookSlot; +use crate::object_api::{ObjectInfo, ObjectOptions}; +use crate::storage_api_contracts::range::HTTPRangeSpec; use bytes::Bytes; +use std::future::Future; use std::sync::Arc; +tokio::task_local! { + static SKIP_GET_OBJECT_BODY_CACHE_HOOK: bool; +} + /// Serves full-object GET bodies from a cache keyed by object identity. /// /// Implementations must validate identity (etag/version/size) against the @@ -62,14 +68,263 @@ pub fn register_get_object_body_cache_hook(hook: Arc ); } +/// Unregister the process-wide GET body cache hook. +/// +/// Config reloads use this when body caching becomes disabled so an adapter +/// retained by the previous configuration cannot continue serving stale hits. +pub fn unregister_get_object_body_cache_hook() { + GET_OBJECT_BODY_CACHE_HOOK.clear(); +} + +/// Probes the registered hook against an already resolved metadata snapshot. +/// Staged GET callers use this once before suppressing the nested reader probe, +/// preserving the legacy hook contract. +#[non_exhaustive] +pub enum GetObjectBodyCacheHookLookup { + Ineligible, + Absent, + Miss, + Hit(Bytes), +} + +/// Returns the complete plaintext length only when the request can safely use +/// the body-cache hook. Callers may use this before conditional decisions so +/// ineligible reads retain the established reader-path error precedence. +pub fn get_object_body_cache_plaintext_len( + range: &Option, + opts: &ObjectOptions, + info: &ObjectInfo, +) -> Option { + crate::set_disk::body_cache_plaintext_len(range, opts, info) +} + +pub async fn lookup_get_object_body_cache_hook( + bucket: &str, + object: &str, + range: &Option, + opts: &ObjectOptions, + info: &ObjectInfo, +) -> GetObjectBodyCacheHookLookup { + let Some(plaintext_len) = get_object_body_cache_plaintext_len(range, opts, info) else { + return GetObjectBodyCacheHookLookup::Ineligible; + }; + let Some(hook) = get_object_body_cache_hook() else { + return GetObjectBodyCacheHookLookup::Absent; + }; + match hook.lookup(bucket, object, info).await { + Some(body) if i64::try_from(body.len()).is_ok_and(|body_len| body_len == plaintext_len) => { + GetObjectBodyCacheHookLookup::Hit(body) + } + Some(_) | None => GetObjectBodyCacheHookLookup::Miss, + } +} + /// The registered hook, if any. pub(crate) fn get_object_body_cache_hook() -> Option> { GET_OBJECT_BODY_CACHE_HOOK.get() } +pub(crate) fn get_object_body_cache_hook_suppressed() -> bool { + SKIP_GET_OBJECT_BODY_CACHE_HOOK.try_with(|skip| *skip).unwrap_or(false) +} + +pub(crate) async fn without_get_object_body_cache_hook(future: F) -> F::Output +where + F: Future, +{ + SKIP_GET_OBJECT_BODY_CACHE_HOOK.scope(true, future).await +} + /// Test-only: unregister the hook so tests can register and clear the slot /// deterministically without leaking a hook into unrelated tests. #[cfg(test)] pub(crate) fn clear_get_object_body_cache_hook() { - GET_OBJECT_BODY_CACHE_HOOK.clear(); + unregister_get_object_body_cache_hook(); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct LegacyHook { + calls: AtomicUsize, + body: Option, + } + + #[async_trait::async_trait] + impl GetObjectBodyCacheHook for LegacyHook { + async fn lookup(&self, _bucket: &str, _object: &str, _info: &ObjectInfo) -> Option { + self.calls.fetch_add(1, Ordering::Relaxed); + self.body.clone() + } + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn staged_probe_preserves_legacy_hook_hit_once() { + clear_get_object_body_cache_hook(); + let hook = Arc::new(LegacyHook { + calls: AtomicUsize::new(0), + body: Some(Bytes::from_static(b"legacy")), + }); + register_get_object_body_cache_hook(Arc::clone(&hook) as Arc); + + let info = ObjectInfo { + size: 6, + actual_size: 6, + ..Default::default() + }; + let GetObjectBodyCacheHookLookup::Hit(body) = + lookup_get_object_body_cache_hook("bucket", "object", &None, &ObjectOptions::default(), &info).await + else { + panic!("legacy hook hit must be returned"); + }; + assert_eq!(body, Bytes::from_static(b"legacy")); + assert_eq!(hook.calls.load(Ordering::Relaxed), 1); + clear_get_object_body_cache_hook(); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn staged_probe_treats_wrong_length_legacy_body_as_authoritative_miss() { + clear_get_object_body_cache_hook(); + let hook = Arc::new(LegacyHook { + calls: AtomicUsize::new(0), + body: Some(Bytes::from_static(b"short")), + }); + register_get_object_body_cache_hook(Arc::clone(&hook) as Arc); + let info = ObjectInfo { + size: 6, + actual_size: 6, + ..Default::default() + }; + + assert!(matches!( + lookup_get_object_body_cache_hook("bucket", "object", &None, &ObjectOptions::default(), &info).await, + GetObjectBodyCacheHookLookup::Miss + )); + assert_eq!(hook.calls.load(Ordering::Relaxed), 1); + clear_get_object_body_cache_hook(); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn staged_probe_preserves_legacy_hook_miss_once() { + clear_get_object_body_cache_hook(); + let hook = Arc::new(LegacyHook { + calls: AtomicUsize::new(0), + body: None, + }); + register_get_object_body_cache_hook(Arc::clone(&hook) as Arc); + + let info = ObjectInfo { + size: 4, + actual_size: 4, + ..Default::default() + }; + assert!(matches!( + lookup_get_object_body_cache_hook("bucket", "object", &None, &ObjectOptions::default(), &info).await, + GetObjectBodyCacheHookLookup::Miss + )); + assert_eq!(hook.calls.load(Ordering::Relaxed), 1); + clear_get_object_body_cache_hook(); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn staged_probe_bypasses_raw_movement_and_restore_reads() { + clear_get_object_body_cache_hook(); + let hook = Arc::new(LegacyHook { + calls: AtomicUsize::new(0), + body: Some(Bytes::from_static(b"body")), + }); + register_get_object_body_cache_hook(Arc::clone(&hook) as Arc); + let info = ObjectInfo { + size: 4, + actual_size: 4, + ..Default::default() + }; + let mut restore = ObjectOptions::default(); + restore.transition.restore_request.days = Some(1); + let cases = [ + ObjectOptions { + raw_data_movement_read: true, + ..Default::default() + }, + ObjectOptions { + data_movement: true, + ..Default::default() + }, + restore, + ]; + + for opts in &cases { + assert!(matches!( + lookup_get_object_body_cache_hook("bucket", "object", &None, opts, &info).await, + GetObjectBodyCacheHookLookup::Ineligible + )); + } + assert_eq!(hook.calls.load(Ordering::Relaxed), 0); + clear_get_object_body_cache_hook(); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn staged_probe_bypasses_pre_hook_early_return_objects() { + clear_get_object_body_cache_hook(); + let hook = Arc::new(LegacyHook { + calls: AtomicUsize::new(0), + body: Some(Bytes::from_static(b"body")), + }); + register_get_object_body_cache_hook(Arc::clone(&hook) as Arc); + let delete_marker = ObjectInfo { + delete_marker: true, + size: 4, + actual_size: 4, + ..Default::default() + }; + let zero = ObjectInfo::default(); + let inline = ObjectInfo { + inlined: true, + size: 4, + actual_size: 4, + parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + number: 1, + ..Default::default() + }]), + ..Default::default() + }; + let version_only = ObjectInfo { + version_only: true, + size: 4, + actual_size: 4, + ..Default::default() + }; + let metadata_only = ObjectInfo { + metadata_only: true, + size: 4, + actual_size: 4, + ..Default::default() + }; + + for info in [&delete_marker, &zero, &inline, &version_only, &metadata_only] { + assert!(matches!( + lookup_get_object_body_cache_hook("bucket", "object", &None, &ObjectOptions::default(), info).await, + GetObjectBodyCacheHookLookup::Ineligible + )); + } + assert_eq!(hook.calls.load(Ordering::Relaxed), 0); + clear_get_object_body_cache_hook(); + } + + #[tokio::test] + async fn staged_reader_scope_suppresses_only_the_nested_probe() { + assert!(!get_object_body_cache_hook_suppressed()); + without_get_object_body_cache_hook(async { + assert!(get_object_body_cache_hook_suppressed()); + }) + .await; + assert!(!get_object_body_cache_hook_suppressed()); + } } diff --git a/crates/ecstore/src/object_api/hook_slot.rs b/crates/ecstore/src/object_api/hook_slot.rs index 9ac2e99e8..6a99ada57 100644 --- a/crates/ecstore/src/object_api/hook_slot.rs +++ b/crates/ecstore/src/object_api/hook_slot.rs @@ -61,8 +61,7 @@ impl HookSlot { self.inner.read().unwrap_or_else(|poisoned| poisoned.into_inner()).clone() } - /// Clears the slot. Test-only: production never unregisters a hook. - #[cfg(test)] + /// Clears the slot during feature disable or test cleanup. pub(crate) fn clear(&self) { *self.inner.write().unwrap_or_else(|poisoned| poisoned.into_inner()) = None; } diff --git a/crates/ecstore/src/object_api/mod.rs b/crates/ecstore/src/object_api/mod.rs index f6efa90a6..4c5807630 100644 --- a/crates/ecstore/src/object_api/mod.rs +++ b/crates/ecstore/src/object_api/mod.rs @@ -60,9 +60,14 @@ mod types; #[cfg(test)] pub(crate) use body_cache_hook::clear_get_object_body_cache_hook; -pub(crate) use body_cache_hook::get_object_body_cache_hook; -pub use body_cache_hook::{GetObjectBodyCacheHook, register_get_object_body_cache_hook}; +pub use body_cache_hook::{ + GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, + register_get_object_body_cache_hook, unregister_get_object_body_cache_hook, +}; +pub(crate) use body_cache_hook::{ + get_object_body_cache_hook, get_object_body_cache_hook_suppressed, without_get_object_body_cache_hook, +}; pub(crate) use object_mutation_hook::notify_object_mutation; -pub use object_mutation_hook::{ObjectMutationHook, register_object_mutation_hook}; +pub use object_mutation_hook::{ObjectMutationHook, register_object_mutation_hook, unregister_object_mutation_hook}; pub use readers::*; pub use types::*; diff --git a/crates/ecstore/src/object_api/object_mutation_hook.rs b/crates/ecstore/src/object_api/object_mutation_hook.rs index d9b0f6424..529561d87 100644 --- a/crates/ecstore/src/object_api/object_mutation_hook.rs +++ b/crates/ecstore/src/object_api/object_mutation_hook.rs @@ -56,6 +56,14 @@ pub fn register_object_mutation_hook(hook: Arc) { ); } +/// Unregister the process-wide object mutation hook. +/// +/// Config reloads use this when body caching becomes disabled so the previous +/// adapter and its cached plaintext bodies are not retained until their TTL. +pub fn unregister_object_mutation_hook() { + OBJECT_MUTATION_HOOK.clear(); +} + /// The registered hook, if any. fn object_mutation_hook() -> Option> { OBJECT_MUTATION_HOOK.get() @@ -71,13 +79,6 @@ pub(crate) async fn notify_object_mutation(bucket: &str, object: &str) { } } -/// Test-only: unregister the hook so tests can register and clear the slot -/// deterministically without leaking a hook into unrelated tests. -#[cfg(test)] -pub(crate) fn clear_object_mutation_hook() { - OBJECT_MUTATION_HOOK.clear(); -} - #[cfg(test)] mod tests { use super::*; @@ -97,7 +98,7 @@ mod tests { #[tokio::test] #[serial_test::serial(object_mutation_hook)] async fn notify_invokes_registered_hook_with_identity() { - clear_object_mutation_hook(); + unregister_object_mutation_hook(); let calls = Arc::new(Mutex::new(Vec::new())); register_object_mutation_hook(Arc::new(RecordingHook { calls: Arc::clone(&calls), @@ -106,14 +107,35 @@ mod tests { notify_object_mutation("bucket", "photos/a.jpg").await; assert_eq!(&*calls.lock().unwrap(), &[("bucket".to_string(), "photos/a.jpg".to_string())]); - clear_object_mutation_hook(); + unregister_object_mutation_hook(); } #[tokio::test] #[serial_test::serial(object_mutation_hook)] - async fn notify_without_registered_hook_is_noop() { - clear_object_mutation_hook(); - // Must not panic when no hook is installed (the cache feature is off). + async fn unregister_prevents_later_notifications() { + unregister_object_mutation_hook(); + let calls = Arc::new(Mutex::new(Vec::new())); + register_object_mutation_hook(Arc::new(RecordingHook { + calls: Arc::clone(&calls), + })); + unregister_object_mutation_hook(); + notify_object_mutation("bucket", "object").await; + + assert!(calls.lock().unwrap().is_empty(), "an unregistered hook must receive no mutation callback"); + } + + #[test] + #[serial_test::serial(object_mutation_hook)] + fn unregister_releases_the_previous_hook() { + unregister_object_mutation_hook(); + let calls = Arc::new(Mutex::new(Vec::new())); + let hook = Arc::new(RecordingHook { calls }); + let weak = Arc::downgrade(&hook); + register_object_mutation_hook(hook); + + assert!(weak.upgrade().is_some()); + unregister_object_mutation_hook(); + assert!(weak.upgrade().is_none()); } } diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index e8baa6545..586ce0fa6 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -307,6 +307,17 @@ pub struct GetObjectReader { } impl GetObjectReader { + /// Builds a fully materialized reader from a cache-coordinated body. + pub fn from_cache_body(mut object_info: ObjectInfo, body: Bytes) -> Result { + object_info.size = i64::try_from(body.len()).map_err(|_| Error::other("cached GET body length exceeds i64::MAX"))?; + Ok(Self { + stream: Box::new(std::io::Cursor::new(body.clone())), + object_info, + buffered_body: Some(body), + body_source: GetObjectBodySource::HookServed, + }) + } + /// True when `buffered_body` is the body the cache hook served. The app /// layer serves it as the object-data-cache source without a second lookup. pub fn is_cache_hook_served(&self) -> bool { @@ -1674,6 +1685,40 @@ mod tests { bytes } + #[tokio::test] + async fn cache_body_uses_plaintext_length_for_compressed_metadata() { + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, + "klauspost/compress/s2".to_string(), + ); + let object_info = ObjectInfo { + size: 3, + actual_size: 11, + user_defined: Arc::new(metadata), + ..Default::default() + }; + assert!(object_info.is_compressed()); + + let body = Bytes::from_static(b"hello world"); + let mut reader = + GetObjectReader::from_cache_body(object_info, body.clone()).expect("cache body length must fit in object metadata"); + + assert_eq!(reader.body_source, GetObjectBodySource::HookServed); + assert_eq!(reader.buffered_body.as_ref(), Some(&body)); + assert_eq!(reader.object_info.size, 11); + assert_eq!(reader.object_info.actual_size, 11); + assert!(reader.object_info.is_compressed()); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("cache body should stream"); + assert_eq!(restored, body); + } + /// Regression for the #4576 fallout: the encrypt side persists a random /// SSE-C nonce, and this reader-side resolver must read it back — falling /// back to the deterministic legacy nonce only when no IV was stored. diff --git a/crates/ecstore/src/services/tier/test_util.rs b/crates/ecstore/src/services/tier/test_util.rs index 5cf4ed39a..84d7d4b6e 100644 --- a/crates/ecstore/src/services/tier/test_util.rs +++ b/crates/ecstore/src/services/tier/test_util.rs @@ -62,7 +62,7 @@ use std::time::Duration; use async_trait::async_trait; use tokio::io::AsyncReadExt; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::{Mutex, Notify, RwLock}; use uuid::Uuid; use crate::client::transition_api::{ReadCloser, ReaderImpl}; @@ -141,6 +141,40 @@ struct MockWarmBackendInner { objects: Mutex>, faults: Mutex, op_log: Mutex>, + put_versions: Mutex>, + remove_versions: Mutex>, + put_barrier: Mutex>>, +} + +#[derive(Default)] +struct MockPutBarrierState { + arrived: Notify, + release: Notify, +} + +/// One-shot barrier that pauses a mock tier PUT after storing its remote body. +pub struct MockPutBarrier { + state: Arc, +} + +impl MockPutBarrier { + /// Wait until the remote body is stored and the PUT is paused before returning. + pub async fn wait_until_paused(&self) { + tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) + .await + .expect("mock tier PUT should reach the deterministic barrier"); + } + + /// Release the paused PUT. + pub fn release(&self) { + self.state.release.notify_one(); + } +} + +impl Drop for MockPutBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + } } /// In-memory [`WarmBackend`] for lifecycle / tiering integration tests. @@ -159,6 +193,13 @@ impl MockWarmBackend { Self::default() } + /// Arm a one-shot pause after the next tier PUT stores its remote body. + pub async fn arm_put_barrier(&self) -> MockPutBarrier { + let state = Arc::new(MockPutBarrierState::default()); + *self.inner.put_barrier.lock().await = Some(Arc::clone(&state)); + MockPutBarrier { state } + } + // ---- fault injection ------------------------------------------------- /// Replace the entire fault configuration. @@ -232,6 +273,16 @@ impl MockWarmBackend { .count() } + /// Return the exact object/version pairs produced by successful tier PUTs. + pub async fn put_versions(&self) -> Vec<(String, String)> { + self.inner.put_versions.lock().await.clone() + } + + /// Return the exact object/version pairs passed to successful tier removes. + pub async fn remove_versions(&self) -> Vec<(String, String)> { + self.inner.remove_versions.lock().await.clone() + } + /// Number of `get` calls recorded — useful to assert restore reads hit the /// local copy rather than the remote tier. pub async fn get_count(&self) -> usize { @@ -358,6 +409,11 @@ impl WarmBackend for MockWarmBackend { self.precondition().await?; let bytes = self.read_bytes(r).await?; let version = self.put_bytes(object, bytes, HashMap::new()).await; + self.inner + .put_versions + .lock() + .await + .push((object.to_string(), version.clone())); self.record(MockWarmOp::Put { object: object.to_string(), }) @@ -398,6 +454,16 @@ impl WarmBackend for MockWarmBackend { metadata.insert("x-amz-object-lock-legal-hold".to_string(), opts.legalhold.as_str().to_string()); } let version = self.put_bytes(object, bytes, metadata).await; + self.inner + .put_versions + .lock() + .await + .push((object.to_string(), version.clone())); + let barrier = self.inner.put_barrier.lock().await.take(); + if let Some(barrier) = barrier { + barrier.arrived.notify_one(); + barrier.release.notified().await; + } self.record(MockWarmOp::Put { object: object.to_string(), }) @@ -427,9 +493,22 @@ impl WarmBackend for MockWarmBackend { Ok(tokio::io::BufReader::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec()))) } - async fn remove(&self, object: &str, _rv: &str) -> Result<(), std::io::Error> { + async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> { self.precondition().await?; - self.inner.objects.lock().await.remove(object); + let mut objects = self.inner.objects.lock().await; + if let Some(stored) = objects.get(object) + && !rv.is_empty() + && stored.remote_version_id != rv + { + return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "NoSuchVersion")); + } + objects.remove(object); + drop(objects); + self.inner + .remove_versions + .lock() + .await + .push((object.to_string(), rv.to_string())); self.record(MockWarmOp::Remove { object: object.to_string(), }) diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index a95b158d6..3484d4368 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -160,7 +160,7 @@ use rustfs_utils::{ }; use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_RESTORE}; use sha2::{Digest, Sha256}; -use std::hash::Hash; +use std::hash::{BuildHasher, Hash, Hasher}; use std::mem::{self}; use std::pin::Pin; use std::sync::OnceLock; @@ -547,6 +547,7 @@ const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750); const GET_OBJECT_METADATA_CACHE_TTL: Duration = Duration::from_secs(2); // Increased from 250ms to 2s const DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: usize = 4096; // Increased from 1024 to 4096 const ENV_RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: &str = "RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES"; +const GET_OBJECT_METADATA_CACHE_FENCE_SHARDS: u16 = 4096; // --- Codec Streaming Configuration --- @@ -623,12 +624,284 @@ mod core; mod ctx; mod metadata; mod ops; +pub(crate) use ops::object::body_cache_plaintext_len; mod read; mod replication; pub(crate) mod shard_source; pub use ops::heal_walk::HealWalkVersion; +pub(crate) struct PreparedGetObjectMetadata { + fi: FileInfo, + files: Vec, + disks: Vec>, + object_info: Option, +} + +impl PreparedGetObjectMetadata { + pub(crate) fn object_info(&self) -> &ObjectInfo { + self.object_info + .as_ref() + .expect("prepared GET metadata must retain its ObjectInfo until consumed") + } + + pub(crate) fn take_object_info(&mut self) -> ObjectInfo { + self.object_info + .take() + .expect("prepared GET metadata ObjectInfo must be consumed exactly once") + } +} + +tokio::task_local! { + static PREPARED_GET_OBJECT_METADATA: std::cell::RefCell>; +} + +#[cfg(test)] +tokio::task_local! { + static GET_OBJECT_INFO_CONVERSIONS: Arc; +} + +fn build_get_object_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo { + #[cfg(test)] + let _ = GET_OBJECT_INFO_CONVERSIONS.try_with(|conversions| { + conversions.fetch_add(1, Ordering::Relaxed); + }); + ObjectInfo::from_file_info(fi, bucket, object, versioned) +} + +fn take_prepared_get_object_metadata() -> Option { + PREPARED_GET_OBJECT_METADATA + .try_with(|prepared| prepared.borrow_mut().take()) + .ok() + .flatten() +} + +async fn with_prepared_get_object_metadata(metadata: PreparedGetObjectMetadata, future: F) -> F::Output +where + F: std::future::Future, +{ + PREPARED_GET_OBJECT_METADATA + .scope(std::cell::RefCell::new(Some(metadata)), future) + .await +} + +#[cfg(test)] +mod prepared_get_object_metadata_tests { + use super::*; + use crate::ecstore_validation_blackbox::make_local_set_disks; + use crate::object_api::{BLOCK_SIZE_V2, PutObjReader}; + use crate::set_disk::core::io_primitives::disk_call_counters; + use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions}; + use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + use http::HeaderMap; + use tokio::io::AsyncReadExt; + + #[tokio::test] + async fn prepared_metadata_is_consumed_exactly_once() { + let metadata = PreparedGetObjectMetadata { + fi: FileInfo::default(), + files: Vec::new(), + disks: Vec::new(), + object_info: None, + }; + + with_prepared_get_object_metadata(metadata, async { + assert!(take_prepared_get_object_metadata().is_some()); + assert!(take_prepared_get_object_metadata().is_none()); + }) + .await; + assert!(take_prepared_get_object_metadata().is_none()); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn prepared_reader_reuses_metadata_fanout_exactly_once() { + let (_dirs, set_disks) = make_local_set_disks(4, 2).await; + let bucket = "prepared-metadata-fanout"; + let object = "prepared-metadata-fanout-object.bin"; + let payload = b"prepared-metadata-fanout-payload-".repeat(40_000); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, object, &mut put_reader, &opts) + .await + .expect("object should be written"); + + let calls = disk_call_counters::observe(object); + let conversions = Arc::new(AtomicU64::new(0)); + let restored = GET_OBJECT_INFO_CONVERSIONS + .scope(Arc::clone(&conversions), async { + let metadata = set_disks + .prepare_get_object_metadata(bucket, object, &opts) + .await + .expect("prepared metadata should resolve"); + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + 4, + "preparation should fan out to each online disk exactly once" + ); + + let mut reader = set_disks + .get_object_reader_with_prepared_metadata(bucket, object, None, HeaderMap::new(), &opts, metadata) + .await + .expect("prepared body reader should open"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("prepared body should stream"); + restored + }) + .await; + + assert_eq!(restored, payload); + assert_eq!( + conversions.load(Ordering::Relaxed), + 1, + "prepared reader must consume the ObjectInfo built during metadata preparation" + ); + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + 4, + "reader construction must consume prepared metadata instead of repeating the fanout" + ); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn prepared_reader_rebuilds_object_info_when_precomputed_value_is_absent() { + let (_dirs, set_disks) = make_local_set_disks(4, 2).await; + let bucket = "prepared-object-info-fallback"; + let object = "prepared-object-info-fallback.bin"; + let payload = b"prepared-object-info-fallback-payload".repeat(4_000); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, object, &mut put_reader, &opts) + .await + .expect("object should be written"); + + let conversions = Arc::new(AtomicU64::new(0)); + let restored = GET_OBJECT_INFO_CONVERSIONS + .scope(Arc::clone(&conversions), async { + let mut metadata = set_disks + .prepare_get_object_metadata(bucket, object, &opts) + .await + .expect("prepared metadata should resolve"); + metadata.object_info = None; + let mut reader = set_disks + .get_object_reader_with_prepared_metadata(bucket, object, None, HeaderMap::new(), &opts, metadata) + .await + .expect("reader should rebuild missing prepared ObjectInfo"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("fallback reader should stream"); + restored + }) + .await; + + assert_eq!(restored, payload); + assert_eq!( + conversions.load(Ordering::Relaxed), + 2, + "missing precomputed ObjectInfo must trigger exactly one structural fallback rebuild" + ); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn prepared_reader_restores_full_body_with_one_offline_shard() { + let (_dirs, set_disks) = make_local_set_disks(4, 2).await; + let bucket = "prepared-reader-offline-shard"; + let object = "prepared-reader-offline-shard-object.bin"; + let payload = (0..(BLOCK_SIZE_V2 + 321)) + .map(|idx| ((idx * 19) % 251) as u8) + .collect::>(); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, object, &mut put_reader, &opts) + .await + .expect("object should be written"); + set_disks.disks.write().await[0] = None; + + let metadata = set_disks + .prepare_get_object_metadata(bucket, object, &opts) + .await + .expect("prepared metadata should tolerate one offline shard"); + let mut reader = set_disks + .get_object_reader_with_prepared_metadata(bucket, object, None, HeaderMap::new(), &opts, metadata) + .await + .expect("prepared body reader should open with one offline shard"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("degraded prepared body should stream"); + + assert_eq!(restored, payload); + } +} + +impl SetDisks { + pub(crate) async fn prepare_get_object_metadata( + &self, + bucket: &str, + object: &str, + opts: &ObjectOptions, + ) -> Result { + let (fi, files, disks) = self.get_object_fileinfo(bucket, object, opts, true, true).await?; + let object_info = build_get_object_info(&fi, bucket, object, opts.versioned || opts.version_suspended); + Ok(PreparedGetObjectMetadata { + fi, + files, + disks, + object_info: Some(object_info), + }) + } + + pub(crate) async fn get_object_reader_with_prepared_metadata( + &self, + bucket: &str, + object: &str, + range: Option, + headers: HeaderMap, + opts: &ObjectOptions, + metadata: PreparedGetObjectMetadata, + ) -> Result { + with_prepared_get_object_metadata(metadata, self.get_object_reader(bucket, object, range, headers, opts)).await + } +} + /// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds) /// Defaults to 30 seconds if not set or invalid /// Lock acquisition timeout. Cached: this is consulted on every object @@ -1742,6 +2015,8 @@ pub struct SetDisks { pub format: FormatV3, disk_health_cache: Arc>>>, get_object_metadata_cache: moka::future::Cache>, + get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState, + get_object_metadata_cache_generations: Arc<[AtomicU64]>, pub lockers: Vec>, local_lock_manager: Arc, /// Per-instance runtime context (Phase 5, backlog#939). @@ -1761,21 +2036,109 @@ pub struct SetDisks { capacity_dirty_generation: Arc, } -#[derive(Clone, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] struct GetObjectMetadataCacheKey { + bucket: Arc, + object: Arc, + generation: u64, + hash: u64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct GetObjectMetadataCacheGeneration { + index: usize, + value: u64, + hash: u64, +} + +#[cfg(test)] +struct MetadataCacheInvalidationProbeState { bucket: String, object: String, + count: AtomicU64, +} + +#[cfg(test)] +struct MetadataCacheInvalidationProbe { + state: Arc, +} + +#[cfg(test)] +static METADATA_CACHE_INVALIDATION_PROBE: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +impl MetadataCacheInvalidationProbe { + fn install(bucket: &str, object: &str) -> Self { + let state = Arc::new(MetadataCacheInvalidationProbeState { + bucket: bucket.to_string(), + object: object.to_string(), + count: AtomicU64::new(0), + }); + let mut slot = METADATA_CACHE_INVALIDATION_PROBE + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("metadata cache invalidation probe mutex should not poison"); + assert!( + slot.is_none(), + "metadata cache invalidation probe must be installed by one test at a time" + ); + *slot = Some(Arc::clone(&state)); + drop(slot); + Self { state } + } + + fn count(&self) -> u64 { + self.state.count.load(Ordering::Acquire) + } +} + +#[cfg(test)] +impl Drop for MetadataCacheInvalidationProbe { + fn drop(&mut self) { + let mut slot = METADATA_CACHE_INVALIDATION_PROBE + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("metadata cache invalidation probe mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +fn record_metadata_cache_invalidation(bucket: &str, object: &str) { + let probe = METADATA_CACHE_INVALIDATION_PROBE + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("metadata cache invalidation probe mutex should not poison") + .as_ref() + .filter(|probe| probe.bucket == bucket && probe.object == object) + .cloned(); + if let Some(probe) = probe { + probe.count.fetch_add(1, Ordering::AcqRel); + } } impl GetObjectMetadataCacheKey { - fn new(bucket: &str, object: &str) -> Self { + fn new(bucket: &str, object: &str, generation: GetObjectMetadataCacheGeneration) -> Self { Self { - bucket: bucket.to_string(), - object: object.to_string(), + bucket: Arc::from(bucket), + object: Arc::from(object), + generation: generation.value, + hash: generation.hash, } } } +impl Hash for GetObjectMetadataCacheKey { + fn hash(&self, state: &mut H) { + self.hash.hash(state); + self.generation.hash(state); + } +} + #[derive(Clone, Debug)] struct GetObjectMetadataCacheEntry { #[allow(dead_code)] // Kept for debugging; moka handles TTL internally @@ -1803,10 +2166,50 @@ impl DiskHealthEntry { } impl SetDisks { + fn get_object_metadata_cache_hash(&self, bucket: &str, object: &str) -> u64 { + let mut hasher = self.get_object_metadata_cache_hash_builder.build_hasher(); + bucket.hash(&mut hasher); + object.hash(&mut hasher); + hasher.finish() + } + + fn get_object_metadata_cache_generation(&self, bucket: &str, object: &str) -> Option { + let hash = self.get_object_metadata_cache_hash(bucket, object); + let hash_bytes = hash.to_le_bytes(); + let index = usize::from(u16::from_le_bytes([hash_bytes[0], hash_bytes[1]]) % GET_OBJECT_METADATA_CACHE_FENCE_SHARDS); + let value = self.get_object_metadata_cache_generations[index].load(Ordering::Acquire); + (value != u64::MAX).then_some(GetObjectMetadataCacheGeneration { index, value, hash }) + } + + fn is_get_object_metadata_cache_generation_current(&self, generation: GetObjectMetadataCacheGeneration) -> bool { + self.get_object_metadata_cache_generations[generation.index].load(Ordering::Acquire) == generation.value + } + async fn invalidate_get_object_metadata_cache(&self, bucket: &str, object: &str) { + let hash = self.get_object_metadata_cache_hash(bucket, object); + let hash_bytes = hash.to_le_bytes(); + let index = usize::from(u16::from_le_bytes([hash_bytes[0], hash_bytes[1]]) % GET_OBJECT_METADATA_CACHE_FENCE_SHARDS); + let generation = &self.get_object_metadata_cache_generations[index]; + let previous = match generation.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| current.checked_add(1)) { + Ok(previous) | Err(previous) => previous, + }; + let previous = GetObjectMetadataCacheGeneration { + index, + value: previous, + hash, + }; self.get_object_metadata_cache - .invalidate(&GetObjectMetadataCacheKey::new(bucket, object)) + .invalidate(&GetObjectMetadataCacheKey::new(bucket, object, previous)) .await; + #[cfg(test)] + record_metadata_cache_invalidation(bucket, object); + } + + fn invalidate_all_get_object_metadata_cache(&self) { + for generation in self.get_object_metadata_cache_generations.iter() { + let _ = generation.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| current.checked_add(1)); + } + self.get_object_metadata_cache.invalidate_all(); } async fn acquire_read_lock_diag(&self, op: &'static str, bucket: &str, object: &str) -> Result { @@ -1953,6 +2356,12 @@ impl SetDisks { .max_capacity(get_object_metadata_cache_max_entries() as u64) .time_to_live(GET_OBJECT_METADATA_CACHE_TTL) .build(), + get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState::new(), + get_object_metadata_cache_generations: Arc::from( + (0..usize::from(GET_OBJECT_METADATA_CACHE_FENCE_SHARDS)) + .map(|_| AtomicU64::new(0)) + .collect::>(), + ), lockers, // Sourced from the instance context so each instance owns its lock // namespace (Phase 5 Slice 3). Single-instance: ctx aliases the diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index b3f66e09a..5d4618685 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -21,8 +21,9 @@ use super::super::*; +use crate::bucket::lifecycle::tier_sweeper::delete_object_from_remote_tier_idempotent; use crate::disk::OldCurrentSize; -use crate::object_api::GetObjectBodySource; +use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed}; /// Length of the full plaintext body when — and only when — this read's output /// is exactly the object's complete plaintext, so the app-layer body cache may @@ -61,6 +62,11 @@ fn full_object_plaintext_len(range: &Option, opts: &ObjectOptions || crate::object_api::restore_request_active(opts) || object_info.is_encrypted() || object_info.is_remote() + || object_info.delete_marker + || object_info.size == 0 + || object_info.version_only + || object_info.metadata_only + || object_info.is_inline_fast_path_eligible() { return None; } @@ -72,6 +78,14 @@ fn full_object_plaintext_len(range: &Option, opts: &ObjectOptions Some(object_info.size) } +pub(crate) fn body_cache_plaintext_len( + range: &Option, + opts: &ObjectOptions, + object_info: &ObjectInfo, +) -> Option { + full_object_plaintext_len(range, opts, object_info) +} + #[async_trait::async_trait] impl crate::storage_api_contracts::object::ObjectIO for SetDisks { type Error = Error; @@ -131,16 +145,21 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { }; let metadata_stage_start = Instant::now(); - let (fi, files, disks) = match self.get_object_fileinfo(bucket, object, opts, true, true).await { - Ok(result) => result, - Err(err) => { - rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_stage_start.elapsed().as_secs_f64()); - record_get_object_pipeline_failure(GET_STAGE_METADATA, classify_storage_error(&err)); - return Err(to_object_err(err, vec![bucket, object])); + let (fi, files, disks, prepared_object_info) = if let Some(prepared) = take_prepared_get_object_metadata() { + (prepared.fi, prepared.files, prepared.disks, prepared.object_info) + } else { + match self.get_object_fileinfo(bucket, object, opts, true, true).await { + Ok((fi, files, disks)) => (fi, files, disks, None), + Err(err) => { + rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_stage_start.elapsed().as_secs_f64()); + record_get_object_pipeline_failure(GET_STAGE_METADATA, classify_storage_error(&err)); + return Err(to_object_err(err, vec![bucket, object])); + } } }; let object_info_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); - let object_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended); + let object_info = prepared_object_info + .unwrap_or_else(|| build_get_object_info(&fi, bucket, object, opts.versioned || opts.version_suspended)); let object_class = classify_get_codec_streaming_object_class(&range, &object_info, &fi); let size_bucket = rustfs_io_metrics::get_object_size_bucket(object_info.size); record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_OBJECT_INFO, object_info_stage_start); @@ -410,6 +429,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { // it forward. let mut body_source = GetObjectBodySource::Unprobed; if let Some(plaintext_len) = full_object_plaintext_len(&range, opts, &object_info) + && !get_object_body_cache_hook_suppressed() && let Some(hook) = get_object_body_cache_hook() { match hook.lookup(bucket, object, &object_info).await { @@ -1222,6 +1242,178 @@ impl SetDisks { } } +async fn cleanup_uncommitted_transition_upload(tier: &str, object: &str, remote_version: &str) { + if let Err(err) = delete_object_from_remote_tier_idempotent(object, remote_version, tier).await { + warn!( + tier, + object, + remote_version, + error = ?err, + "failed to clean uncommitted transition upload" + ); + } +} + +struct TransitionUploadCleanup { + tier: String, + object: String, + remote_version: String, + armed: bool, +} + +impl TransitionUploadCleanup { + fn new(tier: &str, object: &str, remote_version: &str) -> Self { + Self { + tier: tier.to_string(), + object: object.to_string(), + remote_version: remote_version.to_string(), + armed: true, + } + } + + async fn cleanup(&mut self) { + cleanup_uncommitted_transition_upload(&self.tier, &self.object, &self.remote_version).await; + self.armed = false; + } + + fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for TransitionUploadCleanup { + fn drop(&mut self) { + if !self.armed { + return; + } + let tier = self.tier.clone(); + let object = self.object.clone(); + let remote_version = self.remote_version.clone(); + if let Ok(handle) = tokio::runtime::Handle::try_current() { + handle.spawn(async move { + cleanup_uncommitted_transition_upload(&tier, &object, &remote_version).await; + }); + } + } +} + +#[cfg(test)] +struct TransitionCommitBarrierState { + bucket: String, + object: String, + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(test)] +struct TransitionCommitBarrier { + state: Arc, +} + +#[cfg(test)] +static TRANSITION_COMMIT_BARRIER: std::sync::OnceLock>>> = + std::sync::OnceLock::new(); + +#[cfg(test)] +impl TransitionCommitBarrier { + fn install(bucket: &str, object: &str) -> Self { + let state = Arc::new(TransitionCommitBarrierState { + bucket: bucket.to_string(), + object: object.to_string(), + arrived: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let mut slot = TRANSITION_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("transition commit barrier mutex should not poison"); + assert!(slot.is_none(), "transition commit barrier must be installed by one test at a time"); + *slot = Some(Arc::clone(&state)); + drop(slot); + Self { state } + } + + async fn wait_until_paused(&self) { + tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) + .await + .expect("transition should reach the deterministic commit barrier"); + } + + fn release(&self) { + self.state.release.notify_one(); + } +} + +#[cfg(test)] +impl Drop for TransitionCommitBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + let mut slot = TRANSITION_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("transition commit barrier mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +async fn pause_transition_before_local_commit(bucket: &str, object: &str) { + let barrier = TRANSITION_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("transition commit barrier mutex should not poison") + .as_ref() + .filter(|barrier| barrier.bucket == bucket && barrier.object == object) + .cloned(); + if let Some(barrier) = barrier { + barrier.arrived.notify_one(); + barrier.release.notified().await; + } +} + +fn parse_transition_version_id(remote_version: &str) -> std::result::Result, uuid::Error> { + if remote_version.is_empty() { + return Ok(None); + } + Uuid::parse_str(remote_version).map(|version_id| (!version_id.is_nil()).then_some(version_id)) +} + +fn transition_cleanup_remote_version(remote_version: &str, version_id: Option) -> &str { + version_id.map(|_| remote_version).unwrap_or("") +} + +#[cfg(test)] +mod transition_version_id_tests { + use super::{parse_transition_version_id, transition_cleanup_remote_version}; + use uuid::Uuid; + + #[test] + fn normalizes_unversioned_remote_ids() { + assert_eq!(parse_transition_version_id("").expect("empty remote version should be valid"), None); + assert_eq!( + parse_transition_version_id(&Uuid::nil().to_string()).expect("nil remote version should be valid"), + None + ); + assert_eq!(transition_cleanup_remote_version(&Uuid::nil().to_string(), None), ""); + } + + #[test] + fn preserves_valid_remote_id_and_rejects_invalid_text() { + let version_id = Uuid::new_v4(); + assert_eq!( + parse_transition_version_id(&version_id.to_string()).expect("UUID remote version should be valid"), + Some(version_id) + ); + assert_eq!( + transition_cleanup_remote_version(&version_id.to_string(), Some(version_id)), + version_id.to_string() + ); + assert!(parse_transition_version_id("not-a-uuid").is_err()); + } +} + #[async_trait::async_trait] impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { type Error = Error; @@ -1908,7 +2100,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { .await .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?; - self.get_object_metadata_cache.invalidate_all(); + self.invalidate_all_get_object_metadata_cache(); return Ok(ObjectInfo::default()); } @@ -2236,7 +2428,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { if let Some(mod_time1) = opts.mod_time { if let Some(mod_time2) = fi.mod_time.as_ref() { if mod_time1.unix_timestamp() != mod_time2.unix_timestamp() - /*|| transition_etag != stored_etag*/ + || (!transition_etag.is_empty() && transition_etag != stored_etag) { return Err(to_object_err(Error::other(DiskError::FileNotFound), vec![bucket, object])); } @@ -2333,25 +2525,88 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { return Err(StorageError::Io(err)); } let rv = rv?; - fi.transition_status = TRANSITION_COMPLETE.to_string(); - fi.transitioned_objname = dest_obj; - fi.transition_tier = opts.transition.tier.clone(); - fi.transition_version_id = if rv.is_empty() { None } else { Some(Uuid::parse_str(&rv)?) }; + drop(tier_config_mgr); + + let transition_version_id = match parse_transition_version_id(&rv) { + Ok(version_id) => version_id, + Err(err) => { + cleanup_uncommitted_transition_upload(&opts.transition.tier, &dest_obj, &rv).await; + return Err(err.into()); + } + }; + let cleanup_remote_version = transition_cleanup_remote_version(&rv, transition_version_id); + let mut upload_cleanup = TransitionUploadCleanup::new(&opts.transition.tier, &dest_obj, cleanup_remote_version); + + let mut commit_opts = opts.clone(); + commit_opts.no_lock = true; + commit_opts.metadata_cache_safe = false; + let transition_lock_guard = if opts.no_lock { + None + } else { + match self.acquire_write_lock_diag("transition_object_commit", bucket, object).await { + Ok(guard) => Some(guard), + Err(err) => { + upload_cleanup.cleanup().await; + return Err(err); + } + } + }; + self.invalidate_get_object_metadata_cache(bucket, object).await; + let current = self.get_object_fileinfo(bucket, object, &commit_opts, true, false).await; + let (mut current_fi, _, _) = match current { + Ok(current) => current, + Err(err) => { + drop(transition_lock_guard); + upload_cleanup.cleanup().await; + return Err(err); + } + }; + let source_matches = current_fi.version_id == fi.version_id + && current_fi.data_dir == fi.data_dir + && current_fi.mod_time == fi.mod_time + && current_fi.size == fi.size + && rustfs_utils::path::trim_etag(&get_raw_etag(¤t_fi.metadata)) == stored_etag; + if current_fi.transition_status == TRANSITION_COMPLETE || !source_matches { + let already_transitioned = current_fi.transition_status == TRANSITION_COMPLETE; + drop(transition_lock_guard); + upload_cleanup.cleanup().await; + if already_transitioned { + return Ok(()); + } + return Err(to_object_err(Error::other(DiskError::FileNotFound), vec![bucket, object])); + } + + current_fi.transition_status = TRANSITION_COMPLETE.to_string(); + current_fi.transitioned_objname = dest_obj; + current_fi.transition_tier = opts.transition.tier.clone(); + current_fi.transition_version_id = transition_version_id; + fi = current_fi; let event_name = EventName::LifecycleTransition.as_str(); - let mut should_notify_transition = true; - - let disks = self.disk_inventory().await; + if transition_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + drop(transition_lock_guard); + upload_cleanup.cleanup().await; + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "transition_object_commit", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + #[cfg(test)] + pause_transition_before_local_commit(bucket, object).await; + upload_cleanup.disarm(); if let Err(err) = self.delete_object_version(bucket, object, &fi, false).await { - should_notify_transition = false; warn!( bucket = bucket, object = object, error = ?err, - "transition completed on remote tier but source cleanup failed; skipping external lifecycle transition notification" + "transition remote upload completed but local commit failed" ); - } else { - self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks); + self.invalidate_get_object_metadata_cache(bucket, object).await; + drop(transition_lock_guard); + return Err(err); } // delete_object_version persisted transition_status=complete and freed the @@ -2361,6 +2616,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // early-return above and streams the already-deleted local data to the // remote tier again (rustfs/rustfs#4827). self.invalidate_get_object_metadata_cache(bucket, object).await; + drop(transition_lock_guard); + let disks = self.disk_inventory().await; + self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks); for disk in disks.iter() { if let Some(disk) = disk { @@ -2372,17 +2630,15 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { break; } - if should_notify_transition { - let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended); - send_event(EventArgs { - event_name: event_name.to_string(), - bucket_name: bucket.to_string(), - object: obj_info, - user_agent: "Internal: [ILM-Transition]".to_string(), - host: runtime_sources::default_local_node_name(), - ..Default::default() - }); - } + let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended); + send_event(EventArgs { + event_name: event_name.to_string(), + bucket_name: bucket.to_string(), + object: obj_info, + user_agent: "Internal: [ILM-Transition]".to_string(), + host: runtime_sources::default_local_node_name(), + ..Default::default() + }); //let tags = opts.lifecycle_audit_event.tags(); //auditLogLifecycle(ctx, objInfo, ILMTransition, tags, traceFn) Ok(()) @@ -2738,6 +2994,127 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support { } } +#[cfg(all(test, feature = "test-util"))] +mod transition_commit_failure_tests { + use super::hermetic_set_disks_support::hermetic_set_disks; + use super::*; + use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions}; + use crate::disk::DiskAPI as _; + use crate::services::tier::test_util::register_mock_tier; + use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + use http::HeaderMap; + use tokio::io::AsyncReadExt; + + #[tokio::test] + #[serial_test::serial] + async fn local_commit_failure_returns_error_and_preserves_remote_candidate() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "transition-commit-failure-bucket"; + let object = "object.bin"; + let payload = b"transition commit failure must preserve the uploaded candidate".repeat(1024); + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let mut reader = PutObjReader::from_vec(payload.clone()); + let original = set_disks + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("source object should be written"); + let (fi, parts_metadata, online_disks) = set_disks + .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false) + .await + .expect("source metadata should resolve"); + let generation = set_disks + .get_object_metadata_cache_generation(bucket, object) + .expect("metadata cache generation should be active"); + let cache_key = GetObjectMetadataCacheKey::new(bucket, object, generation); + set_disks + .get_object_metadata_cache + .insert( + cache_key.clone(), + Arc::new(GetObjectMetadataCacheEntry { + created_at: Instant::now(), + fi: fi.clone(), + parts_metadata, + online_disks, + read_quorum: 2, + }), + ) + .await; + assert!(set_disks.get_object_metadata_cache.get(&cache_key).await.is_some()); + + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; + let opts = ObjectOptions { + no_lock: true, + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name, + etag: original.etag.clone().unwrap_or_default(), + ..Default::default() + }, + version_id: original.version_id.map(|version| version.to_string()), + mod_time: original.mod_time, + ..Default::default() + }; + + let barrier = TransitionCommitBarrier::install(bucket, object); + let invalidations = MetadataCacheInvalidationProbe::install(bucket, object); + let transition_set = Arc::clone(&set_disks); + let transition = tokio::spawn(async move { transition_set.transition_object(bucket, object, &opts).await }); + barrier.wait_until_paused().await; + assert_eq!(invalidations.count(), 1, "precommit revalidation must fence the old metadata once"); + let saved_disks = { + let mut disks = set_disks.disks.write().await; + let saved = std::mem::take(&mut *disks); + *disks = vec![None; saved.len()]; + saved + }; + barrier.release(); + let result = transition.await.expect("transition task should not panic"); + *set_disks.disks.write().await = saved_disks; + + result.expect_err("local write quorum failure must be returned to the transition worker"); + assert_eq!( + invalidations.count(), + 2, + "local commit failure must fence any partially committed metadata again" + ); + assert_eq!(backend.put_count().await, 1); + assert_eq!( + backend.remove_count().await, + 0, + "ambiguous local commit failure must retain the remote candidate" + ); + assert_eq!(backend.object_count().await, 1); + assert!( + set_disks.get_object_metadata_cache.get(&cache_key).await.is_none(), + "local commit failure must invalidate pre-transition metadata" + ); + + let mut restored = Vec::new(); + set_disks + .get_object_reader( + bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("the local source must remain readable after a fail-before-commit result") + .stream + .read_to_end(&mut restored) + .await + .expect("the local source body should drain"); + assert_eq!(restored, payload); + } +} + #[cfg(test)] mod put_object_tmp_cleanup_tests { //! Regression coverage for backlog#924 (HP-3): the speculative tmp-dir diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 4a3015ecd..610fc6136 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -20,21 +20,22 @@ use crate::diagnostics::get::{ GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS, GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM, GET_METADATA_CACHE_REASON_META_BUCKET, GET_METADATA_CACHE_REASON_NO_LOCK, GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED, GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER, - GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, - GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED, - GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, - GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, - GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, - GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, - GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, - GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, - GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, - GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, - GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, - GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, - GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, - GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure, - record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled, + GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_STALE_PUBLICATION, + GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, + GET_METADATA_CACHE_REASON_VERSIONED, GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, + GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR, + GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, + GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, + GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, + GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR, + GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID, + GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY, + GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP, + GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, GET_STAGE_READER_SETUP_DROP_PENDING, + GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT, + GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, GetObjectFailureReason, classify_disk_error, + get_stage_timer_if_enabled, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path, + record_get_stage_duration_if_enabled, }; use crate::erasure::coding::BitrotReader; use crate::io_support::bitrot::{ @@ -60,6 +61,7 @@ use super::core::io_primitives::*; impl SetDisks { async fn get_object_metadata_cache_bypass_reason( + &self, bucket: &str, opts: &ObjectOptions, read_data: bool, @@ -67,7 +69,8 @@ impl SetDisks { if let Some(reason) = get_object_metadata_cache_request_bypass_reason(bucket, opts, read_data) { return Some(reason); } - runtime_sources::setup_is_dist_erasure() + self.ctx + .is_dist_erasure() .await .then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE) } @@ -80,11 +83,28 @@ impl SetDisks { } async fn lookup_cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> MetadataCacheLookup { - let key = GetObjectMetadataCacheKey::new(bucket, object); + self.lookup_cached_get_object_fileinfo_after_get(bucket, object, || {}).await + } + + async fn lookup_cached_get_object_fileinfo_after_get( + &self, + bucket: &str, + object: &str, + after_get: impl FnOnce(), + ) -> MetadataCacheLookup { + let Some(generation) = self.get_object_metadata_cache_generation(bucket, object) else { + return MetadataCacheLookup::Miss; + }; + let key = GetObjectMetadataCacheKey::new(bucket, object, generation); // moka handles TTL expiry automatically; no is_fresh() check needed let Some(entry) = self.get_object_metadata_cache.get(&key).await else { return MetadataCacheLookup::Miss; }; + after_get(); + if !self.is_get_object_metadata_cache_generation_current(generation) { + self.get_object_metadata_cache.invalidate(&key).await; + return MetadataCacheLookup::Miss; + } if entry.online_disks.iter().filter(|disk| disk.is_some()).count() >= entry.read_quorum { MetadataCacheLookup::Hit(entry) } else { @@ -94,8 +114,8 @@ impl SetDisks { async fn cache_get_object_fileinfo( &self, - bucket: &str, - object: &str, + identity: (&str, &str), + generation: Option, fi: &FileInfo, parts_metadata: &[FileInfo], online_disks: &[Option], @@ -104,23 +124,51 @@ impl SetDisks { if fi.deleted || !fi.is_valid() { return; } + let (bucket, object) = identity; - let key = GetObjectMetadataCacheKey::new(bucket, object); - // moka handles capacity eviction (LRU) automatically - self.get_object_metadata_cache - .insert( - key, - Arc::new(GetObjectMetadataCacheEntry { - created_at: Instant::now(), - fi: fi.clone(), - parts_metadata: parts_metadata.to_vec(), - online_disks: online_disks.to_vec(), - read_quorum, - }), - ) + let Some(generation) = generation.filter(|generation| self.is_get_object_metadata_cache_generation_current(*generation)) + else { + rustfs_io_metrics::record_get_object_metadata_cache_decision( + GET_OBJECT_PATH_SET_DISK, + GET_METADATA_CACHE_DECISION_REJECT, + GET_METADATA_CACHE_REASON_STALE_PUBLICATION, + ); + return; + }; + + let key = GetObjectMetadataCacheKey::new(bucket, object, generation); + let entry = Arc::new(GetObjectMetadataCacheEntry { + created_at: Instant::now(), + fi: fi.clone(), + parts_metadata: parts_metadata.to_vec(), + online_disks: online_disks.to_vec(), + read_quorum, + }); + self.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {}) .await; } + async fn insert_get_object_metadata_cache_entry_after_insert( + &self, + key: GetObjectMetadataCacheKey, + generation: GetObjectMetadataCacheGeneration, + entry: Arc, + after_insert: impl FnOnce(), + ) { + self.get_object_metadata_cache.insert(key.clone(), entry).await; + after_insert(); + if self.is_get_object_metadata_cache_generation_current(generation) { + return; + } + + self.get_object_metadata_cache.invalidate(&key).await; + rustfs_io_metrics::record_get_object_metadata_cache_decision( + GET_OBJECT_PATH_SET_DISK, + GET_METADATA_CACHE_DECISION_REJECT, + GET_METADATA_CACHE_REASON_STALE_PUBLICATION, + ); + } + #[cfg_attr(feature = "hotpath", hotpath::measure)] pub async fn read_version_optimized( &self, @@ -193,7 +241,7 @@ impl SetDisks { let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); let metadata_cache_lookup_start = get_stage_timer_if_enabled(stage_metrics_enabled); - let cache_bypass_reason = Self::get_object_metadata_cache_bypass_reason(bucket, opts, read_data).await; + let cache_bypass_reason = self.get_object_metadata_cache_bypass_reason(bucket, opts, read_data).await; let use_metadata_cache = cache_bypass_reason.is_none(); if let Some(reason) = cache_bypass_reason { rustfs_io_metrics::record_get_object_metadata_cache_decision( @@ -238,6 +286,10 @@ impl SetDisks { metadata_cache_lookup_start, ); + let metadata_cache_generation = use_metadata_cache + .then(|| self.get_object_metadata_cache_generation(bucket, object)) + .flatten(); + let disks = self.disks.read().await; let disks = disks.clone(); @@ -311,8 +363,17 @@ impl SetDisks { ) .await; } else if use_metadata_cache && metadata_fanout_complete { - self.cache_get_object_fileinfo(bucket, object, &fi, &parts_metadata, &op_online_disks, read_quorum) - .await; + #[cfg(test)] + metadata_cache_tests::wait_before_metadata_cache_publish(bucket, object).await; + self.cache_get_object_fileinfo( + (bucket, object), + metadata_cache_generation, + &fi, + &parts_metadata, + &op_online_disks, + read_quorum, + ) + .await; } record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_METADATA_RESOLVE, metadata_resolve_stage_start); // debug!("get_object_fileinfo pick fi {:?}", &fi); @@ -1738,13 +1799,81 @@ mod metadata_cache_tests { use super::*; use rustfs_common::heal_channel::HealAdmissionDropReason; use serial_test::serial; - use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Mutex, OnceLock}; + use tokio::sync::Notify; static SLOW_READ_REPAIR_SUBMITTER_CALLS: AtomicUsize = AtomicUsize::new(0); static DROPPED_READ_REPAIR_SUBMITTER_CALLS: AtomicUsize = AtomicUsize::new(0); static CAPTURED_READ_REPAIR_PRIORITY: Mutex> = Mutex::new(None); static CAPTURED_READ_REPAIR_CALLS: AtomicUsize = AtomicUsize::new(0); + static METADATA_CACHE_PUBLISH_BARRIER: OnceLock>>> = OnceLock::new(); + + struct MetadataCachePublishBarrierState { + bucket: String, + object: String, + arrived: Notify, + release: Notify, + } + + struct MetadataCachePublishBarrier { + state: Arc, + } + + impl MetadataCachePublishBarrier { + fn install(bucket: &str, object: &str) -> Self { + let state = Arc::new(MetadataCachePublishBarrierState { + bucket: bucket.to_string(), + object: object.to_string(), + arrived: Notify::new(), + release: Notify::new(), + }); + let mut slot = METADATA_CACHE_PUBLISH_BARRIER + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("metadata publish barrier mutex should not poison"); + assert!(slot.is_none(), "metadata publish barrier must be installed by one test at a time"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + async fn wait_until_paused(&self) { + tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) + .await + .expect("metadata publication should reach the deterministic barrier"); + } + + fn release(&self) { + self.state.release.notify_one(); + } + } + + impl Drop for MetadataCachePublishBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + let mut slot = METADATA_CACHE_PUBLISH_BARRIER + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("metadata publish barrier mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } + } + + pub(super) async fn wait_before_metadata_cache_publish(bucket: &str, object: &str) { + let barrier = METADATA_CACHE_PUBLISH_BARRIER + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("metadata publish barrier mutex should not poison") + .as_ref() + .filter(|state| state.bucket == bucket && state.object == object) + .cloned(); + if let Some(barrier) = barrier { + barrier.arrived.notify_one(); + barrier.release.notified().await; + } + } fn slow_read_repair_submitter(_request: rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture { SLOW_READ_REPAIR_SUBMITTER_CALLS.fetch_add(1, Ordering::Relaxed); @@ -1771,7 +1900,11 @@ mod metadata_cache_tests { } async fn new_metadata_cache_test_set() -> Arc { - SetDisks::new( + new_metadata_cache_test_set_with_ctx(Arc::new(crate::runtime::instance::InstanceContext::new())).await + } + + async fn new_metadata_cache_test_set_with_ctx(ctx: Arc) -> Arc { + SetDisks::new_with_instance_ctx( "metadata-cache-test".to_string(), Arc::new(RwLock::new(Vec::new())), 4, @@ -1781,6 +1914,7 @@ mod metadata_cache_tests { Vec::new(), FormatV3::new(1, 4), Vec::new(), + ctx, ) .await } @@ -2334,7 +2468,8 @@ mod metadata_cache_tests { let parts_metadata = vec![fi.clone()]; let online_disks = Vec::new(); - set.cache_get_object_fileinfo("bucket", "object", &fi, &parts_metadata, &online_disks, 0) + let generation = set.get_object_metadata_cache_generation("bucket", "object"); + set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, &parts_metadata, &online_disks, 0) .await; let cached = set @@ -2353,7 +2488,8 @@ mod metadata_cache_tests { let mut deleted = valid_test_fileinfo("deleted-object"); deleted.deleted = true; - set.cache_get_object_fileinfo("bucket", "deleted-object", &deleted, &[deleted.clone()], &[], 0) + let generation = set.get_object_metadata_cache_generation("bucket", "deleted-object"); + set.cache_get_object_fileinfo(("bucket", "deleted-object"), generation, &deleted, &[deleted.clone()], &[], 0) .await; assert!( set.cached_get_object_fileinfo("bucket", "deleted-object").await.is_none(), @@ -2361,7 +2497,8 @@ mod metadata_cache_tests { ); let invalid = FileInfo::default(); - set.cache_get_object_fileinfo("bucket", "invalid-object", &invalid, std::slice::from_ref(&invalid), &[], 0) + let generation = set.get_object_metadata_cache_generation("bucket", "invalid-object"); + set.cache_get_object_fileinfo(("bucket", "invalid-object"), generation, &invalid, std::slice::from_ref(&invalid), &[], 0) .await; assert!( set.cached_get_object_fileinfo("bucket", "invalid-object").await.is_none(), @@ -2376,7 +2513,12 @@ mod metadata_cache_tests { set.get_object_metadata_cache .insert( - GetObjectMetadataCacheKey::new("bucket", "object"), + GetObjectMetadataCacheKey::new( + "bucket", + "object", + set.get_object_metadata_cache_generation("bucket", "object") + .expect("metadata cache generation should be active"), + ), Arc::new(GetObjectMetadataCacheEntry { created_at: Instant::now(), fi: fi.clone(), @@ -2401,7 +2543,8 @@ mod metadata_cache_tests { let set = new_metadata_cache_test_set().await; let fi = valid_test_fileinfo("object"); - set.cache_get_object_fileinfo("bucket", "object", &fi, std::slice::from_ref(&fi), &[], 0) + let generation = set.get_object_metadata_cache_generation("bucket", "object"); + set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, std::slice::from_ref(&fi), &[], 0) .await; assert!( @@ -2415,7 +2558,8 @@ mod metadata_cache_tests { let set = new_metadata_cache_test_set().await; let fi = valid_test_fileinfo("object"); - set.cache_get_object_fileinfo("bucket", "object", &fi, std::slice::from_ref(&fi), &[], 0) + let generation = set.get_object_metadata_cache_generation("bucket", "object"); + set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, std::slice::from_ref(&fi), &[], 0) .await; assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_some()); @@ -2426,6 +2570,193 @@ mod metadata_cache_tests { ); } + #[tokio::test] + #[serial(metadata_cache_publish_barrier)] + async fn metadata_cache_production_fanout_cannot_publish_after_invalidation() { + let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks(4, 2).await; + let bucket = "metadata-cache-production-fence"; + let object = "object"; + let disks = set.disks.read().await.clone(); + for disk in disks.iter().flatten() { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let mut reader = PutObjReader::from_vec(vec![7u8; 1024]); + set.put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("test object should be written"); + + let barrier = MetadataCachePublishBarrier::install(bucket, object); + let stale_generation = set + .get_object_metadata_cache_generation(bucket, object) + .expect("metadata cache generation should be active"); + let reader_set = Arc::clone(&set); + let read = tokio::spawn(async move { + reader_set + .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false) + .await + }); + barrier.wait_until_paused().await; + set.invalidate_get_object_metadata_cache(bucket, object).await; + barrier.release(); + read.await + .expect("metadata read task should not panic") + .expect("metadata fanout should still return its selected FileInfo"); + + assert!( + set.get_object_metadata_cache + .get(&GetObjectMetadataCacheKey::new(bucket, object, stale_generation)) + .await + .is_none(), + "production stale publication must not remain under its retired generation key" + ); + assert!( + set.cached_get_object_fileinfo(bucket, object).await.is_none(), + "the production fanout token captured before invalidation must not publish afterward" + ); + } + + #[tokio::test] + async fn metadata_cache_lookup_rechecks_generation_after_get() { + let set = new_metadata_cache_test_set().await; + let fi = valid_test_fileinfo("object"); + let generation = set.get_object_metadata_cache_generation("bucket", "object"); + let generation_index = generation.expect("metadata cache generation should be active").index; + set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, std::slice::from_ref(&fi), &[], 0) + .await; + + let lookup = set + .lookup_cached_get_object_fileinfo_after_get("bucket", "object", || { + set.get_object_metadata_cache_generations[generation_index].fetch_add(1, Ordering::AcqRel); + }) + .await; + + assert!( + matches!(lookup, MetadataCacheLookup::Miss), + "a hit overlapping invalidation must be rejected after the second generation read" + ); + } + + #[tokio::test] + async fn metadata_cache_publication_rechecks_generation_after_insert() { + let set = new_metadata_cache_test_set().await; + let fi = valid_test_fileinfo("object"); + let generation = set + .get_object_metadata_cache_generation("bucket", "object") + .expect("metadata cache generation should be active"); + let generation_index = generation.index; + let key = GetObjectMetadataCacheKey::new("bucket", "object", generation); + let entry = Arc::new(GetObjectMetadataCacheEntry { + created_at: Instant::now(), + fi: fi.clone(), + parts_metadata: vec![fi], + online_disks: Vec::new(), + read_quorum: 0, + }); + + set.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || { + set.get_object_metadata_cache_generations[generation_index].fetch_add(1, Ordering::AcqRel); + }) + .await; + assert!( + set.get_object_metadata_cache + .get(&GetObjectMetadataCacheKey::new("bucket", "object", generation)) + .await + .is_none(), + "post-insert generation change must withdraw the retired entry" + ); + assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none()); + } + + #[tokio::test] + async fn metadata_cache_generation_advances_monotonically() { + let set = new_metadata_cache_test_set().await; + let fi = valid_test_fileinfo("object"); + let initial = set + .get_object_metadata_cache_generation("bucket", "object") + .expect("initial generation should be active"); + + set.invalidate_get_object_metadata_cache("bucket", "object").await; + let first = set + .get_object_metadata_cache_generation("bucket", "object") + .expect("first generation should be active"); + set.invalidate_get_object_metadata_cache("bucket", "object").await; + let second = set + .get_object_metadata_cache_generation("bucket", "object") + .expect("second generation should be active"); + + assert_eq!(first.value, initial.value + 1); + assert_eq!(second.value, first.value + 1); + set.cache_get_object_fileinfo(("bucket", "object"), Some(first), &fi, std::slice::from_ref(&fi), &[], 0) + .await; + assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none()); + } + + #[tokio::test] + async fn metadata_cache_distribution_bypass_uses_set_instance_context() { + let distributed_ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + distributed_ctx + .update_erasure_type(crate::layout::endpoints::SetupType::DistErasure) + .await; + let standalone_ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + standalone_ctx + .update_erasure_type(crate::layout::endpoints::SetupType::Erasure) + .await; + let distributed = new_metadata_cache_test_set_with_ctx(distributed_ctx).await; + let standalone = new_metadata_cache_test_set_with_ctx(standalone_ctx).await; + + assert_eq!( + distributed + .get_object_metadata_cache_bypass_reason("bucket", &ObjectOptions::default(), true) + .await, + Some(GET_METADATA_CACHE_REASON_DIST_ERASURE) + ); + assert_eq!( + standalone + .get_object_metadata_cache_bypass_reason("bucket", &ObjectOptions::default(), true) + .await, + None + ); + } + + #[tokio::test] + async fn metadata_cache_generation_overflow_fails_closed() { + let set = new_metadata_cache_test_set().await; + let fi = valid_test_fileinfo("object"); + let generation_index = set + .get_object_metadata_cache_generation("bucket", "object") + .expect("metadata cache generation should be active") + .index; + set.get_object_metadata_cache_generations[generation_index].store(u64::MAX - 1, Ordering::Release); + let last_generation = set.get_object_metadata_cache_generation("bucket", "object"); + assert_eq!(last_generation.map(|generation| generation.value), Some(u64::MAX - 1)); + + set.invalidate_get_object_metadata_cache("bucket", "object").await; + + assert_eq!(set.get_object_metadata_cache_generation("bucket", "object"), None); + set.cache_get_object_fileinfo(("bucket", "object"), last_generation, &fi, std::slice::from_ref(&fi), &[], 0) + .await; + assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none()); + set.invalidate_get_object_metadata_cache("bucket", "object").await; + assert_eq!( + set.get_object_metadata_cache_generations[generation_index].load(Ordering::Acquire), + u64::MAX, + "overflow must permanently disable publication instead of wrapping" + ); + } + + #[tokio::test] + async fn metadata_cache_invalidate_all_fences_late_insert() { + let set = new_metadata_cache_test_set().await; + let fi = valid_test_fileinfo("object"); + let stale_generation = set.get_object_metadata_cache_generation("bucket", "object"); + + set.invalidate_all_get_object_metadata_cache(); + + set.cache_get_object_fileinfo(("bucket", "object"), stale_generation, &fi, std::slice::from_ref(&fi), &[], 0) + .await; + assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none()); + } + #[tokio::test] async fn get_object_metadata_cache_prunes_when_capacity_is_reached() { // moka handles capacity eviction automatically via the configured max_capacity. @@ -2433,7 +2764,8 @@ mod metadata_cache_tests { let set = new_metadata_cache_test_set().await; let fresh_fi = valid_test_fileinfo("fresh-object"); - set.cache_get_object_fileinfo("bucket", "fresh-object", &fresh_fi, std::slice::from_ref(&fresh_fi), &[], 0) + let generation = set.get_object_metadata_cache_generation("bucket", "fresh-object"); + set.cache_get_object_fileinfo(("bucket", "fresh-object"), generation, &fresh_fi, std::slice::from_ref(&fresh_fi), &[], 0) .await; assert!( diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 181930128..4c00a6420 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -158,6 +158,7 @@ mod list; pub(crate) mod list_objects; mod multipart; mod object; +pub use object::PreparedGetObjectReader; mod peer; mod rebalance; pub(crate) mod utils; diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index d9bd0beed..8af36ea9c 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -31,6 +31,61 @@ use std::{ }; use tokio::io::{AsyncRead, ReadBuf}; +/// A GET whose object identity has been resolved while its namespace read lock +/// remains held, but whose body reader has not been constructed yet. +/// +/// The application can evaluate request preconditions and cache coordination +/// against [`Self::object_info`] before consuming this value. Dropping it +/// releases the read lock without constructing a body reader. +pub struct PreparedGetObjectReader { + pool: Arc, + bucket: String, + object: String, + range: Option, + headers: HeaderMap, + opts: ObjectOptions, + metadata: crate::set_disk::PreparedGetObjectMetadata, + read_lock_guard: Option, +} + +impl PreparedGetObjectReader { + /// Returns the fresh metadata snapshot protected by this prepared read. + pub fn object_info(&self) -> &ObjectInfo { + self.metadata.object_info() + } + + /// Finishes a metadata-only decision without constructing a body reader. + pub fn into_object_info(mut self) -> ObjectInfo { + self.metadata.take_object_info() + } + + /// Replaces the headers used when this prepared value constructs its body reader. + #[must_use] + pub fn with_headers(mut self, headers: HeaderMap) -> Self { + self.headers = headers; + self + } + + /// Constructs the body reader while retaining the metadata snapshot's read + /// lock, then transfers that lock to the returned stream as usual. The + /// staged caller already performed the authoritative app-layer cache probe, + /// so the nested set-disk reader must not probe the same hook again. + pub async fn into_reader(self) -> Result { + let mut reader = + crate::object_api::without_get_object_body_cache_hook(self.pool.get_object_reader_with_prepared_metadata( + &self.bucket, + &self.object, + self.range, + self.headers, + &self.opts, + self.metadata, + )) + .await?; + reader.body_source = crate::object_api::GetObjectBodySource::HookMissed; + Ok(ECStore::attach_read_lock_guard(reader, self.read_lock_guard)) + } +} + struct LockGuardedReader { inner: Box, guard: Option, @@ -394,6 +449,56 @@ fn sorted_unique_delete_object_names(objects: &[ObjectToDelete]) -> Vec<&str> { } impl ECStore { + /// Resolves a GET's object identity without constructing its body reader. + /// + /// This is an additive two-stage counterpart to `get_object_reader`. The + /// existing method remains the compatibility path for callers that do not + /// need a pre-reader decision point. + pub async fn prepare_get_object_reader( + &self, + bucket: &str, + object: &str, + range: Option, + headers: HeaderMap, + opts: &ObjectOptions, + ) -> Result { + check_get_obj_args(bucket, object)?; + + let object = encode_dir_object(object); + let mut opts = opts.clone(); + let read_lock_guard = self + .acquire_object_read_lock_if_needed("prepare_get_object", bucket, &object, &mut opts) + .await?; + + let (metadata, pool) = if self.single_pool() { + let pool = Arc::clone(&self.pools[0]); + let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?; + (metadata, pool) + } else { + let (_, pool_idx) = self + .get_latest_accessible_object_info_with_idx(bucket, &object, &opts) + .await?; + let pool = self + .pools + .get(pool_idx) + .cloned() + .ok_or_else(|| Error::other(format!("resolved GET pool index {pool_idx} is out of bounds")))?; + let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?; + (metadata, pool) + }; + + Ok(PreparedGetObjectReader { + pool, + bucket: bucket.to_owned(), + object, + range, + headers, + opts, + metadata, + read_lock_guard, + }) + } + fn map_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err: rustfs_lock::LockError) -> StorageError { match err { rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable { @@ -1484,15 +1589,43 @@ mod tests { ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_state_to_filemeta, replication_statuses_map, version_purge_statuses_map, }; + use crate::ecstore_validation_blackbox::make_local_set_disks; use crate::layout::{ endpoints::{Endpoints, PoolEndpoints}, format::FormatV3, }; + use crate::object_api::{ + GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource, clear_get_object_body_cache_hook, + lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, + }; + use crate::set_disk::SetDisks; + use crate::storage_api_contracts::bucket::MakeBucketOptions; use bytes::Bytes; use std::io::Cursor; use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::io::AsyncReadExt; + struct CountingMissHook { + calls: AtomicUsize, + } + + #[async_trait::async_trait] + impl GetObjectBodyCacheHook for CountingMissHook { + async fn lookup(&self, _bucket: &str, _object: &str, _info: &ObjectInfo) -> Option { + self.calls.fetch_add(1, Ordering::Relaxed); + None + } + } + + struct BodyCacheHookGuard; + + impl Drop for BodyCacheHookGuard { + fn drop(&mut self) { + clear_get_object_body_cache_hook(); + } + } + #[test] fn delete_marker_data_movement_falls_back_when_only_source_pool_has_object() { let target = select_data_movement_target_pool(Ok(1), 1, true).unwrap(); @@ -2095,6 +2228,244 @@ mod tests { } } + async fn new_prepared_reader_test_store(set_disks: &[Arc]) -> ECStore { + let mut pool_configs = Vec::with_capacity(set_disks.len()); + let mut pools = Vec::with_capacity(set_disks.len()); + + for (pool_idx, set_disks) in set_disks.iter().enumerate() { + let mut endpoints = Endpoints::from(set_disks.set_endpoints.clone()); + for endpoint in endpoints.as_mut() { + endpoint.set_pool_index(pool_idx); + } + let pool_config = PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: set_disks.set_drive_count, + endpoints, + cmd_line: format!("prepared-reader-test-pool-{pool_idx}"), + platform: "test".to_string(), + }; + let disks = set_disks.disks.read().await.clone(); + let pool = Sets::new(disks, &pool_config, &set_disks.format, pool_idx, set_disks.default_parity_count) + .await + .expect("prepared-reader test pool should be created from local disks"); + pool_configs.push(pool_config); + pools.push(pool); + } + + let endpoint_pools = EndpointServerPools::from(pool_configs); + ECStore { + id: Uuid::new_v4(), + disk_map: HashMap::new(), + pools, + peer_sys: S3PeerSys::new(&endpoint_pools), + pool_meta: RwLock::new(PoolMeta::default()), + rebalance_meta: RwLock::new(None), + decommission_cancelers: RwLock::new(Vec::new()), + start_gate: Mutex::new(()), + pool_meta_save_gate: Mutex::new(()), + ctx: crate::runtime::instance::bootstrap_ctx(), + } + } + + async fn assert_prepared_reader_blocks_writer(store: &ECStore, bucket: &str, object: &str) { + let manager = Arc::clone(store.pools[0].disk_set[0].local_lock_manager_for_test()); + let lock = rustfs_lock::NamespaceLock::with_local_manager("prepared-reader-writer".to_string(), manager); + let err = lock + .get_write_lock(rustfs_lock::ObjectKey::new(bucket, object), "competing-writer", Duration::from_millis(50)) + .await + .expect_err("prepared read lock should block the writer"); + assert!(matches!(err, rustfs_lock::LockError::Timeout { .. })); + } + + async fn acquire_prepared_reader_writer(store: &ECStore, bucket: &str, object: &str) -> rustfs_lock::NamespaceLockGuard { + let manager = Arc::clone(store.pools[0].disk_set[0].local_lock_manager_for_test()); + let lock = rustfs_lock::NamespaceLock::with_local_manager("prepared-reader-writer".to_string(), manager); + lock.get_write_lock(rustfs_lock::ObjectKey::new(bucket, object), "competing-writer", Duration::from_secs(1)) + .await + .expect("prepared read lock should have been released") + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn prepared_reader_uses_authoritative_hook_miss_once_and_streams_full_body() { + let (_dirs, set_disks) = make_local_set_disks(4, 2).await; + let store = new_prepared_reader_test_store(&[set_disks]).await; + let bucket = "prepared-reader-hook-miss"; + let object = "object.bin"; + let payload = b"prepared-reader-hook-miss-payload-".repeat(40_000); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + store.pools[0] + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + store.pools[0] + .put_object(bucket, object, &mut put_reader, &opts) + .await + .expect("object should be written"); + + clear_get_object_body_cache_hook(); + let hook = Arc::new(CountingMissHook { + calls: AtomicUsize::new(0), + }); + register_get_object_body_cache_hook(Arc::clone(&hook) as Arc); + let _hook_guard = BodyCacheHookGuard; + + let prepared = store + .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts) + .await + .expect("prepared reader metadata should resolve"); + assert!(matches!( + lookup_get_object_body_cache_hook(bucket, object, &None, &opts, prepared.object_info()).await, + GetObjectBodyCacheHookLookup::Miss + )); + assert_eq!(hook.calls.load(Ordering::Relaxed), 1, "the authoritative probe should call the hook once"); + + let mut reader = prepared.into_reader().await.expect("prepared body reader should open"); + assert_eq!(reader.body_source, GetObjectBodySource::HookMissed); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("prepared body should stream"); + + assert_eq!(restored, payload); + assert_eq!(hook.calls.load(Ordering::Relaxed), 1, "reader construction must not probe the hook again"); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn prepared_reader_holds_namespace_lock_until_eof_or_drop() { + temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async { + let (_dirs, set_disks) = make_local_set_disks(4, 2).await; + let store = new_prepared_reader_test_store(&[set_disks]).await; + let bucket = "prepared-reader-lock-lifetime"; + let object = "object.bin"; + let payload = b"prepared-reader-lock-lifetime-payload-".repeat(40_000); + let put_opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + store.pools[0] + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + store.pools[0] + .put_object(bucket, object, &mut put_reader, &put_opts) + .await + .expect("object should be written"); + + let prepared = store + .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("prepared reader metadata should resolve"); + assert!(prepared.read_lock_guard.is_some()); + assert_prepared_reader_blocks_writer(&store, bucket, object).await; + + let mut reader = prepared.into_reader().await.expect("prepared body reader should open"); + assert_prepared_reader_blocks_writer(&store, bucket, object).await; + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("prepared body should stream"); + assert_eq!(restored, payload); + drop(acquire_prepared_reader_writer(&store, bucket, object).await); + + let prepared = store + .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("second prepared reader metadata should resolve"); + let reader = prepared.into_reader().await.expect("second prepared body reader should open"); + assert_prepared_reader_blocks_writer(&store, bucket, object).await; + drop(reader); + drop(acquire_prepared_reader_writer(&store, bucket, object).await); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn prepared_object_info_releases_namespace_lock_immediately() { + let (_dirs, set_disks) = make_local_set_disks(4, 2).await; + let store = new_prepared_reader_test_store(&[set_disks]).await; + let bucket = "prepared-object-info-lock-release"; + let object = "object.bin"; + let payload = b"prepared-object-info-lock-release".to_vec(); + let put_opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + store.pools[0] + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + store.pools[0] + .put_object(bucket, object, &mut put_reader, &put_opts) + .await + .expect("object should be written"); + + let prepared = store + .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("prepared reader metadata should resolve"); + assert_prepared_reader_blocks_writer(&store, bucket, object).await; + assert_eq!(prepared.into_object_info().size, payload.len() as i64); + + drop(acquire_prepared_reader_writer(&store, bucket, object).await); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn prepared_reader_resolves_object_from_second_pool() { + let (_first_dirs, first_set) = make_local_set_disks(4, 2).await; + let (_second_dirs, second_set) = make_local_set_disks(4, 2).await; + let store = new_prepared_reader_test_store(&[first_set, second_set]).await; + let bucket = "prepared-reader-second-pool"; + let object = "object.bin"; + let payload = b"prepared-reader-second-pool-payload-".repeat(40_000); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + for pool in &store.pools { + pool.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created in each pool"); + } + let mut put_reader = PutObjReader::from_vec(payload.clone()); + store.pools[1] + .put_object(bucket, object, &mut put_reader, &opts) + .await + .expect("object should be written only to the second pool"); + + let prepared = store + .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts) + .await + .expect("prepared reader should resolve the second-pool object"); + assert_eq!(prepared.object_info().size, payload.len() as i64); + let mut reader = prepared.into_reader().await.expect("prepared body reader should open"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("prepared body should stream"); + assert_eq!(restored, payload); + } + // Phase 5 Slice 2 (backlog#939): the instance context flows down the whole // object graph — ECStore, its Sets, and their SetDisks must all carry the // same `Arc` in a single-instance deployment. diff --git a/crates/object-data-cache/src/cache.rs b/crates/object-data-cache/src/cache.rs index 77a7df06c..96ce39bc4 100644 --- a/crates/object-data-cache/src/cache.rs +++ b/crates/object-data-cache/src/cache.rs @@ -14,8 +14,10 @@ use crate::backend::ObjectDataCacheBackendKind; use crate::config::ObjectDataCacheConfig; +use crate::entry::projected_weight; use crate::error::ObjectDataCacheConfigError; use crate::key::{ObjectDataCacheBodyVariant, ObjectDataCacheIdentity, ObjectDataCacheKey}; +use crate::memory::ObjectDataCacheMemoryReservation; use crate::metrics::{ describe_metrics_once, publish_cache_state, record_fill_result, record_hit_bytes, record_invalidation, record_lookup_result, record_plan_decision, @@ -27,6 +29,52 @@ use bytes::Bytes; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Instant; +use tokio::sync::OwnedSemaphorePermit; + +/// Admission token for one body allocation performed before a cold cache fill. +#[derive(Debug)] +pub struct ObjectDataCacheBodyReservation { + pub(crate) memory: ObjectDataCacheMemoryReservation, + pub(crate) permit: OwnedSemaphorePermit, + pub(crate) fill_generation: crate::moka_backend::FillGenerationGuard, + pub(crate) key: ObjectDataCacheKey, + pub(crate) expected_size: u64, +} + +/// A materialized body whose allocation owns its memory claim until the last +/// `Bytes` clone is dropped. +#[derive(Debug)] +pub struct ObjectDataCacheReservedBody { + pub(crate) bytes: Bytes, + pub(crate) permit: OwnedSemaphorePermit, + pub(crate) fill_generation: crate::moka_backend::FillGenerationGuard, + pub(crate) key: ObjectDataCacheKey, + pub(crate) expected_size: u64, +} + +impl ObjectDataCacheBodyReservation { + /// Attaches this reservation to a newly materialized body. + pub fn wrap_bytes(self, bytes: Bytes) -> ObjectDataCacheReservedBody { + ObjectDataCacheReservedBody { + bytes: self.memory.wrap_bytes(bytes), + permit: self.permit, + fill_generation: self.fill_generation, + key: self.key, + expected_size: self.expected_size, + } + } +} + +impl ObjectDataCacheReservedBody { + /// Returns a clone that shares the reservation-owning allocation. + pub fn bytes(&self) -> Bytes { + self.bytes.clone() + } + + pub(crate) fn into_parts(self) -> (Bytes, OwnedSemaphorePermit, crate::moka_backend::FillGenerationGuard) { + (self.bytes, self.permit, self.fill_generation) + } +} /// Minimum spacing between cache-state gauge publishes. Moka's `entry_count` /// and `weighted_size` are cross-segment approximations that only settle after @@ -45,6 +93,9 @@ pub struct ObjectDataCache { backend: ObjectDataCacheBackendKind, config: Arc, stats: Arc, + /// Resolved Moka weighted capacity used by the planner's exact key-aware + /// admission check. Zero for the disabled backend. + max_capacity: u64, /// Effective fill ceiling in bytes: a body larger than this is planned /// `SkipTooLarge` even when it fits `max_entry_bytes`. The app layer sets it /// to `min(max_entry_bytes, seek-support threshold, 64 MiB buffer cap)` so a @@ -72,6 +123,7 @@ impl ObjectDataCache { backend: ObjectDataCacheBackendKind::Noop(NoopBackend), config, stats, + max_capacity: 0, fill_ceiling_bytes: 0, created_at: Instant::now(), last_entry_publish_ms: AtomicU64::new(0), @@ -83,16 +135,19 @@ impl ObjectDataCache { describe_metrics_once(); config.validate()?; let stats = Arc::new(ObjectDataCacheStats::default()); - let backend = if config.is_disabled() { - ObjectDataCacheBackendKind::Noop(NoopBackend) + let (backend, max_capacity) = if config.is_disabled() { + (ObjectDataCacheBackendKind::Noop(NoopBackend), 0) } else { - ObjectDataCacheBackendKind::Moka(Box::new(MokaBackend::new(&config, Arc::clone(&stats))?)) + let backend = MokaBackend::new(&config, Arc::clone(&stats))?; + let max_capacity = backend.max_capacity(); + (ObjectDataCacheBackendKind::Moka(Box::new(backend)), max_capacity) }; Ok(Self { backend, config: Arc::new(config), stats, + max_capacity, fill_ceiling_bytes: 0, created_at: Instant::now(), last_entry_publish_ms: AtomicU64::new(0), @@ -121,14 +176,26 @@ impl ObjectDataCache { /// Produces a lightweight GET plan from request metadata. pub fn plan_get(&self, request: ObjectDataCacheGetRequest<'_>) -> ObjectDataCacheGetPlan { + self.plan_get_inner(request, true) + } + + /// Rebuilds a plan for identity revalidation without counting another GET. + #[doc(hidden)] + pub fn plan_get_untracked(&self, request: ObjectDataCacheGetRequest<'_>) -> ObjectDataCacheGetPlan { + self.plan_get_inner(request, false) + } + + fn plan_get_inner(&self, request: ObjectDataCacheGetRequest<'_>, record_metric: bool) -> ObjectDataCacheGetPlan { if self.config.is_disabled() { - record_plan_decision( - self.backend.as_metric_label(), - self.config.mode, - "disabled", - "mode_disabled", - request.size, - ); + if record_metric { + record_plan_decision( + self.backend.as_metric_label(), + self.config.mode, + "disabled", + "mode_disabled", + request.size, + ); + } return ObjectDataCacheGetPlan::Disabled; } @@ -137,24 +204,34 @@ impl ObjectDataCache { // in-memory GET fill limits could never fill, so admitting it here would // report it "eligible" while it kept a permanent 0% hit rate. if request.size > self.effective_size_ceiling() { - record_plan_decision(self.backend.as_metric_label(), self.config.mode, "skip", "too_large", request.size); + if record_metric { + record_plan_decision(self.backend.as_metric_label(), self.config.mode, "skip", "too_large", request.size); + } return ObjectDataCacheGetPlan::SkipTooLarge; } - record_plan_decision(self.backend.as_metric_label(), self.config.mode, "cacheable", "eligible", request.size); - - ObjectDataCacheGetPlan::Cacheable { - key: ObjectDataCacheKey::with_write_anchors( - request.bucket, - request.object, - request.version_id.as_deref(), - request.etag, - request.size, - request.data_dir_u128, - request.mod_time_unix_nanos, - request.body_variant, - ), + let key = ObjectDataCacheKey::with_write_anchors( + request.bucket, + request.object, + request.version_id.as_deref(), + request.etag, + request.size, + request.data_dir_u128, + request.mod_time_unix_nanos, + request.body_variant, + ); + if projected_weight(&key, request.size) > self.max_capacity { + if record_metric { + record_plan_decision(self.backend.as_metric_label(), self.config.mode, "skip", "too_large", request.size); + } + return ObjectDataCacheGetPlan::SkipTooLarge; } + + if record_metric { + record_plan_decision(self.backend.as_metric_label(), self.config.mode, "cacheable", "eligible", request.size); + } + + ObjectDataCacheGetPlan::Cacheable { key } } /// Looks up an object body from the configured backend. @@ -192,6 +269,61 @@ impl ObjectDataCache { lookup } + /// Performs an internal second-chance lookup without recording another + /// request lookup. Callers must have already performed the authoritative + /// lookup for the current GET. + #[doc(hidden)] + pub async fn peek_body_untracked(&self, plan: &ObjectDataCacheGetPlan) -> ObjectDataCacheLookup { + match &self.backend { + ObjectDataCacheBackendKind::Noop(backend) => backend.lookup_body(plan).await, + ObjectDataCacheBackendKind::Moka(backend) => backend.lookup_body(plan).await, + } + } + + /// Reserves memory and a fill slot before allocating a cold-fill body. + pub fn reserve_body(&self, plan: &ObjectDataCacheGetPlan) -> Option { + if !self.config.fill_enabled() { + return None; + } + match &self.backend { + ObjectDataCacheBackendKind::Noop(_) => None, + ObjectDataCacheBackendKind::Moka(backend) => backend.reserve_body(plan), + } + } + + /// Fills from a body admitted before allocation. + pub async fn fill_reserved_body( + &self, + plan: &ObjectDataCacheGetPlan, + body: ObjectDataCacheReservedBody, + ) -> ObjectDataCacheFillResult { + let fill_bytes = u64::try_from(body.bytes.len()).unwrap_or(u64::MAX); + let fill_start = Instant::now(); + let result = match &self.backend { + ObjectDataCacheBackendKind::Noop(_) => ObjectDataCacheFillResult::SkippedDisabled, + ObjectDataCacheBackendKind::Moka(backend) => backend.fill_reserved_body(plan, body).await, + }; + let (recorded_bytes, duration) = match result { + ObjectDataCacheFillResult::Inserted => { + self.stats.record_fill(); + self.refresh_entry_count(); + (fill_bytes, Some(fill_start.elapsed().as_secs_f64())) + } + ObjectDataCacheFillResult::SkippedInvalidationRace | ObjectDataCacheFillResult::SkippedIdentityOverflow => { + (fill_bytes, Some(fill_start.elapsed().as_secs_f64())) + } + _ => (0, None), + }; + record_fill_result( + self.backend.as_metric_label(), + self.config.mode, + result.as_metric_label(), + recorded_bytes, + duration, + ); + result + } + /// Attempts to fill the cache body for the current plan. pub async fn fill_body(&self, plan: &ObjectDataCacheGetPlan, bytes: Bytes) -> ObjectDataCacheFillResult { let fill_bytes = u64::try_from(bytes.len()).unwrap_or(u64::MAX); @@ -434,6 +566,16 @@ pub enum ObjectDataCacheGetPlan { }, } +impl ObjectDataCacheGetPlan { + /// Returns the stable cache key for a cacheable plan. + pub fn key(&self) -> Option<&ObjectDataCacheKey> { + match self { + Self::Cacheable { key } => Some(key), + Self::Disabled | Self::SkipTooLarge => None, + } + } +} + /// Result of a cache lookup attempt. #[derive(Debug, Clone, PartialEq, Eq)] pub enum ObjectDataCacheLookup { @@ -863,6 +1005,80 @@ mod tests { assert!(matches!(lookup, ObjectDataCacheLookup::Miss)); } + #[tokio::test] + async fn reserved_body_is_bound_to_origin_cache_and_plan() { + let cache_a = fill_enabled_cache(); + let cache_b = fill_enabled_cache(); + let plan = cache_a.plan_get(plain_request("bucket", "object", "etag", 5)); + let wrong_plan = cache_a.plan_get(plain_request("bucket", "other", "etag", 5)); + + let wrong_plan_body = cache_a + .reserve_body(&plan) + .expect("the original plan should be admitted") + .wrap_bytes(Bytes::from_static(b"hello")); + assert_eq!( + cache_a.fill_reserved_body(&wrong_plan, wrong_plan_body).await, + ObjectDataCacheFillResult::SkippedNotCacheable + ); + + let cross_cache_body = cache_a + .reserve_body(&plan) + .expect("the original cache should admit another reservation") + .wrap_bytes(Bytes::from_static(b"hello")); + assert_eq!( + cache_b.fill_reserved_body(&plan, cross_cache_body).await, + ObjectDataCacheFillResult::SkippedNotCacheable + ); + + let wrong_size_body = cache_a + .reserve_body(&plan) + .expect("the original plan should still be admitted") + .wrap_bytes(Bytes::from_static(b"oops")); + assert_eq!( + cache_a.fill_reserved_body(&plan, wrong_size_body).await, + ObjectDataCacheFillResult::SkippedSizeMismatch + ); + assert_eq!(cache_a.stats().fills, 0, "rejected reservations must not count as successful fills"); + assert!(matches!(cache_a.lookup_body(&plan).await, ObjectDataCacheLookup::Miss)); + assert!(matches!(cache_a.lookup_body(&wrong_plan).await, ObjectDataCacheLookup::Miss)); + assert!(matches!(cache_b.lookup_body(&plan).await, ObjectDataCacheLookup::Miss)); + + let body = cache_a + .reserve_body(&plan) + .expect("the matching reservation should be admitted") + .wrap_bytes(Bytes::from_static(b"hello")); + assert_eq!(cache_a.fill_reserved_body(&plan, body).await, ObjectDataCacheFillResult::Inserted); + assert_eq!(cache_a.stats().fills, 1); + assert_eq!(cache_a.lookup_body(&plan).await, ObjectDataCacheLookup::Hit(Bytes::from_static(b"hello"))); + } + + #[test] + fn reserved_size_mismatch_records_fill_outcome_without_bytes() { + let cache = fill_enabled_cache(); + let metrics = capture_metrics(|| async { + let plan = cache.plan_get(plain_request("bucket", "object", "etag", 5)); + let body = cache + .reserve_body(&plan) + .expect("the plan should be admitted before materialization") + .wrap_bytes(Bytes::from_static(b"oops")); + assert_eq!( + cache.fill_reserved_body(&plan, body).await, + ObjectDataCacheFillResult::SkippedSizeMismatch + ); + }); + + assert!(has_counter_with_label( + &metrics, + "rustfs_object_data_cache_fill_total", + ("result", "skipped_size_mismatch") + )); + assert_eq!( + counter_total(&metrics, "rustfs_object_data_cache_fill_bytes_total"), + None, + "a rejected reserved body must not record filled bytes" + ); + } + #[test] fn plan_clamps_size_eligibility_to_fill_ceiling() { // ODC-24 (backlog#1129): a body in the gap between `max_entry_bytes` and @@ -895,6 +1111,34 @@ mod tests { ); } + #[test] + fn planner_rejects_key_whose_projected_weight_exceeds_capacity() { + let config = ObjectDataCacheConfig { + mode: ObjectDataCacheMode::HitOnly, + max_bytes: 8 * 1024, + max_memory_percent: 0, + max_entry_bytes: 4 * 1024, + ..ObjectDataCacheConfig::default() + }; + let cache = ObjectDataCache::new(config).expect("capacity test config should initialize"); + // Body (4096) + fixed fields (bucket=1, version=null=4, etag=1) + + // object (4026) + entry overhead (64) = 8192, exactly capacity. + let boundary_object = "o".repeat(4026); + assert!(matches!( + cache.plan_get(plain_request("b", &boundary_object, "e", 4 * 1024)), + ObjectDataCacheGetPlan::Cacheable { .. } + )); + + // Body (4096) + fixed fields (bucket=1, version=null=4, etag=1) + + // object (4027) + entry overhead (64) = 8193, one byte over capacity. + let overweight_object = "o".repeat(4027); + + assert_eq!( + cache.plan_get(plain_request("b", &overweight_object, "e", 4 * 1024)), + ObjectDataCacheGetPlan::SkipTooLarge + ); + } + #[test] fn plan_carries_mod_time_into_key() { // ODC-06: the resolved modification time must reach the key so two diff --git a/crates/object-data-cache/src/config.rs b/crates/object-data-cache/src/config.rs index 629a59d7c..962450b2c 100644 --- a/crates/object-data-cache/src/config.rs +++ b/crates/object-data-cache/src/config.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::error::ObjectDataCacheConfigError; -use crate::memory::{MemoryBasis, resolve_effective_memory}; +use crate::memory::{EffectiveMemory, MemoryBasis, resolve_effective_memory}; use std::sync::Once; use std::time::Duration; @@ -142,13 +142,21 @@ impl ObjectDataCacheConfig { /// Resolves the effective max capacity in bytes for the cache. pub fn resolved_max_bytes(&self) -> Result { if self.max_bytes > 0 { + validate_entry_fits_capacity( + self.max_bytes, + self.max_entry_bytes, + ObjectDataCacheConfigError::MaxEntryBytesExceedsMaxBytes, + )?; return Ok(self.max_bytes); } // Resolve capacity from the effective (container-aware) total memory so // a pod with a cgroup limit far below the node RAM does not size the // cache to the node. - let effective = resolve_effective_memory(); + self.resolved_max_bytes_for_effective_memory(resolve_effective_memory()) + } + + fn resolved_max_bytes_for_effective_memory(&self, effective: EffectiveMemory) -> Result { let total_memory = effective.total_bytes; let derived = total_memory.saturating_mul(u64::from(self.max_memory_percent)) / 100; let resolved = clamp_derived_max_bytes(derived, total_memory); @@ -159,10 +167,9 @@ impl ObjectDataCacheConfig { // The derived capacity is no longer floored by `max_entry_bytes` (that // used to silently inflate the cache above the safety clamp). If a - // single entry cannot fit, reject rather than inflate. - if self.max_entry_bytes > resolved { - return Err(ObjectDataCacheConfigError::MaxEntryBytesExceedsCapacity); - } + // single entry plus its weight overhead cannot fit, reject rather than + // inflate. + validate_entry_fits_capacity(resolved, self.max_entry_bytes, ObjectDataCacheConfigError::MaxEntryBytesExceedsCapacity)?; log_resolved_capacity_once(resolved, total_memory, effective.basis); @@ -186,8 +193,12 @@ impl ObjectDataCacheConfig { // An explicit capacity must leave room for a full entry plus the // weigher overhead, otherwise moka can never retain the entry while // fills still report success. - if self.max_bytes > 0 && self.max_bytes < self.max_entry_bytes.saturating_add(ENTRY_WEIGHT_OVERHEAD_BYTES) { - return Err(ObjectDataCacheConfigError::MaxEntryBytesExceedsMaxBytes); + if self.max_bytes > 0 { + validate_entry_fits_capacity( + self.max_bytes, + self.max_entry_bytes, + ObjectDataCacheConfigError::MaxEntryBytesExceedsMaxBytes, + )?; } if self.ttl.is_zero() { @@ -249,6 +260,17 @@ impl ObjectDataCacheConfig { } } +fn validate_entry_fits_capacity( + capacity: u64, + max_entry_bytes: u64, + error: ObjectDataCacheConfigError, +) -> Result<(), ObjectDataCacheConfigError> { + match max_entry_bytes.checked_add(ENTRY_WEIGHT_OVERHEAD_BYTES) { + Some(required_capacity) if required_capacity <= capacity => Ok(()), + _ => Err(error), + } +} + fn clamp_derived_max_bytes(derived: u64, total_memory: u64) -> u64 { let percent_cap = total_memory.saturating_mul(DEFAULT_DERIVED_MAX_MEMORY_PERCENT_CAP) / 100; let safe_cap = percent_cap.min(DEFAULT_DERIVED_MAX_BYTES_CAP); @@ -269,8 +291,12 @@ fn log_resolved_capacity_once(resolved_max_bytes: u64, effective_total_bytes: u6 #[cfg(test)] mod tests { - use super::{DEFAULT_DERIVED_MAX_BYTES_CAP, ObjectDataCacheConfig, ObjectDataCacheMode, clamp_derived_max_bytes}; + use super::{ + DEFAULT_DERIVED_MAX_BYTES_CAP, ENTRY_WEIGHT_OVERHEAD_BYTES, ObjectDataCacheConfig, ObjectDataCacheMode, + clamp_derived_max_bytes, + }; use crate::error::ObjectDataCacheConfigError; + use crate::memory::{EffectiveMemory, MemoryBasis}; use std::time::Duration; #[test] @@ -459,6 +485,87 @@ mod tests { assert_eq!(err, ObjectDataCacheConfigError::MaxEntryBytesExceedsCapacity); } + #[test] + fn derived_capacity_rejects_entry_equal_to_capacity() { + const CAPACITY: u64 = 8 * 1024; + let config = ObjectDataCacheConfig { + max_bytes: 0, + max_memory_percent: 10, + max_entry_bytes: CAPACITY, + ..ObjectDataCacheConfig::default() + }; + + let err = config + .resolved_max_bytes_for_effective_memory(EffectiveMemory { + total_bytes: CAPACITY * 10, + available_bytes: CAPACITY * 10, + basis: MemoryBasis::Host, + }) + .expect_err("an entry equal to derived capacity leaves no room for its weight overhead"); + + assert_eq!(err, ObjectDataCacheConfigError::MaxEntryBytesExceedsCapacity); + } + + #[test] + fn explicit_and_derived_capacity_share_entry_margin_matrix() { + const CAPACITY: u64 = 8 * 1024; + let effective_memory = EffectiveMemory { + total_bytes: CAPACITY * 10, + available_bytes: CAPACITY * 10, + basis: MemoryBasis::Host, + }; + + for (max_entry_bytes, fits) in [ + (CAPACITY - ENTRY_WEIGHT_OVERHEAD_BYTES, true), + (CAPACITY - ENTRY_WEIGHT_OVERHEAD_BYTES + 1, false), + (CAPACITY - 1, false), + (CAPACITY, false), + ] { + let explicit = ObjectDataCacheConfig { + max_bytes: CAPACITY, + max_memory_percent: 0, + max_entry_bytes, + ..ObjectDataCacheConfig::default() + }; + let derived = ObjectDataCacheConfig { + max_bytes: 0, + max_memory_percent: 10, + max_entry_bytes, + ..ObjectDataCacheConfig::default() + }; + + if fits { + assert_eq!(explicit.validate(), Ok(()), "explicit max_entry_bytes={max_entry_bytes}"); + assert_eq!( + explicit.resolved_max_bytes(), + Ok(CAPACITY), + "explicit resolved max_entry_bytes={max_entry_bytes}" + ); + assert_eq!( + derived.resolved_max_bytes_for_effective_memory(effective_memory), + Ok(CAPACITY), + "derived max_entry_bytes={max_entry_bytes}" + ); + } else { + assert_eq!( + explicit.validate(), + Err(ObjectDataCacheConfigError::MaxEntryBytesExceedsMaxBytes), + "explicit max_entry_bytes={max_entry_bytes}" + ); + assert_eq!( + explicit.resolved_max_bytes(), + Err(ObjectDataCacheConfigError::MaxEntryBytesExceedsMaxBytes), + "explicit resolved max_entry_bytes={max_entry_bytes}" + ); + assert_eq!( + derived.resolved_max_bytes_for_effective_memory(effective_memory), + Err(ObjectDataCacheConfigError::MaxEntryBytesExceedsCapacity), + "derived max_entry_bytes={max_entry_bytes}" + ); + } + } + } + #[test] fn derived_max_bytes_clamps_to_v3_safe_cap() { let one_tib = 1024_u64 * 1024 * 1024 * 1024; diff --git a/crates/object-data-cache/src/entry.rs b/crates/object-data-cache/src/entry.rs index 011598f5a..66fef16b4 100644 --- a/crates/object-data-cache/src/entry.rs +++ b/crates/object-data-cache/src/entry.rs @@ -13,11 +13,26 @@ // limitations under the License. use bytes::Bytes; -use std::cmp; +use crate::index::ObjectDataCacheGeneration; use crate::key::ObjectDataCacheKey; -const ENTRY_OVERHEAD_BYTES: usize = 64; +const ENTRY_OVERHEAD_BYTES: u64 = 64; + +/// Estimates the weighted capacity charged for a key and planned body. +pub(crate) fn projected_weight(key: &ObjectDataCacheKey, body_bytes: u64) -> u64 { + let key_bytes = key + .bucket + .len() + .saturating_add(key.object.len()) + .saturating_add(key.version_id.len()) + .saturating_add(key.etag.len()); + + u64::try_from(key_bytes) + .unwrap_or(u64::MAX) + .saturating_add(body_bytes) + .saturating_add(ENTRY_OVERHEAD_BYTES) +} /// Cached object body entry. /// @@ -29,12 +44,17 @@ const ENTRY_OVERHEAD_BYTES: usize = 64; #[derive(Debug, Clone)] pub struct ObjectDataCacheEntry { bytes: Bytes, + generation: ObjectDataCacheGeneration, } impl ObjectDataCacheEntry { /// Creates a new cached entry. pub fn new(bytes: Bytes) -> Self { - Self { bytes } + Self { bytes, generation: 0 } + } + + pub(crate) fn with_generation(bytes: Bytes, generation: ObjectDataCacheGeneration) -> Self { + Self { bytes, generation } } /// Returns a clone of the cached body bytes. @@ -42,14 +62,14 @@ impl ObjectDataCacheEntry { self.bytes.clone() } + pub(crate) const fn generation(&self) -> ObjectDataCacheGeneration { + self.generation + } + /// Returns the estimated weighted size for capacity accounting. pub fn estimated_weight(&self, key: &ObjectDataCacheKey) -> u32 { - let key_bytes = key.bucket.len() + key.object.len() + key.version_id.len() + key.etag.len(); - let body_bytes = self.bytes.len(); - let total = key_bytes.saturating_add(body_bytes).saturating_add(ENTRY_OVERHEAD_BYTES); - let clamped = cmp::min(total, u32::MAX as usize); - - u32::try_from(clamped).unwrap_or(u32::MAX) + let body_bytes = u64::try_from(self.bytes.len()).unwrap_or(u64::MAX); + u32::try_from(projected_weight(key, body_bytes)).unwrap_or(u32::MAX) } } diff --git a/crates/object-data-cache/src/index.rs b/crates/object-data-cache/src/index.rs index 2dc1bb3ec..072d61428 100644 --- a/crates/object-data-cache/src/index.rs +++ b/crates/object-data-cache/src/index.rs @@ -15,13 +15,24 @@ use crate::key::ObjectDataCacheKey; use crate::starshard_index::StarshardIdentityIndex; -/// Per-entry generation token used to tell a freshly refilled body apart from a -/// superseded one under the same key. The token is the cache entry's `Arc` -/// pointer captured at fill time: the eviction listener receives the evicted -/// value and can compare its pointer against the token currently tracked, so a -/// deferred or inline eviction of an old generation cannot remove the index key -/// registered for the current generation. +/// Legacy public identity-index token type. +/// +/// The backend uses an internal monotonic `u64` generation instead; keeping this +/// alias preserves the existing public `StarshardIdentityIndex` method shapes. pub(crate) type ObjectDataCacheKeyToken = usize; +pub(crate) type ObjectDataCacheGeneration = u64; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ObjectDataCacheEvictedGeneration { + pub(crate) key: ObjectDataCacheKey, + pub(crate) generation: ObjectDataCacheGeneration, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ObjectDataCacheGenerationalInsertResult { + Inserted { evicted: Vec }, + Duplicate, +} /// Result of inserting a cache key into the identity index. #[derive(Debug, Clone, PartialEq, Eq)] @@ -41,7 +52,7 @@ pub enum ObjectDataCacheIndexInsertResult { #[derive(Debug, Clone)] struct TrackedKey { key: ObjectDataCacheKey, - token: ObjectDataCacheKeyToken, + generation: ObjectDataCacheGeneration, } #[derive(Debug, Clone, Default)] @@ -50,37 +61,41 @@ pub(crate) struct ObjectDataCacheKeySet { } impl ObjectDataCacheKeySet { - pub(crate) fn insert( + pub(crate) fn insert_generation( &mut self, key: ObjectDataCacheKey, - token: ObjectDataCacheKeyToken, + generation: ObjectDataCacheGeneration, max_keys: usize, - ) -> ObjectDataCacheIndexInsertResult { + ) -> ObjectDataCacheGenerationalInsertResult { if let Some(existing) = self.keys.iter_mut().find(|existing| existing.key == key) { - // Refresh the generation token so a later eviction of the superseded + // Refresh the generation so a later eviction of the superseded // body cannot remove the key registered for this new body. - existing.token = token; - return ObjectDataCacheIndexInsertResult::Duplicate; + existing.generation = generation; + return ObjectDataCacheGenerationalInsertResult::Duplicate; } // Bounded eviction: the Vec preserves insertion order, so evicting from // the front drops the oldest keys and keeps hot (recently filled) ones, // rather than clearing the whole identity and rejecting the new key. - let mut evicted_keys = Vec::new(); + let mut evicted = Vec::new(); while self.keys.len() >= max_keys { - evicted_keys.push(self.keys.remove(0).key); + let tracked = self.keys.remove(0); + evicted.push(ObjectDataCacheEvictedGeneration { + key: tracked.key, + generation: tracked.generation, + }); } - self.keys.push(TrackedKey { key, token }); - ObjectDataCacheIndexInsertResult::Inserted { evicted_keys } + self.keys.push(TrackedKey { key, generation }); + ObjectDataCacheGenerationalInsertResult::Inserted { evicted } } /// Removes the key only when its tracked generation token matches, so an /// eviction notification for an old generation leaves a refreshed key intact. - pub(crate) fn remove_evicted_key(&mut self, key: &ObjectDataCacheKey, token: ObjectDataCacheKeyToken) -> bool { + pub(crate) fn remove_generation(&mut self, key: &ObjectDataCacheKey, generation: ObjectDataCacheGeneration) -> bool { let original_len = self.keys.len(); self.keys - .retain(|existing| !(existing.key == *key && existing.token == token)); + .retain(|existing| !(existing.key == *key && existing.generation == generation)); original_len != self.keys.len() } @@ -99,6 +114,12 @@ impl ObjectDataCacheKeySet { self.keys.iter().any(|existing| &existing.key == key) } + pub(crate) fn contains_generation(&self, key: &ObjectDataCacheKey, generation: ObjectDataCacheGeneration) -> bool { + self.keys + .iter() + .any(|existing| &existing.key == key && existing.generation == generation) + } + #[cfg(test)] pub(crate) fn len(&self) -> usize { self.keys.len() @@ -114,7 +135,7 @@ pub type ObjectDataCacheIdentityIndex = StarshardIdentityIndex; #[cfg(test)] mod tests { - use super::{ObjectDataCacheIndexInsertResult, ObjectDataCacheKeySet}; + use super::{ObjectDataCacheGenerationalInsertResult, ObjectDataCacheKeySet}; use crate::key::{ObjectDataCacheBodyVariant, ObjectDataCacheKey}; fn make_key(id: &str) -> ObjectDataCacheKey { @@ -126,16 +147,11 @@ mod tests { let mut set = ObjectDataCacheKeySet::default(); let key = make_key("v1"); - let first = set.insert(key.clone(), 1, 4); - let second = set.insert(key, 2, 4); + let first = set.insert_generation(key.clone(), 1, 4); + let second = set.insert_generation(key, 2, 4); - assert_eq!( - first, - ObjectDataCacheIndexInsertResult::Inserted { - evicted_keys: Vec::new() - } - ); - assert_eq!(second, ObjectDataCacheIndexInsertResult::Duplicate); + assert_eq!(first, ObjectDataCacheGenerationalInsertResult::Inserted { evicted: Vec::new() }); + assert_eq!(second, ObjectDataCacheGenerationalInsertResult::Duplicate); assert_eq!(set.len(), 1); } @@ -145,12 +161,13 @@ mod tests { let key_a = make_key("v1"); let key_b = make_key("v2"); - let _ = set.insert(key_a.clone(), 1, 1); - let result = set.insert(key_b.clone(), 2, 1); + let _ = set.insert_generation(key_a.clone(), 1, 1); + let result = set.insert_generation(key_b.clone(), 2, 1); assert!(matches!( result, - ObjectDataCacheIndexInsertResult::Inserted { evicted_keys } if evicted_keys == vec![key_a] + ObjectDataCacheGenerationalInsertResult::Inserted { evicted } + if evicted.len() == 1 && evicted[0].key == key_a && evicted[0].generation == 1 )); // The new key replaces the evicted one instead of the identity being cleared. assert!(set.contains(&key_b)); @@ -162,14 +179,14 @@ mod tests { let mut set = ObjectDataCacheKeySet::default(); let key = make_key("v1"); - let _ = set.insert(key.clone(), 1, 4); + let _ = set.insert_generation(key.clone(), 1, 4); // A stale eviction notification carrying the old token must not remove the key. - assert!(!set.remove_evicted_key(&key, 2)); + assert!(!set.remove_generation(&key, 2)); assert!(set.contains(&key)); // The matching token removes the key. - assert!(set.remove_evicted_key(&key, 1)); + assert!(set.remove_generation(&key, 1)); assert!(!set.contains(&key)); } @@ -178,11 +195,11 @@ mod tests { let mut set = ObjectDataCacheKeySet::default(); let key = make_key("v1"); - let _ = set.insert(key.clone(), 1, 4); - let _ = set.insert(key.clone(), 2, 4); + let _ = set.insert_generation(key.clone(), 1, 4); + let _ = set.insert_generation(key.clone(), 2, 4); // After the refresh the old token no longer matches. - assert!(!set.remove_evicted_key(&key, 1)); - assert!(set.remove_evicted_key(&key, 2)); + assert!(!set.remove_generation(&key, 1)); + assert!(set.remove_generation(&key, 2)); } } diff --git a/crates/object-data-cache/src/lib.rs b/crates/object-data-cache/src/lib.rs index 13dcadec1..e0ec6497b 100644 --- a/crates/object-data-cache/src/lib.rs +++ b/crates/object-data-cache/src/lib.rs @@ -57,8 +57,9 @@ pub mod starshard_index; pub mod stats; pub use cache::{ - ObjectDataCache, ObjectDataCacheFillResult, ObjectDataCacheGetPlan, ObjectDataCacheGetRequest, - ObjectDataCacheInvalidationReason, ObjectDataCacheInvalidationResult, ObjectDataCacheLookup, + ObjectDataCache, ObjectDataCacheBodyReservation, ObjectDataCacheFillResult, ObjectDataCacheGetPlan, + ObjectDataCacheGetRequest, ObjectDataCacheInvalidationReason, ObjectDataCacheInvalidationResult, ObjectDataCacheLookup, + ObjectDataCacheReservedBody, }; pub use config::{ObjectDataCacheConfig, ObjectDataCacheMode}; pub use error::ObjectDataCacheConfigError; diff --git a/crates/object-data-cache/src/memory.rs b/crates/object-data-cache/src/memory.rs index 1ec3874e2..354991c87 100644 --- a/crates/object-data-cache/src/memory.rs +++ b/crates/object-data-cache/src/memory.rs @@ -15,14 +15,21 @@ use crate::config::ObjectDataCacheConfig; use crate::metrics::record_memory_pressure; use crate::stats::ObjectDataCacheStats; +use bytes::Bytes; +use std::hint::spin_loop; use std::sync::Arc; #[cfg(test)] use std::sync::Mutex; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Duration; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; use sysinfo::System; const DEFAULT_REFRESH_INTERVAL: Duration = Duration::from_secs(5); +const DEFAULT_TELEMETRY_STALENESS: Duration = Duration::from_secs(15); +// Each writer holds the sequence for only a few atomic loads/stores. Eight +// attempts cover brief overlap without prolonged spinning on a Tokio worker; +// exhaustion safely skips only the cache fill. +const MAX_STATE_RETRIES: usize = 8; /// Source used to resolve the effective memory limits. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -115,48 +122,377 @@ impl ObjectDataCacheMemorySnapshot { /// Lock-free memory snapshot shared between the gate and its refresher task. #[derive(Debug)] struct MemorySnapshotCell { + /// Even values identify a stable state; an odd value means a writer owns + /// the short atomic update section. + sequence: AtomicU64, total_bytes: AtomicU64, available_bytes: AtomicU64, - /// Bytes admitted by the gate since the last snapshot refresh. The snapshot - /// is sampled at most every 5 s, so a burst that begins while it still reads - /// high could all pass a plain check-then-act gate and over-allocate before - /// the next refresh. Subtracting this running total from `available_bytes` - /// shrinks the effective budget as the burst proceeds, bounding cumulative - /// admission to the real headroom; the refresh resets it because the fresh - /// reading already reflects those allocations (backlog#1107). - admitted_since_refresh: AtomicU64, + /// Monotonic ordering only; unlike the reservation counters this never + /// decreases or resets at a telemetry boundary. + issued_sequence: AtomicU64, + /// Token-bound claims. A refresh never changes this value, so a token from + /// an old epoch cannot subtract a newer epoch's claim when it is dropped. + live_reserved: AtomicU64, + /// Releases that collided with the bounded state writer. Both the releasing + /// thread and the active writer attempt a bounded drain, so Drop never waits + /// indefinitely and a release cannot be lost when a writer is preempted. + pending_release: AtomicU64, + /// Permanently fails admission closed if release accounting violates its + /// internal bounds instead of risking an under-count. + release_accounting_failed: AtomicBool, + /// Already-materialized buffered allocations without an attachable owner. + /// A sample absorbs only the baseline captured before its blocking read. + sampled_reserved: AtomicU64, + clock_origin: Instant, + last_refresh_millis: AtomicU64, + #[cfg(test)] + test_now_millis: AtomicU64, +} + +#[derive(Debug, Clone, Copy)] +struct MemorySampleBaseline { + issued_sequence: u64, + sampled_reserved: u64, } impl MemorySnapshotCell { fn new(snapshot: ObjectDataCacheMemorySnapshot) -> Self { + let available_bytes = snapshot.available_bytes.min(snapshot.total_bytes); Self { + sequence: AtomicU64::new(0), total_bytes: AtomicU64::new(snapshot.total_bytes), - available_bytes: AtomicU64::new(snapshot.available_bytes), - admitted_since_refresh: AtomicU64::new(0), + available_bytes: AtomicU64::new(available_bytes), + issued_sequence: AtomicU64::new(0), + live_reserved: AtomicU64::new(0), + pending_release: AtomicU64::new(0), + release_accounting_failed: AtomicBool::new(false), + sampled_reserved: AtomicU64::new(0), + clock_origin: Instant::now(), + last_refresh_millis: AtomicU64::new(1), + #[cfg(test)] + test_now_millis: AtomicU64::new(0), } } - /// Stores a fresh snapshot and resets the admitted-bytes counter: the new - /// reading already accounts for whatever was admitted since the last one. + #[cfg(test)] fn store(&self, snapshot: ObjectDataCacheMemorySnapshot) { - self.total_bytes.store(snapshot.total_bytes, Ordering::Relaxed); - self.available_bytes.store(snapshot.available_bytes, Ordering::Relaxed); - self.admitted_since_refresh.store(0, Ordering::Relaxed); + let Some(baseline) = self.begin_sample() else { + return; + }; + self.publish_sample(baseline, snapshot); } - fn load(&self) -> ObjectDataCacheMemorySnapshot { - ObjectDataCacheMemorySnapshot { - total_bytes: self.total_bytes.load(Ordering::Relaxed), - available_bytes: self.available_bytes.load(Ordering::Relaxed), + fn begin_sample(&self) -> Option { + let writer = self.try_write()?; + let baseline = MemorySampleBaseline { + issued_sequence: self.issued_sequence.load(Ordering::Relaxed), + sampled_reserved: self.sampled_reserved.load(Ordering::Relaxed), + }; + writer.commit(); + Some(baseline) + } + + fn publish_sample(&self, baseline: MemorySampleBaseline, snapshot: ObjectDataCacheMemorySnapshot) { + let Some(writer) = self.try_write() else { + return; + }; + let issued_sequence = self.issued_sequence.load(Ordering::Relaxed); + if issued_sequence < baseline.issued_sequence { + return; + } + let sampled_reserved = self.sampled_reserved.load(Ordering::Relaxed); + let Some(sampled_after) = sampled_reserved.checked_sub(baseline.sampled_reserved) else { + return; + }; + self.total_bytes.store(snapshot.total_bytes, Ordering::Relaxed); + self.available_bytes + .store(snapshot.available_bytes.min(snapshot.total_bytes), Ordering::Relaxed); + self.sampled_reserved.store(sampled_after, Ordering::Relaxed); + self.last_refresh_millis.store(self.now_millis(), Ordering::Relaxed); + writer.commit(); + } + + fn load(&self) -> Option { + for _ in 0..MAX_STATE_RETRIES { + let start = self.sequence.load(Ordering::Acquire); + if start & 1 == 1 { + spin_loop(); + continue; + } + + let snapshot = ObjectDataCacheMemorySnapshot { + total_bytes: self.total_bytes.load(Ordering::Relaxed), + available_bytes: self.available_bytes.load(Ordering::Relaxed), + }; + if self.sequence.load(Ordering::Acquire) == start { + return Some(snapshot); + } + } + + None + } + + #[cfg(test)] + fn admitted(&self) -> u64 { + if self.release_accounting_failed.load(Ordering::Acquire) { + return u64::MAX; + } + self.live_reserved + .load(Ordering::Relaxed) + .saturating_sub(self.pending_release.load(Ordering::Acquire)) + .saturating_add(self.sampled_reserved.load(Ordering::Relaxed)) + } + + #[cfg(test)] + fn reserve(&self, bytes: u64) { + let Some(writer) = self.try_write() else { + return; + }; + let claimed = self.sampled_reserved.load(Ordering::Relaxed); + self.sampled_reserved.store(claimed.saturating_add(bytes), Ordering::Relaxed); + writer.commit(); + } + + fn release(&self, bytes: u64) { + if bytes == 0 { + return; + } + if self + .pending_release + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |pending| pending.checked_add(bytes)) + .is_err() + { + self.release_accounting_failed.store(true, Ordering::Release); + return; + } + self.try_settle_pending_releases(); + } + + fn try_settle_pending_releases(&self) { + if self.pending_release.load(Ordering::Acquire) == 0 { + return; + } + let Some(writer) = self.try_write_inner(false) else { + return; + }; + writer.commit(); + } + + fn settle_pending_releases_locked(&self) { + let pending = self.pending_release.swap(0, Ordering::AcqRel); + if pending == 0 { + return; + } + let claimed = self.live_reserved.load(Ordering::Relaxed); + if let Some(remaining) = claimed.checked_sub(pending) { + self.live_reserved.store(remaining, Ordering::Relaxed); + } else { + self.live_reserved.store(u64::MAX, Ordering::Relaxed); + self.release_accounting_failed.store(true, Ordering::Release); } } - fn admitted(&self) -> u64 { - self.admitted_since_refresh.load(Ordering::Relaxed) + fn materialize_without_owner(&self, bytes: u64) -> bool { + let Some(writer) = self.try_write() else { + return false; + }; + let live = self.live_reserved.load(Ordering::Relaxed); + let sampled = self.sampled_reserved.load(Ordering::Relaxed); + let Some(live_after) = live.checked_sub(bytes) else { + return false; + }; + let Some(sampled_after) = sampled.checked_add(bytes) else { + return false; + }; + self.live_reserved.store(live_after, Ordering::Relaxed); + self.sampled_reserved.store(sampled_after, Ordering::Relaxed); + writer.commit(); + true } - fn reserve(&self, bytes: u64) { - self.admitted_since_refresh.fetch_add(bytes, Ordering::Relaxed); + fn telemetry_age(&self) -> Option { + let refreshed = self.last_refresh_millis.load(Ordering::Acquire); + if refreshed == 0 { + return None; + } + Some(Duration::from_millis(self.now_millis().saturating_sub(refreshed))) + } + + fn now_millis(&self) -> u64 { + #[cfg(test)] + { + let now = self.test_now_millis.load(Ordering::Relaxed); + if now != 0 { + return now; + } + } + u64::try_from(self.clock_origin.elapsed().as_millis()) + .unwrap_or(u64::MAX) + .saturating_add(1) + } + + #[cfg(test)] + fn set_test_now(&self, now: Duration) { + self.test_now_millis + .store(u64::try_from(now.as_millis()).unwrap_or(u64::MAX), Ordering::Relaxed); + } + + #[cfg(test)] + fn store_with_hook(&self, snapshot: ObjectDataCacheMemorySnapshot, after_total: impl FnOnce()) { + let Some(writer) = self.try_write() else { + return; + }; + self.total_bytes.store(snapshot.total_bytes, Ordering::Relaxed); + after_total(); + self.available_bytes + .store(snapshot.available_bytes.min(snapshot.total_bytes), Ordering::Relaxed); + self.last_refresh_millis.store(self.now_millis(), Ordering::Relaxed); + writer.commit(); + } + + fn try_write(&self) -> Option> { + self.try_write_inner(true) + } + + fn try_write_inner(&self, settle_pending_on_drop: bool) -> Option> { + for _ in 0..MAX_STATE_RETRIES { + let sequence = self.sequence.load(Ordering::Acquire); + if sequence & 1 == 1 { + spin_loop(); + continue; + } + let next_sequence = sequence.checked_add(2)?; + if self + .sequence + .compare_exchange_weak(sequence, sequence + 1, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + self.settle_pending_releases_locked(); + let previous = MemoryState { + total_bytes: self.total_bytes.load(Ordering::Relaxed), + available_bytes: self.available_bytes.load(Ordering::Relaxed), + issued_sequence: self.issued_sequence.load(Ordering::Relaxed), + live_reserved: self.live_reserved.load(Ordering::Relaxed), + sampled_reserved: self.sampled_reserved.load(Ordering::Relaxed), + last_refresh_millis: self.last_refresh_millis.load(Ordering::Relaxed), + }; + return Some(MemoryStateWriteGuard { + cell: self, + next_sequence, + previous, + committed: false, + settle_pending_on_drop, + }); + } + } + + None + } +} + +struct MemoryState { + total_bytes: u64, + available_bytes: u64, + issued_sequence: u64, + live_reserved: u64, + sampled_reserved: u64, + last_refresh_millis: u64, +} + +struct MemoryStateWriteGuard<'a> { + cell: &'a MemorySnapshotCell, + next_sequence: u64, + previous: MemoryState, + committed: bool, + settle_pending_on_drop: bool, +} + +impl MemoryStateWriteGuard<'_> { + fn commit(mut self) { + self.committed = true; + } +} + +impl Drop for MemoryStateWriteGuard<'_> { + fn drop(&mut self) { + if !self.committed { + self.cell.total_bytes.store(self.previous.total_bytes, Ordering::Relaxed); + self.cell + .available_bytes + .store(self.previous.available_bytes, Ordering::Relaxed); + self.cell + .issued_sequence + .store(self.previous.issued_sequence, Ordering::Relaxed); + self.cell.live_reserved.store(self.previous.live_reserved, Ordering::Relaxed); + self.cell + .sampled_reserved + .store(self.previous.sampled_reserved, Ordering::Relaxed); + self.cell + .last_refresh_millis + .store(self.previous.last_refresh_millis, Ordering::Relaxed); + } + self.cell.sequence.store(self.next_sequence, Ordering::Release); + if self.settle_pending_on_drop { + self.cell.try_settle_pending_releases(); + } + } +} + +/// An exclusive claim on memory admitted for one cache allocation. +/// +/// Dropping the token releases the claim. Callers that only use the legacy +/// boolean admission API retain the claim until the next telemetry refresh. +#[derive(Debug)] +#[must_use = "dropping the reservation releases the admitted memory"] +pub struct ObjectDataCacheMemoryReservation { + snapshot: Option>, + bytes: u64, +} + +impl ObjectDataCacheMemoryReservation { + /// Attaches this reservation to a newly allocated body. + /// + /// Call this before cloning `bytes`. Every clone of the returned value then + /// shares one allocation owner, and the reservation is released only after + /// its last clone is dropped. + pub fn wrap_bytes(self, bytes: Bytes) -> Bytes { + if self.snapshot.is_none() { + return bytes; + } + Bytes::from_owner(ReservedBytes { + bytes, + _reservation: self, + }) + } + + pub(crate) fn until_refresh(mut self) -> bool { + let Some(snapshot) = &self.snapshot else { + return true; + }; + if !snapshot.materialize_without_owner(self.bytes) { + return false; + } + self.snapshot = None; + true + } +} + +impl Drop for ObjectDataCacheMemoryReservation { + fn drop(&mut self) { + if let Some(snapshot) = &self.snapshot { + snapshot.release(self.bytes); + } + } +} + +struct ReservedBytes { + bytes: Bytes, + _reservation: ObjectDataCacheMemoryReservation, +} + +impl AsRef<[u8]> for ReservedBytes { + fn as_ref(&self) -> &[u8] { + self.bytes.as_ref() } } @@ -170,13 +506,12 @@ impl Drop for RefresherGuard { } } -/// Memory gate that keeps a cheap, lock-free snapshot for fill-path checks. +/// Memory gate that keeps a cheap atomic snapshot and reservation state. /// /// The snapshot is sampled off the fill path by a dedicated periodic refresher /// (the private `spawn_refresher` task) that /// runs the blocking `sysinfo` read on a `spawn_blocking` thread. `allows_fill` -/// only reads atomics, so it never blocks a tokio worker and concurrent fills -/// never serialize on a refresh. +/// only performs bounded atomic operations, so it never blocks a tokio worker. #[derive(Debug)] pub struct ObjectDataCacheMemoryGate { snapshot: Arc, @@ -232,14 +567,20 @@ impl ObjectDataCacheMemoryGate { ticker.tick().await; loop { ticker.tick().await; + let Some(baseline) = snapshot.begin_sample() else { + continue; + }; // Run the blocking /proc/meminfo read off the async worker. A // join error only happens if the runtime is shutting down; keep // the last snapshot rather than clobbering it in that case. if let Ok(effective) = tokio::task::spawn_blocking(resolve_effective_memory).await { - snapshot.store(ObjectDataCacheMemorySnapshot { - total_bytes: effective.total_bytes, - available_bytes: effective.available_bytes, - }); + snapshot.publish_sample( + baseline, + ObjectDataCacheMemorySnapshot { + total_bytes: effective.total_bytes, + available_bytes: effective.available_bytes, + }, + ); } } }); @@ -255,65 +596,144 @@ impl ObjectDataCacheMemoryGate { } } - self.snapshot.load() + self.snapshot.load().unwrap_or_default() + } + + /// Returns the age of the last successfully published telemetry sample. + pub fn telemetry_age(&self) -> Option { + self.snapshot.telemetry_age() + } + + /// Atomically claims memory for one cache allocation. + /// + /// The returned token owns the claim and releases it on drop. Contention is + /// retried a bounded number of times; exhaustion fails closed by skipping + /// this cache fill without affecting the underlying object read. + pub fn try_claim(&self, required_bytes: u64, cache_growth_headroom: u64) -> Option { + self.try_claim_after(required_bytes, cache_growth_headroom, || {}) + } + + fn try_claim_after( + &self, + required_bytes: u64, + _cache_growth_headroom: u64, + before_claim: impl FnOnce(), + ) -> Option { + if self.min_free_memory_percent == 0 { + return Some(ObjectDataCacheMemoryReservation { + snapshot: None, + bytes: 0, + }); + } + + before_claim(); + let Some(writer) = self.snapshot.try_write() else { + record_memory_pressure(&self.stats, "moka"); + return None; + }; + if self.snapshot.release_accounting_failed.load(Ordering::Acquire) { + writer.commit(); + record_memory_pressure(&self.stats, "moka"); + return None; + } + if self + .snapshot + .telemetry_age() + .is_none_or(|age| age > DEFAULT_TELEMETRY_STALENESS) + { + writer.commit(); + record_memory_pressure(&self.stats, "moka"); + return None; + } + let snapshot = { + #[cfg(test)] + if let Some(snapshot) = *lock_or_recover(&self.test_override) { + snapshot + } else { + ObjectDataCacheMemorySnapshot { + total_bytes: self.snapshot.total_bytes.load(Ordering::Relaxed), + available_bytes: self.snapshot.available_bytes.load(Ordering::Relaxed), + } + } + #[cfg(not(test))] + ObjectDataCacheMemorySnapshot { + total_bytes: self.snapshot.total_bytes.load(Ordering::Relaxed), + available_bytes: self.snapshot.available_bytes.load(Ordering::Relaxed), + } + }; + if snapshot.total_bytes == 0 { + writer.commit(); + record_memory_pressure(&self.stats, "moka"); + return None; + } + let live_reserved = self.snapshot.live_reserved.load(Ordering::Relaxed); + let sampled_reserved = self.snapshot.sampled_reserved.load(Ordering::Relaxed); + let Some(claimed) = live_reserved.checked_add(sampled_reserved) else { + writer.commit(); + record_memory_pressure(&self.stats, "moka"); + return None; + }; + let Some(claimed_after) = claimed.checked_add(required_bytes) else { + writer.commit(); + record_memory_pressure(&self.stats, "moka"); + return None; + }; + let Some(available_after) = snapshot.available_bytes.checked_sub(claimed_after) else { + writer.commit(); + record_memory_pressure(&self.stats, "moka"); + return None; + }; + let min_free = u64::from(self.min_free_memory_percent); + let has_percent_budget = u128::from(available_after) * 100 >= u128::from(snapshot.total_bytes) * u128::from(min_free); + + if !has_percent_budget { + writer.commit(); + record_memory_pressure(&self.stats, "moka"); + return None; + } + + let Some(live_after) = live_reserved.checked_add(required_bytes) else { + writer.commit(); + record_memory_pressure(&self.stats, "moka"); + return None; + }; + let issued_sequence = self.snapshot.issued_sequence.load(Ordering::Relaxed); + let Some(next_issued_sequence) = issued_sequence.checked_add(1) else { + writer.commit(); + record_memory_pressure(&self.stats, "moka"); + return None; + }; + self.snapshot.live_reserved.store(live_after, Ordering::Relaxed); + self.snapshot.issued_sequence.store(next_issued_sequence, Ordering::Relaxed); + writer.commit(); + Some(ObjectDataCacheMemoryReservation { + snapshot: Some(Arc::clone(&self.snapshot)), + bytes: required_bytes, + }) } /// Returns true when the fill path may proceed under current memory pressure. /// - /// This is lock-free and does no blocking sysinfo read: it only reads the - /// atomic snapshot maintained by the periodic refresher. + /// This does no blocking sysinfo read: it uses the atomic snapshot and a + /// bounded reservation update maintained by the periodic refresher. /// - /// `cache_growth_headroom` is how many more bytes the cache itself can hold - /// before it is at capacity (`max_capacity - weighted_size()`). It caps how - /// far the in-window reservation may shrink the budget: see the reservation - /// note below. + /// `cache_growth_headroom` remains for source compatibility. Reservations + /// now follow live allocations, so cache capacity cannot cap the peak memory + /// held by an evicted entry whose response body is still alive. pub fn allows_fill(&self, required_bytes: u64, cache_growth_headroom: u64) -> bool { - // A zero floor opts out of the gate, so fill admission never depends on - // a live memory reading — which differs between a host and a container. - // This must short-circuit before any snapshot read (see 51a97a81c). - if self.min_free_memory_percent == 0 { - return true; - } + self.try_claim_buffered(required_bytes, cache_growth_headroom) + } - let snapshot = self.snapshot(); - if snapshot.total_bytes == 0 { - return true; - } - - // Effective budget is the snapshot's available memory minus what the - // gate has already admitted since that snapshot was taken. This bounds a - // burst that arrives faster than the 5 s refresh: each admission shrinks - // the budget the next one sees, so cumulative admission cannot exceed the - // real headroom even though every fill reads the same (stale) snapshot. - // - // `admitted_since_refresh` counts GROSS admitted bytes and never rolls - // back on a fill that is later evicted, cancelled, or loses the - // invalidation race; it only resets on the 5 s refresh. Under sustained - // churn (net footprint flat, far below capacity) the raw counter would - // balloon past the real memory the cache consumes and falsely trip the - // gate, skipping the hottest fills until the next refresh. The cache can - // never hold more than `max_capacity`, so a burst adds at most - // `cache_growth_headroom` bytes of real memory before moka evicts to stay - // bounded (net-zero churn beyond that point). Capping the deduction there - // keeps the reservation honest without treating gross churn as growth - // (backlog#1212). - let reserved = self.snapshot.admitted().min(cache_growth_headroom); - let effective_available = snapshot.available_bytes.saturating_sub(reserved); - - let min_free = u64::from(self.min_free_memory_percent); - let has_percent_budget = effective_available.saturating_mul(100) >= snapshot.total_bytes.saturating_mul(min_free); - let has_entry_budget = effective_available >= required_bytes; - let allowed = has_percent_budget && has_entry_budget; - - if allowed { - // Reserve the admitted bytes so a concurrent fill sees a smaller - // budget; the reservation clears on the next snapshot refresh. - self.snapshot.reserve(required_bytes); + pub(crate) fn try_claim_buffered(&self, required_bytes: u64, cache_growth_headroom: u64) -> bool { + let Some(reservation) = self.try_claim(required_bytes, cache_growth_headroom) else { + return false; + }; + if reservation.until_refresh() { + true } else { record_memory_pressure(&self.stats, "moka"); + false } - - allowed } #[cfg(test)] @@ -324,14 +744,46 @@ impl ObjectDataCacheMemoryGate { /// Writes the atomic snapshot directly, bypassing the `test_override` read /// path so a test can observe whether `allows_fill` mutates the snapshot. #[cfg(test)] - fn store_raw_snapshot_for_test(&self, snapshot: ObjectDataCacheMemorySnapshot) { + pub(crate) fn store_raw_snapshot_for_test(&self, snapshot: ObjectDataCacheMemorySnapshot) { self.snapshot.store(snapshot); } /// Reads the atomic snapshot directly, bypassing `test_override`. #[cfg(test)] fn raw_snapshot_for_test(&self) -> ObjectDataCacheMemorySnapshot { - self.snapshot.load() + self.snapshot.load().unwrap_or_default() + } + + #[cfg(test)] + fn try_claim_with_hook( + &self, + required_bytes: u64, + cache_growth_headroom: u64, + before_claim: impl FnOnce(), + ) -> Option { + self.try_claim_after(required_bytes, cache_growth_headroom, before_claim) + } + + #[cfg(test)] + pub(crate) fn claimed_bytes_for_test(&self) -> u64 { + self.snapshot.admitted() + } + + #[cfg(test)] + fn begin_sample_for_test(&self) -> MemorySampleBaseline { + self.snapshot + .begin_sample() + .expect("test sample should acquire the state writer") + } + + #[cfg(test)] + fn publish_sample_for_test(&self, baseline: MemorySampleBaseline, snapshot: ObjectDataCacheMemorySnapshot) { + self.snapshot.publish_sample(baseline, snapshot); + } + + #[cfg(test)] + fn set_test_now(&self, now: Duration) { + self.snapshot.set_test_now(now); } } @@ -345,14 +797,449 @@ fn lock_or_recover(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { #[cfg(test)] mod tests { - use super::{MemoryBasis, ObjectDataCacheMemoryGate, ObjectDataCacheMemorySnapshot, select_effective_memory}; + use super::{ + DEFAULT_TELEMETRY_STALENESS, MemoryBasis, ObjectDataCacheMemoryGate, ObjectDataCacheMemorySnapshot, + select_effective_memory, + }; use crate::config::ObjectDataCacheConfig; use crate::stats::ObjectDataCacheStats; + use bytes::Bytes; + use std::panic::{AssertUnwindSafe, catch_unwind}; use std::sync::Arc; + use std::sync::Barrier; + use std::sync::atomic::Ordering; + use std::thread; use std::time::Duration; const GIB: u64 = 1024 * 1024 * 1024; + #[test] + fn memory_gate_concurrent_claims_are_linearizable() { + const CLAIMERS: usize = 8; + let gate = Arc::new(ObjectDataCacheMemoryGate::new( + &ObjectDataCacheConfig::default(), + Arc::new(ObjectDataCacheStats::default()), + )); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + })); + let ready = Arc::new(Barrier::new(CLAIMERS)); + + let mut handles = Vec::with_capacity(CLAIMERS); + for _ in 0..CLAIMERS { + let gate = Arc::clone(&gate); + let ready = Arc::clone(&ready); + handles.push(thread::spawn(move || { + gate.try_claim_with_hook(300, u64::MAX, || { + ready.wait(); + }) + })); + } + let claims: Vec<_> = handles + .into_iter() + .filter_map(|handle| handle.join().expect("claim thread should not panic")) + .collect(); + + assert_eq!(claims.len(), 1, "only one 300-byte claim preserves the shared floor"); + assert_eq!(gate.claimed_bytes_for_test(), 300); + drop(claims); + assert_eq!(gate.claimed_bytes_for_test(), 0, "dropping the owner releases its claim"); + } + + #[test] + fn memory_reservation_token_lifecycle_is_idempotent() { + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::new(ObjectDataCacheStats::default())); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + })); + + let mut claim = Some(gate.try_claim(100, u64::MAX).expect("100-byte claim should fit")); + assert_eq!(gate.claimed_bytes_for_test(), 100); + drop(claim.take()); + drop(claim); + + assert_eq!(gate.claimed_bytes_for_test(), 0, "the claim is released exactly once"); + } + + #[test] + fn memory_release_debt_settles_after_active_writer_exits() { + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::new(ObjectDataCacheStats::default())); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + })); + let claim = gate + .try_claim(300, 0) + .expect("the initial claim should fit at the memory floor"); + let writer = gate.snapshot.try_write().expect("the test must hold the memory epoch writer"); + + drop(claim); + assert_eq!(gate.snapshot.live_reserved.load(Ordering::Relaxed), 300); + assert_eq!(gate.snapshot.pending_release.load(Ordering::Acquire), 300); + assert_eq!(gate.claimed_bytes_for_test(), 0, "release debt must immediately offset the dead claim"); + + drop(writer); + assert_eq!(gate.claimed_bytes_for_test(), 0, "writer exit must settle the deferred release"); + assert_eq!(gate.snapshot.pending_release.load(Ordering::Acquire), 0); + assert!(gate.try_claim(300, 0).is_some(), "settled release debt must restore admission"); + } + + #[test] + fn memory_release_debt_overflow_fails_admission_closed() { + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::new(ObjectDataCacheStats::default())); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + })); + gate.snapshot.pending_release.store(u64::MAX, Ordering::Release); + + gate.snapshot.release(1); + + assert!(gate.snapshot.release_accounting_failed.load(Ordering::Acquire)); + assert!(gate.try_claim(1, 0).is_none(), "overflowed release accounting must reject future fills"); + } + + #[test] + fn memory_release_debt_exceeding_live_claim_fails_admission_closed() { + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::new(ObjectDataCacheStats::default())); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + })); + gate.snapshot.live_reserved.store(1, Ordering::Relaxed); + gate.snapshot.pending_release.store(2, Ordering::Release); + + gate.snapshot.try_settle_pending_releases(); + + assert!(gate.snapshot.release_accounting_failed.load(Ordering::Acquire)); + assert_eq!(gate.snapshot.live_reserved.load(Ordering::Relaxed), u64::MAX); + assert!(gate.try_claim(1, 0).is_none(), "underflowed release accounting must reject future fills"); + } + + #[test] + fn memory_gate_claimed_bytes_never_exceed_epoch_budget() { + const CLAIMERS: usize = 32; + let gate = Arc::new(ObjectDataCacheMemoryGate::new( + &ObjectDataCacheConfig::default(), + Arc::new(ObjectDataCacheStats::default()), + )); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + })); + let ready = Arc::new(Barrier::new(CLAIMERS)); + + let mut handles = Vec::with_capacity(CLAIMERS); + for _ in 0..CLAIMERS { + let gate = Arc::clone(&gate); + let ready = Arc::clone(&ready); + handles.push(thread::spawn(move || { + gate.try_claim_with_hook(100, u64::MAX, || { + ready.wait(); + }) + })); + } + let claims: Vec<_> = handles + .into_iter() + .filter_map(|handle| handle.join().expect("claim thread should not panic")) + .collect(); + + assert_eq!(gate.claimed_bytes_for_test(), u64::try_from(claims.len()).unwrap_or(u64::MAX) * 100); + assert!(gate.claimed_bytes_for_test() <= 500); + } + + #[test] + fn memory_claim_recomputes_when_epoch_changes() { + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::new(ObjectDataCacheStats::default())); + gate.set_test_snapshot(None); + gate.store_raw_snapshot_for_test(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 100, + }); + + let claim = gate.try_claim_with_hook(100, u64::MAX, || { + gate.store_raw_snapshot_for_test(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + }); + }); + + assert!(claim.is_some(), "claim must use the epoch published before linearization"); + } + + #[test] + fn memory_snapshot_publish_preserves_post_sample_claim() { + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::new(ObjectDataCacheStats::default())); + gate.store_raw_snapshot_for_test(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + }); + let sample = gate.begin_sample_for_test(); + let claim = gate.try_claim(100, 0).expect("post-sample claim should fit"); + assert!(claim.until_refresh(), "materialized claim should enter the telemetry ledger"); + + gate.publish_sample_for_test( + sample, + ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + }, + ); + + assert_eq!(gate.claimed_bytes_for_test(), 100, "post-sample claim must carry into the new epoch"); + } + + #[test] + fn memory_snapshot_carries_pre_sample_unmaterialized_claim() { + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::new(ObjectDataCacheStats::default())); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + })); + let claim = gate.try_claim(100, 0).expect("pre-sample claim should fit"); + let sample = gate.begin_sample_for_test(); + + gate.publish_sample_for_test( + sample, + ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 400, + }, + ); + + assert_eq!( + gate.claimed_bytes_for_test(), + 100, + "unmaterialized claim cannot be absorbed by the sample" + ); + drop(claim); + assert_eq!(gate.claimed_bytes_for_test(), 0); + } + + #[test] + fn memory_gate_telemetry_expiry_degrades_and_recovers() { + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::new(ObjectDataCacheStats::default())); + gate.set_test_snapshot(None); + gate.set_test_now(Duration::from_secs(1)); + gate.store_raw_snapshot_for_test(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + }); + assert!(gate.try_claim(100, 0).is_some()); + + gate.set_test_now(DEFAULT_TELEMETRY_STALENESS + Duration::from_secs(2)); + assert!(gate.try_claim(100, 0).is_none(), "expired telemetry must fail closed for cache fills"); + + gate.store_raw_snapshot_for_test(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + }); + assert!(gate.try_claim(100, 0).is_some(), "a valid refresh must recover admission"); + } + + #[test] + fn memory_snapshot_reader_never_observes_mixed_epoch() { + let gate = Arc::new(ObjectDataCacheMemoryGate::new( + &ObjectDataCacheConfig::default(), + Arc::new(ObjectDataCacheStats::default()), + )); + gate.set_test_snapshot(None); + gate.store_raw_snapshot_for_test(ObjectDataCacheMemorySnapshot { + total_bytes: 64 * GIB, + available_bytes: 40 * GIB, + }); + let entered = Arc::new(Barrier::new(2)); + let release = Arc::new(Barrier::new(2)); + let writer = { + let gate = Arc::clone(&gate); + let entered = Arc::clone(&entered); + let release = Arc::clone(&release); + thread::spawn(move || { + gate.snapshot.store_with_hook( + ObjectDataCacheMemorySnapshot { + total_bytes: 2 * GIB, + available_bytes: 100 * 1024 * 1024, + }, + || { + entered.wait(); + release.wait(); + }, + ); + }) + }; + + entered.wait(); + assert_eq!( + gate.raw_snapshot_for_test(), + ObjectDataCacheMemorySnapshot::default(), + "a bounded reader must fail closed while an epoch is incomplete" + ); + release.wait(); + writer.join().expect("snapshot writer should not panic"); + + assert_eq!( + gate.raw_snapshot_for_test(), + ObjectDataCacheMemorySnapshot { + total_bytes: 2 * GIB, + available_bytes: 100 * 1024 * 1024, + } + ); + } + + #[test] + fn memory_snapshot_interrupted_writer_rolls_back_partial_epoch() { + let initial = ObjectDataCacheMemorySnapshot { + total_bytes: 64 * GIB, + available_bytes: 40 * GIB, + }; + let snapshot = super::MemorySnapshotCell::new(initial); + + let interrupted = catch_unwind(AssertUnwindSafe(|| { + snapshot.store_with_hook( + ObjectDataCacheMemorySnapshot { + total_bytes: 2 * GIB, + available_bytes: 100 * 1024 * 1024, + }, + || panic!("interrupt writer before tuple publication"), + ); + })); + + assert!(interrupted.is_err()); + assert_eq!(snapshot.load(), Some(initial)); + } + + #[test] + fn memory_claim_sequence_overflow_fails_closed() { + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::new(ObjectDataCacheStats::default())); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + })); + gate.snapshot.issued_sequence.store(u64::MAX, Ordering::Relaxed); + + assert!(gate.try_claim(100, 0).is_none()); + assert_eq!(gate.claimed_bytes_for_test(), 0); + } + + #[test] + fn memory_live_reserved_overflow_fails_closed() { + let gate = ObjectDataCacheMemoryGate::new( + &ObjectDataCacheConfig { + min_free_memory_percent: 1, + ..ObjectDataCacheConfig::default() + }, + Arc::new(ObjectDataCacheStats::default()), + ); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: u64::MAX, + available_bytes: u64::MAX, + })); + gate.snapshot.live_reserved.store(u64::MAX - 1, Ordering::Relaxed); + + assert!(gate.try_claim(2, 0).is_none()); + assert_eq!(gate.claimed_bytes_for_test(), u64::MAX - 1); + } + + #[test] + fn memory_old_epoch_release_preserves_new_epoch_claim() { + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::new(ObjectDataCacheStats::default())); + gate.set_test_snapshot(None); + gate.store_raw_snapshot_for_test(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + }); + let old_epoch = gate.try_claim(100, 0).expect("old epoch claim should fit"); + let sample = gate.begin_sample_for_test(); + gate.publish_sample_for_test( + sample, + ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 400, + }, + ); + let new_epoch = gate.try_claim(100, 0).expect("new epoch claim should fit"); + + drop(old_epoch); + assert_eq!(gate.claimed_bytes_for_test(), 100, "old token must not subtract the new epoch claim"); + drop(new_epoch); + assert_eq!(gate.claimed_bytes_for_test(), 0); + } + + #[test] + fn memory_gate_zero_telemetry_fails_closed_when_enabled() { + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::new(ObjectDataCacheStats::default())); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot::default())); + + assert!(gate.try_claim(1, 0).is_none()); + } + + #[test] + fn memory_gate_preserves_post_admission_floor_at_exact_boundary() { + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::new(ObjectDataCacheStats::default())); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + })); + + let boundary = gate.try_claim(300, 0); + assert!(boundary.is_some(), "a claim ending exactly at the 20% floor must fit"); + drop(boundary); + assert!(gate.try_claim(301, u64::MAX).is_none(), "a claim crossing the floor must fail"); + } + + #[test] + fn full_cache_replacement_reserves_peak_live_bytes_until_last_body_owner_drops() { + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::new(ObjectDataCacheStats::default())); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 1_000, + })); + + let old_cached = gate + .try_claim(300, 0) + .expect("old allocation should fit") + .wrap_bytes(Bytes::from(vec![0; 300])); + let old_response = old_cached.clone(); + let replacement = gate + .try_claim(300, 0) + .expect("replacement allocation should fit while the old response is live") + .wrap_bytes(Bytes::from(vec![0; 300])); + assert_eq!(gate.claimed_bytes_for_test(), 600, "replacement peak includes both live allocations"); + + drop(old_cached); + assert_eq!( + gate.claimed_bytes_for_test(), + 600, + "eviction cannot release the response owner's allocation" + ); + drop(replacement); + assert_eq!(gate.claimed_bytes_for_test(), 300); + drop(old_response); + assert_eq!(gate.claimed_bytes_for_test(), 0); + } + + #[test] + fn eviction_completion_does_not_release_live_response_body_reservation() { + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::new(ObjectDataCacheStats::default())); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + })); + + let cached = gate + .try_claim(100, 0) + .expect("allocation should fit") + .wrap_bytes(Bytes::from(vec![0; 100])); + let response = cached.clone(); + drop(cached); + + assert_eq!(gate.claimed_bytes_for_test(), 100); + drop(response); + assert_eq!(gate.claimed_bytes_for_test(), 0); + } + #[test] fn select_effective_memory_prefers_constraining_cgroup() { let effective = select_effective_memory(64 * GIB, 40 * GIB, Some((2 * GIB, GIB))); @@ -451,59 +1338,37 @@ mod tests { assert_eq!(stats.snapshot().memory_pressure_events, 1); } - // backlog#1212: `admitted_since_refresh` counts GROSS admitted bytes and - // never rolls back on an evicted/cancelled/lost-race fill, so a churn window - // (net footprint flat, far below capacity) balloons the raw counter past the - // memory the cache actually holds. Deducting it wholesale falsely trips the - // gate; capping the deduction at the cache's growth headroom fixes it. + // backlog#1331 supersedes the cache-capacity deduction from backlog#1212: + // an evicted cache entry can still have a live response clone, so cache + // growth headroom is not a safe bound on process memory ownership. #[test] - fn reservation_deduction_capped_at_growth_headroom_avoids_false_pressure() { + fn cache_growth_headroom_does_not_hide_outstanding_claims() { let stats = Arc::new(ObjectDataCacheStats::default()); let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::clone(&stats)); gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { total_bytes: 1_000_000, available_bytes: 500_000, // 50% free, well above the 20% floor })); - // A churn window admitted far more GROSS bytes than the cache can hold; - // repeated insert/evict never rolled the counter back, so it now dwarfs - // the real available memory even though the live footprint stays tiny. gate.snapshot.reserve(10_000_000); - // Uncapped, the raw gross counter swamps the budget and falsely signals - // memory pressure even though the cache's net footprint is flat. - assert!( - !gate.allows_fill(1_000, u64::MAX), - "raw gross admitted-bytes deduction should falsely suppress (the bug being fixed)" - ); - - // Capping the deduction at the cache's growth headroom (net size flat, - // far below capacity) restores admission: gross churn is no longer - // mistaken for real memory growth. - assert!( - gate.allows_fill(1_000, 100_000), - "capping the reservation at cache growth headroom must not falsely suppress" - ); + assert!(!gate.allows_fill(1_000, u64::MAX)); + assert!(!gate.allows_fill(1_000, 0), "zero cache headroom must not erase a live allocation claim"); } - // backlog#1212: capping the deduction must not defeat the reservation under - // genuine cache growth. When the cache still has room to grow, the in-window - // reservation must still shrink the budget so a burst cannot over-admit. #[test] - fn reservation_still_bounds_burst_within_growth_headroom() { + fn reservation_still_bounds_burst_independent_of_growth_headroom() { let stats = Arc::new(ObjectDataCacheStats::default()); let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::clone(&stats)); gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { total_bytes: 1_000_000, available_bytes: 300_000, // 30% free; floor is 20% = 200_000 })); - // The cache can still grow well past the reserved amount, so the cap does - // not bind and the reservation is deducted in full. gate.snapshot.reserve(150_000); // 300_000 available - 150_000 reserved = 150_000 effective, below the // 200_000 floor: the reservation must still suppress the fill. assert!( - !gate.allows_fill(1_000, u64::MAX), + !gate.allows_fill(1_000, 0), "reservation must still bound a burst while the cache can genuinely grow" ); } @@ -551,7 +1416,7 @@ mod tests { // real host reading (total memory is always far above 1 byte). let mut refreshed = false; for _ in 0..200 { - if snapshot.load().total_bytes > 1 { + if snapshot.load().is_some_and(|snapshot| snapshot.total_bytes > 1) { refreshed = true; break; } diff --git a/crates/object-data-cache/src/moka_backend.rs b/crates/object-data-cache/src/moka_backend.rs index 3a19a3a64..81abaf719 100644 --- a/crates/object-data-cache/src/moka_backend.rs +++ b/crates/object-data-cache/src/moka_backend.rs @@ -12,18 +12,171 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::cache::{ObjectDataCacheFillResult, ObjectDataCacheGetPlan, ObjectDataCacheInvalidationResult, ObjectDataCacheLookup}; +use crate::cache::{ + ObjectDataCacheBodyReservation, ObjectDataCacheFillResult, ObjectDataCacheGetPlan, ObjectDataCacheInvalidationResult, + ObjectDataCacheLookup, ObjectDataCacheReservedBody, +}; use crate::config::ObjectDataCacheConfig; -use crate::entry::ObjectDataCacheEntry; -use crate::index::{ObjectDataCacheIdentityIndex, ObjectDataCacheIndexInsertResult, ObjectDataCacheKeyToken}; +use crate::entry::{ObjectDataCacheEntry, projected_weight}; +use crate::index::{ObjectDataCacheGenerationalInsertResult, ObjectDataCacheIdentityIndex}; use crate::key::{ObjectDataCacheIdentity, ObjectDataCacheKey}; use crate::memory::ObjectDataCacheMemoryGate; use crate::singleflight::{ObjectDataCacheSingleflight, ObjectDataCacheSingleflightAcquire}; use crate::stats::ObjectDataCacheStats; use bytes::Bytes; use moka::future::{Cache, FutureExt}; -use std::sync::Arc; -use tokio::sync::Semaphore; +use moka::ops::compute; +use std::future::ready; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, watch}; + +#[derive(Debug)] +struct ClearFence { + state: Mutex, + quiesced: Notify, +} + +#[derive(Debug)] +struct ClearFenceState { + /// `None` permanently rejects fills after an impossible counter failure. + generation: Option, + active_fills: usize, + clear: Option>, +} + +#[derive(Debug)] +struct ClearOperation { + completion: watch::Sender>, +} + +#[derive(Debug)] +pub(crate) struct FillGenerationGuard { + fence: Arc, + generation: u64, +} + +impl ClearFence { + fn new() -> Self { + Self { + state: Mutex::new(ClearFenceState { + generation: Some(0), + active_fills: 0, + clear: None, + }), + quiesced: Notify::new(), + } + } + + fn try_register_fill(self: &Arc) -> Option { + let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if state.clear.is_some() { + return None; + } + let generation = state.generation?; + let Some(active_fills) = state.active_fills.checked_add(1) else { + state.generation = None; + return None; + }; + state.active_fills = active_fills; + Some(FillGenerationGuard { + fence: Arc::clone(self), + generation, + }) + } + + fn begin_clear(&self) -> (Arc, bool) { + let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(operation) = &state.clear { + return (Arc::clone(operation), false); + } + + let operation = Arc::new(ClearOperation::new()); + state.clear = Some(Arc::clone(&operation)); + state.generation = state.generation.and_then(|generation| generation.checked_add(1)); + (operation, true) + } + + async fn wait_for_active_fills(&self) { + loop { + let notified = self.quiesced.notified(); + if self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .active_fills + == 0 + { + return; + } + notified.await; + } + } + + fn finish_clear(&self, operation: &Arc, result: ObjectDataCacheInvalidationResult) { + let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if !state.clear.as_ref().is_some_and(|current| Arc::ptr_eq(current, operation)) { + return; + } + + let next_generation = state.generation.and_then(|generation| generation.checked_add(1)); + state.generation = next_generation; + operation.complete(result); + if next_generation.is_some() { + state.clear = None; + } + } +} + +impl FillGenerationGuard { + fn belongs_to(&self, fence: &Arc) -> bool { + Arc::ptr_eq(&self.fence, fence) + } +} + +impl ClearOperation { + fn new() -> Self { + let (completion, _) = watch::channel(None); + Self { completion } + } + + async fn wait(&self) -> ObjectDataCacheInvalidationResult { + let mut completion = self.completion.subscribe(); + loop { + if let Some(result) = *completion.borrow_and_update() { + return result; + } + let _ = completion.changed().await; + } + } + + fn complete(&self, result: ObjectDataCacheInvalidationResult) { + self.completion.send_replace(Some(result)); + } +} + +impl FillGenerationGuard { + fn is_current(&self) -> bool { + let state = self.fence.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + state.clear.is_none() && state.generation == Some(self.generation) + } +} + +impl Drop for FillGenerationGuard { + fn drop(&mut self) { + let mut state = self.fence.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let Some(active_fills) = state.active_fills.checked_sub(1) else { + state.generation = None; + return; + }; + state.active_fills = active_fills; + if state.active_fills == 0 { + // There is exactly one owned clear drain. `notify_one` retains a + // permit if it reaches zero between the state check and `.await`. + self.fence.quiesced.notify_one(); + } + } +} /// Weighted Moka backend for reusable object bodies. #[derive(Debug)] @@ -32,17 +185,33 @@ pub struct MokaBackend { index: Arc, singleflight: ObjectDataCacheSingleflight, memory_gate: ObjectDataCacheMemoryGate, - /// Cache capacity in weighted bytes. Used to derive how much the cache can - /// still grow (`max_capacity - weighted_size()`), which caps the memory - /// gate's in-window reservation deduction (backlog#1212). + /// Cache capacity in weighted bytes. The derived growth headroom remains an + /// input to the compatible memory-gate API, although live allocation claims + /// are no longer capped by cache residency (backlog#1331). max_capacity: u64, /// Bounds the number of concurrent distinct-key fills. Singleflight only /// dedups per key, so without this limiter distinct-key fills are unbounded. fill_semaphore: Arc, + next_generation: AtomicU64, + /// Serializes global clears and fences fill publication by generation. + clear_fence: Arc, /// Test-only barrier that pauses a fill between the index insert and the /// cache insert so the fill-vs-invalidation race can be driven deterministically. #[cfg(test)] fill_barrier: std::sync::Mutex>>, + /// Test-only barrier that pauses identity-budget eviction after the index + /// selects a victim and before the cache removal starts. + #[cfg(test)] + identity_eviction_barrier: std::sync::Mutex>>, + /// Test-only blocking rendezvous inside the conditional compute closure. + #[cfg(test)] + identity_compute_barrier: std::sync::Mutex>>, + #[cfg(test)] + fill_before_index_barrier: std::sync::Mutex>>, + #[cfg(test)] + clear_barrier: std::sync::Mutex>>, + #[cfg(test)] + clear_joined: Arc, } /// Test-only rendezvous that pauses a fill between the index insert and the @@ -53,6 +222,45 @@ pub struct MokaBackend { struct FillBarrier { reached: tokio::sync::Semaphore, release: tokio::sync::Semaphore, + completed: tokio::sync::Semaphore, +} + +#[cfg(test)] +#[derive(Debug)] +struct ComputeBarrier { + reached: tokio::sync::Semaphore, + released: std::sync::Mutex, + release: std::sync::Condvar, +} + +#[cfg(test)] +impl ComputeBarrier { + fn new() -> Self { + Self { + reached: tokio::sync::Semaphore::new(0), + released: std::sync::Mutex::new(false), + release: std::sync::Condvar::new(), + } + } + + fn wait(&self) { + self.reached.add_permits(1); + let mut released = self.released.lock().unwrap_or_else(|p| p.into_inner()); + while !*released { + released = self.release.wait(released).unwrap_or_else(|p| p.into_inner()); + } + } + + async fn wait_until_reached(&self) { + if let Ok(permit) = self.reached.acquire().await { + permit.forget(); + } + } + + fn release(&self) { + *self.released.lock().unwrap_or_else(|p| p.into_inner()) = true; + self.release.notify_all(); + } } #[cfg(test)] @@ -61,6 +269,7 @@ impl FillBarrier { Self { reached: tokio::sync::Semaphore::new(0), release: tokio::sync::Semaphore::new(0), + completed: tokio::sync::Semaphore::new(0), } } @@ -83,6 +292,16 @@ impl FillBarrier { fn release(&self) { self.release.add_permits(1); } + + fn complete(&self) { + self.completed.add_permits(1); + } + + async fn wait_until_completed(&self) { + if let Ok(permit) = self.completed.acquire().await { + permit.forget(); + } + } } impl MokaBackend { @@ -136,8 +355,7 @@ impl MokaBackend { // still the one it tracks. An inline `Expired` upsert or a // deferred `Size` notification for a superseded generation // then cannot remove the key a fresh refill just registered. - let token = Arc::as_ptr(&value) as ObjectDataCacheKeyToken; - index.remove_evicted_key(&identity, &key, token).await; + index.remove_generation(&identity, &key, value.generation()).await; } .boxed() }) @@ -150,8 +368,20 @@ impl MokaBackend { memory_gate: ObjectDataCacheMemoryGate::new(config, stats), max_capacity, fill_semaphore: Arc::new(Semaphore::new(fill_permits)), + next_generation: AtomicU64::new(0), + clear_fence: Arc::new(ClearFence::new()), #[cfg(test)] fill_barrier: std::sync::Mutex::new(None), + #[cfg(test)] + identity_eviction_barrier: std::sync::Mutex::new(None), + #[cfg(test)] + identity_compute_barrier: std::sync::Mutex::new(None), + #[cfg(test)] + fill_before_index_barrier: std::sync::Mutex::new(None), + #[cfg(test)] + clear_barrier: std::sync::Mutex::new(None), + #[cfg(test)] + clear_joined: Arc::new(Semaphore::new(0)), }) } @@ -163,6 +393,43 @@ impl MokaBackend { barrier } + #[cfg(test)] + fn install_identity_eviction_barrier(&self) -> Arc { + let barrier = Arc::new(FillBarrier::new()); + *self.identity_eviction_barrier.lock().unwrap_or_else(|p| p.into_inner()) = Some(Arc::clone(&barrier)); + barrier + } + + #[cfg(test)] + fn install_identity_compute_barrier(&self) -> Arc { + let barrier = Arc::new(ComputeBarrier::new()); + *self.identity_compute_barrier.lock().unwrap_or_else(|p| p.into_inner()) = Some(Arc::clone(&barrier)); + barrier + } + + #[cfg(test)] + fn install_fill_before_index_barrier(&self) -> Arc { + let barrier = Arc::new(FillBarrier::new()); + *self + .fill_before_index_barrier + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(Arc::clone(&barrier)); + barrier + } + + #[cfg(test)] + fn install_clear_barrier(&self) -> Arc { + let barrier = Arc::new(FillBarrier::new()); + *self.clear_barrier.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(Arc::clone(&barrier)); + barrier + } + + #[cfg(test)] + async fn wait_until_clear_joined(&self) { + if let Ok(permit) = self.clear_joined.acquire().await { + permit.forget(); + } + } /// Returns the current cache entry count. pub fn entry_count(&self) -> u64 { self.cache.entry_count() @@ -173,6 +440,10 @@ impl MokaBackend { self.cache.weighted_size() } + pub(crate) const fn max_capacity(&self) -> u64 { + self.max_capacity + } + /// Looks up a cached body for the supplied plan. pub async fn lookup_body(&self, plan: &ObjectDataCacheGetPlan) -> ObjectDataCacheLookup { let ObjectDataCacheGetPlan::Cacheable { key } = plan else { @@ -185,11 +456,67 @@ impl MokaBackend { } } + pub(crate) fn reserve_body(&self, plan: &ObjectDataCacheGetPlan) -> Option { + let ObjectDataCacheGetPlan::Cacheable { key } = plan else { + return None; + }; + if projected_weight(key, key.size) > self.max_capacity { + return None; + } + // Register with the clear fence before the caller allocates or reads + // the body. A clear that starts after this point must wait for this + // reservation to publish or be dropped, so an old cold read cannot + // become visible after clear returns. + let fill_generation = self.clear_fence.try_register_fill()?; + let permit = Arc::clone(&self.fill_semaphore).try_acquire_owned().ok()?; + let cache_growth_headroom = self.max_capacity.saturating_sub(self.cache.weighted_size()); + let memory = self.memory_gate.try_claim(key.size, cache_growth_headroom)?; + Some(ObjectDataCacheBodyReservation { + memory, + permit, + fill_generation, + key: key.clone(), + expected_size: key.size, + }) + } + /// Inserts a cached body for the supplied plan. pub async fn fill_body(&self, plan: &ObjectDataCacheGetPlan, bytes: Bytes) -> ObjectDataCacheFillResult { + self.fill_body_inner(plan, bytes, None).await + } + + pub(crate) async fn fill_reserved_body( + &self, + plan: &ObjectDataCacheGetPlan, + body: ObjectDataCacheReservedBody, + ) -> ObjectDataCacheFillResult { let ObjectDataCacheGetPlan::Cacheable { key } = plan else { return ObjectDataCacheFillResult::SkippedNotCacheable; }; + if !body.fill_generation.belongs_to(&self.clear_fence) || body.key != *key { + return ObjectDataCacheFillResult::SkippedNotCacheable; + } + if u64::try_from(body.bytes.len()).unwrap_or(u64::MAX) != body.expected_size { + return ObjectDataCacheFillResult::SkippedSizeMismatch; + } + let (bytes, permit, fill_generation) = body.into_parts(); + self.fill_body_inner(plan, bytes, Some((permit, fill_generation))).await + } + + async fn fill_body_inner( + &self, + plan: &ObjectDataCacheGetPlan, + bytes: Bytes, + reservation: Option<(OwnedSemaphorePermit, FillGenerationGuard)>, + ) -> ObjectDataCacheFillResult { + let is_pre_reserved = reservation.is_some(); + let ObjectDataCacheGetPlan::Cacheable { key } = plan else { + return ObjectDataCacheFillResult::SkippedNotCacheable; + }; + let body_bytes = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + if projected_weight(key, body_bytes) > self.max_capacity { + return ObjectDataCacheFillResult::SkippedNotCacheable; + } // The caller already owns the body, so a non-leader gains nothing by // waiting for another request's leader — it just skips its own fill. @@ -201,24 +528,51 @@ impl MokaBackend { return ObjectDataCacheFillResult::JoinedInflightFill; }; + // Registration and the clear transition share one short state lock, + // closing the check/register race without holding a lock across body + // processing or cache/index I/O. Only the singleflight leader can + // publish, so duplicate fills stay off this global hot-path lock. + let (reserved_permit, fill_generation) = match reservation { + Some((permit, fill_generation)) => (Some(permit), fill_generation), + None => { + let Some(fill_generation) = self.clear_fence.try_register_fill() else { + return leader.finish(ObjectDataCacheFillResult::SkippedInvalidationRace); + }; + (None, fill_generation) + } + }; + + #[cfg(test)] + let fill_before_index_barrier = self + .fill_before_index_barrier + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(); + #[cfg(test)] + if let Some(barrier) = fill_before_index_barrier { + barrier.wait().await; + } + + if !fill_generation.is_current() { + return leader.finish(ObjectDataCacheFillResult::SkippedInvalidationRace); + } + // Bound distinct-key fill concurrency after winning leadership and // before the memory-gate check. Reject rather than queue: queuing would // reintroduce the GET-latency coupling this path is meant to avoid. - let permit = match Arc::clone(&self.fill_semaphore).try_acquire_owned() { - Ok(permit) => permit, - Err(_) => return leader.finish(ObjectDataCacheFillResult::SkippedFillConcurrency), + let permit = match reserved_permit { + Some(permit) => permit, + None => match Arc::clone(&self.fill_semaphore).try_acquire_owned() { + Ok(permit) => permit, + Err(_) => return leader.finish(ObjectDataCacheFillResult::SkippedFillConcurrency), + }, }; - // How much the cache can still grow before it is at capacity. This caps - // the gate's in-window reservation so sustained gross churn (net size - // flat, far below capacity) cannot be mistaken for real memory growth - // and falsely skip fills (backlog#1212). weighted_size() is moka's - // lazily-maintained approximation, which is all this bound needs. + // Keep passing cache growth headroom for compatibility with the public + // gate API. Allocation-scoped reservations deliberately do not cap by + // this value because an evicted entry can still have live body clones. let cache_growth_headroom = self.max_capacity.saturating_sub(self.cache.weighted_size()); - if !self - .memory_gate - .allows_fill(u64::try_from(bytes.len()).unwrap_or(u64::MAX), cache_growth_headroom) - { + if !is_pre_reserved && !self.memory_gate.try_claim_buffered(body_bytes, cache_growth_headroom) { return leader.finish(ObjectDataCacheFillResult::SkippedMemoryPressure); } @@ -232,10 +586,16 @@ impl MokaBackend { }) .await; - // Build the entry up front so its Arc pointer can serve as the - // generation token registered in the index alongside the key. - let entry = Arc::new(ObjectDataCacheEntry::new(bytes)); - let token = Arc::as_ptr(&entry) as ObjectDataCacheKeyToken; + // The counter supplies uniqueness only; cache/index publication provides + // the synchronization, so relaxed ordering is sufficient here. + let generation = match self + .next_generation + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| current.checked_add(1)) + { + Ok(previous) => previous + 1, + Err(_) => return leader.finish(ObjectDataCacheFillResult::SkippedIdentityOverflow), + }; + let entry = Arc::new(ObjectDataCacheEntry::with_generation(bytes, generation)); // Run the register/insert/recheck/undo sequence in a spawned task and // await its handle. The GET path already detaches the whole fill from @@ -244,27 +604,74 @@ impl MokaBackend { // if the enclosing fill future is aborted (e.g. the detached fill task // is cancelled). Aborting the outer future then only detaches this // JoinHandle; the task still finishes the undo, so a stale body cannot - // survive with no index entry. Dropping the leader releases the key. + // survive with no index entry. The owned singleflight leader moves into + // the task as well, keeping later same-key fills out until publication + // or rollback is complete. let cache = self.cache.clone(); let index = Arc::clone(&self.index); let fill_key = key.clone(); let fill_identity = identity.clone(); #[cfg(test)] let fill_barrier = self.fill_barrier.lock().unwrap_or_else(|p| p.into_inner()).clone(); + #[cfg(test)] + let identity_eviction_barrier = self + .identity_eviction_barrier + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clone(); + #[cfg(test)] + let identity_compute_barrier = self + .identity_compute_barrier + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clone(); let handle = tokio::spawn(async move { // Hold the fill-concurrency permit until the register/insert/undo // sequence completes, then release it on task exit. let _permit = permit; + // Keep the generation registered through the complete + // register/publish/recheck/undo sequence. If the outer caller is + // cancelled, this owned task still acknowledges the clear fence. + let fill_generation = fill_generation; + if !fill_generation.is_current() { + return leader.finish(ObjectDataCacheFillResult::SkippedInvalidationRace); + } // Register the key in the identity index BEFORE the entry becomes // visible in the cache, so a concurrent invalidation always finds it. - match index.insert(fill_identity.clone(), fill_key.clone(), token).await { - ObjectDataCacheIndexInsertResult::Inserted { evicted_keys } => { - for evicted in evicted_keys { - cache.remove(&evicted).await; + match index + .insert_generation(fill_identity.clone(), fill_key.clone(), generation) + .await + { + ObjectDataCacheGenerationalInsertResult::Inserted { evicted } => { + for evicted in evicted { + #[cfg(test)] + if let Some(barrier) = &identity_eviction_barrier { + barrier.wait().await; + } + let expected_generation = evicted.generation; + #[cfg(test)] + let compute_barrier = identity_compute_barrier.clone(); + let _ = cache + .entry(evicted.key) + .and_compute_with(move |current| { + let remove = current + .as_ref() + .is_some_and(|entry| entry.value().generation() == expected_generation); + #[cfg(test)] + if let Some(barrier) = compute_barrier { + barrier.wait(); + } + ready(if remove { compute::Op::Remove } else { compute::Op::Nop }) + }) + .await; + #[cfg(test)] + if let Some(barrier) = &identity_eviction_barrier { + barrier.complete(); + } } } - ObjectDataCacheIndexInsertResult::Duplicate => {} + ObjectDataCacheGenerationalInsertResult::Duplicate => {} } #[cfg(test)] @@ -272,22 +679,35 @@ impl MokaBackend { barrier.wait().await; } - cache.insert(fill_key.clone(), entry).await; + if !fill_generation.is_current() { + // Leave the registered key for the owned clear drain. Besides + // making the acknowledgement visible to clear, this preserves + // the pre-existing invalidation outcome count for admin stats. + return leader.finish(ObjectDataCacheFillResult::SkippedInvalidationRace); + } + + let _ = cache + .entry(fill_key.clone()) + .and_compute_with(move |_| ready(compute::Op::Put(entry))) + .await; // An invalidation may have raced between the index and cache // inserts; re-check the index and undo the fill so the stale // body cannot outlive the invalidation. - if index.contains_key(&fill_identity, &fill_key).await { + let indexed = index.contains_generation(&fill_identity, &fill_key, generation).await; + let result = if fill_generation.is_current() && indexed { ObjectDataCacheFillResult::Inserted } else { cache.remove(&fill_key).await; + if fill_generation.is_current() { + index.remove_generation(&fill_identity, &fill_key, generation).await; + } ObjectDataCacheFillResult::SkippedInvalidationRace - } + }; + leader.finish(result) }); - let result = handle.await.unwrap_or(ObjectDataCacheFillResult::SkippedInvalidationRace); - - leader.finish(result) + handle.await.unwrap_or(ObjectDataCacheFillResult::SkippedInvalidationRace) } /// Conservatively invalidates all cached keys matching the object identity. @@ -338,15 +758,52 @@ impl MokaBackend { Self::invalidation_result(removed) } - /// Drops every cached body and resets the identity index. The index scan - /// yields the tracked key count for the outcome label, then the cache is - /// drained explicitly so even a body that is no longer tracked by the - /// index cannot survive the clear path. Rare admin `clear()` path only. + /// Drops every cached body and resets the identity index. Concurrent clear + /// callers join one owned drain, so cancelling a caller cannot cancel the + /// operation or reopen its fill generation. Lookups remain lock-free and + /// may observe entries until the drain removes them; fills started while + /// the clear is active skip publication. pub async fn clear(&self) -> ObjectDataCacheInvalidationResult { - let keys_to_remove = self.index.remove_matching(|_| true).await; + let (operation, should_start) = self.clear_fence.begin_clear(); + #[cfg(test)] + if !should_start { + self.clear_joined.add_permits(1); + } + if should_start { + let cache = self.cache.clone(); + let index = Arc::clone(&self.index); + let clear_fence = Arc::clone(&self.clear_fence); + let owned_operation = Arc::clone(&operation); + #[cfg(test)] + let clear_barrier = self + .clear_barrier + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .take(); + + tokio::spawn(async move { + #[cfg(test)] + if let Some(barrier) = clear_barrier { + barrier.wait().await; + } + + clear_fence.wait_for_active_fills().await; + let result = Self::drain_all(cache, index).await; + clear_fence.finish_clear(&owned_operation, result); + }); + } + + operation.wait().await + } + + async fn drain_all( + cache: Cache>, + index: Arc, + ) -> ObjectDataCacheInvalidationResult { + let keys_to_remove = index.remove_matching(|_| true).await; let removed = keys_to_remove.len(); for key in keys_to_remove { - self.cache.remove(&key).await; + cache.remove(&key).await; } // Moka maintenance is still needed to retire queued removals, but clear @@ -356,16 +813,16 @@ impl MokaBackend { // entries visible to iteration and a fallback invalidate_all fence so // delayed internal writes cannot strand a counted entry. for _ in 0..256 { - self.cache.run_pending_tasks().await; - let lingering_keys: Vec<_> = self.cache.iter().map(|(key, _)| key.as_ref().clone()).collect(); + cache.run_pending_tasks().await; + let lingering_keys: Vec<_> = cache.iter().map(|(key, _)| key.as_ref().clone()).collect(); for key in lingering_keys { - self.cache.remove(&key).await; + cache.remove(&key).await; } - self.cache.invalidate_all(); - self.cache.run_pending_tasks().await; + cache.invalidate_all(); + cache.run_pending_tasks().await; - if self.cache.entry_count() == 0 { + if cache.entry_count() == 0 { break; } tokio::task::yield_now().await; @@ -384,7 +841,7 @@ impl MokaBackend { #[cfg(test)] mod tests { - use super::MokaBackend; + use super::{ClearFence, FillGenerationGuard, MokaBackend}; use crate::cache::{ ObjectDataCacheFillResult, ObjectDataCacheGetPlan, ObjectDataCacheInvalidationResult, ObjectDataCacheLookup, }; @@ -423,8 +880,12 @@ mod tests { } fn cacheable_plan(object: &str, etag: &str) -> ObjectDataCacheGetPlan { + cacheable_plan_with_size(object, etag, 5) + } + + fn cacheable_plan_with_size(object: &str, etag: &str, size: u64) -> ObjectDataCacheGetPlan { ObjectDataCacheGetPlan::Cacheable { - key: ObjectDataCacheKey::new("bucket", object, None, etag, 5, ObjectDataCacheBodyVariant::FullObjectPlainV1), + key: ObjectDataCacheKey::new("bucket", object, None, etag, size, ObjectDataCacheBodyVariant::FullObjectPlainV1), } } @@ -434,6 +895,19 @@ mod tests { } } + fn identity_budget_fixture() -> (Arc, ObjectDataCacheGetPlan, ObjectDataCacheGetPlan, ObjectDataCacheGetPlan) { + let mut config = enabled_config(); + config.ttl = Duration::from_secs(30); + config.time_to_idle = Duration::from_secs(30); + config.identity_keys_max = 2; + ( + Arc::new(MokaBackend::new(&config, Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build")), + versioned_plan("object", "v1", "etag-1"), + versioned_plan("object", "v2", "etag-2"), + versioned_plan("object", "v3", "etag-3"), + ) + } + #[tokio::test] async fn moka_backend_round_trips_cached_body() { let backend = @@ -447,6 +921,44 @@ mod tests { assert!(matches!(lookup, ObjectDataCacheLookup::Hit(ref bytes) if bytes.as_ref() == b"hello")); } + #[tokio::test] + async fn moka_backend_never_reports_inserted_for_overweight_entry() { + let mut config = enabled_config(); + config.max_bytes = 8 * 1024; + config.max_entry_bytes = 4 * 1024; + let backend = MokaBackend::new(&config, Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"); + // Projected weight is 8193 bytes, one byte over max_capacity. + let object = "o".repeat(4022); + let plan = cacheable_plan_with_size(&object, "e", 4 * 1024); + let body = Bytes::from(vec![0_u8; 4 * 1024]); + + let fill = backend.fill_body(&plan, body).await; + backend.cache.run_pending_tasks().await; + + assert_eq!(fill, ObjectDataCacheFillResult::SkippedNotCacheable); + assert!(matches!(backend.lookup_body(&plan).await, ObjectDataCacheLookup::Miss)); + } + + #[tokio::test] + async fn accepted_boundary_entry_remains_hit_after_pending_maintenance() { + let mut config = enabled_config(); + config.max_bytes = 8 * 1024; + config.max_entry_bytes = 4 * 1024; + let backend = MokaBackend::new(&config, Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"); + // Projected weight is exactly max_capacity: 4096 + 6 + 4021 + 4 + 1 + 64. + let object = "o".repeat(4021); + let plan = cacheable_plan_with_size(&object, "e", 4 * 1024); + let body = Bytes::from(vec![0_u8; 4 * 1024]); + + let fill = backend.fill_body(&plan, body).await; + backend.cache.run_pending_tasks().await; + let lookup = backend.lookup_body(&plan).await; + + assert_eq!(fill, ObjectDataCacheFillResult::Inserted); + assert_eq!(backend.weighted_size(), config.max_bytes, "Moka must charge the exact projected weight"); + assert!(matches!(lookup, ObjectDataCacheLookup::Hit(ref bytes) if bytes.len() == 4 * 1024)); + } + #[tokio::test] async fn moka_backend_invalidates_matching_identity() { let backend = @@ -587,6 +1099,267 @@ mod tests { assert_eq!(backend.clear().await, ObjectDataCacheInvalidationResult::NoOp); } + #[tokio::test] + async fn clear_fences_fill_started_before_index_registration() { + let backend = Arc::new( + MokaBackend::new(&enabled_config(), Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"), + ); + let fill_barrier = backend.install_fill_before_index_barrier(); + let clear_barrier = backend.install_clear_barrier(); + let plan = cacheable_plan("object", "etag-a"); + + let fill_backend = Arc::clone(&backend); + let fill_plan = plan.clone(); + let fill = tokio::spawn(async move { fill_backend.fill_body(&fill_plan, Bytes::from_static(b"hello")).await }); + fill_barrier.wait_until_reached().await; + + let clear_backend = Arc::clone(&backend); + let clear = tokio::spawn(async move { clear_backend.clear().await }); + clear_barrier.wait_until_reached().await; + clear_barrier.release(); + assert!( + !clear.is_finished(), + "clear must wait for a pre-registration fill to acknowledge the fence" + ); + + fill_barrier.release(); + assert_eq!( + fill.await.expect("fill task should finish"), + ObjectDataCacheFillResult::SkippedInvalidationRace + ); + assert_eq!(clear.await.expect("clear task should finish"), ObjectDataCacheInvalidationResult::NoOp); + assert!(matches!(backend.lookup_body(&plan).await, ObjectDataCacheLookup::Miss)); + assert_eq!(backend.index.identity_count().await, 0); + } + + #[tokio::test] + async fn clear_fences_body_reserved_before_materialization() { + let backend = Arc::new( + MokaBackend::new(&enabled_config(), Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"), + ); + let clear_barrier = backend.install_clear_barrier(); + let plan = cacheable_plan("reserved-object", "etag-a"); + let reservation = backend + .reserve_body(&plan) + .expect("reservation must register before body reads"); + + let clear_backend = Arc::clone(&backend); + let clear = tokio::spawn(async move { clear_backend.clear().await }); + clear_barrier.wait_until_reached().await; + clear_barrier.release(); + assert!( + !clear.is_finished(), + "clear must wait while a pre-materialization reservation can still publish" + ); + + let body = reservation.wrap_bytes(Bytes::from_static(b"hello")); + assert_eq!( + backend.fill_reserved_body(&plan, body).await, + ObjectDataCacheFillResult::SkippedInvalidationRace + ); + assert_eq!(clear.await.expect("clear task should finish"), ObjectDataCacheInvalidationResult::NoOp); + assert!(matches!(backend.lookup_body(&plan).await, ObjectDataCacheLookup::Miss)); + assert_eq!(backend.index.identity_count().await, 0); + } + + #[tokio::test] + async fn clear_fences_fill_registered_before_cache_publish() { + let backend = Arc::new( + MokaBackend::new(&enabled_config(), Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"), + ); + let fill_barrier = backend.install_fill_barrier(); + let clear_barrier = backend.install_clear_barrier(); + let plan = cacheable_plan("object", "etag-a"); + + let fill_backend = Arc::clone(&backend); + let fill_plan = plan.clone(); + let fill = tokio::spawn(async move { fill_backend.fill_body(&fill_plan, Bytes::from_static(b"hello")).await }); + fill_barrier.wait_until_reached().await; + + let clear_backend = Arc::clone(&backend); + let clear = tokio::spawn(async move { clear_backend.clear().await }); + clear_barrier.wait_until_reached().await; + clear_barrier.release(); + assert!(!clear.is_finished(), "clear must wait for a registered fill to acknowledge the fence"); + + fill_barrier.release(); + assert_eq!( + fill.await.expect("fill task should finish"), + ObjectDataCacheFillResult::SkippedInvalidationRace + ); + assert_eq!( + clear.await.expect("clear task should finish"), + ObjectDataCacheInvalidationResult::Removed { keys: 1 } + ); + assert!(matches!(backend.lookup_body(&plan).await, ObjectDataCacheLookup::Miss)); + assert_eq!(backend.index.identity_count().await, 0); + } + + #[tokio::test] + async fn fill_started_during_clear_skips_without_waiting() { + let backend = Arc::new( + MokaBackend::new(&enabled_config(), Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"), + ); + let clear_barrier = backend.install_clear_barrier(); + let clear_backend = Arc::clone(&backend); + let clear = tokio::spawn(async move { clear_backend.clear().await }); + clear_barrier.wait_until_reached().await; + + assert_eq!( + backend + .fill_body(&cacheable_plan("object", "etag-a"), Bytes::from_static(b"hello")) + .await, + ObjectDataCacheFillResult::SkippedInvalidationRace + ); + + clear_barrier.release(); + assert_eq!(clear.await.expect("clear task should finish"), ObjectDataCacheInvalidationResult::NoOp); + } + + #[tokio::test] + async fn concurrent_clear_joins_same_operation() { + let backend = Arc::new( + MokaBackend::new(&enabled_config(), Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"), + ); + let _ = backend + .fill_body(&cacheable_plan("object", "etag-a"), Bytes::from_static(b"hello")) + .await; + let clear_barrier = backend.install_clear_barrier(); + + let first_backend = Arc::clone(&backend); + let first = tokio::spawn(async move { first_backend.clear().await }); + clear_barrier.wait_until_reached().await; + let second_backend = Arc::clone(&backend); + let second = tokio::spawn(async move { second_backend.clear().await }); + backend.wait_until_clear_joined().await; + assert!(!second.is_finished(), "a concurrent clear must join the active drain"); + + clear_barrier.release(); + let expected = ObjectDataCacheInvalidationResult::Removed { keys: 1 }; + assert_eq!(first.await.expect("first clear should finish"), expected); + assert_eq!(second.await.expect("second clear should finish"), expected); + } + + #[tokio::test] + async fn cancelled_clear_does_not_reopen_old_generation() { + let backend = Arc::new( + MokaBackend::new(&enabled_config(), Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"), + ); + let clear_barrier = backend.install_clear_barrier(); + let first_backend = Arc::clone(&backend); + let first = tokio::spawn(async move { first_backend.clear().await }); + clear_barrier.wait_until_reached().await; + first.abort(); + + assert_eq!( + backend + .fill_body(&cacheable_plan("during-clear", "etag-a"), Bytes::from_static(b"hello")) + .await, + ObjectDataCacheFillResult::SkippedInvalidationRace + ); + + let second_backend = Arc::clone(&backend); + let second = tokio::spawn(async move { second_backend.clear().await }); + backend.wait_until_clear_joined().await; + second.abort(); + clear_barrier.release(); + assert_eq!(backend.clear().await, ObjectDataCacheInvalidationResult::NoOp); + assert_eq!( + backend + .fill_body(&cacheable_plan("after-clear", "etag-b"), Bytes::from_static(b"hello")) + .await, + ObjectDataCacheFillResult::Inserted + ); + } + + #[tokio::test] + async fn clear_completion_reopens_generation_atomically() { + let fence = Arc::new(ClearFence::new()); + let (operation, should_start) = fence.begin_clear(); + assert!(should_start); + + fence.finish_clear(&operation, ObjectDataCacheInvalidationResult::NoOp); + + assert_eq!(operation.wait().await, ObjectDataCacheInvalidationResult::NoOp); + assert!(fence.try_register_fill().is_some()); + } + + #[tokio::test] + async fn cancelled_fill_acknowledges_clear_fence() { + let backend = Arc::new( + MokaBackend::new(&enabled_config(), Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"), + ); + let fill_barrier = backend.install_fill_barrier(); + let plan = cacheable_plan("object", "etag-a"); + let fill_backend = Arc::clone(&backend); + let fill_plan = plan.clone(); + let fill = tokio::spawn(async move { fill_backend.fill_body(&fill_plan, Bytes::from_static(b"hello")).await }); + fill_barrier.wait_until_reached().await; + fill.abort(); + let _ = fill.await; + assert_eq!( + backend.clear_fence.state.lock().unwrap().active_fills, + 1, + "the detached publish task must retain its active-fill acknowledgement" + ); + assert_eq!( + tokio::time::timeout(Duration::from_millis(100), backend.fill_body(&plan, Bytes::from_static(b"newer")),) + .await + .expect("same-key retry must not wait for detached publication"), + ObjectDataCacheFillResult::JoinedInflightFill, + "the detached publication must retain same-key singleflight ownership", + ); + + let clear_barrier = backend.install_clear_barrier(); + let clear_backend = Arc::clone(&backend); + let clear = tokio::spawn(async move { clear_backend.clear().await }); + clear_barrier.wait_until_reached().await; + clear_barrier.release(); + assert!(!clear.is_finished(), "clear must wait for the detached publish task"); + + fill_barrier.release(); + assert_eq!( + clear.await.expect("clear task should finish"), + ObjectDataCacheInvalidationResult::Removed { keys: 1 } + ); + assert!(matches!(backend.lookup_body(&plan).await, ObjectDataCacheLookup::Miss)); + assert_eq!(backend.index.identity_count().await, 0); + } + + #[test] + fn clear_fence_generation_overflow_is_fail_closed() { + for generation in [u64::MAX, u64::MAX - 1] { + let fence = Arc::new(ClearFence::new()); + fence.state.lock().unwrap().generation = Some(generation); + + let (operation, should_start) = fence.begin_clear(); + assert!(should_start); + fence.finish_clear(&operation, ObjectDataCacheInvalidationResult::NoOp); + + assert!(fence.try_register_fill().is_none()); + assert!(fence.state.lock().unwrap().generation.is_none()); + let (joined, should_start) = fence.begin_clear(); + assert!(!should_start, "a failed generation must retain the permanent clear fence"); + assert!(Arc::ptr_eq(&operation, &joined)); + } + } + + #[test] + fn clear_fence_active_counter_failure_is_fail_closed() { + let overflow = Arc::new(ClearFence::new()); + overflow.state.lock().unwrap().active_fills = usize::MAX; + assert!(overflow.try_register_fill().is_none()); + assert!(overflow.state.lock().unwrap().generation.is_none()); + + let underflow = Arc::new(ClearFence::new()); + drop(FillGenerationGuard { + fence: Arc::clone(&underflow), + generation: 0, + }); + assert!(underflow.state.lock().unwrap().generation.is_none()); + assert!(underflow.try_register_fill().is_none()); + } + #[tokio::test] async fn moka_backend_expires_entries_by_ttl() { let backend = @@ -663,6 +1436,94 @@ mod tests { assert_eq!(stats.snapshot().memory_pressure_events, 1); } + #[test] + fn reserve_body_rejects_before_materialization_under_memory_pressure() { + let stats = Arc::new(ObjectDataCacheStats::default()); + let backend = MokaBackend::new(&memory_gated_config(), Arc::clone(&stats)).expect("moka backend should build"); + backend + .memory_gate + .set_test_snapshot(Some(crate::memory::ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 100, + })); + + assert!(backend.reserve_body(&cacheable_plan("object", "etag-a")).is_none()); + assert_eq!(backend.memory_gate.claimed_bytes_for_test(), 0); + assert_eq!(stats.snapshot().memory_pressure_events, 1); + } + + #[tokio::test] + async fn reserved_body_claim_survives_cache_eviction_until_last_clone_drops() { + let backend = MokaBackend::new(&memory_gated_config(), Arc::new(ObjectDataCacheStats::default())) + .expect("moka backend should build"); + backend + .memory_gate + .set_test_snapshot(Some(crate::memory::ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + })); + let plan = cacheable_plan("object", "etag-a"); + let body = backend + .reserve_body(&plan) + .expect("the allocation should fit the memory budget") + .wrap_bytes(Bytes::from_static(b"hello")); + assert_eq!(backend.memory_gate.claimed_bytes_for_test(), 5); + + assert_eq!(backend.fill_reserved_body(&plan, body).await, ObjectDataCacheFillResult::Inserted); + assert_eq!( + backend.memory_gate.claimed_bytes_for_test(), + 5, + "reserved fill must not claim the same allocation twice" + ); + let ObjectDataCacheLookup::Hit(response_body) = backend.lookup_body(&plan).await else { + panic!("the reserved body should be cached"); + }; + let _ = backend + .invalidate_object(&ObjectDataCacheIdentity::new("bucket", "object")) + .await; + backend.cache.run_pending_tasks().await; + assert_eq!( + backend.memory_gate.claimed_bytes_for_test(), + 5, + "the response clone still owns the cached allocation" + ); + + drop(response_body); + backend.cache.run_pending_tasks().await; + assert_eq!(backend.memory_gate.claimed_bytes_for_test(), 0); + } + + #[tokio::test] + async fn moka_buffered_fill_holds_conservative_claim_until_refresh() { + let backend = MokaBackend::new(&memory_gated_config(), Arc::new(ObjectDataCacheStats::default())) + .expect("moka backend should build"); + let snapshot = crate::memory::ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + }; + backend.memory_gate.set_test_snapshot(Some(snapshot)); + let plan = cacheable_plan("object", "etag-a"); + + assert_eq!( + backend.fill_body(&plan, Bytes::from_static(b"hello")).await, + ObjectDataCacheFillResult::Inserted + ); + assert_eq!(backend.memory_gate.claimed_bytes_for_test(), 5); + + let _ = backend + .invalidate_object(&ObjectDataCacheIdentity::new("bucket", "object")) + .await; + backend.cache.run_pending_tasks().await; + assert_eq!( + backend.memory_gate.claimed_bytes_for_test(), + 5, + "an already-buffered body cannot be retrofitted with an allocation owner" + ); + + backend.memory_gate.store_raw_snapshot_for_test(snapshot); + assert_eq!(backend.memory_gate.claimed_bytes_for_test(), 0); + } + // ODC-15 + ODC-18: a concurrent duplicate fill for the same key must NOT // wait for the in-flight leader (it already owns the body), and it must be // reported as JoinedInflightFill — not Inserted — so it records no fill @@ -808,15 +1669,7 @@ mod tests { // and still caches the newest body, instead of clearing the whole identity. #[tokio::test] async fn moka_backend_identity_budget_evicts_oldest_and_caches_newest() { - let mut config = enabled_config(); - config.ttl = Duration::from_secs(30); - config.time_to_idle = Duration::from_secs(30); - config.identity_keys_max = 2; - let backend = MokaBackend::new(&config, Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"); - - let plan_v1 = versioned_plan("object", "v1", "etag-1"); - let plan_v2 = versioned_plan("object", "v2", "etag-2"); - let plan_v3 = versioned_plan("object", "v3", "etag-3"); + let (backend, plan_v1, plan_v2, plan_v3) = identity_budget_fixture(); assert_eq!( backend.fill_body(&plan_v1, Bytes::from_static(b"aaaaa")).await, @@ -843,6 +1696,216 @@ mod tests { ); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn identity_budget_eviction_preserves_refill_before_conditional_delete() { + let (backend, plan_v1, plan_v2, plan_v3) = identity_budget_fixture(); + + assert_eq!( + backend.fill_body(&plan_v1, Bytes::from_static(b"old-a")).await, + ObjectDataCacheFillResult::Inserted + ); + assert_eq!( + backend.fill_body(&plan_v2, Bytes::from_static(b"body-b")).await, + ObjectDataCacheFillResult::Inserted + ); + + let barrier = backend.install_identity_eviction_barrier(); + let fill_v3 = { + let backend = Arc::clone(&backend); + let plan_v3 = plan_v3.clone(); + tokio::spawn(async move { backend.fill_body(&plan_v3, Bytes::from_static(b"body-c")).await }) + }; + + barrier.wait_until_reached().await; + *backend.identity_eviction_barrier.lock().unwrap_or_else(|p| p.into_inner()) = None; + assert_eq!( + backend.fill_body(&plan_v1, Bytes::from_static(b"new-a")).await, + ObjectDataCacheFillResult::Inserted + ); + assert!(matches!( + backend.lookup_body(&plan_v1).await, + ObjectDataCacheLookup::Hit(ref bytes) if bytes.as_ref() == b"new-a" + )); + + barrier.release(); + assert_eq!(fill_v3.await.expect("v3 fill task should complete"), ObjectDataCacheFillResult::Inserted); + assert!(matches!( + backend.lookup_body(&plan_v1).await, + ObjectDataCacheLookup::Hit(ref bytes) if bytes.as_ref() == b"new-a" + )); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn identity_budget_conditional_delete_is_atomic_with_refill() { + let (backend, plan_v1, plan_v2, plan_v3) = identity_budget_fixture(); + + assert_eq!( + backend.fill_body(&plan_v1, Bytes::from_static(b"old-a")).await, + ObjectDataCacheFillResult::Inserted + ); + assert_eq!( + backend.fill_body(&plan_v2, Bytes::from_static(b"body-b")).await, + ObjectDataCacheFillResult::Inserted + ); + + let compute_barrier = backend.install_identity_compute_barrier(); + let fill_v3 = { + let backend = Arc::clone(&backend); + let plan_v3 = plan_v3.clone(); + tokio::spawn(async move { backend.fill_body(&plan_v3, Bytes::from_static(b"body-c")).await }) + }; + compute_barrier.wait_until_reached().await; + *backend.identity_compute_barrier.lock().unwrap_or_else(|p| p.into_inner()) = None; + + let fill_barrier = backend.install_fill_barrier(); + let refill_v1 = { + let backend = Arc::clone(&backend); + let plan_v1 = plan_v1.clone(); + tokio::spawn(async move { backend.fill_body(&plan_v1, Bytes::from_static(b"new-a")).await }) + }; + fill_barrier.wait_until_reached().await; + fill_barrier.release(); + tokio::task::yield_now().await; + assert!(!refill_v1.is_finished(), "the refill must serialize behind the same-key compute"); + + compute_barrier.release(); + assert_eq!(fill_v3.await.expect("v3 fill task should complete"), ObjectDataCacheFillResult::Inserted); + assert_eq!( + refill_v1.await.expect("v1 refill task should complete"), + ObjectDataCacheFillResult::Inserted + ); + assert!(matches!( + backend.lookup_body(&plan_v1).await, + ObjectDataCacheLookup::Hit(ref bytes) if bytes.as_ref() == b"new-a" + )); + } + + #[tokio::test] + async fn identity_budget_eviction_keeps_index_and_cache_generation_consistent() { + let (backend, plan_v1, plan_v2, plan_v3) = identity_budget_fixture(); + + assert_eq!( + backend.fill_body(&plan_v1, Bytes::from_static(b"body-a")).await, + ObjectDataCacheFillResult::Inserted + ); + assert_eq!( + backend.fill_body(&plan_v2, Bytes::from_static(b"body-b")).await, + ObjectDataCacheFillResult::Inserted + ); + assert_eq!( + backend.fill_body(&plan_v3, Bytes::from_static(b"body-c")).await, + ObjectDataCacheFillResult::Inserted + ); + + let ObjectDataCacheGetPlan::Cacheable { key: key_v1 } = &plan_v1 else { + panic!("versioned plan should be cacheable"); + }; + let ObjectDataCacheGetPlan::Cacheable { key: key_v2 } = &plan_v2 else { + panic!("versioned plan should be cacheable"); + }; + let ObjectDataCacheGetPlan::Cacheable { key: key_v3 } = &plan_v3 else { + panic!("versioned plan should be cacheable"); + }; + let identity = ObjectDataCacheIdentity::new("bucket", "object"); + let entry_v2 = backend.cache.get(key_v2).await.expect("v2 should remain cached"); + let entry_v3 = backend.cache.get(key_v3).await.expect("v3 should remain cached"); + + assert!(backend.cache.get(key_v1).await.is_none(), "evicted v1 must not remain cached"); + assert!(!backend.index.contains_key(&identity, key_v1).await, "evicted v1 must not remain indexed"); + assert!( + backend + .index + .contains_generation(&identity, key_v2, entry_v2.generation()) + .await, + "v2 index generation must match its cache entry" + ); + assert!( + backend + .index + .contains_generation(&identity, key_v3, entry_v3.generation()) + .await, + "v3 index generation must match its cache entry" + ); + } + + #[tokio::test] + async fn cache_generation_is_monotonic_across_clear_and_overflow_fails_closed() { + let mut config = enabled_config(); + config.ttl = Duration::from_secs(30); + config.time_to_idle = Duration::from_secs(30); + let backend = MokaBackend::new(&config, Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"); + let plan_v1 = versioned_plan("object", "v1", "etag-1"); + let plan_v2 = versioned_plan("object", "v2", "etag-2"); + let ObjectDataCacheGetPlan::Cacheable { key: key_v1 } = &plan_v1 else { + panic!("versioned plan should be cacheable"); + }; + + assert_eq!( + backend.fill_body(&plan_v1, Bytes::from_static(b"old-a")).await, + ObjectDataCacheFillResult::Inserted + ); + let first_generation = backend + .cache + .get(key_v1) + .await + .expect("first generation should be cached") + .generation(); + assert_eq!(backend.clear().await, ObjectDataCacheInvalidationResult::Removed { keys: 1 }); + assert_eq!( + backend.fill_body(&plan_v1, Bytes::from_static(b"new-a")).await, + ObjectDataCacheFillResult::Inserted + ); + let second_generation = backend + .cache + .get(key_v1) + .await + .expect("second generation should be cached") + .generation(); + assert!(second_generation > first_generation, "clear must not reset the generation counter"); + + backend.next_generation.store(u64::MAX, std::sync::atomic::Ordering::Relaxed); + assert_eq!( + backend.fill_body(&plan_v2, Bytes::from_static(b"body-b")).await, + ObjectDataCacheFillResult::SkippedIdentityOverflow + ); + assert!(matches!(backend.lookup_body(&plan_v2).await, ObjectDataCacheLookup::Miss)); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn identity_budget_eviction_preserves_refill_after_caller_cancellation() { + let (backend, plan_v1, plan_v2, plan_v3) = identity_budget_fixture(); + + assert_eq!( + backend.fill_body(&plan_v1, Bytes::from_static(b"old-a")).await, + ObjectDataCacheFillResult::Inserted + ); + assert_eq!( + backend.fill_body(&plan_v2, Bytes::from_static(b"body-b")).await, + ObjectDataCacheFillResult::Inserted + ); + + let barrier = backend.install_identity_eviction_barrier(); + let fill_v3 = { + let backend = Arc::clone(&backend); + let plan_v3 = plan_v3.clone(); + tokio::spawn(async move { backend.fill_body(&plan_v3, Bytes::from_static(b"body-c")).await }) + }; + barrier.wait_until_reached().await; + fill_v3.abort(); + *backend.identity_eviction_barrier.lock().unwrap_or_else(|p| p.into_inner()) = None; + assert_eq!( + backend.fill_body(&plan_v1, Bytes::from_static(b"new-a")).await, + ObjectDataCacheFillResult::Inserted + ); + + barrier.release(); + barrier.wait_until_completed().await; + assert!(matches!( + backend.lookup_body(&plan_v1).await, + ObjectDataCacheLookup::Hit(ref bytes) if bytes.as_ref() == b"new-a" + )); + } + // ODC-31: the fill-vs-invalidation recheck. Pause the fill between the index // insert and the cache insert, invalidate the identity, then resume; the fill // must undo itself and report the race, leaving nothing cached. diff --git a/crates/object-data-cache/src/singleflight.rs b/crates/object-data-cache/src/singleflight.rs index f7ec5c363..6aa0dba3f 100644 --- a/crates/object-data-cache/src/singleflight.rs +++ b/crates/object-data-cache/src/singleflight.rs @@ -33,7 +33,7 @@ fn lock_fills(fills: &FillSet) -> std::sync::MutexGuard<'_, HashSet, stats: Arc, } @@ -41,7 +41,7 @@ impl ObjectDataCacheSingleflight { /// Creates a new singleflight controller. pub fn new(stats: Arc) -> Self { Self { - fills: Mutex::new(HashSet::new()), + fills: Arc::new(Mutex::new(HashSet::new())), stats, } } @@ -53,7 +53,7 @@ impl ObjectDataCacheSingleflight { /// [`Busy`](ObjectDataCacheSingleflightAcquire::Busy). The caller already /// owns the body, so a `Busy` outcome skips the redundant fill rather than /// waiting for another request's leader to finish. - pub fn try_acquire(&self, key: ObjectDataCacheKey) -> ObjectDataCacheSingleflightAcquire<'_> { + pub fn try_acquire(&self, key: ObjectDataCacheKey) -> ObjectDataCacheSingleflightAcquire<'static> { // Keep the critical section to the map mutation only; emit metrics after // dropping the guard so the recorder round-trip never serializes fills. let inflight_len = { @@ -75,9 +75,10 @@ impl ObjectDataCacheSingleflight { set_inflight_fills(&self.stats, "moka", len); ObjectDataCacheSingleflightAcquire::Leader(ObjectDataCacheSingleflightLeader { key, - fills: &self.fills, + fills: Arc::clone(&self.fills), stats: Arc::clone(&self.stats), finished: false, + lifetime: std::marker::PhantomData, }) } } @@ -104,12 +105,13 @@ pub enum ObjectDataCacheSingleflightAcquire<'a> { /// Leader handle for a singleflight fill operation. pub struct ObjectDataCacheSingleflightLeader<'a> { key: ObjectDataCacheKey, - fills: &'a FillSet, + fills: Arc, stats: Arc, finished: bool, + lifetime: std::marker::PhantomData<&'a ()>, } -impl<'a> ObjectDataCacheSingleflightLeader<'a> { +impl ObjectDataCacheSingleflightLeader<'_> { /// Completes the leader operation and releases the key. pub fn finish(mut self, result: ObjectDataCacheFillResult) -> ObjectDataCacheFillResult { self.remove_entry(); @@ -121,7 +123,7 @@ impl<'a> ObjectDataCacheSingleflightLeader<'a> { // Capture the length under the guard, then emit the gauge after dropping // it so the recorder round-trip stays out of the critical section. let len = { - let mut fills = lock_fills(self.fills); + let mut fills = lock_fills(&self.fills); fills.remove(&self.key); fills.len() }; @@ -129,7 +131,7 @@ impl<'a> ObjectDataCacheSingleflightLeader<'a> { } } -impl<'a> Drop for ObjectDataCacheSingleflightLeader<'a> { +impl Drop for ObjectDataCacheSingleflightLeader<'_> { fn drop(&mut self) { // A leader dropped without finish() was cancelled mid-fill (e.g. the // fill task was aborted). Release the key so a later fill can become the diff --git a/crates/object-data-cache/src/starshard_index.rs b/crates/object-data-cache/src/starshard_index.rs index d867b222e..fdfe7962e 100644 --- a/crates/object-data-cache/src/starshard_index.rs +++ b/crates/object-data-cache/src/starshard_index.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::index::{ObjectDataCacheIndexInsertResult, ObjectDataCacheKeySet, ObjectDataCacheKeyToken}; +use crate::index::{ + ObjectDataCacheGeneration, ObjectDataCacheGenerationalInsertResult, ObjectDataCacheIndexInsertResult, ObjectDataCacheKeySet, + ObjectDataCacheKeyToken, +}; use crate::key::{ObjectDataCacheIdentity, ObjectDataCacheKey}; use starshard::{AsyncShardedHashMap, DEFAULT_SHARDS, SnapshotMode}; use std::collections::hash_map::RandomState; @@ -56,6 +59,21 @@ impl StarshardIdentityIndex { key: ObjectDataCacheKey, token: ObjectDataCacheKeyToken, ) -> ObjectDataCacheIndexInsertResult { + let generation = u64::try_from(token).unwrap_or(u64::MAX); + match self.insert_generation(identity, key, generation).await { + ObjectDataCacheGenerationalInsertResult::Inserted { evicted } => ObjectDataCacheIndexInsertResult::Inserted { + evicted_keys: evicted.into_iter().map(|entry| entry.key).collect(), + }, + ObjectDataCacheGenerationalInsertResult::Duplicate => ObjectDataCacheIndexInsertResult::Duplicate, + } + } + + pub(crate) async fn insert_generation( + &self, + identity: ObjectDataCacheIdentity, + key: ObjectDataCacheKey, + generation: ObjectDataCacheGeneration, + ) -> ObjectDataCacheGenerationalInsertResult { let max_keys = self.max_keys_per_identity; loop { let mut outcome = None; @@ -65,7 +83,7 @@ impl StarshardIdentityIndex { let _ = self .by_object .compute_if_present(&identity, move |mut key_set| { - let result = key_set.insert(key, token, max_keys); + let result = key_set.insert_generation(key, generation, max_keys); // Insert never empties the set (it either dedups or // bounded-evicts and adds the new key), but keep the // guard so an already-empty entry is not republished. @@ -80,7 +98,7 @@ impl StarshardIdentityIndex { // Identity not tracked yet: publish a fresh single-key set. let mut fresh = ObjectDataCacheKeySet::default(); - let result = fresh.insert(key.clone(), token, max_keys); + let result = fresh.insert_generation(key.clone(), generation, max_keys); let final_set = self.by_object.compute_if_absent(identity.clone(), move || fresh).await; if final_set.contains(&key) { return result; @@ -146,6 +164,16 @@ impl StarshardIdentityIndex { identity: &ObjectDataCacheIdentity, key: &ObjectDataCacheKey, token: ObjectDataCacheKeyToken, + ) -> bool { + self.remove_generation(identity, key, u64::try_from(token).unwrap_or(u64::MAX)) + .await + } + + pub(crate) async fn remove_generation( + &self, + identity: &ObjectDataCacheIdentity, + key: &ObjectDataCacheKey, + generation: ObjectDataCacheGeneration, ) -> bool { let mut removed = false; { @@ -153,7 +181,7 @@ impl StarshardIdentityIndex { let _ = self .by_object .compute_if_present(identity, move |mut key_set| { - *removed = key_set.remove_evicted_key(key, token); + *removed = key_set.remove_generation(key, generation); (!key_set.is_empty()).then_some(key_set) }) .await; @@ -169,6 +197,18 @@ impl StarshardIdentityIndex { .is_some_and(|key_set| key_set.contains(key)) } + pub(crate) async fn contains_generation( + &self, + identity: &ObjectDataCacheIdentity, + key: &ObjectDataCacheKey, + generation: ObjectDataCacheGeneration, + ) -> bool { + self.by_object + .get(identity) + .await + .is_some_and(|key_set| key_set.contains_generation(key, generation)) + } + /// Returns the number of tracked identities. #[cfg(test)] pub(crate) async fn identity_count(&self) -> usize { diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 0debdf0b8..99cb6a670 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -206,6 +206,7 @@ proptest = "1" tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] } temp-env = { workspace = true, features = ["async_closure"] } tracing-subscriber = { workspace = true, features = ["env-filter", "time"] } +metrics-util = { version = "0.20", features = ["debugging"] } opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] } rsa = { workspace = true } rcgen = { workspace = true } diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index 76b289290..c38e0e0b6 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -960,7 +960,7 @@ async fn duplicate_transition_task_skips_already_transitioned_version() { .await .into_iter() .filter_map(|op| match op { - MockWarmOp::Put { object } => Some(object), + MockWarmOp::Put { object, .. } => Some(object), _ => None, }) .collect(); @@ -997,6 +997,190 @@ async fn duplicate_transition_task_skips_already_transitioned_version() { assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, payload); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] +async fn transition_rejects_stale_etag_before_remote_upload() { + let (_disk_paths, ecstore) = setup_test_env().await; + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&tier_name).await; + let bucket = format!("test-api-transition-stale-etag-{}", &Uuid::new_v4().simple().to_string()[..8]); + let object = "test/transition-stale-etag.txt"; + (*ecstore) + .make_bucket(bucket.as_str(), &MakeBucketOptions::default()) + .await + .expect("unversioned test bucket should be created"); + let original = upload_test_object(&ecstore, bucket.as_str(), object, b"original transition candidate").await; + let replacement_payload = b"replacement committed before the stale transition starts"; + let replacement = upload_test_object(&ecstore, bucket.as_str(), object, replacement_payload).await; + let transition_opts = ObjectOptions { + transition: lifecycle::lifecycle_contract::TransitionOptions { + status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(), + tier: tier_name, + etag: original.etag.clone().unwrap_or_default(), + ..Default::default() + }, + version_id: replacement.version_id.map(|version| version.to_string()), + mod_time: replacement.mod_time, + ..Default::default() + }; + + ecstore + .transition_object(bucket.as_str(), object, &transition_opts) + .await + .expect_err("stale ETag must reject the transition before remote upload"); + assert_eq!(backend.put_count().await, 0, "rejected stale work must not upload a remote candidate"); + assert_eq!( + read_object_bytes(&ecstore, bucket.as_str(), object).await, + replacement_payload, + "the replacement object must remain local and readable" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] +async fn transition_commit_rejects_replaced_source() { + let (_disk_paths, ecstore) = setup_test_env().await; + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&tier_name).await; + let bucket = format!("test-api-transition-replace-{}", &Uuid::new_v4().simple().to_string()[..8]); + let object = "test/transition-replaced-source.txt"; + let original_payload = b"transition source that must not replace a concurrent overwrite"; + let replacement_payload = b"concurrent replacement must remain the authoritative local object"; + + (*ecstore) + .make_bucket(bucket.as_str(), &MakeBucketOptions::default()) + .await + .expect("unversioned test bucket should be created"); + let original = upload_test_object(&ecstore, bucket.as_str(), object, original_payload).await; + let transition_opts = ObjectOptions { + transition: lifecycle::lifecycle_contract::TransitionOptions { + status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(), + tier: tier_name.clone(), + etag: original.etag.clone().unwrap_or_default(), + ..Default::default() + }, + version_id: original.version_id.map(|version| version.to_string()), + mod_time: original.mod_time, + ..Default::default() + }; + + let put_barrier = backend.arm_put_barrier().await; + let transition_store = Arc::clone(&ecstore); + let transition_bucket = bucket.clone(); + let transition = tokio::spawn(async move { + transition_store + .transition_object(&transition_bucket, object, &transition_opts) + .await + }); + put_barrier.wait_until_paused().await; + + upload_test_object(&ecstore, bucket.as_str(), object, replacement_payload).await; + put_barrier.release(); + + transition + .await + .expect("transition task should not panic") + .expect_err("transition must reject a source replaced during remote upload"); + assert_eq!( + backend.put_count().await, + 1, + "the stale transition should upload exactly one remote candidate" + ); + assert_eq!( + backend.remove_count().await, + 1, + "the uncommitted remote candidate must be removed after identity revalidation fails" + ); + assert_eq!( + backend.remove_versions().await, + backend.put_versions().await, + "cleanup must delete the exact object version returned by the tier PUT" + ); + assert_eq!( + backend.object_count().await, + 0, + "the rejected transition must not leave a remote candidate" + ); + assert_eq!( + read_object_bytes(&ecstore, bucket.as_str(), object).await, + replacement_payload, + "the replacement object must remain readable and unmodified" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] +async fn cancelled_transition_waiting_for_prepared_reader_cleans_remote() { + temp_env::async_with_vars([(ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async { + let (_disk_paths, ecstore) = setup_test_env().await; + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&tier_name).await; + let bucket = format!("test-api-transition-reader-lock-{}", &Uuid::new_v4().simple().to_string()[..8]); + let object = "test/transition-reader-lock.txt"; + let payload = b"prepared reader must keep transition commit behind the namespace lock".repeat(1024); + + (*ecstore) + .make_bucket(bucket.as_str(), &MakeBucketOptions::default()) + .await + .expect("unversioned test bucket should be created"); + let original = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await; + let transition_opts = ObjectOptions { + transition: lifecycle::lifecycle_contract::TransitionOptions { + status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(), + tier: tier_name, + etag: original.etag.clone().unwrap_or_default(), + ..Default::default() + }, + version_id: original.version_id.map(|version| version.to_string()), + mod_time: original.mod_time, + ..Default::default() + }; + + let put_barrier = backend.arm_put_barrier().await; + let transition_store = Arc::clone(&ecstore); + let transition_bucket = bucket.clone(); + let mut transition = tokio::spawn(async move { + transition_store + .transition_object(&transition_bucket, object, &transition_opts) + .await + }); + put_barrier.wait_until_paused().await; + + let prepared_reader = ecstore + .prepare_get_object_reader(bucket.as_str(), object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("prepared reader should resolve while the local source still exists"); + put_barrier.release(); + assert!( + tokio::time::timeout(Duration::from_millis(200), &mut transition) + .await + .is_err(), + "transition commit must wait while a prepared reader owns the namespace read lock" + ); + + transition.abort(); + assert!( + transition + .await + .expect_err("transition task should be cancelled") + .is_cancelled(), + "transition cancellation should not be reported as a task panic" + ); + drop(prepared_reader); + assert_eq!(backend.put_count().await, 1, "the transition should upload exactly one remote candidate"); + assert!( + backend.wait_for_object_count(0, TRANSITION_WAIT_TIMEOUT).await, + "cancelling while the commit waits for the namespace lock must clean the exact uploaded candidate" + ); + assert_eq!(backend.remove_versions().await, backend.put_versions().await); + assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, payload); + }) + .await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[serial] #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"] diff --git a/rustfs/src/app/object_data_cache/adapter.rs b/rustfs/src/app/object_data_cache/adapter.rs index b3f5a79e9..9d6585fd0 100644 --- a/rustfs/src/app/object_data_cache/adapter.rs +++ b/rustfs/src/app/object_data_cache/adapter.rs @@ -40,6 +40,7 @@ struct ObjectDataCacheEnvValues { #[derive(Debug, Clone)] pub(crate) struct ObjectDataCacheAdapter { cache: Arc, + cold_fill: Arc, } impl ObjectDataCacheAdapter { @@ -49,9 +50,10 @@ impl ObjectDataCacheAdapter { /// in-memory GET fill limits so a `max_entry_bytes` above them is not /// reported as eligible while fill could never materialize the body. pub(crate) fn new(config: ObjectDataCacheConfig) -> Result { - let fill_ceiling = resolve_fill_ceiling_bytes(config.max_entry_bytes); + let fill_ceiling = resolve_fill_ceiling_bytes(config.mode, config.max_entry_bytes, config.max_bytes); Ok(Self { cache: Arc::new(ObjectDataCache::new(config)?.with_fill_ceiling_bytes(fill_ceiling)), + cold_fill: Arc::new(super::ColdFillCoordinator::default()), }) } @@ -78,6 +80,7 @@ impl ObjectDataCacheAdapter { pub(crate) fn disabled() -> Self { Self { cache: Arc::new(ObjectDataCache::disabled()), + cold_fill: Arc::new(super::ColdFillCoordinator::default()), } } @@ -107,16 +110,47 @@ impl ObjectDataCacheAdapter { self.cache.plan_get(request) } + /// Rebuilds a plan for producer identity revalidation without counting a + /// second request-level planning decision. + pub(crate) fn plan_get_untracked(&self, request: ObjectDataCacheGetRequest<'_>) -> ObjectDataCacheGetPlan { + self.cache.plan_get_untracked(request) + } + /// Executes an engine-level cache lookup. pub(crate) async fn lookup_body(&self, plan: &ObjectDataCacheGetPlan) -> ObjectDataCacheLookup { self.cache.lookup_body(plan).await } + /// Rechecks a completed cold fill without counting a second lookup for the + /// same GET request. + pub(crate) async fn peek_body_untracked(&self, plan: &ObjectDataCacheGetPlan) -> ObjectDataCacheLookup { + self.cache.peek_body_untracked(plan).await + } + /// Executes an engine-level cache fill. pub(crate) async fn fill_body(&self, plan: &ObjectDataCacheGetPlan, bytes: Bytes) -> ObjectDataCacheFillResult { self.cache.fill_body(plan, bytes).await } + pub(crate) fn cold_fill_coordinator(&self) -> Arc { + Arc::clone(&self.cold_fill) + } + + pub(crate) fn reserve_body( + &self, + plan: &ObjectDataCacheGetPlan, + ) -> Option { + self.cache.reserve_body(plan) + } + + pub(crate) async fn fill_reserved_body( + &self, + plan: &ObjectDataCacheGetPlan, + body: rustfs_object_data_cache::ObjectDataCacheReservedBody, + ) -> ObjectDataCacheFillResult { + self.cache.fill_reserved_body(plan, body).await + } + /// Executes an engine-level object invalidation. pub(crate) async fn invalidate_object( &self, @@ -165,12 +199,13 @@ impl ObjectDataCacheAdapter { fn from_config_or_disabled(config: ObjectDataCacheConfig, source: &str) -> Arc { let mode = config.mode; let max_entry_bytes = config.max_entry_bytes; + let max_bytes = config.max_bytes; // `Self::new` applies the ODC-24 fill-ceiling clamp; this startup path // only adds the resolved-config logging on top. match Self::new(config) { Ok(adapter) => { if !adapter.is_disabled() { - let ceiling = resolve_fill_ceiling_bytes(max_entry_bytes); + let ceiling = resolve_fill_ceiling_bytes(mode, max_entry_bytes, max_bytes); // ODC-19: log the resolved mode at startup, and warn when the // operator explicitly selected the never-filling HitOnly mode. tracing::info!(source, ?mode, "object data cache enabled"); @@ -211,7 +246,14 @@ impl ObjectDataCacheAdapter { /// (backlog#1129 / ODC-24). The concurrency-driven shrink of the fill threshold /// is dynamic and cannot be captured here, so this static ceiling is the /// conservative floor. -fn resolve_fill_ceiling_bytes(max_entry_bytes: u64) -> u64 { +fn resolve_fill_ceiling_bytes(mode: ObjectDataCacheMode, max_entry_bytes: u64, max_bytes: u64) -> u64 { + if mode == ObjectDataCacheMode::FillMaterializeEnabled { + return if max_bytes == 0 { + max_entry_bytes + } else { + max_entry_bytes.min(max_bytes) + }; + } let seek_threshold = crate::app::object_usecase::object_seek_support_threshold() as u64; let hard_cap = u64::try_from(crate::app::object_usecase::MAX_GET_OBJECT_MEMORY_BUFFER_BYTES).unwrap_or(u64::MAX); max_entry_bytes.min(seek_threshold).min(hard_cap) @@ -481,7 +523,7 @@ mod tests { // ODC-24 (backlog#1129): a max_entry_bytes above the in-memory GET fill // limits is clamped down; the ceiling never exceeds the 64 MiB hard cap. let huge = 512 * 1024 * 1024; - let ceiling = super::resolve_fill_ceiling_bytes(huge); + let ceiling = super::resolve_fill_ceiling_bytes(ObjectDataCacheMode::FillBufferedOnly, huge, huge); assert!(ceiling < huge, "a huge max_entry_bytes must be clamped down"); assert!( ceiling <= u64::try_from(crate::app::object_usecase::MAX_GET_OBJECT_MEMORY_BUFFER_BYTES).unwrap(), @@ -493,7 +535,7 @@ mod tests { fn resolve_fill_ceiling_leaves_small_max_entry_bytes_untouched() { // A small max_entry_bytes is already below the fill limits, so it is the // binding constraint and passes through unchanged. - assert_eq!(super::resolve_fill_ceiling_bytes(4096), 4096); + assert_eq!(super::resolve_fill_ceiling_bytes(ObjectDataCacheMode::FillBufferedOnly, 4096, 8192), 4096); } #[test] @@ -528,6 +570,48 @@ mod tests { assert_eq!(plan, rustfs_object_data_cache::ObjectDataCacheGetPlan::SkipTooLarge); } + #[test] + fn materialize_mode_allows_384_mib_when_capacity_is_two_gib() { + let adapter = ObjectDataCacheAdapter::from_config_or_disabled( + ObjectDataCacheConfig { + mode: ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 2 * 1024 * 1024 * 1024, + max_memory_percent: 0, + max_entry_bytes: 512 * 1024 * 1024, + ..ObjectDataCacheConfig::default() + }, + "test", + ); + + let plan = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "bucket", + object: "object", + version_id: None, + etag: "etag", + size: 384 * 1024 * 1024, + data_dir_u128: None, + mod_time_unix_nanos: 0, + body_variant: rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + assert!(matches!(plan, rustfs_object_data_cache::ObjectDataCacheGetPlan::Cacheable { .. })); + } + + #[test] + fn materialize_mode_keeps_entry_weight_capacity_validation() { + let adapter = ObjectDataCacheAdapter::from_config_or_disabled( + ObjectDataCacheConfig { + mode: ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 512 * 1024 * 1024, + max_memory_percent: 0, + max_entry_bytes: 512 * 1024 * 1024, + ..ObjectDataCacheConfig::default() + }, + "test", + ); + + assert!(adapter.is_disabled()); + } + #[test] fn object_data_cache_explicit_mode_wins_over_enable_true() { let config = object_data_cache_config_from_values(ObjectDataCacheEnvValues { diff --git a/rustfs/src/app/object_data_cache/cold_fill.rs b/rustfs/src/app/object_data_cache/cold_fill.rs new file mode 100644 index 000000000..b54ab7014 --- /dev/null +++ b/rustfs/src/app/object_data_cache/cold_fill.rs @@ -0,0 +1,1213 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); + +use crate::app::storage_api::object_usecase::error::StorageError; +use bytes::Bytes; +use rustfs_object_data_cache::ObjectDataCacheKey; +use std::collections::HashMap; +use std::future::Future; +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Instant; +use tokio::sync::Notify; +use tokio_util::sync::CancellationToken; + +const MAX_PRODUCER_ATTEMPTS: u8 = 2; +const MAX_ACTIVE_SESSIONS: usize = 1024; +const MAX_WAITERS_PER_SESSION: usize = 2048; +const MAX_GLOBAL_WAITERS: usize = 8192; +const SESSION_SHARDS: usize = 64; + +macro_rules! cold_fill_counter_increment { + ($name:literal, $value:expr $(, $label_key:literal => $label_value:literal)*) => { + if rustfs_io_metrics::metrics_enabled() { + #[cfg(not(test))] + { + static HANDLE: std::sync::LazyLock = + std::sync::LazyLock::new(|| metrics::counter!($name $(, $label_key => $label_value)*)); + HANDLE.increment($value); + } + #[cfg(test)] + metrics::counter!($name $(, $label_key => $label_value)*).increment($value); + } + }; +} + +macro_rules! cold_fill_gauge_adjust { + ($name:literal, $method:ident, $value:expr) => { + #[cfg(not(test))] + { + static HANDLE: std::sync::LazyLock = std::sync::LazyLock::new(|| metrics::gauge!($name)); + HANDLE.$method($value); + } + #[cfg(test)] + metrics::gauge!($name).$method($value); + }; +} + +macro_rules! cold_fill_histogram_record { + ($name:literal, $value:expr) => { + if rustfs_io_metrics::metrics_enabled() { + #[cfg(not(test))] + { + static HANDLE: std::sync::LazyLock = std::sync::LazyLock::new(|| metrics::histogram!($name)); + HANDLE.record($value); + } + #[cfg(test)] + metrics::histogram!($name).record($value); + } + }; +} + +#[derive(Clone, Copy)] +pub(crate) enum ColdFillDiskPermitOwner { + Producer, + Follower, +} + +tokio::task_local! { + static COLD_FILL_DISK_PERMIT_OWNER: ColdFillDiskPermitOwner; +} + +pub(crate) fn current_cold_fill_disk_permit_owner() -> Option { + COLD_FILL_DISK_PERMIT_OWNER.try_with(|owner| *owner).ok() +} + +#[cfg(test)] +pub(crate) async fn scope_cold_fill_disk_permit_owner_for_test( + owner: ColdFillDiskPermitOwner, + future: F, +) -> F::Output { + COLD_FILL_DISK_PERMIT_OWNER.scope(owner, future).await +} + +#[derive(Debug)] +pub(crate) struct ColdFillCoordinator { + sessions: [Mutex>>; SESSION_SHARDS], + active_sessions: AtomicUsize, + global_waiters: AtomicUsize, + next_session_id: AtomicU64, +} + +#[derive(Debug)] +struct ColdFillSession { + id: u64, + producer_deadline: Option, + state: Mutex, + changed: Notify, + cancelled: CancellationToken, +} + +#[derive(Debug)] +struct ColdFillSessionState { + producer_active: bool, + reader_started: bool, + attempts: u8, + waiters: usize, + consumers: usize, + result: Option, + completed_at: Option, +} + +#[derive(Debug, Clone)] +enum ColdFillResult { + Ready(Result), + Bypass, +} + +pub(crate) enum ColdFillRole { + Produce(ColdFillProducer), + Wait(ColdFillWaiter), + Bypass, + Rejected, +} + +pub(crate) enum ColdFillWaitOutcome { + Produce(ColdFillProducer), + Ready(Result), + Bypass, + DeadlineExceeded, +} + +#[derive(Debug)] +pub(crate) enum ColdFillCoordinateOutcome { + Ready(Result), + Bypass, + Rejected, +} + +#[derive(Debug, Clone)] +pub(crate) enum ColdFillError { + Storage(StorageError), + DiskAdmissionClosed, +} + +pub(crate) struct ColdFillProducer { + coordinator: Arc, + key: ObjectDataCacheKey, + session: Arc, + finished: bool, + consumer_transferred: bool, +} + +pub(crate) struct ColdFillWaiter { + coordinator: Arc, + key: ObjectDataCacheKey, + session: Arc, + global_waiter_counted: bool, + waiter_metric_recorded: bool, + consumer_counted: bool, +} + +impl ColdFillCoordinator { + #[cfg(test)] + pub(crate) fn global_waiter_count_for_test(&self) -> usize { + self.global_waiters.load(Ordering::Relaxed) + } + + #[cfg(test)] + pub(crate) fn active_session_count_for_test(&self) -> usize { + self.active_sessions.load(Ordering::Relaxed) + } + + #[cfg(test)] + pub(crate) fn join(self: &Arc, key: ObjectDataCacheKey) -> ColdFillRole { + self.join_with_producer_deadline(key, None) + } + + fn join_with_producer_deadline( + self: &Arc, + key: ObjectDataCacheKey, + producer_deadline: Option, + ) -> ColdFillRole { + // Global lock order: a shard's session-map mutex may be followed by a + // session-state mutex. Code that needs the map after inspecting state + // must release state first and then call remove_if_current. + let mut sessions = self.sessions[self.session_shard(&key)] + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(session) = sessions.get(&key) { + let mut state = session.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if state.consumers == 0 || session.cancelled.is_cancelled() { + drop(state); + drop(sessions); + return ColdFillRole::Bypass; + } + let Some(consumers) = state.consumers.checked_add(1) else { + drop(state); + drop(sessions); + cold_fill_counter_increment!("rustfs_object_data_cache_cold_fill_waiter_rejected_total", 1); + return ColdFillRole::Rejected; + }; + if state.waiters >= MAX_WAITERS_PER_SESSION || !self.reserve_global_waiter() { + drop(state); + drop(sessions); + cold_fill_counter_increment!("rustfs_object_data_cache_cold_fill_waiter_rejected_total", 1); + return ColdFillRole::Rejected; + } + state.consumers = consumers; + state.waiters += 1; + let session_waiters = state.waiters; + drop(state); + let session = Arc::clone(session); + drop(sessions); + let waiter_metric_recorded = self.record_waiter_join(session_waiters); + return ColdFillRole::Wait(ColdFillWaiter { + coordinator: Arc::clone(self), + key, + session, + global_waiter_counted: true, + waiter_metric_recorded, + consumer_counted: true, + }); + } + + if self + .active_sessions + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + (current < MAX_ACTIVE_SESSIONS).then_some(current + 1) + }) + .is_err() + { + drop(sessions); + cold_fill_counter_increment!("rustfs_object_data_cache_cold_fill_session_rejected_total", 1); + return ColdFillRole::Rejected; + } + + let session_id = match self + .next_session_id + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| current.checked_add(1)) + { + Ok(session_id) => session_id, + Err(_) => { + self.active_sessions.fetch_sub(1, Ordering::Relaxed); + drop(sessions); + cold_fill_counter_increment!("rustfs_object_data_cache_cold_fill_session_rejected_total", 1); + return ColdFillRole::Rejected; + } + }; + let session = Arc::new(ColdFillSession { + id: session_id, + producer_deadline, + state: Mutex::new(ColdFillSessionState { + producer_active: true, + reader_started: false, + attempts: 1, + waiters: 0, + consumers: 1, + result: None, + completed_at: None, + }), + changed: Notify::new(), + cancelled: CancellationToken::new(), + }); + sessions.insert(key.clone(), Arc::clone(&session)); + drop(sessions); + cold_fill_counter_increment!( + "rustfs_object_data_cache_cold_fill_producer_started_total", + 1, + "attempt" => "initial" + ); + ColdFillRole::Produce(ColdFillProducer { + coordinator: Arc::clone(self), + key, + session, + finished: false, + consumer_transferred: false, + }) + } + + fn remove_if_current(&self, key: &ObjectDataCacheKey, session: &Arc) { + let mut sessions = self.sessions[self.session_shard(key)] + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if sessions.get(key).is_some_and(|current| current.id == session.id) { + sessions.remove(key); + self.active_sessions.fetch_sub(1, Ordering::Relaxed); + } + } + + fn session_shard(&self, key: &ObjectDataCacheKey) -> usize { + let mut hasher = DefaultHasher::new(); + key.hash(&mut hasher); + usize::try_from(hasher.finish()).unwrap_or(usize::MAX) % SESSION_SHARDS + } + + fn reserve_global_waiter(&self) -> bool { + let result = self + .global_waiters + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + (current < MAX_GLOBAL_WAITERS).then_some(current + 1) + }); + if let Ok(previous) = result { + let _ = previous; + true + } else { + false + } + } + + fn release_global_waiter(&self, waiter_metric_recorded: bool) { + self.global_waiters.fetch_sub(1, Ordering::Relaxed); + if waiter_metric_recorded { + cold_fill_gauge_adjust!("rustfs_object_data_cache_cold_fill_waiters", decrement, 1.0); + } + } + + fn record_waiter_join(&self, session_waiters: usize) -> bool { + let waiter_metric_recorded = rustfs_io_metrics::metrics_enabled(); + if waiter_metric_recorded { + cold_fill_gauge_adjust!("rustfs_object_data_cache_cold_fill_waiters", increment, 1.0); + } + cold_fill_histogram_record!("rustfs_object_data_cache_cold_fill_waiters_per_session", session_waiters as f64); + waiter_metric_recorded + } +} + +impl Default for ColdFillCoordinator { + fn default() -> Self { + Self { + sessions: std::array::from_fn(|_| Mutex::new(HashMap::new())), + active_sessions: AtomicUsize::new(0), + global_waiters: AtomicUsize::new(0), + next_session_id: AtomicU64::new(1), + } + } +} + +pub(crate) async fn coordinate_cold_fill( + coordinator: &Arc, + key: ObjectDataCacheKey, + waiter_deadline: Option, + proposed_producer_deadline: Option, + mut start: F, +) -> ColdFillCoordinateOutcome +where + F: FnMut(ColdFillProducer) -> Fut, + Fut: Future + Send + 'static, +{ + let mut role = coordinator.join_with_producer_deadline(key, proposed_producer_deadline); + loop { + match role { + ColdFillRole::Produce(mut producer) => { + let waiter = producer.waiter(); + tokio::spawn(COLD_FILL_DISK_PERMIT_OWNER.scope(ColdFillDiskPermitOwner::Producer, start(producer))); + match COLD_FILL_DISK_PERMIT_OWNER + .scope(ColdFillDiskPermitOwner::Follower, waiter.wait_until(waiter_deadline)) + .await + { + ColdFillWaitOutcome::Ready(result) => return ColdFillCoordinateOutcome::Ready(result), + ColdFillWaitOutcome::Bypass => return ColdFillCoordinateOutcome::Bypass, + ColdFillWaitOutcome::DeadlineExceeded => { + return ColdFillCoordinateOutcome::Ready(Err(ColdFillError::Storage(StorageError::Timeout))); + } + ColdFillWaitOutcome::Produce(producer) => role = ColdFillRole::Produce(producer), + } + } + ColdFillRole::Wait(waiter) => match COLD_FILL_DISK_PERMIT_OWNER + .scope(ColdFillDiskPermitOwner::Follower, waiter.wait_until(waiter_deadline)) + .await + { + ColdFillWaitOutcome::Produce(producer) => role = ColdFillRole::Produce(producer), + ColdFillWaitOutcome::Ready(result) => return ColdFillCoordinateOutcome::Ready(result), + ColdFillWaitOutcome::Bypass => return ColdFillCoordinateOutcome::Bypass, + ColdFillWaitOutcome::DeadlineExceeded => { + return ColdFillCoordinateOutcome::Ready(Err(ColdFillError::Storage(StorageError::Timeout))); + } + }, + ColdFillRole::Bypass => return ColdFillCoordinateOutcome::Bypass, + ColdFillRole::Rejected => return ColdFillCoordinateOutcome::Rejected, + } + } +} + +impl ColdFillProducer { + pub(crate) fn waiter(&mut self) -> ColdFillWaiter { + self.consumer_transferred = true; + ColdFillWaiter { + coordinator: Arc::clone(&self.coordinator), + key: self.key.clone(), + session: Arc::clone(&self.session), + global_waiter_counted: false, + waiter_metric_recorded: false, + consumer_counted: true, + } + } + + pub(crate) fn cancellation_token(&self) -> CancellationToken { + self.session.cancelled.clone() + } + + pub(crate) fn deadline(&self) -> Option { + self.session.producer_deadline + } + + pub(crate) fn mark_reader_started(&self) { + let mut state = self.session.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + state.reader_started = true; + } + + pub(crate) fn finish(self, result: Result) { + self.finish_shared(result.map_err(ColdFillError::Storage)); + } + + pub(crate) fn finish_shared(mut self, result: Result) { + let release_consumer = !self.consumer_transferred; + self.consumer_transferred = true; + let last_consumer; + { + let mut state = self.session.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if release_consumer { + state.consumers = state.consumers.saturating_sub(1); + } + state.result = Some(ColdFillResult::Ready(result)); + state.completed_at = rustfs_io_metrics::metrics_enabled().then(Instant::now); + state.producer_active = false; + last_consumer = state.consumers == 0; + } + self.finished = true; + self.session.changed.notify_waiters(); + self.coordinator.remove_if_current(&self.key, &self.session); + if last_consumer { + self.session.cancelled.cancel(); + } + } + + pub(crate) fn relinquish_or_finish(mut self, error: ColdFillError) { + let retryable; + { + let mut state = self.session.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + retryable = state.consumers > 0 && !state.reader_started && state.attempts < MAX_PRODUCER_ATTEMPTS; + if retryable { + state.producer_active = false; + } else { + state.result = Some(ColdFillResult::Ready(Err(error))); + state.completed_at = rustfs_io_metrics::metrics_enabled().then(Instant::now); + state.producer_active = false; + } + } + self.finished = true; + self.session.changed.notify_waiters(); + if !retryable { + self.coordinator.remove_if_current(&self.key, &self.session); + } + } + + pub(crate) fn bypass(mut self) { + let release_consumer = !self.consumer_transferred; + self.consumer_transferred = true; + let last_consumer; + { + let mut state = self.session.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if release_consumer { + state.consumers = state.consumers.saturating_sub(1); + } + state.result = Some(ColdFillResult::Bypass); + state.completed_at = rustfs_io_metrics::metrics_enabled().then(Instant::now); + state.producer_active = false; + last_consumer = state.consumers == 0; + } + self.finished = true; + self.session.changed.notify_waiters(); + self.coordinator.remove_if_current(&self.key, &self.session); + if last_consumer { + self.session.cancelled.cancel(); + } + } +} + +impl Drop for ColdFillProducer { + fn drop(&mut self) { + if self.finished { + return; + } + + let release_consumer = !self.consumer_transferred; + self.consumer_transferred = true; + let (retryable, last_consumer) = { + let mut state = self.session.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if release_consumer { + state.consumers = state.consumers.saturating_sub(1); + } + if !state.reader_started && state.attempts < MAX_PRODUCER_ATTEMPTS && state.consumers > 0 { + state.producer_active = false; + (true, state.consumers == 0) + } else { + state.result = Some(ColdFillResult::Ready(Err(ColdFillError::Storage(StorageError::OperationCanceled)))); + state.completed_at = rustfs_io_metrics::metrics_enabled().then(Instant::now); + state.producer_active = false; + (false, state.consumers == 0) + } + }; + if last_consumer { + self.session.cancelled.cancel(); + } + self.session.changed.notify_waiters(); + if !retryable || last_consumer { + self.coordinator.remove_if_current(&self.key, &self.session); + } + } +} + +impl ColdFillWaiter { + async fn wait_until(self, deadline: Option) -> ColdFillWaitOutcome { + match deadline { + Some(deadline) => tokio::time::timeout_at(deadline, self.wait()) + .await + .unwrap_or(ColdFillWaitOutcome::DeadlineExceeded), + None => self.wait().await, + } + } + + pub(crate) async fn wait(mut self) -> ColdFillWaitOutcome { + loop { + let session = Arc::clone(&self.session); + let notified = session.changed.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + { + let mut state = session.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(result) = &state.result { + let outcome = match result { + ColdFillResult::Ready(result) => ColdFillWaitOutcome::Ready(result.clone()), + ColdFillResult::Bypass => ColdFillWaitOutcome::Bypass, + }; + let (release_global, release_waiter_metric) = self.release_waiter_slot(&mut state, true); + let last_consumer = state.consumers == 0; + let producer_active = state.producer_active; + let completed_at = state.completed_at; + drop(state); + self.finish_release(release_global, release_waiter_metric, last_consumer, producer_active); + if let Some(completed_at) = completed_at { + cold_fill_histogram_record!( + "rustfs_object_data_cache_cold_fill_completion_handoff_seconds", + completed_at.elapsed().as_secs_f64() + ); + } + return outcome; + } + if !state.producer_active && !state.reader_started && state.attempts < MAX_PRODUCER_ATTEMPTS { + state.producer_active = true; + state.attempts += 1; + let (release_global, release_waiter_metric) = self.release_waiter_slot(&mut state, false); + self.consumer_counted = false; + drop(state); + if release_global { + self.coordinator.release_global_waiter(release_waiter_metric); + } + cold_fill_counter_increment!( + "rustfs_object_data_cache_cold_fill_producer_started_total", + 1, + "attempt" => "successor" + ); + return ColdFillWaitOutcome::Produce(ColdFillProducer { + coordinator: Arc::clone(&self.coordinator), + key: self.key.clone(), + session: Arc::clone(&self.session), + finished: false, + consumer_transferred: false, + }); + } + } + notified.await; + } + } + + fn release_waiter_slot(&mut self, state: &mut ColdFillSessionState, release_consumer: bool) -> (bool, bool) { + let release_global = self.global_waiter_counted; + let release_waiter_metric = release_global && self.waiter_metric_recorded; + if self.global_waiter_counted { + state.waiters = state.waiters.saturating_sub(1); + self.global_waiter_counted = false; + self.waiter_metric_recorded = false; + } + if release_consumer && self.consumer_counted { + state.consumers = state.consumers.saturating_sub(1); + self.consumer_counted = false; + } + (release_global, release_waiter_metric) + } + + fn finish_release(&self, release_global: bool, release_waiter_metric: bool, last_consumer: bool, producer_active: bool) { + if release_global { + self.coordinator.release_global_waiter(release_waiter_metric); + } + if last_consumer { + self.session.cancelled.cancel(); + if !producer_active { + self.coordinator.remove_if_current(&self.key, &self.session); + } + } + } +} + +impl Drop for ColdFillWaiter { + fn drop(&mut self) { + if !self.global_waiter_counted && !self.consumer_counted { + return; + } + let session = Arc::clone(&self.session); + let mut state = session.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let (release_global, release_waiter_metric) = self.release_waiter_slot(&mut state, true); + let last_consumer = state.consumers == 0; + let producer_active = state.producer_active; + drop(state); + self.finish_release(release_global, release_waiter_metric, last_consumer, producer_active); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rustfs_object_data_cache::ObjectDataCacheBodyVariant; + + fn key() -> ObjectDataCacheKey { + ObjectDataCacheKey::new("bucket", "object", None, "etag", 4, ObjectDataCacheBodyVariant::FullObjectPlainV1) + } + + fn indexed_key(index: usize) -> ObjectDataCacheKey { + ObjectDataCacheKey::new( + "bucket", + format!("object-{index}"), + None, + "etag", + 4, + ObjectDataCacheBodyVariant::FullObjectPlainV1, + ) + } + + #[test] + #[serial_test::serial(cold_fill_metrics_gate)] + fn cold_fill_metrics_follow_runtime_gate_and_release_waiter_gauge() { + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; + + async fn start_session( + coordinator: Arc, + key: ObjectDataCacheKey, + ) -> (ColdFillProducer, ColdFillWaiter) { + let ColdFillRole::Produce(producer) = coordinator.join(key.clone()) else { + panic!("first request must own the producer role"); + }; + let ColdFillRole::Wait(waiter) = coordinator.join(key) else { + panic!("second request must join as a waiter"); + }; + (producer, waiter) + } + + let metrics_was_enabled = rustfs_io_metrics::metrics_enabled(); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("metric test runtime must build"); + + metrics::with_local_recorder(&recorder, || { + runtime.block_on(async { + rustfs_io_metrics::set_metrics_enabled(false); + let (producer, waiter) = start_session(Arc::new(ColdFillCoordinator::default()), indexed_key(1)).await; + rustfs_io_metrics::set_metrics_enabled(true); + producer.finish(Ok(Bytes::from_static(b"body"))); + assert!(matches!(waiter.wait().await, ColdFillWaitOutcome::Ready(Ok(_)))); + }); + }); + assert!( + snapshotter + .snapshot() + .into_vec() + .iter() + .all(|(composite, _, _, _)| { composite.key().name() != "rustfs_object_data_cache_cold_fill_waiters" }), + "a waiter admitted while metrics were disabled must not record an unmatched decrement" + ); + + metrics::with_local_recorder(&recorder, || { + runtime.block_on(async { + rustfs_io_metrics::set_metrics_enabled(true); + let (producer, waiter) = start_session(Arc::new(ColdFillCoordinator::default()), indexed_key(2)).await; + rustfs_io_metrics::set_metrics_enabled(false); + producer.finish(Ok(Bytes::from_static(b"body"))); + assert!(matches!(waiter.wait().await, ColdFillWaitOutcome::Ready(Ok(_)))); + + rustfs_io_metrics::set_metrics_enabled(true); + let (producer, waiter) = start_session(Arc::new(ColdFillCoordinator::default()), indexed_key(3)).await; + producer.finish(Ok(Bytes::from_static(b"body"))); + assert!(matches!(waiter.wait().await, ColdFillWaitOutcome::Ready(Ok(_)))); + }); + }); + let snapshot = snapshotter.snapshot().into_vec(); + let waiter_gauge = snapshot + .iter() + .find_map(|(composite, _unit, _description, value)| { + (composite.key().name() == "rustfs_object_data_cache_cold_fill_waiters").then_some(value) + }) + .expect("enabled metrics must record the waiter gauge"); + let DebugValue::Gauge(waiter_gauge) = waiter_gauge else { + panic!("waiter metric must be a gauge"); + }; + assert_eq!(waiter_gauge.into_inner(), 0.0); + for name in [ + "rustfs_object_data_cache_cold_fill_waiters_per_session", + "rustfs_object_data_cache_cold_fill_completion_handoff_seconds", + ] { + assert!( + snapshot.iter().any(|(composite, _, _, _)| composite.key().name() == name), + "enabled metrics must record {name}" + ); + } + rustfs_io_metrics::set_metrics_enabled(metrics_was_enabled); + } + + #[tokio::test] + async fn cold_fill_waiter_observes_shared_result() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let ColdFillRole::Produce(producer) = coordinator.join(key()) else { + panic!("first join must produce"); + }; + let ColdFillRole::Wait(waiter) = coordinator.join(key()) else { + panic!("second join must wait"); + }; + producer.finish(Ok(Bytes::from_static(b"body"))); + let ColdFillWaitOutcome::Ready(result) = waiter.wait().await else { + panic!("completed session must be ready"); + }; + assert_eq!(result.expect("shared cold fill should succeed"), Bytes::from_static(b"body")); + } + + #[tokio::test] + async fn cold_fill_only_promotes_before_reader_start() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let ColdFillRole::Produce(producer) = coordinator.join(key()) else { + panic!("first join must produce"); + }; + let ColdFillRole::Wait(waiter) = coordinator.join(key()) else { + panic!("second join must wait"); + }; + drop(producer); + let promoted = match waiter.wait().await { + ColdFillWaitOutcome::Produce(producer) => producer, + ColdFillWaitOutcome::Ready(_) | ColdFillWaitOutcome::Bypass | ColdFillWaitOutcome::DeadlineExceeded => { + panic!("second attempt must promote") + } + }; + let ColdFillRole::Wait(final_waiter) = coordinator.join(key()) else { + panic!("third join must wait for promoted producer"); + }; + promoted.mark_reader_started(); + drop(promoted); + let ColdFillWaitOutcome::Ready(Err(ColdFillError::Storage(StorageError::OperationCanceled))) = final_waiter.wait().await + else { + panic!("reader-started producer cancellation must be terminal"); + }; + let ColdFillRole::Produce(_) = coordinator.join(key()) else { + panic!("terminal failed session must be removed"); + }; + } + + #[tokio::test] + async fn cold_fill_bypass_wakes_waiters() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let ColdFillRole::Produce(producer) = coordinator.join(key()) else { + panic!("first join must produce"); + }; + let ColdFillRole::Wait(waiter) = coordinator.join(key()) else { + panic!("second join must wait"); + }; + producer.bypass(); + assert!(matches!(waiter.wait().await, ColdFillWaitOutcome::Bypass)); + } + + #[tokio::test] + async fn one_follower_cancel_does_not_cancel_producer_or_other_followers() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let ColdFillRole::Produce(mut producer) = coordinator.join(key()) else { + panic!("first request must produce"); + }; + let leader = producer.waiter(); + let ColdFillRole::Wait(cancelled_follower) = coordinator.join(key()) else { + panic!("second request must follow"); + }; + let ColdFillRole::Wait(surviving_follower) = coordinator.join(key()) else { + panic!("third request must follow"); + }; + + drop(cancelled_follower); + assert!(!producer.cancellation_token().is_cancelled()); + producer.finish(Ok(Bytes::from_static(b"body"))); + + for waiter in [leader, surviving_follower] { + let ColdFillWaitOutcome::Ready(Ok(body)) = waiter.wait().await else { + panic!("remaining consumers must receive the shared result"); + }; + assert_eq!(body, Bytes::from_static(b"body")); + } + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + + #[tokio::test] + async fn aborted_pre_reader_producer_promotes_only_one_successor() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let ColdFillRole::Produce(producer) = coordinator.join(key()) else { + panic!("first request must produce"); + }; + let ColdFillRole::Wait(first_follower) = coordinator.join(key()) else { + panic!("second request must follow"); + }; + let ColdFillRole::Wait(second_follower) = coordinator.join(key()) else { + panic!("third request must follow"); + }; + let task = tokio::spawn(async move { + let _producer = producer; + std::future::pending::<()>().await; + }); + task.abort(); + let _ = task.await; + + let ColdFillWaitOutcome::Produce(successor) = first_follower.wait().await else { + panic!("exactly one follower must become the successor"); + }; + let second_task = tokio::spawn(async move { second_follower.wait().await }); + tokio::task::yield_now().await; + assert!(!second_task.is_finished(), "the other follower must wait for the unique successor"); + successor.finish(Ok(Bytes::from_static(b"body"))); + let ColdFillWaitOutcome::Ready(Ok(body)) = second_task.await.expect("second follower task must join") else { + panic!("the waiting follower must receive the successor result"); + }; + assert_eq!(body, Bytes::from_static(b"body")); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + + #[tokio::test] + async fn sole_leader_waiter_promotes_after_pre_reader_producer_abort_or_panic() { + for panic_producer in [false, true] { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let ColdFillRole::Produce(mut producer) = coordinator.join(key()) else { + panic!("first request must produce"); + }; + let leader = producer.waiter(); + let producer_task = tokio::spawn(async move { + if panic_producer { + panic!("simulated pre-reader producer panic"); + } + let _producer = producer; + std::future::pending::<()>().await; + }); + if !panic_producer { + producer_task.abort(); + } + assert!(producer_task.await.is_err(), "producer task must abort or panic"); + + let ColdFillWaitOutcome::Produce(successor) = leader.wait().await else { + panic!("the sole leader waiter must become the successor"); + }; + successor.finish(Ok(Bytes::from_static(b"body"))); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + } + + #[tokio::test] + async fn aborted_post_reader_producer_is_terminal() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let ColdFillRole::Produce(producer) = coordinator.join(key()) else { + panic!("first request must produce"); + }; + let ColdFillRole::Wait(follower) = coordinator.join(key()) else { + panic!("second request must follow"); + }; + let task = tokio::spawn(async move { + producer.mark_reader_started(); + let _producer = producer; + std::future::pending::<()>().await; + }); + tokio::task::yield_now().await; + task.abort(); + let _ = task.await; + + assert!(matches!( + follower.wait().await, + ColdFillWaitOutcome::Ready(Err(ColdFillError::Storage(StorageError::OperationCanceled))) + )); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + + #[test] + fn cold_fill_waiter_limit_rejects_without_starting_another_producer() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let ColdFillRole::Produce(_producer) = coordinator.join(key()) else { + panic!("first join must produce"); + }; + let mut waiters = Vec::with_capacity(MAX_WAITERS_PER_SESSION); + for _ in 0..MAX_WAITERS_PER_SESSION { + let ColdFillRole::Wait(waiter) = coordinator.join(key()) else { + panic!("waiter below the cap must join"); + }; + waiters.push(waiter); + } + assert!(matches!(coordinator.join(key()), ColdFillRole::Rejected)); + drop(waiters.pop()); + assert!(matches!(coordinator.join(key()), ColdFillRole::Wait(_))); + } + + #[test] + fn canceled_unobserved_producers_release_session_capacity() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + for index in 0..MAX_ACTIVE_SESSIONS { + let ColdFillRole::Produce(producer) = coordinator.join(indexed_key(index)) else { + panic!("unobserved key must initially produce"); + }; + drop(producer); + } + assert_eq!(coordinator.active_sessions.load(Ordering::Relaxed), 0); + assert!(matches!(coordinator.join(indexed_key(MAX_ACTIVE_SESSIONS)), ColdFillRole::Produce(_))); + } + + #[test] + fn last_consumer_cancels_and_late_request_bypasses_until_producer_ack() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let ColdFillRole::Produce(mut producer) = coordinator.join(key()) else { + panic!("first request must produce"); + }; + let leader = producer.waiter(); + drop(leader); + + assert!(producer.cancellation_token().is_cancelled()); + assert!(matches!(coordinator.join(key()), ColdFillRole::Bypass)); + drop(producer); + + assert_eq!(coordinator.active_sessions.load(Ordering::Relaxed), 0); + assert!(matches!(coordinator.join(key()), ColdFillRole::Produce(_))); + } + + #[test] + fn last_consumer_then_producer_relinquish_removes_abandoned_session() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let ColdFillRole::Produce(mut producer) = coordinator.join(key()) else { + panic!("first request must produce"); + }; + drop(producer.waiter()); + + producer.relinquish_or_finish(ColdFillError::Storage(StorageError::Timeout)); + + assert_eq!(coordinator.active_session_count_for_test(), 0); + assert!(matches!(coordinator.join(key()), ColdFillRole::Produce(_))); + } + + #[test] + fn producer_and_last_waiter_drop_remove_retryable_session() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let ColdFillRole::Produce(producer) = coordinator.join(key()) else { + panic!("first request must produce"); + }; + let ColdFillRole::Wait(waiter) = coordinator.join(key()) else { + panic!("second request must wait"); + }; + + drop(producer); + drop(waiter); + + assert_eq!(coordinator.active_sessions.load(Ordering::Relaxed), 0); + assert!(matches!(coordinator.join(key()), ColdFillRole::Produce(_))); + } + + #[test] + fn session_ids_are_monotonic_and_overflow_fails_closed() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let ColdFillRole::Produce(first) = coordinator.join(indexed_key(1)) else { + panic!("first session must produce"); + }; + let first_id = first.session.id; + drop(first); + let ColdFillRole::Produce(second) = coordinator.join(indexed_key(2)) else { + panic!("second session must produce"); + }; + assert!(second.session.id > first_id); + drop(second); + + coordinator.next_session_id.store(u64::MAX, Ordering::Relaxed); + assert!(matches!(coordinator.join(indexed_key(3)), ColdFillRole::Rejected)); + assert!(matches!(coordinator.join(indexed_key(4)), ColdFillRole::Rejected)); + assert_eq!(coordinator.active_sessions.load(Ordering::Relaxed), 0); + } + + #[test] + fn late_old_session_remove_cannot_delete_new_same_key_session() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let session_key = key(); + let ColdFillRole::Produce(old_producer) = coordinator.join(session_key.clone()) else { + panic!("old session must produce"); + }; + let old_session = Arc::clone(&old_producer.session); + old_producer.bypass(); + + let ColdFillRole::Produce(new_producer) = coordinator.join(session_key.clone()) else { + panic!("new session must produce after old completion"); + }; + assert_ne!(old_session.id, new_producer.session.id); + coordinator.remove_if_current(&session_key, &old_session); + + assert_eq!(coordinator.active_session_count_for_test(), 1); + assert!(matches!(coordinator.join(session_key), ColdFillRole::Wait(_))); + drop(new_producer); + } + + #[tokio::test] + async fn second_pre_reader_failure_is_terminal_and_removes_session() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let attempts = Arc::new(AtomicUsize::new(0)); + let outcome = coordinate_cold_fill(&coordinator, key(), None, None, { + let attempts = Arc::clone(&attempts); + move |producer| { + let attempts = Arc::clone(&attempts); + async move { + let error = ColdFillError::Storage(StorageError::other("pre-reader failure")); + attempts.fetch_add(1, Ordering::Relaxed); + producer.relinquish_or_finish(error); + } + } + }) + .await; + + assert!(matches!( + outcome, + ColdFillCoordinateOutcome::Ready(Err(ColdFillError::Storage(StorageError::Io(_)))) + )); + assert_eq!(attempts.load(Ordering::Relaxed), usize::from(MAX_PRODUCER_ATTEMPTS)); + assert_eq!(coordinator.active_sessions.load(Ordering::Relaxed), 0); + assert!(matches!(coordinator.join(key()), ColdFillRole::Produce(_))); + } + + #[tokio::test] + async fn cold_fill_accepts_two_thousand_requests_with_one_producer() { + const REQUESTS: usize = 2000; + let coordinator = Arc::new(ColdFillCoordinator::default()); + let ColdFillRole::Produce(producer) = coordinator.join(key()) else { + panic!("first request must produce"); + }; + let mut waiters = Vec::with_capacity(REQUESTS - 1); + for _ in 1..REQUESTS { + let ColdFillRole::Wait(waiter) = coordinator.join(key()) else { + panic!("every follower in the acceptance matrix must wait"); + }; + waiters.push(waiter); + } + assert_eq!(coordinator.global_waiters.load(Ordering::Relaxed), REQUESTS - 1); + + producer.finish(Ok(Bytes::from_static(b"body"))); + for waiter in waiters { + let ColdFillWaitOutcome::Ready(Ok(body)) = waiter.wait().await else { + panic!("every follower must receive the shared body"); + }; + assert_eq!(body, Bytes::from_static(b"body")); + } + assert_eq!(coordinator.global_waiters.load(Ordering::Relaxed), 0); + } + + #[tokio::test(start_paused = true)] + async fn leader_waiter_deadline_does_not_cancel_longer_lived_follower() { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let producer_gate = Arc::new(tokio::sync::Semaphore::new(0)); + let producer_starts = Arc::new(AtomicUsize::new(0)); + let now = tokio::time::Instant::now(); + let producer_deadline = now + std::time::Duration::from_secs(1); + + let leader = { + let coordinator = Arc::clone(&coordinator); + let producer_gate = Arc::clone(&producer_gate); + let producer_starts = Arc::clone(&producer_starts); + tokio::spawn(async move { + coordinate_cold_fill( + &coordinator, + key(), + Some(now + std::time::Duration::from_millis(20)), + Some(producer_deadline), + move |producer| { + let producer_gate = Arc::clone(&producer_gate); + let producer_starts = Arc::clone(&producer_starts); + async move { + assert!(matches!(current_cold_fill_disk_permit_owner(), Some(ColdFillDiskPermitOwner::Producer))); + producer_starts.fetch_add(1, Ordering::Relaxed); + assert_eq!(producer.deadline(), Some(producer_deadline)); + let permit = producer_gate.acquire().await.expect("producer barrier must remain open"); + permit.forget(); + producer.finish(Ok(Bytes::from_static(b"body"))); + } + }, + ) + .await + }) + }; + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while coordinator.active_session_count_for_test() != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("leader must publish its session"); + + let follower = { + let coordinator = Arc::clone(&coordinator); + let producer_starts = Arc::clone(&producer_starts); + tokio::spawn(async move { + coordinate_cold_fill( + &coordinator, + key(), + Some(now + std::time::Duration::from_secs(1)), + Some(now + std::time::Duration::from_secs(2)), + move |producer| { + let producer_starts = Arc::clone(&producer_starts); + async move { + producer_starts.fetch_add(1, Ordering::Relaxed); + producer.finish(Ok(Bytes::from_static(b"unexpected"))); + } + }, + ) + .await + }) + }; + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while coordinator.global_waiter_count_for_test() != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("follower must join before the leader deadline"); + + tokio::time::advance(std::time::Duration::from_millis(20)).await; + assert!(matches!( + leader.await.expect("leader task must complete"), + ColdFillCoordinateOutcome::Ready(Err(ColdFillError::Storage(StorageError::Timeout))) + )); + + producer_gate.add_permits(1); + let ColdFillCoordinateOutcome::Ready(Ok(body)) = follower.await.expect("follower task must complete") else { + panic!("follower must receive the shared producer body"); + }; + assert_eq!(body, Bytes::from_static(b"body")); + assert_eq!(producer_starts.load(Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn cold_fill_production_seam_runs_permit_reader_and_allocation_once_per_key() { + const REQUESTS: usize = 2000; + for key_count in [1_usize, 4, 32] { + let coordinator = Arc::new(ColdFillCoordinator::default()); + let producer_gate = Arc::new(tokio::sync::Semaphore::new(0)); + let permits = Arc::new(AtomicUsize::new(0)); + let readers = Arc::new(AtomicUsize::new(0)); + let allocations = Arc::new(AtomicUsize::new(0)); + let mut tasks = tokio::task::JoinSet::new(); + + for request in 0..REQUESTS { + let coordinator = Arc::clone(&coordinator); + let producer_gate = Arc::clone(&producer_gate); + let permits = Arc::clone(&permits); + let readers = Arc::clone(&readers); + let allocations = Arc::clone(&allocations); + tasks.spawn(async move { + let outcome = + coordinate_cold_fill(&coordinator, indexed_key(request % key_count), None, None, move |producer| { + let producer_gate = Arc::clone(&producer_gate); + let permits = Arc::clone(&permits); + let readers = Arc::clone(&readers); + let allocations = Arc::clone(&allocations); + async move { + permits.fetch_add(1, Ordering::Relaxed); + readers.fetch_add(1, Ordering::Relaxed); + allocations.fetch_add(1, Ordering::Relaxed); + let permit = producer_gate.acquire().await.expect("test producer gate must stay open"); + permit.forget(); + producer.finish(Ok(Bytes::from_static(b"body"))); + } + }) + .await; + assert!(matches!(outcome, ColdFillCoordinateOutcome::Ready(Ok(_)))); + }); + } + + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while permits.load(Ordering::Relaxed) < key_count + || coordinator.global_waiters.load(Ordering::Relaxed) < REQUESTS - key_count + { + tokio::task::yield_now().await; + } + }) + .await + .expect("all matrix requests must join before the watchdog expires"); + producer_gate.add_permits(key_count); + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while let Some(result) = tasks.join_next().await { + result.expect("cold-fill matrix request must complete"); + } + }) + .await + .expect("all matrix responses must complete before the watchdog expires"); + + assert_eq!(permits.load(Ordering::Relaxed), key_count); + assert_eq!(readers.load(Ordering::Relaxed), key_count); + assert_eq!(allocations.load(Ordering::Relaxed), key_count); + } + } +} diff --git a/rustfs/src/app/object_data_cache/hook.rs b/rustfs/src/app/object_data_cache/hook.rs index df21f08ba..27be70f74 100644 --- a/rustfs/src/app/object_data_cache/hook.rs +++ b/rustfs/src/app/object_data_cache/hook.rs @@ -20,13 +20,18 @@ //! saves the erasure read/decode. use crate::app::object_data_cache::{ - GetObjectBodyCacheLookup, GetObjectBodyCacheRequest, ObjectDataCacheAdapter, build_get_object_body_cache_plan, - lookup_get_object_body_cache_hit, + GetObjectBodyCacheLookup, GetObjectBodyCachePlan, GetObjectBodyCacheRequest, ObjectDataCacheAdapter, + build_get_object_body_cache_plan, lookup_get_object_body_cache_hit, }; use crate::app::storage_api::object_usecase::StorageObjectInfo; +use crate::app::storage_api::object_usecase::StorageObjectOptions; +use crate::app::storage_api::object_usecase::contract::range::HTTPRangeSpec; use crate::storage::sse::contains_managed_encryption_metadata; use crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::LifecycleOps as _; -use crate::storage::storage_api::ecstore_object::{GetObjectBodyCacheHook, register_get_object_body_cache_hook}; +use crate::storage::storage_api::ecstore_object::{ + GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, + unregister_get_object_body_cache_hook, +}; use bytes::Bytes; use rustfs_utils::http::headers::SSEC_ALGORITHM_HEADER; use std::sync::Arc; @@ -36,10 +41,38 @@ pub(crate) struct ObjectDataCacheBodyHook { adapter: Arc, } -/// Registers the body-cache hook into ecstore. No-op for a disabled cache so -/// the hot path keeps a single `None` branch when the feature is off. +#[derive(Clone)] +struct PreplannedLookup { + adapter: Arc, + plan: GetObjectBodyCachePlan, +} + +tokio::task_local! { + static PREPLANNED_LOOKUP: PreplannedLookup; +} + +pub(crate) async fn lookup_preplanned_get_object_body_cache_hook( + adapter: Arc, + plan: GetObjectBodyCachePlan, + bucket: &str, + object: &str, + range: &Option, + opts: &StorageObjectOptions, + info: &StorageObjectInfo, +) -> GetObjectBodyCacheHookLookup { + PREPLANNED_LOOKUP + .scope( + PreplannedLookup { adapter, plan }, + lookup_get_object_body_cache_hook(bucket, object, range, opts, info), + ) + .await +} + +/// Registers the body-cache hook into ecstore, or removes the previous hook +/// when a config reload disables caching. pub(crate) fn register_object_data_cache_body_hook(adapter: Arc) { if adapter.is_disabled() { + unregister_get_object_body_cache_hook(); return; } register_get_object_body_cache_hook(Arc::new(ObjectDataCacheBodyHook { adapter })); @@ -58,18 +91,24 @@ impl GetObjectBodyCacheHook for ObjectDataCacheBodyHook { if info.is_remote() || object_metadata_indicates_encryption(&info.user_defined) { return None; } - let response_content_length = info.get_actual_size().ok()?; - let request = GetObjectBodyCacheRequest { - bucket, - key: object, - info, - response_content_length, - has_range: false, - part_number: None, - encryption_applied: false, + let preplanned = PREPLANNED_LOOKUP.try_with(Clone::clone).ok(); + let (adapter, plan) = match preplanned { + Some(preplanned) => (preplanned.adapter, preplanned.plan), + None => { + let response_content_length = info.get_actual_size().ok()?; + let request = GetObjectBodyCacheRequest { + bucket, + key: object, + info, + response_content_length, + has_range: false, + part_number: None, + encryption_applied: false, + }; + (Arc::clone(&self.adapter), build_get_object_body_cache_plan(&self.adapter, request)) + } }; - let plan = build_get_object_body_cache_plan(&self.adapter, request); - match lookup_get_object_body_cache_hit(&self.adapter, &plan).await { + match lookup_get_object_body_cache_hit(&adapter, &plan).await { GetObjectBodyCacheLookup::Hit(bytes) => Some(bytes), GetObjectBodyCacheLookup::Disabled | GetObjectBodyCacheLookup::Skip | GetObjectBodyCacheLookup::Miss => None, } @@ -79,6 +118,8 @@ impl GetObjectBodyCacheHook for ObjectDataCacheBodyHook { #[cfg(test)] mod tests { use super::*; + use metrics_util::MetricKind; + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; use rustfs_object_data_cache::{ObjectDataCacheConfig, ObjectDataCacheMode}; fn hit_only_adapter() -> Arc { @@ -129,6 +170,99 @@ mod tests { assert_eq!(hit, body); } + #[test] + #[serial_test::serial] + fn preplanned_hook_lookup_records_one_plan_for_one_get() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("hook metric runtime must build"); + metrics::with_local_recorder(&recorder, || { + runtime.block_on(async { + let adapter = hit_only_adapter(); + let info = plain_info(5); + let plan = build_get_object_body_cache_plan( + &adapter, + GetObjectBodyCacheRequest { + bucket: "b", + key: "k", + info: &info, + response_content_length: 5, + has_range: false, + part_number: None, + encryption_applied: false, + }, + ); + register_get_object_body_cache_hook(Arc::new(ObjectDataCacheBodyHook { + adapter: Arc::clone(&adapter), + })); + let result = lookup_preplanned_get_object_body_cache_hook( + adapter, + plan, + "b", + "k", + &None, + &StorageObjectOptions::default(), + &info, + ) + .await; + assert!(matches!(result, GetObjectBodyCacheHookLookup::Miss)); + unregister_get_object_body_cache_hook(); + }); + }); + + let plans = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter_map(|(composite, _unit, _description, value)| { + (composite.kind() == MetricKind::Counter && composite.key().name() == "rustfs_object_data_cache_plan_total") + .then_some(value) + }) + .map(|value| match value { + DebugValue::Counter(value) => value, + _ => panic!("plan metric must be a counter"), + }) + .sum::(); + assert_eq!(plans, 1); + } + + #[tokio::test] + #[serial_test::serial] + async fn disabled_reload_unregisters_previous_enabled_hook() { + let adapter = hit_only_adapter(); + let info = plain_info(5); + let plan = build_get_object_body_cache_plan( + &adapter, + GetObjectBodyCacheRequest { + bucket: "b", + key: "k", + info: &info, + response_content_length: 5, + has_range: false, + part_number: None, + encryption_applied: false, + }, + ); + let body = Bytes::from_static(b"hello"); + let _ = crate::app::object_data_cache::fill_get_object_body_cache_from_buffered_body(&adapter, &plan, &body).await; + register_object_data_cache_body_hook(adapter); + assert!(matches!( + lookup_get_object_body_cache_hook("b", "k", &None, &StorageObjectOptions::default(), &info).await, + GetObjectBodyCacheHookLookup::Hit(_) + )); + + let disabled = Arc::new(ObjectDataCacheAdapter::new(ObjectDataCacheConfig::default()).expect("disabled adapter")); + register_object_data_cache_body_hook(disabled); + + assert!(matches!( + lookup_get_object_body_cache_hook("b", "k", &None, &StorageObjectOptions::default(), &info).await, + GetObjectBodyCacheHookLookup::Absent + )); + } + #[tokio::test] async fn hook_lookup_skips_encrypted_objects() { let adapter = hit_only_adapter(); diff --git a/rustfs/src/app/object_data_cache/mod.rs b/rustfs/src/app/object_data_cache/mod.rs index b05feaffe..c5f288af1 100644 --- a/rustfs/src/app/object_data_cache/mod.rs +++ b/rustfs/src/app/object_data_cache/mod.rs @@ -16,6 +16,7 @@ mod adapter; mod body; +mod cold_fill; mod hook; mod invalidation; mod mutation_hook; @@ -26,7 +27,13 @@ pub(crate) use body::{ GetObjectBodyCacheLookup, fill_get_object_body_cache_from_buffered_body, fill_get_object_body_cache_from_materialized_body, lookup_get_object_body_cache_hit, }; -pub(crate) use hook::register_object_data_cache_body_hook; +pub(crate) use cold_fill::{ + ColdFillCoordinateOutcome, ColdFillCoordinator, ColdFillDiskPermitOwner, ColdFillError, ColdFillProducer, + coordinate_cold_fill, current_cold_fill_disk_permit_owner, +}; +#[cfg(test)] +pub(crate) use cold_fill::{ColdFillRole, ColdFillWaitOutcome, scope_cold_fill_disk_permit_owner_for_test}; +pub(crate) use hook::{lookup_preplanned_get_object_body_cache_hook, register_object_data_cache_body_hook}; pub(crate) use invalidation::{ invalidate_object_data_cache_after_complete_multipart_success, invalidate_object_data_cache_after_copy_success, invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_after_put_success, @@ -35,4 +42,7 @@ pub(crate) use invalidation::{ invalidate_object_data_cache_prefix_after_delete, invalidate_object_data_cache_prefix_before_mutation, }; pub(crate) use mutation_hook::register_object_data_cache_mutation_hook; -pub(crate) use planner::{GetObjectBodyCachePlan, GetObjectBodyCacheRequest, build_get_object_body_cache_plan}; +pub(crate) use planner::{ + GetObjectBodyCachePlan, GetObjectBodyCacheRequest, build_get_object_body_cache_plan, + build_get_object_body_cache_plan_for_revalidation, +}; diff --git a/rustfs/src/app/object_data_cache/mutation_hook.rs b/rustfs/src/app/object_data_cache/mutation_hook.rs index e1a4f4b8f..9744b0c12 100644 --- a/rustfs/src/app/object_data_cache/mutation_hook.rs +++ b/rustfs/src/app/object_data_cache/mutation_hook.rs @@ -20,7 +20,9 @@ //! dead bytes resident until TTL (ODC-26, backlog#1131). use crate::app::object_data_cache::ObjectDataCacheAdapter; -use crate::storage::storage_api::ecstore_object::{ObjectMutationHook, register_object_mutation_hook}; +use crate::storage::storage_api::ecstore_object::{ + ObjectMutationHook, register_object_mutation_hook, unregister_object_mutation_hook, +}; use rustfs_object_data_cache::{ObjectDataCacheIdentity, ObjectDataCacheInvalidationReason}; use std::sync::Arc; @@ -29,11 +31,11 @@ pub(crate) struct ObjectDataCacheMutationHook { adapter: Arc, } -/// Registers the mutation hook into ecstore. No-op for a disabled cache so the -/// delete paths keep a single `None` branch when the feature is off, matching -/// [`register_object_data_cache_body_hook`](super::register_object_data_cache_body_hook). +/// Registers the mutation hook into ecstore, or removes the previous hook when +/// a config reload disables caching. pub(crate) fn register_object_data_cache_mutation_hook(adapter: Arc) { if adapter.is_disabled() { + unregister_object_mutation_hook(); return; } register_object_mutation_hook(Arc::new(ObjectDataCacheMutationHook { adapter })); @@ -99,4 +101,22 @@ mod tests { assert!(matches!(adapter.lookup_body(&plan).await, ObjectDataCacheLookup::Miss)); } + + #[test] + #[serial_test::serial(object_mutation_hook)] + fn disabled_reload_unregisters_and_releases_previous_adapter() { + let adapter = fill_enabled_adapter(); + let weak = Arc::downgrade(&adapter); + register_object_data_cache_mutation_hook(adapter); + + let disabled = Arc::new(ObjectDataCacheAdapter::disabled()); + let disabled_weak = Arc::downgrade(&disabled); + register_object_data_cache_mutation_hook(disabled); + + assert!(weak.upgrade().is_none()); + assert!( + disabled_weak.upgrade().is_none(), + "disabled reload must clear the mutation-hook slot instead of registering the disabled adapter" + ); + } } diff --git a/rustfs/src/app/object_data_cache/planner.rs b/rustfs/src/app/object_data_cache/planner.rs index 1495fb794..bead6a723 100644 --- a/rustfs/src/app/object_data_cache/planner.rs +++ b/rustfs/src/app/object_data_cache/planner.rs @@ -39,10 +39,34 @@ pub(crate) enum GetObjectBodyCachePlan { Cacheable(ObjectDataCacheGetPlan), } +impl GetObjectBodyCachePlan { + pub(crate) fn key(&self) -> Option<&rustfs_object_data_cache::ObjectDataCacheKey> { + match self { + Self::Cacheable(plan) => plan.key(), + Self::Disabled | Self::Skip => None, + } + } +} + /// Builds a conservative body-cache plan for a GET request. pub(crate) fn build_get_object_body_cache_plan( adapter: &ObjectDataCacheAdapter, request: GetObjectBodyCacheRequest<'_>, +) -> GetObjectBodyCachePlan { + build_get_object_body_cache_plan_inner(adapter, request, true) +} + +pub(crate) fn build_get_object_body_cache_plan_for_revalidation( + adapter: &ObjectDataCacheAdapter, + request: GetObjectBodyCacheRequest<'_>, +) -> GetObjectBodyCachePlan { + build_get_object_body_cache_plan_inner(adapter, request, false) +} + +fn build_get_object_body_cache_plan_inner( + adapter: &ObjectDataCacheAdapter, + request: GetObjectBodyCacheRequest<'_>, + record_metric: bool, ) -> GetObjectBodyCachePlan { if adapter.is_disabled() { return GetObjectBodyCachePlan::Disabled; @@ -51,6 +75,7 @@ pub(crate) fn build_get_object_body_cache_plan( if request.has_range || request.part_number.is_some() || request.encryption_applied + || request.info.is_encrypted() || request.info.delete_marker || request.info.version_only || request.info.metadata_only @@ -105,7 +130,12 @@ pub(crate) fn build_get_object_body_cache_plan( body_variant: ObjectDataCacheBodyVariant::FullObjectPlainV1, }; - match adapter.plan_get(engine_request) { + let engine_plan = if record_metric { + adapter.plan_get(engine_request) + } else { + adapter.plan_get_untracked(engine_request) + }; + match engine_plan { ObjectDataCacheGetPlan::Disabled | ObjectDataCacheGetPlan::SkipTooLarge => GetObjectBodyCachePlan::Skip, plan @ ObjectDataCacheGetPlan::Cacheable { .. } => GetObjectBodyCachePlan::Cacheable(plan), } @@ -113,8 +143,13 @@ pub(crate) fn build_get_object_body_cache_plan( #[cfg(test)] mod tests { - use super::{GetObjectBodyCachePlan, GetObjectBodyCacheRequest, build_get_object_body_cache_plan}; + use super::{ + GetObjectBodyCachePlan, GetObjectBodyCacheRequest, build_get_object_body_cache_plan, + build_get_object_body_cache_plan_for_revalidation, + }; use crate::app::object_data_cache::ObjectDataCacheAdapter; + use metrics_util::MetricKind; + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; use rustfs_object_data_cache::{ObjectDataCacheConfig, ObjectDataCacheMode}; fn enabled_adapter() -> ObjectDataCacheAdapter { @@ -151,6 +186,63 @@ mod tests { assert!(matches!(plan, GetObjectBodyCachePlan::Disabled)); } + #[test] + fn revalidation_plan_does_not_count_a_second_get() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + let adapter = enabled_adapter(); + let info = crate::storage::storage_api::StorageObjectInfo { + etag: Some("etag".to_string()), + size: 4, + actual_size: 4, + ..Default::default() + }; + let request = GetObjectBodyCacheRequest { + bucket: "bucket", + key: "object", + info: &info, + response_content_length: 4, + has_range: false, + part_number: None, + encryption_applied: false, + }; + assert!(matches!( + build_get_object_body_cache_plan(&adapter, request), + GetObjectBodyCachePlan::Cacheable(_) + )); + assert!(matches!( + build_get_object_body_cache_plan_for_revalidation(&adapter, request), + GetObjectBodyCachePlan::Cacheable(_) + )); + assert!(matches!( + build_get_object_body_cache_plan_for_revalidation( + &adapter, + GetObjectBodyCacheRequest { + response_content_length: 9 * 1024 * 1024, + ..request + } + ), + GetObjectBodyCachePlan::Skip + )); + }); + + let plans = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter_map(|(composite, _unit, _description, value)| { + (composite.kind() == MetricKind::Counter && composite.key().name() == "rustfs_object_data_cache_plan_total") + .then_some(value) + }) + .map(|value| match value { + DebugValue::Counter(value) => value, + _ => panic!("plan metric must be a counter"), + }) + .sum::(); + assert_eq!(plans, 1); + } + #[test] fn plan_skips_range_requests() { let adapter = enabled_adapter(); @@ -176,6 +268,102 @@ mod tests { assert!(matches!(plan, GetObjectBodyCachePlan::Skip)); } + #[test] + fn cold_fill_bypass_variants_never_join_session() { + let adapter = enabled_adapter(); + let coordinator = adapter.cold_fill_coordinator(); + let info = crate::storage::storage_api::StorageObjectInfo { + etag: Some("etag".to_string()), + size: 4, + ..Default::default() + }; + + for (has_range, part_number, encryption_applied) in [(true, None, false), (false, Some(1), false), (false, None, true)] { + let plan = build_get_object_body_cache_plan( + &adapter, + GetObjectBodyCacheRequest { + bucket: "bucket", + key: "object", + info: &info, + response_content_length: 4, + has_range, + part_number, + encryption_applied, + }, + ); + assert!(matches!(plan, GetObjectBodyCachePlan::Skip)); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + + let mut remote = info; + remote.transitioned_object.status = "complete".to_string(); + let remote_plan = build_get_object_body_cache_plan( + &adapter, + GetObjectBodyCacheRequest { + bucket: "bucket", + key: "object", + info: &remote, + response_content_length: 4, + has_range: false, + part_number: None, + encryption_applied: false, + }, + ); + assert!(matches!(remote_plan, GetObjectBodyCachePlan::Skip)); + assert_eq!(coordinator.active_session_count_for_test(), 0); + + let encrypted = crate::storage::storage_api::StorageObjectInfo { + etag: Some("etag".to_string()), + size: 4, + user_defined: std::sync::Arc::new(std::collections::HashMap::from([( + "x-amz-server-side-encryption".to_string(), + "AES256".to_string(), + )])), + ..Default::default() + }; + let encrypted_plan = build_get_object_body_cache_plan( + &adapter, + GetObjectBodyCacheRequest { + bucket: "bucket", + key: "object", + info: &encrypted, + response_content_length: 4, + has_range: false, + part_number: None, + encryption_applied: false, + }, + ); + assert!(matches!(encrypted_plan, GetObjectBodyCachePlan::Skip)); + assert_eq!(coordinator.active_session_count_for_test(), 0); + + let compressed = crate::storage::storage_api::StorageObjectInfo { + etag: Some("etag".to_string()), + size: 4, + user_defined: std::sync::Arc::new(std::collections::HashMap::from([( + rustfs_utils::http::SUFFIX_COMPRESSION.to_string(), + "klauspost/compress/s2".to_string(), + )])), + ..Default::default() + }; + let compressed_plan = build_get_object_body_cache_plan( + &adapter, + GetObjectBodyCacheRequest { + bucket: "bucket", + key: "compressed-object", + info: &compressed, + response_content_length: 4, + has_range: false, + part_number: None, + encryption_applied: false, + }, + ); + assert!( + matches!(compressed_plan, GetObjectBodyCachePlan::Cacheable(_)), + "a decoded full-object compressed read must remain cacheable" + ); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + #[test] fn plan_skips_when_etag_is_missing() { let adapter = enabled_adapter(); diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 85252ed81..de6ccd611 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -74,6 +74,11 @@ use super::storage_api::object_usecase::error::{ use super::storage_api::object_usecase::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children}; use super::storage_api::object_usecase::helper::{OperationHelper, spawn_background_with_context}; use super::storage_api::object_usecase::io::{DynReader, HashReader, WritePlan, compression_metadata_value, wrap_reader}; +#[cfg(test)] +use super::storage_api::object_usecase::object_cache::GetObjectBodySource; +#[cfg(test)] +use super::storage_api::object_usecase::object_cache::lookup_get_object_body_cache_hook; +use super::storage_api::object_usecase::object_cache::{GetObjectBodyCacheHookLookup, get_object_body_cache_plaintext_len}; use super::storage_api::object_usecase::object_utils::to_s3s_etag; use super::storage_api::object_usecase::options::{ copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime_with_object_name, @@ -191,22 +196,175 @@ use tracing::{debug, error, instrument, warn}; use uuid::Uuid; use super::storage_api::object_usecase::{ - StorageDeletedObject, StorageObjectInfo as ObjectInfo, StorageObjectLockDeleteOptions, StorageObjectOptions as ObjectOptions, - StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader, + GetObjectReader, StorageDeletedObject, StorageObjectInfo as ObjectInfo, StorageObjectLockDeleteOptions, + StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader, }; use crate::app::object_data_cache::{ - GetObjectBodyCacheLookup, GetObjectBodyCachePlan, GetObjectBodyCacheRequest, ObjectDataCacheAdapter, - build_get_object_body_cache_plan, fill_get_object_body_cache_from_buffered_body, - fill_get_object_body_cache_from_materialized_body, invalidate_object_data_cache_after_copy_success, - invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_after_put_success, - invalidate_object_data_cache_before_mutation, invalidate_object_data_cache_objects_after_delete_success, - invalidate_object_data_cache_objects_before_mutation, invalidate_object_data_cache_prefix_after_delete, - invalidate_object_data_cache_prefix_before_mutation, lookup_get_object_body_cache_hit, + ColdFillCoordinateOutcome, ColdFillDiskPermitOwner, ColdFillError, ColdFillProducer, GetObjectBodyCacheLookup, + GetObjectBodyCachePlan, GetObjectBodyCacheRequest, ObjectDataCacheAdapter, build_get_object_body_cache_plan, + build_get_object_body_cache_plan_for_revalidation, coordinate_cold_fill, current_cold_fill_disk_permit_owner, + fill_get_object_body_cache_from_buffered_body, fill_get_object_body_cache_from_materialized_body, + invalidate_object_data_cache_after_copy_success, invalidate_object_data_cache_after_delete_success, + invalidate_object_data_cache_after_put_success, invalidate_object_data_cache_before_mutation, + invalidate_object_data_cache_objects_after_delete_success, invalidate_object_data_cache_objects_before_mutation, + invalidate_object_data_cache_prefix_after_delete, invalidate_object_data_cache_prefix_before_mutation, + lookup_get_object_body_cache_hit, lookup_preplanned_get_object_body_cache_hook, }; +#[cfg(test)] +use crate::app::object_data_cache::{ColdFillRole, ColdFillWaitOutcome, scope_cold_fill_disk_permit_owner_for_test}; type S3StdError = Box; +struct ColdFillDiskPermitMetric { + owner: ColdFillDiskPermitOwner, + metric_recorded: bool, +} + +#[cfg(test)] +static COLD_FILL_FOLLOWER_DISK_PERMITS_FOR_TEST: AtomicU64 = AtomicU64::new(0); + +#[cfg(test)] +struct ColdFillPublicationBarrier { + reached: tokio::sync::Semaphore, + release: tokio::sync::Semaphore, +} + +#[cfg(test)] +type ColdFillPublicationBarrierState = Option<(rustfs_object_data_cache::ObjectDataCacheKey, Arc)>; + +#[cfg(test)] +static COLD_FILL_PUBLICATION_BARRIER: OnceLock> = OnceLock::new(); + +#[cfg(test)] +type ColdFillReaderOpenProbeState = Option<(rustfs_object_data_cache::ObjectDataCacheKey, Arc)>; + +#[cfg(test)] +static COLD_FILL_READER_OPEN_PROBE: OnceLock> = OnceLock::new(); + +fn adjust_cold_fill_disk_permit_metric(owner: ColdFillDiskPermitOwner, acquired: bool) { + macro_rules! adjust_gauge { + ($name:literal) => {{ + #[cfg(not(test))] + let gauge = { + static HANDLE: std::sync::LazyLock = std::sync::LazyLock::new(|| metrics::gauge!($name)); + &*HANDLE + }; + #[cfg(test)] + let gauge = metrics::gauge!($name); + if acquired { + gauge.increment(1.0); + } else { + gauge.decrement(1.0); + } + }}; + } + + match owner { + ColdFillDiskPermitOwner::Producer => { + adjust_gauge!("rustfs_object_data_cache_cold_fill_producer_disk_permits"); + } + ColdFillDiskPermitOwner::Follower => { + adjust_gauge!("rustfs_object_data_cache_cold_fill_follower_disk_permits"); + } + } +} + +#[cfg(test)] +async fn wait_cold_fill_publication_barrier(plan: &rustfs_object_data_cache::ObjectDataCacheGetPlan) { + let Some(key) = plan.key() else { + return; + }; + let barrier = COLD_FILL_PUBLICATION_BARRIER + .get_or_init(|| Mutex::new(None)) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .as_ref() + .filter(|(barrier_key, _)| barrier_key == key) + .map(|(_, barrier)| Arc::clone(barrier)); + if let Some(barrier) = barrier { + barrier.reached.add_permits(1); + if let Ok(permit) = barrier.release.acquire().await { + permit.forget(); + } + } +} + +#[cfg(test)] +fn record_cold_fill_reader_open_for_test(plan: &rustfs_object_data_cache::ObjectDataCacheGetPlan) { + let Some(key) = plan.key() else { + return; + }; + let probe = COLD_FILL_READER_OPEN_PROBE + .get_or_init(|| Mutex::new(None)) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .as_ref() + .filter(|(probe_key, _)| probe_key == key) + .map(|(_, count)| Arc::clone(count)); + if let Some(count) = probe { + count.fetch_add(1, Ordering::Relaxed); + } +} + +impl ColdFillDiskPermitMetric { + fn new(owner: ColdFillDiskPermitOwner) -> Self { + let metric_recorded = rustfs_io_metrics::metrics_enabled(); + if metric_recorded { + adjust_cold_fill_disk_permit_metric(owner, true); + } + #[cfg(test)] + if matches!(owner, ColdFillDiskPermitOwner::Follower) { + COLD_FILL_FOLLOWER_DISK_PERMITS_FOR_TEST.fetch_add(1, Ordering::Relaxed); + } + Self { owner, metric_recorded } + } +} + +impl Drop for ColdFillDiskPermitMetric { + fn drop(&mut self) { + if self.metric_recorded { + adjust_cold_fill_disk_permit_metric(self.owner, false); + } + #[cfg(test)] + if matches!(self.owner, ColdFillDiskPermitOwner::Follower) { + COLD_FILL_FOLLOWER_DISK_PERMITS_FOR_TEST.fetch_sub(1, Ordering::Relaxed); + } + } +} + +struct GetObjectDiskPermit { + permit: Option, + metric: Option, +} + +impl GetObjectDiskPermit { + fn new(permit: OwnedSemaphorePermit) -> Self { + Self { + permit: Some(permit), + metric: current_cold_fill_disk_permit_owner().map(ColdFillDiskPermitMetric::new), + } + } + + fn release(&mut self) { + self.permit.take(); + self.metric.take(); + } +} + +impl From for GetObjectDiskPermit { + fn from(permit: OwnedSemaphorePermit) -> Self { + Self::new(permit) + } +} + +impl Drop for GetObjectDiskPermit { + fn drop(&mut self) { + self.release(); + } +} + const ACCEPT_RANGES_BYTES: &str = "bytes"; +const COLD_FILL_HARD_MAX_DURATION: Duration = Duration::from_secs(10 * 60); pub(crate) const MAX_GET_OBJECT_MEMORY_BUFFER_BYTES: i64 = 64 * 1024 * 1024; const MEDIUM_CONCURRENCY_GET_OBJECT_MEMORY_BUFFER_BYTES: i64 = 8 * 1024 * 1024; const HIGH_CONCURRENCY_GET_OBJECT_MEMORY_BUFFER_BYTES: i64 = 4 * 1024 * 1024; @@ -428,12 +586,18 @@ struct GetObjectBootstrap { struct GetObjectIoPlanning { /// `None` when inline fast path skips disk I/O semaphore. - disk_permit: Option, + disk_permit: Option, permit_wait_duration: Duration, queue_status: concurrency::IoQueueStatus, queue_utilization: f64, } +#[derive(Clone, Copy)] +struct GetObjectRequestTimeout<'a> { + wrapper: &'a RequestTimeoutWrapper, + policy: &'a GetObjectTimeoutPolicy, +} + struct GetObjectRequestContext { bucket: String, key: String, @@ -453,6 +617,7 @@ struct GetObjectReadSetup { /// ODC-16: the cache hook probed this read (served or missed), so the app /// layer must skip its own lookup. cache_hook_probed: bool, + cache_fill_allowed: bool, rs: Option, content_type: Option, last_modified: Option, @@ -681,12 +846,12 @@ pin_project! { struct DiskReadPermitReader { #[pin] inner: R, - disk_permit: Option, + disk_permit: Option, } } impl DiskReadPermitReader { - fn new(inner: R, disk_permit: OwnedSemaphorePermit) -> Self { + fn new(inner: R, disk_permit: GetObjectDiskPermit) -> Self { Self { inner, disk_permit: Some(disk_permit), @@ -705,8 +870,12 @@ where let poll = this.inner.poll_read(cx, buf); // EOF: no more disk reads can happen through this stream, so release // the permit instead of holding it until the client drops the body. - if had_capacity && matches!(poll, Poll::Ready(Ok(()))) && buf.filled().len() == filled_before { - this.disk_permit.take(); + if had_capacity + && matches!(poll, Poll::Ready(Ok(()))) + && buf.filled().len() == filled_before + && let Some(mut disk_permit) = this.disk_permit.take() + { + disk_permit.release(); } poll } @@ -865,7 +1034,7 @@ enum StrictMaterializeError { impl std::fmt::Display for StrictMaterializeError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::LengthMismatch { expected, actual } => { + Self::LengthMismatch { expected, actual, .. } => { write!(f, "materialized length mismatch: expected {expected}, got {actual}") } Self::Read { consumed, source } => { @@ -876,18 +1045,18 @@ impl std::fmt::Display for StrictMaterializeError { } impl StrictMaterializeError { - fn into_s3_error(self, response_content_length: i64) -> S3Error { + fn into_storage_error(self) -> StorageError { match self { - Self::LengthMismatch { expected, actual } => ApiError::from(StorageError::other(format!( - "GET object materialized length mismatch: declared content length {response_content_length}, expected {expected}, got {actual}" - ))) - .into(), - Self::Read { consumed, source } => ApiError::from(StorageError::other(format!( - "Failed to read object body into memory after {consumed} bytes: {source}" - ))) - .into(), + Self::LengthMismatch { expected, actual, .. } if actual < expected => StorageError::LessData, + Self::LengthMismatch { .. } => StorageError::MoreData, + Self::Read { source, .. } if source.kind() == std::io::ErrorKind::TimedOut => StorageError::Timeout, + Self::Read { source, .. } => StorageError::Io(std::io::Error::new(source.kind(), "object body read failed")), } } + + fn into_s3_error(self, _response_content_length: i64) -> S3Error { + ApiError::from(self.into_storage_error()).into() + } } /// Strictly materialize an object body into memory, enforcing an exact-length @@ -912,31 +1081,259 @@ async fn strict_materialize_object_body( where R: AsyncRead + Unpin, { + // Stop filling before the Vec reaches capacity. Calling `read_to_end` on a + // bounded reader can still reserve beyond `expected` before observing EOF. + // The over-long probe below stays outside this Vec so the admitted body + // allocation remains exactly `expected` bytes. let mut buf = Vec::with_capacity(expected); - // Read one byte past the declared length so an over-long stream is detected - // rather than silently truncated to `Content-Length`. - let mut bounded = tokio::io::AsyncReadExt::take(reader, expected as u64 + 1); + let mut reader = reader; let read_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); - let read_result = tokio::io::AsyncReadExt::read_to_end(&mut bounded, &mut buf).await; + let read_result = loop { + if buf.len() == expected { + break Ok(()); + } + match tokio::io::AsyncReadExt::read_buf(&mut reader, &mut buf).await { + Ok(0) => break Ok(()), + Ok(_) => {} + Err(source) => break Err(source), + } + }; + let actual = buf.len(); + let probe_result = if read_result.is_ok() && actual == expected { + let mut probe = [0_u8; 1]; + tokio::io::AsyncReadExt::read(&mut reader, &mut probe).await + } else { + Ok(0) + }; record_get_object_s3_handler_stage_duration(stage, read_start); - match read_result { - Ok(_) => { - if buf.len() == expected { + match (read_result, probe_result) { + (Ok(_), Ok(extra)) => { + let actual = actual.saturating_add(extra); + if actual == expected { Ok(buf) } else { - Err(StrictMaterializeError::LengthMismatch { - expected, - actual: buf.len(), - }) + Err(StrictMaterializeError::LengthMismatch { expected, actual }) } } - Err(source) => Err(StrictMaterializeError::Read { - consumed: buf.len(), + (Err(source), _) | (_, Err(source)) => Err(StrictMaterializeError::Read { + consumed: actual, source, }), } } +struct ColdFillProducerExecution { + expected: usize, + deadline: Option, + adapter: Arc, + engine_plan: rustfs_object_data_cache::ObjectDataCacheGetPlan, +} + +enum ColdFillStartupWaitError { + Cancelled, + DeadlineExceeded, +} + +async fn await_cold_fill_startup( + future: F, + cancellation: &tokio_util::sync::CancellationToken, + deadline: Option, +) -> Result +where + F: Future, +{ + tokio::pin!(future); + match deadline { + Some(deadline) => { + tokio::select! { + biased; + _ = cancellation.cancelled() => Err(ColdFillStartupWaitError::Cancelled), + result = tokio::time::timeout_at(deadline, &mut future) => { + result.map_err(|_| ColdFillStartupWaitError::DeadlineExceeded) + } + } + } + None => { + tokio::select! { + biased; + _ = cancellation.cancelled() => Err(ColdFillStartupWaitError::Cancelled), + result = &mut future => Ok(result), + } + } + } +} + +async fn start_cold_fill_producer( + producer: ColdFillProducer, + reservation: Option, + acquire_io: AcquireIo, + open_reader: OpenReader, + execution: ColdFillProducerExecution, +) where + AcquireIo: FnOnce() -> AcquireIoFuture, + AcquireIoFuture: Future>, + OpenReader: FnOnce() -> OpenReaderFuture, + OpenReaderFuture: Future>, +{ + let ColdFillProducerExecution { + expected, + deadline, + adapter, + engine_plan, + } = execution; + let hard_deadline = tokio::time::Instant::now() + COLD_FILL_HARD_MAX_DURATION; + let deadline = deadline.map_or(hard_deadline, |request_deadline| request_deadline.min(hard_deadline)); + let cancellation = producer.cancellation_token(); + let Some(reservation) = reservation else { + producer.bypass(); + return; + }; + let acquire = acquire_io(); + tokio::pin!(acquire); + let producer_io = tokio::select! { + _ = cancellation.cancelled() => { + producer.finish(Err(StorageError::OperationCanceled)); + return; + } + result = tokio::time::timeout_at(deadline, &mut acquire) => match result { + Ok(result) => result, + Err(_) => { + producer.relinquish_or_finish(ColdFillError::Storage(StorageError::Timeout)); + return; + } + } + }; + let producer_io = match producer_io { + Ok(io) => io, + Err(err) => { + producer.relinquish_or_finish(err); + return; + } + }; + + let open = open_reader(); + tokio::pin!(open); + let reader = match tokio::select! { + _ = cancellation.cancelled() => Err(StorageError::OperationCanceled), + result = tokio::time::timeout_at(deadline, &mut open) => { + result.unwrap_or(Err(StorageError::Timeout)) + } + } { + Ok(reader) => reader, + Err(err) => { + producer.relinquish_or_finish(ColdFillError::Storage(err)); + return; + } + }; + producer.mark_reader_started(); + let materialize = async move { + let GetObjectReader { + stream, buffered_body, .. + } = reader; + let body = if let Some(body) = buffered_body { + if body.len() == expected { + body + } else { + return Err(StorageError::other(format!( + "cold-fill buffered body length mismatch: expected {expected}, got {}", + body.len() + ))); + } + } else { + let stream = if let Some(permit) = producer_io.disk_permit { + wrap_reader(DiskReadPermitReader::new(stream, permit)) + } else { + stream + }; + Bytes::from( + strict_materialize_object_body(stream, expected, GET_OBJECT_STAGE_BODY_CACHE_MATERIALIZE_READ) + .await + .map_err(StrictMaterializeError::into_storage_error)?, + ) + }; + Ok::<_, StorageError>((body, reservation)) + }; + let materialized = tokio::select! { + _ = cancellation.cancelled() => Err(StorageError::OperationCanceled), + result = tokio::time::timeout_at(deadline, materialize) => { + result.unwrap_or(Err(StorageError::Timeout)) + } + }; + let result = match materialized { + Ok((body, reservation)) => { + if cancellation.is_cancelled() { + producer.finish(Err(StorageError::OperationCanceled)); + return; + } + if deadline <= tokio::time::Instant::now() { + producer.finish(Err(StorageError::Timeout)); + return; + } + let reserved = reservation.wrap_bytes(body); + let shared = reserved.bytes(); + let publish = async { + #[cfg(test)] + wait_cold_fill_publication_barrier(&engine_plan).await; + adapter.fill_reserved_body(&engine_plan, reserved).await + }; + tokio::pin!(publish); + tokio::select! { + _ = cancellation.cancelled() => Err(StorageError::OperationCanceled), + _ = tokio::time::sleep_until(deadline) => { + Err(StorageError::Timeout) + } + _ = &mut publish => Ok(shared), + } + } + Err(err) => Err(err), + }; + producer.finish(result); +} + +fn cold_fill_deadline( + wrapper: &RequestTimeoutWrapper, + timeout_config: &GetObjectTimeoutPolicy, + response_size: u64, +) -> Option { + if !timeout_config.is_timeout_enabled() { + return None; + } + Some(tokio::time::Instant::now() + wrapper.remaining_time_for_size(Some(response_size)).unwrap_or(Duration::ZERO)) +} + +fn cold_fill_producer_deadline(timeout_config: &GetObjectTimeoutPolicy, response_size: u64) -> tokio::time::Instant { + let now = tokio::time::Instant::now(); + let hard_deadline = now + COLD_FILL_HARD_MAX_DURATION; + if timeout_config.is_timeout_enabled() { + (now + timeout_config.calculate_timeout_for_size(response_size)).min(hard_deadline) + } else { + hard_deadline + } +} + +async fn lookup_cold_fill_second_chance( + adapter: &ObjectDataCacheAdapter, + plan: &rustfs_object_data_cache::ObjectDataCacheGetPlan, +) -> Option { + match adapter.peek_body_untracked(plan).await { + rustfs_object_data_cache::ObjectDataCacheLookup::Hit(body) => Some(body), + _ => None, + } +} + +fn retain_cold_fill_producer_for_matching_plan( + producer: ColdFillProducer, + current: &GetObjectBodyCachePlan, + expected: &rustfs_object_data_cache::ObjectDataCacheGetPlan, +) -> Option { + if current == &GetObjectBodyCachePlan::Cacheable(expected.clone()) { + Some(producer) + } else { + producer.bypass(); + None + } +} + impl futures::Stream for GetObjectReaderStream where R: AsyncRead, @@ -2866,6 +3263,11 @@ impl DefaultObjectUsecase { Ok(()) } + fn validate_get_object_before_cold_fill(headers: &HeaderMap, part_number: Option, info: &ObjectInfo) -> S3Result<()> { + check_preconditions(headers, info)?; + Self::validate_get_object_part_number(part_number, info) + } + /// How long a GET waits for a disk read permit before degrading to a /// permit-less read. Cached: consulted per GET. Zero disables the bound. fn disk_permit_wait_timeout() -> Duration { @@ -2880,8 +3282,7 @@ impl DefaultObjectUsecase { async fn acquire_get_object_io_planning( manager: &ConcurrencyManager, - wrapper: &RequestTimeoutWrapper, - timeout_config: &GetObjectTimeoutPolicy, + request_timeout: Option>, bucket: &str, key: &str, ) -> S3Result { @@ -2929,13 +3330,15 @@ impl DefaultObjectUsecase { }; let permit_wait_duration = permit_wait_start.elapsed(); - Self::ensure_get_object_not_timed_out( - wrapper, - timeout_config, - bucket, - key, - GetObjectTimeoutStage::DiskPermitWait { permit_wait_duration }, - )?; + if let Some(timeout) = request_timeout { + Self::ensure_get_object_not_timed_out( + timeout.wrapper, + timeout.policy, + bucket, + key, + GetObjectTimeoutStage::DiskPermitWait { permit_wait_duration }, + )?; + } let queue_status = manager.io_queue_status(); let queue_snapshot = GetObjectQueueSnapshot::from_available_permits( @@ -2963,16 +3366,50 @@ impl DefaultObjectUsecase { } } - Self::ensure_get_object_not_timed_out(wrapper, timeout_config, bucket, key, GetObjectTimeoutStage::BeforeRead)?; + if let Some(timeout) = request_timeout { + Self::ensure_get_object_not_timed_out( + timeout.wrapper, + timeout.policy, + bucket, + key, + GetObjectTimeoutStage::BeforeRead, + )?; + } Ok(GetObjectIoPlanning { - disk_permit, + disk_permit: disk_permit.map(GetObjectDiskPermit::new), permit_wait_duration, queue_status, queue_utilization, }) } + async fn acquire_cold_fill_io_planning( + manager: &'static ConcurrencyManager, + bucket: &str, + key: &str, + ) -> Result { + match Self::acquire_get_object_io_planning(manager, None, bucket, key).await { + Ok(io) => Ok(io), + Err(err) if err.code() == &S3ErrorCode::SlowDown => Err(ColdFillError::Storage(StorageError::SlowDown)), + Err(_) => Err(ColdFillError::DiskAdmissionClosed), + } + } + + fn get_object_io_planning_without_disk(manager: &ConcurrencyManager) -> GetObjectIoPlanning { + let queue_status = manager.io_queue_status(); + let queue_snapshot = GetObjectQueueSnapshot::from_available_permits( + queue_status.total_permits, + queue_status.total_permits.saturating_sub(queue_status.permits_in_use), + ); + GetObjectIoPlanning { + disk_permit: None, + permit_wait_duration: Duration::ZERO, + queue_utilization: queue_snapshot.utilization_percent(), + queue_status, + } + } + async fn prepare_get_object_request_context(req: &S3Request) -> S3Result { // Clone only the fields this path needs instead of the whole input. let bucket = req.input.bucket.clone(); @@ -3006,8 +3443,9 @@ impl DefaultObjectUsecase { } #[allow(clippy::too_many_arguments)] async fn prepare_get_object_read_execution( + &self, req: &S3Request, - manager: &ConcurrencyManager, + manager: &'static ConcurrencyManager, wrapper: &RequestTimeoutWrapper, timeout_config: &GetObjectTimeoutPolicy, bucket: &str, @@ -3016,8 +3454,6 @@ impl DefaultObjectUsecase { opts: &ObjectOptions, part_number: Option, ) -> S3Result { - let h = req.headers.clone(); - // SF05: Store lookup first (cached via SF01 moka cache). let store_lookup_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); let store = get_validated_store(bucket).await?; @@ -3029,52 +3465,338 @@ impl DefaultObjectUsecase { ); } - // Acquire the GET disk permit before ECStore reader setup. Reader setup - // may perform metadata fanout, materialize direct-memory bodies, or - // start legacy duplex background reads; all of that work belongs inside - // the admission boundary. Fully materialized bodies release the permit - // when the unused wrapped reader is dropped during body construction, - // while streaming bodies keep it until EOF or client drop. - let io_planning = Self::acquire_get_object_io_planning(manager, wrapper, timeout_config, bucket, key).await?; - let read_start = std::time::Instant::now(); let read_stage_start = rustfs_io_metrics::get_stage_metrics_enabled().then_some(read_start); - let read_setup = Self::prepare_get_object_read( - req, - &store, - manager, - bucket, - key, - rs, - h, - opts, - part_number, - read_start, - read_stage_start, - ) - .await?; + let cache_adapter = self.object_data_cache(); + if cache_adapter.is_disabled() || !cache_adapter.materialize_fill_enabled() { + let io_planning = Self::acquire_get_object_io_planning( + manager, + Some(GetObjectRequestTimeout { + wrapper, + policy: timeout_config, + }), + bucket, + key, + ) + .await?; + let reader = store + .get_object_reader(bucket, key, rs.clone(), req.headers.clone(), opts) + .await + .map_err(map_get_object_reader_error)?; + let read_setup = + Self::finish_get_object_read(req, manager, bucket, key, rs, part_number, read_start, reader, true).await?; + return Ok(GetObjectPreparedRead { io_planning, read_setup }); + } - Ok(GetObjectPreparedRead { io_planning, read_setup }) - } + // Preserve the legacy metadata-fanout bound without making followers + // hold a body-transfer permit while they wait on the cold-fill session. + let mut metadata_admission = Some( + Self::acquire_get_object_io_planning( + manager, + Some(GetObjectRequestTimeout { + wrapper, + policy: timeout_config, + }), + bucket, + key, + ) + .await?, + ); + let mut prepared = Some( + store + .prepare_get_object_reader(bucket, key, rs.clone(), HeaderMap::new(), opts) + .await + .map_err(map_get_object_reader_error)?, + ); + let mut cache_fill_allowed = true; + let mut legacy_hook_missed = false; + 'snapshot: { + let info = prepared + .as_ref() + .ok_or_else(|| s3_error!(InternalError, "prepared metadata snapshot is unavailable"))? + .object_info(); + // Preconditions, cache planning, and the authoritative hook lookup all + // run against one namespace-locked metadata snapshot. Cacheable misses + // release both the lock and short admission before joining cold fill. + let Some(response_content_length) = get_object_body_cache_plaintext_len(&rs, opts, info) else { + break 'snapshot; + }; + let cache_plan = build_get_object_body_cache_plan( + &cache_adapter, + GetObjectBodyCacheRequest { + bucket, + key, + info, + response_content_length, + has_range: rs.is_some(), + part_number, + encryption_applied: info.is_encrypted(), + }, + ); - #[allow(clippy::too_many_arguments)] - async fn prepare_get_object_read( - req: &S3Request, - store: &ECStore, - manager: &ConcurrencyManager, - bucket: &str, - key: &str, - mut rs: Option, - h: HeaderMap, - opts: &ObjectOptions, - part_number: Option, - read_start: std::time::Instant, - read_stage_start: Option, - ) -> S3Result { - let reader = store - .get_object_reader(bucket, key, rs.clone(), h, opts) - .await - .map_err(map_get_object_reader_error)?; + // The legacy hook is evaluated once, before cold-fill coordination. + // In-session producer retries never re-enter this snapshot block. + let legacy_probe = lookup_preplanned_get_object_body_cache_hook( + Arc::clone(&cache_adapter), + cache_plan.clone(), + bucket, + key, + &rs, + opts, + info, + ) + .await; + if matches!(legacy_probe, GetObjectBodyCacheHookLookup::Ineligible) { + break 'snapshot; + } + Self::validate_get_object_before_cold_fill(&req.headers, part_number, info)?; + if let GetObjectBodyCacheHookLookup::Hit(body) = legacy_probe { + drop(metadata_admission.take()); + let info = prepared + .take() + .ok_or_else(|| s3_error!(InternalError, "prepared cache-hit reader is unavailable"))? + .into_object_info(); + let reader = GetObjectReader::from_cache_body(info, body).map_err(ApiError::from)?; + let read_setup = + Self::finish_get_object_read(req, manager, bucket, key, rs, part_number, read_start, reader, true).await?; + return Ok(GetObjectPreparedRead { + io_planning: Self::get_object_io_planning_without_disk(manager), + read_setup, + }); + } + if matches!(legacy_probe, GetObjectBodyCacheHookLookup::Miss) { + legacy_hook_missed = true; + } + if !legacy_hook_missed + && let GetObjectBodyCacheLookup::Hit(body) = lookup_get_object_body_cache_hit(&cache_adapter, &cache_plan).await + { + drop(metadata_admission.take()); + let info = prepared + .take() + .ok_or_else(|| s3_error!(InternalError, "prepared cache-hit reader is unavailable"))? + .into_object_info(); + let reader = GetObjectReader::from_cache_body(info, body).map_err(ApiError::from)?; + let read_setup = + Self::finish_get_object_read(req, manager, bucket, key, rs, part_number, read_start, reader, true).await?; + return Ok(GetObjectPreparedRead { + io_planning: Self::get_object_io_planning_without_disk(manager), + read_setup, + }); + } + + let GetObjectBodyCachePlan::Cacheable(engine_plan) = &cache_plan else { + break 'snapshot; + }; + let Some(cache_key) = cache_plan.key().cloned() else { + break 'snapshot; + }; + let expected = usize::try_from(response_content_length) + .map_err(|_| s3_error!(InternalError, "cold-fill body length is not representable"))?; + let response_size = u64::try_from(response_content_length) + .map_err(|_| s3_error!(InternalError, "cold-fill body length is negative"))?; + let waiter_deadline = cold_fill_deadline(wrapper, timeout_config, response_size); + let proposed_producer_deadline = cold_fill_producer_deadline(timeout_config, response_size); + let coordinator = cache_adapter.cold_fill_coordinator(); + let info = prepared + .take() + .ok_or_else(|| s3_error!(InternalError, "prepared cold-fill reader is unavailable"))? + .into_object_info(); + drop(metadata_admission.take()); + let outcome = coordinate_cold_fill(&coordinator, cache_key, waiter_deadline, Some(proposed_producer_deadline), { + let adapter = &cache_adapter; + let headers = &req.headers; + let store = &store; + let range = &rs; + move |producer| { + let adapter = Arc::clone(adapter); + let engine_plan = engine_plan.clone(); + let h = headers.clone(); + let store = Arc::clone(store); + let range = range.clone(); + let bucket = bucket.to_owned(); + let key = key.to_owned(); + let opts = opts.clone(); + async move { + let producer_deadline = producer.deadline(); + let cancellation = producer.cancellation_token(); + let second_chance = match await_cold_fill_startup( + lookup_cold_fill_second_chance(&adapter, &engine_plan), + &cancellation, + producer_deadline, + ) + .await + { + Ok(body) => body, + Err(ColdFillStartupWaitError::Cancelled) => { + producer.finish(Err(StorageError::OperationCanceled)); + return; + } + Err(ColdFillStartupWaitError::DeadlineExceeded) => { + producer.relinquish_or_finish(ColdFillError::Storage(StorageError::Timeout)); + return; + } + }; + if let Some(body) = second_chance { + producer.finish_shared(Ok(body)); + return; + } + + let acquire = Self::acquire_cold_fill_io_planning(manager, &bucket, &key); + let producer_io = match await_cold_fill_startup(acquire, &cancellation, producer_deadline).await { + Ok(result) => result, + Err(ColdFillStartupWaitError::Cancelled) => { + producer.finish(Err(StorageError::OperationCanceled)); + return; + } + Err(ColdFillStartupWaitError::DeadlineExceeded) => { + producer.relinquish_or_finish(ColdFillError::Storage(StorageError::Timeout)); + return; + } + }; + let producer_io = match producer_io { + Ok(io) => io, + Err(err) => { + producer.finish_shared(Err(err)); + return; + } + }; + + let prepare = store.prepare_get_object_reader(&bucket, &key, range.clone(), HeaderMap::new(), &opts); + let prepared = match match await_cold_fill_startup(prepare, &cancellation, producer_deadline).await { + Ok(result) => result, + Err(ColdFillStartupWaitError::Cancelled) => { + producer.finish(Err(StorageError::OperationCanceled)); + return; + } + Err(ColdFillStartupWaitError::DeadlineExceeded) => { + producer.relinquish_or_finish(ColdFillError::Storage(StorageError::Timeout)); + return; + } + } { + Ok(prepared) => prepared, + Err(err) => { + producer.relinquish_or_finish(ColdFillError::Storage(err)); + return; + } + }; + let current_info = prepared.object_info(); + let current_length = match current_info.get_actual_size() { + Ok(length) => length, + Err(err) => { + let _ = err; + producer.finish_shared(Err(ColdFillError::Storage(StorageError::FileCorrupt))); + return; + } + }; + let current_plan = build_get_object_body_cache_plan_for_revalidation( + &adapter, + GetObjectBodyCacheRequest { + bucket: &bucket, + key: &key, + info: current_info, + response_content_length: current_length, + has_range: range.is_some(), + part_number, + encryption_applied: current_info.is_encrypted(), + }, + ); + let Some(producer) = retain_cold_fill_producer_for_matching_plan(producer, ¤t_plan, &engine_plan) + else { + return; + }; + + let reservation = adapter.reserve_body(&engine_plan); + #[cfg(test)] + let reader_open_plan = engine_plan.clone(); + start_cold_fill_producer( + producer, + reservation, + || async move { Ok(producer_io) }, + || { + #[cfg(test)] + record_cold_fill_reader_open_for_test(&reader_open_plan); + prepared.with_headers(h).into_reader() + }, + ColdFillProducerExecution { + expected, + deadline: producer_deadline, + adapter, + engine_plan, + }, + ) + .await; + } + } + }) + .await; + + match outcome { + ColdFillCoordinateOutcome::Ready(result) => { + let body = match result { + Ok(body) => body, + Err(ColdFillError::Storage(err)) => return Err(map_get_object_reader_error(err).into()), + Err(ColdFillError::DiskAdmissionClosed) => { + return Err(s3_error!(InternalError, "disk read semaphore closed")); + } + }; + let reader = GetObjectReader::from_cache_body(info, body).map_err(ApiError::from)?; + let read_setup = + Self::finish_get_object_read(req, manager, bucket, key, rs, part_number, read_start, reader, true) + .await?; + return Ok(GetObjectPreparedRead { + io_planning: Self::get_object_io_planning_without_disk(manager), + read_setup, + }); + } + ColdFillCoordinateOutcome::Bypass => { + cache_fill_allowed = false; + break 'snapshot; + } + ColdFillCoordinateOutcome::Rejected => return Err(ApiError::from(StorageError::SlowDown).into()), + } + } + + let (io_planning, reader) = if let Some(prepared) = prepared.take() { + let io_planning = metadata_admission + .take() + .ok_or_else(|| s3_error!(InternalError, "prepared metadata admission is unavailable"))?; + let reader = prepared + .with_headers(req.headers.clone()) + .into_reader() + .await + .map_err(map_get_object_reader_error)?; + (io_planning, reader) + } else { + let io_planning = Self::acquire_get_object_io_planning( + manager, + Some(GetObjectRequestTimeout { + wrapper, + policy: timeout_config, + }), + bucket, + key, + ) + .await?; + let reader = if legacy_hook_missed { + store + .prepare_get_object_reader(bucket, key, rs.clone(), HeaderMap::new(), opts) + .await + .map_err(map_get_object_reader_error)? + .with_headers(req.headers.clone()) + .into_reader() + .await + .map_err(map_get_object_reader_error)? + } else { + store + .get_object_reader(bucket, key, rs.clone(), req.headers.clone(), opts) + .await + .map_err(map_get_object_reader_error)? + }; + (io_planning, reader) + }; + let read_setup = + Self::finish_get_object_read(req, manager, bucket, key, rs, part_number, read_start, reader, cache_fill_allowed) + .await?; if let Some(read_stage_start) = read_stage_start { rustfs_io_metrics::record_get_object_stage_duration( "s3_handler", @@ -3082,7 +3804,21 @@ impl DefaultObjectUsecase { read_stage_start.elapsed().as_secs_f64(), ); } + Ok(GetObjectPreparedRead { io_planning, read_setup }) + } + #[allow(clippy::too_many_arguments)] + async fn finish_get_object_read( + req: &S3Request, + manager: &ConcurrencyManager, + bucket: &str, + key: &str, + mut rs: Option, + part_number: Option, + read_start: std::time::Instant, + reader: GetObjectReader, + cache_fill_allowed: bool, + ) -> S3Result { // ODC-16: capture whether the ecstore cache hook already probed this // read, so the app layer does not repeat the lookup it ran after fresh // metadata resolution. @@ -3203,6 +3939,7 @@ impl DefaultObjectUsecase { buffered_body, cache_hook_served, cache_hook_probed, + cache_fill_allowed, rs, content_type, last_modified, @@ -3493,6 +4230,7 @@ impl DefaultObjectUsecase { buffered_body: Option, cache_hook_served: bool, cache_hook_probed: bool, + cache_fill_allowed: bool, bucket: &str, key: &str, mut lifecycle: GetObjectBodyLifecycle, @@ -3500,6 +4238,37 @@ impl DefaultObjectUsecase { where R: AsyncRead + Send + Sync + Unpin + 'static, { + // ODC-16 (backlog#1121): when the ecstore hook or shared cold fill + // already supplied this body, the request-level plan was built before + // the authoritative lookup. Serve it without planning a second time. + if cache_hook_served && let Some(bytes) = buffered_body.clone() { + return Ok(Self::build_memory_bytes_blob( + bytes, + response_content_length, + GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE, + lifecycle, + )); + } + + if !cache_fill_allowed { + return Self::build_get_object_body( + final_stream, + info, + response_content_length, + optimal_buffer_size, + enable_readahead, + concurrent_requests, + part_number, + has_range, + encryption_applied, + buffered_body, + bucket, + key, + lifecycle, + ) + .await; + } + let cache_request = GetObjectBodyCacheRequest { bucket, key, @@ -3511,19 +4280,6 @@ impl DefaultObjectUsecase { }; let cache_plan = build_get_object_body_cache_plan(cache_adapter, cache_request); - // ODC-16 (backlog#1121): when the ecstore hook already served this body - // from the cache, serve it straight through as the object-data-cache - // source. Re-running the lookup here would record a second hit, double - // the hit_bytes, and do redundant moka work for one hook-served GET. - if cache_hook_served && let Some(bytes) = buffered_body.clone() { - return Ok(Self::build_memory_bytes_blob( - bytes, - response_content_length, - GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE, - lifecycle, - )); - } - // ODC-16: only look up when the hook did not probe this read. When it did // probe (a served body handled above, or a miss), its result is // authoritative because it ran after fresh metadata resolution, so the @@ -3548,14 +4304,14 @@ impl DefaultObjectUsecase { // detached task (Bytes is a cheap clone) and return immediately. For // a non-cacheable plan the fill is a pure metric-only skip with no // I/O, so record it inline to preserve observability. - if matches!(cache_plan, GetObjectBodyCachePlan::Cacheable(_)) { + if cache_fill_allowed && matches!(cache_plan, GetObjectBodyCachePlan::Cacheable(_)) { let cache_adapter = cache_adapter.clone(); let cache_plan = cache_plan.clone(); let fill_bytes = buffered_body.clone(); tokio::spawn(async move { let _ = fill_get_object_body_cache_from_buffered_body(&cache_adapter, &cache_plan, &fill_bytes).await; }); - } else { + } else if cache_fill_allowed { let _ = fill_get_object_body_cache_from_buffered_body(cache_adapter, &cache_plan, &buffered_body).await; } @@ -3568,6 +4324,7 @@ impl DefaultObjectUsecase { } let should_materialize_for_cache = cache_adapter.materialize_fill_enabled() + && cache_fill_allowed && matches!(cache_plan, GetObjectBodyCachePlan::Cacheable(_)) && should_materialize_get_object_body_for_cache( info, @@ -4430,6 +5187,7 @@ impl DefaultObjectUsecase { buffered_body: Option, cache_hook_served: bool, cache_hook_probed: bool, + cache_fill_allowed: bool, rs: Option, content_type: Option, last_modified: Option, @@ -4484,6 +5242,7 @@ impl DefaultObjectUsecase { buffered_body, cache_hook_served, cache_hook_probed, + cache_fill_allowed, bucket, key, lifecycle, @@ -4620,18 +5379,9 @@ impl DefaultObjectUsecase { let manager = get_concurrency_manager(); - let prepared_read = match Self::prepare_get_object_read_execution( - &req, - manager, - &wrapper, - &timeout_config, - &bucket, - &key, - rs, - &opts, - part_number, - ) - .await + let prepared_read = match self + .prepare_get_object_read_execution(&req, manager, &wrapper, &timeout_config, &bucket, &key, rs, &opts, part_number) + .await { Ok(prepared_read) => prepared_read, Err(err) => { @@ -4653,6 +5403,7 @@ impl DefaultObjectUsecase { buffered_body, cache_hook_served, cache_hook_probed, + cache_fill_allowed, rs, content_type, last_modified, @@ -4687,6 +5438,7 @@ impl DefaultObjectUsecase { buffered_body, cache_hook_served, cache_hook_probed, + cache_fill_allowed, rs, content_type, last_modified, @@ -7200,6 +7952,34 @@ mod tests { assert_eq!(throttle.claim(IO_QUEUE_CONGESTION_WARN_INTERVAL_MS + 1), None); } + #[tokio::test(start_paused = true)] + async fn cold_fill_disk_admission_preserves_slow_down() { + let manager = Box::leak(Box::new(ConcurrencyManager::with_disk_read_caps_for_test(1, 1))); + let primary = match manager.admit_disk_read(Duration::from_millis(1)).await.unwrap() { + DiskReadAdmission::Primary(permit) => permit, + other => panic!("expected primary admission, got {other:?}"), + }; + let degraded = match manager.admit_disk_read(Duration::from_millis(1)).await.unwrap() { + DiskReadAdmission::Degraded(permit) => permit, + other => panic!("expected degraded admission, got {other:?}"), + }; + + let result = DefaultObjectUsecase::acquire_cold_fill_io_planning(manager, "bucket", "object").await; + assert!(matches!(result, Err(ColdFillError::Storage(StorageError::SlowDown)))); + + drop(degraded); + drop(primary); + } + + #[tokio::test] + async fn cold_fill_closed_disk_admission_is_not_slow_down() { + let manager = Box::leak(Box::new(ConcurrencyManager::with_disk_read_caps_for_test(1, 1))); + manager.close_disk_read_admission_for_test(); + + let result = DefaultObjectUsecase::acquire_cold_fill_io_planning(manager, "bucket", "object").await; + assert!(matches!(result, Err(ColdFillError::DiskAdmissionClosed))); + } + // classify_response_checksums is the single point that splits decrypted checksum // pairs into the five s3s-typed fields and the additional-algorithm `extra` // headers, replacing five copies of the loop. Lock its behaviour (#1252). @@ -7748,6 +8528,37 @@ mod tests { data: std::io::Cursor>, } + struct ColdFillMatrixReader { + inner: tokio::io::DuplexStream, + first_poll_recorded: bool, + completion_recorded: bool, + first_polls: Arc, + completed: Arc, + bytes_read: Arc, + } + + impl AsyncRead for ColdFillMatrixReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if !self.first_poll_recorded { + self.first_poll_recorded = true; + self.first_polls.fetch_add(1, AtomicOrdering::Relaxed); + } + let before = buf.filled().len(); + match Pin::new(&mut self.inner).poll_read(cx, buf) { + Poll::Ready(Ok(())) => { + let read = buf.filled().len().saturating_sub(before); + self.bytes_read.fetch_add(read, AtomicOrdering::Relaxed); + if read == 0 && !self.completion_recorded { + self.completion_recorded = true; + self.completed.fetch_add(1, AtomicOrdering::Relaxed); + } + Poll::Ready(Ok(())) + } + other => other, + } + } + } + impl AsyncRead for DataProbeReader { fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { self.reads.fetch_add(1, AtomicOrdering::Relaxed); @@ -7827,12 +8638,46 @@ mod tests { .await .expect("exact-length read must materialize"); assert_eq!(buf, b"hello"); + assert_eq!(buf.capacity(), 5, "exact materialization must allocate only the declared body length"); + + let exact_large = vec![7_u8; 64 * 1024]; + let buf = strict_materialize_object_body( + std::io::Cursor::new(exact_large.clone()), + exact_large.len(), + GET_OBJECT_STAGE_BODY_SEEK_BUFFER_READ, + ) + .await + .expect("64 KiB exact-length read must materialize"); + assert_eq!(buf.capacity(), exact_large.len()); + + let mut overlong_large = exact_large; + overlong_large.push(9); + let overlong = strict_materialize_object_body( + std::io::Cursor::new(overlong_large), + 64 * 1024, + GET_OBJECT_STAGE_BODY_SEEK_BUFFER_READ, + ) + .await; + assert!(matches!( + overlong, + Err(StrictMaterializeError::LengthMismatch { + expected: 65_536, + actual: 65_537 + }) + )); // Short read (actual = expected - 1): a clean EOF before the declared // length must be a hard error, never a truncated served body. let short = strict_materialize_object_body(cursor_reader(b"hell"), 5, GET_OBJECT_STAGE_BODY_SEEK_BUFFER_READ).await; assert!( - matches!(short, Err(StrictMaterializeError::LengthMismatch { expected: 5, actual: 4 })), + matches!( + short, + Err(StrictMaterializeError::LengthMismatch { + expected: 5, + actual: 4, + .. + }) + ), "short read must fail with a length mismatch, got {short:?}", short = short.as_ref().map(|b| b.len()) ); @@ -7860,6 +8705,1478 @@ mod tests { ); } + #[test] + fn cold_fill_zero_timeout_policy_disables_deadline() { + let policy = GetObjectTimeoutPolicy { + get_object_timeout: Duration::ZERO, + ..GetObjectTimeoutPolicy::default() + }; + let wrapper = RequestTimeoutWrapper::with_request_id(policy.clone(), "cold-fill-zero-timeout"); + assert!(cold_fill_deadline(&wrapper, &policy, 1).is_none()); + } + + #[tokio::test(start_paused = true)] + async fn cold_fill_producer_deadline_is_capped_at_ten_minutes() { + let disabled = GetObjectTimeoutPolicy { + get_object_timeout: Duration::ZERO, + ..GetObjectTimeoutPolicy::default() + }; + let now = tokio::time::Instant::now(); + assert_eq!(cold_fill_producer_deadline(&disabled, 1) - now, Duration::from_secs(600)); + + let long = GetObjectTimeoutPolicy { + get_object_timeout: Duration::from_secs(3600), + enable_dynamic_timeout: false, + ..GetObjectTimeoutPolicy::default() + }; + let now = tokio::time::Instant::now(); + assert_eq!(cold_fill_producer_deadline(&long, 1) - now, Duration::from_secs(600)); + } + + #[tokio::test] + async fn cold_fill_startup_wait_stops_when_last_consumer_cancels() { + let cancellation = tokio_util::sync::CancellationToken::new(); + let waiting = tokio::spawn({ + let cancellation = cancellation.clone(); + async move { await_cold_fill_startup(std::future::pending::<()>(), &cancellation, None).await } + }); + tokio::task::yield_now().await; + + cancellation.cancel(); + + let result = tokio::time::timeout(Duration::from_secs(1), waiting) + .await + .expect("startup wait must observe cancellation") + .expect("startup wait task must not panic"); + assert!(matches!(result, Err(ColdFillStartupWaitError::Cancelled))); + } + + #[tokio::test(start_paused = true)] + async fn cold_fill_startup_wait_with_deadline_still_observes_cancellation() { + let cancellation = tokio_util::sync::CancellationToken::new(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + let waiting = tokio::spawn({ + let cancellation = cancellation.clone(); + async move { await_cold_fill_startup(std::future::pending::<()>(), &cancellation, Some(deadline)).await } + }); + tokio::task::yield_now().await; + + cancellation.cancel(); + + let result = waiting.await.expect("startup wait task must not panic"); + assert!(matches!(result, Err(ColdFillStartupWaitError::Cancelled))); + } + + #[tokio::test(start_paused = true)] + async fn cold_fill_startup_wait_reports_deadline_exceeded() { + let cancellation = tokio_util::sync::CancellationToken::new(); + let deadline = tokio::time::Instant::now() + Duration::from_millis(1); + + let result = await_cold_fill_startup(std::future::pending::<()>(), &cancellation, Some(deadline)).await; + + assert!(matches!(result, Err(ColdFillStartupWaitError::DeadlineExceeded))); + } + + #[tokio::test] + async fn cold_fill_late_miss_second_chance_hits_without_reader() { + let adapter = ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 1024 * 1024, + max_memory_percent: 0, + max_entry_bytes: 1024, + min_free_memory_percent: 0, + fill_concurrency_max: 1, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("second-chance cache config must be valid"); + let plan = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "late-bucket", + object: "late-object", + version_id: None, + etag: "late-etag", + size: 4, + data_dir_u128: Some(1), + mod_time_unix_nanos: 1, + body_variant: rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + assert!(matches!( + adapter.lookup_body(&plan).await, + rustfs_object_data_cache::ObjectDataCacheLookup::Miss + )); + let request_lookups = adapter.cache().stats().lookups; + assert_eq!(request_lookups, 1, "the authoritative request lookup must be counted once"); + + let reservation = adapter.reserve_body(&plan).expect("late producer must reserve"); + let reserved = reservation.wrap_bytes(Bytes::from_static(b"body")); + let _ = adapter.fill_reserved_body(&plan, reserved).await; + let coordinator = adapter.cold_fill_coordinator(); + let cache_key = plan.key().cloned().expect("late plan must be cacheable"); + let adapter = Arc::new(adapter); + let readers = Arc::new(AtomicUsize::new(0)); + let outcome = coordinate_cold_fill(&coordinator, cache_key, None, None, { + let adapter = Arc::clone(&adapter); + let readers = Arc::clone(&readers); + move |producer| { + let adapter = Arc::clone(&adapter); + let plan = plan.clone(); + let readers = Arc::clone(&readers); + async move { + if let Some(body) = lookup_cold_fill_second_chance(&adapter, &plan).await { + producer.finish_shared(Ok(body)); + return; + } + readers.fetch_add(1, AtomicOrdering::Relaxed); + producer.bypass(); + } + } + }) + .await; + let ColdFillCoordinateOutcome::Ready(Ok(body)) = outcome else { + panic!("late request must observe the completed fill, got {outcome:?}"); + }; + assert_eq!(body, Bytes::from_static(b"body")); + assert_eq!( + adapter.cache().stats().lookups, + request_lookups, + "the producer second chance must not count another request lookup" + ); + assert_eq!(readers.load(AtomicOrdering::Relaxed), 0); + } + + #[tokio::test] + async fn cold_fill_timeout_is_shared_and_releases_resources() { + let adapter = Arc::new( + ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 1024 * 1024, + max_memory_percent: 0, + max_entry_bytes: 1024, + min_free_memory_percent: 0, + fill_concurrency_max: 1, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("timeout cache config must be valid"), + ); + let plan = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "timeout-bucket", + object: "timeout-object", + version_id: None, + etag: "timeout-etag", + size: 1, + data_dir_u128: Some(1), + mod_time_unix_nanos: 1, + body_variant: rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + let key = plan.key().cloned().expect("timeout body must be cacheable"); + let coordinator = adapter.cold_fill_coordinator(); + let ColdFillRole::Produce(mut producer) = coordinator.join(key.clone()) else { + panic!("first timeout request must produce"); + }; + let leader = producer.waiter(); + let reservation = adapter.reserve_body(&plan); + let disk_permits = Arc::new(tokio::sync::Semaphore::new(1)); + let disk_gate = Arc::clone(&disk_permits); + let readers = Arc::new(AtomicUsize::new(0)); + let reader_count = Arc::clone(&readers); + let producer_task = tokio::spawn(start_cold_fill_producer( + producer, + reservation, + move || async move { + let permit = disk_gate + .acquire_owned() + .await + .map_err(|_| ColdFillError::DiskAdmissionClosed)?; + let mut io = DefaultObjectUsecase::get_object_io_planning_without_disk(get_concurrency_manager()); + io.disk_permit = Some(permit.into()); + Ok(io) + }, + move || async move { + reader_count.fetch_add(1, AtomicOrdering::Relaxed); + Ok(GetObjectReader { + stream: Box::new(PendingReader), + object_info: ObjectInfo { + size: 1, + actual_size: 1, + ..Default::default() + }, + buffered_body: None, + body_source: GetObjectBodySource::HookMissed, + }) + }, + ColdFillProducerExecution { + expected: 1, + deadline: Some(tokio::time::Instant::now() + Duration::from_millis(20)), + adapter: Arc::clone(&adapter), + engine_plan: plan.clone(), + }, + )); + tokio::time::timeout(Duration::from_secs(1), async { + while readers.load(AtomicOrdering::Relaxed) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("producer reader must open"); + let ColdFillRole::Wait(follower) = coordinator.join(key.clone()) else { + panic!("second timeout request must follow"); + }; + + let (leader_result, follower_result) = + tokio::time::timeout(Duration::from_secs(2), async { tokio::join!(leader.wait(), follower.wait()) }) + .await + .expect("typed timeout must wake all waiters"); + assert!(matches!( + leader_result, + ColdFillWaitOutcome::Ready(Err(ColdFillError::Storage(StorageError::Timeout))) + )); + assert!(matches!( + follower_result, + ColdFillWaitOutcome::Ready(Err(ColdFillError::Storage(StorageError::Timeout))) + )); + assert_eq!(readers.load(AtomicOrdering::Relaxed), 1); + assert_eq!(disk_permits.available_permits(), 1); + assert_eq!(coordinator.global_waiter_count_for_test(), 0); + assert_eq!(coordinator.active_session_count_for_test(), 0); + assert!(matches!( + adapter.lookup_body(&plan).await, + rustfs_object_data_cache::ObjectDataCacheLookup::Miss + )); + producer_task.await.expect("producer task must join"); + assert!(adapter.reserve_body(&plan).is_some(), "timeout must release the body reservation"); + let ColdFillRole::Produce(successor) = coordinator.join(key) else { + panic!("timeout must release the session for a successor"); + }; + drop(successor); + } + + #[tokio::test] + async fn cold_fill_survives_leader_request_cancellation_without_second_producer() { + let adapter = Arc::new( + ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 1024 * 1024, + max_memory_percent: 0, + max_entry_bytes: 1024, + min_free_memory_percent: 0, + fill_concurrency_max: 1, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("cancellation cache config must be valid"), + ); + let plan = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "cancel-bucket", + object: "cancel-object", + version_id: None, + etag: "cancel-etag", + size: 4, + data_dir_u128: Some(1), + mod_time_unix_nanos: 1, + body_variant: rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + let key = plan.key().cloned().expect("cancellation body must be cacheable"); + let coordinator = adapter.cold_fill_coordinator(); + let ColdFillRole::Produce(mut producer) = coordinator.join(key.clone()) else { + panic!("first cancellation request must produce"); + }; + let leader = producer.waiter(); + let reservation = adapter.reserve_body(&plan); + let readers = Arc::new(AtomicUsize::new(0)); + let reader_count = Arc::clone(&readers); + let writer_slot = Arc::new(Mutex::new(None)); + let writer_output = Arc::clone(&writer_slot); + let producer_task = tokio::spawn(start_cold_fill_producer( + producer, + reservation, + || async { Ok(DefaultObjectUsecase::get_object_io_planning_without_disk(get_concurrency_manager())) }, + move || async move { + reader_count.fetch_add(1, AtomicOrdering::Relaxed); + let (writer, reader) = tokio::io::duplex(16); + *writer_output.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(writer); + Ok(GetObjectReader { + stream: Box::new(reader), + object_info: ObjectInfo { + size: 4, + actual_size: 4, + ..Default::default() + }, + buffered_body: None, + body_source: GetObjectBodySource::HookMissed, + }) + }, + ColdFillProducerExecution { + expected: 4, + deadline: None, + adapter: Arc::clone(&adapter), + engine_plan: plan.clone(), + }, + )); + tokio::time::timeout(Duration::from_secs(1), async { + while readers.load(AtomicOrdering::Relaxed) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("cancellation producer reader must open"); + let ColdFillRole::Wait(follower) = coordinator.join(key.clone()) else { + panic!("second cancellation request must follow"); + }; + drop(leader); + assert_eq!(readers.load(AtomicOrdering::Relaxed), 1); + let ColdFillRole::Wait(late) = coordinator.join(key) else { + panic!("leader cancellation must not open a successor session"); + }; + drop(late); + + let mut writer = writer_slot + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .take() + .expect("reader factory must publish writer"); + tokio::io::AsyncWriteExt::write_all(&mut writer, b"body") + .await + .expect("body write must succeed"); + tokio::io::AsyncWriteExt::shutdown(&mut writer) + .await + .expect("body writer must close"); + let ColdFillWaitOutcome::Ready(result) = follower.wait().await else { + panic!("follower must receive producer result"); + }; + assert_eq!(result.expect("surviving producer must succeed"), Bytes::from_static(b"body")); + producer_task.await.expect("producer task must join"); + assert_eq!(readers.load(AtomicOrdering::Relaxed), 1); + } + + #[tokio::test] + async fn cold_fill_reservation_rejection_streams_without_materializing() { + let coordinator = Arc::new(crate::app::object_data_cache::ColdFillCoordinator::default()); + let plan = rustfs_object_data_cache::ObjectDataCacheGetPlan::Disabled; + let ColdFillRole::Produce(mut producer) = coordinator.join(rustfs_object_data_cache::ObjectDataCacheKey::new( + "bucket", + "object", + None, + "etag", + 4, + rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + )) else { + panic!("first rejected reservation request must produce"); + }; + let leader = producer.waiter(); + let permits = Arc::new(AtomicUsize::new(0)); + let readers = Arc::new(AtomicUsize::new(0)); + let permit_count = Arc::clone(&permits); + let reader_count = Arc::clone(&readers); + start_cold_fill_producer( + producer, + None, + move || async move { + permit_count.fetch_add(1, AtomicOrdering::Relaxed); + Ok(DefaultObjectUsecase::get_object_io_planning_without_disk(get_concurrency_manager())) + }, + move || async move { + reader_count.fetch_add(1, AtomicOrdering::Relaxed); + Err(StorageError::other("reader must not open")) + }, + ColdFillProducerExecution { + expected: 4, + deadline: None, + adapter: Arc::new(ObjectDataCacheAdapter::disabled()), + engine_plan: plan, + }, + ) + .await; + assert!(matches!(leader.wait().await, ColdFillWaitOutcome::Bypass)); + assert_eq!(permits.load(AtomicOrdering::Relaxed), 0); + assert_eq!(readers.load(AtomicOrdering::Relaxed), 0); + + let fallback_reads = Arc::new(AtomicUsize::new(0)); + let fallback_reader = DataProbeReader { + reads: Arc::clone(&fallback_reads), + data: std::io::Cursor::new(b"body".to_vec()), + }; + let info = ObjectInfo { + size: 4, + actual_size: 4, + ..Default::default() + }; + let mut fallback_body = DefaultObjectUsecase::build_get_object_body( + fallback_reader, + &info, + 4, + 128 * 1024, + false, + 1, + None, + false, + false, + None, + "bucket", + "object", + GetObjectBodyLifecycle::disabled(), + ) + .await + .expect("reservation bypass must construct the normal streaming fallback"); + let chunk = fallback_body + .next() + .await + .expect("fallback stream must yield a body chunk") + .expect("fallback stream must not fail"); + assert_eq!(chunk, Bytes::from_static(b"body")); + assert!(fallback_reads.load(AtomicOrdering::Relaxed) > 0); + assert_eq!(readers.load(AtomicOrdering::Relaxed), 0, "cold-fill materialization must remain unopened"); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + + #[tokio::test] + async fn cold_fill_internal_movement_and_restore_reads_never_join_sessions() { + let coordinator = Arc::new(crate::app::object_data_cache::ColdFillCoordinator::default()); + let info = ObjectInfo { + size: 4, + actual_size: 4, + ..Default::default() + }; + let mut restore = ObjectOptions::default(); + restore.transition.restore_request.days = Some(1); + let cases = [ + ObjectOptions { + raw_data_movement_read: true, + ..Default::default() + }, + ObjectOptions { + data_movement: true, + ..Default::default() + }, + restore, + ]; + + for opts in &cases { + assert!(matches!( + lookup_get_object_body_cache_hook("bucket", "object", &None, opts, &info).await, + GetObjectBodyCacheHookLookup::Ineligible + )); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + + let delete_marker = ObjectInfo { + delete_marker: true, + etag: Some("delete-marker-etag".to_string()), + ..Default::default() + }; + let delete_marker_part = ObjectOptions { + part_number: Some(2), + ..Default::default() + }; + assert!(matches!( + lookup_get_object_body_cache_hook("bucket", "object", &None, &delete_marker_part, &delete_marker).await, + GetObjectBodyCacheHookLookup::Ineligible + )); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + + #[tokio::test] + async fn cold_fill_generation_change_bypasses_before_opening_body() { + let adapter = Arc::new( + ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 1024 * 1024, + max_memory_percent: 0, + max_entry_bytes: 1024, + min_free_memory_percent: 0, + fill_concurrency_max: 1, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("generation retry cache config must be valid"), + ); + let request = |data_dir_u128| rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "generation-bucket", + object: "generation-object", + version_id: None, + etag: "generation-etag", + size: 4, + data_dir_u128: Some(data_dir_u128), + mod_time_unix_nanos: 1, + body_variant: rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + }; + let initial_plan = adapter.plan_get(request(1)); + let changed_plan = GetObjectBodyCachePlan::Cacheable(adapter.plan_get(request(2))); + let cache_key = initial_plan.key().cloned().expect("initial generation must be cacheable"); + let coordinator = adapter.cold_fill_coordinator(); + let body_opens = Arc::new(AtomicUsize::new(0)); + let producer_attempts = Arc::new(AtomicUsize::new(0)); + + let outcome = coordinate_cold_fill(&coordinator, cache_key, None, None, { + let body_opens = Arc::clone(&body_opens); + let producer_attempts = Arc::clone(&producer_attempts); + move |producer| { + let body_opens = Arc::clone(&body_opens); + let producer_attempts = Arc::clone(&producer_attempts); + let changed_plan = changed_plan.clone(); + let initial_plan = initial_plan.clone(); + async move { + producer_attempts.fetch_add(1, AtomicOrdering::Relaxed); + let Some(producer) = retain_cold_fill_producer_for_matching_plan(producer, &changed_plan, &initial_plan) + else { + return; + }; + body_opens.fetch_add(1, AtomicOrdering::Relaxed); + producer.bypass(); + } + } + }) + .await; + + assert!(matches!(outcome, ColdFillCoordinateOutcome::Bypass)); + assert_eq!(producer_attempts.load(AtomicOrdering::Relaxed), 1); + assert_eq!(body_opens.load(AtomicOrdering::Relaxed), 0); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + + async fn real_cold_fill_test_context() -> (Arc, Arc) { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + if current_app_context().is_none() { + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + } + let ambient = current_app_context().expect("real cold-fill tests require an ambient AppContext"); + let context = temp_env::with_vars( + [ + (rustfs_config::ENV_OBJECT_DATA_CACHE_ENABLE, Some("true")), + (rustfs_config::ENV_OBJECT_DATA_CACHE_MODE, Some("fill_materialize_enabled")), + (rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_BYTES, Some("8388608")), + (rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_ENTRY_BYTES, Some("2097152")), + (rustfs_config::ENV_OBJECT_DATA_CACHE_MIN_FREE_MEMORY_PERCENT, Some("0")), + ], + || Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms())), + ); + assert!(context.object_data_cache().materialize_fill_enabled()); + (store, context) + } + + async fn put_real_cold_fill_object(store: &Arc, bucket: &str, object: &str, body: &[u8]) -> ObjectInfo { + let mut reader = PutObjReader::from_vec(body.to_vec()); + store + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("real cold-fill test object must be written") + } + + fn real_cold_fill_plan( + adapter: &ObjectDataCacheAdapter, + bucket: &str, + object: &str, + info: &ObjectInfo, + ) -> rustfs_object_data_cache::ObjectDataCacheGetPlan { + let length = info + .get_actual_size() + .expect("real cold-fill test metadata must expose plaintext size"); + let GetObjectBodyCachePlan::Cacheable(plan) = build_get_object_body_cache_plan( + adapter, + GetObjectBodyCacheRequest { + bucket, + key: object, + info, + response_content_length: length, + has_range: false, + part_number: None, + encryption_applied: false, + }, + ) else { + panic!("real cold-fill test object must be cacheable"); + }; + plan + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn execute_get_object_rejects_conditions_before_joining_cold_fill() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let (store, context) = real_cold_fill_test_context().await; + let bucket = format!("cold-condition-{}", Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("real cold-fill condition bucket must be created"); + let body = vec![b'a'; 1_300_000]; + let info = put_real_cold_fill_object(&store, &bucket, object, &body).await; + let adapter = context.object_data_cache(); + let plan = real_cold_fill_plan(&adapter, &bucket, object, &info); + let coordinator = adapter.cold_fill_coordinator(); + let ColdFillRole::Produce(producer) = + coordinator.join(plan.key().cloned().expect("real cold-fill plan must expose its key")) + else { + panic!("test must reserve the initial cold-fill producer"); + }; + + let input = GetObjectInput::builder() + .bucket(bucket) + .key(object.to_string()) + .build() + .expect("real cold-fill GET input must build"); + let mut req = build_request(input, Method::GET); + let etag = info.etag.expect("real cold-fill test object must have an ETag"); + req.headers.insert( + http::header::IF_NONE_MATCH, + HeaderValue::from_str(&format!("\"{etag}\"")).expect("ETag header must be valid"), + ); + let usecase = DefaultObjectUsecase::with_context(Some(context)); + let result = tokio::time::timeout(Duration::from_secs(2), usecase.execute_get_object(req)) + .await + .expect("conditional GET must not wait for the reserved cold-fill session") + .expect_err("matching If-None-Match must reject the GET"); + + assert_eq!(result.code(), &S3ErrorCode::NotModified); + assert_eq!(coordinator.global_waiter_count_for_test(), 0); + drop(producer); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn execute_get_object_maps_cold_fill_session_rejection_to_slow_down_without_opening_reader() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let (store, context) = real_cold_fill_test_context().await; + let bucket = format!("cold-rejected-{}", Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("real cold-fill rejection bucket must be created"); + let body = vec![b'a'; 1_300_000]; + let info = put_real_cold_fill_object(&store, &bucket, object, &body).await; + let adapter = context.object_data_cache(); + let plan = real_cold_fill_plan(&adapter, &bucket, object, &info); + let cache_key = plan.key().cloned().expect("real cold-fill plan must expose its key"); + let coordinator = adapter.cold_fill_coordinator(); + let mut held_producers = Vec::new(); + for index in 0..2048 { + let saturation_key = rustfs_object_data_cache::ObjectDataCacheKey::new( + "cold-fill-saturation", + format!("object-{index}"), + None, + "etag", + 4, + rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + ); + match coordinator.join(saturation_key) { + ColdFillRole::Produce(producer) => held_producers.push(producer), + ColdFillRole::Rejected => break, + ColdFillRole::Wait(_) | ColdFillRole::Bypass => panic!("unique saturation keys must produce or reject"), + } + } + assert_eq!(coordinator.active_session_count_for_test(), held_producers.len()); + assert!(!held_producers.is_empty(), "saturation must reserve cold-fill sessions"); + + let reader_opens = Arc::new(AtomicU64::new(0)); + *COLD_FILL_READER_OPEN_PROBE + .get_or_init(|| Mutex::new(None)) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some((cache_key, Arc::clone(&reader_opens))); + let input = GetObjectInput::builder() + .bucket(bucket) + .key(object.to_string()) + .build() + .expect("real cold-fill rejection GET input must build"); + let usecase = DefaultObjectUsecase::with_context(Some(context)); + let result = tokio::time::timeout(Duration::from_secs(2), usecase.execute_get_object(build_request(input, Method::GET))) + .await + .expect("rejected real GET must not wait for a cold-fill session") + .expect_err("rejected real GET must return an S3 error"); + *COLD_FILL_READER_OPEN_PROBE + .get_or_init(|| Mutex::new(None)) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = None; + + assert_eq!(result.code(), &S3ErrorCode::SlowDown); + assert_eq!(reader_opens.load(Ordering::Relaxed), 0, "rejected GET must not open its body reader"); + assert_eq!(coordinator.active_session_count_for_test(), held_producers.len()); + drop(held_producers); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn execute_get_object_generation_change_bypasses_old_cold_fill_plan() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let (store, context) = real_cold_fill_test_context().await; + let bucket = format!("cold-generation-{}", Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("real cold-fill generation bucket must be created"); + let initial_body = vec![b'a'; 1_300_000]; + let changed_body = vec![b'b'; initial_body.len()]; + let initial_info = put_real_cold_fill_object(&store, &bucket, object, &initial_body).await; + let adapter = context.object_data_cache(); + let initial_plan = real_cold_fill_plan(&adapter, &bucket, object, &initial_info); + let coordinator = adapter.cold_fill_coordinator(); + let ColdFillRole::Produce(producer) = + coordinator.join(initial_plan.key().cloned().expect("real cold-fill plan must expose its key")) + else { + panic!("test must reserve the initial cold-fill producer"); + }; + + let input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(object.to_string()) + .build() + .expect("real cold-fill GET input must build"); + let usecase = DefaultObjectUsecase::with_context(Some(context)); + let request = tokio::spawn(async move { usecase.execute_get_object(build_request(input, Method::GET)).await }); + tokio::time::timeout(Duration::from_secs(2), async { + while coordinator.global_waiter_count_for_test() != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("real GET must join the reserved cold-fill session"); + + let changed_info = put_real_cold_fill_object(&store, &bucket, object, &changed_body).await; + assert_ne!(initial_info.etag, changed_info.etag); + producer.relinquish_or_finish(ColdFillError::Storage(StorageError::Timeout)); + + let mut response = tokio::time::timeout(Duration::from_secs(10), request) + .await + .expect("generation-changing GET must complete") + .expect("generation-changing GET task must join") + .expect("generation-changing GET must fall back successfully"); + let mut response_body = response.output.body.take().expect("GET response must include a body"); + let mut actual = Vec::with_capacity(changed_body.len()); + while let Some(chunk) = response_body.next().await { + actual.extend_from_slice(&chunk.expect("fallback body chunk must be readable")); + } + + assert_eq!(actual, changed_body); + assert!(matches!( + adapter.lookup_body(&initial_plan).await, + rustfs_object_data_cache::ObjectDataCacheLookup::Miss + )); + assert_eq!(coordinator.global_waiter_count_for_test(), 0); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + + #[tokio::test] + async fn cold_fill_open_error_retries_once_then_single_successor_succeeds() { + let adapter = Arc::new( + ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 1024 * 1024, + max_memory_percent: 0, + max_entry_bytes: 1024, + min_free_memory_percent: 0, + fill_concurrency_max: 1, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("open retry cache config must be valid"), + ); + let plan = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "open-retry-bucket", + object: "open-retry-object", + version_id: None, + etag: "open-retry-etag", + size: 4, + data_dir_u128: Some(1), + mod_time_unix_nanos: 1, + body_variant: rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + let cache_key = plan.key().cloned().expect("open retry plan must be cacheable"); + let coordinator = adapter.cold_fill_coordinator(); + let open_attempts = Arc::new(AtomicUsize::new(0)); + let open_attempts_for_start = Arc::clone(&open_attempts); + + let outcome = coordinate_cold_fill(&coordinator, cache_key, None, None, move |producer| { + let reservation = adapter.reserve_body(&plan); + let adapter = Arc::clone(&adapter); + let plan = plan.clone(); + let open_attempts = Arc::clone(&open_attempts_for_start); + async move { + start_cold_fill_producer( + producer, + reservation, + || async { Ok(DefaultObjectUsecase::get_object_io_planning_without_disk(get_concurrency_manager())) }, + move || async move { + let attempt = open_attempts.fetch_add(1, AtomicOrdering::Relaxed); + if attempt == 0 { + return Err(StorageError::other("first open fails")); + } + Ok(GetObjectReader { + stream: Box::new(std::io::Cursor::new(Vec::::new())), + object_info: ObjectInfo { + size: 4, + actual_size: 4, + ..Default::default() + }, + buffered_body: Some(Bytes::from_static(b"body")), + body_source: GetObjectBodySource::HookMissed, + }) + }, + ColdFillProducerExecution { + expected: 4, + deadline: None, + adapter, + engine_plan: plan, + }, + ) + .await + } + }) + .await; + + let ColdFillCoordinateOutcome::Ready(Ok(body)) = outcome else { + panic!("the unique successor must publish the body"); + }; + assert_eq!(body, Bytes::from_static(b"body")); + assert_eq!(open_attempts.load(AtomicOrdering::Relaxed), 2); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + + #[tokio::test] + async fn cold_fill_open_timeout_retries_once_then_is_terminal() { + tokio::time::pause(); + let adapter = Arc::new( + ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 1024 * 1024, + max_memory_percent: 0, + max_entry_bytes: 1024, + min_free_memory_percent: 0, + fill_concurrency_max: 1, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("open timeout cache config must be valid"), + ); + let plan = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "open-timeout-bucket", + object: "open-timeout-object", + version_id: None, + etag: "open-timeout-etag", + size: 4, + data_dir_u128: Some(1), + mod_time_unix_nanos: 1, + body_variant: rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + let cache_key = plan.key().cloned().expect("open timeout plan must be cacheable"); + let coordinator = adapter.cold_fill_coordinator(); + let open_attempts = Arc::new(AtomicUsize::new(0)); + + let deadline = tokio::time::Instant::now() + Duration::from_millis(10); + let task = tokio::spawn({ + let adapter = Arc::clone(&adapter); + let coordinator = Arc::clone(&coordinator); + let plan = plan.clone(); + let open_attempts = Arc::clone(&open_attempts); + async move { + coordinate_cold_fill(&coordinator, cache_key, None, Some(deadline), move |producer| { + let adapter = Arc::clone(&adapter); + let plan = plan.clone(); + let open_attempts = Arc::clone(&open_attempts); + let reservation = adapter.reserve_body(&plan); + let producer_deadline = producer.deadline(); + async move { + start_cold_fill_producer( + producer, + reservation, + || async { Ok(DefaultObjectUsecase::get_object_io_planning_without_disk(get_concurrency_manager())) }, + move || async move { + open_attempts.fetch_add(1, AtomicOrdering::Relaxed); + std::future::pending::>().await + }, + ColdFillProducerExecution { + expected: 4, + deadline: producer_deadline, + adapter, + engine_plan: plan, + }, + ) + .await + } + }) + .await + } + }); + while open_attempts.load(AtomicOrdering::Relaxed) == 0 { + tokio::task::yield_now().await; + } + tokio::time::advance(Duration::from_millis(11)).await; + let outcome = task.await.expect("open timeout task must join"); + assert!(matches!( + outcome, + ColdFillCoordinateOutcome::Ready(Err(ColdFillError::Storage(StorageError::Timeout))) + )); + + assert_eq!(open_attempts.load(AtomicOrdering::Relaxed), 2); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + + #[tokio::test] + async fn cold_fill_pre_reader_failure_promotes_one_of_two_thousand_waiters() { + const REQUESTS: usize = 2000; + let adapter = Arc::new( + ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 1024 * 1024, + max_memory_percent: 0, + max_entry_bytes: 1024, + min_free_memory_percent: 0, + fill_concurrency_max: 1, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("successor cache config must be valid"), + ); + let plan = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "successor-bucket", + object: "successor-object", + version_id: None, + etag: "successor-etag", + size: 4, + data_dir_u128: Some(1), + mod_time_unix_nanos: 1, + body_variant: rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + let cache_key = plan.key().cloned().expect("successor plan must be cacheable"); + let coordinator = adapter.cold_fill_coordinator(); + let admission_attempts = Arc::new(AtomicUsize::new(0)); + let open_attempts = Arc::new(AtomicUsize::new(0)); + let first_open_release = Arc::new(tokio::sync::Semaphore::new(0)); + let mut tasks = tokio::task::JoinSet::new(); + + for _ in 0..REQUESTS { + let adapter = Arc::clone(&adapter); + let coordinator = Arc::clone(&coordinator); + let cache_key = cache_key.clone(); + let plan = plan.clone(); + let admission_attempts = Arc::clone(&admission_attempts); + let open_attempts = Arc::clone(&open_attempts); + let first_open_release = Arc::clone(&first_open_release); + tasks.spawn(async move { + coordinate_cold_fill(&coordinator, cache_key, None, None, move |producer| { + let reservation = adapter.reserve_body(&plan); + let adapter = Arc::clone(&adapter); + let plan = plan.clone(); + let admission_attempts = Arc::clone(&admission_attempts); + let open_attempts = Arc::clone(&open_attempts); + let first_open_release = Arc::clone(&first_open_release); + async move { + start_cold_fill_producer( + producer, + reservation, + move || async move { + admission_attempts.fetch_add(1, AtomicOrdering::Relaxed); + Ok(DefaultObjectUsecase::get_object_io_planning_without_disk(get_concurrency_manager())) + }, + move || async move { + if open_attempts.fetch_add(1, AtomicOrdering::Relaxed) == 0 { + first_open_release + .acquire() + .await + .expect("first open release gate must remain open") + .forget(); + return Err(StorageError::other("first open fails")); + } + Ok(GetObjectReader { + stream: Box::new(std::io::Cursor::new(Vec::::new())), + object_info: ObjectInfo { + size: 4, + actual_size: 4, + ..Default::default() + }, + buffered_body: Some(Bytes::from_static(b"body")), + body_source: GetObjectBodySource::HookMissed, + }) + }, + ColdFillProducerExecution { + expected: 4, + deadline: None, + adapter, + engine_plan: plan, + }, + ) + .await + } + }) + .await + }); + } + + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if coordinator.global_waiter_count_for_test() == REQUESTS - 1 && open_attempts.load(AtomicOrdering::Relaxed) == 1 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("all followers must join before the first open fails"); + first_open_release.add_permits(1); + + while let Some(result) = tasks.join_next().await { + let ColdFillCoordinateOutcome::Ready(Ok(body)) = result.expect("successor request task must join") else { + panic!("all followers must receive the successor body"); + }; + assert_eq!(body, Bytes::from_static(b"body")); + } + assert_eq!(admission_attempts.load(AtomicOrdering::Relaxed), 2); + assert_eq!(open_attempts.load(AtomicOrdering::Relaxed), 2); + assert_eq!(coordinator.global_waiter_count_for_test(), 0); + assert_eq!(coordinator.active_session_count_for_test(), 0); + } + + fn install_cold_fill_publication_barrier( + plan: &rustfs_object_data_cache::ObjectDataCacheGetPlan, + ) -> Arc { + let barrier = Arc::new(ColdFillPublicationBarrier { + reached: tokio::sync::Semaphore::new(0), + release: tokio::sync::Semaphore::new(0), + }); + let key = plan.key().cloned().expect("publication barrier plan must be cacheable"); + *COLD_FILL_PUBLICATION_BARRIER + .get_or_init(|| Mutex::new(None)) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some((key, Arc::clone(&barrier))); + barrier + } + + fn clear_cold_fill_publication_barrier() { + *COLD_FILL_PUBLICATION_BARRIER + .get_or_init(|| Mutex::new(None)) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = None; + } + + fn publication_test_adapter() -> Arc { + Arc::new( + ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 1024 * 1024, + max_memory_percent: 0, + max_entry_bytes: 1024, + min_free_memory_percent: 0, + fill_concurrency_max: 1, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("publication cache config must be valid"), + ) + } + + fn publication_test_plan(adapter: &ObjectDataCacheAdapter, object: &str) -> rustfs_object_data_cache::ObjectDataCacheGetPlan { + adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "publication-bucket", + object, + version_id: None, + etag: "publication-etag", + size: 4, + data_dir_u128: Some(1), + mod_time_unix_nanos: 1, + body_variant: rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + }) + } + + #[tokio::test] + #[serial_test::serial(cold_fill_publication_barrier)] + async fn cold_fill_last_consumer_cancel_releases_session_before_publication_barrier() { + let adapter = publication_test_adapter(); + let plan = publication_test_plan(&adapter, "cancel"); + let barrier = install_cold_fill_publication_barrier(&plan); + let coordinator = adapter.cold_fill_coordinator(); + let key = plan.key().cloned().expect("publication plan must be cacheable"); + let ColdFillRole::Produce(mut producer) = coordinator.join(key) else { + panic!("publication request must produce"); + }; + let leader = producer.waiter(); + let reservation = adapter.reserve_body(&plan); + let disk_permits = Arc::new(tokio::sync::Semaphore::new(1)); + let disk_gate = Arc::clone(&disk_permits); + let producer_task = tokio::spawn(scope_cold_fill_disk_permit_owner_for_test( + ColdFillDiskPermitOwner::Producer, + start_cold_fill_producer( + producer, + reservation, + move || async move { + let permit = disk_gate + .acquire_owned() + .await + .map_err(|_| ColdFillError::DiskAdmissionClosed)?; + let mut io = DefaultObjectUsecase::get_object_io_planning_without_disk(get_concurrency_manager()); + io.disk_permit = Some(permit.into()); + Ok(io) + }, + || async { + Ok(GetObjectReader { + stream: Box::new(std::io::Cursor::new(b"body".to_vec())), + object_info: ObjectInfo { + size: 4, + actual_size: 4, + ..Default::default() + }, + buffered_body: None, + body_source: GetObjectBodySource::HookMissed, + }) + }, + ColdFillProducerExecution { + expected: 4, + deadline: None, + adapter: Arc::clone(&adapter), + engine_plan: plan.clone(), + }, + ), + )); + + let reached = barrier.reached.acquire().await.expect("publication barrier must remain open"); + reached.forget(); + assert_eq!( + disk_permits.available_permits(), + 1, + "the producer disk permit and its gauge guard must end before publication" + ); + let clear_adapter = Arc::clone(&adapter); + let clear = tokio::spawn(async move { + clear_adapter + .clear(rustfs_object_data_cache::ObjectDataCacheInvalidationReason::Manual) + .await + }); + tokio::task::yield_now().await; + assert!(!clear.is_finished(), "clear must wait while publication owns its reservation"); + drop(leader); + tokio::time::timeout(Duration::from_secs(1), async { + while coordinator.active_session_count_for_test() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("last-consumer cancellation must release the session immediately"); + tokio::time::timeout(Duration::from_secs(1), clear) + .await + .expect("clear must finish after publication cancellation") + .expect("clear task must join"); + producer_task.await.expect("producer task must join"); + + barrier.release.add_permits(1); + clear_cold_fill_publication_barrier(); + drop(adapter.reserve_body(&plan).expect("publication reservation must be released")); + } + + #[tokio::test(start_paused = true)] + #[serial_test::serial(cold_fill_publication_barrier)] + async fn cold_fill_hard_deadline_releases_session_at_publication_barrier() { + let adapter = publication_test_adapter(); + let plan = publication_test_plan(&adapter, "deadline"); + let barrier = install_cold_fill_publication_barrier(&plan); + let coordinator = adapter.cold_fill_coordinator(); + let key = plan.key().cloned().expect("publication plan must be cacheable"); + let ColdFillRole::Produce(mut producer) = coordinator.join(key) else { + panic!("publication request must produce"); + }; + let leader = producer.waiter(); + let reservation = adapter.reserve_body(&plan); + let deadline = tokio::time::Instant::now() + Duration::from_millis(20); + let producer_task = tokio::spawn(start_cold_fill_producer( + producer, + reservation, + || async { Ok(DefaultObjectUsecase::get_object_io_planning_without_disk(get_concurrency_manager())) }, + || async { + Ok(GetObjectReader { + stream: Box::new(std::io::Cursor::new(Vec::::new())), + object_info: ObjectInfo { + size: 4, + actual_size: 4, + ..Default::default() + }, + buffered_body: Some(Bytes::from_static(b"body")), + body_source: GetObjectBodySource::HookMissed, + }) + }, + ColdFillProducerExecution { + expected: 4, + deadline: Some(deadline), + adapter: Arc::clone(&adapter), + engine_plan: plan.clone(), + }, + )); + + let reached = barrier.reached.acquire().await.expect("publication barrier must remain open"); + reached.forget(); + tokio::time::advance(Duration::from_millis(20)).await; + assert!(matches!( + leader.wait().await, + ColdFillWaitOutcome::Ready(Err(ColdFillError::Storage(StorageError::Timeout))) + )); + assert_eq!(coordinator.active_session_count_for_test(), 0); + producer_task.await.expect("producer task must join"); + + barrier.release.add_permits(1); + clear_cold_fill_publication_barrier(); + drop( + adapter + .reserve_body(&plan) + .expect("deadline must release the publication reservation"), + ); + tokio::time::timeout( + Duration::from_secs(1), + adapter.clear(rustfs_object_data_cache::ObjectDataCacheInvalidationReason::Manual), + ) + .await + .expect("clear must complete after publication deadline"); + } + + #[tokio::test(start_paused = true)] + async fn cold_fill_without_request_timeout_stops_at_ten_minute_hard_cap() { + let adapter = publication_test_adapter(); + let plan = publication_test_plan(&adapter, "hard-cap"); + let coordinator = adapter.cold_fill_coordinator(); + let key = plan.key().cloned().expect("hard-cap plan must be cacheable"); + let ColdFillRole::Produce(mut producer) = coordinator.join(key) else { + panic!("hard-cap request must produce"); + }; + let leader = producer.waiter(); + let reservation = adapter.reserve_body(&plan); + let producer_task = tokio::spawn(start_cold_fill_producer( + producer, + reservation, + || async { Ok(DefaultObjectUsecase::get_object_io_planning_without_disk(get_concurrency_manager())) }, + || async { + Ok(GetObjectReader { + stream: Box::new(PendingReader), + object_info: ObjectInfo { + size: 4, + actual_size: 4, + ..Default::default() + }, + buffered_body: None, + body_source: GetObjectBodySource::HookMissed, + }) + }, + ColdFillProducerExecution { + expected: 4, + deadline: None, + adapter: Arc::clone(&adapter), + engine_plan: plan.clone(), + }, + )); + let wait = tokio::spawn(async move { leader.wait().await }); + + tokio::time::advance(Duration::from_secs(599)).await; + tokio::task::yield_now().await; + assert!(!wait.is_finished(), "hard cap must not fire before 600 seconds"); + assert!(adapter.reserve_body(&plan).is_none(), "reservation must remain owned before the hard cap"); + + tokio::time::advance(Duration::from_secs(1)).await; + assert!(matches!( + wait.await.expect("hard-cap waiter must join"), + ColdFillWaitOutcome::Ready(Err(ColdFillError::Storage(StorageError::Timeout))) + )); + producer_task.await.expect("producer task must join"); + assert_eq!(coordinator.active_session_count_for_test(), 0); + drop( + adapter + .reserve_body(&plan) + .expect("hard cap must release the body reservation"), + ); + } + + #[tokio::test] + async fn build_get_object_body_with_cache_same_key_cold_fill_consumes_one_reader() { + const REQUESTS: usize = 2000; + const BODY_BYTES: usize = 64 * 1024; + const BODY_BYTES_U64: u64 = 64 * 1024; + const BODY_BYTES_I64: i64 = 64 * 1024; + + for key_count in [1_usize, 4, 32] { + let adapter = Arc::new( + ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 128 * 1024 * 1024, + max_memory_percent: 0, + max_entry_bytes: 1024 * 1024, + min_free_memory_percent: 0, + fill_concurrency_per_cpu: 64, + fill_concurrency_max: 64, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("matrix cache config must be valid"), + ); + let coordinator = adapter.cold_fill_coordinator(); + let disk_permits = Arc::new(tokio::sync::Semaphore::new(key_count)); + let writers = Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(key_count))); + let permit_acquires = Arc::new(AtomicUsize::new(0)); + let reader_factories = Arc::new(AtomicUsize::new(0)); + let first_polls = Arc::new(AtomicUsize::new(0)); + let completed = Arc::new(AtomicUsize::new(0)); + let bytes_read = Arc::new(AtomicUsize::new(0)); + let mut tasks = tokio::task::JoinSet::new(); + + for request in 0..REQUESTS { + let key_index = request % key_count; + let object = format!("matrix-object-{key_index}"); + let engine_plan = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "matrix-bucket", + object: &object, + version_id: None, + etag: "matrix-etag", + size: BODY_BYTES_U64, + data_dir_u128: Some(u128::try_from(key_index).unwrap_or(u128::MAX) + 1), + mod_time_unix_nanos: 1, + body_variant: rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + let cache_key = engine_plan.key().cloned().expect("matrix body must be cacheable"); + let adapter = Arc::clone(&adapter); + let coordinator = Arc::clone(&coordinator); + let disk_permits = Arc::clone(&disk_permits); + let writers = Arc::clone(&writers); + let permit_acquires = Arc::clone(&permit_acquires); + let reader_factories = Arc::clone(&reader_factories); + let first_polls = Arc::clone(&first_polls); + let completed = Arc::clone(&completed); + let bytes_read = Arc::clone(&bytes_read); + tasks.spawn(async move { + let outcome = coordinate_cold_fill(&coordinator, cache_key, None, None, move |producer| { + let reservation = adapter.reserve_body(&engine_plan); + let adapter = Arc::clone(&adapter); + let disk_permits = Arc::clone(&disk_permits); + let writers = Arc::clone(&writers); + let permit_acquires = Arc::clone(&permit_acquires); + let reader_factories = Arc::clone(&reader_factories); + let first_polls = Arc::clone(&first_polls); + let completed = Arc::clone(&completed); + let bytes_read = Arc::clone(&bytes_read); + let fill_plan = engine_plan.clone(); + async move { + start_cold_fill_producer( + producer, + reservation, + || async move { + permit_acquires.fetch_add(1, AtomicOrdering::Relaxed); + let permit = disk_permits + .acquire_owned() + .await + .map_err(|_| ColdFillError::DiskAdmissionClosed)?; + let mut io = + DefaultObjectUsecase::get_object_io_planning_without_disk(get_concurrency_manager()); + io.disk_permit = Some(permit.into()); + Ok(io) + }, + || async move { + reader_factories.fetch_add(1, AtomicOrdering::Relaxed); + let (writer, reader) = tokio::io::duplex(BODY_BYTES * 2); + writers.lock().await.push(writer); + Ok(GetObjectReader { + stream: Box::new(ColdFillMatrixReader { + inner: reader, + first_poll_recorded: false, + completion_recorded: false, + first_polls, + completed, + bytes_read, + }), + object_info: ObjectInfo { + size: BODY_BYTES_I64, + actual_size: BODY_BYTES_I64, + ..Default::default() + }, + buffered_body: None, + body_source: GetObjectBodySource::HookMissed, + }) + }, + ColdFillProducerExecution { + expected: BODY_BYTES, + deadline: None, + adapter, + engine_plan: fill_plan, + }, + ) + .await + } + }) + .await; + let ColdFillCoordinateOutcome::Ready(Ok(body)) = outcome else { + panic!("matrix request must receive the shared body, got {outcome:?}"); + }; + assert_eq!(body.len(), BODY_BYTES); + assert!(body.iter().all(|byte| *byte == 7)); + (key_index, body.as_ptr() as usize) + }); + } + + tokio::time::timeout(Duration::from_secs(30), async { + loop { + if writers.lock().await.len() == key_count + && coordinator.global_waiter_count_for_test() == REQUESTS - key_count + && first_polls.load(AtomicOrdering::Relaxed) == key_count + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("all matrix followers must join before releasing bodies"); + + let mut body_writers = std::mem::take(&mut *writers.lock().await); + let body = vec![7_u8; BODY_BYTES]; + for writer in &mut body_writers { + tokio::io::AsyncWriteExt::write_all(writer, &body) + .await + .expect("matrix body write must succeed"); + tokio::io::AsyncWriteExt::shutdown(writer) + .await + .expect("matrix body writer must close"); + } + let mut backing_pointers = std::collections::HashMap::>::new(); + tokio::time::timeout(Duration::from_secs(30), async { + while let Some(result) = tasks.join_next().await { + let (key_index, body_pointer) = result.expect("matrix GET task must complete"); + backing_pointers.entry(key_index).or_default().insert(body_pointer); + } + }) + .await + .expect("matrix GET tasks must complete before the watchdog"); + + assert_eq!(permit_acquires.load(AtomicOrdering::Relaxed), key_count); + assert_eq!(reader_factories.load(AtomicOrdering::Relaxed), key_count); + assert_eq!(first_polls.load(AtomicOrdering::Relaxed), key_count); + assert_eq!(completed.load(AtomicOrdering::Relaxed), key_count); + assert_eq!(bytes_read.load(AtomicOrdering::Relaxed), key_count * BODY_BYTES); + assert_eq!(backing_pointers.len(), key_count); + assert!( + backing_pointers.values().all(|pointers| pointers.len() == 1), + "all followers of one key must share one backing allocation" + ); + assert_eq!( + backing_pointers + .values() + .flatten() + .copied() + .collect::>() + .len(), + key_count + ); + assert_eq!(coordinator.global_waiter_count_for_test(), 0); + assert_eq!(coordinator.active_session_count_for_test(), 0); + assert_eq!(disk_permits.available_permits(), key_count); + + for key_index in 0..key_count { + let object = format!("matrix-object-{key_index}"); + let plan = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "matrix-bucket", + object: &object, + version_id: None, + etag: "matrix-etag", + size: BODY_BYTES_U64, + data_dir_u128: Some(u128::try_from(key_index).unwrap_or(u128::MAX) + 1), + mod_time_unix_nanos: 1, + body_variant: rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + assert!(matches!( + adapter.lookup_body(&plan).await, + rustfs_object_data_cache::ObjectDataCacheLookup::Hit(_) + )); + } + } + } + // #1324: the in-memory (buffered/cache) source is guarded by // MemoryTrackedBytesStream. A buffer whose length disagrees with the declared // content length must yield a stream error on first poll instead of a clean @@ -8114,13 +10431,116 @@ mod tests { .await .expect("test semaphore should grant owned permit"); - let reader = DiskReadPermitReader::new(std::io::Cursor::new(Vec::::new()), permit); + let reader = DiskReadPermitReader::new(std::io::Cursor::new(Vec::::new()), permit.into()); assert_eq!(semaphore.available_permits(), 0); drop(reader); assert_eq!(semaphore.available_permits(), 1); } + #[tokio::test] + #[serial_test::serial(cold_fill_metrics_gate)] + async fn cold_fill_follower_disk_permit_metric_tracks_actual_permit_lifetime() { + COLD_FILL_FOLLOWER_DISK_PERMITS_FOR_TEST.store(0, Ordering::Relaxed); + let semaphore = Arc::new(tokio::sync::Semaphore::new(1)); + scope_cold_fill_disk_permit_owner_for_test(ColdFillDiskPermitOwner::Follower, async { + let permit = semaphore + .clone() + .acquire_owned() + .await + .expect("follower test semaphore must grant an owned permit"); + let tracked = GetObjectDiskPermit::new(permit); + assert_eq!(semaphore.available_permits(), 0); + assert_eq!(COLD_FILL_FOLLOWER_DISK_PERMITS_FOR_TEST.load(Ordering::Relaxed), 1); + + drop(tracked); + assert_eq!(semaphore.available_permits(), 1); + assert_eq!(COLD_FILL_FOLLOWER_DISK_PERMITS_FOR_TEST.load(Ordering::Relaxed), 0); + }) + .await; + } + + #[test] + #[serial_test::serial(cold_fill_metrics_gate)] + fn cold_fill_disk_permit_metrics_obey_gate_and_return_to_zero() { + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; + + let metrics_was_enabled = rustfs_io_metrics::metrics_enabled(); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("metric test runtime must build"); + metrics::with_local_recorder(&recorder, || { + runtime.block_on(async { + rustfs_io_metrics::set_metrics_enabled(false); + let semaphore = Arc::new(tokio::sync::Semaphore::new(1)); + scope_cold_fill_disk_permit_owner_for_test(ColdFillDiskPermitOwner::Follower, async { + let permit = semaphore + .clone() + .acquire_owned() + .await + .expect("metric test permit must be available"); + let tracked = GetObjectDiskPermit::new(permit); + rustfs_io_metrics::set_metrics_enabled(true); + drop(tracked); + }) + .await; + assert!( + snapshotter.snapshot().into_vec().into_iter().all(|(composite, _, _, _)| { + !composite.key().name().starts_with("rustfs_object_data_cache_cold_fill_") + }), + "a permit acquired while metrics were disabled must not record an unmatched decrement" + ); + + scope_cold_fill_disk_permit_owner_for_test(ColdFillDiskPermitOwner::Producer, async { + let permit = semaphore + .clone() + .acquire_owned() + .await + .expect("metric test permit must be available"); + let tracked = GetObjectDiskPermit::new(permit); + rustfs_io_metrics::set_metrics_enabled(false); + drop(tracked); + }) + .await; + rustfs_io_metrics::set_metrics_enabled(true); + scope_cold_fill_disk_permit_owner_for_test(ColdFillDiskPermitOwner::Follower, async { + let permit = semaphore.acquire_owned().await.expect("metric test permit must be available"); + let tracked = GetObjectDiskPermit::new(permit); + let _replacement = crate::app::object_data_cache::ColdFillCoordinator::default(); + drop(tracked); + }) + .await; + }); + }); + + let values = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter_map(|(composite, _unit, _description, value)| { + composite + .key() + .name() + .starts_with("rustfs_object_data_cache_cold_fill_") + .then_some((composite.key().name().to_string(), value)) + }) + .collect::>(); + assert_eq!(values.len(), 2); + for name in [ + "rustfs_object_data_cache_cold_fill_producer_disk_permits", + "rustfs_object_data_cache_cold_fill_follower_disk_permits", + ] { + let DebugValue::Gauge(value) = values.get(name).unwrap_or_else(|| panic!("missing {name} gauge")) else { + panic!("{name} must be a gauge"); + }; + assert_eq!(value.into_inner(), 0.0, "{name} must return to zero after permit drop"); + } + rustfs_io_metrics::set_metrics_enabled(metrics_was_enabled); + } + #[tokio::test] async fn build_get_object_body_keeps_large_objects_on_streaming_path_without_preread() { let reads = Arc::new(AtomicUsize::new(0)); @@ -8277,6 +10697,7 @@ mod tests { None, false, false, + true, "test-bucket", "cached-object", GetObjectBodyLifecycle::disabled(), @@ -8337,6 +10758,7 @@ mod tests { None, false, false, + true, "test-bucket", "cached-object", GetObjectBodyLifecycle::disabled(), @@ -8396,6 +10818,7 @@ mod tests { Some(Bytes::from_static(b"hello")), false, false, + true, "test-bucket", "cached-object", GetObjectBodyLifecycle::disabled(), @@ -8421,6 +10844,7 @@ mod tests { None, false, false, + true, "test-bucket", "cached-object", GetObjectBodyLifecycle::disabled(), @@ -8485,6 +10909,7 @@ mod tests { Some(Bytes::from_static(b"oops")), false, false, + true, "test-bucket", "cached-object", GetObjectBodyLifecycle::disabled(), @@ -8566,6 +10991,7 @@ mod tests { Some(hit_body), /* cache_hook_served */ true, /* cache_hook_probed */ true, + /* cache_fill_allowed */ true, "test-bucket", "hook-served", GetObjectBodyLifecycle::disabled(), @@ -8623,6 +11049,7 @@ mod tests { Some(Bytes::from_static(b"hello")), /* cache_hook_served */ false, /* cache_hook_probed */ true, + /* cache_fill_allowed */ true, "test-bucket", "hook-missed", GetObjectBodyLifecycle::disabled(), @@ -8682,6 +11109,7 @@ mod tests { None, false, false, + true, "test-bucket", "materialized-object", GetObjectBodyLifecycle::disabled(), @@ -8707,6 +11135,7 @@ mod tests { None, false, false, + true, "test-bucket", "materialized-object", GetObjectBodyLifecycle::disabled(), @@ -8766,6 +11195,7 @@ mod tests { None, false, false, + true, "test-bucket", "mismatch-object", GetObjectBodyLifecycle::disabled(), @@ -8818,6 +11248,7 @@ mod tests { None, false, false, + true, "test-bucket", "short-object", GetObjectBodyLifecycle::disabled(), @@ -8868,6 +11299,7 @@ mod tests { None, false, false, + true, "test-bucket", "partial-read-object", GetObjectBodyLifecycle::disabled(), @@ -8998,6 +11430,7 @@ mod tests { None, false, false, + true, "test-bucket", "too-large-object", GetObjectBodyLifecycle::disabled(), @@ -9348,7 +11781,7 @@ mod tests { let permit = semaphore.clone().acquire_owned().await.expect("acquire permit"); assert_eq!(semaphore.available_permits(), 0); - let mut reader = DiskReadPermitReader::new(std::io::Cursor::new(b"hello".to_vec()), permit); + let mut reader = DiskReadPermitReader::new(std::io::Cursor::new(b"hello".to_vec()), permit.into()); let mut body = Vec::new(); reader.read_to_end(&mut body).await.expect("read body"); assert_eq!(body, b"hello"); @@ -9747,6 +12180,7 @@ mod tests { None, false, false, + true, None, None, None, @@ -9819,6 +12253,53 @@ mod tests { assert!(DefaultObjectUsecase::validate_get_object_part_number(Some(1), &info).is_ok()); } + #[test] + fn cold_fill_conditions_fail_before_phase_probe_advances() { + fn run_phase_probe(headers: &HeaderMap, info: &ObjectInfo) -> (S3Result<()>, [usize; 3]) { + let coordination = AtomicUsize::new(0); + let permit = AtomicUsize::new(0); + let reader = AtomicUsize::new(0); + let result = DefaultObjectUsecase::validate_get_object_before_cold_fill(headers, None, info); + if result.is_ok() { + coordination.fetch_add(1, AtomicOrdering::Relaxed); + permit.fetch_add(1, AtomicOrdering::Relaxed); + reader.fetch_add(1, AtomicOrdering::Relaxed); + } + ( + result, + [ + coordination.load(AtomicOrdering::Relaxed), + permit.load(AtomicOrdering::Relaxed), + reader.load(AtomicOrdering::Relaxed), + ], + ) + } + + let info = ObjectInfo { + etag: Some("phase-etag".to_string()), + parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + number: 1, + ..Default::default() + }]), + ..Default::default() + }; + + let mut not_modified = HeaderMap::new(); + not_modified.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("\"phase-etag\"")); + let (result, phases) = run_phase_probe(¬_modified, &info); + assert_eq!(result.expect_err("matching If-None-Match must reject").code(), &S3ErrorCode::NotModified); + assert_eq!(phases, [0, 0, 0]); + + let mut precondition_failed = HeaderMap::new(); + precondition_failed.insert(http::header::IF_MATCH, HeaderValue::from_static("\"other-etag\"")); + let (result, phases) = run_phase_probe(&precondition_failed, &info); + assert_eq!( + result.expect_err("mismatched If-Match must reject").code(), + &S3ErrorCode::PreconditionFailed + ); + assert_eq!(phases, [0, 0, 0]); + } + #[tokio::test] async fn execute_get_object_rejects_range_with_part_number() { let input = GetObjectInput::builder() diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index ffa00abbf..8f3bb651b 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -953,6 +953,16 @@ pub(crate) mod bucket_usecase { } pub(crate) mod object_usecase { + pub(crate) mod object_cache { + #[cfg(test)] + pub(crate) use crate::storage::storage_api::ecstore_object::GetObjectBodySource; + #[cfg(test)] + pub(crate) use crate::storage::storage_api::ecstore_object::lookup_get_object_body_cache_hook; + pub(crate) use crate::storage::storage_api::ecstore_object::{ + GetObjectBodyCacheHookLookup, get_object_body_cache_plaintext_len, + }; + } + pub(crate) mod contract { #[cfg(test)] pub(crate) mod http { @@ -977,12 +987,12 @@ pub(crate) mod object_usecase { object_utils, options, request_context, s3_api, set_disk, sse, storage_class, timeout_wrapper, }; pub(crate) use crate::storage::storage_api::{ - ECStore, OldCurrentSize, RFC1123, StorageDeletedObject, StorageObjectInfo, StorageObjectLockDeleteOptions, - StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, check_preconditions, get_validated_store, - has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, - remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_object_lock_enabled, - validate_object_key, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, - wrap_response_with_cors, + ECStore, GetObjectReader, OldCurrentSize, RFC1123, StorageDeletedObject, StorageObjectInfo, + StorageObjectLockDeleteOptions, StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, check_preconditions, + get_validated_store, has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention, + parse_part_number_i32_to_usize, remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, + validate_bucket_object_lock_enabled, validate_object_key, validate_sse_headers_for_read, validate_sse_headers_for_write, + validate_ssec_for_read, wrap_response_with_cors, }; } diff --git a/rustfs/src/storage/concurrency/manager.rs b/rustfs/src/storage/concurrency/manager.rs index 3794e76d6..13b509f93 100644 --- a/rustfs/src/storage/concurrency/manager.rs +++ b/rustfs/src/storage/concurrency/manager.rs @@ -193,6 +193,12 @@ impl ConcurrencyManager { manager } + #[cfg(test)] + pub(crate) fn close_disk_read_admission_for_test(&self) { + self.disk_read_semaphore.close(); + self.degraded_read_semaphore.close(); + } + /// Track a GetObject request pub fn track_request() -> GetObjectGuard { GetObjectGuard::new() diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index f7b636eda..d4a04959f 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -469,8 +469,12 @@ pub(crate) mod ecstore_rpc { } pub(crate) mod ecstore_object { + #[cfg(test)] + pub(crate) use rustfs_ecstore::api::object::GetObjectBodySource; pub(crate) use rustfs_ecstore::api::object::{ - GetObjectBodyCacheHook, ObjectMutationHook, register_get_object_body_cache_hook, register_object_mutation_hook, + GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, ObjectMutationHook, get_object_body_cache_plaintext_len, + lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook, + unregister_get_object_body_cache_hook, unregister_object_mutation_hook, }; } diff --git a/scripts/check_object_data_cache_follower_samples.py b/scripts/check_object_data_cache_follower_samples.py new file mode 100644 index 000000000..7f80e39f5 --- /dev/null +++ b/scripts/check_object_data_cache_follower_samples.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +import argparse +import math +import pathlib +import tempfile + + +def check_samples(samples_path: pathlib.Path, ready_path: pathlib.Path, scrape_seconds: float) -> tuple[int, float]: + samples = [ + tuple(map(float, line.split())) + for line in samples_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + ready_epoch = float(ready_path.read_text(encoding="utf-8").strip()) + fresh: dict[float, float] = {} + for timestamp, value in samples: + if not math.isfinite(timestamp) or not math.isfinite(value): + raise ValueError("sample timestamp and value must be finite") + if timestamp >= ready_epoch: + if timestamp in fresh and fresh[timestamp] != value: + raise ValueError(f"conflicting values for Prometheus timestamp {timestamp:.3f}") + fresh[timestamp] = value + timestamps = sorted(fresh) + values = [fresh[timestamp] for timestamp in timestamps] + if len(timestamps) < 3: + raise ValueError(f"only {len(timestamps)} distinct fresh follower-permit scrapes; need at least 3") + span = timestamps[-1] - timestamps[0] + if span < 2 * scrape_seconds: + raise ValueError( + f"observation span {span:.3f}s is shorter than two configured scrape intervals ({2 * scrape_seconds:.3f}s)" + ) + if any(value != 0 for value in values): + raise ValueError(f"follower permit gauge was non-zero; max={max(values)}") + return len(timestamps), 0.0 + + +def expect_failure(samples: str, ready: str, scrape_seconds: float) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + samples_path = root / "samples" + ready_path = root / "ready" + samples_path.write_text(samples, encoding="utf-8") + ready_path.write_text(ready, encoding="utf-8") + try: + check_samples(samples_path, ready_path, scrape_seconds) + except ValueError: + return + raise AssertionError("invalid follower sample matrix unexpectedly passed") + + +def self_test() -> None: + expect_failure("1000 0\n1000 0\n1000 0\n", "999", 5) + expect_failure("1000 0\n1005 1\n1010 0\n", "999", 5) + expect_failure("1000 0\n1005 0\n1010 0\n", "1010", 5) + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + samples_path = root / "samples" + ready_path = root / "ready" + samples_path.write_text("1000 0\n1005 0\n1010 0\n", encoding="utf-8") + ready_path.write_text("999", encoding="utf-8") + assert check_samples(samples_path, ready_path, 5) == (3, 0.0) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--samples", type=pathlib.Path) + parser.add_argument("--ready", type=pathlib.Path) + parser.add_argument("--scrape-seconds", type=float) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + if args.self_test: + self_test() + return + if args.samples is None or args.ready is None or args.scrape_seconds is None: + parser.error("--samples, --ready, and --scrape-seconds are required") + count, maximum = check_samples(args.samples, args.ready, args.scrape_seconds) + print(f"{count},{maximum:g}") + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_object_data_cache_cold_stampede.sh b/scripts/validate_object_data_cache_cold_stampede.sh new file mode 100755 index 000000000..9cd51acb3 --- /dev/null +++ b/scripts/validate_object_data_cache_cold_stampede.sh @@ -0,0 +1,687 @@ +#!/usr/bin/env bash + +# Manual/nightly validation for cold object-data-cache stampedes. +# +# This gate deliberately requires a dedicated RustFS instance, an exact 28 GiB +# cgroup v2 limit, pre-provisioned objects, and authoritative Prometheus queries. +# It never substitutes process I/O counters or synthetic values for server +# metrics. With no --run flag (or with missing prerequisites outside --strict), +# it reports SKIP and exits successfully. + +set -euo pipefail + +readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +readonly REQUESTS=2000 +readonly KEY_MATRIX="1 4 32" +readonly REQUIRED_MEMORY_BYTES=$((28 * 1024 * 1024 * 1024)) +readonly OBJECT_SIZE=$((384 * 1024 * 1024)) +readonly READER_BYTES_QUERY='sum(rustfs_io_get_object_reader_bytes_total) or vector(0)' +readonly FOLLOWER_PERMIT_QUERY='sum(rustfs_object_data_cache_cold_fill_follower_disk_permits)' + +RUN=0 +SELF_TEST=0 +STRICT=0 +ACK_ISOLATED=0 +ROUNDS=3 +ENDPOINT="" +BUCKET="" +KEY_PREFIX="" +EXPECTED_SHA256="" +REGION="us-east-1" +PROMETHEUS_QUERY_URL="" +PROMETHEUS_SCRAPE_SECONDS="" +RESET_COMMAND="" +CACHE_OFF_COMMAND="" +CACHE_ON_COMMAND="" +SERVER_PID="" +CGROUP_PATH="" +SAMPLE_INTERVAL="0.25" +METRICS_SETTLE_SECONDS="5" +LOAD_TIMEOUT_SECONDS="1800" +OUT_DIR="" + +usage() { + cat <<'USAGE' +Usage: + validate_object_data_cache_cold_stampede.sh --run --ack-isolated \ + --endpoint URL --bucket NAME --key-prefix PREFIX \ + --expected-sha256 HEX --prometheus-query-url URL \ + --prometheus-scrape-seconds SECONDS \ + --reset-command COMMAND --cache-off-command COMMAND \ + --cache-on-command COMMAND --server-pid PID [--cgroup-path PATH] [options] + +Required run contract: + * N is fixed at 2000; K is fixed at 1, 4, and 32; each K runs >= 3 rounds. + * PREFIX/object-00 through PREFIX/object-31 must already exist, each exactly + 384 MiB with identical SHA-256 content. + * COMMAND must synchronously make the selected objects cold before returning. + * The RustFS process must be isolated from unrelated traffic and run in a + dedicated cgroup v2 with memory.max exactly 28 GiB. + * The built-in first-party reader-byte counter and follower-permit gauge must + each return exactly one Prometheus vector sample. + * Cache switch commands must return only after the mode is effective and must + keep the same RustFS PID inside the dedicated cgroup. + * Every response must match SHA-256, ETag, Content-Length, and its per-key + stable header contract. Date, x-amz-id-2, x-amz-request-id, + x-minio-request-id, x-rustfs-request-id, Connection, Keep-Alive, and + Transfer-Encoding are explicitly excluded as volatile/transport headers. + +Built-in PromQL: + reader bytes: sum(rustfs_io_get_object_reader_bytes_total) or vector(0) + follower permits: sum(rustfs_object_data_cache_cold_fill_follower_disk_permits) + +Options: + --self-test Validate the follower sample append/check chain locally + --rounds N Rounds per K (default: 3; minimum: 3) + --region REGION SigV4 region (default: us-east-1) + --sample-interval SECONDS Follower gauge polling interval (default: 0.25) + --prometheus-scrape-seconds N Configured scrape interval; required + --metrics-settle-seconds N Wait for final Prometheus scrape (default: 5) + --load-timeout-seconds N Per-round load timeout (default: 1800) + --out-dir PATH Artifact directory (default: temporary) + --strict Missing prerequisite is FAIL instead of SKIP + -h, --help Show this help + +Credentials are read only from AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and +optional AWS_SESSION_TOKEN. They are never written to the result artifacts. +USAGE +} + +skip_or_fail() { + local message=$1 + if ((STRICT)); then + printf 'FAIL: %s\n' "$message" >&2 + exit 1 + fi + printf 'SKIP: %s\n' "$message" + exit 0 +} + +fail() { + printf 'FAIL: %s\n' "$1" >&2 + exit 1 +} + +need_value() { + (($# >= 2)) || fail "option $1 requires a value" +} + +append_follower_sample() { + local samples_file=$1 + local sample=$2 + local sample_epoch sample_value trailing + read -r sample_epoch sample_value trailing <<<"$sample" + [[ -n $sample_epoch && -n $sample_value && -z $trailing ]] || return 1 + printf '%s %s\n' "$sample_epoch" "$sample_value" >>"$samples_file" +} + +follower_samples_check() { + local samples_file=$1 + local ready_file=$2 + local scrape_seconds=${3:-$PROMETHEUS_SCRAPE_SECONDS} + python3 "$SCRIPT_DIR/check_object_data_cache_follower_samples.py" \ + --samples "$samples_file" --ready "$ready_file" \ + --scrape-seconds "$scrape_seconds" +} + +run_operator_command() { + local mode=$1 + local round=$2 + local key_count=$3 + local command=$4 + env -u AWS_ACCESS_KEY_ID -u AWS_SECRET_ACCESS_KEY -u AWS_SESSION_TOKEN \ + RUSTFS_STAMPEDE_MODE="$mode" RUSTFS_STAMPEDE_ROUND="$round" RUSTFS_STAMPEDE_K="$key_count" \ + bash -c "$command" +} + +follower_samples_self_test() ( + command -v python3 >/dev/null 2>&1 || fail "python3 is required for --self-test" + local temp_dir samples_file ready_file result sample + temp_dir=$(mktemp -d "${TMPDIR:-/tmp}/rustfs-follower-samples.XXXXXX") + trap 'rm -rf -- "$temp_dir"' EXIT + samples_file="$temp_dir/follower.samples" + ready_file="$temp_dir/ready" + printf '999\n' >"$ready_file" + + : >"$samples_file" + for sample in '1000 0' '1000 0' '1000 0'; do + append_follower_sample "$samples_file" "$sample" || fail "self-test could not append duplicate follower sample" + done + [[ $(<"$samples_file") == $'1000 0\n1000 0\n1000 0' ]] || fail "duplicate Prometheus timestamps were not preserved by the append chain" + if follower_samples_check "$samples_file" "$ready_file" 5 >/dev/null 2>&1; then + fail "duplicate Prometheus timestamps unexpectedly passed the follower sample checker" + fi + + : >"$samples_file" + for sample in '1000 0' '1000 1' '1005 0' '1010 0'; do + append_follower_sample "$samples_file" "$sample" || fail "self-test could not append conflicting follower sample" + done + if follower_samples_check "$samples_file" "$ready_file" 5 >/dev/null 2>&1; then + fail "conflicting values for one Prometheus timestamp unexpectedly passed the follower sample checker" + fi + + : >"$samples_file" + for sample in '1000 0' '1005 0' '1010 0'; do + append_follower_sample "$samples_file" "$sample" || fail "self-test could not append fresh follower sample" + done + [[ $(<"$samples_file") == $'1000 0\n1005 0\n1010 0' ]] || fail "fresh Prometheus timestamps were not preserved by the append chain" + result=$(follower_samples_check "$samples_file" "$ready_file" 5) || fail "fresh Prometheus timestamps unexpectedly failed the follower sample checker" + [[ $result == '3,0' ]] || fail "fresh Prometheus timestamps returned unexpected result: $result" + + result=$( + AWS_ACCESS_KEY_ID=self-test-access \ + AWS_SECRET_ACCESS_KEY=self-test-secret \ + AWS_SESSION_TOKEN=self-test-session \ + run_operator_command self-test 7 11 \ + 'printf "%s|%s|%s|%s|%s|%s" "$RUSTFS_STAMPEDE_MODE" "$RUSTFS_STAMPEDE_ROUND" "$RUSTFS_STAMPEDE_K" "${AWS_ACCESS_KEY_ID-unset}" "${AWS_SECRET_ACCESS_KEY-unset}" "${AWS_SESSION_TOKEN-unset}"' + ) || fail "operator command self-test failed" + [[ $result == 'self-test|7|11|unset|unset|unset' ]] || fail "operator command inherited AWS credentials: $result" + printf 'PASS: cold stampede script self-test\n' +) + +while (($#)); do + case "$1" in + --run) RUN=1 ;; + --self-test) SELF_TEST=1 ;; + --strict) STRICT=1 ;; + --ack-isolated) ACK_ISOLATED=1 ;; + --endpoint) need_value "$@"; ENDPOINT=$2; shift ;; + --bucket) need_value "$@"; BUCKET=$2; shift ;; + --key-prefix) need_value "$@"; KEY_PREFIX=$2; shift ;; + --expected-sha256) need_value "$@"; EXPECTED_SHA256=$2; shift ;; + --region) need_value "$@"; REGION=$2; shift ;; + --prometheus-query-url) need_value "$@"; PROMETHEUS_QUERY_URL=$2; shift ;; + --prometheus-scrape-seconds) need_value "$@"; PROMETHEUS_SCRAPE_SECONDS=$2; shift ;; + --reset-command) need_value "$@"; RESET_COMMAND=$2; shift ;; + --cache-off-command) need_value "$@"; CACHE_OFF_COMMAND=$2; shift ;; + --cache-on-command) need_value "$@"; CACHE_ON_COMMAND=$2; shift ;; + --server-pid) need_value "$@"; SERVER_PID=$2; shift ;; + --cgroup-path) need_value "$@"; CGROUP_PATH=$2; shift ;; + --rounds) need_value "$@"; ROUNDS=$2; shift ;; + --sample-interval) need_value "$@"; SAMPLE_INTERVAL=$2; shift ;; + --metrics-settle-seconds) need_value "$@"; METRICS_SETTLE_SECONDS=$2; shift ;; + --load-timeout-seconds) need_value "$@"; LOAD_TIMEOUT_SECONDS=$2; shift ;; + --out-dir) need_value "$@"; OUT_DIR=$2; shift ;; + -h|--help) usage; exit 0 ;; + *) fail "unknown option: $1" ;; + esac + shift +done + +if ((SELF_TEST)); then + follower_samples_self_test + exit 0 +fi + +((RUN)) || skip_or_fail "not started; pass --run after reading --help" +((ACK_ISOLATED)) || skip_or_fail "--ack-isolated is required because global counters must have no unrelated traffic" + +[[ $(uname -s) == Linux ]] || skip_or_fail "Linux with cgroup v2 is required" +for command in bash curl python3; do + command -v "$command" >/dev/null 2>&1 || skip_or_fail "required command is unavailable: $command" +done + +[[ -n $ENDPOINT ]] || skip_or_fail "--endpoint is required" +[[ -n $BUCKET ]] || skip_or_fail "--bucket is required" +[[ -n $KEY_PREFIX ]] || skip_or_fail "--key-prefix is required" +[[ $EXPECTED_SHA256 =~ ^[[:xdigit:]]{64}$ ]] || skip_or_fail "--expected-sha256 must be exactly 64 hexadecimal characters" +[[ -n $PROMETHEUS_QUERY_URL ]] || skip_or_fail "--prometheus-query-url is required" +[[ -n $PROMETHEUS_SCRAPE_SECONDS ]] || skip_or_fail "--prometheus-scrape-seconds is required" +[[ -n $RESET_COMMAND ]] || skip_or_fail "--reset-command is required to prove each round starts cold" +[[ -n $CACHE_OFF_COMMAND ]] || skip_or_fail "--cache-off-command is required" +[[ -n $CACHE_ON_COMMAND ]] || skip_or_fail "--cache-on-command is required" +[[ -n ${AWS_ACCESS_KEY_ID:-} ]] || skip_or_fail "AWS_ACCESS_KEY_ID is required" +[[ -n ${AWS_SECRET_ACCESS_KEY:-} ]] || skip_or_fail "AWS_SECRET_ACCESS_KEY is required" +if ! [[ $ROUNDS =~ ^[0-9]+$ ]] || ((ROUNDS < 3)); then + fail "--rounds must be an integer >= 3" +fi +[[ $METRICS_SETTLE_SECONDS =~ ^[0-9]+$ ]] || fail "--metrics-settle-seconds must be a non-negative integer" +if ! [[ $LOAD_TIMEOUT_SECONDS =~ ^[0-9]+$ ]] || ((LOAD_TIMEOUT_SECONDS == 0)); then + fail "--load-timeout-seconds must be positive" +fi +python3 - "$SAMPLE_INTERVAL" "$PROMETHEUS_SCRAPE_SECONDS" <<'PY' || fail "sample and scrape intervals must be finite positive numbers" +import math, sys +values = [float(value) for value in sys.argv[1:]] +raise SystemExit(0 if all(math.isfinite(value) and value > 0 for value in values) else 1) +PY +python3 - "$METRICS_SETTLE_SECONDS" "$PROMETHEUS_SCRAPE_SECONDS" <<'PY' \ + || fail "--metrics-settle-seconds must be at least one configured Prometheus scrape interval" +import sys +raise SystemExit(0 if float(sys.argv[1]) >= float(sys.argv[2]) else 1) +PY + +[[ $SERVER_PID =~ ^[0-9]+$ ]] || skip_or_fail "--server-pid is required to bind OOM evidence to RustFS" +[[ -r /proc/$SERVER_PID/cgroup ]] || skip_or_fail "cannot read /proc/$SERVER_PID/cgroup" +if [[ -z $CGROUP_PATH ]]; then + cgroup_relative=$(awk -F: '$1 == "0" { print $3; exit }' "/proc/$SERVER_PID/cgroup") + [[ -n $cgroup_relative ]] || skip_or_fail "server process is not in a cgroup v2 hierarchy" + CGROUP_PATH="/sys/fs/cgroup${cgroup_relative}" +fi + +[[ -r $CGROUP_PATH/memory.max ]] || skip_or_fail "cannot read $CGROUP_PATH/memory.max" +[[ -r $CGROUP_PATH/memory.events ]] || skip_or_fail "cannot read $CGROUP_PATH/memory.events" +[[ -r $CGROUP_PATH/cgroup.procs ]] || skip_or_fail "cannot read $CGROUP_PATH/cgroup.procs" +memory_max=$(<"$CGROUP_PATH/memory.max") +[[ $memory_max =~ ^[0-9]+$ ]] || skip_or_fail "memory.max must be finite, not '$memory_max'" +((memory_max == REQUIRED_MEMORY_BYTES)) || skip_or_fail "memory.max must equal 28 GiB ($REQUIRED_MEMORY_BYTES), got $memory_max" + +server_in_cgroup() { + awk -v pid="$SERVER_PID" '$1 == pid { found = 1; exit } END { exit !found }' \ + "$CGROUP_PATH/cgroup.procs" +} +server_in_cgroup || skip_or_fail "RustFS PID $SERVER_PID is not in $CGROUP_PATH" + +nofile_limit=$(ulimit -n) +[[ $nofile_limit =~ ^[0-9]+$ ]] || skip_or_fail "unable to determine the open-file limit" +((nofile_limit >= REQUESTS + 256)) || skip_or_fail "open-file limit must be at least $((REQUESTS + 256)), got $nofile_limit" + +if [[ -z $OUT_DIR ]]; then + OUT_DIR=$(mktemp -d "${TMPDIR:-/tmp}/rustfs-cold-stampede.XXXXXX") +else + mkdir -p "$OUT_DIR" +fi +readonly OUT_DIR +readonly RESULTS_CSV="$OUT_DIR/results.csv" +LOAD_PID="" + +cleanup_load() { + if [[ -n $LOAD_PID ]] && kill -0 "$LOAD_PID" 2>/dev/null; then + kill "$LOAD_PID" 2>/dev/null || true + wait "$LOAD_PID" 2>/dev/null || true + fi +} +trap cleanup_load EXIT + +prometheus_sample() { + local query=$1 + local response + response=$(curl --fail --silent --show-error --get \ + --data-urlencode "query=$query" "$PROMETHEUS_QUERY_URL") || return 1 + python3 -c ' +import json, math, sys +doc = json.load(sys.stdin) +if doc.get("status") != "success": + raise SystemExit("Prometheus query was not successful") +result = doc.get("data", {}).get("result", []) +if len(result) != 1 or "value" not in result[0]: + raise SystemExit(f"expected exactly one vector sample, got {len(result)}") +timestamp = float(result[0]["value"][0]) +value = float(result[0]["value"][1]) +if not math.isfinite(timestamp) or not math.isfinite(value) or value < 0: + raise SystemExit(f"invalid metric value: {value}") +print(format(timestamp, ".17g"), format(value, ".17g")) +' <<<"$response" +} + +prometheus_value() { + local timestamp value + read -r timestamp value < <(prometheus_sample "$1") || return 1 + [[ -n $timestamp && -n $value ]] || return 1 + printf '%s\n' "$value" +} + +cgroup_event() { + local name=$1 + awk -v name="$name" '$1 == name { print $2; found = 1; exit } END { if (!found) exit 1 }' \ + "$CGROUP_PATH/memory.events" +} + +numeric_delta_check() { + local before=$1 + local after=$2 + local expected_min=$3 + local expected_max=$4 + python3 - "$before" "$after" "$expected_min" "$expected_max" <<'PY' +import math, sys +before, after, lower, upper = map(float, sys.argv[1:]) +delta = after - before +if not math.isfinite(delta) or delta < lower or delta > upper: + raise SystemExit(f"reader byte delta {delta:.0f} outside [{lower:.0f}, {upper:.0f}]") +print(f"{delta:.0f}") +PY +} + +numeric_positive_delta() { + local before=$1 + local after=$2 + python3 - "$before" "$after" <<'PY' +import math, sys +before, after = map(float, sys.argv[1:]) +delta = after - before +if not math.isfinite(delta) or delta <= 0: + raise SystemExit(f"reader byte baseline delta must be positive, got {delta:.0f}") +print(f"{delta:.0f}") +PY +} + +run_load() { + local key_count=$1 + local request_count=$2 + local summary_file=$3 + local ready_file=$4 + local reference_file=${5:-} + python3 - "$ENDPOINT" "$BUCKET" "$KEY_PREFIX" "$EXPECTED_SHA256" "$REGION" \ + "$OBJECT_SIZE" "$request_count" "$key_count" "$LOAD_TIMEOUT_SECONDS" "$ready_file" "$reference_file" \ + >"$summary_file" <<'PY' +import asyncio +import datetime +import hashlib +import hmac +import json +import os +import ssl +import sys +import time +import urllib.parse + +( + endpoint, bucket, key_prefix, expected_sha, region, object_size, + request_count, key_count, timeout_seconds, ready_file, reference_file, +) = sys.argv[1:] +object_size = int(object_size) +request_count = int(request_count) +key_count = int(key_count) +timeout_seconds = int(timeout_seconds) +expected_sha = expected_sha.lower() +access_key = os.environ["AWS_ACCESS_KEY_ID"] +secret_key = os.environ["AWS_SECRET_ACCESS_KEY"] +session_token = os.environ.get("AWS_SESSION_TOKEN") + +parsed = urllib.parse.urlsplit(endpoint) +if parsed.scheme not in ("http", "https") or not parsed.hostname: + raise SystemExit("endpoint must be an absolute http(s) URL") +port = parsed.port or (443 if parsed.scheme == "https" else 80) +host_header = parsed.netloc +base_path = parsed.path.rstrip("/") +payload_hash = hashlib.sha256(b"").hexdigest() +tls_context = ssl.create_default_context() if parsed.scheme == "https" else None +volatile_headers = { + "connection", + "date", + "keep-alive", + "transfer-encoding", + "x-amz-id-2", + "x-amz-request-id", + "x-minio-request-id", + "x-rustfs-request-id", +} + +def sign_headers(method, canonical_uri): + now = datetime.datetime.now(datetime.timezone.utc) + amz_date = now.strftime("%Y%m%dT%H%M%SZ") + date_stamp = now.strftime("%Y%m%d") + headers = { + "host": host_header, + "x-amz-content-sha256": payload_hash, + "x-amz-date": amz_date, + } + if session_token: + headers["x-amz-security-token"] = session_token + signed_headers = ";".join(sorted(headers)) + canonical_headers = "".join(f"{name}:{headers[name]}\n" for name in sorted(headers)) + canonical_request = "\n".join(( + method, canonical_uri, "", canonical_headers, signed_headers, payload_hash, + )) + scope = f"{date_stamp}/{region}/s3/aws4_request" + string_to_sign = "\n".join(( + "AWS4-HMAC-SHA256", amz_date, scope, + hashlib.sha256(canonical_request.encode()).hexdigest(), + )) + def digest(key, message): + return hmac.new(key, message.encode(), hashlib.sha256).digest() + signing_key = digest(digest(digest(digest( + ("AWS4" + secret_key).encode(), date_stamp), region), "s3"), "aws4_request") + signature = hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest() + headers["authorization"] = ( + f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, " + f"SignedHeaders={signed_headers}, Signature={signature}" + ) + return headers + +async def read_one(index, release, connected, connected_lock, all_connected): + key = f"{key_prefix.rstrip('/')}/object-{index % key_count:02d}" + raw_path = f"{base_path}/{bucket}/{key}" + canonical_uri = urllib.parse.quote(raw_path, safe="/-_.~") + reader, writer = await asyncio.wait_for( + asyncio.open_connection(parsed.hostname, port, ssl=tls_context), timeout=60, + ) + async with connected_lock: + connected[0] += 1 + if connected[0] == request_count: + all_connected.set() + await release.wait() + headers = sign_headers("GET", canonical_uri) + request = [f"GET {canonical_uri} HTTP/1.1", "Connection: close"] + request.extend(f"{name}: {value}" for name, value in headers.items()) + writer.write(("\r\n".join(request) + "\r\n\r\n").encode()) + await writer.drain() + status = (await reader.readline()).decode("latin1").rstrip("\r\n") + parts = status.split(" ", 2) + if len(parts) < 2 or parts[1] != "200": + raise RuntimeError(f"request {index}: unexpected status {status!r}") + response_headers = {} + while True: + line = await reader.readline() + if line in (b"\r\n", b"\n", b""): + break + name, value = line.decode("latin1").split(":", 1) + response_headers[name.strip().lower()] = value.strip() + if "transfer-encoding" in response_headers: + raise RuntimeError(f"request {index}: chunked responses are not accepted") + length = int(response_headers.get("content-length", "-1")) + if length != object_size: + raise RuntimeError(f"request {index}: content-length {length}, expected {object_size}") + digest = hashlib.sha256() + remaining = length + while remaining: + chunk = await reader.read(min(64 * 1024, remaining)) + if not chunk: + raise RuntimeError(f"request {index}: body ended with {remaining} bytes missing") + digest.update(chunk) + remaining -= len(chunk) + writer.close() + await writer.wait_closed() + actual = digest.hexdigest() + if actual != expected_sha: + raise RuntimeError(f"request {index}: SHA-256 {actual}, expected {expected_sha}") + etag = response_headers.get("etag") + if not etag: + raise RuntimeError(f"request {index}: missing ETag") + semantic_headers = { + name: value + for name, value in response_headers.items() + if name not in volatile_headers and name not in {"content-length", "etag"} + } + return { + "bytes": length, + "etag": etag, + "key_index": index % key_count, + "semantic_headers": semantic_headers, + "sha256": actual, + } + +async def main(): + release = asyncio.Event() + all_connected = asyncio.Event() + connected = [0] + connected_lock = asyncio.Lock() + tasks = [asyncio.create_task(read_one( + index, release, connected, connected_lock, all_connected, + )) for index in range(request_count)] + started = time.monotonic() + await asyncio.wait_for(all_connected.wait(), timeout=300) + with open(ready_file, "x", encoding="utf-8") as handle: + handle.write(str(time.time())) + release.set() + responses = await asyncio.wait_for(asyncio.gather(*tasks), timeout=timeout_seconds) + reference_contracts = {} + if reference_file: + with open(reference_file, encoding="utf-8") as handle: + reference_contracts = json.load(handle).get("contracts", {}) + contracts = {} + for index, response in enumerate(responses): + key = str(response["key_index"]) + contract = { + "etag": response["etag"], + "semantic_headers": response["semantic_headers"], + } + expected = reference_contracts.get(key) or contracts.setdefault(key, contract) + if contract != expected: + raise RuntimeError( + f"request {index}: stable response contract differs: actual={contract!r}, expected={expected!r}" + ) + print(json.dumps({ + "requests": len(responses), + "keys": key_count, + "bytes": sum(response["bytes"] for response in responses), + "contracts": contracts or reference_contracts, + "sha256_failures": 0, + "duration_seconds": time.monotonic() - started, + }, sort_keys=True)) + +asyncio.run(main()) +PY +} + +summary_fields() { + local summary_file=$1 + python3 - "$summary_file" <<'PY' +import json, sys +with open(sys.argv[1], encoding="utf-8") as handle: + doc = json.load(handle) +if doc.get("sha256_failures") != 0: + raise SystemExit("SHA-256 failures were reported") +if not doc.get("contracts"): + raise SystemExit("no ETag/stable-header response contract was recorded") +print(doc["requests"], doc["duration_seconds"]) +PY +} + +switch_cache_mode() { + local mode=$1 + local command=$2 + local round=$3 + local key_count=$4 + run_operator_command "$mode" "$round" "$key_count" "$command" \ + || fail "cache-$mode command failed for round=$round K=$key_count" + server_in_cgroup || fail "cache-$mode command moved RustFS PID $SERVER_PID out of $CGROUP_PATH" +} + +reset_cache() { + local mode=$1 + local round=$2 + local key_count=$3 + run_operator_command "$mode" "$round" "$key_count" "$RESET_COMMAND" \ + || fail "reset command failed for mode=$mode round=$round K=$key_count" + server_in_cgroup || fail "reset command moved RustFS PID $SERVER_PID out of $CGROUP_PATH" +} + +printf 'mode,k,round,requests,object_size,baseline_reader_bytes,reader_bytes_delta,reader_bytes_limit,follower_samples,follower_max,oom_delta,oom_kill_delta,memory_peak_before,memory_peak_after,duration_seconds\n' >"$RESULTS_CSV" +overall_oom_before=$(cgroup_event oom) || fail "memory.events has no oom field" +overall_oom_kill_before=$(cgroup_event oom_kill) || fail "memory.events has no oom_kill field" + +printf 'INFO: artifacts: %s\n' "$OUT_DIR" +printf 'INFO: fixed matrix N=%d, K={1,4,32}, rounds=%d, object_size=%d\n' \ + "$REQUESTS" "$ROUNDS" "$OBJECT_SIZE" + +for ((round = 1; round <= ROUNDS; round++)); do + for key_count in $KEY_MATRIX; do + baseline_prefix="cache-off-k${key_count}-round${round}" + baseline_summary="$OUT_DIR/$baseline_prefix-load.json" + baseline_ready="$OUT_DIR/$baseline_prefix-ready" + + printf 'INFO: K=%d round=%d: switching cache off for one request per key baseline\n' "$key_count" "$round" + switch_cache_mode off "$CACHE_OFF_COMMAND" "$round" "$key_count" + reset_cache off "$round" "$key_count" + baseline_reader_before=$(prometheus_value "$READER_BYTES_QUERY") || fail "reader query failed before cache-off K=$key_count round=$round" + baseline_oom_before=$(cgroup_event oom) || fail "cannot read oom before baseline" + baseline_oom_kill_before=$(cgroup_event oom_kill) || fail "cannot read oom_kill before baseline" + baseline_peak_before=$(cat "$CGROUP_PATH/memory.peak" 2>/dev/null || printf 'NA') + + run_load "$key_count" "$key_count" "$baseline_summary" "$baseline_ready" + sleep "$METRICS_SETTLE_SECONDS" + baseline_reader_after=$(prometheus_value "$READER_BYTES_QUERY") || fail "reader query failed after cache-off K=$key_count round=$round" + baseline_reader_delta=$(numeric_positive_delta "$baseline_reader_before" "$baseline_reader_after") \ + || fail "cache-off reader baseline is invalid for K=$key_count round=$round" + baseline_oom_after=$(cgroup_event oom) || fail "cannot read oom after baseline" + baseline_oom_kill_after=$(cgroup_event oom_kill) || fail "cannot read oom_kill after baseline" + baseline_oom_delta=$((baseline_oom_after - baseline_oom_before)) + baseline_oom_kill_delta=$((baseline_oom_kill_after - baseline_oom_kill_before)) + ((baseline_oom_delta == 0 && baseline_oom_kill_delta == 0)) \ + || fail "OOM event delta is non-zero for cache-off K=$key_count round=$round" + baseline_peak_after=$(cat "$CGROUP_PATH/memory.peak" 2>/dev/null || printf 'NA') + read -r baseline_requests baseline_duration < <(summary_fields "$baseline_summary") + ((baseline_requests == key_count)) \ + || fail "cache-off baseline completed $baseline_requests requests, expected $key_count" + printf 'cache_off,%d,%d,%d,%d,%s,%s,NA,NA,NA,%d,%d,%s,%s,%s\n' \ + "$key_count" "$round" "$key_count" "$OBJECT_SIZE" "$baseline_reader_delta" "$baseline_reader_delta" \ + "$baseline_oom_delta" "$baseline_oom_kill_delta" "$baseline_peak_before" "$baseline_peak_after" "$baseline_duration" \ + >>"$RESULTS_CSV" + + prefix="k${key_count}-round${round}" + summary_file="$OUT_DIR/$prefix-load.json" + ready_file="$OUT_DIR/$prefix-ready" + follower_file="$OUT_DIR/$prefix-follower.samples" + : >"$follower_file" + + printf 'INFO: K=%d round=%d: switching cache on and resetting\n' "$key_count" "$round" + switch_cache_mode on "$CACHE_ON_COMMAND" "$round" "$key_count" + reset_cache on "$round" "$key_count" + + reader_before=$(prometheus_value "$READER_BYTES_QUERY") || fail "reader query failed before K=$key_count round=$round" + oom_before=$(cgroup_event oom) || fail "cannot read oom before round" + oom_kill_before=$(cgroup_event oom_kill) || fail "cannot read oom_kill before round" + memory_peak_before=$(cat "$CGROUP_PATH/memory.peak" 2>/dev/null || printf 'NA') + + run_load "$key_count" "$REQUESTS" "$summary_file" "$ready_file" "$baseline_summary" & + load_pid=$! + LOAD_PID=$load_pid + while kill -0 "$load_pid" 2>/dev/null; do + if [[ -e $ready_file ]]; then + if sample=$(prometheus_sample "$FOLLOWER_PERMIT_QUERY" 2>/dev/null); then + append_follower_sample "$follower_file" "$sample" || fail "invalid follower sample for K=$key_count round=$round" + fi + fi + sleep "$SAMPLE_INTERVAL" + done + wait "$load_pid" || fail "load or SHA-256 validation failed for K=$key_count round=$round (see $summary_file)" + LOAD_PID="" + + sleep "$METRICS_SETTLE_SECONDS" + reader_after=$(prometheus_value "$READER_BYTES_QUERY") || fail "reader query failed after K=$key_count round=$round" + expected_reader_bytes=$(((baseline_reader_delta * 90) / 100)) + reader_limit=$(((baseline_reader_delta * 110 + 99) / 100)) + reader_delta=$(numeric_delta_check "$reader_before" "$reader_after" \ + "$expected_reader_bytes" "$reader_limit") || fail "reader byte invariant failed for K=$key_count round=$round" + follower_result=$(follower_samples_check "$follower_file" "$ready_file") || fail "follower permit invariant failed for K=$key_count round=$round" + IFS=, read -r follower_samples follower_max <<<"$follower_result" + + oom_after=$(cgroup_event oom) || fail "cannot read oom after round" + oom_kill_after=$(cgroup_event oom_kill) || fail "cannot read oom_kill after round" + server_in_cgroup || fail "RustFS PID $SERVER_PID left $CGROUP_PATH during the run" + oom_delta=$((oom_after - oom_before)) + oom_kill_delta=$((oom_kill_after - oom_kill_before)) + ((oom_delta == 0 && oom_kill_delta == 0)) || fail "OOM event delta is non-zero for K=$key_count round=$round" + memory_peak_after=$(cat "$CGROUP_PATH/memory.peak" 2>/dev/null || printf 'NA') + + read -r completed_requests duration_seconds < <(summary_fields "$summary_file") + ((completed_requests == REQUESTS)) || fail "completed request count is $completed_requests, expected $REQUESTS" + + printf 'cache_on,%d,%d,%d,%d,%s,%s,%d,%s,%s,%d,%d,%s,%s,%s\n' \ + "$key_count" "$round" "$completed_requests" "$OBJECT_SIZE" "$baseline_reader_delta" "$reader_delta" \ + "$reader_limit" "$follower_samples" "$follower_max" "$oom_delta" \ + "$oom_kill_delta" "$memory_peak_before" "$memory_peak_after" "$duration_seconds" \ + >>"$RESULTS_CSV" + printf 'PASS: K=%d round=%d SHA-256=%s reader_delta=%s follower_max=0 OOM_delta=0\n' \ + "$key_count" "$round" "$EXPECTED_SHA256" "$reader_delta" + done +done + +overall_oom_after=$(cgroup_event oom) || fail "cannot read final oom count" +overall_oom_kill_after=$(cgroup_event oom_kill) || fail "cannot read final oom_kill count" +server_in_cgroup || fail "RustFS PID $SERVER_PID left $CGROUP_PATH during the run" +((overall_oom_after == overall_oom_before)) || fail "overall oom delta is non-zero" +((overall_oom_kill_after == overall_oom_kill_before)) || fail "overall oom_kill delta is non-zero" + +printf 'PASS: cold stampede matrix completed; results: %s\n' "$RESULTS_CSV"