diff --git a/Cargo.lock b/Cargo.lock index edb391254..d2ececfec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8831,6 +8831,7 @@ dependencies = [ "rustfs-madmin", "rustfs-notify", "rustfs-object-capacity", + "rustfs-object-data-cache", "rustfs-obs", "rustfs-policy", "rustfs-protocols", @@ -9445,6 +9446,19 @@ dependencies = [ "walkdir", ] +[[package]] +name = "rustfs-object-data-cache" +version = "1.0.0-beta.8" +dependencies = [ + "bytes", + "metrics", + "moka", + "starshard", + "sysinfo", + "thiserror 2.0.18", + "tokio", +] + [[package]] name = "rustfs-obs" version = "1.0.0-beta.8" diff --git a/Cargo.toml b/Cargo.toml index 4ae15f2e8..53db5d73f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ members = [ "crates/notify", # Notification system for events "crates/obs", # Observability utilities "crates/object-capacity", # Capacity scan and refresh core + "crates/object-data-cache", # Long-term object body cache engine "crates/policy", # Policy management "crates/protocols", # Protocol implementations (FTPS, SFTP, etc.) "crates/protos", # Protocol buffer definitions @@ -103,6 +104,7 @@ rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.8" } rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.8" } rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.8" } rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.8" } +rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.8" } rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.8" } rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.8" } rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.8" } diff --git a/crates/config/src/constants/runtime.rs b/crates/config/src/constants/runtime.rs index c10ff93b5..1a50bd146 100644 --- a/crates/config/src/constants/runtime.rs +++ b/crates/config/src/constants/runtime.rs @@ -110,3 +110,16 @@ pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: usize = 4 * 1024 * 1024; /// Default is set to 10 MiB. pub const ENV_OBJECT_SEEK_SUPPORT_THRESHOLD: &str = "RUSTFS_OBJECT_SEEK_SUPPORT_THRESHOLD"; pub const DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD: usize = 10 * 1024 * 1024; + +// Object data cache configuration +pub const ENV_OBJECT_DATA_CACHE_ENABLE: &str = "RUSTFS_OBJECT_DATA_CACHE_ENABLE"; +pub const ENV_OBJECT_DATA_CACHE_MODE: &str = "RUSTFS_OBJECT_DATA_CACHE_MODE"; +pub const ENV_OBJECT_DATA_CACHE_MAX_BYTES: &str = "RUSTFS_OBJECT_DATA_CACHE_MAX_BYTES"; +pub const ENV_OBJECT_DATA_CACHE_MAX_MEMORY_PERCENT: &str = "RUSTFS_OBJECT_DATA_CACHE_MAX_MEMORY_PERCENT"; +pub const ENV_OBJECT_DATA_CACHE_MAX_ENTRY_BYTES: &str = "RUSTFS_OBJECT_DATA_CACHE_MAX_ENTRY_BYTES"; +pub const ENV_OBJECT_DATA_CACHE_TTL_SECS: &str = "RUSTFS_OBJECT_DATA_CACHE_TTL_SECS"; +pub const ENV_OBJECT_DATA_CACHE_TIME_TO_IDLE_SECS: &str = "RUSTFS_OBJECT_DATA_CACHE_TIME_TO_IDLE_SECS"; +pub const ENV_OBJECT_DATA_CACHE_MIN_FREE_MEMORY_PERCENT: &str = "RUSTFS_OBJECT_DATA_CACHE_MIN_FREE_MEMORY_PERCENT"; +pub const ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_PER_CPU: &str = "RUSTFS_OBJECT_DATA_CACHE_FILL_CONCURRENCY_PER_CPU"; +pub const ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_MAX: &str = "RUSTFS_OBJECT_DATA_CACHE_FILL_CONCURRENCY_MAX"; +pub const ENV_OBJECT_DATA_CACHE_IDENTITY_KEYS_MAX: &str = "RUSTFS_OBJECT_DATA_CACHE_IDENTITY_KEYS_MAX"; diff --git a/crates/object-data-cache/Cargo.toml b/crates/object-data-cache/Cargo.toml new file mode 100644 index 000000000..be4303461 --- /dev/null +++ b/crates/object-data-cache/Cargo.toml @@ -0,0 +1,43 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[package] +name = "rustfs-object-data-cache" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +homepage.workspace = true +description = "Long-term object body cache engine for RustFS." +keywords = ["cache", "object-storage", "rustfs"] +categories = ["web-programming", "development-tools"] + +[lib] +doctest = false + +[dependencies] +bytes.workspace = true +metrics = { workspace = true } +moka = { workspace = true, features = ["future"] } +starshard.workspace = true +sysinfo = { workspace = true, features = ["multithread"] } +thiserror.workspace = true +tokio = { workspace = true, features = ["sync"] } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt", "time"] } + +[lints] +workspace = true diff --git a/crates/object-data-cache/src/backend.rs b/crates/object-data-cache/src/backend.rs new file mode 100644 index 000000000..67407bbe6 --- /dev/null +++ b/crates/object-data-cache/src/backend.rs @@ -0,0 +1,40 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::moka_backend::MokaBackend; +use crate::noop::NoopBackend; + +/// Backend dispatch for the object data cache. +#[derive(Debug)] +pub enum ObjectDataCacheBackendKind { + /// No-op backend used while the feature is disabled. + Noop(NoopBackend), + /// Moka backend used when cache lookups are enabled. + Moka(Box), +} + +impl Default for ObjectDataCacheBackendKind { + fn default() -> Self { + Self::Noop(NoopBackend) + } +} + +impl ObjectDataCacheBackendKind { + pub(crate) const fn as_metric_label(&self) -> &'static str { + match self { + Self::Noop(_) => "noop", + Self::Moka(_) => "moka", + } + } +} diff --git a/crates/object-data-cache/src/cache.rs b/crates/object-data-cache/src/cache.rs new file mode 100644 index 000000000..9437df50b --- /dev/null +++ b/crates/object-data-cache/src/cache.rs @@ -0,0 +1,374 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::backend::ObjectDataCacheBackendKind; +use crate::config::ObjectDataCacheConfig; +use crate::error::ObjectDataCacheConfigError; +use crate::key::{ObjectDataCacheBodyVariant, ObjectDataCacheIdentity, ObjectDataCacheKey}; +use crate::metrics::{ + describe_metrics_once, publish_cache_state, record_fill_result, record_hit_bytes, record_invalidation, + record_request_decision, +}; +use crate::moka_backend::MokaBackend; +use crate::noop::NoopBackend; +use crate::stats::{ObjectDataCacheStats, ObjectDataCacheStatsSnapshot}; +use bytes::Bytes; +use std::sync::Arc; +use std::time::Instant; + +/// Protocol-neutral cache facade for object body reuse. +#[derive(Debug)] +pub struct ObjectDataCache { + backend: ObjectDataCacheBackendKind, + config: Arc, + stats: Arc, +} + +impl ObjectDataCache { + /// Creates a disabled cache facade without requiring configuration parsing. + pub fn disabled() -> Self { + let config = Arc::new(ObjectDataCacheConfig::default()); + let stats = Arc::new(ObjectDataCacheStats::default()); + + Self { + backend: ObjectDataCacheBackendKind::Noop(NoopBackend), + config, + stats, + } + } + + /// Creates a new cache facade. + pub fn new(config: ObjectDataCacheConfig) -> Result { + describe_metrics_once(); + config.validate()?; + let stats = Arc::new(ObjectDataCacheStats::default()); + let backend = if config.is_disabled() { + ObjectDataCacheBackendKind::Noop(NoopBackend) + } else { + ObjectDataCacheBackendKind::Moka(Box::new(MokaBackend::new(&config, Arc::clone(&stats))?)) + }; + + Ok(Self { + backend, + config: Arc::new(config), + stats, + }) + } + + /// Produces a lightweight GET plan from request metadata. + pub fn plan_get(&self, request: ObjectDataCacheGetRequest<'_>) -> ObjectDataCacheGetPlan { + if self.config.is_disabled() { + record_request_decision( + self.backend.as_metric_label(), + self.config.mode, + "disabled", + "mode_disabled", + request.size, + ); + return ObjectDataCacheGetPlan::Disabled; + } + + if request.size > self.config.max_entry_bytes { + record_request_decision(self.backend.as_metric_label(), self.config.mode, "skip", "too_large", request.size); + return ObjectDataCacheGetPlan::SkipTooLarge; + } + + record_request_decision(self.backend.as_metric_label(), self.config.mode, "cacheable", "eligible", request.size); + + ObjectDataCacheGetPlan::Cacheable { + key: ObjectDataCacheKey::new( + request.bucket, + request.object, + request.version_id.as_deref(), + request.etag, + request.size, + request.body_variant, + ), + } + } + + /// Looks up an object body from the configured backend. + pub async fn lookup_body(&self, plan: &ObjectDataCacheGetPlan) -> ObjectDataCacheLookup { + let lookup = match &self.backend { + ObjectDataCacheBackendKind::Noop(backend) => backend.lookup_body(plan).await, + ObjectDataCacheBackendKind::Moka(backend) => backend.lookup_body(plan).await, + }; + + self.stats.record_lookup(matches!(lookup, ObjectDataCacheLookup::Hit(_))); + self.refresh_entry_count(); + match &lookup { + ObjectDataCacheLookup::Hit(bytes) => { + let size_bytes = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + record_request_decision(self.backend.as_metric_label(), self.config.mode, "hit", "cache_hit", size_bytes); + record_hit_bytes(self.backend.as_metric_label(), self.config.mode, size_bytes); + } + ObjectDataCacheLookup::Miss => { + let size_bytes = match plan { + ObjectDataCacheGetPlan::Cacheable { key } => key.size, + _ => 0, + }; + record_request_decision(self.backend.as_metric_label(), self.config.mode, "miss", "cache_miss", size_bytes); + } + ObjectDataCacheLookup::SkipDisabled => { + record_request_decision(self.backend.as_metric_label(), self.config.mode, "skip", "lookup_disabled", 0); + } + ObjectDataCacheLookup::SkipNotCacheable => { + record_request_decision(self.backend.as_metric_label(), self.config.mode, "skip", "lookup_not_cacheable", 0); + } + } + + lookup + } + + /// Attempts to fill the cache body for the current plan. + pub async fn fill_body(&self, plan: &ObjectDataCacheGetPlan, bytes: Bytes) -> ObjectDataCacheFillResult { + let fill_bytes = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + if !self.config.fill_enabled() { + record_fill_result(self.backend.as_metric_label(), self.config.mode, "skipped_by_mode", fill_bytes, 0.0); + return ObjectDataCacheFillResult::SkippedByMode; + } + + if let ObjectDataCacheGetPlan::Cacheable { key } = plan + && fill_bytes != key.size + { + let result = ObjectDataCacheFillResult::SkippedSizeMismatch; + record_fill_result( + self.backend.as_metric_label(), + self.config.mode, + result.as_metric_label(), + fill_bytes, + 0.0, + ); + return result; + } + + let fill_start = Instant::now(); + let result = match &self.backend { + ObjectDataCacheBackendKind::Noop(backend) => backend.fill_body(plan).await, + ObjectDataCacheBackendKind::Moka(backend) => backend.fill_body(plan, bytes).await, + }; + + if matches!(result, ObjectDataCacheFillResult::Inserted) { + self.stats.record_fill(); + } + self.refresh_entry_count(); + record_fill_result( + self.backend.as_metric_label(), + self.config.mode, + result.as_metric_label(), + fill_bytes, + fill_start.elapsed().as_secs_f64(), + ); + + result + } + + /// Invalidates all cache entries associated with the object identity. + pub async fn invalidate_object( + &self, + _identity: ObjectDataCacheIdentity, + _reason: ObjectDataCacheInvalidationReason, + ) -> ObjectDataCacheInvalidationResult { + let result = match &self.backend { + ObjectDataCacheBackendKind::Noop(backend) => backend.invalidate_object().await, + ObjectDataCacheBackendKind::Moka(backend) => backend.invalidate_object(&_identity).await, + }; + + self.stats.record_invalidation(); + self.refresh_entry_count(); + record_invalidation(self.backend.as_metric_label(), _reason.as_metric_label()); + + result + } + + /// Returns the current stats snapshot. + pub fn stats(&self) -> ObjectDataCacheStatsSnapshot { + self.stats.snapshot() + } + + /// Returns true when the cache facade is fully disabled. + pub fn is_disabled(&self) -> bool { + self.config.is_disabled() + } + + /// Returns true when the cache mode allows materialize fill. + pub fn materialize_fill_enabled(&self) -> bool { + matches!(self.config.mode, crate::config::ObjectDataCacheMode::FillMaterializeEnabled) + } + + fn refresh_entry_count(&self) { + let (entries, weighted_bytes) = match &self.backend { + ObjectDataCacheBackendKind::Noop(_) => (0, 0), + ObjectDataCacheBackendKind::Moka(backend) => (backend.entry_count(), backend.weighted_size()), + }; + self.stats.set_entries(entries); + publish_cache_state(self.backend.as_metric_label(), entries, weighted_bytes); + } +} + +/// Protocol-neutral GET request metadata for cache planning. +#[derive(Debug, Clone)] +pub struct ObjectDataCacheGetRequest<'a> { + /// Bucket name. + pub bucket: &'a str, + /// Object key. + pub object: &'a str, + /// Optional version id. + pub version_id: Option, + /// Object ETag. + pub etag: &'a str, + /// Object size in bytes. + pub size: u64, + /// Supported response body variant. + pub body_variant: ObjectDataCacheBodyVariant, +} + +/// Planning result for a cache-aware GET. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ObjectDataCacheGetPlan { + /// Cache is globally disabled. + Disabled, + /// Object body exceeds the configured cacheable entry size. + SkipTooLarge, + /// Request is eligible for cache lookup and fill. + Cacheable { + /// Stable cache key for the request. + key: ObjectDataCacheKey, + }, +} + +/// Result of a cache lookup attempt. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ObjectDataCacheLookup { + /// Cache is disabled. + SkipDisabled, + /// Request was not cacheable under the current plan. + SkipNotCacheable, + /// Cache did not contain a matching object body. + Miss, + /// Cache returned a reusable object body. + Hit(Bytes), +} + +/// Result of a cache fill attempt. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ObjectDataCacheFillResult { + /// Cache is disabled. + SkippedDisabled, + /// Cache mode does not currently allow fill. + SkippedByMode, + /// Request was not cacheable under the current plan. + SkippedNotCacheable, + /// Fill was skipped because the local memory gate rejected it. + SkippedMemoryPressure, + /// Fill was skipped because the per-identity key budget overflowed and was conservatively cleared. + SkippedIdentityOverflow, + /// Fill was skipped because the provided body length did not match the cache key identity. + SkippedSizeMismatch, + /// Fill waiters were released without a published leader result. + SkippedSingleflightClosed, + /// Fill was undone because an invalidation raced with the insert. + SkippedInvalidationRace, + /// The cache entry was inserted successfully. + Inserted, +} + +impl ObjectDataCacheFillResult { + pub(crate) const fn as_metric_label(&self) -> &'static str { + match self { + Self::SkippedDisabled => "skipped_disabled", + Self::SkippedByMode => "skipped_by_mode", + Self::SkippedNotCacheable => "skipped_not_cacheable", + Self::SkippedMemoryPressure => "skipped_memory_pressure", + Self::SkippedIdentityOverflow => "skipped_identity_overflow", + Self::SkippedSizeMismatch => "skipped_size_mismatch", + Self::SkippedSingleflightClosed => "skipped_singleflight_closed", + Self::SkippedInvalidationRace => "skipped_invalidation_race", + Self::Inserted => "inserted", + } + } +} + +/// Invalidation reason placeholder for the skeleton. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ObjectDataCacheInvalidationReason { + /// Conservative invalidation before a mutating write or delete begins. + BeforeMutation, + /// Invalidation after a successful PutObject write. + AfterPutSuccess, + /// Invalidation after a successful delete. + AfterDeleteSuccess, + /// Invalidation after a successful copy destination write. + AfterCopySuccess, + /// Invalidation after a successful complete multipart upload. + AfterCompleteMultipartSuccess, + /// Manual invalidation requested by the caller. + Manual, +} + +impl ObjectDataCacheInvalidationReason { + pub(crate) const fn as_metric_label(self) -> &'static str { + match self { + Self::BeforeMutation => "before_mutation", + Self::AfterPutSuccess => "after_put_success", + Self::AfterDeleteSuccess => "after_delete_success", + Self::AfterCopySuccess => "after_copy_success", + Self::AfterCompleteMultipartSuccess => "after_complete_multipart_success", + Self::Manual => "manual", + } + } +} + +/// Result of an invalidation request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ObjectDataCacheInvalidationResult { + /// Invalidation completed successfully. + Success, +} + +#[cfg(test)] +mod tests { + use super::{ObjectDataCache, ObjectDataCacheFillResult, ObjectDataCacheGetRequest, ObjectDataCacheLookup}; + use crate::config::{ObjectDataCacheConfig, ObjectDataCacheMode}; + use crate::key::ObjectDataCacheBodyVariant; + use bytes::Bytes; + + fn fill_enabled_cache() -> ObjectDataCache { + let config = ObjectDataCacheConfig { + mode: ObjectDataCacheMode::FillBufferedOnly, + max_bytes: 8_388_608, + ..ObjectDataCacheConfig::default() + }; + ObjectDataCache::new(config).expect("fill-enabled cache config should initialize") + } + + #[tokio::test] + async fn fill_body_rejects_size_mismatch() { + let cache = fill_enabled_cache(); + let plan = cache.plan_get(ObjectDataCacheGetRequest { + bucket: "bucket", + object: "object", + version_id: None, + etag: "etag", + size: 5, + body_variant: ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + + let fill = cache.fill_body(&plan, Bytes::from_static(b"oops")).await; + let lookup = cache.lookup_body(&plan).await; + + assert_eq!(fill, ObjectDataCacheFillResult::SkippedSizeMismatch); + assert!(matches!(lookup, ObjectDataCacheLookup::Miss)); + } +} diff --git a/crates/object-data-cache/src/config.rs b/crates/object-data-cache/src/config.rs new file mode 100644 index 000000000..d5ea7bc73 --- /dev/null +++ b/crates/object-data-cache/src/config.rs @@ -0,0 +1,311 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::error::ObjectDataCacheConfigError; +use std::time::Duration; +use sysinfo::System; + +const DEFAULT_DERIVED_MAX_MEMORY_PERCENT_CAP: u64 = 10; +const DEFAULT_DERIVED_MAX_BYTES_CAP: u64 = 64 * 1024 * 1024 * 1024; + +/// Runtime mode for the object data cache. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ObjectDataCacheMode { + /// Cache is completely disabled. + #[default] + Disabled, + /// Cache lookups are allowed, but cache fill remains disabled. + HitOnly, + /// Cache fill is only allowed from an existing buffered body. + FillBufferedOnly, + /// Cache fill may materialize the final body stream exactly once. + FillMaterializeEnabled, +} + +impl ObjectDataCacheMode { + pub(crate) const fn as_metric_label(self) -> &'static str { + match self { + Self::Disabled => "disabled", + Self::HitOnly => "hit_only", + Self::FillBufferedOnly => "fill_buffered_only", + Self::FillMaterializeEnabled => "fill_materialize_enabled", + } + } +} + +/// Object data cache configuration. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObjectDataCacheConfig { + /// Runtime mode gate for the cache engine. + pub mode: ObjectDataCacheMode, + /// Explicit byte-capacity override. Zero means derive from memory percent. + pub max_bytes: u64, + /// Memory percent used when `max_bytes` is zero. + pub max_memory_percent: u8, + /// Maximum cacheable entry size in bytes. + pub max_entry_bytes: u64, + /// Time-to-live for a cache entry. + pub ttl: Duration, + /// Time-to-idle for a cache entry. + pub time_to_idle: Duration, + /// Minimum free memory percent before fill is paused. + pub min_free_memory_percent: u8, + /// Fill concurrency multiplier applied to CPU count. + pub fill_concurrency_per_cpu: u16, + /// Absolute fill concurrency cap. + pub fill_concurrency_max: u16, + /// Conservative cap for keys attached to one object identity. + pub identity_keys_max: u16, +} + +impl Default for ObjectDataCacheConfig { + fn default() -> Self { + Self { + mode: ObjectDataCacheMode::Disabled, + max_bytes: 0, + max_memory_percent: 5, + max_entry_bytes: 1_048_576, + ttl: Duration::from_secs(60), + time_to_idle: Duration::from_secs(30), + min_free_memory_percent: 20, + fill_concurrency_per_cpu: 1, + fill_concurrency_max: 32, + identity_keys_max: 16, + } + } +} + +impl ObjectDataCacheConfig { + /// Returns true when the cache is effectively disabled. + pub const fn is_disabled(&self) -> bool { + matches!(self.mode, ObjectDataCacheMode::Disabled) + } + + /// Returns true when the cache mode allows lookups. + pub const fn lookup_enabled(&self) -> bool { + !self.is_disabled() + } + + /// Returns true when the cache mode allows fills. + pub const fn fill_enabled(&self) -> bool { + matches!( + self.mode, + ObjectDataCacheMode::FillBufferedOnly | ObjectDataCacheMode::FillMaterializeEnabled + ) + } + + /// Resolves the effective max capacity in bytes for the cache. + pub fn resolved_max_bytes(&self) -> Result { + if self.max_bytes > 0 { + return Ok(self.max_bytes); + } + + let mut system = System::new(); + system.refresh_memory(); + let total_memory = system.total_memory(); + let derived = total_memory.saturating_mul(u64::from(self.max_memory_percent)) / 100; + let resolved = clamp_derived_max_bytes(derived, total_memory, self.max_entry_bytes); + + if resolved == 0 { + return Err(ObjectDataCacheConfigError::ZeroResolvedMaxBytes); + } + + Ok(resolved) + } + + /// Validates the configuration for internal consistency. + pub fn validate(&self) -> Result<(), ObjectDataCacheConfigError> { + if self.max_bytes == 0 && (self.max_memory_percent == 0 || self.max_memory_percent > 100) { + return Err(ObjectDataCacheConfigError::InvalidMaxMemoryPercent); + } + + if self.max_entry_bytes == 0 { + return Err(ObjectDataCacheConfigError::ZeroMaxEntryBytes); + } + + if self.ttl.is_zero() { + return Err(ObjectDataCacheConfigError::ZeroTimeToLiveSecs); + } + + if self.time_to_idle.is_zero() { + return Err(ObjectDataCacheConfigError::ZeroTimeToIdleSecs); + } + + if self.min_free_memory_percent == 0 || self.min_free_memory_percent > 100 { + return Err(ObjectDataCacheConfigError::InvalidMinFreeMemoryPercent); + } + + if self.fill_concurrency_per_cpu == 0 { + return Err(ObjectDataCacheConfigError::ZeroFillConcurrencyPerCpu); + } + + if self.fill_concurrency_max == 0 { + return Err(ObjectDataCacheConfigError::ZeroFillConcurrencyMax); + } + + if self.fill_concurrency_max < self.fill_concurrency_per_cpu { + return Err(ObjectDataCacheConfigError::FillConcurrencyMaxTooSmall); + } + + if self.identity_keys_max == 0 { + return Err(ObjectDataCacheConfigError::ZeroIdentityKeysMax); + } + + Ok(()) + } +} + +fn clamp_derived_max_bytes(derived: u64, total_memory: u64, max_entry_bytes: u64) -> u64 { + let percent_cap = total_memory.saturating_mul(DEFAULT_DERIVED_MAX_MEMORY_PERCENT_CAP) / 100; + let safe_cap = percent_cap.min(DEFAULT_DERIVED_MAX_BYTES_CAP).max(max_entry_bytes); + + derived.min(safe_cap).max(max_entry_bytes) +} + +#[cfg(test)] +mod tests { + use super::{DEFAULT_DERIVED_MAX_BYTES_CAP, ObjectDataCacheConfig, ObjectDataCacheMode, clamp_derived_max_bytes}; + use crate::error::ObjectDataCacheConfigError; + use std::time::Duration; + + #[test] + fn default_config_matches_v3_baseline() { + let config = ObjectDataCacheConfig::default(); + + assert!(matches!(config.mode, ObjectDataCacheMode::Disabled)); + assert_eq!(config.max_bytes, 0); + assert_eq!(config.max_memory_percent, 5); + assert_eq!(config.max_entry_bytes, 1_048_576); + assert_eq!(config.ttl, Duration::from_secs(60)); + assert_eq!(config.time_to_idle, Duration::from_secs(30)); + assert_eq!(config.min_free_memory_percent, 20); + assert_eq!(config.fill_concurrency_per_cpu, 1); + assert_eq!(config.fill_concurrency_max, 32); + assert_eq!(config.identity_keys_max, 16); + } + + #[test] + fn validate_rejects_invalid_memory_percent_when_capacity_is_derived() { + let config = ObjectDataCacheConfig { + max_memory_percent: 0, + ..ObjectDataCacheConfig::default() + }; + + let err = config + .validate() + .expect_err("derived capacity requires a non-zero memory percent"); + + assert_eq!(err, ObjectDataCacheConfigError::InvalidMaxMemoryPercent); + } + + #[test] + fn validate_rejects_zero_entry_size() { + let config = ObjectDataCacheConfig { + max_entry_bytes: 0, + ..ObjectDataCacheConfig::default() + }; + + let err = config.validate().expect_err("entry size must stay positive"); + + assert_eq!(err, ObjectDataCacheConfigError::ZeroMaxEntryBytes); + } + + #[test] + fn validate_rejects_zero_ttl() { + let config = ObjectDataCacheConfig { + ttl: Duration::ZERO, + ..ObjectDataCacheConfig::default() + }; + + let err = config.validate().expect_err("ttl must stay positive"); + + assert_eq!(err, ObjectDataCacheConfigError::ZeroTimeToLiveSecs); + } + + #[test] + fn validate_rejects_zero_time_to_idle() { + let config = ObjectDataCacheConfig { + time_to_idle: Duration::ZERO, + ..ObjectDataCacheConfig::default() + }; + + let err = config.validate().expect_err("time-to-idle must stay positive"); + + assert_eq!(err, ObjectDataCacheConfigError::ZeroTimeToIdleSecs); + } + + #[test] + fn validate_rejects_invalid_fill_concurrency_bounds() { + let config = ObjectDataCacheConfig { + fill_concurrency_per_cpu: 2, + fill_concurrency_max: 1, + ..ObjectDataCacheConfig::default() + }; + + let err = config + .validate() + .expect_err("max fill concurrency must not be smaller than per-cpu factor"); + + assert_eq!(err, ObjectDataCacheConfigError::FillConcurrencyMaxTooSmall); + } + + #[test] + fn validate_accepts_explicit_byte_cap() { + let config = ObjectDataCacheConfig { + mode: ObjectDataCacheMode::HitOnly, + max_bytes: 4_194_304, + max_memory_percent: 0, + ..ObjectDataCacheConfig::default() + }; + + assert!(config.validate().is_ok()); + } + + #[test] + fn resolved_max_bytes_prefers_explicit_cap() { + let config = ObjectDataCacheConfig { + max_bytes: 4_194_304, + ..ObjectDataCacheConfig::default() + }; + + let resolved = config + .resolved_max_bytes() + .expect("explicit max_bytes should be returned directly"); + + assert_eq!(resolved, 4_194_304); + } + + #[test] + fn resolved_max_bytes_is_at_least_max_entry_bytes() { + let config = ObjectDataCacheConfig { + max_bytes: 0, + max_memory_percent: 1, + max_entry_bytes: 8_388_608, + ..ObjectDataCacheConfig::default() + }; + + let resolved = config.resolved_max_bytes().expect("derived capacity should stay positive"); + + assert!(resolved >= config.max_entry_bytes); + } + + #[test] + fn derived_max_bytes_clamps_to_v3_safe_cap() { + let one_tib = 1024_u64 * 1024 * 1024 * 1024; + let derived = one_tib / 2; + let resolved = clamp_derived_max_bytes(derived, one_tib, 1_048_576); + + assert_eq!(resolved, DEFAULT_DERIVED_MAX_BYTES_CAP); + } +} diff --git a/crates/object-data-cache/src/entry.rs b/crates/object-data-cache/src/entry.rs new file mode 100644 index 000000000..164123916 --- /dev/null +++ b/crates/object-data-cache/src/entry.rs @@ -0,0 +1,113 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bytes::Bytes; +use std::cmp; +use std::sync::Arc; +use std::time::Instant; + +use crate::key::ObjectDataCacheKey; + +const ENTRY_OVERHEAD_BYTES: usize = 64; + +/// Cached object body entry. +#[derive(Debug, Clone)] +pub struct ObjectDataCacheEntry { + bytes: Bytes, + content_length: u64, + etag: Arc, + inserted_at: Instant, +} + +impl ObjectDataCacheEntry { + /// Creates a new cached entry. + pub fn new(bytes: Bytes, content_length: u64, etag: Arc) -> Self { + Self { + bytes, + content_length, + etag, + inserted_at: Instant::now(), + } + } + + /// Returns a clone of the cached body bytes. + pub fn bytes(&self) -> Bytes { + self.bytes.clone() + } + + /// Returns the recorded content length. + pub const fn content_length(&self) -> u64 { + self.content_length + } + + /// Returns the cached etag reference. + pub fn etag(&self) -> &Arc { + &self.etag + } + + /// Returns the insertion timestamp. + pub const fn inserted_at(&self) -> Instant { + self.inserted_at + } + + /// Returns the estimated weighted size for capacity accounting. + pub fn estimated_weight(&self, key: &ObjectDataCacheKey) -> u32 { + let key_bytes = key.bucket.len() + key.object.len() + key.version_id.len() + key.etag.len(); + let body_bytes = self.bytes.len(); + let total = key_bytes.saturating_add(body_bytes).saturating_add(ENTRY_OVERHEAD_BYTES); + let clamped = cmp::min(total, u32::MAX as usize); + + u32::try_from(clamped).unwrap_or(u32::MAX) + } +} + +#[cfg(test)] +mod tests { + use super::ObjectDataCacheEntry; + use crate::key::{ObjectDataCacheBodyVariant, ObjectDataCacheKey}; + use bytes::Bytes; + use std::sync::Arc; + + #[test] + fn estimated_weight_includes_body_and_key_bytes() { + let key = + ObjectDataCacheKey::new("bucket", "object", Some("vid"), "etag", 5, ObjectDataCacheBodyVariant::FullObjectPlainV1); + let entry = ObjectDataCacheEntry::new(Bytes::from_static(b"hello"), 5, Arc::::from("etag")); + + let weight = entry.estimated_weight(&key); + + assert!(weight >= 5); + } + + #[test] + fn estimated_weight_clamps_to_u32_max() { + let huge_bucket = "b".repeat(1024); + let huge_object = "o".repeat(1024); + let huge_etag = "e".repeat(1024); + let key = ObjectDataCacheKey::new( + huge_bucket.as_str(), + huge_object.as_str(), + Some("version"), + huge_etag.as_str(), + 1, + ObjectDataCacheBodyVariant::FullObjectPlainV1, + ); + let huge = vec![0u8; (u32::MAX as usize).saturating_add(1024)]; + let entry = ObjectDataCacheEntry::new(Bytes::from(huge), 1, Arc::::from("etag")); + + let weight = entry.estimated_weight(&key); + + assert_eq!(weight, u32::MAX); + } +} diff --git a/crates/object-data-cache/src/error.rs b/crates/object-data-cache/src/error.rs new file mode 100644 index 000000000..2f4eda4c2 --- /dev/null +++ b/crates/object-data-cache/src/error.rs @@ -0,0 +1,59 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use thiserror::Error; + +/// Configuration errors for the object data cache engine. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum ObjectDataCacheConfigError { + /// The configured memory percentage exceeded the supported range. + #[error("object data cache max_memory_percent must be in 1..=100")] + InvalidMaxMemoryPercent, + + /// The configured entry size exceeded the supported range. + #[error("object data cache max_entry_bytes must be greater than 0")] + ZeroMaxEntryBytes, + + /// The configured time-to-live cannot be zero. + #[error("object data cache ttl_secs must be greater than 0")] + ZeroTimeToLiveSecs, + + /// The configured time-to-idle cannot be zero. + #[error("object data cache time_to_idle_secs must be greater than 0")] + ZeroTimeToIdleSecs, + + /// The configured minimum free memory percentage exceeded the supported range. + #[error("object data cache min_free_memory_percent must be in 1..=100")] + InvalidMinFreeMemoryPercent, + + /// The configured fill concurrency per CPU exceeded the supported range. + #[error("object data cache fill_concurrency_per_cpu must be greater than 0")] + ZeroFillConcurrencyPerCpu, + + /// The configured fill concurrency maximum exceeded the supported range. + #[error("object data cache fill_concurrency_max must be greater than 0")] + ZeroFillConcurrencyMax, + + /// The configured fill concurrency bounds are internally inconsistent. + #[error("object data cache fill_concurrency_max must be at least fill_concurrency_per_cpu")] + FillConcurrencyMaxTooSmall, + + /// The configured identity key-set cap exceeded the supported range. + #[error("object data cache identity_keys_max must be greater than 0")] + ZeroIdentityKeysMax, + + /// Failed to resolve a non-zero cache capacity from the runtime environment. + #[error("object data cache could not resolve a positive max capacity")] + ZeroResolvedMaxBytes, +} diff --git a/crates/object-data-cache/src/index.rs b/crates/object-data-cache/src/index.rs new file mode 100644 index 000000000..f09f85e65 --- /dev/null +++ b/crates/object-data-cache/src/index.rs @@ -0,0 +1,127 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::key::ObjectDataCacheKey; +use crate::starshard_index::StarshardIdentityIndex; + +/// Result of inserting a cache key into the identity index. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ObjectDataCacheIndexInsertResult { + /// The key was inserted into the identity set. + Inserted, + /// The key was already tracked for the identity. + Duplicate, + /// The identity exceeded its configured key budget and was conservatively cleared. + Overflow { + /// Keys removed while clearing the identity. + cleared_keys: Vec, + }, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct ObjectDataCacheKeySet { + keys: Vec, +} + +impl ObjectDataCacheKeySet { + pub(crate) fn insert(&mut self, key: ObjectDataCacheKey, max_keys: usize) -> ObjectDataCacheIndexInsertResult { + if self.keys.iter().any(|existing| existing == &key) { + return ObjectDataCacheIndexInsertResult::Duplicate; + } + + if self.keys.len() >= max_keys { + let cleared_keys = self.drain(); + return ObjectDataCacheIndexInsertResult::Overflow { cleared_keys }; + } + + self.keys.push(key); + ObjectDataCacheIndexInsertResult::Inserted + } + + pub(crate) fn remove_key(&mut self, key: &ObjectDataCacheKey) -> bool { + let original_len = self.keys.len(); + self.keys.retain(|existing| existing != key); + original_len != self.keys.len() + } + + pub(crate) fn retain(&mut self, mut keep: F) + where + F: FnMut(&ObjectDataCacheKey) -> bool, + { + self.keys.retain(|key| keep(key)); + } + + pub(crate) fn is_empty(&self) -> bool { + self.keys.is_empty() + } + + pub(crate) fn contains(&self, key: &ObjectDataCacheKey) -> bool { + self.keys.iter().any(|existing| existing == key) + } + + #[cfg(test)] + pub(crate) fn len(&self) -> usize { + self.keys.len() + } + + pub(crate) fn cloned(&self) -> Vec { + self.keys.clone() + } + + pub(crate) fn drain(&mut self) -> Vec { + std::mem::take(&mut self.keys) + } +} + +/// Public identity-index façade used by the cache backend. +pub type ObjectDataCacheIdentityIndex = StarshardIdentityIndex; + +#[cfg(test)] +mod tests { + use super::{ObjectDataCacheIndexInsertResult, ObjectDataCacheKeySet}; + use crate::key::{ObjectDataCacheBodyVariant, ObjectDataCacheKey}; + + fn make_key(id: &str) -> ObjectDataCacheKey { + ObjectDataCacheKey::new("bucket", "object", Some(id), "etag", 1, ObjectDataCacheBodyVariant::FullObjectPlainV1) + } + + #[test] + fn key_set_deduplicates_existing_key() { + let mut set = ObjectDataCacheKeySet::default(); + let key = make_key("v1"); + + let first = set.insert(key.clone(), 4); + let second = set.insert(key, 4); + + assert_eq!(first, ObjectDataCacheIndexInsertResult::Inserted); + assert_eq!(second, ObjectDataCacheIndexInsertResult::Duplicate); + assert_eq!(set.len(), 1); + } + + #[test] + fn key_set_overflow_clears_existing_keys() { + let mut set = ObjectDataCacheKeySet::default(); + let key_a = make_key("v1"); + let key_b = make_key("v2"); + + let _ = set.insert(key_a.clone(), 1); + let result = set.insert(key_b, 1); + + assert!(matches!( + result, + ObjectDataCacheIndexInsertResult::Overflow { cleared_keys } if cleared_keys == vec![key_a] + )); + assert!(set.is_empty()); + } +} diff --git a/crates/object-data-cache/src/key.rs b/crates/object-data-cache/src/key.rs new file mode 100644 index 000000000..4c669ecbf --- /dev/null +++ b/crates/object-data-cache/src/key.rs @@ -0,0 +1,118 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +/// Canonical synthetic version id for unversioned or latest-only object bodies. +pub const NULL_VERSION_ID: &str = "null"; + +/// Response body variant supported by the cache. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum ObjectDataCacheBodyVariant { + /// Full plain object body. + #[default] + FullObjectPlainV1, +} + +/// Stable cache key for a reusable object body. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ObjectDataCacheKey { + /// Bucket name. + pub bucket: Arc, + /// Object key. + pub object: Arc, + /// Canonical version id, using `"null"` for unversioned bodies. + pub version_id: Arc, + /// Object ETag. + pub etag: Arc, + /// Object size in bytes. + pub size: u64, + /// Cached body semantics. + pub body_variant: ObjectDataCacheBodyVariant, +} + +/// Identity used for conservative invalidation. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ObjectDataCacheIdentity { + /// Bucket name. + pub bucket: Arc, + /// Object key. + pub object: Arc, +} + +impl ObjectDataCacheKey { + /// Creates a new stable object data cache key. + pub fn new( + bucket: impl Into>, + object: impl Into>, + version_id: Option<&str>, + etag: impl Into>, + size: u64, + body_variant: ObjectDataCacheBodyVariant, + ) -> Self { + Self { + bucket: bucket.into(), + object: object.into(), + version_id: version_id.map_or_else(|| Arc::::from(NULL_VERSION_ID), Arc::::from), + etag: etag.into(), + size, + body_variant, + } + } + + /// Returns true when this key targets the canonical unversioned body variant. + pub fn is_null_version(&self) -> bool { + self.version_id.as_ref() == NULL_VERSION_ID + } +} + +impl ObjectDataCacheIdentity { + /// Creates a conservative invalidation identity. + pub fn new(bucket: impl Into>, object: impl Into>) -> Self { + Self { + bucket: bucket.into(), + object: object.into(), + } + } +} + +#[cfg(test)] +mod tests { + use super::{NULL_VERSION_ID, ObjectDataCacheBodyVariant, ObjectDataCacheIdentity, ObjectDataCacheKey}; + + #[test] + fn key_uses_canonical_null_version_for_missing_version_id() { + let key = ObjectDataCacheKey::new("bucket", "object", None, "etag", 42, ObjectDataCacheBodyVariant::FullObjectPlainV1); + + assert_eq!(key.version_id.as_ref(), NULL_VERSION_ID); + assert!(key.is_null_version()); + } + + #[test] + fn key_distinguishes_explicit_version_ids() { + let latest = ObjectDataCacheKey::new("bucket", "object", None, "etag", 42, ObjectDataCacheBodyVariant::FullObjectPlainV1); + let versioned = + ObjectDataCacheKey::new("bucket", "object", Some("3d2"), "etag", 42, ObjectDataCacheBodyVariant::FullObjectPlainV1); + + assert_ne!(latest, versioned); + } + + #[test] + fn identity_new_preserves_bucket_and_object() { + let identity = ObjectDataCacheIdentity::new("bucket", "object"); + + assert_eq!(identity.bucket.as_ref(), "bucket"); + assert_eq!(identity.object.as_ref(), "object"); + } +} diff --git a/crates/object-data-cache/src/lib.rs b/crates/object-data-cache/src/lib.rs new file mode 100644 index 000000000..62888c18c --- /dev/null +++ b/crates/object-data-cache/src/lib.rs @@ -0,0 +1,43 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Engine-only object body cache contracts for RustFS. +//! +//! This crate intentionally contains only a minimal skeleton for the initial +//! rollout phase. App-layer semantics and storage-specific read paths stay in +//! the `rustfs` crate. + +pub mod backend; +pub mod cache; +pub mod config; +pub mod entry; +pub mod error; +pub mod index; +pub mod key; +pub mod memory; +pub mod metrics; +pub mod moka_backend; +pub mod noop; +pub mod singleflight; +pub mod starshard_index; +pub mod stats; + +pub use cache::{ + ObjectDataCache, ObjectDataCacheFillResult, ObjectDataCacheGetPlan, ObjectDataCacheGetRequest, + ObjectDataCacheInvalidationReason, ObjectDataCacheInvalidationResult, ObjectDataCacheLookup, +}; +pub use config::{ObjectDataCacheConfig, ObjectDataCacheMode}; +pub use error::ObjectDataCacheConfigError; +pub use key::{NULL_VERSION_ID, ObjectDataCacheBodyVariant, ObjectDataCacheIdentity, ObjectDataCacheKey}; +pub use stats::{ObjectDataCacheStats, ObjectDataCacheStatsSnapshot}; diff --git a/crates/object-data-cache/src/memory.rs b/crates/object-data-cache/src/memory.rs new file mode 100644 index 000000000..5b4c99256 --- /dev/null +++ b/crates/object-data-cache/src/memory.rs @@ -0,0 +1,184 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::config::ObjectDataCacheConfig; +use crate::metrics::record_memory_pressure; +use crate::stats::ObjectDataCacheStats; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; +use sysinfo::System; + +const DEFAULT_REFRESH_INTERVAL: Duration = Duration::from_secs(5); + +/// Immutable memory snapshot used by the cache fill gate. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ObjectDataCacheMemorySnapshot { + /// Total system memory in bytes. + pub total_bytes: u64, + /// Available system memory in bytes. + pub available_bytes: u64, +} + +impl ObjectDataCacheMemorySnapshot { + /// Returns the available memory percentage. + pub fn available_percent(&self) -> u8 { + if self.total_bytes == 0 { + return 0; + } + + let percent = self.available_bytes.saturating_mul(100) / self.total_bytes; + u8::try_from(percent.min(100)).unwrap_or(100) + } +} + +/// Memory gate that keeps a cheap snapshot for fill-path checks. +#[derive(Debug)] +pub struct ObjectDataCacheMemoryGate { + system: Mutex, + last_refresh: Mutex, + snapshot_total_bytes: AtomicU64, + snapshot_available_bytes: AtomicU64, + min_free_memory_percent: u8, + refresh_interval: Duration, + stats: Arc, + #[cfg(test)] + test_override: Mutex>, +} + +impl ObjectDataCacheMemoryGate { + /// Creates a new memory gate. + pub fn new(config: &ObjectDataCacheConfig, stats: Arc) -> Self { + let mut system = System::new(); + system.refresh_memory(); + let snapshot = ObjectDataCacheMemorySnapshot { + total_bytes: system.total_memory(), + available_bytes: system.available_memory(), + }; + + Self { + system: Mutex::new(system), + last_refresh: Mutex::new(Instant::now()), + snapshot_total_bytes: AtomicU64::new(snapshot.total_bytes), + snapshot_available_bytes: AtomicU64::new(snapshot.available_bytes), + min_free_memory_percent: config.min_free_memory_percent, + refresh_interval: DEFAULT_REFRESH_INTERVAL, + stats, + #[cfg(test)] + test_override: Mutex::new(None), + } + } + + /// Returns the current atomic memory snapshot. + pub fn snapshot(&self) -> ObjectDataCacheMemorySnapshot { + #[cfg(test)] + { + if let Some(snapshot) = *lock_or_recover(&self.test_override) { + return snapshot; + } + } + + ObjectDataCacheMemorySnapshot { + total_bytes: self.snapshot_total_bytes.load(Ordering::Relaxed), + available_bytes: self.snapshot_available_bytes.load(Ordering::Relaxed), + } + } + + /// Refreshes the snapshot if it is stale. + pub fn refresh_if_stale(&self) { + { + let last_refresh = lock_or_recover(&self.last_refresh); + if last_refresh.elapsed() < self.refresh_interval { + return; + } + } + + let mut system = lock_or_recover(&self.system); + system.refresh_memory(); + let snapshot = ObjectDataCacheMemorySnapshot { + total_bytes: system.total_memory(), + available_bytes: system.available_memory(), + }; + + self.snapshot_total_bytes.store(snapshot.total_bytes, Ordering::Relaxed); + self.snapshot_available_bytes + .store(snapshot.available_bytes, Ordering::Relaxed); + *lock_or_recover(&self.last_refresh) = Instant::now(); + } + + /// Returns true when the fill path may proceed under current memory pressure. + pub fn allows_fill(&self, required_bytes: u64) -> bool { + self.refresh_if_stale(); + let snapshot = self.snapshot(); + if snapshot.total_bytes == 0 { + return true; + } + + let min_free = u64::from(self.min_free_memory_percent); + let has_percent_budget = snapshot.available_bytes.saturating_mul(100) >= snapshot.total_bytes.saturating_mul(min_free); + let has_entry_budget = snapshot.available_bytes >= required_bytes; + let allowed = has_percent_budget && has_entry_budget; + + if !allowed { + record_memory_pressure(&self.stats, "moka"); + } + + allowed + } + + #[cfg(test)] + pub fn set_test_snapshot(&self, snapshot: Option) { + *lock_or_recover(&self.test_override) = snapshot; + } +} + +fn lock_or_recover(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + match mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +#[cfg(test)] +mod tests { + use super::{ObjectDataCacheMemoryGate, ObjectDataCacheMemorySnapshot}; + use crate::config::ObjectDataCacheConfig; + use crate::stats::ObjectDataCacheStats; + use std::sync::Arc; + + #[test] + fn allows_fill_when_memory_snapshot_has_headroom() { + let stats = Arc::new(ObjectDataCacheStats::default()); + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::clone(&stats)); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 500, + })); + + assert!(gate.allows_fill(100)); + } + + #[test] + fn blocks_fill_under_memory_pressure() { + let stats = Arc::new(ObjectDataCacheStats::default()); + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::clone(&stats)); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 100, + })); + + assert!(!gate.allows_fill(128)); + assert_eq!(stats.snapshot().memory_pressure_events, 1); + } +} diff --git a/crates/object-data-cache/src/metrics.rs b/crates/object-data-cache/src/metrics.rs new file mode 100644 index 000000000..b09a49783 --- /dev/null +++ b/crates/object-data-cache/src/metrics.rs @@ -0,0 +1,156 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::config::ObjectDataCacheMode; +use crate::stats::ObjectDataCacheStats; +use metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram}; +use std::sync::{Arc, OnceLock}; + +const METRIC_REQUESTS_TOTAL: &str = "rustfs_object_data_cache_requests_total"; +const METRIC_FILLS_TOTAL: &str = "rustfs_object_data_cache_fill_total"; +const METRIC_FILL_DURATION_SECONDS: &str = "rustfs_object_data_cache_fill_duration_seconds"; +const METRIC_FILL_BYTES_TOTAL: &str = "rustfs_object_data_cache_fill_bytes_total"; +const METRIC_HIT_BYTES_TOTAL: &str = "rustfs_object_data_cache_hit_bytes_total"; +const METRIC_ENTRIES: &str = "rustfs_object_data_cache_entries"; +const METRIC_WEIGHTED_BYTES: &str = "rustfs_object_data_cache_weighted_bytes"; +const METRIC_INFLIGHT_FILLS: &str = "rustfs_object_data_cache_inflight_fills"; +const METRIC_INVALIDATIONS_TOTAL: &str = "rustfs_object_data_cache_invalidations_total"; +const METRIC_MEMORY_PRESSURE_TOTAL: &str = "rustfs_object_data_cache_memory_pressure_total"; + +pub(crate) fn describe_metrics_once() { + static DESCRIBED: OnceLock<()> = OnceLock::new(); + let _ = DESCRIBED.get_or_init(|| { + describe_counter!( + METRIC_REQUESTS_TOTAL, + "Object data cache request decisions labeled by backend, mode, decision, reason, and size class." + ); + describe_counter!( + METRIC_FILLS_TOTAL, + "Object data cache fill outcomes by backend, mode, result, and size class." + ); + describe_histogram!(METRIC_FILL_DURATION_SECONDS, "Object data cache fill duration in seconds."); + describe_counter!(METRIC_FILL_BYTES_TOTAL, "Total bytes submitted to object data cache fill operations."); + describe_counter!(METRIC_HIT_BYTES_TOTAL, "Total bytes served from object data cache hits."); + describe_gauge!(METRIC_ENTRIES, "Current object data cache entry count."); + describe_gauge!(METRIC_WEIGHTED_BYTES, "Approximate weighted bytes held by the object data cache."); + describe_gauge!(METRIC_INFLIGHT_FILLS, "Current number of in-flight object data cache fills."); + describe_counter!( + METRIC_INVALIDATIONS_TOTAL, + "Object data cache invalidation attempts by backend and reason." + ); + describe_counter!(METRIC_MEMORY_PRESSURE_TOTAL, "Object data cache fill skips caused by memory pressure."); + }); +} + +pub(crate) const fn size_class(size_bytes: u64) -> &'static str { + if size_bytes <= 4 * 1024 { + "le_4k" + } else if size_bytes <= 64 * 1024 { + "le_64k" + } else if size_bytes <= 256 * 1024 { + "le_256k" + } else if size_bytes <= 1024 * 1024 { + "le_1m" + } else if size_bytes <= 4 * 1024 * 1024 { + "le_4m" + } else { + "gt_4m" + } +} + +pub(crate) fn record_request_decision( + backend: &'static str, + mode: ObjectDataCacheMode, + decision: &'static str, + reason: &'static str, + size_bytes: u64, +) { + counter!( + METRIC_REQUESTS_TOTAL, + "backend" => backend, + "mode" => mode.as_metric_label(), + "decision" => decision, + "reason" => reason, + "size_class" => size_class(size_bytes), + ) + .increment(1); +} + +pub(crate) fn record_fill_result( + backend: &'static str, + mode: ObjectDataCacheMode, + result: &'static str, + size_bytes: u64, + duration_seconds: f64, +) { + let size_class = size_class(size_bytes); + counter!( + METRIC_FILLS_TOTAL, + "backend" => backend, + "mode" => mode.as_metric_label(), + "result" => result, + "size_class" => size_class, + ) + .increment(1); + histogram!( + METRIC_FILL_DURATION_SECONDS, + "backend" => backend, + "mode" => mode.as_metric_label(), + "result" => result, + "size_class" => size_class, + ) + .record(duration_seconds); + counter!( + METRIC_FILL_BYTES_TOTAL, + "backend" => backend, + "mode" => mode.as_metric_label(), + "result" => result, + "size_class" => size_class, + ) + .increment(size_bytes); +} + +pub(crate) fn record_hit_bytes(backend: &'static str, mode: ObjectDataCacheMode, size_bytes: u64) { + counter!( + METRIC_HIT_BYTES_TOTAL, + "backend" => backend, + "mode" => mode.as_metric_label(), + "size_class" => size_class(size_bytes), + ) + .increment(size_bytes); +} + +pub(crate) fn publish_cache_state(backend: &'static str, entries: u64, weighted_bytes: u64) { + gauge!(METRIC_ENTRIES, "backend" => backend).set(entries as f64); + gauge!(METRIC_WEIGHTED_BYTES, "backend" => backend).set(weighted_bytes as f64); +} + +pub(crate) fn set_inflight_fills(stats: &Arc, backend: &'static str, count: usize) { + stats.set_inflight_fills(count); + let count_u64 = u64::try_from(count).unwrap_or(u64::MAX); + gauge!(METRIC_INFLIGHT_FILLS, "backend" => backend).set(count_u64 as f64); +} + +pub(crate) fn record_singleflight_join(stats: &Arc) { + stats.record_singleflight_join(); +} + +pub(crate) fn record_memory_pressure(stats: &Arc, backend: &'static str) { + stats.record_memory_pressure(); + counter!(METRIC_MEMORY_PRESSURE_TOTAL, "backend" => backend).increment(1); +} + +pub(crate) fn record_invalidation(backend: &'static str, reason: &'static str) { + counter!(METRIC_INVALIDATIONS_TOTAL, "backend" => backend, "reason" => reason).increment(1); +} diff --git a/crates/object-data-cache/src/moka_backend.rs b/crates/object-data-cache/src/moka_backend.rs new file mode 100644 index 000000000..3bdfaa908 --- /dev/null +++ b/crates/object-data-cache/src/moka_backend.rs @@ -0,0 +1,288 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::cache::{ObjectDataCacheFillResult, ObjectDataCacheGetPlan, ObjectDataCacheInvalidationResult, ObjectDataCacheLookup}; +use crate::config::ObjectDataCacheConfig; +use crate::entry::ObjectDataCacheEntry; +use crate::index::{ObjectDataCacheIdentityIndex, ObjectDataCacheIndexInsertResult}; +use crate::key::ObjectDataCacheIdentity; +use crate::memory::ObjectDataCacheMemoryGate; +use crate::singleflight::{ObjectDataCacheSingleflight, ObjectDataCacheSingleflightAcquire}; +use crate::stats::ObjectDataCacheStats; +use bytes::Bytes; +use moka::future::Cache; +use std::sync::Arc; + +/// Weighted Moka backend for reusable object bodies. +#[derive(Debug)] +pub struct MokaBackend { + cache: Cache>, + index: ObjectDataCacheIdentityIndex, + singleflight: ObjectDataCacheSingleflight, + memory_gate: ObjectDataCacheMemoryGate, +} + +impl MokaBackend { + /// Creates a new backend from the validated configuration. + pub fn new( + config: &ObjectDataCacheConfig, + stats: Arc, + ) -> Result { + let max_capacity = config.resolved_max_bytes()?; + let ttl = config.ttl; + let time_to_idle = config.time_to_idle; + let cache = Cache::builder() + .max_capacity(max_capacity) + .weigher(|key, value: &Arc| value.estimated_weight(key)) + .time_to_live(ttl) + .time_to_idle(time_to_idle) + .build(); + + Ok(Self { + cache, + index: ObjectDataCacheIdentityIndex::new(usize::from(config.identity_keys_max)), + singleflight: ObjectDataCacheSingleflight::new(Arc::clone(&stats)), + memory_gate: ObjectDataCacheMemoryGate::new(config, stats), + }) + } + + /// Returns the current cache entry count. + pub fn entry_count(&self) -> u64 { + self.cache.entry_count() + } + + /// Returns the approximate weighted size of cached entries. + pub fn weighted_size(&self) -> u64 { + self.cache.weighted_size() + } + + /// Looks up a cached body for the supplied plan. + pub async fn lookup_body(&self, plan: &ObjectDataCacheGetPlan) -> ObjectDataCacheLookup { + let ObjectDataCacheGetPlan::Cacheable { key } = plan else { + return ObjectDataCacheLookup::SkipNotCacheable; + }; + + match self.cache.get(key).await { + Some(entry) => ObjectDataCacheLookup::Hit(entry.bytes()), + None => ObjectDataCacheLookup::Miss, + } + } + + /// Inserts a cached body for the supplied plan. + pub async fn fill_body(&self, plan: &ObjectDataCacheGetPlan, bytes: Bytes) -> ObjectDataCacheFillResult { + let ObjectDataCacheGetPlan::Cacheable { key } = plan else { + return ObjectDataCacheFillResult::SkippedNotCacheable; + }; + + let fill_state = self.singleflight.acquire(key.clone()).await; + let ObjectDataCacheSingleflightAcquire::Leader(leader) = fill_state else { + let ObjectDataCacheSingleflightAcquire::Waiter(waiter) = fill_state else { + unreachable!(); + }; + return waiter.wait().await; + }; + + if !self.memory_gate.allows_fill(u64::try_from(bytes.len()).unwrap_or(u64::MAX)) { + return leader.finish(ObjectDataCacheFillResult::SkippedMemoryPressure).await; + } + + let identity = ObjectDataCacheIdentity::new(Arc::clone(&key.bucket), Arc::clone(&key.object)); + self.index + .prune_missing(&identity, |candidate| self.cache.contains_key(candidate)) + .await; + + // Register the key in the identity index BEFORE the entry becomes + // visible in the cache, so a concurrent invalidation always finds it. + let result = match self.index.insert(identity.clone(), key.clone()).await { + ObjectDataCacheIndexInsertResult::Inserted | ObjectDataCacheIndexInsertResult::Duplicate => { + let entry = Arc::new(ObjectDataCacheEntry::new(bytes, key.size, Arc::clone(&key.etag))); + self.cache.insert(key.clone(), entry).await; + + // An invalidation may have raced between the index and cache + // inserts; re-check the index and undo the fill so the stale + // body cannot outlive the invalidation. + if self.index.contains_key(&identity, key).await { + ObjectDataCacheFillResult::Inserted + } else { + self.cache.remove(key).await; + ObjectDataCacheFillResult::SkippedInvalidationRace + } + } + ObjectDataCacheIndexInsertResult::Overflow { cleared_keys } => { + for stale_key in cleared_keys { + self.cache.remove(&stale_key).await; + } + ObjectDataCacheFillResult::SkippedIdentityOverflow + } + }; + + leader.finish(result).await + } + + /// Conservatively invalidates all cached keys matching the object identity. + /// + /// The identity index is authoritative: fills register the key in the + /// index before the entry becomes visible in the cache (and undo the fill + /// if an invalidation raced in between), so no full-cache scan fallback is + /// needed when the index has no entry for the identity. + pub async fn invalidate_object(&self, identity: &ObjectDataCacheIdentity) -> ObjectDataCacheInvalidationResult { + let keys_to_remove = self.index.remove_identity(identity).await; + for key in keys_to_remove { + self.cache.remove(&key).await; + } + + ObjectDataCacheInvalidationResult::Success + } +} + +#[cfg(test)] +mod tests { + use super::MokaBackend; + use crate::cache::{ + ObjectDataCacheFillResult, ObjectDataCacheGetPlan, ObjectDataCacheInvalidationResult, ObjectDataCacheLookup, + }; + use crate::config::{ObjectDataCacheConfig, ObjectDataCacheMode}; + use crate::key::{ObjectDataCacheBodyVariant, ObjectDataCacheIdentity, ObjectDataCacheKey}; + use crate::stats::ObjectDataCacheStats; + use bytes::Bytes; + use std::sync::Arc; + use std::time::Duration; + + fn enabled_config() -> ObjectDataCacheConfig { + ObjectDataCacheConfig { + mode: ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 8_388_608, + max_memory_percent: 5, + max_entry_bytes: 1_048_576, + ttl: Duration::from_millis(100), + time_to_idle: Duration::from_millis(100), + min_free_memory_percent: 20, + fill_concurrency_per_cpu: 1, + fill_concurrency_max: 32, + identity_keys_max: 16, + } + } + + fn cacheable_plan(object: &str, etag: &str) -> ObjectDataCacheGetPlan { + ObjectDataCacheGetPlan::Cacheable { + key: ObjectDataCacheKey::new("bucket", object, None, etag, 5, ObjectDataCacheBodyVariant::FullObjectPlainV1), + } + } + + #[tokio::test] + async fn moka_backend_round_trips_cached_body() { + let backend = + MokaBackend::new(&enabled_config(), Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"); + let plan = cacheable_plan("object", "etag-a"); + + let fill = backend.fill_body(&plan, Bytes::from_static(b"hello")).await; + let lookup = backend.lookup_body(&plan).await; + + assert!(matches!(fill, ObjectDataCacheFillResult::Inserted)); + assert!(matches!(lookup, ObjectDataCacheLookup::Hit(ref bytes) if bytes.as_ref() == b"hello")); + } + + #[tokio::test] + async fn moka_backend_invalidates_matching_identity() { + let backend = + MokaBackend::new(&enabled_config(), Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"); + let plan_a = cacheable_plan("object-a", "etag-a"); + let plan_b = cacheable_plan("object-b", "etag-b"); + + let _ = backend.fill_body(&plan_a, Bytes::from_static(b"aaaaa")).await; + let _ = backend.fill_body(&plan_b, Bytes::from_static(b"bbbbb")).await; + + let result = backend + .invalidate_object(&ObjectDataCacheIdentity::new("bucket", "object-a")) + .await; + let lookup_a = backend.lookup_body(&plan_a).await; + let lookup_b = backend.lookup_body(&plan_b).await; + + assert!(matches!(result, ObjectDataCacheInvalidationResult::Success)); + assert!(matches!(lookup_a, ObjectDataCacheLookup::Miss)); + assert!(matches!(lookup_b, ObjectDataCacheLookup::Hit(_))); + } + + #[tokio::test] + async fn moka_backend_expires_entries_by_ttl() { + let backend = + MokaBackend::new(&enabled_config(), Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"); + let plan = cacheable_plan("object", "etag-a"); + + let _ = backend.fill_body(&plan, Bytes::from_static(b"hello")).await; + tokio::time::sleep(Duration::from_millis(150)).await; + + let lookup = backend.lookup_body(&plan).await; + + assert!(matches!(lookup, ObjectDataCacheLookup::Miss)); + } + + #[tokio::test] + async fn moka_backend_expires_entries_by_tti() { + let mut config = enabled_config(); + config.ttl = Duration::from_secs(30); + config.time_to_idle = Duration::from_millis(100); + let backend = MokaBackend::new(&config, Arc::new(ObjectDataCacheStats::default())).expect("moka backend should build"); + let plan = cacheable_plan("object", "etag-a"); + + let _ = backend.fill_body(&plan, Bytes::from_static(b"hello")).await; + tokio::time::sleep(Duration::from_millis(150)).await; + + let lookup = backend.lookup_body(&plan).await; + + assert!(matches!(lookup, ObjectDataCacheLookup::Miss)); + } + + #[tokio::test] + async fn moka_backend_skips_fill_under_memory_pressure() { + let stats = Arc::new(ObjectDataCacheStats::default()); + let backend = MokaBackend::new(&enabled_config(), Arc::clone(&stats)).expect("moka backend should build"); + backend + .memory_gate + .set_test_snapshot(Some(crate::memory::ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 100, + })); + let plan = cacheable_plan("object", "etag-a"); + + let result = backend.fill_body(&plan, Bytes::from_static(b"hello")).await; + let lookup = backend.lookup_body(&plan).await; + + assert_eq!(result, ObjectDataCacheFillResult::SkippedMemoryPressure); + assert!(matches!(lookup, ObjectDataCacheLookup::Miss)); + assert_eq!(stats.snapshot().memory_pressure_events, 1); + } + + #[tokio::test] + async fn moka_backend_singleflight_waiter_observes_leader_result() { + let stats = Arc::new(ObjectDataCacheStats::default()); + let backend = Arc::new(MokaBackend::new(&enabled_config(), Arc::clone(&stats)).expect("moka backend should build")); + let plan = cacheable_plan("object", "etag-a"); + let leader_plan = plan.clone(); + let plan_clone = plan.clone(); + let first = Arc::clone(&backend); + let second = Arc::clone(&backend); + + let leader = tokio::spawn(async move { first.fill_body(&leader_plan, Bytes::from_static(b"hello")).await }); + let waiter = tokio::spawn(async move { second.fill_body(&plan_clone, Bytes::from_static(b"hello")).await }); + + let leader_result = leader.await.expect("leader task should complete"); + let waiter_result = waiter.await.expect("waiter task should complete"); + let lookup = backend.lookup_body(&plan).await; + + assert_eq!(leader_result, ObjectDataCacheFillResult::Inserted); + assert_eq!(waiter_result, ObjectDataCacheFillResult::Inserted); + assert!(matches!(lookup, ObjectDataCacheLookup::Hit(ref bytes) if bytes.as_ref() == b"hello")); + } +} diff --git a/crates/object-data-cache/src/noop.rs b/crates/object-data-cache/src/noop.rs new file mode 100644 index 000000000..0905aa559 --- /dev/null +++ b/crates/object-data-cache/src/noop.rs @@ -0,0 +1,58 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::cache::{ObjectDataCacheFillResult, ObjectDataCacheGetPlan, ObjectDataCacheInvalidationResult, ObjectDataCacheLookup}; + +/// Lightweight no-op backend for the initial skeleton. +#[derive(Debug, Default)] +pub struct NoopBackend; + +impl NoopBackend { + /// Returns a disabled lookup result. + pub async fn lookup_body(&self, _plan: &ObjectDataCacheGetPlan) -> ObjectDataCacheLookup { + ObjectDataCacheLookup::SkipDisabled + } + + /// Returns a skipped fill result. + pub async fn fill_body(&self, _plan: &ObjectDataCacheGetPlan) -> ObjectDataCacheFillResult { + ObjectDataCacheFillResult::SkippedDisabled + } + + /// Returns a successful no-op invalidation result. + pub async fn invalidate_object(&self) -> ObjectDataCacheInvalidationResult { + ObjectDataCacheInvalidationResult::Success + } +} + +#[cfg(test)] +mod tests { + use super::NoopBackend; + use crate::cache::{ + ObjectDataCacheFillResult, ObjectDataCacheGetPlan, ObjectDataCacheInvalidationResult, ObjectDataCacheLookup, + }; + + #[tokio::test] + async fn noop_backend_returns_disabled_results() { + let backend = NoopBackend; + let plan = ObjectDataCacheGetPlan::Disabled; + + let lookup = backend.lookup_body(&plan).await; + let fill = backend.fill_body(&plan).await; + let invalidation = backend.invalidate_object().await; + + assert!(matches!(lookup, ObjectDataCacheLookup::SkipDisabled)); + assert!(matches!(fill, ObjectDataCacheFillResult::SkippedDisabled)); + assert!(matches!(invalidation, ObjectDataCacheInvalidationResult::Success)); + } +} diff --git a/crates/object-data-cache/src/singleflight.rs b/crates/object-data-cache/src/singleflight.rs new file mode 100644 index 000000000..d02d954ce --- /dev/null +++ b/crates/object-data-cache/src/singleflight.rs @@ -0,0 +1,198 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::cache::ObjectDataCacheFillResult; +use crate::key::ObjectDataCacheKey; +use crate::metrics::{record_singleflight_join, set_inflight_fills}; +use crate::stats::ObjectDataCacheStats; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use tokio::sync::watch; + +type FillMap = Mutex>>>; + +fn lock_fills( + fills: &FillMap, +) -> std::sync::MutexGuard<'_, HashMap>>> { + fills.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// Shared singleflight controller for cache fill operations. +#[derive(Debug)] +pub struct ObjectDataCacheSingleflight { + fills: FillMap, + stats: Arc, +} + +impl ObjectDataCacheSingleflight { + /// Creates a new singleflight controller. + pub fn new(stats: Arc) -> Self { + Self { + fills: Mutex::new(HashMap::new()), + stats, + } + } + + /// Acquires a leader-or-waiter role for the supplied cache key. + pub async fn acquire(&self, key: ObjectDataCacheKey) -> ObjectDataCacheSingleflightAcquire<'_> { + let mut fills = lock_fills(&self.fills); + if let Some(sender) = fills.get(&key) { + record_singleflight_join(&self.stats); + return ObjectDataCacheSingleflightAcquire::Waiter(ObjectDataCacheSingleflightWaiter { rx: sender.subscribe() }); + } + + let (tx, _rx) = watch::channel(None); + fills.insert(key.clone(), tx.clone()); + set_inflight_fills(&self.stats, "moka", fills.len()); + drop(fills); + + ObjectDataCacheSingleflightAcquire::Leader(ObjectDataCacheSingleflightLeader { + key, + tx, + fills: &self.fills, + stats: Arc::clone(&self.stats), + finished: false, + }) + } +} + +/// Leader or waiter result from the singleflight map. +pub enum ObjectDataCacheSingleflightAcquire<'a> { + /// Caller is responsible for performing the fill operation. + Leader(ObjectDataCacheSingleflightLeader<'a>), + /// Caller must wait for the leader's result. + Waiter(ObjectDataCacheSingleflightWaiter), +} + +/// Leader handle for a singleflight fill operation. +pub struct ObjectDataCacheSingleflightLeader<'a> { + key: ObjectDataCacheKey, + tx: watch::Sender>, + fills: &'a FillMap, + stats: Arc, + finished: bool, +} + +impl<'a> ObjectDataCacheSingleflightLeader<'a> { + /// Completes the leader operation and publishes the shared result. + pub async fn finish(mut self, result: ObjectDataCacheFillResult) -> ObjectDataCacheFillResult { + self.remove_entry(); + self.finished = true; + let _ = self.tx.send(Some(result.clone())); + result + } + + fn remove_entry(&self) { + let mut fills = lock_fills(self.fills); + fills.remove(&self.key); + set_inflight_fills(&self.stats, "moka", fills.len()); + } +} + +impl<'a> Drop for ObjectDataCacheSingleflightLeader<'a> { + fn drop(&mut self) { + // A leader dropped without finish() was cancelled mid-fill (e.g. the + // GET request future was aborted). Remove the map entry so its sender + // clone is released and waiters observe the closed channel instead of + // blocking forever, and so a later fill can become the new leader. + if !self.finished { + self.remove_entry(); + } + } +} + +/// Waiter handle for a singleflight fill operation. +pub struct ObjectDataCacheSingleflightWaiter { + rx: watch::Receiver>, +} + +impl ObjectDataCacheSingleflightWaiter { + /// Waits for the shared fill result. + pub async fn wait(mut self) -> ObjectDataCacheFillResult { + if self.rx.wait_for(|value| value.is_some()).await.is_err() { + return ObjectDataCacheFillResult::SkippedSingleflightClosed; + } + + self.rx + .borrow() + .as_ref() + .cloned() + .unwrap_or(ObjectDataCacheFillResult::SkippedSingleflightClosed) + } +} + +#[cfg(test)] +mod tests { + use super::{ObjectDataCacheSingleflight, ObjectDataCacheSingleflightAcquire}; + use crate::cache::ObjectDataCacheFillResult; + use crate::key::{ObjectDataCacheBodyVariant, ObjectDataCacheKey}; + use crate::stats::ObjectDataCacheStats; + use std::sync::Arc; + + fn key() -> ObjectDataCacheKey { + ObjectDataCacheKey::new("bucket", "object", None, "etag", 1, ObjectDataCacheBodyVariant::FullObjectPlainV1) + } + + #[tokio::test] + async fn singleflight_returns_waiter_for_second_caller() { + let stats = Arc::new(ObjectDataCacheStats::default()); + let singleflight = ObjectDataCacheSingleflight::new(Arc::clone(&stats)); + + let first = singleflight.acquire(key()).await; + let second = singleflight.acquire(key()).await; + + let leader = match first { + ObjectDataCacheSingleflightAcquire::Leader(leader) => leader, + ObjectDataCacheSingleflightAcquire::Waiter(_) => panic!("first caller must become leader"), + }; + let waiter = match second { + ObjectDataCacheSingleflightAcquire::Leader(_) => panic!("second caller must become waiter"), + ObjectDataCacheSingleflightAcquire::Waiter(waiter) => waiter, + }; + + let waiter_task = tokio::spawn(async move { waiter.wait().await }); + let leader_result = leader.finish(ObjectDataCacheFillResult::Inserted).await; + let waiter_result = waiter_task.await.expect("waiter task should complete"); + + assert_eq!(leader_result, ObjectDataCacheFillResult::Inserted); + assert_eq!(waiter_result, ObjectDataCacheFillResult::Inserted); + assert_eq!(stats.snapshot().singleflight_joins, 1); + } + + #[tokio::test] + async fn cancelled_leader_releases_key_and_unblocks_waiters() { + let stats = Arc::new(ObjectDataCacheStats::default()); + let singleflight = ObjectDataCacheSingleflight::new(Arc::clone(&stats)); + + let leader = match singleflight.acquire(key()).await { + ObjectDataCacheSingleflightAcquire::Leader(leader) => leader, + ObjectDataCacheSingleflightAcquire::Waiter(_) => panic!("first caller must become leader"), + }; + let waiter = match singleflight.acquire(key()).await { + ObjectDataCacheSingleflightAcquire::Leader(_) => panic!("second caller must become waiter"), + ObjectDataCacheSingleflightAcquire::Waiter(waiter) => waiter, + }; + + // Dropping without finish() simulates the leader future being cancelled. + drop(leader); + + let waiter_result = waiter.wait().await; + assert_eq!(waiter_result, ObjectDataCacheFillResult::SkippedSingleflightClosed); + + assert!( + matches!(singleflight.acquire(key()).await, ObjectDataCacheSingleflightAcquire::Leader(_)), + "key must be released after a cancelled leader" + ); + } +} diff --git a/crates/object-data-cache/src/starshard_index.rs b/crates/object-data-cache/src/starshard_index.rs new file mode 100644 index 000000000..f3a01e9b4 --- /dev/null +++ b/crates/object-data-cache/src/starshard_index.rs @@ -0,0 +1,215 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::index::{ObjectDataCacheIndexInsertResult, ObjectDataCacheKeySet}; +use crate::key::{ObjectDataCacheIdentity, ObjectDataCacheKey}; +use starshard::{AsyncShardedHashMap, DEFAULT_SHARDS, SnapshotMode}; +use std::collections::hash_map::RandomState; +use std::sync::Arc; + +/// Async starshard-backed identity -> keys index. +#[derive(Clone)] +pub struct StarshardIdentityIndex { + by_object: Arc>, + max_keys_per_identity: usize, +} + +impl std::fmt::Debug for StarshardIdentityIndex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StarshardIdentityIndex") + .field("max_keys_per_identity", &self.max_keys_per_identity) + .finish() + } +} + +impl StarshardIdentityIndex { + /// Creates a new identity index. + pub fn new(max_keys_per_identity: usize) -> Self { + Self { + by_object: Arc::new(AsyncShardedHashMap::with_shards_and_hasher_and_snapshot_mode( + DEFAULT_SHARDS, + RandomState::new(), + SnapshotMode::Cached, + )), + max_keys_per_identity, + } + } + + /// Inserts a key into the identity index. + /// + /// Runs the read-modify-write under the shard write lock so concurrent + /// fills for the same identity cannot drop each other's keys. + pub async fn insert(&self, identity: ObjectDataCacheIdentity, key: ObjectDataCacheKey) -> ObjectDataCacheIndexInsertResult { + let max_keys = self.max_keys_per_identity; + loop { + let mut outcome = None; + { + let outcome = &mut outcome; + let key = key.clone(); + let _ = self + .by_object + .compute_if_present(&identity, move |mut key_set| { + let result = key_set.insert(key, max_keys); + let keep = !matches!(result, ObjectDataCacheIndexInsertResult::Overflow { .. }) && !key_set.is_empty(); + *outcome = Some(result); + keep.then_some(key_set) + }) + .await; + } + if let Some(result) = outcome { + return result; + } + + // Identity not tracked yet: publish a fresh single-key set. + let mut fresh = ObjectDataCacheKeySet::default(); + let result = fresh.insert(key.clone(), max_keys); + if !matches!(result, ObjectDataCacheIndexInsertResult::Inserted) { + return result; + } + let final_set = self.by_object.compute_if_absent(identity.clone(), move || fresh).await; + if final_set.contains(&key) { + return ObjectDataCacheIndexInsertResult::Inserted; + } + // Lost the race to a concurrent insert; retry against the now + // present entry. + } + } + + /// Removes all keys tracked for an identity. + pub async fn remove_identity(&self, identity: &ObjectDataCacheIdentity) -> Vec { + self.by_object + .remove(identity) + .await + .map_or_else(Vec::new, |set| set.cloned()) + } + + /// Removes a single key tracked under an identity. + pub async fn remove_key(&self, identity: &ObjectDataCacheIdentity, key: &ObjectDataCacheKey) -> bool { + let mut removed = false; + { + let removed = &mut removed; + let _ = self + .by_object + .compute_if_present(identity, move |mut key_set| { + *removed = key_set.remove_key(key); + (!key_set.is_empty()).then_some(key_set) + }) + .await; + } + removed + } + + /// Returns whether the identity currently tracks the supplied key. + pub async fn contains_key(&self, identity: &ObjectDataCacheIdentity, key: &ObjectDataCacheKey) -> bool { + self.by_object + .get(identity) + .await + .is_some_and(|key_set| key_set.contains(key)) + } + + /// Removes index keys that no longer exist in the cache. + pub async fn prune_missing(&self, identity: &ObjectDataCacheIdentity, mut key_exists: F) + where + F: FnMut(&ObjectDataCacheKey) -> bool, + { + let _ = self + .by_object + .compute_if_present(identity, move |mut key_set| { + key_set.retain(|key| key_exists(key)); + (!key_set.is_empty()).then_some(key_set) + }) + .await; + } +} + +#[cfg(test)] +mod tests { + use super::StarshardIdentityIndex; + use crate::index::ObjectDataCacheIndexInsertResult; + use crate::key::{ObjectDataCacheBodyVariant, ObjectDataCacheIdentity, ObjectDataCacheKey}; + + fn identity() -> ObjectDataCacheIdentity { + ObjectDataCacheIdentity::new("bucket", "object") + } + + fn key(id: &str) -> ObjectDataCacheKey { + ObjectDataCacheKey::new("bucket", "object", Some(id), "etag", 1, ObjectDataCacheBodyVariant::FullObjectPlainV1) + } + + #[tokio::test] + async fn identity_index_removes_all_keys_for_identity() { + let index = StarshardIdentityIndex::new(4); + let identity = identity(); + let key_a = key("v1"); + let key_b = key("v2"); + + let _ = index.insert(identity.clone(), key_a.clone()).await; + let _ = index.insert(identity.clone(), key_b.clone()).await; + let removed = index.remove_identity(&identity).await; + + assert_eq!(removed, vec![key_a, key_b]); + } + + #[tokio::test] + async fn identity_index_prunes_stale_keys() { + let index = StarshardIdentityIndex::new(4); + let identity = identity(); + let key_a = key("v1"); + let key_b = key("v2"); + + let _ = index.insert(identity.clone(), key_a.clone()).await; + let _ = index.insert(identity.clone(), key_b.clone()).await; + index.prune_missing(&identity, |candidate| candidate == &key_b).await; + let removed = index.remove_identity(&identity).await; + + assert_eq!(removed, vec![key_b]); + } + + #[tokio::test] + async fn identity_index_concurrent_inserts_keep_all_keys() { + let index = StarshardIdentityIndex::new(64); + let identity = identity(); + + let mut handles = Vec::new(); + for i in 0..32 { + let index = index.clone(); + let identity = identity.clone(); + handles.push(tokio::spawn(async move { index.insert(identity, key(&format!("v{i}"))).await })); + } + for handle in handles { + handle.await.expect("insert task should complete"); + } + + let removed = index.remove_identity(&identity).await; + assert_eq!(removed.len(), 32, "no concurrent insert may drop another fill's key"); + } + + #[tokio::test] + async fn identity_index_overflow_clears_identity() { + let index = StarshardIdentityIndex::new(1); + let identity = identity(); + let key_a = key("v1"); + let key_b = key("v2"); + + let _ = index.insert(identity.clone(), key_a.clone()).await; + let result = index.insert(identity.clone(), key_b).await; + let removed = index.remove_identity(&identity).await; + + assert!(matches!( + result, + ObjectDataCacheIndexInsertResult::Overflow { cleared_keys } if cleared_keys == vec![key_a] + )); + assert!(removed.is_empty()); + } +} diff --git a/crates/object-data-cache/src/stats.rs b/crates/object-data-cache/src/stats.rs new file mode 100644 index 000000000..f8db51794 --- /dev/null +++ b/crates/object-data-cache/src/stats.rs @@ -0,0 +1,133 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Cache statistics holder for the engine skeleton. +#[derive(Debug, Default)] +pub struct ObjectDataCacheStats { + entries: AtomicU64, + lookups: AtomicU64, + hits: AtomicU64, + fills: AtomicU64, + invalidations: AtomicU64, + inflight_fills: AtomicU64, + singleflight_joins: AtomicU64, + memory_pressure_events: AtomicU64, +} + +/// Immutable snapshot of cache statistics. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ObjectDataCacheStatsSnapshot { + /// Number of cached entries. + pub entries: u64, + /// Total lookup attempts. + pub lookups: u64, + /// Total cache hits. + pub hits: u64, + /// Total fill attempts. + pub fills: u64, + /// Total invalidation attempts. + pub invalidations: u64, + /// Current number of in-flight fills. + pub inflight_fills: u64, + /// Number of singleflight joiners. + pub singleflight_joins: u64, + /// Number of times fill was skipped under memory pressure. + pub memory_pressure_events: u64, +} + +impl ObjectDataCacheStats { + /// Updates the current entry count snapshot. + pub fn set_entries(&self, entries: u64) { + self.entries.store(entries, Ordering::Relaxed); + } + + /// Records a cache lookup attempt. + pub fn record_lookup(&self, hit: bool) { + self.lookups.fetch_add(1, Ordering::Relaxed); + if hit { + self.hits.fetch_add(1, Ordering::Relaxed); + } + } + + /// Records a cache fill attempt. + pub fn record_fill(&self) { + self.fills.fetch_add(1, Ordering::Relaxed); + } + + /// Records a cache invalidation attempt. + pub fn record_invalidation(&self) { + self.invalidations.fetch_add(1, Ordering::Relaxed); + } + + /// Sets the current number of in-flight fills. + pub fn set_inflight_fills(&self, inflight_fills: usize) { + let value = u64::try_from(inflight_fills).unwrap_or(u64::MAX); + self.inflight_fills.store(value, Ordering::Relaxed); + } + + /// Records a singleflight join. + pub fn record_singleflight_join(&self) { + self.singleflight_joins.fetch_add(1, Ordering::Relaxed); + } + + /// Records a memory pressure event. + pub fn record_memory_pressure(&self) { + self.memory_pressure_events.fetch_add(1, Ordering::Relaxed); + } + + /// Returns the current immutable stats snapshot. + pub fn snapshot(&self) -> ObjectDataCacheStatsSnapshot { + ObjectDataCacheStatsSnapshot { + entries: self.entries.load(Ordering::Relaxed), + lookups: self.lookups.load(Ordering::Relaxed), + hits: self.hits.load(Ordering::Relaxed), + fills: self.fills.load(Ordering::Relaxed), + invalidations: self.invalidations.load(Ordering::Relaxed), + inflight_fills: self.inflight_fills.load(Ordering::Relaxed), + singleflight_joins: self.singleflight_joins.load(Ordering::Relaxed), + memory_pressure_events: self.memory_pressure_events.load(Ordering::Relaxed), + } + } +} + +#[cfg(test)] +mod tests { + use super::ObjectDataCacheStats; + + #[test] + fn snapshot_reflects_recorded_counters() { + let stats = ObjectDataCacheStats::default(); + stats.set_entries(3); + stats.record_lookup(true); + stats.record_lookup(false); + stats.record_fill(); + stats.record_invalidation(); + stats.set_inflight_fills(2); + stats.record_singleflight_join(); + stats.record_memory_pressure(); + + let snapshot = stats.snapshot(); + + assert_eq!(snapshot.entries, 3); + assert_eq!(snapshot.lookups, 2); + assert_eq!(snapshot.hits, 1); + assert_eq!(snapshot.fills, 1); + assert_eq!(snapshot.invalidations, 1); + assert_eq!(snapshot.inflight_fills, 2); + assert_eq!(snapshot.singleflight_joins, 1); + assert_eq!(snapshot.memory_pressure_events, 1); + } +} diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 4f6a3ae0e..7452989b9 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -102,6 +102,7 @@ rustfs-zip = { workspace = true } rustfs-io-core = { workspace = true } rustfs-io-metrics = { workspace = true } rustfs-object-capacity = { workspace = true } +rustfs-object-data-cache = { workspace = true } rustfs-concurrency = { workspace = true } rustfs-scanner = { workspace = true } tempfile = { workspace = true } diff --git a/rustfs/src/app/context.rs b/rustfs/src/app/context.rs index 1d13a9c7f..0d5688c0b 100644 --- a/rustfs/src/app/context.rs +++ b/rustfs/src/app/context.rs @@ -32,6 +32,7 @@ use super::storage_api::context::runtime::{ ScannerMetricsReport, StorageClassConfig, TierConfigMgr, TransitionState, }; use super::storage_api::context::{ECStore, EndpointServerPools}; +use crate::app::object_data_cache::ObjectDataCacheAdapter; use crate::config::RustFSBufferConfig; use rustfs_config::server_config::Config; use rustfs_credentials::Credentials; @@ -125,6 +126,21 @@ pub fn resolve_object_store_handle_for_context(context: Option<&AppContext>) -> context.map(|context| context.object_store()) } +/// Resolve object data cache adapter using AppContext-first precedence. +#[expect( + dead_code, + reason = "ST-05 exposes the global resolver; app use sites start in later cache phases" +)] +pub(crate) fn resolve_object_data_cache_handle() -> Option> { + let context = get_global_app_context(); + resolve_object_data_cache_handle_for_context(context.as_deref()) +} + +/// Resolve object data cache adapter using an explicit AppContext. +pub(crate) fn resolve_object_data_cache_handle_for_context(context: Option<&AppContext>) -> Option> { + context.map(|context| context.object_data_cache()) +} + /// Resolve notify interface using AppContext-first precedence. pub fn resolve_notify_interface() -> Option> { let context = get_global_app_context(); diff --git a/rustfs/src/app/context/global.rs b/rustfs/src/app/context/global.rs index b5f334e4d..da703b664 100644 --- a/rustfs/src/app/context/global.rs +++ b/rustfs/src/app/context/global.rs @@ -33,6 +33,7 @@ use super::interfaces::{ ReplicationPoolInterface, ReplicationStatsInterface, RuntimePortInterface, S3SelectDbInterface, ScannerMetricsInterface, ServerConfigInterface, StorageClassInterface, TierConfigInterface, TransitionStateInterface, }; +use crate::app::object_data_cache::ObjectDataCacheAdapter; use rustfs_iam::{oidc::OidcSys, store::object::ObjectStore, sys::IamSys}; use rustfs_kms::KmsServiceManager; use std::sync::{Arc, OnceLock}; @@ -72,10 +73,13 @@ pub struct AppContext { server_config: Arc, storage_class: Arc, buffer_config: Arc, + object_data_cache: Arc, } impl AppContext { pub fn new(object_store: Arc, iam: Arc, kms: Arc) -> Self { + let object_data_cache = ObjectDataCacheAdapter::from_env_or_disabled(); + Self { object_store, iam, @@ -108,6 +112,7 @@ impl AppContext { server_config: default_server_config_interface(), storage_class: default_storage_class_interface(), buffer_config: default_buffer_config_interface(), + object_data_cache, } } @@ -249,6 +254,10 @@ impl AppContext { pub fn buffer_config(&self) -> Arc { self.buffer_config.clone() } + + pub(crate) fn object_data_cache(&self) -> Arc { + Arc::clone(&self.object_data_cache) + } } #[cfg(test)] @@ -320,6 +329,7 @@ impl AppContext { server_config: interfaces.server_config, storage_class: interfaces.storage_class, buffer_config: interfaces.buffer_config, + object_data_cache: ObjectDataCacheAdapter::disabled_arc(), } } } diff --git a/rustfs/src/app/context/handles.rs b/rustfs/src/app/context/handles.rs index d2f95fa4b..addfd9ae8 100644 --- a/rustfs/src/app/context/handles.rs +++ b/rustfs/src/app/context/handles.rs @@ -432,6 +432,10 @@ pub fn default_notify_interface() -> Arc { Arc::new(NotifyHandle) } +pub(crate) fn default_object_data_cache_handle() -> Arc { + crate::app::object_data_cache::ObjectDataCacheAdapter::disabled_arc() +} + pub fn default_notification_system_interface() -> Arc { Arc::new(NotificationSystemHandle) } diff --git a/rustfs/src/app/mod.rs b/rustfs/src/app/mod.rs index 8f282ea64..dcfd01a51 100644 --- a/rustfs/src/app/mod.rs +++ b/rustfs/src/app/mod.rs @@ -19,6 +19,7 @@ pub mod admin_usecase; pub mod bucket_usecase; pub mod context; pub mod multipart_usecase; +pub(crate) mod object_data_cache; pub mod object_usecase; pub(crate) mod runtime_sources; mod select_object; diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 3c130ae4c..a86314471 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -55,8 +55,14 @@ use super::storage_api::multipart_usecase::sse::{ mark_encrypted_multipart_metadata, sse_decryption, sse_prepare_encryption, }; use super::storage_api::multipart_usecase::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader}; +use crate::app::object_data_cache::{ + ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success, + invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_before_mutation, +}; use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write}; -use crate::app::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context}; +use crate::app::runtime_sources::{ + AppContext, current_app_context, current_object_data_cache_for_context, current_object_store_handle_for_context, +}; use crate::capacity::record_capacity_write; use crate::error::ApiError; use crate::table_catalog; @@ -297,6 +303,10 @@ impl DefaultMultipartUsecase { current_object_store_handle_for_context(self.context.as_deref()) } + fn object_data_cache(&self) -> Arc { + current_object_data_cache_for_context(self.context.as_deref()) + } + #[instrument(level = "debug", skip(self))] pub async fn execute_abort_multipart_upload( &self, @@ -447,6 +457,8 @@ impl DefaultMultipartUsecase { .get_multipart_info(&bucket, &key, &upload_id, &ObjectOptions::default()) .await .map_err(ApiError::from)?; + let cache_adapter = self.object_data_cache(); + let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await; let server_side_encryption = multipart_info .user_defined @@ -466,6 +478,7 @@ impl DefaultMultipartUsecase { .complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, &opts) .await .map_err(ApiError::from)?; + let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await; record_capacity_write(Some(capacity_scope_token)).await; // check quota after completing multipart upload @@ -480,6 +493,7 @@ impl DefaultMultipartUsecase { if !check_result.allowed { // Quota exceeded, delete the completed object let _ = store.delete_object(&bucket, &key, ObjectOptions::default()).await; + let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await; return Err(S3Error::with_message( S3ErrorCode::InvalidRequest, format!( diff --git a/rustfs/src/app/object_data_cache/adapter.rs b/rustfs/src/app/object_data_cache/adapter.rs new file mode 100644 index 000000000..14d2ad415 --- /dev/null +++ b/rustfs/src/app/object_data_cache/adapter.rs @@ -0,0 +1,345 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bytes::Bytes; +use rustfs_object_data_cache::{ + ObjectDataCache, ObjectDataCacheConfig, ObjectDataCacheConfigError, ObjectDataCacheFillResult, ObjectDataCacheGetPlan, + ObjectDataCacheGetRequest, ObjectDataCacheIdentity, ObjectDataCacheInvalidationReason, ObjectDataCacheInvalidationResult, + ObjectDataCacheLookup, ObjectDataCacheMode, +}; +use std::{sync::Arc, time::Duration}; +use tracing::warn; + +#[derive(Debug, Default)] +struct ObjectDataCacheEnvValues { + enabled: Option, + mode: Option, + max_bytes: Option, + max_memory_percent: Option, + max_entry_bytes: Option, + ttl_secs: Option, + time_to_idle_secs: Option, + min_free_memory_percent: Option, + fill_concurrency_per_cpu: Option, + fill_concurrency_max: Option, + identity_keys_max: Option, +} + +/// App-layer wrapper around the engine-only object data cache. +#[derive(Debug, Clone)] +pub(crate) struct ObjectDataCacheAdapter { + cache: Arc, +} + +impl ObjectDataCacheAdapter { + /// Creates an adapter from validated cache configuration. + pub(crate) fn new(config: ObjectDataCacheConfig) -> Result { + Ok(Self { + cache: Arc::new(ObjectDataCache::new(config)?), + }) + } + + /// Creates an adapter from runtime environment variables, falling back to + /// no-op on invalid input so startup behavior stays conservative. + pub(crate) fn from_env_or_disabled() -> Arc { + Self::from_config_or_disabled(object_data_cache_config_from_env(), "env") + } + + /// Creates a disabled no-op adapter. + pub(crate) fn disabled() -> Self { + Self { + cache: Arc::new(ObjectDataCache::disabled()), + } + } + + /// Creates a shared disabled no-op adapter. + pub(crate) fn disabled_arc() -> Arc { + Arc::new(Self::disabled()) + } + + /// Returns the underlying shared cache handle. + #[cfg(test)] + pub(crate) fn cache(&self) -> Arc { + Arc::clone(&self.cache) + } + + /// Returns true when the adapter is wired to a disabled cache. + pub(crate) fn is_disabled(&self) -> bool { + self.cache.is_disabled() + } + + /// Returns true when the adapter allows materialize fill. + pub(crate) fn materialize_fill_enabled(&self) -> bool { + self.cache.materialize_fill_enabled() + } + + /// Builds an engine-level GET plan. + pub(crate) fn plan_get(&self, request: ObjectDataCacheGetRequest<'_>) -> ObjectDataCacheGetPlan { + self.cache.plan_get(request) + } + + /// Executes an engine-level cache lookup. + pub(crate) async fn lookup_body(&self, plan: &ObjectDataCacheGetPlan) -> ObjectDataCacheLookup { + self.cache.lookup_body(plan).await + } + + /// Executes an engine-level cache fill. + pub(crate) async fn fill_body(&self, plan: &ObjectDataCacheGetPlan, bytes: Bytes) -> ObjectDataCacheFillResult { + self.cache.fill_body(plan, bytes).await + } + + /// Executes an engine-level object invalidation. + pub(crate) async fn invalidate_object( + &self, + identity: ObjectDataCacheIdentity, + reason: ObjectDataCacheInvalidationReason, + ) -> ObjectDataCacheInvalidationResult { + self.cache.invalidate_object(identity, reason).await + } +} + +impl ObjectDataCacheAdapter { + fn from_config_or_disabled(config: ObjectDataCacheConfig, source: &str) -> Arc { + match Self::new(config) { + Ok(adapter) => Arc::new(adapter), + Err(err) => { + warn!( + error = %err, + source, + "object data cache disabled because configuration is invalid" + ); + Self::disabled_arc() + } + } + } +} + +impl Default for ObjectDataCacheAdapter { + fn default() -> Self { + Self::disabled() + } +} + +fn object_data_cache_config_from_env() -> ObjectDataCacheConfig { + object_data_cache_config_from_values(ObjectDataCacheEnvValues { + enabled: rustfs_utils::get_env_opt_bool(rustfs_config::ENV_OBJECT_DATA_CACHE_ENABLE), + mode: rustfs_utils::get_env_opt_str(rustfs_config::ENV_OBJECT_DATA_CACHE_MODE), + max_bytes: rustfs_utils::get_env_opt_u64(rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_BYTES), + max_memory_percent: rustfs_utils::get_env_opt_u8(rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_MEMORY_PERCENT), + max_entry_bytes: rustfs_utils::get_env_opt_u64(rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_ENTRY_BYTES), + ttl_secs: rustfs_utils::get_env_opt_u64(rustfs_config::ENV_OBJECT_DATA_CACHE_TTL_SECS), + time_to_idle_secs: rustfs_utils::get_env_opt_u64(rustfs_config::ENV_OBJECT_DATA_CACHE_TIME_TO_IDLE_SECS), + min_free_memory_percent: rustfs_utils::get_env_opt_u8(rustfs_config::ENV_OBJECT_DATA_CACHE_MIN_FREE_MEMORY_PERCENT), + fill_concurrency_per_cpu: rustfs_utils::get_env_opt_u16(rustfs_config::ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_PER_CPU), + fill_concurrency_max: rustfs_utils::get_env_opt_u16(rustfs_config::ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_MAX), + identity_keys_max: rustfs_utils::get_env_opt_u16(rustfs_config::ENV_OBJECT_DATA_CACHE_IDENTITY_KEYS_MAX), + }) +} + +fn object_data_cache_config_from_values(values: ObjectDataCacheEnvValues) -> ObjectDataCacheConfig { + let mut config = ObjectDataCacheConfig::default(); + + let mode_explicit = values.mode.is_some(); + if let Some(mode) = values.mode { + config.mode = parse_object_data_cache_mode(&mode).unwrap_or_else(|| { + warn!( + value = %mode, + supported = "disabled,hit_only,fill_buffered_only,fill_materialize_enabled", + "invalid object data cache mode; cache remains disabled" + ); + ObjectDataCacheMode::Disabled + }); + } + match values.enabled { + Some(false) => { + config.mode = ObjectDataCacheMode::Disabled; + } + Some(true) if !mode_explicit => { + // Enable flag without an explicit mode: start at the safest + // enabled stage instead of silently staying disabled. + config.mode = ObjectDataCacheMode::HitOnly; + } + _ => {} + } + + if let Some(max_bytes) = values.max_bytes { + config.max_bytes = max_bytes; + } + if let Some(max_memory_percent) = values.max_memory_percent { + config.max_memory_percent = max_memory_percent; + } + if let Some(max_entry_bytes) = values.max_entry_bytes { + config.max_entry_bytes = max_entry_bytes; + } + if let Some(ttl_secs) = values.ttl_secs { + config.ttl = Duration::from_secs(ttl_secs); + } + if let Some(time_to_idle_secs) = values.time_to_idle_secs { + config.time_to_idle = Duration::from_secs(time_to_idle_secs); + } + if let Some(min_free_memory_percent) = values.min_free_memory_percent { + config.min_free_memory_percent = min_free_memory_percent; + } + if let Some(fill_concurrency_per_cpu) = values.fill_concurrency_per_cpu { + config.fill_concurrency_per_cpu = fill_concurrency_per_cpu; + } + if let Some(fill_concurrency_max) = values.fill_concurrency_max { + config.fill_concurrency_max = fill_concurrency_max; + } + if let Some(identity_keys_max) = values.identity_keys_max { + config.identity_keys_max = identity_keys_max; + } + + config +} + +fn parse_object_data_cache_mode(value: &str) -> Option { + match value.trim().to_ascii_lowercase().replace('-', "_").as_str() { + "disabled" | "off" | "none" => Some(ObjectDataCacheMode::Disabled), + "hit_only" => Some(ObjectDataCacheMode::HitOnly), + "fill_buffered_only" => Some(ObjectDataCacheMode::FillBufferedOnly), + "fill_materialize_enabled" => Some(ObjectDataCacheMode::FillMaterializeEnabled), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::{ + ObjectDataCacheAdapter, ObjectDataCacheEnvValues, object_data_cache_config_from_values, parse_object_data_cache_mode, + }; + use rustfs_object_data_cache::{ObjectDataCacheConfig, ObjectDataCacheMode}; + use std::time::Duration; + + #[test] + fn disabled_adapter_exposes_disabled_engine() { + let adapter = ObjectDataCacheAdapter::disabled(); + + assert!(adapter.is_disabled()); + assert!(adapter.cache().is_disabled()); + } + + #[test] + fn adapter_new_accepts_default_disabled_config() { + let adapter = ObjectDataCacheAdapter::new(ObjectDataCacheConfig::default()).expect("disabled config should initialize"); + + assert!(adapter.is_disabled()); + } + + #[test] + fn object_data_cache_mode_parser_accepts_supported_modes() { + assert_eq!(parse_object_data_cache_mode("disabled"), Some(ObjectDataCacheMode::Disabled)); + assert_eq!(parse_object_data_cache_mode("hit-only"), Some(ObjectDataCacheMode::HitOnly)); + assert_eq!( + parse_object_data_cache_mode("fill_buffered_only"), + Some(ObjectDataCacheMode::FillBufferedOnly) + ); + assert_eq!( + parse_object_data_cache_mode("fill-materialize-enabled"), + Some(ObjectDataCacheMode::FillMaterializeEnabled) + ); + } + + #[test] + fn object_data_cache_env_defaults_stay_disabled() { + let config = object_data_cache_config_from_values(ObjectDataCacheEnvValues::default()); + + assert_eq!(config, ObjectDataCacheConfig::default()); + } + + #[test] + fn object_data_cache_env_values_override_defaults() { + let config = object_data_cache_config_from_values(ObjectDataCacheEnvValues { + enabled: Some(true), + mode: Some("fill_materialize_enabled".to_string()), + max_bytes: Some(8_388_608), + max_memory_percent: Some(10), + max_entry_bytes: Some(2_097_152), + ttl_secs: Some(120), + time_to_idle_secs: Some(45), + min_free_memory_percent: Some(30), + fill_concurrency_per_cpu: Some(2), + fill_concurrency_max: Some(64), + identity_keys_max: Some(32), + }); + + assert_eq!(config.mode, ObjectDataCacheMode::FillMaterializeEnabled); + assert_eq!(config.max_bytes, 8_388_608); + assert_eq!(config.max_memory_percent, 10); + assert_eq!(config.max_entry_bytes, 2_097_152); + assert_eq!(config.ttl, Duration::from_secs(120)); + assert_eq!(config.time_to_idle, Duration::from_secs(45)); + assert_eq!(config.min_free_memory_percent, 30); + assert_eq!(config.fill_concurrency_per_cpu, 2); + assert_eq!(config.fill_concurrency_max, 64); + assert_eq!(config.identity_keys_max, 32); + } + + #[test] + fn invalid_object_data_cache_mode_falls_back_to_disabled() { + let config = object_data_cache_config_from_values(ObjectDataCacheEnvValues { + mode: Some("enabled".to_string()), + ..ObjectDataCacheEnvValues::default() + }); + + assert_eq!(config.mode, ObjectDataCacheMode::Disabled); + } + + #[test] + fn object_data_cache_enable_false_overrides_mode() { + let config = object_data_cache_config_from_values(ObjectDataCacheEnvValues { + enabled: Some(false), + mode: Some("fill_materialize_enabled".to_string()), + ..ObjectDataCacheEnvValues::default() + }); + + assert_eq!(config.mode, ObjectDataCacheMode::Disabled); + } + + #[test] + fn object_data_cache_enable_true_without_mode_defaults_to_hit_only() { + let config = object_data_cache_config_from_values(ObjectDataCacheEnvValues { + enabled: Some(true), + ..ObjectDataCacheEnvValues::default() + }); + + assert_eq!(config.mode, ObjectDataCacheMode::HitOnly); + } + + #[test] + fn object_data_cache_explicit_mode_wins_over_enable_true() { + let config = object_data_cache_config_from_values(ObjectDataCacheEnvValues { + enabled: Some(true), + mode: Some("disabled".to_string()), + ..ObjectDataCacheEnvValues::default() + }); + + assert_eq!(config.mode, ObjectDataCacheMode::Disabled); + } + + #[test] + fn invalid_object_data_cache_config_builds_disabled_adapter() { + let adapter = ObjectDataCacheAdapter::from_config_or_disabled( + ObjectDataCacheConfig { + mode: ObjectDataCacheMode::FillBufferedOnly, + fill_concurrency_per_cpu: 2, + fill_concurrency_max: 1, + ..ObjectDataCacheConfig::default() + }, + "test", + ); + + assert!(adapter.is_disabled()); + } +} diff --git a/rustfs/src/app/object_data_cache/body.rs b/rustfs/src/app/object_data_cache/body.rs new file mode 100644 index 000000000..70689e70f --- /dev/null +++ b/rustfs/src/app/object_data_cache/body.rs @@ -0,0 +1,235 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Body handoff glue for the object data cache adapter. + +use crate::app::object_data_cache::ObjectDataCacheAdapter; +use crate::app::object_data_cache::planner::GetObjectBodyCachePlan; +use bytes::Bytes; +use rustfs_object_data_cache::{ObjectDataCacheFillResult, ObjectDataCacheLookup}; + +/// Result of an app-layer GET body cache lookup. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum GetObjectBodyCacheLookup { + Disabled, + Skip, + Miss, + Hit(Bytes), +} + +/// Attempts a conservative cache lookup for the GET response body. +pub(crate) async fn lookup_get_object_body_cache_hit( + adapter: &ObjectDataCacheAdapter, + plan: &GetObjectBodyCachePlan, +) -> GetObjectBodyCacheLookup { + match plan { + GetObjectBodyCachePlan::Disabled => GetObjectBodyCacheLookup::Disabled, + GetObjectBodyCachePlan::Skip => GetObjectBodyCacheLookup::Skip, + GetObjectBodyCachePlan::Cacheable(plan) => match adapter.lookup_body(plan).await { + ObjectDataCacheLookup::Hit(bytes) => GetObjectBodyCacheLookup::Hit(bytes), + ObjectDataCacheLookup::Miss => GetObjectBodyCacheLookup::Miss, + ObjectDataCacheLookup::SkipDisabled => GetObjectBodyCacheLookup::Disabled, + ObjectDataCacheLookup::SkipNotCacheable => GetObjectBodyCacheLookup::Skip, + }, + } +} + +/// Attempts a conservative cache fill from an already buffered full object body. +pub(crate) async fn fill_get_object_body_cache_from_buffered_body( + adapter: &ObjectDataCacheAdapter, + plan: &GetObjectBodyCachePlan, + buffered_body: &Bytes, +) -> ObjectDataCacheFillResult { + fill_get_object_body_cache_from_bytes(adapter, plan, buffered_body).await +} + +/// Attempts a conservative cache fill from an already materialized full object body. +pub(crate) async fn fill_get_object_body_cache_from_materialized_body( + adapter: &ObjectDataCacheAdapter, + plan: &GetObjectBodyCachePlan, + materialized_body: &Bytes, +) -> ObjectDataCacheFillResult { + fill_get_object_body_cache_from_bytes(adapter, plan, materialized_body).await +} + +async fn fill_get_object_body_cache_from_bytes( + adapter: &ObjectDataCacheAdapter, + plan: &GetObjectBodyCachePlan, + body: &Bytes, +) -> ObjectDataCacheFillResult { + match plan { + GetObjectBodyCachePlan::Disabled => ObjectDataCacheFillResult::SkippedDisabled, + GetObjectBodyCachePlan::Skip => ObjectDataCacheFillResult::SkippedNotCacheable, + GetObjectBodyCachePlan::Cacheable(plan) => adapter.fill_body(plan, body.clone()).await, + } +} + +#[cfg(test)] +mod tests { + use super::{ + GetObjectBodyCacheLookup, fill_get_object_body_cache_from_buffered_body, + fill_get_object_body_cache_from_materialized_body, lookup_get_object_body_cache_hit, + }; + use crate::app::object_data_cache::ObjectDataCacheAdapter; + use crate::app::object_data_cache::planner::{GetObjectBodyCacheRequest, build_get_object_body_cache_plan}; + use bytes::Bytes; + use rustfs_object_data_cache::{ + ObjectDataCacheBodyVariant, ObjectDataCacheConfig, ObjectDataCacheFillResult, ObjectDataCacheMode, + }; + + fn enabled_fill_adapter() -> ObjectDataCacheAdapter { + let config = ObjectDataCacheConfig { + mode: ObjectDataCacheMode::FillBufferedOnly, + max_bytes: 8_388_608, + ..ObjectDataCacheConfig::default() + }; + ObjectDataCacheAdapter::new(config).expect("fill-enabled config should build adapter") + } + + #[tokio::test] + async fn body_lookup_returns_hit_when_cache_contains_matching_body() { + let adapter = enabled_fill_adapter(); + let info = crate::storage::storage_api::StorageObjectInfo { + etag: Some("etag".to_string()), + size: 5, + ..Default::default() + }; + let request = GetObjectBodyCacheRequest { + bucket: "bucket", + key: "object", + info: &info, + response_content_length: 5, + has_range: false, + part_number: None, + encryption_applied: false, + }; + let cache_plan = build_get_object_body_cache_plan(&adapter, request); + + let plan = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "bucket", + object: "object", + version_id: None, + etag: "etag", + size: 5, + body_variant: ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + let fill = adapter.cache().fill_body(&plan, Bytes::from_static(b"hello")).await; + let lookup = lookup_get_object_body_cache_hit(&adapter, &cache_plan).await; + + assert_eq!(fill, ObjectDataCacheFillResult::Inserted); + assert!(matches!(lookup, GetObjectBodyCacheLookup::Hit(ref bytes) if bytes.as_ref() == b"hello")); + } + + #[tokio::test] + async fn body_lookup_returns_skip_for_part_requests() { + let adapter = enabled_fill_adapter(); + let info = crate::storage::storage_api::StorageObjectInfo { + etag: Some("etag".to_string()), + size: 5, + ..Default::default() + }; + + let request = GetObjectBodyCacheRequest { + bucket: "bucket", + key: "object", + info: &info, + response_content_length: 5, + has_range: false, + part_number: Some(1), + encryption_applied: false, + }; + let cache_plan = build_get_object_body_cache_plan(&adapter, request); + let lookup = lookup_get_object_body_cache_hit(&adapter, &cache_plan).await; + + assert!(matches!(lookup, GetObjectBodyCacheLookup::Skip)); + } + + #[tokio::test] + async fn buffered_body_fill_populates_cache_for_later_hit() { + let adapter = enabled_fill_adapter(); + let info = crate::storage::storage_api::StorageObjectInfo { + etag: Some("etag".to_string()), + size: 5, + ..Default::default() + }; + let request = GetObjectBodyCacheRequest { + bucket: "bucket", + key: "object", + info: &info, + response_content_length: 5, + has_range: false, + part_number: None, + encryption_applied: false, + }; + let cache_plan = build_get_object_body_cache_plan(&adapter, request); + + let fill = fill_get_object_body_cache_from_buffered_body(&adapter, &cache_plan, &Bytes::from_static(b"hello")).await; + let lookup = lookup_get_object_body_cache_hit(&adapter, &cache_plan).await; + + assert_eq!(fill, ObjectDataCacheFillResult::Inserted); + assert!(matches!(lookup, GetObjectBodyCacheLookup::Hit(ref bytes) if bytes.as_ref() == b"hello")); + } + + #[tokio::test] + async fn buffered_body_fill_rejects_size_mismatch() { + let adapter = enabled_fill_adapter(); + let info = crate::storage::storage_api::StorageObjectInfo { + etag: Some("etag".to_string()), + size: 5, + ..Default::default() + }; + let request = GetObjectBodyCacheRequest { + bucket: "bucket", + key: "object", + info: &info, + response_content_length: 5, + has_range: false, + part_number: None, + encryption_applied: false, + }; + let cache_plan = build_get_object_body_cache_plan(&adapter, request); + + let fill = fill_get_object_body_cache_from_buffered_body(&adapter, &cache_plan, &Bytes::from_static(b"oops")).await; + let lookup = lookup_get_object_body_cache_hit(&adapter, &cache_plan).await; + + assert_eq!(fill, ObjectDataCacheFillResult::SkippedSizeMismatch); + assert!(matches!(lookup, GetObjectBodyCacheLookup::Miss)); + } + + #[tokio::test] + async fn materialized_body_fill_populates_cache_for_later_hit() { + let adapter = enabled_fill_adapter(); + let info = crate::storage::storage_api::StorageObjectInfo { + etag: Some("etag".to_string()), + size: 5, + ..Default::default() + }; + let request = GetObjectBodyCacheRequest { + bucket: "bucket", + key: "object", + info: &info, + response_content_length: 5, + has_range: false, + part_number: None, + encryption_applied: false, + }; + let cache_plan = build_get_object_body_cache_plan(&adapter, request); + + let fill = fill_get_object_body_cache_from_materialized_body(&adapter, &cache_plan, &Bytes::from_static(b"hello")).await; + let lookup = lookup_get_object_body_cache_hit(&adapter, &cache_plan).await; + + assert_eq!(fill, ObjectDataCacheFillResult::Inserted); + assert!(matches!(lookup, GetObjectBodyCacheLookup::Hit(ref bytes) if bytes.as_ref() == b"hello")); + } +} diff --git a/rustfs/src/app/object_data_cache/invalidation.rs b/rustfs/src/app/object_data_cache/invalidation.rs new file mode 100644 index 000000000..0ea479053 --- /dev/null +++ b/rustfs/src/app/object_data_cache/invalidation.rs @@ -0,0 +1,185 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Invalidation glue for the object data cache adapter. + +use crate::app::object_data_cache::ObjectDataCacheAdapter; +use rustfs_object_data_cache::{ObjectDataCacheIdentity, ObjectDataCacheInvalidationReason, ObjectDataCacheInvalidationResult}; + +/// Invalidates a single object before a mutating operation begins. +pub(crate) async fn invalidate_object_data_cache_before_mutation( + adapter: &ObjectDataCacheAdapter, + bucket: &str, + key: &str, +) -> ObjectDataCacheInvalidationResult { + invalidate_object_data_cache_object(adapter, bucket, key, ObjectDataCacheInvalidationReason::BeforeMutation).await +} + +/// Invalidates a single object after a successful `PutObject`. +pub(crate) async fn invalidate_object_data_cache_after_put_success( + adapter: &ObjectDataCacheAdapter, + bucket: &str, + key: &str, +) -> ObjectDataCacheInvalidationResult { + invalidate_object_data_cache_object(adapter, bucket, key, ObjectDataCacheInvalidationReason::AfterPutSuccess).await +} + +/// Invalidates a single object after a successful delete. +pub(crate) async fn invalidate_object_data_cache_after_delete_success( + adapter: &ObjectDataCacheAdapter, + bucket: &str, + key: &str, +) -> ObjectDataCacheInvalidationResult { + invalidate_object_data_cache_object(adapter, bucket, key, ObjectDataCacheInvalidationReason::AfterDeleteSuccess).await +} + +/// Invalidates a single object after a successful copy destination write. +pub(crate) async fn invalidate_object_data_cache_after_copy_success( + adapter: &ObjectDataCacheAdapter, + bucket: &str, + key: &str, +) -> ObjectDataCacheInvalidationResult { + invalidate_object_data_cache_object(adapter, bucket, key, ObjectDataCacheInvalidationReason::AfterCopySuccess).await +} + +/// Invalidates a single object after a successful multipart completion. +pub(crate) async fn invalidate_object_data_cache_after_complete_multipart_success( + adapter: &ObjectDataCacheAdapter, + bucket: &str, + key: &str, +) -> ObjectDataCacheInvalidationResult { + invalidate_object_data_cache_object(adapter, bucket, key, ObjectDataCacheInvalidationReason::AfterCompleteMultipartSuccess) + .await +} + +/// Invalidates a single object identity through the app-layer cache adapter. +pub(crate) async fn invalidate_object_data_cache_object( + adapter: &ObjectDataCacheAdapter, + bucket: &str, + key: &str, + reason: ObjectDataCacheInvalidationReason, +) -> ObjectDataCacheInvalidationResult { + adapter + .invalidate_object(ObjectDataCacheIdentity::new(bucket, key), reason) + .await +} + +/// Invalidates many object identities before a mutating operation begins. +pub(crate) async fn invalidate_object_data_cache_objects_before_mutation<'a, I>( + adapter: &ObjectDataCacheAdapter, + bucket: &str, + keys: I, +) where + I: IntoIterator, +{ + invalidate_object_data_cache_objects(adapter, bucket, keys, ObjectDataCacheInvalidationReason::BeforeMutation).await; +} + +/// Invalidates many object identities after successful deletes. +pub(crate) async fn invalidate_object_data_cache_objects_after_delete_success<'a, I>( + adapter: &ObjectDataCacheAdapter, + bucket: &str, + keys: I, +) where + I: IntoIterator, +{ + invalidate_object_data_cache_objects(adapter, bucket, keys, ObjectDataCacheInvalidationReason::AfterDeleteSuccess).await; +} + +/// Invalidates many object identities through the app-layer cache adapter. +pub(crate) async fn invalidate_object_data_cache_objects<'a, I>( + adapter: &ObjectDataCacheAdapter, + bucket: &str, + keys: I, + reason: ObjectDataCacheInvalidationReason, +) where + I: IntoIterator, +{ + for key in keys { + let _ = invalidate_object_data_cache_object(adapter, bucket, key.as_str(), reason).await; + } +} + +#[cfg(test)] +mod tests { + use super::{ + invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_before_mutation, + invalidate_object_data_cache_objects_before_mutation, + }; + use crate::app::object_data_cache::ObjectDataCacheAdapter; + use bytes::Bytes; + use rustfs_object_data_cache::{ + ObjectDataCacheBodyVariant, ObjectDataCacheConfig, ObjectDataCacheFillResult, ObjectDataCacheInvalidationResult, + ObjectDataCacheLookup, ObjectDataCacheMode, + }; + + fn enabled_adapter() -> ObjectDataCacheAdapter { + let config = ObjectDataCacheConfig { + mode: ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 8_388_608, + ..ObjectDataCacheConfig::default() + }; + ObjectDataCacheAdapter::new(config).expect("enabled config should build adapter") + } + + #[tokio::test] + async fn single_object_invalidation_removes_cached_body() { + let adapter = enabled_adapter(); + let plan = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "bucket", + object: "object", + version_id: None, + etag: "etag", + size: 5, + body_variant: ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + let fill = adapter.fill_body(&plan, Bytes::from_static(b"hello")).await; + let _ = invalidate_object_data_cache_before_mutation(&adapter, "bucket", "object").await; + let invalidation = invalidate_object_data_cache_after_delete_success(&adapter, "bucket", "object").await; + let lookup = adapter.lookup_body(&plan).await; + + assert_eq!(fill, ObjectDataCacheFillResult::Inserted); + assert_eq!(invalidation, ObjectDataCacheInvalidationResult::Success); + assert!(matches!(lookup, ObjectDataCacheLookup::Miss)); + } + + #[tokio::test] + async fn batch_invalidation_removes_each_requested_identity() { + let adapter = enabled_adapter(); + let plan_a = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "bucket", + object: "a", + version_id: None, + etag: "etag-a", + size: 5, + body_variant: ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + let plan_b = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "bucket", + object: "b", + version_id: None, + etag: "etag-b", + size: 5, + body_variant: ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + let _ = adapter.fill_body(&plan_a, Bytes::from_static(b"aaaaa")).await; + let _ = adapter.fill_body(&plan_b, Bytes::from_static(b"bbbbb")).await; + + let keys = ["a".to_string(), "b".to_string()]; + invalidate_object_data_cache_objects_before_mutation(&adapter, "bucket", keys.iter()).await; + + assert!(matches!(adapter.lookup_body(&plan_a).await, ObjectDataCacheLookup::Miss)); + assert!(matches!(adapter.lookup_body(&plan_b).await, ObjectDataCacheLookup::Miss)); + } +} diff --git a/rustfs/src/app/object_data_cache/mod.rs b/rustfs/src/app/object_data_cache/mod.rs new file mode 100644 index 000000000..922d22af1 --- /dev/null +++ b/rustfs/src/app/object_data_cache/mod.rs @@ -0,0 +1,33 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! App-layer object data cache adapter boundary. + +mod adapter; +mod body; +mod invalidation; +mod planner; + +pub(crate) use adapter::ObjectDataCacheAdapter; +pub(crate) use body::{ + GetObjectBodyCacheLookup, fill_get_object_body_cache_from_buffered_body, fill_get_object_body_cache_from_materialized_body, + lookup_get_object_body_cache_hit, +}; +pub(crate) use invalidation::{ + invalidate_object_data_cache_after_complete_multipart_success, invalidate_object_data_cache_after_copy_success, + invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_after_put_success, + invalidate_object_data_cache_before_mutation, invalidate_object_data_cache_objects_after_delete_success, + invalidate_object_data_cache_objects_before_mutation, +}; +pub(crate) use planner::{GetObjectBodyCachePlan, GetObjectBodyCacheRequest, build_get_object_body_cache_plan}; diff --git a/rustfs/src/app/object_data_cache/planner.rs b/rustfs/src/app/object_data_cache/planner.rs new file mode 100644 index 000000000..534edd507 --- /dev/null +++ b/rustfs/src/app/object_data_cache/planner.rs @@ -0,0 +1,205 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! GET planning glue for the object data cache adapter. + +use crate::app::object_data_cache::ObjectDataCacheAdapter; +use crate::app::storage_api::object_usecase::StorageObjectInfo; +use rustfs_object_data_cache::{ObjectDataCacheBodyVariant, ObjectDataCacheGetPlan, ObjectDataCacheGetRequest}; + +/// App-layer GET request snapshot used for cache planning. +#[derive(Clone, Copy)] +pub(crate) struct GetObjectBodyCacheRequest<'a> { + pub(crate) bucket: &'a str, + pub(crate) key: &'a str, + pub(crate) info: &'a StorageObjectInfo, + pub(crate) response_content_length: i64, + pub(crate) has_range: bool, + pub(crate) part_number: Option, + pub(crate) encryption_applied: bool, +} + +/// Planning result for app-layer GET cache lookup. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum GetObjectBodyCachePlan { + Disabled, + Skip, + Cacheable(ObjectDataCacheGetPlan), +} + +/// Builds a conservative body-cache plan for a GET request. +pub(crate) fn build_get_object_body_cache_plan( + adapter: &ObjectDataCacheAdapter, + request: GetObjectBodyCacheRequest<'_>, +) -> GetObjectBodyCachePlan { + if adapter.is_disabled() { + return GetObjectBodyCachePlan::Disabled; + } + + if request.has_range + || request.part_number.is_some() + || request.encryption_applied + || request.info.delete_marker + || request.info.version_only + || request.info.metadata_only + || request.response_content_length < 0 + { + return GetObjectBodyCachePlan::Skip; + } + + let Some(etag) = request.info.etag.as_deref() else { + return GetObjectBodyCachePlan::Skip; + }; + + let Ok(size) = u64::try_from(request.response_content_length) else { + return GetObjectBodyCachePlan::Skip; + }; + + // Nil version ids mean "no value" (see CLAUDE.md); map them to None so the + // engine canonicalizes to the same "null" key as unversioned reads. + let version_id = request + .info + .version_id + .filter(|version_id| !version_id.is_nil()) + .map(|version_id| version_id.to_string()); + let engine_request = ObjectDataCacheGetRequest { + bucket: request.bucket, + object: request.key, + version_id, + etag, + size, + body_variant: ObjectDataCacheBodyVariant::FullObjectPlainV1, + }; + + match adapter.plan_get(engine_request) { + ObjectDataCacheGetPlan::Disabled | ObjectDataCacheGetPlan::SkipTooLarge => GetObjectBodyCachePlan::Skip, + plan @ ObjectDataCacheGetPlan::Cacheable { .. } => GetObjectBodyCachePlan::Cacheable(plan), + } +} + +#[cfg(test)] +mod tests { + use super::{GetObjectBodyCachePlan, GetObjectBodyCacheRequest, build_get_object_body_cache_plan}; + use crate::app::object_data_cache::ObjectDataCacheAdapter; + use rustfs_object_data_cache::{ObjectDataCacheConfig, ObjectDataCacheMode}; + + fn enabled_adapter() -> ObjectDataCacheAdapter { + let config = ObjectDataCacheConfig { + mode: ObjectDataCacheMode::HitOnly, + max_bytes: 8_388_608, + ..ObjectDataCacheConfig::default() + }; + ObjectDataCacheAdapter::new(config).expect("hit-only config should build adapter") + } + + #[test] + fn plan_is_disabled_when_adapter_is_disabled() { + let adapter = ObjectDataCacheAdapter::disabled(); + let info = crate::storage::storage_api::StorageObjectInfo { + etag: Some("etag".to_string()), + size: 4, + ..Default::default() + }; + + let plan = build_get_object_body_cache_plan( + &adapter, + GetObjectBodyCacheRequest { + bucket: "bucket", + key: "object", + info: &info, + response_content_length: 4, + has_range: false, + part_number: None, + encryption_applied: false, + }, + ); + + assert!(matches!(plan, GetObjectBodyCachePlan::Disabled)); + } + + #[test] + fn plan_skips_range_requests() { + let adapter = enabled_adapter(); + let info = crate::storage::storage_api::StorageObjectInfo { + etag: Some("etag".to_string()), + size: 4, + ..Default::default() + }; + + let plan = build_get_object_body_cache_plan( + &adapter, + GetObjectBodyCacheRequest { + bucket: "bucket", + key: "object", + info: &info, + response_content_length: 4, + has_range: true, + part_number: None, + encryption_applied: false, + }, + ); + + assert!(matches!(plan, GetObjectBodyCachePlan::Skip)); + } + + #[test] + fn plan_skips_when_etag_is_missing() { + let adapter = enabled_adapter(); + let info = crate::storage::storage_api::StorageObjectInfo { + etag: None, + size: 4, + ..Default::default() + }; + + let plan = build_get_object_body_cache_plan( + &adapter, + GetObjectBodyCacheRequest { + bucket: "bucket", + key: "object", + info: &info, + response_content_length: 4, + has_range: false, + part_number: None, + encryption_applied: false, + }, + ); + + assert!(matches!(plan, GetObjectBodyCachePlan::Skip)); + } + + #[test] + fn plan_is_cacheable_for_plain_full_object() { + let adapter = enabled_adapter(); + let info = crate::storage::storage_api::StorageObjectInfo { + etag: Some("etag".to_string()), + size: 4, + ..Default::default() + }; + + let plan = build_get_object_body_cache_plan( + &adapter, + GetObjectBodyCacheRequest { + bucket: "bucket", + key: "object", + info: &info, + response_content_length: 4, + has_range: false, + part_number: None, + encryption_applied: false, + }, + ); + + assert!(matches!(plan, GetObjectBodyCachePlan::Cacheable(_))); + } +} diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 65d8a4547..b56e9839a 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -94,7 +94,7 @@ use super::storage_api::object_usecase::{ }; use crate::app::runtime_sources::{ AppContext, current_app_context, current_expiry_state_handle, current_notify_interface_for_context, - current_object_store_handle_for_context, + current_object_data_cache_for_context, current_object_store_handle_for_context, }; use crate::config::RustFSBufferConfig; use crate::delete_tail_activity::{DeleteTailActivityGuard, DeleteTailStage}; @@ -189,6 +189,14 @@ use super::storage_api::object_usecase::{ StorageDeletedObject, StorageObjectInfo as ObjectInfo, StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader, }; +use crate::app::object_data_cache::{ + GetObjectBodyCacheLookup, GetObjectBodyCachePlan, GetObjectBodyCacheRequest, ObjectDataCacheAdapter, + build_get_object_body_cache_plan, fill_get_object_body_cache_from_buffered_body, + fill_get_object_body_cache_from_materialized_body, invalidate_object_data_cache_after_copy_success, + invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_after_put_success, + invalidate_object_data_cache_before_mutation, invalidate_object_data_cache_objects_after_delete_success, + invalidate_object_data_cache_objects_before_mutation, lookup_get_object_body_cache_hit, +}; type S3StdError = Box; @@ -393,8 +401,11 @@ const GET_READER_STREAM_POLL_READY_DATA: &str = "ready_data"; const GET_READER_STREAM_POLL_READY_EMPTY: &str = "ready_empty"; const GET_READER_STREAM_POLL_READY_ERROR: &str = "ready_error"; const GET_MEMORY_BODY_SOURCE_BUFFERED_BODY: &str = "buffered_body"; +const GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE: &str = "object_data_cache"; +const GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE_MATERIALIZED: &str = "object_data_cache_materialized"; const GET_MEMORY_BODY_SOURCE_SEEK_BUFFER: &str = "seek_buffer"; const GET_MEMORY_BODY_SOURCE_ENCRYPTED_BUFFER: &str = "encrypted_buffer"; +const GET_OBJECT_STAGE_BODY_CACHE_MATERIALIZE_READ: &str = "body_cache_materialize_read"; fn get_reader_stream_buffer_size_override() -> Option { static GET_READER_STREAM_BUFFER_SIZE_OVERRIDE: OnceLock> = OnceLock::new(); @@ -1286,6 +1297,25 @@ fn should_buffer_get_object_in_memory( ) } +fn should_materialize_get_object_body_for_cache( + info: &ObjectInfo, + response_content_length: i64, + part_number: Option, + has_range: bool, + concurrent_requests: usize, +) -> bool { + let configured_threshold = object_seek_support_threshold() as i64; + should_buffer_get_object_in_memory_with_threshold( + info, + response_content_length, + part_number, + has_range, + configured_threshold, + concurrent_requests, + true, + ) +} + fn should_buffer_get_object_in_memory_with_threshold( _info: &ObjectInfo, response_content_length: i64, @@ -2045,6 +2075,10 @@ impl DefaultObjectUsecase { current_object_store_handle_for_context(self.context.as_deref()) } + fn object_data_cache(&self) -> Arc { + current_object_data_cache_for_context(self.context.as_deref()) + } + fn base_buffer_size(&self) -> usize { self.context .clone() @@ -2788,6 +2822,147 @@ impl DefaultObjectUsecase { )) } + #[allow(clippy::too_many_arguments)] + async fn build_get_object_body_with_cache( + cache_adapter: &ObjectDataCacheAdapter, + mut final_stream: R, + info: &ObjectInfo, + response_content_length: i64, + optimal_buffer_size: usize, + enable_readahead: bool, + concurrent_requests: usize, + part_number: Option, + has_range: bool, + encryption_applied: bool, + buffered_body: Option, + bucket: &str, + key: &str, + ) -> S3Result> + where + R: AsyncRead + Send + Sync + Unpin + 'static, + { + let cache_request = GetObjectBodyCacheRequest { + bucket, + key, + info, + response_content_length, + has_range, + part_number, + encryption_applied, + }; + let cache_plan = build_get_object_body_cache_plan(cache_adapter, cache_request); + + match lookup_get_object_body_cache_hit(cache_adapter, &cache_plan).await { + GetObjectBodyCacheLookup::Hit(bytes) => { + return Ok(Self::build_memory_bytes_blob( + bytes, + response_content_length, + optimal_buffer_size, + GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE, + )); + } + GetObjectBodyCacheLookup::Disabled | GetObjectBodyCacheLookup::Skip | GetObjectBodyCacheLookup::Miss => {} + } + + if let Some(buffered_body) = buffered_body { + let _fill_result = fill_get_object_body_cache_from_buffered_body(cache_adapter, &cache_plan, &buffered_body).await; + + return Ok(Self::build_memory_bytes_blob( + buffered_body, + response_content_length, + optimal_buffer_size, + GET_MEMORY_BODY_SOURCE_BUFFERED_BODY, + )); + } + + let should_materialize_for_cache = cache_adapter.materialize_fill_enabled() + && matches!(cache_plan, GetObjectBodyCachePlan::Cacheable(_)) + && should_materialize_get_object_body_for_cache( + info, + response_content_length, + part_number, + has_range, + concurrent_requests, + ); + + if should_materialize_for_cache { + let Ok(materialized_capacity) = usize::try_from(response_content_length) else { + warn!( + expected = response_content_length, + "GetObject materialize-fill skipped because content length is not representable" + ); + return Self::build_get_object_body( + final_stream, + info, + response_content_length, + optimal_buffer_size, + enable_readahead, + concurrent_requests, + part_number, + has_range, + encryption_applied, + None, + bucket, + key, + ) + .await; + }; + let mut buf = Vec::with_capacity(materialized_capacity); + let buffer_read_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); + let read_result = tokio::io::AsyncReadExt::read_to_end(&mut final_stream, &mut buf).await; + record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_MATERIALIZE_READ, buffer_read_start); + + match read_result { + Ok(_) => { + if buf.len() != materialized_capacity { + warn!( + expected = response_content_length, + actual = buf.len(), + "Object size mismatch during materialize-fill read" + ); + } + + let bytes = Bytes::from(buf); + let _fill_result = + fill_get_object_body_cache_from_materialized_body(cache_adapter, &cache_plan, &bytes).await; + + return Ok(Self::build_memory_bytes_blob( + bytes, + response_content_length, + optimal_buffer_size, + GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE_MATERIALIZED, + )); + } + Err(e) => { + error!(error = %e, "GetObject materialize-fill buffering failed"); + // The stream is partially consumed; falling back to the + // streaming path would send a body missing its prefix, so + // fail the request like the encrypted-buffer path does. + return Err(ApiError::from(StorageError::other(format!( + "Failed to read object body for cache materialization: {e}" + ))) + .into()); + } + } + } + + Self::build_get_object_body( + final_stream, + info, + response_content_length, + optimal_buffer_size, + enable_readahead, + concurrent_requests, + part_number, + has_range, + encryption_applied, + None, + bucket, + key, + ) + .await + } + fn put_object_execution_context(req: &S3Request) -> (EventName, QuotaOperation, &'static str) { if req.extensions.get::().is_some() { (put_event_name_for_post_object(true), QuotaOperation::PostObject, "POST") @@ -3199,6 +3374,9 @@ impl DefaultObjectUsecase { ); } + let cache_adapter = self.object_data_cache(); + let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await; + let store_put_watchdog = tokio_util::sync::CancellationToken::new(); spawn_traced({ let store_put_watchdog = store_put_watchdog.clone(); @@ -3277,6 +3455,7 @@ impl DefaultObjectUsecase { }; maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await; + let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await; let put_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; // Fast in-memory update for immediate quota and admin usage consistency @@ -3513,9 +3692,11 @@ impl DefaultObjectUsecase { optimal_buffer_size, enable_readahead, } = strategy; + let cache_adapter = self.object_data_cache(); let body_build_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now); - let body = Self::build_get_object_body( + let body = Self::build_get_object_body_with_cache( + &cache_adapter, final_stream, &info, response_content_length, @@ -4285,6 +4466,8 @@ impl DefaultObjectUsecase { self.check_bucket_quota(&bucket, QuotaOperation::CopyObject, src_info.size as u64) .await?; let has_bucket_metadata = self.bucket_metadata_sys().is_some(); + let cache_adapter = self.object_data_cache(); + let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await; let oi = store .copy_object(&src_bucket, &src_key, &bucket, &key, &mut src_info, &src_opts, &dst_opts) @@ -4292,6 +4475,7 @@ impl DefaultObjectUsecase { .map_err(ApiError::from)?; maybe_enqueue_transition_immediate(&oi, LcEventSrc::S3CopyObject).await; + let _ = invalidate_object_data_cache_after_copy_success(&cache_adapter, &bucket, &key).await; let dest_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; // Update quota tracking after successful copy @@ -4514,6 +4698,13 @@ impl DefaultObjectUsecase { existing_object_infos.push(gerr.is_none().then_some(goi)); } + let cache_adapter = self.object_data_cache(); + let cache_keys_before_delete = object_to_delete + .iter() + .map(|object| object.object_name.clone()) + .collect::>(); + invalidate_object_data_cache_objects_before_mutation(&cache_adapter, &bucket, cache_keys_before_delete.iter()).await; + let (mut dobjs, errs) = store .delete_objects( &bucket, @@ -4616,6 +4807,11 @@ impl DefaultObjectUsecase { }, }) .collect(); + let deleted_cache_keys = delete_results + .iter() + .filter_map(|result| result.delete_object.as_ref().map(|deleted| deleted.object_name.clone())) + .collect::>(); + invalidate_object_data_cache_objects_after_delete_success(&cache_adapter, &bucket, deleted_cache_keys.iter()).await; let errors = delete_results .iter() @@ -4787,6 +4983,9 @@ impl DefaultObjectUsecase { } }; + let cache_adapter = self.object_data_cache(); + let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await; + let obj_info = { match store.delete_object(&bucket, &key, opts.clone()).await { Ok(obj) => obj, @@ -4816,6 +5015,7 @@ impl DefaultObjectUsecase { "failed to persist transitioned object cleanup journal" ); } + let _ = invalidate_object_data_cache_after_delete_success(&cache_adapter, &bucket, &key).await; // Fast in-memory update for immediate quota and admin usage consistency if delete_creates_delete_marker(&opts) { @@ -5829,6 +6029,8 @@ impl DefaultObjectUsecase { hrd = write_plan.apply(hrd, actual_size).map_err(ApiError::from)?; opts.user_defined.extend(metadata); let mut reader = PutObjReader::new(hrd); + let cache_adapter = self.object_data_cache(); + let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &fpath).await; let obj_info = match store.put_object(&bucket, &fpath, &mut reader, &opts).await { Ok(info) => info, @@ -5840,6 +6042,7 @@ impl DefaultObjectUsecase { return Err(ApiError::from(e).into()); } }; + let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &fpath).await; if !wrote_any_entry { rustfs_scanner::record_dirty_usage_bucket(&bucket); wrote_any_entry = true; @@ -6355,6 +6558,33 @@ mod tests { } } + struct DataProbeReader { + reads: Arc, + data: std::io::Cursor>, + } + + impl AsyncRead for DataProbeReader { + fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + self.reads.fetch_add(1, AtomicOrdering::Relaxed); + + let remaining = buf.remaining(); + if remaining == 0 { + return Poll::Ready(Ok(())); + } + + let position = usize::try_from(self.data.position()).unwrap_or(usize::MAX); + let source = self.data.get_ref(); + if position >= source.len() { + return Poll::Ready(Ok(())); + } + + let end = position.saturating_add(remaining).min(source.len()); + buf.put_slice(&source[position..end]); + self.data.set_position(u64::try_from(end).unwrap_or(u64::MAX)); + Poll::Ready(Ok(())) + } + } + struct PendingReader; impl AsyncRead for PendingReader { @@ -6501,6 +6731,374 @@ mod tests { ); } + #[tokio::test] + async fn build_get_object_body_with_cache_uses_cached_body_without_reader_preread() { + let reads = Arc::new(AtomicUsize::new(0)); + let reader = ReadProbeReader { + reads: Arc::clone(&reads), + }; + let info = ObjectInfo { + size: 5, + etag: Some("etag".to_string()), + ..Default::default() + }; + let adapter = + crate::app::object_data_cache::ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillBufferedOnly, + max_bytes: 8_388_608, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("fill-enabled cache adapter should initialize"); + let plan = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "test-bucket", + object: "cached-object", + version_id: None, + etag: "etag", + size: 5, + body_variant: rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + let fill = adapter.cache().fill_body(&plan, Bytes::from_static(b"hello")).await; + + assert_eq!(fill, rustfs_object_data_cache::ObjectDataCacheFillResult::Inserted); + + let body = DefaultObjectUsecase::build_get_object_body_with_cache( + &adapter, + reader, + &info, + 5, + 128 * 1024, + false, + 1, + None, + false, + false, + None, + "test-bucket", + "cached-object", + ) + .await + .expect("cache hit body handoff should succeed"); + + assert!(body.is_some()); + assert_eq!( + reads.load(AtomicOrdering::Relaxed), + 0, + "cache hit body handoff must not read from the fallback reader" + ); + } + + #[tokio::test] + async fn build_get_object_body_with_cache_rejects_size_mismatch_fill() { + let reads = Arc::new(AtomicUsize::new(0)); + let reader = ReadProbeReader { + reads: Arc::clone(&reads), + }; + let info = ObjectInfo { + size: 5, + etag: Some("etag".to_string()), + ..Default::default() + }; + let adapter = + crate::app::object_data_cache::ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillBufferedOnly, + max_bytes: 8_388_608, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("fill-enabled cache adapter should initialize"); + let plan = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "test-bucket", + object: "cached-object", + version_id: None, + etag: "etag", + size: 5, + body_variant: rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + let fill = adapter.cache().fill_body(&plan, Bytes::from_static(b"oops")).await; + + let body = DefaultObjectUsecase::build_get_object_body_with_cache( + &adapter, + reader, + &info, + 5, + 128 * 1024, + false, + 1, + None, + false, + false, + None, + "test-bucket", + "cached-object", + ) + .await + .expect("size-mismatched direct fill should not create a cache hit"); + let lookup_after_mismatch = adapter.lookup_body(&plan).await; + + assert_eq!(fill, rustfs_object_data_cache::ObjectDataCacheFillResult::SkippedSizeMismatch); + assert!(body.is_some()); + assert_eq!( + reads.load(AtomicOrdering::Relaxed), + 0, + "size-mismatched rejected fill should construct the fallback stream without pre-reading" + ); + assert!( + matches!(lookup_after_mismatch, rustfs_object_data_cache::ObjectDataCacheLookup::Miss), + "size-mismatched fill must not leave a reusable cache entry" + ); + } + + #[tokio::test] + async fn build_get_object_body_with_cache_fills_from_buffered_body_without_reader_preread() { + let first_reads = Arc::new(AtomicUsize::new(0)); + let first_reader = ReadProbeReader { + reads: Arc::clone(&first_reads), + }; + let second_reads = Arc::new(AtomicUsize::new(0)); + let second_reader = ReadProbeReader { + reads: Arc::clone(&second_reads), + }; + let info = ObjectInfo { + size: 5, + etag: Some("etag".to_string()), + ..Default::default() + }; + let adapter = + crate::app::object_data_cache::ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillBufferedOnly, + max_bytes: 8_388_608, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("fill-enabled cache adapter should initialize"); + + let first_body = DefaultObjectUsecase::build_get_object_body_with_cache( + &adapter, + first_reader, + &info, + 5, + 128 * 1024, + false, + 1, + None, + false, + false, + Some(Bytes::from_static(b"hello")), + "test-bucket", + "cached-object", + ) + .await + .expect("buffered-body handoff should succeed"); + + let second_body = DefaultObjectUsecase::build_get_object_body_with_cache( + &adapter, + second_reader, + &info, + 5, + 128 * 1024, + false, + 1, + None, + false, + false, + None, + "test-bucket", + "cached-object", + ) + .await + .expect("follow-up cache hit should succeed"); + + assert!(first_body.is_some()); + assert!(second_body.is_some()); + assert_eq!( + first_reads.load(AtomicOrdering::Relaxed), + 0, + "buffered-body fill path must not read from the fallback reader" + ); + assert_eq!( + second_reads.load(AtomicOrdering::Relaxed), + 0, + "cache hit after buffered-body fill must not read from the fallback reader" + ); + } + + #[tokio::test] + async fn build_get_object_body_with_cache_skips_buffered_fill_on_size_mismatch() { + let reads = Arc::new(AtomicUsize::new(0)); + let reader = ReadProbeReader { + reads: Arc::clone(&reads), + }; + let info = ObjectInfo { + size: 5, + etag: Some("etag".to_string()), + ..Default::default() + }; + let adapter = + crate::app::object_data_cache::ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillBufferedOnly, + max_bytes: 8_388_608, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("fill-enabled cache adapter should initialize"); + let plan = adapter.plan_get(rustfs_object_data_cache::ObjectDataCacheGetRequest { + bucket: "test-bucket", + object: "cached-object", + version_id: None, + etag: "etag", + size: 5, + body_variant: rustfs_object_data_cache::ObjectDataCacheBodyVariant::FullObjectPlainV1, + }); + + let body = DefaultObjectUsecase::build_get_object_body_with_cache( + &adapter, + reader, + &info, + 5, + 128 * 1024, + false, + 1, + None, + false, + false, + Some(Bytes::from_static(b"oops")), + "test-bucket", + "cached-object", + ) + .await + .expect("size-mismatched buffered-body handoff should still return a response body"); + let lookup = adapter.lookup_body(&plan).await; + + assert!(body.is_some()); + assert_eq!( + reads.load(AtomicOrdering::Relaxed), + 0, + "buffered-body handoff must not read from the fallback reader" + ); + assert!( + matches!(lookup, rustfs_object_data_cache::ObjectDataCacheLookup::Miss), + "size-mismatched buffered body must not be filled into cache" + ); + } + + #[tokio::test] + async fn build_get_object_body_with_cache_materializes_once_and_hits_later() { + let first_reads = Arc::new(AtomicUsize::new(0)); + let first_reader = DataProbeReader { + reads: Arc::clone(&first_reads), + data: std::io::Cursor::new(b"hello".to_vec()), + }; + let second_reads = Arc::new(AtomicUsize::new(0)); + let second_reader = ReadProbeReader { + reads: Arc::clone(&second_reads), + }; + let info = ObjectInfo { + size: 5, + etag: Some("etag".to_string()), + ..Default::default() + }; + let adapter = + crate::app::object_data_cache::ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 8_388_608, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("materialize-fill cache adapter should initialize"); + + let first_body = DefaultObjectUsecase::build_get_object_body_with_cache( + &adapter, + first_reader, + &info, + 5, + 128 * 1024, + false, + 1, + None, + false, + false, + None, + "test-bucket", + "materialized-object", + ) + .await + .expect("materialize-fill handoff should succeed"); + + let second_body = DefaultObjectUsecase::build_get_object_body_with_cache( + &adapter, + second_reader, + &info, + 5, + 128 * 1024, + false, + 1, + None, + false, + false, + None, + "test-bucket", + "materialized-object", + ) + .await + .expect("follow-up cache hit should succeed"); + + assert!(first_body.is_some()); + assert!(second_body.is_some()); + assert_eq!( + first_reads.load(AtomicOrdering::Relaxed), + 2, + "materialize-fill path should read the source stream once to data and once for EOF" + ); + assert_eq!( + second_reads.load(AtomicOrdering::Relaxed), + 0, + "cache hit after materialize-fill must not read from the fallback reader" + ); + } + + #[tokio::test] + async fn build_get_object_body_with_cache_skips_materialize_when_too_large_for_cache() { + let reads = Arc::new(AtomicUsize::new(0)); + let reader = DataProbeReader { + reads: Arc::clone(&reads), + data: std::io::Cursor::new(b"hello".to_vec()), + }; + let info = ObjectInfo { + size: 5, + etag: Some("etag".to_string()), + ..Default::default() + }; + let adapter = + crate::app::object_data_cache::ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { + mode: rustfs_object_data_cache::ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 8_388_608, + max_entry_bytes: 4, + ..rustfs_object_data_cache::ObjectDataCacheConfig::default() + }) + .expect("materialize-fill cache adapter should initialize"); + + let body = DefaultObjectUsecase::build_get_object_body_with_cache( + &adapter, + reader, + &info, + 5, + 128 * 1024, + false, + 1, + None, + false, + false, + None, + "test-bucket", + "too-large-object", + ) + .await + .expect("too-large cache candidate should use streaming fallback"); + + assert!(body.is_some()); + assert_eq!( + reads.load(AtomicOrdering::Relaxed), + 0, + "too-large materialize-fill candidate must not pre-read the fallback reader" + ); + } + #[tokio::test] async fn build_get_object_body_keeps_small_plain_objects_on_streaming_path_by_default() { let reads = Arc::new(AtomicUsize::new(0)); diff --git a/rustfs/src/app/runtime_sources.rs b/rustfs/src/app/runtime_sources.rs index 174a73ea9..4aa70ef29 100644 --- a/rustfs/src/app/runtime_sources.rs +++ b/rustfs/src/app/runtime_sources.rs @@ -12,13 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::app::object_data_cache::ObjectDataCacheAdapter; use crate::app::storage_api::runtime_sources::ExpiryState; #[cfg(test)] use crate::app::storage_api::runtime_sources::TierConfigMgr; use crate::runtime_sources as root_runtime_sources; pub(crate) use crate::runtime_sources::{ AppContext, current_encryption_service, current_endpoints_handle, current_notification_system, - current_object_store_handle_for_context, + current_object_data_cache_handle_for_context, current_object_store_handle_for_context, }; use rustfs_s3select_api::{QueryResult, server::dbms::DatabaseManagerSystem}; use s3s::dto::SelectObjectContentInput; @@ -36,6 +37,11 @@ pub(crate) fn current_notify_interface_for_context( .unwrap_or_else(root_runtime_sources::fallback_notify_interface) } +pub(crate) fn current_object_data_cache_for_context(app_context: Option<&AppContext>) -> Arc { + current_object_data_cache_handle_for_context(app_context) + .unwrap_or_else(root_runtime_sources::fallback_object_data_cache_handle) +} + pub(crate) async fn current_s3select_db( input: SelectObjectContentInput, enable_debug: bool, diff --git a/rustfs/src/runtime_sources.rs b/rustfs/src/runtime_sources.rs index f909bcdcf..5a2c0ed5a 100644 --- a/rustfs/src/runtime_sources.rs +++ b/rustfs/src/runtime_sources.rs @@ -17,6 +17,7 @@ use std::sync::Arc; pub(crate) use context::{ AppContext, NotifyInterface, default_notify_interface as fallback_notify_interface, + default_object_data_cache_handle as fallback_object_data_cache_handle, default_outbound_tls_runtime_interface as fallback_outbound_tls_runtime_interface, default_s3select_db_interface as fallback_s3select_db_interface, default_scanner_metrics_interface as fallback_scanner_metrics_interface, @@ -35,6 +36,7 @@ pub(crate) use context::{ resolve_notification_system_for_context as current_notification_system_for_context, resolve_notify_interface as current_notify_interface, resolve_notify_interface_for_context as current_notify_interface_for_context, + resolve_object_data_cache_handle_for_context as current_object_data_cache_handle_for_context, resolve_object_store_handle as current_object_store_handle, resolve_object_store_handle_for_context as current_object_store_handle_for_context, resolve_oidc_handle as current_oidc_handle, diff --git a/rustfs/src/storage/concurrency/manager.rs b/rustfs/src/storage/concurrency/manager.rs index 543b05cf8..aabe85268 100644 --- a/rustfs/src/storage/concurrency/manager.rs +++ b/rustfs/src/storage/concurrency/manager.rs @@ -988,10 +988,12 @@ mod integration_tests { StorageMedia::Hdd => config.hdd_buffer_cap, StorageMedia::Unknown => config.ssd_buffer_cap, }; - let expected_max = media_cap; - // Large base buffer should be constrained by the active storage media cap. - assert_eq!(strategy.buffer_size, expected_max, "Buffer should be capped by the active media profile"); + // Large base buffer should be constrained by the storage media cap. + // The final safety clamp is [32KiB, media_cap.max(MI_B)], so it never + // lowers the result below the media cap (e.g. NVMe's 2MiB cap stays + // effective even though the global floor clamp is 1MiB). + assert_eq!(strategy.buffer_size, media_cap, "Buffer should be capped by the storage media profile"); } #[tokio::test] diff --git a/scripts/check_logging_guardrails.sh b/scripts/check_logging_guardrails.sh index 53daffd87..1aeb25656 100755 --- a/scripts/check_logging_guardrails.sh +++ b/scripts/check_logging_guardrails.sh @@ -43,7 +43,7 @@ checked_files=( "crates/targets/src/target/webhook.rs" "crates/ecstore/src/store/peer.rs" "crates/ecstore/src/store/init.rs" - "crates/ecstore/src/tier/tier.rs" + "crates/ecstore/src/services/tier/tier.rs" "crates/heal/src/heal/manager.rs" "crates/heal/src/heal/storage.rs" "crates/heal/src/heal/task.rs" diff --git a/scripts/run_get_codec_streaming_smoke.sh b/scripts/run_get_codec_streaming_smoke.sh index dd4009f9b..18a231f8d 100755 --- a/scripts/run_get_codec_streaming_smoke.sh +++ b/scripts/run_get_codec_streaming_smoke.sh @@ -26,6 +26,9 @@ ROUND_COOLDOWN_SECS=20 WARP_OBJECTS="" WARP_OBJECT_LIFECYCLE="per-round" WARP_PREPARE_DURATION="1s" +WARP_MODE="get" +WARP_EXTRA_ARGS="" +WARP_WARMUP_GET_BEFORE_BENCH=false GET_OBJECT_METADATA_CACHE_MAX_ENTRIES="" GET_SMALL_OBJECT_DIRECT_MEMORY="" GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD="" @@ -161,6 +164,13 @@ Core options: --duration warp duration per round (default: 30s) --warp-objects Number of objects prepared by warp for each size (default: warp default) + --warp-mode warp workload mode used by the benchmark + (default: get) + --warp-extra-args Extra arguments passed to the warp benchmark + command after lifecycle arguments + --warp-warmup-get-before-bench Run a GET warmup with --noclear before the + measured benchmark. Useful for read/write + invalidation workloads. --warp-object-lifecycle Object lifecycle mode for warp GET: per-round|prepare-once|existing-only (default: per-round) @@ -319,6 +329,9 @@ parse_args() { --concurrency) CONCURRENCY="$2"; shift 2 ;; --duration) DURATION="$2"; shift 2 ;; --warp-objects) WARP_OBJECTS="$2"; shift 2 ;; + --warp-mode) WARP_MODE="$2"; shift 2 ;; + --warp-extra-args) WARP_EXTRA_ARGS="$2"; shift 2 ;; + --warp-warmup-get-before-bench) WARP_WARMUP_GET_BEFORE_BENCH=true; shift ;; --warp-object-lifecycle) WARP_OBJECT_LIFECYCLE="$2"; shift 2 ;; --warp-prepare-duration) WARP_PREPARE_DURATION="$2"; shift 2 ;; --metadata-cache-max-entries) GET_OBJECT_METADATA_CACHE_MAX_ENTRIES="$2"; shift 2 ;; @@ -415,6 +428,10 @@ validate_args() { if [[ -n "$WARP_OBJECTS" ]]; then validate_positive_int "$WARP_OBJECTS" "--warp-objects" fi + case "$WARP_MODE" in + get|mixed) ;; + *) die "--warp-mode must be get or mixed" ;; + esac case "$WARP_OBJECT_LIFECYCLE" in per-round|prepare-once|existing-only) ;; *) die "--warp-object-lifecycle must be per-round, prepare-once, or existing-only" ;; @@ -940,6 +957,9 @@ rounds=${ROUNDS} retry_per_round=${RETRY_PER_ROUND} round_cooldown_secs=${ROUND_COOLDOWN_SECS} warp_objects=${WARP_OBJECTS} +warp_mode=${WARP_MODE} +warp_extra_args=${WARP_EXTRA_ARGS} +warp_warmup_get_before_bench=${WARP_WARMUP_GET_BEFORE_BENCH} warp_object_lifecycle=${WARP_OBJECT_LIFECYCLE} warp_prepare_duration=${WARP_PREPARE_DURATION} rustfs_bin=${RUSTFS_BIN} @@ -1207,6 +1227,63 @@ prepare_warp_existing_objects() { done < <(benchmark_sizes) } +warmup_warp_get_before_bench() { + local profile="$1" + local profile_dir="${OUT_DIR}/${profile}" + local warmup_dir="${profile_dir}/warp_warmup_get" + + if [[ "$WARP_WARMUP_GET_BEFORE_BENCH" != "true" ]]; then + return 0 + fi + + mkdir -p "$warmup_dir" + + local size bucket size_slug + while IFS= read -r size; do + [[ -n "$size" ]] || continue + bucket="$(bucket_for_size "$size")" + size_slug="$(sanitize_bucket_suffix "$size")" + + local cmd=( + "$WARP_BIN" get + --host "$ADDRESS" + --access-key "$ACCESS_KEY" + --secret-key "$SECRET_KEY" + --bucket "$bucket" + --obj.size "$size" + --concurrent "$CONCURRENCY" + --duration "$WARP_PREPARE_DURATION" + --region "$REGION" + --noclear + ) + if [[ -n "$WARP_OBJECTS" ]]; then + cmd+=(--objects "$WARP_OBJECTS") + fi + + { + printf 'profile=%s\n' "$profile" + printf 'mode=warmup-get-before-bench\n' + printf 'size=%s\n' "$size" + printf 'duration=%s\n' "$WARP_PREPARE_DURATION" + printf 'bucket=%s\n' "$bucket" + printf 'command=' + printf '%q ' "${cmd[@]}" + printf '\n' + } >"${warmup_dir}/manifest-${size_slug}.env" + + if [[ "$DRY_RUN" == "true" ]]; then + log "[DRY-RUN] warmup GET before benchmark profile=${profile} size=${size} bucket=${bucket}" + printf '[DRY-RUN] ' >"${warmup_dir}/warmup-${size_slug}.log" + printf '%q ' "${cmd[@]}" >>"${warmup_dir}/warmup-${size_slug}.log" + printf '\n' >>"${warmup_dir}/warmup-${size_slug}.log" + continue + fi + + log "Running warmup GET before benchmark profile=${profile} size=${size} bucket=${bucket}..." + "${cmd[@]}" >"${warmup_dir}/warmup-${size_slug}.log" 2>&1 + done < <(benchmark_sizes) +} + run_bench() { local profile="$1" local baseline_csv="${2:-}" @@ -1221,7 +1298,7 @@ run_bench() { --bucket "$BUCKET" --region "$REGION" --warp-bin "$WARP_BIN" - --warp-mode get + --warp-mode "$WARP_MODE" --sizes "$SIZES" --concurrency "$CONCURRENCY" --duration "$DURATION" @@ -1234,19 +1311,24 @@ run_bench() { if [[ -n "$baseline_csv" ]]; then cmd+=(--baseline-csv "$baseline_csv") fi + local lifecycle_args="" if [[ "$WARP_OBJECT_LIFECYCLE" == "per-round" && -n "$WARP_OBJECTS" ]]; then - cmd+=(--extra-args "--objects ${WARP_OBJECTS}") - fi - if [[ "$WARP_OBJECT_LIFECYCLE" != "per-round" ]]; then - local lifecycle_args="--list-existing --noclear" + lifecycle_args="--objects ${WARP_OBJECTS}" + elif [[ "$WARP_OBJECT_LIFECYCLE" != "per-round" ]]; then + lifecycle_args="--list-existing --noclear" if [[ -n "$WARP_OBJECTS" ]]; then lifecycle_args="${lifecycle_args} --objects ${WARP_OBJECTS}" fi - cmd+=(--extra-args "$lifecycle_args") if [[ "$(benchmark_size_count)" -gt 1 ]]; then cmd+=(--bucket-size-suffix) fi fi + if [[ -n "$WARP_EXTRA_ARGS" ]]; then + lifecycle_args="${lifecycle_args:+${lifecycle_args} }${WARP_EXTRA_ARGS}" + fi + if [[ -n "$lifecycle_args" ]]; then + cmd+=(--extra-args "$lifecycle_args") + fi if [[ "$DIAGNOSTIC_METRICS" == "true" && -z "$DIAGNOSTIC_PROMETHEUS_QUERY_URL" ]]; then cmd+=( --service-metrics-url "$DIAGNOSTIC_METRICS_URL" @@ -3925,6 +4007,7 @@ run_profile() { stop_server start_server "$profile" prepare_warp_existing_objects "$profile" + warmup_warp_get_before_bench "$profile" capture_service_metrics_snapshot "$profile" before if run_bench "$profile" "$baseline_csv"; then : diff --git a/scripts/run_object_data_cache_bench.sh b/scripts/run_object_data_cache_bench.sh new file mode 100755 index 000000000..105acded9 --- /dev/null +++ b/scripts/run_object_data_cache_bench.sh @@ -0,0 +1,1458 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Dedicated GET benchmark harness for the object data cache rollout gate. +# +# This script intentionally reuses run_get_codec_streaming_smoke.sh for local +# RustFS lifecycle and warp orchestration, then adds object-cache-specific +# mode sequencing and metric acceptance checks. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +GET_BENCH="${PROJECT_ROOT}/scripts/run_get_codec_streaming_smoke.sh" + +ADDRESS_BASE="127.0.0.1:19130" +ACCESS_KEY="rustfsadmin" +SECRET_KEY="rustfsadmin" +BUCKET="rustfs-object-cache-bench" +REGION="us-east-1" +SIZES="256KiB,1MiB" +CONCURRENCY=16 +CONCURRENCY_LIST="" +DURATION="15s" +ROUNDS=2 +RETRY_PER_ROUND=1 +ROUND_COOLDOWN_SECS=5 +WARP_OBJECTS=128 +WARP_PREPARE_DURATION="3s" +MODES="hit_only,fill_buffered_only" +WORKLOADS="warm" +PROFILES="current" +MATRIX_PRESET="quick-gate" +BUFFERED_FILL_DIRECT_MEMORY_THRESHOLD=1048576 +OUT_DIR="" +RUSTFS_BIN="${PROJECT_ROOT}/target/release/rustfs" +WARP_BIN="warp" +PYTHON_BIN="python3" +SKIP_BUILD=false +DRY_RUN=false + +SERVICE_METRICS_URL="" +SERVICE_PROMETHEUS_QUERY_URL="" +SERVICE_PROMETHEUS_QUERY='{__name__=~"rustfs_object_data_cache_.*"}' +SERVICE_METRICS_FILTER_REGEX="rustfs_object_data_cache_" +SERVICE_METRICS_CAPTURE_ATTEMPTS=5 +SERVICE_METRICS_CAPTURE_RETRY_SECS=1 +SERVICE_METRICS_CONNECT_TIMEOUT_SECS=2 +SERVICE_METRICS_MAX_TIME_SECS=15 +SERVICE_METRICS_SETTLE_SECS=2 +OBS_ENDPOINT="${RUSTFS_OBS_ENDPOINT:-}" +OBS_METRIC_ENDPOINT="${RUSTFS_OBS_METRIC_ENDPOINT:-}" +OBS_METER_INTERVAL="${RUSTFS_OBS_METER_INTERVAL:-1}" +OBS_SERVICE_NAME_PREFIX="${RUSTFS_OBS_SERVICE_NAME:-RustFS-object-cache}" +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" +ALLOW_MISSING_CACHE_METRICS=false + +SIZES_SET=false +MODES_SET=false +CONCURRENCY_LIST_SET=false +WORKLOADS_SET=false +PROFILES_SET=false +CASE_CLEANUP=true + +OBJECT_CACHE_MAX_BYTES="" +OBJECT_CACHE_MAX_MEMORY_PERCENT="" +OBJECT_CACHE_MAX_ENTRY_BYTES="" +OBJECT_CACHE_TTL_SECS="" +OBJECT_CACHE_TIME_TO_IDLE_SECS="" +OBJECT_CACHE_MIN_FREE_MEMORY_PERCENT="" +OBJECT_CACHE_FILL_CONCURRENCY_PER_CPU="" +OBJECT_CACHE_FILL_CONCURRENCY_MAX="" +OBJECT_CACHE_IDENTITY_KEYS_MAX="" + +RUN_FAILURES=0 + +usage() { + cat <<'USAGE' +Usage: + scripts/run_object_data_cache_bench.sh [options] + +Purpose: + Run the object data cache rollout benchmark matrix and verify that the + cache-specific metrics prove the expected behavior before default enablement. + +Default quick-gate mode matrix: + hit_only + fill_buffered_only + +Core options: + --modes Cache modes to run + (default quick-gate: hit_only,fill_buffered_only) + --matrix-preset + quick-gate runs the default rollout gate only: + hit_only -> fill_buffered_only. + materialize-experimental runs only the explicit + fill_materialize_enabled experiment. + Expand the documented ST-10 matrix. st10-full + uses sizes 4KiB,64KiB,256KiB,1MiB,4MiB, + concurrency 1,8,16,32,64, workloads + cold,warm,mixed_80_20,write_after_read, and + profiles cpu_mem_1_2,cpu_mem_1_4,cpu_mem_1_8. + st10-full excludes fill_materialize_enabled + unless --modes explicitly includes it. + --address-base First local RustFS address. The port is + incremented per matrix case + (default: 127.0.0.1:19130) + --bucket Benchmark bucket prefix + --sizes Object sizes (default: 256KiB,1MiB) + --concurrency warp concurrency (default: 16) + --concurrency-list warp concurrency matrix. Overrides + --concurrency when set. + --workloads Workloads: cold,warm,mixed_80_20, + write_after_read (default: warm) + --profiles Capacity profiles: current,cpu_mem_1_2, + cpu_mem_1_4,cpu_mem_1_8 (default: current) + --duration warp duration per round (default: 15s) + --rounds benchmark rounds per size (default: 2) + --retry-per-round failed-attempt retries per round (default: 1) + --round-cooldown-secs cooldown after each round (default: 5) + --warp-objects minimum objects prepared and reused by warp. + The harness raises this per case when + concurrency/workload requires more. + --warp-prepare-duration prepare-once warmup duration (default: 3s) + --buffered-fill-direct-memory-threshold + Direct-memory threshold used only for + fill_buffered_only to force a buffered_body + producer (default: 1048576) + --out-dir output directory + +Metrics gate options: + --service-metrics-url Prometheus text scrape URL + --service-prometheus-query-url + Prometheus HTTP API /api/v1/query URL + --service-prometheus-query PromQL for object-cache metrics + --service-metrics-filter-regex + --service-metrics-settle-secs + --diagnostic-obs-endpoint + --diagnostic-obs-metric-endpoint + RUSTFS_OBS_METRIC_ENDPOINT for OTLP export + --diagnostic-obs-meter-interval + --diagnostic-obs-service-name-prefix + --allow-missing-cache-metrics Run perf-only if no service metrics endpoint is + provided. Strict rollout acceptance is skipped. + --keep-case-artifacts Keep per-case local RustFS data and warp logs. + By default they are removed after each case + once summary CSVs have been written. + +Object cache env overrides: + --object-cache-max-bytes + --object-cache-max-memory-percent + --object-cache-max-entry-bytes + --object-cache-ttl-secs + --object-cache-time-to-idle-secs + --object-cache-min-free-memory-percent + --object-cache-fill-concurrency-per-cpu + --object-cache-fill-concurrency-max + --object-cache-identity-keys-max + +Binary/options: + --rustfs-bin RustFS binary (default: target/release/rustfs) + --warp-bin warp binary (default: warp) + --python-bin Python binary for summary parsing + --skip-build do not build RustFS release binary + --dry-run print child commands without starting RustFS + +Output: + /environment.txt + /mode_summary.csv + /cache_metrics_summary.csv + /cache_metrics_acceptance.csv + /default_enablement_readiness.md + //legacy/warp/median_summary.csv + //legacy/service-metrics/{before,after}.prom + +Strict acceptance: + - non-disabled modes must expose rustfs_object_data_cache_requests_total + - hit_only must not insert fills + - fill_buffered_only must prove inserted fill, hit bytes, and weighted bytes + - fill_materialize_enabled must prove inserted fill, hit bytes, and weighted bytes +USAGE +} + +log() { + printf '%s\n' "$*" +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + die "command not found: $1" + fi +} + +validate_positive_int() { + local value="$1" + local name="$2" + if ! [[ "$value" =~ ^[0-9]+$ ]] || [[ "$value" -le 0 ]]; then + die "$name must be a positive integer, got: $value" + fi +} + +validate_non_negative_int() { + local value="$1" + local name="$2" + if ! [[ "$value" =~ ^[0-9]+$ ]]; then + die "$name must be a non-negative integer, got: $value" + fi +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --modes) MODES="$2"; MODES_SET=true; shift 2 ;; + --matrix-preset) MATRIX_PRESET="$2"; shift 2 ;; + --address-base) ADDRESS_BASE="$2"; shift 2 ;; + --access-key) ACCESS_KEY="$2"; shift 2 ;; + --secret-key) SECRET_KEY="$2"; shift 2 ;; + --bucket) BUCKET="$2"; shift 2 ;; + --region) REGION="$2"; shift 2 ;; + --sizes) SIZES="$2"; SIZES_SET=true; shift 2 ;; + --concurrency) CONCURRENCY="$2"; shift 2 ;; + --concurrency-list) CONCURRENCY_LIST="$2"; CONCURRENCY_LIST_SET=true; shift 2 ;; + --workloads) WORKLOADS="$2"; WORKLOADS_SET=true; shift 2 ;; + --profiles) PROFILES="$2"; PROFILES_SET=true; shift 2 ;; + --duration) DURATION="$2"; shift 2 ;; + --rounds) ROUNDS="$2"; shift 2 ;; + --retry-per-round) RETRY_PER_ROUND="$2"; shift 2 ;; + --round-cooldown-secs) ROUND_COOLDOWN_SECS="$2"; shift 2 ;; + --warp-objects) WARP_OBJECTS="$2"; shift 2 ;; + --warp-prepare-duration) WARP_PREPARE_DURATION="$2"; shift 2 ;; + --buffered-fill-direct-memory-threshold) BUFFERED_FILL_DIRECT_MEMORY_THRESHOLD="$2"; shift 2 ;; + --out-dir) OUT_DIR="$2"; shift 2 ;; + --service-metrics-url) SERVICE_METRICS_URL="$2"; shift 2 ;; + --service-prometheus-query-url) SERVICE_PROMETHEUS_QUERY_URL="$2"; shift 2 ;; + --service-prometheus-query) SERVICE_PROMETHEUS_QUERY="$2"; shift 2 ;; + --service-metrics-filter-regex) SERVICE_METRICS_FILTER_REGEX="$2"; shift 2 ;; + --service-metrics-attempts) SERVICE_METRICS_CAPTURE_ATTEMPTS="$2"; shift 2 ;; + --service-metrics-retry-secs) SERVICE_METRICS_CAPTURE_RETRY_SECS="$2"; shift 2 ;; + --service-metrics-connect-timeout-secs) SERVICE_METRICS_CONNECT_TIMEOUT_SECS="$2"; shift 2 ;; + --service-metrics-max-time-secs) SERVICE_METRICS_MAX_TIME_SECS="$2"; shift 2 ;; + --service-metrics-settle-secs) SERVICE_METRICS_SETTLE_SECS="$2"; shift 2 ;; + --diagnostic-obs-endpoint) OBS_ENDPOINT="$2"; shift 2 ;; + --diagnostic-obs-metric-endpoint) OBS_METRIC_ENDPOINT="$2"; shift 2 ;; + --diagnostic-obs-meter-interval) OBS_METER_INTERVAL="$2"; shift 2 ;; + --diagnostic-obs-service-name-prefix) OBS_SERVICE_NAME_PREFIX="$2"; shift 2 ;; + --allow-missing-cache-metrics) ALLOW_MISSING_CACHE_METRICS=true; shift ;; + --keep-case-artifacts) CASE_CLEANUP=false; shift ;; + --object-cache-max-bytes) OBJECT_CACHE_MAX_BYTES="$2"; shift 2 ;; + --object-cache-max-memory-percent) OBJECT_CACHE_MAX_MEMORY_PERCENT="$2"; shift 2 ;; + --object-cache-max-entry-bytes) OBJECT_CACHE_MAX_ENTRY_BYTES="$2"; shift 2 ;; + --object-cache-ttl-secs) OBJECT_CACHE_TTL_SECS="$2"; shift 2 ;; + --object-cache-time-to-idle-secs) OBJECT_CACHE_TIME_TO_IDLE_SECS="$2"; shift 2 ;; + --object-cache-min-free-memory-percent) OBJECT_CACHE_MIN_FREE_MEMORY_PERCENT="$2"; shift 2 ;; + --object-cache-fill-concurrency-per-cpu) OBJECT_CACHE_FILL_CONCURRENCY_PER_CPU="$2"; shift 2 ;; + --object-cache-fill-concurrency-max) OBJECT_CACHE_FILL_CONCURRENCY_MAX="$2"; shift 2 ;; + --object-cache-identity-keys-max) OBJECT_CACHE_IDENTITY_KEYS_MAX="$2"; shift 2 ;; + --rustfs-bin) RUSTFS_BIN="$2"; shift 2 ;; + --warp-bin) WARP_BIN="$2"; shift 2 ;; + --python-bin) PYTHON_BIN="$2"; shift 2 ;; + --skip-build) SKIP_BUILD=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage; exit 0 ;; + *) + usage >&2 + die "unknown arg: $1" + ;; + esac + done +} + +apply_matrix_preset() { + case "$MATRIX_PRESET" in + default|quick-gate) + MATRIX_PRESET="quick-gate" + if [[ "$MODES_SET" != "true" ]]; then + MODES="hit_only,fill_buffered_only" + fi + if [[ "$SIZES_SET" != "true" ]]; then + SIZES="256KiB,1MiB" + fi + if [[ "$CONCURRENCY_LIST_SET" != "true" ]]; then + CONCURRENCY_LIST="8,16" + fi + if [[ "$WORKLOADS_SET" != "true" ]]; then + WORKLOADS="warm" + fi + if [[ "$PROFILES_SET" != "true" ]]; then + PROFILES="current" + fi + ;; + materialize-experimental) + if [[ "$MODES_SET" != "true" ]]; then + MODES="fill_materialize_enabled" + fi + if [[ "$SIZES_SET" != "true" ]]; then + SIZES="256KiB,1MiB" + fi + if [[ "$CONCURRENCY_LIST_SET" != "true" ]]; then + CONCURRENCY_LIST="8,16" + fi + if [[ "$WORKLOADS_SET" != "true" ]]; then + WORKLOADS="warm" + fi + if [[ "$PROFILES_SET" != "true" ]]; then + PROFILES="current" + fi + ;; + st10-full) + if [[ "$MODES_SET" != "true" ]]; then + MODES="disabled,hit_only,fill_buffered_only" + fi + if [[ "$SIZES_SET" != "true" ]]; then + SIZES="4KiB,64KiB,256KiB,1MiB,4MiB" + fi + if [[ "$CONCURRENCY_LIST_SET" != "true" ]]; then + CONCURRENCY_LIST="1,8,16,32,64" + fi + if [[ "$WORKLOADS_SET" != "true" ]]; then + WORKLOADS="cold,warm,mixed_80_20,write_after_read" + fi + if [[ "$PROFILES_SET" != "true" ]]; then + PROFILES="cpu_mem_1_2,cpu_mem_1_4,cpu_mem_1_8" + fi + ;; + *) + die "--matrix-preset must be quick-gate, materialize-experimental, or st10-full" + ;; + esac + + if [[ -z "$CONCURRENCY_LIST" ]]; then + CONCURRENCY_LIST="$CONCURRENCY" + fi +} + +validate_mode() { + case "$1" in + disabled|hit_only|fill_buffered_only|fill_materialize_enabled) ;; + *) die "unsupported object cache mode: $1" ;; + esac +} + +validate_workload() { + case "$1" in + cold|warm|mixed_80_20|write_after_read) ;; + *) die "unsupported object cache workload: $1" ;; + esac +} + +validate_profile() { + case "$1" in + current|cpu_mem_1_2|cpu_mem_1_4|cpu_mem_1_8) ;; + *) die "unsupported object cache profile: $1" ;; + esac +} + +validate_args() { + [[ -n "$MODES" ]] || die "--modes must not be empty" + [[ -n "$ADDRESS_BASE" ]] || die "--address-base must not be empty" + [[ -n "$ACCESS_KEY" ]] || die "--access-key must not be empty" + [[ -n "$SECRET_KEY" ]] || die "--secret-key must not be empty" + [[ -n "$BUCKET" ]] || die "--bucket must not be empty" + [[ -n "$SIZES" ]] || die "--sizes must not be empty" + [[ -n "$CONCURRENCY_LIST" ]] || die "--concurrency-list must not be empty" + [[ -n "$WORKLOADS" ]] || die "--workloads must not be empty" + [[ -n "$PROFILES" ]] || die "--profiles must not be empty" + validate_positive_int "$CONCURRENCY" "--concurrency" + validate_positive_int "$ROUNDS" "--rounds" + validate_positive_int "$RETRY_PER_ROUND" "--retry-per-round" + validate_non_negative_int "$ROUND_COOLDOWN_SECS" "--round-cooldown-secs" + validate_positive_int "$WARP_OBJECTS" "--warp-objects" + validate_positive_int "$BUFFERED_FILL_DIRECT_MEMORY_THRESHOLD" "--buffered-fill-direct-memory-threshold" + validate_positive_int "$SERVICE_METRICS_CAPTURE_ATTEMPTS" "--service-metrics-attempts" + validate_non_negative_int "$SERVICE_METRICS_CAPTURE_RETRY_SECS" "--service-metrics-retry-secs" + validate_positive_int "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS" "--service-metrics-connect-timeout-secs" + validate_positive_int "$SERVICE_METRICS_MAX_TIME_SECS" "--service-metrics-max-time-secs" + validate_non_negative_int "$SERVICE_METRICS_SETTLE_SECS" "--service-metrics-settle-secs" + validate_positive_int "$OBS_METER_INTERVAL" "--diagnostic-obs-meter-interval" + + local raw mode + IFS=',' read -r -a mode_list <<< "$MODES" + [[ "${#mode_list[@]}" -gt 0 ]] || die "--modes must contain at least one mode" + for raw in "${mode_list[@]}"; do + mode="${raw//[[:space:]]/}" + [[ -n "$mode" ]] || continue + validate_mode "$mode" + done + + local concurrency_value + IFS=',' read -r -a concurrency_values <<< "$CONCURRENCY_LIST" + [[ "${#concurrency_values[@]}" -gt 0 ]] || die "--concurrency-list must contain at least one value" + for raw in "${concurrency_values[@]}"; do + concurrency_value="${raw//[[:space:]]/}" + [[ -n "$concurrency_value" ]] || continue + validate_positive_int "$concurrency_value" "--concurrency-list value" + done + + local workload + IFS=',' read -r -a workload_list <<< "$WORKLOADS" + [[ "${#workload_list[@]}" -gt 0 ]] || die "--workloads must contain at least one workload" + for raw in "${workload_list[@]}"; do + workload="${raw//[[:space:]]/}" + [[ -n "$workload" ]] || continue + validate_workload "$workload" + done + + local profile + IFS=',' read -r -a profile_list <<< "$PROFILES" + [[ "${#profile_list[@]}" -gt 0 ]] || die "--profiles must contain at least one profile" + for raw in "${profile_list[@]}"; do + profile="${raw//[[:space:]]/}" + [[ -n "$profile" ]] || continue + validate_profile "$profile" + done + + if [[ -n "$SERVICE_METRICS_URL" && -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then + die "--service-metrics-url and --service-prometheus-query-url are mutually exclusive" + fi + if [[ "$ALLOW_MISSING_CACHE_METRICS" != "true" && -z "$SERVICE_METRICS_URL" && -z "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then + die "strict cache metrics acceptance requires --service-metrics-url or --service-prometheus-query-url; use --allow-missing-cache-metrics for perf-only runs" + fi + + [[ -x "$GET_BENCH" ]] || die "GET benchmark script is not executable: $GET_BENCH" + require_cmd git + require_cmd "$PYTHON_BIN" + if [[ "$DRY_RUN" != "true" ]]; then + require_cmd "$WARP_BIN" + if [[ "$SKIP_BUILD" != "true" ]]; then + require_cmd cargo + fi + fi +} + +setup_output() { + if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="${PROJECT_ROOT}/target/bench/object-data-cache-$(date +%Y%m%d-%H%M%S)" + fi + mkdir -p "$OUT_DIR" + echo "mode,workload,profile,concurrency,warp_objects,address,bucket,status,case_dir" >"$(matrix_case_csv)" +} + +host_part() { + printf '%s' "$ADDRESS_BASE" | awk -F: '{ print $1 }' +} + +port_part() { + printf '%s' "$ADDRESS_BASE" | awk -F: '{ print $NF }' +} + +address_for_index() { + local index="$1" + local host port + host="$(host_part)" + port="$(port_part)" + if ! [[ "$port" =~ ^[0-9]+$ ]]; then + die "address-base must end with a numeric port: $ADDRESS_BASE" + fi + printf '%s:%s\n' "$host" "$((port + index))" +} + +sanitize_name() { + printf '%s' "$1" | tr '[:upper:]_' '[:lower:]-' | tr -c 'a-z0-9-' '-' +} + +csv_values() { + local csv="$1" + local raw value + IFS=',' read -r -a value_list <<< "$csv" + for raw in "${value_list[@]}"; do + value="${raw//[[:space:]]/}" + [[ -n "$value" ]] || continue + printf '%s\n' "$value" + done +} + +mode_values() { + csv_values "$MODES" +} + +workload_values() { + csv_values "$WORKLOADS" +} + +profile_values() { + csv_values "$PROFILES" +} + +concurrency_values() { + csv_values "$CONCURRENCY_LIST" +} + +matrix_case_path() { + local mode="$1" + local workload="$2" + local profile="$3" + local concurrency="$4" + printf '%s/%s/%s/%s/concurrency-%s\n' \ + "$OUT_DIR" "$(sanitize_name "$mode")" "$(sanitize_name "$workload")" "$(sanitize_name "$profile")" "$concurrency" +} + +matrix_case_csv() { + printf '%s/matrix_cases.csv\n' "$OUT_DIR" +} + +case_bucket_name() { + local index="$1" + local prefix suffix max_prefix + prefix="$(sanitize_name "$BUCKET")" + suffix="m${index}" + max_prefix=$((63 - ${#suffix} - 1)) + if [[ "$max_prefix" -lt 3 ]]; then + die "--bucket leaves no room for a unique matrix suffix after S3 bucket length normalization" + fi + if [[ "${#prefix}" -gt "$max_prefix" ]]; then + prefix="${prefix:0:${max_prefix}}" + prefix="${prefix%-}" + fi + printf '%s-%s\n' "$prefix" "$suffix" +} + +record_matrix_case() { + local mode="$1" + local workload="$2" + local profile="$3" + local concurrency="$4" + local warp_objects="$5" + local address="$6" + local bucket="$7" + local status="$8" + local case_dir="$9" + + printf '%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \ + "$mode" "$workload" "$profile" "$concurrency" "$warp_objects" "$address" "$bucket" "$status" "$case_dir" >>"$(matrix_case_csv)" +} + +write_environment() { + local branch git_head dirty_count rustc_version cargo_version command_line + branch="$(git -C "$PROJECT_ROOT" rev-parse --abbrev-ref HEAD)" + git_head="$(git -C "$PROJECT_ROOT" rev-parse HEAD)" + dirty_count="$(git -C "$PROJECT_ROOT" status --porcelain | awk 'END { print NR + 0 }')" + rustc_version="$(rustc --version 2>/dev/null || echo unavailable)" + cargo_version="$(cargo --version 2>/dev/null || echo unavailable)" + command_line="$(printf '%q ' "${BASH_SOURCE[0]}" "$@")" + + cat >"${OUT_DIR}/environment.txt" < 4 else "" +sizes = [value.strip() for value in sys.argv[5].split(",") if value.strip()] +concurrency_values = [value.strip() for value in sys.argv[6].split(",") if value.strip()] +workloads = [value.strip() for value in sys.argv[7].split(",") if value.strip()] +profiles = [value.strip() for value in sys.argv[8].split(",") if value.strip()] +DEFAULT_ROLLOUT_REQUIRED_MODES = {"hit_only", "fill_buffered_only"} +EXPERIMENTAL_MODES = {"fill_materialize_enabled"} +ST10_REQUIRED_SIZES = ["4KiB", "64KiB", "256KiB", "1MiB", "4MiB"] +ST10_REQUIRED_CONCURRENCY = ["1", "8", "16", "32", "64"] +ST10_REQUIRED_WORKLOADS = ["cold", "warm", "mixed_80_20", "write_after_read"] +ST10_REQUIRED_PROFILES = ["cpu_mem_1_2", "cpu_mem_1_4", "cpu_mem_1_8"] + +LINE = re.compile( + r'^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+' + r'([-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?)' + r'(?:\s+[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?)?\s*$' +) + + +def read_prom(path): + rows = {} + if not path.exists() or path.stat().st_size == 0: + return rows + for raw in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if not raw or raw.startswith("#"): + continue + match = LINE.match(raw) + if match is None: + continue + metric, labels_raw, value_raw = match.groups() + labels = labels_raw or "" + rows[(metric, labels)] = float(value_raw) + return rows + + +def labels_to_dict(labels): + result = {} + if not labels: + return result + parts = re.findall(r'([A-Za-z0-9_.-]+)="((?:\\.|[^"\\])*)"', labels) + for key, value in parts: + result[key] = value.replace(r'\"', '"').replace(r'\\', '\\') + return result + + +def sanitize_mode(mode): + return "".join(ch if ("a" <= ch <= "z" or "0" <= ch <= "9") else "-" for ch in mode.lower().replace("_", "-")) + + +def labels_belong_to_mode(labels, mode): + label_map = labels_to_dict(labels) + mode_slug = sanitize_mode(mode) + attribution_values = [label_map.get(key, "") for key in ("job", "service_name", "otel_scope_name")] + has_attribution = any(attribution_values) + mode_label_matches = label_map.get("mode") == mode + attribution_matches_mode = any(mode in value or mode_slug in value for value in attribution_values) + attribution_matches_run = not run_id or any(run_id in value for value in attribution_values) + if has_attribution: + return attribution_matches_run and (mode_label_matches or attribution_matches_mode) + + if mode_label_matches: + return True + + return False + + +def labels_have_mode_attribution(labels): + label_map = labels_to_dict(labels) + return any(label_map.get(key) for key in ("mode", "job", "service_name", "otel_scope_name")) + + +def labels_match_mode_or_unattributed(labels, mode): + if labels_belong_to_mode(labels, mode): + return True + return not labels_have_mode_attribution(labels) + + +def metric_matches(metric, expected): + return metric == expected or metric == f"{expected}_total" + + +def counter_delta(before, after, expected, expected_mode, **wanted_labels): + total = 0.0 + for (metric, labels), after_value in after.items(): + if not metric_matches(metric, expected): + continue + if not labels_match_mode_or_unattributed(labels, expected_mode): + continue + label_map = labels_to_dict(labels) + if all(label_map.get(key) == expected_value for key, expected_value in wanted_labels.items()): + before_value = before.get((metric, labels), 0.0) + delta = after_value - before_value + if delta > 0: + total += delta + return total + + +def counter_observed(after, expected, expected_mode, **wanted_labels): + total = 0.0 + for (metric, labels), value in after.items(): + if not metric_matches(metric, expected): + continue + if not labels_match_mode_or_unattributed(labels, expected_mode): + continue + label_map = labels_to_dict(labels) + if all(label_map.get(key) == expected_value for key, expected_value in wanted_labels.items()): + total += value + return total + + +def gauge_max(before, after, expected, mode): + values = [] + for rows in (before, after): + for (metric, labels), value in rows.items(): + if metric_matches(metric, expected): + if not labels_match_mode_or_unattributed(labels, mode): + continue + values.append(value) + return max(values) if values else 0.0 + + +def metric_delta_rows(mode, before, after): + keys = sorted(set(before) | set(after)) + for metric, labels in keys: + if not labels_match_mode_or_unattributed(labels, mode): + continue + before_value = before.get((metric, labels), 0.0) + after_value = after.get((metric, labels), 0.0) + yield { + "mode": mode, + "metric": metric, + "labels": labels, + "before": f"{before_value:.12g}", + "after": f"{after_value:.12g}", + "delta": f"{after_value - before_value:.12g}", + } + + +def read_cases(): + cases_path = out_dir / "matrix_cases.csv" + if not cases_path.exists(): + return [ + { + "mode": mode, + "workload": "warm", + "profile": "current", + "concurrency": "", + "address": "", + "bucket": "", + "status": "unknown", + "case_dir": str(out_dir / mode), + } + for mode in modes + ] + with cases_path.open("r", encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) + + +def case_metrics(case): + metrics_dir = pathlib.Path(case["case_dir"]) / "legacy" / "service-metrics" + before = read_prom(metrics_dir / "before.prom") + after = read_prom(metrics_dir / "after.prom") + mode = case["mode"] + requests = counter_delta(before, after, "rustfs_object_data_cache_requests_total", mode, mode=mode) + misses = counter_delta(before, after, "rustfs_object_data_cache_requests_total", mode, mode=mode, decision="miss") + cacheable = counter_delta(before, after, "rustfs_object_data_cache_requests_total", mode, mode=mode, decision="cacheable") + hits = counter_delta(before, after, "rustfs_object_data_cache_requests_total", mode, mode=mode, decision="hit") + hit_bytes = counter_delta(before, after, "rustfs_object_data_cache_hit_bytes_total", mode, mode=mode) + fill_inserted = counter_observed(after, "rustfs_object_data_cache_fill_total", mode, mode=mode, result="inserted") + fill_skipped_by_mode = counter_observed( + after, "rustfs_object_data_cache_fill_total", mode, mode=mode, result="skipped_by_mode" + ) + weighted_bytes = gauge_max(before, after, "rustfs_object_data_cache_weighted_bytes", mode) + entries = gauge_max(before, after, "rustfs_object_data_cache_entries", mode) + return before, after, { + "requests_total": requests, + "misses": misses, + "cacheable": cacheable, + "hits": hits, + "hit_bytes_total": hit_bytes, + "fill_inserted": fill_inserted, + "fill_skipped_by_mode": fill_skipped_by_mode, + "weighted_bytes": weighted_bytes, + "entries": entries, + } + + +def zero_values(): + return { + "requests_total": 0.0, + "misses": 0.0, + "cacheable": 0.0, + "hits": 0.0, + "hit_bytes_total": 0.0, + "fill_inserted": 0.0, + "fill_skipped_by_mode": 0.0, + "weighted_bytes": 0.0, + "entries": 0.0, + } + + +def merge_values(left, right): + merged = dict(left) + for key in ( + "requests_total", + "misses", + "cacheable", + "hits", + "hit_bytes_total", + "fill_inserted", + "fill_skipped_by_mode", + ): + merged[key] += right[key] + merged["weighted_bytes"] = max(merged["weighted_bytes"], right["weighted_bytes"]) + merged["entries"] = max(merged["entries"], right["entries"]) + return merged + + +def acceptance(mode, values): + if not strict: + return "skipped", "perf_only_run_allow_missing_cache_metrics" + if mode == "disabled": + return "pass", "disabled baseline does not require object-cache metrics" + if values["requests_total"] <= 0: + return "fail", "missing object-cache request metrics" + if mode == "hit_only": + if values["fill_inserted"] > 0: + return "fail", "hit_only inserted cache fills" + return "pass", "hit_only produced cache decisions without fills" + if mode == "fill_buffered_only": + missing = [] + if values["fill_inserted"] <= 0: + missing.append("fill_inserted") + if values["hit_bytes_total"] <= 0: + missing.append("hit_bytes_total") + if values["weighted_bytes"] <= 0: + missing.append("weighted_bytes") + if missing: + return "fail", "missing required metrics: " + "|".join(missing) + return "pass", "buffered fill mode proved fill, hit, and resident bytes" + if mode == "fill_materialize_enabled": + missing = [] + if values["fill_inserted"] <= 0: + missing.append("fill_inserted") + if values["hit_bytes_total"] <= 0: + missing.append("hit_bytes_total") + if values["weighted_bytes"] <= 0: + missing.append("weighted_bytes") + if missing: + return "fail", "missing required metrics: " + "|".join(missing) + return "pass", "materialize mode proved fill, hit, and resident bytes" + return "fail", "unknown mode" + + +cases = read_cases() +case_metrics_rows = [] +mode_totals = {mode: zero_values() for mode in modes} + +with (out_dir / "mode_summary.csv").open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle) + writer.writerow([ + "mode", + "workload", + "profile", + "size", + "tool", + "concurrency", + "successful_rounds", + "failed_rounds", + "median_throughput_bps", + "median_reqps", + "median_latency_ms", + "median_req_p90_ms", + "median_req_p99_ms", + "source_csv", + ]) + for case in cases: + mode = case["mode"] + median_path = pathlib.Path(case["case_dir"]) / "legacy" / "warp" / "median_summary.csv" + if not median_path.exists(): + writer.writerow([ + mode, + case["workload"], + case["profile"], + "N/A", + "N/A", + case["concurrency"], + 0, + "N/A", + "N/A", + "N/A", + "N/A", + "N/A", + "N/A", + str(median_path), + ]) + continue + with median_path.open("r", encoding="utf-8", newline="") as source: + for row in csv.DictReader(source): + writer.writerow([ + mode, + case["workload"], + case["profile"], + row.get("size", "N/A"), + row.get("tool", "N/A"), + row.get("concurrency", "N/A"), + row.get("successful_rounds", "N/A"), + row.get("failed_rounds", "N/A"), + row.get("median_throughput_bps", "N/A"), + row.get("median_reqps", "N/A"), + row.get("median_latency_ms", "N/A"), + row.get("median_req_p90_ms", "N/A"), + row.get("median_req_p99_ms", "N/A"), + str(median_path), + ]) + +with (out_dir / "cache_metrics_summary.csv").open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter( + handle, + fieldnames=["mode", "workload", "profile", "concurrency", "metric", "labels", "before", "after", "delta"], + ) + writer.writeheader() + for case in cases: + before, after, values = case_metrics(case) + case_metrics_rows.append((case, values)) + mode_totals[case["mode"]] = merge_values(mode_totals.get(case["mode"], zero_values()), values) + for row in metric_delta_rows(case["mode"], before, after): + row["mode"] = case["mode"] + row["workload"] = case["workload"] + row["profile"] = case["profile"] + row["concurrency"] = case["concurrency"] + writer.writerow(row) + +with (out_dir / "cache_metrics_acceptance_by_case.csv").open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle) + writer.writerow([ + "mode", + "workload", + "profile", + "concurrency", + "case_status", + "requests_total", + "misses", + "cacheable", + "hits", + "hit_bytes_total", + "fill_inserted", + "fill_skipped_by_mode", + "weighted_bytes", + "entries", + ]) + for case, values in case_metrics_rows: + writer.writerow([ + case["mode"], + case["workload"], + case["profile"], + case["concurrency"], + case["status"], + f'{values["requests_total"]:.12g}', + f'{values["misses"]:.12g}', + f'{values["cacheable"]:.12g}', + f'{values["hits"]:.12g}', + f'{values["hit_bytes_total"]:.12g}', + f'{values["fill_inserted"]:.12g}', + f'{values["fill_skipped_by_mode"]:.12g}', + f'{values["weighted_bytes"]:.12g}', + f'{values["entries"]:.12g}', + ]) + +all_acceptance = [] +for mode in modes: + values = mode_totals.get(mode, zero_values()) + status, notes = acceptance(mode, values) + all_acceptance.append((mode, status, notes, values)) + +with (out_dir / "cache_metrics_acceptance.csv").open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle) + writer.writerow([ + "mode", + "status", + "requests_total", + "misses", + "cacheable", + "hits", + "hit_bytes_total", + "fill_inserted", + "fill_skipped_by_mode", + "weighted_bytes", + "entries", + "notes", + ]) + for mode, status, notes, values in all_acceptance: + writer.writerow([ + mode, + status, + f'{values["requests_total"]:.12g}', + f'{values["misses"]:.12g}', + f'{values["cacheable"]:.12g}', + f'{values["hits"]:.12g}', + f'{values["hit_bytes_total"]:.12g}', + f'{values["fill_inserted"]:.12g}', + f'{values["fill_skipped_by_mode"]:.12g}', + f'{values["weighted_bytes"]:.12g}', + f'{values["entries"]:.12g}', + notes, + ]) + +missing_sizes = [value for value in ST10_REQUIRED_SIZES if value not in set(sizes)] +missing_concurrency = [value for value in ST10_REQUIRED_CONCURRENCY if value not in set(concurrency_values)] +missing_workloads = [value for value in ST10_REQUIRED_WORKLOADS if value not in set(workloads)] +missing_profiles = [value for value in ST10_REQUIRED_PROFILES if value not in set(profiles)] +case_failed = any(case.get("status") != "ok" for case in cases) +matrix_coverage_complete = not (missing_sizes or missing_concurrency or missing_workloads or missing_profiles) + +with (out_dir / "matrix_coverage.md").open("w", encoding="utf-8") as handle: + handle.write("# Object Data Cache ST-10 Matrix Coverage\n\n") + handle.write(f"- sizes_under_test: {', '.join(sizes)}\n") + handle.write(f"- concurrency_under_test: {', '.join(concurrency_values)}\n") + handle.write(f"- workloads_under_test: {', '.join(workloads)}\n") + handle.write(f"- profiles_under_test: {', '.join(profiles)}\n") + handle.write(f"- st10_full_coverage: {str(matrix_coverage_complete).lower()}\n") + if missing_sizes: + handle.write(f"- missing_sizes: {', '.join(missing_sizes)}\n") + if missing_concurrency: + handle.write(f"- missing_concurrency: {', '.join(missing_concurrency)}\n") + if missing_workloads: + handle.write(f"- missing_workloads: {', '.join(missing_workloads)}\n") + if missing_profiles: + handle.write(f"- missing_profiles: {', '.join(missing_profiles)}\n") + handle.write("\n") + handle.write("## Workload Semantics\n\n") + handle.write("- cold: GET benchmark with per-round object lifecycle.\n") + handle.write("- warm: GET benchmark with prepare-once object lifecycle.\n") + handle.write("- mixed_80_20: warp mixed with GET=80, PUT=20, STAT=0, DELETE=0.\n") + handle.write("- write_after_read: GET warmup with --noclear, then warp mixed with GET=50, PUT=50, STAT=0, DELETE=0.\n") + handle.write("\n") + handle.write("## Profile Semantics\n\n") + handle.write("- current: server defaults or explicit CLI env overrides.\n") + handle.write("- cpu_mem_1_2: 3% memory, 1MiB max entry, TTL 30s, TTI 15s.\n") + handle.write("- cpu_mem_1_4: 5% memory, 1MiB max entry, TTL 60s, TTI 30s.\n") + handle.write("- cpu_mem_1_8: 6% memory, 4MiB max entry, TTL 120s, TTI 60s.\n") + +strict_failed = case_failed or any(status == "fail" for _mode, status, _notes, _values in all_acceptance) +default_required_under_test = [ + (mode, status) for mode, status, _notes, _values in all_acceptance if mode in DEFAULT_ROLLOUT_REQUIRED_MODES +] +default_rollout_failed = any(status == "fail" for _mode, status in default_required_under_test) +default_rollout_missing = sorted(DEFAULT_ROLLOUT_REQUIRED_MODES - set(modes)) +experimental_failed = any(status == "fail" for mode, status, _notes, _values in all_acceptance if mode in EXPERIMENTAL_MODES) +overall_status = "skipped" if not strict else ("fail" if strict_failed else "pass") +default_rollout_status = ( + "skipped" + if not strict + else ("incomplete" if default_rollout_missing else ("fail" if default_rollout_failed else "pass")) +) +with (out_dir / "default_enablement_readiness.md").open("w", encoding="utf-8") as handle: + handle.write("# Object Data Cache Rollout Readiness\n\n") + handle.write(f"- strict_metrics_acceptance: {str(strict).lower()}\n") + handle.write(f"- metrics_overall_status: {overall_status}\n") + handle.write(f"- default_buffered_rollout_status: {default_rollout_status}\n") + handle.write(f"- matrix_case_failures: {str(case_failed).lower()}\n") + handle.write(f"- st10_full_matrix_coverage: {str(matrix_coverage_complete).lower()}\n") + handle.write(f"- modes_under_test: {', '.join(modes)}\n") + handle.write(f"- workloads_under_test: {', '.join(workloads)}\n") + handle.write(f"- profiles_under_test: {', '.join(profiles)}\n") + handle.write(f"- concurrency_under_test: {', '.join(concurrency_values)}\n") + handle.write(f"- sizes_under_test: {', '.join(sizes)}\n") + handle.write("- default_required_modes: hit_only, fill_buffered_only\n") + handle.write("- experimental_modes: fill_materialize_enabled\n") + if default_rollout_missing: + handle.write(f"- missing_default_required_modes: {', '.join(default_rollout_missing)}\n") + handle.write("- required_metrics: requests_total, fill_total, hit_bytes_total, weighted_bytes\n\n") + handle.write("## Mode Acceptance\n\n") + handle.write("| mode | status | requests | fill_inserted | hit_bytes | weighted_bytes | notes |\n") + handle.write("| --- | --- | ---: | ---: | ---: | ---: | --- |\n") + for mode, status, notes, values in all_acceptance: + handle.write( + f"| {mode} | {status} | {values['requests_total']:.12g} | " + f"{values['fill_inserted']:.12g} | {values['hit_bytes_total']:.12g} | " + f"{values['weighted_bytes']:.12g} | {notes} |\n" + ) + handle.write("\n") + if strict_failed: + if default_rollout_failed or default_rollout_missing: + handle.write("Conclusion: do not advance the default buffered rollout.\n") + elif experimental_failed: + handle.write( + "Conclusion: default buffered rollout metrics passed, but experimental materialize metrics failed; " + "keep materialize disabled unless investigated separately.\n" + ) + else: + handle.write("Conclusion: strict metrics gate failed; inspect per-mode failures before rollout.\n") + elif strict: + handle.write( + "Conclusion: default buffered rollout metrics passed. Keep fill_materialize_enabled as an explicit " + "experimental mode; metrics success alone is not sufficient for default materialize enablement.\n" + ) + else: + handle.write("Conclusion: perf-only run; rollout remains blocked until strict metrics gate passes.\n") + +if strict_failed: + raise SystemExit(2) +PY +} + +main() { + parse_args "$@" + apply_matrix_preset + validate_args + setup_output + write_environment "$@" + preflight_prometheus_query + + local index=0 + local mode workload profile concurrency case_dir address bucket status + while IFS= read -r mode; do + while IFS= read -r workload; do + while IFS= read -r profile; do + while IFS= read -r concurrency; do + case_dir="$(matrix_case_path "$mode" "$workload" "$profile" "$concurrency")" + address="$(address_for_index "$index")" + bucket="$(case_bucket_name "$index")" + if run_case "$mode" "$index" "$workload" "$profile" "$concurrency"; then + status="ok" + else + status="failed" + RUN_FAILURES=$((RUN_FAILURES + 1)) + fi + record_matrix_case "$mode" "$workload" "$profile" "$concurrency" \ + "$(effective_warp_objects "$workload" "$concurrency")" "$address" "$bucket" "$status" "$case_dir" + cleanup_case_artifacts "$case_dir" + index=$((index + 1)) + done < <(concurrency_values) + done < <(profile_values) + done < <(workload_values) + done < <(mode_values) + + if write_summaries; then + : + else + RUN_FAILURES=$((RUN_FAILURES + 1)) + fi + + log "Output dir: ${OUT_DIR}" + log "Mode summary: ${OUT_DIR}/mode_summary.csv" + log "Matrix cases: ${OUT_DIR}/matrix_cases.csv" + log "Matrix coverage: ${OUT_DIR}/matrix_coverage.md" + log "Cache metrics summary: ${OUT_DIR}/cache_metrics_summary.csv" + log "Cache metrics acceptance: ${OUT_DIR}/cache_metrics_acceptance.csv" + log "Cache metrics acceptance by case: ${OUT_DIR}/cache_metrics_acceptance_by_case.csv" + log "Default enablement readiness: ${OUT_DIR}/default_enablement_readiness.md" + + if [[ "$RUN_FAILURES" -gt 0 ]]; then + die "object data cache benchmark completed with ${RUN_FAILURES} failure(s)" + fi +} + +main "$@"