mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-11 23:56:53 +00:00
fix(cache): harden object data cache coordination (#5004)
* fix(cache): enforce projected entry capacity Refs: rustfs/backlog#1335 Co-Authored-By: heihutu <heihutu@gmail.com> * fix(cache): fence identity budget eviction by generation Refs rustfs/backlog#1334. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(cache): fence clear against concurrent fills Refs rustfs/backlog#1333 Co-Authored-By: heihutu <heihutu@gmail.com> * fix(cache): linearize memory reservation claims Co-Authored-By: heihutu <heihutu@gmail.com> * fix(cache): retain allocation memory claims Co-Authored-By: heihutu <heihutu@gmail.com> * fix(cache): publish memory snapshots by epoch Co-Authored-By: heihutu <heihutu@gmail.com> * fix(cache): coordinate cold object fills Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): fence metadata cache transition races Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
@@ -353,9 +353,12 @@ pub mod notification {
|
||||
|
||||
pub mod object {
|
||||
pub use crate::object_api::{
|
||||
BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectReader, ObjectInfo, ObjectMutationHook, ObjectOptions,
|
||||
PutObjReader, RangedDecompressReader, StreamConsumer, register_get_object_body_cache_hook, register_object_mutation_hook,
|
||||
BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource,
|
||||
GetObjectReader, ObjectInfo, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, StreamConsumer,
|
||||
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
};
|
||||
pub use crate::store::PreparedGetObjectReader;
|
||||
}
|
||||
|
||||
pub mod rebalance {
|
||||
|
||||
@@ -35,7 +35,7 @@ use crate::{
|
||||
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
|
||||
runtime::instance::{InstanceContext, bootstrap_ctx},
|
||||
runtime::sources as runtime_sources,
|
||||
set_disk::SetDisks,
|
||||
set_disk::{PreparedGetObjectMetadata, SetDisks},
|
||||
store::init_format::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
|
||||
};
|
||||
use futures::{
|
||||
@@ -403,6 +403,31 @@ impl crate::storage_api_contracts::object::ObjectIO for Sets {
|
||||
}
|
||||
|
||||
impl Sets {
|
||||
pub(crate) async fn prepare_get_object_reader_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PreparedGetObjectMetadata> {
|
||||
self.get_disks_by_key(object)
|
||||
.prepare_get_object_metadata(bucket, object, opts)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_object_reader_with_prepared_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
headers: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
metadata: PreparedGetObjectMetadata,
|
||||
) -> Result<GetObjectReader> {
|
||||
self.get_disks_by_key(object)
|
||||
.get_object_reader_with_prepared_metadata(bucket, object, range, headers, opts, metadata)
|
||||
.await
|
||||
}
|
||||
|
||||
/// `put_object` plus the rename_data old-size backfill
|
||||
/// (rustfs/backlog#1009); see `SetDisks::put_object_with_old_current_size`.
|
||||
pub async fn put_object_with_old_current_size(
|
||||
|
||||
@@ -137,6 +137,7 @@ pub(crate) const GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED: &str = "not_fou
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_NOT_READ_DATA: &str = "not_read_data";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_PART_NUMBER: &str = "part_number";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ: &str = "raw_data_movement_read";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_STALE_PUBLICATION: &str = "stale_publication";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_USABLE: &str = "usable";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_VERSION_ID: &str = "version_id";
|
||||
pub(crate) const GET_METADATA_CACHE_REASON_VERSION_SUSPENDED: &str = "version_suspended";
|
||||
@@ -386,6 +387,7 @@ mod tests {
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_NOT_READ_DATA, "not_read_data");
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_PART_NUMBER, "part_number");
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, "raw_data_movement_read");
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_STALE_PUBLICATION, "stale_publication");
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_USABLE, "usable");
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_VERSION_ID, "version_id");
|
||||
assert_eq!(GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, "version_suspended");
|
||||
|
||||
@@ -20,11 +20,17 @@
|
||||
//! probing earlier would require a second metadata fan-out, and probing later
|
||||
//! (after the reader is built) means a hit no longer saves any disk I/O.
|
||||
|
||||
use crate::object_api::ObjectInfo;
|
||||
use crate::object_api::hook_slot::HookSlot;
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
use bytes::Bytes;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
|
||||
tokio::task_local! {
|
||||
static SKIP_GET_OBJECT_BODY_CACHE_HOOK: bool;
|
||||
}
|
||||
|
||||
/// Serves full-object GET bodies from a cache keyed by object identity.
|
||||
///
|
||||
/// Implementations must validate identity (etag/version/size) against the
|
||||
@@ -62,14 +68,263 @@ pub fn register_get_object_body_cache_hook(hook: Arc<dyn GetObjectBodyCacheHook>
|
||||
);
|
||||
}
|
||||
|
||||
/// Unregister the process-wide GET body cache hook.
|
||||
///
|
||||
/// Config reloads use this when body caching becomes disabled so an adapter
|
||||
/// retained by the previous configuration cannot continue serving stale hits.
|
||||
pub fn unregister_get_object_body_cache_hook() {
|
||||
GET_OBJECT_BODY_CACHE_HOOK.clear();
|
||||
}
|
||||
|
||||
/// Probes the registered hook against an already resolved metadata snapshot.
|
||||
/// Staged GET callers use this once before suppressing the nested reader probe,
|
||||
/// preserving the legacy hook contract.
|
||||
#[non_exhaustive]
|
||||
pub enum GetObjectBodyCacheHookLookup {
|
||||
Ineligible,
|
||||
Absent,
|
||||
Miss,
|
||||
Hit(Bytes),
|
||||
}
|
||||
|
||||
/// Returns the complete plaintext length only when the request can safely use
|
||||
/// the body-cache hook. Callers may use this before conditional decisions so
|
||||
/// ineligible reads retain the established reader-path error precedence.
|
||||
pub fn get_object_body_cache_plaintext_len(
|
||||
range: &Option<HTTPRangeSpec>,
|
||||
opts: &ObjectOptions,
|
||||
info: &ObjectInfo,
|
||||
) -> Option<i64> {
|
||||
crate::set_disk::body_cache_plaintext_len(range, opts, info)
|
||||
}
|
||||
|
||||
pub async fn lookup_get_object_body_cache_hook(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: &Option<HTTPRangeSpec>,
|
||||
opts: &ObjectOptions,
|
||||
info: &ObjectInfo,
|
||||
) -> GetObjectBodyCacheHookLookup {
|
||||
let Some(plaintext_len) = get_object_body_cache_plaintext_len(range, opts, info) else {
|
||||
return GetObjectBodyCacheHookLookup::Ineligible;
|
||||
};
|
||||
let Some(hook) = get_object_body_cache_hook() else {
|
||||
return GetObjectBodyCacheHookLookup::Absent;
|
||||
};
|
||||
match hook.lookup(bucket, object, info).await {
|
||||
Some(body) if i64::try_from(body.len()).is_ok_and(|body_len| body_len == plaintext_len) => {
|
||||
GetObjectBodyCacheHookLookup::Hit(body)
|
||||
}
|
||||
Some(_) | None => GetObjectBodyCacheHookLookup::Miss,
|
||||
}
|
||||
}
|
||||
|
||||
/// The registered hook, if any.
|
||||
pub(crate) fn get_object_body_cache_hook() -> Option<Arc<dyn GetObjectBodyCacheHook>> {
|
||||
GET_OBJECT_BODY_CACHE_HOOK.get()
|
||||
}
|
||||
|
||||
pub(crate) fn get_object_body_cache_hook_suppressed() -> bool {
|
||||
SKIP_GET_OBJECT_BODY_CACHE_HOOK.try_with(|skip| *skip).unwrap_or(false)
|
||||
}
|
||||
|
||||
pub(crate) async fn without_get_object_body_cache_hook<F>(future: F) -> F::Output
|
||||
where
|
||||
F: Future,
|
||||
{
|
||||
SKIP_GET_OBJECT_BODY_CACHE_HOOK.scope(true, future).await
|
||||
}
|
||||
|
||||
/// Test-only: unregister the hook so tests can register and clear the slot
|
||||
/// deterministically without leaking a hook into unrelated tests.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn clear_get_object_body_cache_hook() {
|
||||
GET_OBJECT_BODY_CACHE_HOOK.clear();
|
||||
unregister_get_object_body_cache_hook();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
struct LegacyHook {
|
||||
calls: AtomicUsize,
|
||||
body: Option<Bytes>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl GetObjectBodyCacheHook for LegacyHook {
|
||||
async fn lookup(&self, _bucket: &str, _object: &str, _info: &ObjectInfo) -> Option<Bytes> {
|
||||
self.calls.fetch_add(1, Ordering::Relaxed);
|
||||
self.body.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn staged_probe_preserves_legacy_hook_hit_once() {
|
||||
clear_get_object_body_cache_hook();
|
||||
let hook = Arc::new(LegacyHook {
|
||||
calls: AtomicUsize::new(0),
|
||||
body: Some(Bytes::from_static(b"legacy")),
|
||||
});
|
||||
register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
|
||||
|
||||
let info = ObjectInfo {
|
||||
size: 6,
|
||||
actual_size: 6,
|
||||
..Default::default()
|
||||
};
|
||||
let GetObjectBodyCacheHookLookup::Hit(body) =
|
||||
lookup_get_object_body_cache_hook("bucket", "object", &None, &ObjectOptions::default(), &info).await
|
||||
else {
|
||||
panic!("legacy hook hit must be returned");
|
||||
};
|
||||
assert_eq!(body, Bytes::from_static(b"legacy"));
|
||||
assert_eq!(hook.calls.load(Ordering::Relaxed), 1);
|
||||
clear_get_object_body_cache_hook();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn staged_probe_treats_wrong_length_legacy_body_as_authoritative_miss() {
|
||||
clear_get_object_body_cache_hook();
|
||||
let hook = Arc::new(LegacyHook {
|
||||
calls: AtomicUsize::new(0),
|
||||
body: Some(Bytes::from_static(b"short")),
|
||||
});
|
||||
register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
|
||||
let info = ObjectInfo {
|
||||
size: 6,
|
||||
actual_size: 6,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(matches!(
|
||||
lookup_get_object_body_cache_hook("bucket", "object", &None, &ObjectOptions::default(), &info).await,
|
||||
GetObjectBodyCacheHookLookup::Miss
|
||||
));
|
||||
assert_eq!(hook.calls.load(Ordering::Relaxed), 1);
|
||||
clear_get_object_body_cache_hook();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn staged_probe_preserves_legacy_hook_miss_once() {
|
||||
clear_get_object_body_cache_hook();
|
||||
let hook = Arc::new(LegacyHook {
|
||||
calls: AtomicUsize::new(0),
|
||||
body: None,
|
||||
});
|
||||
register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
|
||||
|
||||
let info = ObjectInfo {
|
||||
size: 4,
|
||||
actual_size: 4,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(matches!(
|
||||
lookup_get_object_body_cache_hook("bucket", "object", &None, &ObjectOptions::default(), &info).await,
|
||||
GetObjectBodyCacheHookLookup::Miss
|
||||
));
|
||||
assert_eq!(hook.calls.load(Ordering::Relaxed), 1);
|
||||
clear_get_object_body_cache_hook();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn staged_probe_bypasses_raw_movement_and_restore_reads() {
|
||||
clear_get_object_body_cache_hook();
|
||||
let hook = Arc::new(LegacyHook {
|
||||
calls: AtomicUsize::new(0),
|
||||
body: Some(Bytes::from_static(b"body")),
|
||||
});
|
||||
register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
|
||||
let info = ObjectInfo {
|
||||
size: 4,
|
||||
actual_size: 4,
|
||||
..Default::default()
|
||||
};
|
||||
let mut restore = ObjectOptions::default();
|
||||
restore.transition.restore_request.days = Some(1);
|
||||
let cases = [
|
||||
ObjectOptions {
|
||||
raw_data_movement_read: true,
|
||||
..Default::default()
|
||||
},
|
||||
ObjectOptions {
|
||||
data_movement: true,
|
||||
..Default::default()
|
||||
},
|
||||
restore,
|
||||
];
|
||||
|
||||
for opts in &cases {
|
||||
assert!(matches!(
|
||||
lookup_get_object_body_cache_hook("bucket", "object", &None, opts, &info).await,
|
||||
GetObjectBodyCacheHookLookup::Ineligible
|
||||
));
|
||||
}
|
||||
assert_eq!(hook.calls.load(Ordering::Relaxed), 0);
|
||||
clear_get_object_body_cache_hook();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn staged_probe_bypasses_pre_hook_early_return_objects() {
|
||||
clear_get_object_body_cache_hook();
|
||||
let hook = Arc::new(LegacyHook {
|
||||
calls: AtomicUsize::new(0),
|
||||
body: Some(Bytes::from_static(b"body")),
|
||||
});
|
||||
register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
|
||||
let delete_marker = ObjectInfo {
|
||||
delete_marker: true,
|
||||
size: 4,
|
||||
actual_size: 4,
|
||||
..Default::default()
|
||||
};
|
||||
let zero = ObjectInfo::default();
|
||||
let inline = ObjectInfo {
|
||||
inlined: true,
|
||||
size: 4,
|
||||
actual_size: 4,
|
||||
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
|
||||
number: 1,
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
let version_only = ObjectInfo {
|
||||
version_only: true,
|
||||
size: 4,
|
||||
actual_size: 4,
|
||||
..Default::default()
|
||||
};
|
||||
let metadata_only = ObjectInfo {
|
||||
metadata_only: true,
|
||||
size: 4,
|
||||
actual_size: 4,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for info in [&delete_marker, &zero, &inline, &version_only, &metadata_only] {
|
||||
assert!(matches!(
|
||||
lookup_get_object_body_cache_hook("bucket", "object", &None, &ObjectOptions::default(), info).await,
|
||||
GetObjectBodyCacheHookLookup::Ineligible
|
||||
));
|
||||
}
|
||||
assert_eq!(hook.calls.load(Ordering::Relaxed), 0);
|
||||
clear_get_object_body_cache_hook();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn staged_reader_scope_suppresses_only_the_nested_probe() {
|
||||
assert!(!get_object_body_cache_hook_suppressed());
|
||||
without_get_object_body_cache_hook(async {
|
||||
assert!(get_object_body_cache_hook_suppressed());
|
||||
})
|
||||
.await;
|
||||
assert!(!get_object_body_cache_hook_suppressed());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,8 +61,7 @@ impl<T: ?Sized> HookSlot<T> {
|
||||
self.inner.read().unwrap_or_else(|poisoned| poisoned.into_inner()).clone()
|
||||
}
|
||||
|
||||
/// Clears the slot. Test-only: production never unregisters a hook.
|
||||
#[cfg(test)]
|
||||
/// Clears the slot during feature disable or test cleanup.
|
||||
pub(crate) fn clear(&self) {
|
||||
*self.inner.write().unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
|
||||
}
|
||||
|
||||
@@ -60,9 +60,14 @@ mod types;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use body_cache_hook::clear_get_object_body_cache_hook;
|
||||
pub(crate) use body_cache_hook::get_object_body_cache_hook;
|
||||
pub use body_cache_hook::{GetObjectBodyCacheHook, register_get_object_body_cache_hook};
|
||||
pub use body_cache_hook::{
|
||||
GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
|
||||
register_get_object_body_cache_hook, unregister_get_object_body_cache_hook,
|
||||
};
|
||||
pub(crate) use body_cache_hook::{
|
||||
get_object_body_cache_hook, get_object_body_cache_hook_suppressed, without_get_object_body_cache_hook,
|
||||
};
|
||||
pub(crate) use object_mutation_hook::notify_object_mutation;
|
||||
pub use object_mutation_hook::{ObjectMutationHook, register_object_mutation_hook};
|
||||
pub use object_mutation_hook::{ObjectMutationHook, register_object_mutation_hook, unregister_object_mutation_hook};
|
||||
pub use readers::*;
|
||||
pub use types::*;
|
||||
|
||||
@@ -56,6 +56,14 @@ pub fn register_object_mutation_hook(hook: Arc<dyn ObjectMutationHook>) {
|
||||
);
|
||||
}
|
||||
|
||||
/// Unregister the process-wide object mutation hook.
|
||||
///
|
||||
/// Config reloads use this when body caching becomes disabled so the previous
|
||||
/// adapter and its cached plaintext bodies are not retained until their TTL.
|
||||
pub fn unregister_object_mutation_hook() {
|
||||
OBJECT_MUTATION_HOOK.clear();
|
||||
}
|
||||
|
||||
/// The registered hook, if any.
|
||||
fn object_mutation_hook() -> Option<Arc<dyn ObjectMutationHook>> {
|
||||
OBJECT_MUTATION_HOOK.get()
|
||||
@@ -71,13 +79,6 @@ pub(crate) async fn notify_object_mutation(bucket: &str, object: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Test-only: unregister the hook so tests can register and clear the slot
|
||||
/// deterministically without leaking a hook into unrelated tests.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn clear_object_mutation_hook() {
|
||||
OBJECT_MUTATION_HOOK.clear();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -97,7 +98,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(object_mutation_hook)]
|
||||
async fn notify_invokes_registered_hook_with_identity() {
|
||||
clear_object_mutation_hook();
|
||||
unregister_object_mutation_hook();
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
register_object_mutation_hook(Arc::new(RecordingHook {
|
||||
calls: Arc::clone(&calls),
|
||||
@@ -106,14 +107,35 @@ mod tests {
|
||||
notify_object_mutation("bucket", "photos/a.jpg").await;
|
||||
|
||||
assert_eq!(&*calls.lock().unwrap(), &[("bucket".to_string(), "photos/a.jpg".to_string())]);
|
||||
clear_object_mutation_hook();
|
||||
unregister_object_mutation_hook();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(object_mutation_hook)]
|
||||
async fn notify_without_registered_hook_is_noop() {
|
||||
clear_object_mutation_hook();
|
||||
// Must not panic when no hook is installed (the cache feature is off).
|
||||
async fn unregister_prevents_later_notifications() {
|
||||
unregister_object_mutation_hook();
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
register_object_mutation_hook(Arc::new(RecordingHook {
|
||||
calls: Arc::clone(&calls),
|
||||
}));
|
||||
unregister_object_mutation_hook();
|
||||
|
||||
notify_object_mutation("bucket", "object").await;
|
||||
|
||||
assert!(calls.lock().unwrap().is_empty(), "an unregistered hook must receive no mutation callback");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(object_mutation_hook)]
|
||||
fn unregister_releases_the_previous_hook() {
|
||||
unregister_object_mutation_hook();
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
let hook = Arc::new(RecordingHook { calls });
|
||||
let weak = Arc::downgrade(&hook);
|
||||
register_object_mutation_hook(hook);
|
||||
|
||||
assert!(weak.upgrade().is_some());
|
||||
unregister_object_mutation_hook();
|
||||
assert!(weak.upgrade().is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -307,6 +307,17 @@ pub struct GetObjectReader {
|
||||
}
|
||||
|
||||
impl GetObjectReader {
|
||||
/// Builds a fully materialized reader from a cache-coordinated body.
|
||||
pub fn from_cache_body(mut object_info: ObjectInfo, body: Bytes) -> Result<Self> {
|
||||
object_info.size = i64::try_from(body.len()).map_err(|_| Error::other("cached GET body length exceeds i64::MAX"))?;
|
||||
Ok(Self {
|
||||
stream: Box::new(std::io::Cursor::new(body.clone())),
|
||||
object_info,
|
||||
buffered_body: Some(body),
|
||||
body_source: GetObjectBodySource::HookServed,
|
||||
})
|
||||
}
|
||||
|
||||
/// True when `buffered_body` is the body the cache hook served. The app
|
||||
/// layer serves it as the object-data-cache source without a second lookup.
|
||||
pub fn is_cache_hook_served(&self) -> bool {
|
||||
@@ -1674,6 +1685,40 @@ mod tests {
|
||||
bytes
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_body_uses_plaintext_length_for_compressed_metadata() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
let object_info = ObjectInfo {
|
||||
size: 3,
|
||||
actual_size: 11,
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(object_info.is_compressed());
|
||||
|
||||
let body = Bytes::from_static(b"hello world");
|
||||
let mut reader =
|
||||
GetObjectReader::from_cache_body(object_info, body.clone()).expect("cache body length must fit in object metadata");
|
||||
|
||||
assert_eq!(reader.body_source, GetObjectBodySource::HookServed);
|
||||
assert_eq!(reader.buffered_body.as_ref(), Some(&body));
|
||||
assert_eq!(reader.object_info.size, 11);
|
||||
assert_eq!(reader.object_info.actual_size, 11);
|
||||
assert!(reader.object_info.is_compressed());
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("cache body should stream");
|
||||
assert_eq!(restored, body);
|
||||
}
|
||||
|
||||
/// Regression for the #4576 fallout: the encrypt side persists a random
|
||||
/// SSE-C nonce, and this reader-side resolver must read it back — falling
|
||||
/// back to the deterministic legacy nonce only when no IV was stored.
|
||||
|
||||
@@ -62,7 +62,7 @@ use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tokio::sync::{Mutex, Notify, RwLock};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::client::transition_api::{ReadCloser, ReaderImpl};
|
||||
@@ -141,6 +141,40 @@ struct MockWarmBackendInner {
|
||||
objects: Mutex<HashMap<String, MockStoredObject>>,
|
||||
faults: Mutex<FaultConfig>,
|
||||
op_log: Mutex<Vec<MockWarmOp>>,
|
||||
put_versions: Mutex<Vec<(String, String)>>,
|
||||
remove_versions: Mutex<Vec<(String, String)>>,
|
||||
put_barrier: Mutex<Option<Arc<MockPutBarrierState>>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct MockPutBarrierState {
|
||||
arrived: Notify,
|
||||
release: Notify,
|
||||
}
|
||||
|
||||
/// One-shot barrier that pauses a mock tier PUT after storing its remote body.
|
||||
pub struct MockPutBarrier {
|
||||
state: Arc<MockPutBarrierState>,
|
||||
}
|
||||
|
||||
impl MockPutBarrier {
|
||||
/// Wait until the remote body is stored and the PUT is paused before returning.
|
||||
pub async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("mock tier PUT should reach the deterministic barrier");
|
||||
}
|
||||
|
||||
/// Release the paused PUT.
|
||||
pub fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MockPutBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
/// In-memory [`WarmBackend`] for lifecycle / tiering integration tests.
|
||||
@@ -159,6 +193,13 @@ impl MockWarmBackend {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Arm a one-shot pause after the next tier PUT stores its remote body.
|
||||
pub async fn arm_put_barrier(&self) -> MockPutBarrier {
|
||||
let state = Arc::new(MockPutBarrierState::default());
|
||||
*self.inner.put_barrier.lock().await = Some(Arc::clone(&state));
|
||||
MockPutBarrier { state }
|
||||
}
|
||||
|
||||
// ---- fault injection -------------------------------------------------
|
||||
|
||||
/// Replace the entire fault configuration.
|
||||
@@ -232,6 +273,16 @@ impl MockWarmBackend {
|
||||
.count()
|
||||
}
|
||||
|
||||
/// Return the exact object/version pairs produced by successful tier PUTs.
|
||||
pub async fn put_versions(&self) -> Vec<(String, String)> {
|
||||
self.inner.put_versions.lock().await.clone()
|
||||
}
|
||||
|
||||
/// Return the exact object/version pairs passed to successful tier removes.
|
||||
pub async fn remove_versions(&self) -> Vec<(String, String)> {
|
||||
self.inner.remove_versions.lock().await.clone()
|
||||
}
|
||||
|
||||
/// Number of `get` calls recorded — useful to assert restore reads hit the
|
||||
/// local copy rather than the remote tier.
|
||||
pub async fn get_count(&self) -> usize {
|
||||
@@ -358,6 +409,11 @@ impl WarmBackend for MockWarmBackend {
|
||||
self.precondition().await?;
|
||||
let bytes = self.read_bytes(r).await?;
|
||||
let version = self.put_bytes(object, bytes, HashMap::new()).await;
|
||||
self.inner
|
||||
.put_versions
|
||||
.lock()
|
||||
.await
|
||||
.push((object.to_string(), version.clone()));
|
||||
self.record(MockWarmOp::Put {
|
||||
object: object.to_string(),
|
||||
})
|
||||
@@ -398,6 +454,16 @@ impl WarmBackend for MockWarmBackend {
|
||||
metadata.insert("x-amz-object-lock-legal-hold".to_string(), opts.legalhold.as_str().to_string());
|
||||
}
|
||||
let version = self.put_bytes(object, bytes, metadata).await;
|
||||
self.inner
|
||||
.put_versions
|
||||
.lock()
|
||||
.await
|
||||
.push((object.to_string(), version.clone()));
|
||||
let barrier = self.inner.put_barrier.lock().await.take();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
self.record(MockWarmOp::Put {
|
||||
object: object.to_string(),
|
||||
})
|
||||
@@ -427,9 +493,22 @@ impl WarmBackend for MockWarmBackend {
|
||||
Ok(tokio::io::BufReader::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, _rv: &str) -> Result<(), std::io::Error> {
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
self.precondition().await?;
|
||||
self.inner.objects.lock().await.remove(object);
|
||||
let mut objects = self.inner.objects.lock().await;
|
||||
if let Some(stored) = objects.get(object)
|
||||
&& !rv.is_empty()
|
||||
&& stored.remote_version_id != rv
|
||||
{
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "NoSuchVersion"));
|
||||
}
|
||||
objects.remove(object);
|
||||
drop(objects);
|
||||
self.inner
|
||||
.remove_versions
|
||||
.lock()
|
||||
.await
|
||||
.push((object.to_string(), rv.to_string()));
|
||||
self.record(MockWarmOp::Remove {
|
||||
object: object.to_string(),
|
||||
})
|
||||
|
||||
@@ -160,7 +160,7 @@ use rustfs_utils::{
|
||||
};
|
||||
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_RESTORE};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::hash::Hash;
|
||||
use std::hash::{BuildHasher, Hash, Hasher};
|
||||
use std::mem::{self};
|
||||
use std::pin::Pin;
|
||||
use std::sync::OnceLock;
|
||||
@@ -547,6 +547,7 @@ const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750);
|
||||
const GET_OBJECT_METADATA_CACHE_TTL: Duration = Duration::from_secs(2); // Increased from 250ms to 2s
|
||||
const DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: usize = 4096; // Increased from 1024 to 4096
|
||||
const ENV_RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: &str = "RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES";
|
||||
const GET_OBJECT_METADATA_CACHE_FENCE_SHARDS: u16 = 4096;
|
||||
|
||||
// --- Codec Streaming Configuration ---
|
||||
|
||||
@@ -623,12 +624,284 @@ mod core;
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
mod read;
|
||||
mod replication;
|
||||
pub(crate) mod shard_source;
|
||||
|
||||
pub use ops::heal_walk::HealWalkVersion;
|
||||
|
||||
pub(crate) struct PreparedGetObjectMetadata {
|
||||
fi: FileInfo,
|
||||
files: Vec<FileInfo>,
|
||||
disks: Vec<Option<DiskStore>>,
|
||||
object_info: Option<ObjectInfo>,
|
||||
}
|
||||
|
||||
impl PreparedGetObjectMetadata {
|
||||
pub(crate) fn object_info(&self) -> &ObjectInfo {
|
||||
self.object_info
|
||||
.as_ref()
|
||||
.expect("prepared GET metadata must retain its ObjectInfo until consumed")
|
||||
}
|
||||
|
||||
pub(crate) fn take_object_info(&mut self) -> ObjectInfo {
|
||||
self.object_info
|
||||
.take()
|
||||
.expect("prepared GET metadata ObjectInfo must be consumed exactly once")
|
||||
}
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static PREPARED_GET_OBJECT_METADATA: std::cell::RefCell<Option<PreparedGetObjectMetadata>>;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
static GET_OBJECT_INFO_CONVERSIONS: Arc<AtomicU64>;
|
||||
}
|
||||
|
||||
fn build_get_object_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
|
||||
#[cfg(test)]
|
||||
let _ = GET_OBJECT_INFO_CONVERSIONS.try_with(|conversions| {
|
||||
conversions.fetch_add(1, Ordering::Relaxed);
|
||||
});
|
||||
ObjectInfo::from_file_info(fi, bucket, object, versioned)
|
||||
}
|
||||
|
||||
fn take_prepared_get_object_metadata() -> Option<PreparedGetObjectMetadata> {
|
||||
PREPARED_GET_OBJECT_METADATA
|
||||
.try_with(|prepared| prepared.borrow_mut().take())
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
async fn with_prepared_get_object_metadata<F>(metadata: PreparedGetObjectMetadata, future: F) -> F::Output
|
||||
where
|
||||
F: std::future::Future,
|
||||
{
|
||||
PREPARED_GET_OBJECT_METADATA
|
||||
.scope(std::cell::RefCell::new(Some(metadata)), future)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod prepared_get_object_metadata_tests {
|
||||
use super::*;
|
||||
use crate::ecstore_validation_blackbox::make_local_set_disks;
|
||||
use crate::object_api::{BLOCK_SIZE_V2, PutObjReader};
|
||||
use crate::set_disk::core::io_primitives::disk_call_counters;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use http::HeaderMap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_metadata_is_consumed_exactly_once() {
|
||||
let metadata = PreparedGetObjectMetadata {
|
||||
fi: FileInfo::default(),
|
||||
files: Vec::new(),
|
||||
disks: Vec::new(),
|
||||
object_info: None,
|
||||
};
|
||||
|
||||
with_prepared_get_object_metadata(metadata, async {
|
||||
assert!(take_prepared_get_object_metadata().is_some());
|
||||
assert!(take_prepared_get_object_metadata().is_none());
|
||||
})
|
||||
.await;
|
||||
assert!(take_prepared_get_object_metadata().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn prepared_reader_reuses_metadata_fanout_exactly_once() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let bucket = "prepared-metadata-fanout";
|
||||
let object = "prepared-metadata-fanout-object.bin";
|
||||
let payload = b"prepared-metadata-fanout-payload-".repeat(40_000);
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let conversions = Arc::new(AtomicU64::new(0));
|
||||
let restored = GET_OBJECT_INFO_CONVERSIONS
|
||||
.scope(Arc::clone(&conversions), async {
|
||||
let metadata = set_disks
|
||||
.prepare_get_object_metadata(bucket, object, &opts)
|
||||
.await
|
||||
.expect("prepared metadata should resolve");
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
4,
|
||||
"preparation should fan out to each online disk exactly once"
|
||||
);
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader_with_prepared_metadata(bucket, object, None, HeaderMap::new(), &opts, metadata)
|
||||
.await
|
||||
.expect("prepared body reader should open");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("prepared body should stream");
|
||||
restored
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(restored, payload);
|
||||
assert_eq!(
|
||||
conversions.load(Ordering::Relaxed),
|
||||
1,
|
||||
"prepared reader must consume the ObjectInfo built during metadata preparation"
|
||||
);
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
4,
|
||||
"reader construction must consume prepared metadata instead of repeating the fanout"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn prepared_reader_rebuilds_object_info_when_precomputed_value_is_absent() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let bucket = "prepared-object-info-fallback";
|
||||
let object = "prepared-object-info-fallback.bin";
|
||||
let payload = b"prepared-object-info-fallback-payload".repeat(4_000);
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
let conversions = Arc::new(AtomicU64::new(0));
|
||||
let restored = GET_OBJECT_INFO_CONVERSIONS
|
||||
.scope(Arc::clone(&conversions), async {
|
||||
let mut metadata = set_disks
|
||||
.prepare_get_object_metadata(bucket, object, &opts)
|
||||
.await
|
||||
.expect("prepared metadata should resolve");
|
||||
metadata.object_info = None;
|
||||
let mut reader = set_disks
|
||||
.get_object_reader_with_prepared_metadata(bucket, object, None, HeaderMap::new(), &opts, metadata)
|
||||
.await
|
||||
.expect("reader should rebuild missing prepared ObjectInfo");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("fallback reader should stream");
|
||||
restored
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(restored, payload);
|
||||
assert_eq!(
|
||||
conversions.load(Ordering::Relaxed),
|
||||
2,
|
||||
"missing precomputed ObjectInfo must trigger exactly one structural fallback rebuild"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn prepared_reader_restores_full_body_with_one_offline_shard() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let bucket = "prepared-reader-offline-shard";
|
||||
let object = "prepared-reader-offline-shard-object.bin";
|
||||
let payload = (0..(BLOCK_SIZE_V2 + 321))
|
||||
.map(|idx| ((idx * 19) % 251) as u8)
|
||||
.collect::<Vec<_>>();
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
set_disks.disks.write().await[0] = None;
|
||||
|
||||
let metadata = set_disks
|
||||
.prepare_get_object_metadata(bucket, object, &opts)
|
||||
.await
|
||||
.expect("prepared metadata should tolerate one offline shard");
|
||||
let mut reader = set_disks
|
||||
.get_object_reader_with_prepared_metadata(bucket, object, None, HeaderMap::new(), &opts, metadata)
|
||||
.await
|
||||
.expect("prepared body reader should open with one offline shard");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("degraded prepared body should stream");
|
||||
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(crate) async fn prepare_get_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PreparedGetObjectMetadata> {
|
||||
let (fi, files, disks) = self.get_object_fileinfo(bucket, object, opts, true, true).await?;
|
||||
let object_info = build_get_object_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
Ok(PreparedGetObjectMetadata {
|
||||
fi,
|
||||
files,
|
||||
disks,
|
||||
object_info: Some(object_info),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn get_object_reader_with_prepared_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
headers: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
metadata: PreparedGetObjectMetadata,
|
||||
) -> Result<GetObjectReader> {
|
||||
with_prepared_get_object_metadata(metadata, self.get_object_reader(bucket, object, range, headers, opts)).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds)
|
||||
/// Defaults to 30 seconds if not set or invalid
|
||||
/// Lock acquisition timeout. Cached: this is consulted on every object
|
||||
@@ -1742,6 +2015,8 @@ pub struct SetDisks {
|
||||
pub format: FormatV3,
|
||||
disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
|
||||
get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>,
|
||||
get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState,
|
||||
get_object_metadata_cache_generations: Arc<[AtomicU64]>,
|
||||
pub lockers: Vec<Arc<dyn LockClient>>,
|
||||
local_lock_manager: Arc<rustfs_lock::GlobalLockManager>,
|
||||
/// Per-instance runtime context (Phase 5, backlog#939).
|
||||
@@ -1761,21 +2036,109 @@ pub struct SetDisks {
|
||||
capacity_dirty_generation: Arc<AtomicU64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
struct GetObjectMetadataCacheKey {
|
||||
bucket: Arc<str>,
|
||||
object: Arc<str>,
|
||||
generation: u64,
|
||||
hash: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct GetObjectMetadataCacheGeneration {
|
||||
index: usize,
|
||||
value: u64,
|
||||
hash: u64,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct MetadataCacheInvalidationProbeState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
count: AtomicU64,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct MetadataCacheInvalidationProbe {
|
||||
state: Arc<MetadataCacheInvalidationProbeState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static METADATA_CACHE_INVALIDATION_PROBE: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<MetadataCacheInvalidationProbeState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl MetadataCacheInvalidationProbe {
|
||||
fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(MetadataCacheInvalidationProbeState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
count: AtomicU64::new(0),
|
||||
});
|
||||
let mut slot = METADATA_CACHE_INVALIDATION_PROBE
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("metadata cache invalidation probe mutex should not poison");
|
||||
assert!(
|
||||
slot.is_none(),
|
||||
"metadata cache invalidation probe must be installed by one test at a time"
|
||||
);
|
||||
*slot = Some(Arc::clone(&state));
|
||||
drop(slot);
|
||||
Self { state }
|
||||
}
|
||||
|
||||
fn count(&self) -> u64 {
|
||||
self.state.count.load(Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for MetadataCacheInvalidationProbe {
|
||||
fn drop(&mut self) {
|
||||
let mut slot = METADATA_CACHE_INVALIDATION_PROBE
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("metadata cache invalidation probe mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn record_metadata_cache_invalidation(bucket: &str, object: &str) {
|
||||
let probe = METADATA_CACHE_INVALIDATION_PROBE
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("metadata cache invalidation probe mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|probe| probe.bucket == bucket && probe.object == object)
|
||||
.cloned();
|
||||
if let Some(probe) = probe {
|
||||
probe.count.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
impl GetObjectMetadataCacheKey {
|
||||
fn new(bucket: &str, object: &str) -> Self {
|
||||
fn new(bucket: &str, object: &str, generation: GetObjectMetadataCacheGeneration) -> Self {
|
||||
Self {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object),
|
||||
generation: generation.value,
|
||||
hash: generation.hash,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Hash for GetObjectMetadataCacheKey {
|
||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||
self.hash.hash(state);
|
||||
self.generation.hash(state);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct GetObjectMetadataCacheEntry {
|
||||
#[allow(dead_code)] // Kept for debugging; moka handles TTL internally
|
||||
@@ -1803,10 +2166,50 @@ impl DiskHealthEntry {
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
fn get_object_metadata_cache_hash(&self, bucket: &str, object: &str) -> u64 {
|
||||
let mut hasher = self.get_object_metadata_cache_hash_builder.build_hasher();
|
||||
bucket.hash(&mut hasher);
|
||||
object.hash(&mut hasher);
|
||||
hasher.finish()
|
||||
}
|
||||
|
||||
fn get_object_metadata_cache_generation(&self, bucket: &str, object: &str) -> Option<GetObjectMetadataCacheGeneration> {
|
||||
let hash = self.get_object_metadata_cache_hash(bucket, object);
|
||||
let hash_bytes = hash.to_le_bytes();
|
||||
let index = usize::from(u16::from_le_bytes([hash_bytes[0], hash_bytes[1]]) % GET_OBJECT_METADATA_CACHE_FENCE_SHARDS);
|
||||
let value = self.get_object_metadata_cache_generations[index].load(Ordering::Acquire);
|
||||
(value != u64::MAX).then_some(GetObjectMetadataCacheGeneration { index, value, hash })
|
||||
}
|
||||
|
||||
fn is_get_object_metadata_cache_generation_current(&self, generation: GetObjectMetadataCacheGeneration) -> bool {
|
||||
self.get_object_metadata_cache_generations[generation.index].load(Ordering::Acquire) == generation.value
|
||||
}
|
||||
|
||||
async fn invalidate_get_object_metadata_cache(&self, bucket: &str, object: &str) {
|
||||
let hash = self.get_object_metadata_cache_hash(bucket, object);
|
||||
let hash_bytes = hash.to_le_bytes();
|
||||
let index = usize::from(u16::from_le_bytes([hash_bytes[0], hash_bytes[1]]) % GET_OBJECT_METADATA_CACHE_FENCE_SHARDS);
|
||||
let generation = &self.get_object_metadata_cache_generations[index];
|
||||
let previous = match generation.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| current.checked_add(1)) {
|
||||
Ok(previous) | Err(previous) => previous,
|
||||
};
|
||||
let previous = GetObjectMetadataCacheGeneration {
|
||||
index,
|
||||
value: previous,
|
||||
hash,
|
||||
};
|
||||
self.get_object_metadata_cache
|
||||
.invalidate(&GetObjectMetadataCacheKey::new(bucket, object))
|
||||
.invalidate(&GetObjectMetadataCacheKey::new(bucket, object, previous))
|
||||
.await;
|
||||
#[cfg(test)]
|
||||
record_metadata_cache_invalidation(bucket, object);
|
||||
}
|
||||
|
||||
fn invalidate_all_get_object_metadata_cache(&self) {
|
||||
for generation in self.get_object_metadata_cache_generations.iter() {
|
||||
let _ = generation.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| current.checked_add(1));
|
||||
}
|
||||
self.get_object_metadata_cache.invalidate_all();
|
||||
}
|
||||
|
||||
async fn acquire_read_lock_diag(&self, op: &'static str, bucket: &str, object: &str) -> Result<ObjectLockDiagGuard> {
|
||||
@@ -1953,6 +2356,12 @@ impl SetDisks {
|
||||
.max_capacity(get_object_metadata_cache_max_entries() as u64)
|
||||
.time_to_live(GET_OBJECT_METADATA_CACHE_TTL)
|
||||
.build(),
|
||||
get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState::new(),
|
||||
get_object_metadata_cache_generations: Arc::from(
|
||||
(0..usize::from(GET_OBJECT_METADATA_CACHE_FENCE_SHARDS))
|
||||
.map(|_| AtomicU64::new(0))
|
||||
.collect::<Vec<_>>(),
|
||||
),
|
||||
lockers,
|
||||
// Sourced from the instance context so each instance owns its lock
|
||||
// namespace (Phase 5 Slice 3). Single-instance: ctx aliases the
|
||||
|
||||
@@ -21,8 +21,9 @@
|
||||
|
||||
use super::super::*;
|
||||
|
||||
use crate::bucket::lifecycle::tier_sweeper::delete_object_from_remote_tier_idempotent;
|
||||
use crate::disk::OldCurrentSize;
|
||||
use crate::object_api::GetObjectBodySource;
|
||||
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
|
||||
|
||||
/// Length of the full plaintext body when — and only when — this read's output
|
||||
/// is exactly the object's complete plaintext, so the app-layer body cache may
|
||||
@@ -61,6 +62,11 @@ fn full_object_plaintext_len(range: &Option<HTTPRangeSpec>, opts: &ObjectOptions
|
||||
|| crate::object_api::restore_request_active(opts)
|
||||
|| object_info.is_encrypted()
|
||||
|| object_info.is_remote()
|
||||
|| object_info.delete_marker
|
||||
|| object_info.size == 0
|
||||
|| object_info.version_only
|
||||
|| object_info.metadata_only
|
||||
|| object_info.is_inline_fast_path_eligible()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
@@ -72,6 +78,14 @@ fn full_object_plaintext_len(range: &Option<HTTPRangeSpec>, opts: &ObjectOptions
|
||||
Some(object_info.size)
|
||||
}
|
||||
|
||||
pub(crate) fn body_cache_plaintext_len(
|
||||
range: &Option<HTTPRangeSpec>,
|
||||
opts: &ObjectOptions,
|
||||
object_info: &ObjectInfo,
|
||||
) -> Option<i64> {
|
||||
full_object_plaintext_len(range, opts, object_info)
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
type Error = Error;
|
||||
@@ -131,16 +145,21 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
};
|
||||
|
||||
let metadata_stage_start = Instant::now();
|
||||
let (fi, files, disks) = match self.get_object_fileinfo(bucket, object, opts, true, true).await {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_stage_start.elapsed().as_secs_f64());
|
||||
record_get_object_pipeline_failure(GET_STAGE_METADATA, classify_storage_error(&err));
|
||||
return Err(to_object_err(err, vec![bucket, object]));
|
||||
let (fi, files, disks, prepared_object_info) = if let Some(prepared) = take_prepared_get_object_metadata() {
|
||||
(prepared.fi, prepared.files, prepared.disks, prepared.object_info)
|
||||
} else {
|
||||
match self.get_object_fileinfo(bucket, object, opts, true, true).await {
|
||||
Ok((fi, files, disks)) => (fi, files, disks, None),
|
||||
Err(err) => {
|
||||
rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_stage_start.elapsed().as_secs_f64());
|
||||
record_get_object_pipeline_failure(GET_STAGE_METADATA, classify_storage_error(&err));
|
||||
return Err(to_object_err(err, vec![bucket, object]));
|
||||
}
|
||||
}
|
||||
};
|
||||
let object_info_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
let object_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
let object_info = prepared_object_info
|
||||
.unwrap_or_else(|| build_get_object_info(&fi, bucket, object, opts.versioned || opts.version_suspended));
|
||||
let object_class = classify_get_codec_streaming_object_class(&range, &object_info, &fi);
|
||||
let size_bucket = rustfs_io_metrics::get_object_size_bucket(object_info.size);
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_OBJECT_INFO, object_info_stage_start);
|
||||
@@ -410,6 +429,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
// it forward.
|
||||
let mut body_source = GetObjectBodySource::Unprobed;
|
||||
if let Some(plaintext_len) = full_object_plaintext_len(&range, opts, &object_info)
|
||||
&& !get_object_body_cache_hook_suppressed()
|
||||
&& let Some(hook) = get_object_body_cache_hook()
|
||||
{
|
||||
match hook.lookup(bucket, object, &object_info).await {
|
||||
@@ -1222,6 +1242,178 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_uncommitted_transition_upload(tier: &str, object: &str, remote_version: &str) {
|
||||
if let Err(err) = delete_object_from_remote_tier_idempotent(object, remote_version, tier).await {
|
||||
warn!(
|
||||
tier,
|
||||
object,
|
||||
remote_version,
|
||||
error = ?err,
|
||||
"failed to clean uncommitted transition upload"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
struct TransitionUploadCleanup {
|
||||
tier: String,
|
||||
object: String,
|
||||
remote_version: String,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
impl TransitionUploadCleanup {
|
||||
fn new(tier: &str, object: &str, remote_version: &str) -> Self {
|
||||
Self {
|
||||
tier: tier.to_string(),
|
||||
object: object.to_string(),
|
||||
remote_version: remote_version.to_string(),
|
||||
armed: true,
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup(&mut self) {
|
||||
cleanup_uncommitted_transition_upload(&self.tier, &self.object, &self.remote_version).await;
|
||||
self.armed = false;
|
||||
}
|
||||
|
||||
fn disarm(&mut self) {
|
||||
self.armed = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TransitionUploadCleanup {
|
||||
fn drop(&mut self) {
|
||||
if !self.armed {
|
||||
return;
|
||||
}
|
||||
let tier = self.tier.clone();
|
||||
let object = self.object.clone();
|
||||
let remote_version = self.remote_version.clone();
|
||||
if let Ok(handle) = tokio::runtime::Handle::try_current() {
|
||||
handle.spawn(async move {
|
||||
cleanup_uncommitted_transition_upload(&tier, &object, &remote_version).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct TransitionCommitBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct TransitionCommitBarrier {
|
||||
state: Arc<TransitionCommitBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static TRANSITION_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<TransitionCommitBarrierState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl TransitionCommitBarrier {
|
||||
fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(TransitionCommitBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut slot = TRANSITION_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition commit barrier mutex should not poison");
|
||||
assert!(slot.is_none(), "transition commit barrier must be installed by one test at a time");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
drop(slot);
|
||||
Self { state }
|
||||
}
|
||||
|
||||
async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("transition should reach the deterministic commit barrier");
|
||||
}
|
||||
|
||||
fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for TransitionCommitBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
let mut slot = TRANSITION_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition commit barrier mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_transition_before_local_commit(bucket: &str, object: &str) {
|
||||
let barrier = TRANSITION_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition commit barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_transition_version_id(remote_version: &str) -> std::result::Result<Option<Uuid>, uuid::Error> {
|
||||
if remote_version.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
Uuid::parse_str(remote_version).map(|version_id| (!version_id.is_nil()).then_some(version_id))
|
||||
}
|
||||
|
||||
fn transition_cleanup_remote_version(remote_version: &str, version_id: Option<Uuid>) -> &str {
|
||||
version_id.map(|_| remote_version).unwrap_or("")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod transition_version_id_tests {
|
||||
use super::{parse_transition_version_id, transition_cleanup_remote_version};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
fn normalizes_unversioned_remote_ids() {
|
||||
assert_eq!(parse_transition_version_id("").expect("empty remote version should be valid"), None);
|
||||
assert_eq!(
|
||||
parse_transition_version_id(&Uuid::nil().to_string()).expect("nil remote version should be valid"),
|
||||
None
|
||||
);
|
||||
assert_eq!(transition_cleanup_remote_version(&Uuid::nil().to_string(), None), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_valid_remote_id_and_rejects_invalid_text() {
|
||||
let version_id = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
parse_transition_version_id(&version_id.to_string()).expect("UUID remote version should be valid"),
|
||||
Some(version_id)
|
||||
);
|
||||
assert_eq!(
|
||||
transition_cleanup_remote_version(&version_id.to_string(), Some(version_id)),
|
||||
version_id.to_string()
|
||||
);
|
||||
assert!(parse_transition_version_id("not-a-uuid").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
type Error = Error;
|
||||
@@ -1908,7 +2100,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
.await
|
||||
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
|
||||
|
||||
self.get_object_metadata_cache.invalidate_all();
|
||||
self.invalidate_all_get_object_metadata_cache();
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
|
||||
@@ -2236,7 +2428,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
if let Some(mod_time1) = opts.mod_time {
|
||||
if let Some(mod_time2) = fi.mod_time.as_ref() {
|
||||
if mod_time1.unix_timestamp() != mod_time2.unix_timestamp()
|
||||
/*|| transition_etag != stored_etag*/
|
||||
|| (!transition_etag.is_empty() && transition_etag != stored_etag)
|
||||
{
|
||||
return Err(to_object_err(Error::other(DiskError::FileNotFound), vec![bucket, object]));
|
||||
}
|
||||
@@ -2333,25 +2525,88 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
return Err(StorageError::Io(err));
|
||||
}
|
||||
let rv = rv?;
|
||||
fi.transition_status = TRANSITION_COMPLETE.to_string();
|
||||
fi.transitioned_objname = dest_obj;
|
||||
fi.transition_tier = opts.transition.tier.clone();
|
||||
fi.transition_version_id = if rv.is_empty() { None } else { Some(Uuid::parse_str(&rv)?) };
|
||||
drop(tier_config_mgr);
|
||||
|
||||
let transition_version_id = match parse_transition_version_id(&rv) {
|
||||
Ok(version_id) => version_id,
|
||||
Err(err) => {
|
||||
cleanup_uncommitted_transition_upload(&opts.transition.tier, &dest_obj, &rv).await;
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
let cleanup_remote_version = transition_cleanup_remote_version(&rv, transition_version_id);
|
||||
let mut upload_cleanup = TransitionUploadCleanup::new(&opts.transition.tier, &dest_obj, cleanup_remote_version);
|
||||
|
||||
let mut commit_opts = opts.clone();
|
||||
commit_opts.no_lock = true;
|
||||
commit_opts.metadata_cache_safe = false;
|
||||
let transition_lock_guard = if opts.no_lock {
|
||||
None
|
||||
} else {
|
||||
match self.acquire_write_lock_diag("transition_object_commit", bucket, object).await {
|
||||
Ok(guard) => Some(guard),
|
||||
Err(err) => {
|
||||
upload_cleanup.cleanup().await;
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
};
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
let current = self.get_object_fileinfo(bucket, object, &commit_opts, true, false).await;
|
||||
let (mut current_fi, _, _) = match current {
|
||||
Ok(current) => current,
|
||||
Err(err) => {
|
||||
drop(transition_lock_guard);
|
||||
upload_cleanup.cleanup().await;
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let source_matches = current_fi.version_id == fi.version_id
|
||||
&& current_fi.data_dir == fi.data_dir
|
||||
&& current_fi.mod_time == fi.mod_time
|
||||
&& current_fi.size == fi.size
|
||||
&& rustfs_utils::path::trim_etag(&get_raw_etag(¤t_fi.metadata)) == stored_etag;
|
||||
if current_fi.transition_status == TRANSITION_COMPLETE || !source_matches {
|
||||
let already_transitioned = current_fi.transition_status == TRANSITION_COMPLETE;
|
||||
drop(transition_lock_guard);
|
||||
upload_cleanup.cleanup().await;
|
||||
if already_transitioned {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(to_object_err(Error::other(DiskError::FileNotFound), vec![bucket, object]));
|
||||
}
|
||||
|
||||
current_fi.transition_status = TRANSITION_COMPLETE.to_string();
|
||||
current_fi.transitioned_objname = dest_obj;
|
||||
current_fi.transition_tier = opts.transition.tier.clone();
|
||||
current_fi.transition_version_id = transition_version_id;
|
||||
fi = current_fi;
|
||||
let event_name = EventName::LifecycleTransition.as_str();
|
||||
let mut should_notify_transition = true;
|
||||
|
||||
let disks = self.disk_inventory().await;
|
||||
|
||||
if transition_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
drop(transition_lock_guard);
|
||||
upload_cleanup.cleanup().await;
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "transition_object_commit",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
#[cfg(test)]
|
||||
pause_transition_before_local_commit(bucket, object).await;
|
||||
upload_cleanup.disarm();
|
||||
if let Err(err) = self.delete_object_version(bucket, object, &fi, false).await {
|
||||
should_notify_transition = false;
|
||||
warn!(
|
||||
bucket = bucket,
|
||||
object = object,
|
||||
error = ?err,
|
||||
"transition completed on remote tier but source cleanup failed; skipping external lifecycle transition notification"
|
||||
"transition remote upload completed but local commit failed"
|
||||
);
|
||||
} else {
|
||||
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
drop(transition_lock_guard);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
// delete_object_version persisted transition_status=complete and freed the
|
||||
@@ -2361,6 +2616,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
// early-return above and streams the already-deleted local data to the
|
||||
// remote tier again (rustfs/rustfs#4827).
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
drop(transition_lock_guard);
|
||||
let disks = self.disk_inventory().await;
|
||||
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||
|
||||
for disk in disks.iter() {
|
||||
if let Some(disk) = disk {
|
||||
@@ -2372,17 +2630,15 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
break;
|
||||
}
|
||||
|
||||
if should_notify_transition {
|
||||
let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
send_event(EventArgs {
|
||||
event_name: event_name.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: obj_info,
|
||||
user_agent: "Internal: [ILM-Transition]".to_string(),
|
||||
host: runtime_sources::default_local_node_name(),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
send_event(EventArgs {
|
||||
event_name: event_name.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: obj_info,
|
||||
user_agent: "Internal: [ILM-Transition]".to_string(),
|
||||
host: runtime_sources::default_local_node_name(),
|
||||
..Default::default()
|
||||
});
|
||||
//let tags = opts.lifecycle_audit_event.tags();
|
||||
//auditLogLifecycle(ctx, objInfo, ILMTransition, tags, traceFn)
|
||||
Ok(())
|
||||
@@ -2738,6 +2994,127 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
mod transition_commit_failure_tests {
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks;
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions};
|
||||
use crate::disk::DiskAPI as _;
|
||||
use crate::services::tier::test_util::register_mock_tier;
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use http::HeaderMap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn local_commit_failure_returns_error_and_preserves_remote_candidate() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "transition-commit-failure-bucket";
|
||||
let object = "object.bin";
|
||||
let payload = b"transition commit failure must preserve the uploaded candidate".repeat(1024);
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
let original = set_disks
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
let (fi, parts_metadata, online_disks) = set_disks
|
||||
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
|
||||
.await
|
||||
.expect("source metadata should resolve");
|
||||
let generation = set_disks
|
||||
.get_object_metadata_cache_generation(bucket, object)
|
||||
.expect("metadata cache generation should be active");
|
||||
let cache_key = GetObjectMetadataCacheKey::new(bucket, object, generation);
|
||||
set_disks
|
||||
.get_object_metadata_cache
|
||||
.insert(
|
||||
cache_key.clone(),
|
||||
Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi: fi.clone(),
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
read_quorum: 2,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert!(set_disks.get_object_metadata_cache.get(&cache_key).await.is_some());
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name,
|
||||
etag: original.etag.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: original.version_id.map(|version| version.to_string()),
|
||||
mod_time: original.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let barrier = TransitionCommitBarrier::install(bucket, object);
|
||||
let invalidations = MetadataCacheInvalidationProbe::install(bucket, object);
|
||||
let transition_set = Arc::clone(&set_disks);
|
||||
let transition = tokio::spawn(async move { transition_set.transition_object(bucket, object, &opts).await });
|
||||
barrier.wait_until_paused().await;
|
||||
assert_eq!(invalidations.count(), 1, "precommit revalidation must fence the old metadata once");
|
||||
let saved_disks = {
|
||||
let mut disks = set_disks.disks.write().await;
|
||||
let saved = std::mem::take(&mut *disks);
|
||||
*disks = vec![None; saved.len()];
|
||||
saved
|
||||
};
|
||||
barrier.release();
|
||||
let result = transition.await.expect("transition task should not panic");
|
||||
*set_disks.disks.write().await = saved_disks;
|
||||
|
||||
result.expect_err("local write quorum failure must be returned to the transition worker");
|
||||
assert_eq!(
|
||||
invalidations.count(),
|
||||
2,
|
||||
"local commit failure must fence any partially committed metadata again"
|
||||
);
|
||||
assert_eq!(backend.put_count().await, 1);
|
||||
assert_eq!(
|
||||
backend.remove_count().await,
|
||||
0,
|
||||
"ambiguous local commit failure must retain the remote candidate"
|
||||
);
|
||||
assert_eq!(backend.object_count().await, 1);
|
||||
assert!(
|
||||
set_disks.get_object_metadata_cache.get(&cache_key).await.is_none(),
|
||||
"local commit failure must invalidate pre-transition metadata"
|
||||
);
|
||||
|
||||
let mut restored = Vec::new();
|
||||
set_disks
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the local source must remain readable after a fail-before-commit result")
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("the local source body should drain");
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod put_object_tmp_cleanup_tests {
|
||||
//! Regression coverage for backlog#924 (HP-3): the speculative tmp-dir
|
||||
|
||||
@@ -20,21 +20,22 @@ use crate::diagnostics::get::{
|
||||
GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS, GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM,
|
||||
GET_METADATA_CACHE_REASON_META_BUCKET, GET_METADATA_CACHE_REASON_NO_LOCK, GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED,
|
||||
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER,
|
||||
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID,
|
||||
GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED,
|
||||
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
|
||||
GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
|
||||
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
|
||||
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
|
||||
GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND,
|
||||
GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT,
|
||||
GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING,
|
||||
GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE,
|
||||
GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
|
||||
GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
|
||||
GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
|
||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
||||
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
||||
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_STALE_PUBLICATION,
|
||||
GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, GET_METADATA_CACHE_REASON_VERSION_SUSPENDED,
|
||||
GET_METADATA_CACHE_REASON_VERSIONED, GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA,
|
||||
GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR,
|
||||
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND,
|
||||
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
|
||||
GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
|
||||
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
|
||||
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
|
||||
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP,
|
||||
GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, GET_STAGE_READER_SETUP_DROP_PENDING,
|
||||
GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT,
|
||||
GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, GetObjectFailureReason, classify_disk_error,
|
||||
get_stage_timer_if_enabled, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path,
|
||||
record_get_stage_duration_if_enabled,
|
||||
};
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use crate::io_support::bitrot::{
|
||||
@@ -60,6 +61,7 @@ use super::core::io_primitives::*;
|
||||
|
||||
impl SetDisks {
|
||||
async fn get_object_metadata_cache_bypass_reason(
|
||||
&self,
|
||||
bucket: &str,
|
||||
opts: &ObjectOptions,
|
||||
read_data: bool,
|
||||
@@ -67,7 +69,8 @@ impl SetDisks {
|
||||
if let Some(reason) = get_object_metadata_cache_request_bypass_reason(bucket, opts, read_data) {
|
||||
return Some(reason);
|
||||
}
|
||||
runtime_sources::setup_is_dist_erasure()
|
||||
self.ctx
|
||||
.is_dist_erasure()
|
||||
.await
|
||||
.then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE)
|
||||
}
|
||||
@@ -80,11 +83,28 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
async fn lookup_cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> MetadataCacheLookup {
|
||||
let key = GetObjectMetadataCacheKey::new(bucket, object);
|
||||
self.lookup_cached_get_object_fileinfo_after_get(bucket, object, || {}).await
|
||||
}
|
||||
|
||||
async fn lookup_cached_get_object_fileinfo_after_get(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
after_get: impl FnOnce(),
|
||||
) -> MetadataCacheLookup {
|
||||
let Some(generation) = self.get_object_metadata_cache_generation(bucket, object) else {
|
||||
return MetadataCacheLookup::Miss;
|
||||
};
|
||||
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
|
||||
// moka handles TTL expiry automatically; no is_fresh() check needed
|
||||
let Some(entry) = self.get_object_metadata_cache.get(&key).await else {
|
||||
return MetadataCacheLookup::Miss;
|
||||
};
|
||||
after_get();
|
||||
if !self.is_get_object_metadata_cache_generation_current(generation) {
|
||||
self.get_object_metadata_cache.invalidate(&key).await;
|
||||
return MetadataCacheLookup::Miss;
|
||||
}
|
||||
if entry.online_disks.iter().filter(|disk| disk.is_some()).count() >= entry.read_quorum {
|
||||
MetadataCacheLookup::Hit(entry)
|
||||
} else {
|
||||
@@ -94,8 +114,8 @@ impl SetDisks {
|
||||
|
||||
async fn cache_get_object_fileinfo(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
identity: (&str, &str),
|
||||
generation: Option<GetObjectMetadataCacheGeneration>,
|
||||
fi: &FileInfo,
|
||||
parts_metadata: &[FileInfo],
|
||||
online_disks: &[Option<DiskStore>],
|
||||
@@ -104,23 +124,51 @@ impl SetDisks {
|
||||
if fi.deleted || !fi.is_valid() {
|
||||
return;
|
||||
}
|
||||
let (bucket, object) = identity;
|
||||
|
||||
let key = GetObjectMetadataCacheKey::new(bucket, object);
|
||||
// moka handles capacity eviction (LRU) automatically
|
||||
self.get_object_metadata_cache
|
||||
.insert(
|
||||
key,
|
||||
Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi: fi.clone(),
|
||||
parts_metadata: parts_metadata.to_vec(),
|
||||
online_disks: online_disks.to_vec(),
|
||||
read_quorum,
|
||||
}),
|
||||
)
|
||||
let Some(generation) = generation.filter(|generation| self.is_get_object_metadata_cache_generation_current(*generation))
|
||||
else {
|
||||
rustfs_io_metrics::record_get_object_metadata_cache_decision(
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
GET_METADATA_CACHE_DECISION_REJECT,
|
||||
GET_METADATA_CACHE_REASON_STALE_PUBLICATION,
|
||||
);
|
||||
return;
|
||||
};
|
||||
|
||||
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
|
||||
let entry = Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi: fi.clone(),
|
||||
parts_metadata: parts_metadata.to_vec(),
|
||||
online_disks: online_disks.to_vec(),
|
||||
read_quorum,
|
||||
});
|
||||
self.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {})
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn insert_get_object_metadata_cache_entry_after_insert(
|
||||
&self,
|
||||
key: GetObjectMetadataCacheKey,
|
||||
generation: GetObjectMetadataCacheGeneration,
|
||||
entry: Arc<GetObjectMetadataCacheEntry>,
|
||||
after_insert: impl FnOnce(),
|
||||
) {
|
||||
self.get_object_metadata_cache.insert(key.clone(), entry).await;
|
||||
after_insert();
|
||||
if self.is_get_object_metadata_cache_generation_current(generation) {
|
||||
return;
|
||||
}
|
||||
|
||||
self.get_object_metadata_cache.invalidate(&key).await;
|
||||
rustfs_io_metrics::record_get_object_metadata_cache_decision(
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
GET_METADATA_CACHE_DECISION_REJECT,
|
||||
GET_METADATA_CACHE_REASON_STALE_PUBLICATION,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg_attr(feature = "hotpath", hotpath::measure)]
|
||||
pub async fn read_version_optimized(
|
||||
&self,
|
||||
@@ -193,7 +241,7 @@ impl SetDisks {
|
||||
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
|
||||
let metadata_cache_lookup_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
let cache_bypass_reason = Self::get_object_metadata_cache_bypass_reason(bucket, opts, read_data).await;
|
||||
let cache_bypass_reason = self.get_object_metadata_cache_bypass_reason(bucket, opts, read_data).await;
|
||||
let use_metadata_cache = cache_bypass_reason.is_none();
|
||||
if let Some(reason) = cache_bypass_reason {
|
||||
rustfs_io_metrics::record_get_object_metadata_cache_decision(
|
||||
@@ -238,6 +286,10 @@ impl SetDisks {
|
||||
metadata_cache_lookup_start,
|
||||
);
|
||||
|
||||
let metadata_cache_generation = use_metadata_cache
|
||||
.then(|| self.get_object_metadata_cache_generation(bucket, object))
|
||||
.flatten();
|
||||
|
||||
let disks = self.disks.read().await;
|
||||
|
||||
let disks = disks.clone();
|
||||
@@ -311,8 +363,17 @@ impl SetDisks {
|
||||
)
|
||||
.await;
|
||||
} else if use_metadata_cache && metadata_fanout_complete {
|
||||
self.cache_get_object_fileinfo(bucket, object, &fi, &parts_metadata, &op_online_disks, read_quorum)
|
||||
.await;
|
||||
#[cfg(test)]
|
||||
metadata_cache_tests::wait_before_metadata_cache_publish(bucket, object).await;
|
||||
self.cache_get_object_fileinfo(
|
||||
(bucket, object),
|
||||
metadata_cache_generation,
|
||||
&fi,
|
||||
&parts_metadata,
|
||||
&op_online_disks,
|
||||
read_quorum,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_METADATA_RESOLVE, metadata_resolve_stage_start);
|
||||
// debug!("get_object_fileinfo pick fi {:?}", &fi);
|
||||
@@ -1738,13 +1799,81 @@ mod metadata_cache_tests {
|
||||
use super::*;
|
||||
use rustfs_common::heal_channel::HealAdmissionDropReason;
|
||||
use serial_test::serial;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
static SLOW_READ_REPAIR_SUBMITTER_CALLS: AtomicUsize = AtomicUsize::new(0);
|
||||
static DROPPED_READ_REPAIR_SUBMITTER_CALLS: AtomicUsize = AtomicUsize::new(0);
|
||||
static CAPTURED_READ_REPAIR_PRIORITY: Mutex<Option<HealChannelPriority>> = Mutex::new(None);
|
||||
static CAPTURED_READ_REPAIR_CALLS: AtomicUsize = AtomicUsize::new(0);
|
||||
static METADATA_CACHE_PUBLISH_BARRIER: OnceLock<Mutex<Option<Arc<MetadataCachePublishBarrierState>>>> = OnceLock::new();
|
||||
|
||||
struct MetadataCachePublishBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
arrived: Notify,
|
||||
release: Notify,
|
||||
}
|
||||
|
||||
struct MetadataCachePublishBarrier {
|
||||
state: Arc<MetadataCachePublishBarrierState>,
|
||||
}
|
||||
|
||||
impl MetadataCachePublishBarrier {
|
||||
fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(MetadataCachePublishBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
arrived: Notify::new(),
|
||||
release: Notify::new(),
|
||||
});
|
||||
let mut slot = METADATA_CACHE_PUBLISH_BARRIER
|
||||
.get_or_init(|| Mutex::new(None))
|
||||
.lock()
|
||||
.expect("metadata publish barrier mutex should not poison");
|
||||
assert!(slot.is_none(), "metadata publish barrier must be installed by one test at a time");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
Self { state }
|
||||
}
|
||||
|
||||
async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("metadata publication should reach the deterministic barrier");
|
||||
}
|
||||
|
||||
fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MetadataCachePublishBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
let mut slot = METADATA_CACHE_PUBLISH_BARRIER
|
||||
.get_or_init(|| Mutex::new(None))
|
||||
.lock()
|
||||
.expect("metadata publish barrier mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn wait_before_metadata_cache_publish(bucket: &str, object: &str) {
|
||||
let barrier = METADATA_CACHE_PUBLISH_BARRIER
|
||||
.get_or_init(|| Mutex::new(None))
|
||||
.lock()
|
||||
.expect("metadata publish barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|state| state.bucket == bucket && state.object == object)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
fn slow_read_repair_submitter(_request: rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
|
||||
SLOW_READ_REPAIR_SUBMITTER_CALLS.fetch_add(1, Ordering::Relaxed);
|
||||
@@ -1771,7 +1900,11 @@ mod metadata_cache_tests {
|
||||
}
|
||||
|
||||
async fn new_metadata_cache_test_set() -> Arc<SetDisks> {
|
||||
SetDisks::new(
|
||||
new_metadata_cache_test_set_with_ctx(Arc::new(crate::runtime::instance::InstanceContext::new())).await
|
||||
}
|
||||
|
||||
async fn new_metadata_cache_test_set_with_ctx(ctx: Arc<crate::runtime::instance::InstanceContext>) -> Arc<SetDisks> {
|
||||
SetDisks::new_with_instance_ctx(
|
||||
"metadata-cache-test".to_string(),
|
||||
Arc::new(RwLock::new(Vec::new())),
|
||||
4,
|
||||
@@ -1781,6 +1914,7 @@ mod metadata_cache_tests {
|
||||
Vec::new(),
|
||||
FormatV3::new(1, 4),
|
||||
Vec::new(),
|
||||
ctx,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2334,7 +2468,8 @@ mod metadata_cache_tests {
|
||||
let parts_metadata = vec![fi.clone()];
|
||||
let online_disks = Vec::new();
|
||||
|
||||
set.cache_get_object_fileinfo("bucket", "object", &fi, &parts_metadata, &online_disks, 0)
|
||||
let generation = set.get_object_metadata_cache_generation("bucket", "object");
|
||||
set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, &parts_metadata, &online_disks, 0)
|
||||
.await;
|
||||
|
||||
let cached = set
|
||||
@@ -2353,7 +2488,8 @@ mod metadata_cache_tests {
|
||||
|
||||
let mut deleted = valid_test_fileinfo("deleted-object");
|
||||
deleted.deleted = true;
|
||||
set.cache_get_object_fileinfo("bucket", "deleted-object", &deleted, &[deleted.clone()], &[], 0)
|
||||
let generation = set.get_object_metadata_cache_generation("bucket", "deleted-object");
|
||||
set.cache_get_object_fileinfo(("bucket", "deleted-object"), generation, &deleted, &[deleted.clone()], &[], 0)
|
||||
.await;
|
||||
assert!(
|
||||
set.cached_get_object_fileinfo("bucket", "deleted-object").await.is_none(),
|
||||
@@ -2361,7 +2497,8 @@ mod metadata_cache_tests {
|
||||
);
|
||||
|
||||
let invalid = FileInfo::default();
|
||||
set.cache_get_object_fileinfo("bucket", "invalid-object", &invalid, std::slice::from_ref(&invalid), &[], 0)
|
||||
let generation = set.get_object_metadata_cache_generation("bucket", "invalid-object");
|
||||
set.cache_get_object_fileinfo(("bucket", "invalid-object"), generation, &invalid, std::slice::from_ref(&invalid), &[], 0)
|
||||
.await;
|
||||
assert!(
|
||||
set.cached_get_object_fileinfo("bucket", "invalid-object").await.is_none(),
|
||||
@@ -2376,7 +2513,12 @@ mod metadata_cache_tests {
|
||||
|
||||
set.get_object_metadata_cache
|
||||
.insert(
|
||||
GetObjectMetadataCacheKey::new("bucket", "object"),
|
||||
GetObjectMetadataCacheKey::new(
|
||||
"bucket",
|
||||
"object",
|
||||
set.get_object_metadata_cache_generation("bucket", "object")
|
||||
.expect("metadata cache generation should be active"),
|
||||
),
|
||||
Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi: fi.clone(),
|
||||
@@ -2401,7 +2543,8 @@ mod metadata_cache_tests {
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
let fi = valid_test_fileinfo("object");
|
||||
|
||||
set.cache_get_object_fileinfo("bucket", "object", &fi, std::slice::from_ref(&fi), &[], 0)
|
||||
let generation = set.get_object_metadata_cache_generation("bucket", "object");
|
||||
set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, std::slice::from_ref(&fi), &[], 0)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
@@ -2415,7 +2558,8 @@ mod metadata_cache_tests {
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
let fi = valid_test_fileinfo("object");
|
||||
|
||||
set.cache_get_object_fileinfo("bucket", "object", &fi, std::slice::from_ref(&fi), &[], 0)
|
||||
let generation = set.get_object_metadata_cache_generation("bucket", "object");
|
||||
set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, std::slice::from_ref(&fi), &[], 0)
|
||||
.await;
|
||||
assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_some());
|
||||
|
||||
@@ -2426,6 +2570,193 @@ mod metadata_cache_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(metadata_cache_publish_barrier)]
|
||||
async fn metadata_cache_production_fanout_cannot_publish_after_invalidation() {
|
||||
let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks(4, 2).await;
|
||||
let bucket = "metadata-cache-production-fence";
|
||||
let object = "object";
|
||||
let disks = set.disks.read().await.clone();
|
||||
for disk in disks.iter().flatten() {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
let mut reader = PutObjReader::from_vec(vec![7u8; 1024]);
|
||||
set.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("test object should be written");
|
||||
|
||||
let barrier = MetadataCachePublishBarrier::install(bucket, object);
|
||||
let stale_generation = set
|
||||
.get_object_metadata_cache_generation(bucket, object)
|
||||
.expect("metadata cache generation should be active");
|
||||
let reader_set = Arc::clone(&set);
|
||||
let read = tokio::spawn(async move {
|
||||
reader_set
|
||||
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
set.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
barrier.release();
|
||||
read.await
|
||||
.expect("metadata read task should not panic")
|
||||
.expect("metadata fanout should still return its selected FileInfo");
|
||||
|
||||
assert!(
|
||||
set.get_object_metadata_cache
|
||||
.get(&GetObjectMetadataCacheKey::new(bucket, object, stale_generation))
|
||||
.await
|
||||
.is_none(),
|
||||
"production stale publication must not remain under its retired generation key"
|
||||
);
|
||||
assert!(
|
||||
set.cached_get_object_fileinfo(bucket, object).await.is_none(),
|
||||
"the production fanout token captured before invalidation must not publish afterward"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_cache_lookup_rechecks_generation_after_get() {
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
let fi = valid_test_fileinfo("object");
|
||||
let generation = set.get_object_metadata_cache_generation("bucket", "object");
|
||||
let generation_index = generation.expect("metadata cache generation should be active").index;
|
||||
set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, std::slice::from_ref(&fi), &[], 0)
|
||||
.await;
|
||||
|
||||
let lookup = set
|
||||
.lookup_cached_get_object_fileinfo_after_get("bucket", "object", || {
|
||||
set.get_object_metadata_cache_generations[generation_index].fetch_add(1, Ordering::AcqRel);
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(lookup, MetadataCacheLookup::Miss),
|
||||
"a hit overlapping invalidation must be rejected after the second generation read"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_cache_publication_rechecks_generation_after_insert() {
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
let fi = valid_test_fileinfo("object");
|
||||
let generation = set
|
||||
.get_object_metadata_cache_generation("bucket", "object")
|
||||
.expect("metadata cache generation should be active");
|
||||
let generation_index = generation.index;
|
||||
let key = GetObjectMetadataCacheKey::new("bucket", "object", generation);
|
||||
let entry = Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi: fi.clone(),
|
||||
parts_metadata: vec![fi],
|
||||
online_disks: Vec::new(),
|
||||
read_quorum: 0,
|
||||
});
|
||||
|
||||
set.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {
|
||||
set.get_object_metadata_cache_generations[generation_index].fetch_add(1, Ordering::AcqRel);
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
set.get_object_metadata_cache
|
||||
.get(&GetObjectMetadataCacheKey::new("bucket", "object", generation))
|
||||
.await
|
||||
.is_none(),
|
||||
"post-insert generation change must withdraw the retired entry"
|
||||
);
|
||||
assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_cache_generation_advances_monotonically() {
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
let fi = valid_test_fileinfo("object");
|
||||
let initial = set
|
||||
.get_object_metadata_cache_generation("bucket", "object")
|
||||
.expect("initial generation should be active");
|
||||
|
||||
set.invalidate_get_object_metadata_cache("bucket", "object").await;
|
||||
let first = set
|
||||
.get_object_metadata_cache_generation("bucket", "object")
|
||||
.expect("first generation should be active");
|
||||
set.invalidate_get_object_metadata_cache("bucket", "object").await;
|
||||
let second = set
|
||||
.get_object_metadata_cache_generation("bucket", "object")
|
||||
.expect("second generation should be active");
|
||||
|
||||
assert_eq!(first.value, initial.value + 1);
|
||||
assert_eq!(second.value, first.value + 1);
|
||||
set.cache_get_object_fileinfo(("bucket", "object"), Some(first), &fi, std::slice::from_ref(&fi), &[], 0)
|
||||
.await;
|
||||
assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_cache_distribution_bypass_uses_set_instance_context() {
|
||||
let distributed_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
distributed_ctx
|
||||
.update_erasure_type(crate::layout::endpoints::SetupType::DistErasure)
|
||||
.await;
|
||||
let standalone_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
standalone_ctx
|
||||
.update_erasure_type(crate::layout::endpoints::SetupType::Erasure)
|
||||
.await;
|
||||
let distributed = new_metadata_cache_test_set_with_ctx(distributed_ctx).await;
|
||||
let standalone = new_metadata_cache_test_set_with_ctx(standalone_ctx).await;
|
||||
|
||||
assert_eq!(
|
||||
distributed
|
||||
.get_object_metadata_cache_bypass_reason("bucket", &ObjectOptions::default(), true)
|
||||
.await,
|
||||
Some(GET_METADATA_CACHE_REASON_DIST_ERASURE)
|
||||
);
|
||||
assert_eq!(
|
||||
standalone
|
||||
.get_object_metadata_cache_bypass_reason("bucket", &ObjectOptions::default(), true)
|
||||
.await,
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_cache_generation_overflow_fails_closed() {
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
let fi = valid_test_fileinfo("object");
|
||||
let generation_index = set
|
||||
.get_object_metadata_cache_generation("bucket", "object")
|
||||
.expect("metadata cache generation should be active")
|
||||
.index;
|
||||
set.get_object_metadata_cache_generations[generation_index].store(u64::MAX - 1, Ordering::Release);
|
||||
let last_generation = set.get_object_metadata_cache_generation("bucket", "object");
|
||||
assert_eq!(last_generation.map(|generation| generation.value), Some(u64::MAX - 1));
|
||||
|
||||
set.invalidate_get_object_metadata_cache("bucket", "object").await;
|
||||
|
||||
assert_eq!(set.get_object_metadata_cache_generation("bucket", "object"), None);
|
||||
set.cache_get_object_fileinfo(("bucket", "object"), last_generation, &fi, std::slice::from_ref(&fi), &[], 0)
|
||||
.await;
|
||||
assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none());
|
||||
set.invalidate_get_object_metadata_cache("bucket", "object").await;
|
||||
assert_eq!(
|
||||
set.get_object_metadata_cache_generations[generation_index].load(Ordering::Acquire),
|
||||
u64::MAX,
|
||||
"overflow must permanently disable publication instead of wrapping"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_cache_invalidate_all_fences_late_insert() {
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
let fi = valid_test_fileinfo("object");
|
||||
let stale_generation = set.get_object_metadata_cache_generation("bucket", "object");
|
||||
|
||||
set.invalidate_all_get_object_metadata_cache();
|
||||
|
||||
set.cache_get_object_fileinfo(("bucket", "object"), stale_generation, &fi, std::slice::from_ref(&fi), &[], 0)
|
||||
.await;
|
||||
assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_metadata_cache_prunes_when_capacity_is_reached() {
|
||||
// moka handles capacity eviction automatically via the configured max_capacity.
|
||||
@@ -2433,7 +2764,8 @@ mod metadata_cache_tests {
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
let fresh_fi = valid_test_fileinfo("fresh-object");
|
||||
|
||||
set.cache_get_object_fileinfo("bucket", "fresh-object", &fresh_fi, std::slice::from_ref(&fresh_fi), &[], 0)
|
||||
let generation = set.get_object_metadata_cache_generation("bucket", "fresh-object");
|
||||
set.cache_get_object_fileinfo(("bucket", "fresh-object"), generation, &fresh_fi, std::slice::from_ref(&fresh_fi), &[], 0)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
|
||||
@@ -158,6 +158,7 @@ mod list;
|
||||
pub(crate) mod list_objects;
|
||||
mod multipart;
|
||||
mod object;
|
||||
pub use object::PreparedGetObjectReader;
|
||||
mod peer;
|
||||
mod rebalance;
|
||||
pub(crate) mod utils;
|
||||
|
||||
@@ -31,6 +31,61 @@ use std::{
|
||||
};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
|
||||
/// A GET whose object identity has been resolved while its namespace read lock
|
||||
/// remains held, but whose body reader has not been constructed yet.
|
||||
///
|
||||
/// The application can evaluate request preconditions and cache coordination
|
||||
/// against [`Self::object_info`] before consuming this value. Dropping it
|
||||
/// releases the read lock without constructing a body reader.
|
||||
pub struct PreparedGetObjectReader {
|
||||
pool: Arc<Sets>,
|
||||
bucket: String,
|
||||
object: String,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
headers: HeaderMap,
|
||||
opts: ObjectOptions,
|
||||
metadata: crate::set_disk::PreparedGetObjectMetadata,
|
||||
read_lock_guard: Option<ObjectLockDiagGuard>,
|
||||
}
|
||||
|
||||
impl PreparedGetObjectReader {
|
||||
/// Returns the fresh metadata snapshot protected by this prepared read.
|
||||
pub fn object_info(&self) -> &ObjectInfo {
|
||||
self.metadata.object_info()
|
||||
}
|
||||
|
||||
/// Finishes a metadata-only decision without constructing a body reader.
|
||||
pub fn into_object_info(mut self) -> ObjectInfo {
|
||||
self.metadata.take_object_info()
|
||||
}
|
||||
|
||||
/// Replaces the headers used when this prepared value constructs its body reader.
|
||||
#[must_use]
|
||||
pub fn with_headers(mut self, headers: HeaderMap) -> Self {
|
||||
self.headers = headers;
|
||||
self
|
||||
}
|
||||
|
||||
/// Constructs the body reader while retaining the metadata snapshot's read
|
||||
/// lock, then transfers that lock to the returned stream as usual. The
|
||||
/// staged caller already performed the authoritative app-layer cache probe,
|
||||
/// so the nested set-disk reader must not probe the same hook again.
|
||||
pub async fn into_reader(self) -> Result<GetObjectReader> {
|
||||
let mut reader =
|
||||
crate::object_api::without_get_object_body_cache_hook(self.pool.get_object_reader_with_prepared_metadata(
|
||||
&self.bucket,
|
||||
&self.object,
|
||||
self.range,
|
||||
self.headers,
|
||||
&self.opts,
|
||||
self.metadata,
|
||||
))
|
||||
.await?;
|
||||
reader.body_source = crate::object_api::GetObjectBodySource::HookMissed;
|
||||
Ok(ECStore::attach_read_lock_guard(reader, self.read_lock_guard))
|
||||
}
|
||||
}
|
||||
|
||||
struct LockGuardedReader {
|
||||
inner: Box<dyn AsyncRead + Unpin + Send + Sync>,
|
||||
guard: Option<ObjectLockDiagGuard>,
|
||||
@@ -394,6 +449,56 @@ fn sorted_unique_delete_object_names(objects: &[ObjectToDelete]) -> Vec<&str> {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Resolves a GET's object identity without constructing its body reader.
|
||||
///
|
||||
/// This is an additive two-stage counterpart to `get_object_reader`. The
|
||||
/// existing method remains the compatibility path for callers that do not
|
||||
/// need a pre-reader decision point.
|
||||
pub async fn prepare_get_object_reader(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
headers: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PreparedGetObjectReader> {
|
||||
check_get_obj_args(bucket, object)?;
|
||||
|
||||
let object = encode_dir_object(object);
|
||||
let mut opts = opts.clone();
|
||||
let read_lock_guard = self
|
||||
.acquire_object_read_lock_if_needed("prepare_get_object", bucket, &object, &mut opts)
|
||||
.await?;
|
||||
|
||||
let (metadata, pool) = if self.single_pool() {
|
||||
let pool = Arc::clone(&self.pools[0]);
|
||||
let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?;
|
||||
(metadata, pool)
|
||||
} else {
|
||||
let (_, pool_idx) = self
|
||||
.get_latest_accessible_object_info_with_idx(bucket, &object, &opts)
|
||||
.await?;
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_idx)
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::other(format!("resolved GET pool index {pool_idx} is out of bounds")))?;
|
||||
let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?;
|
||||
(metadata, pool)
|
||||
};
|
||||
|
||||
Ok(PreparedGetObjectReader {
|
||||
pool,
|
||||
bucket: bucket.to_owned(),
|
||||
object,
|
||||
range,
|
||||
headers,
|
||||
opts,
|
||||
metadata,
|
||||
read_lock_guard,
|
||||
})
|
||||
}
|
||||
|
||||
fn map_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err: rustfs_lock::LockError) -> StorageError {
|
||||
match err {
|
||||
rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable {
|
||||
@@ -1484,15 +1589,43 @@ mod tests {
|
||||
ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_state_to_filemeta, replication_statuses_map,
|
||||
version_purge_statuses_map,
|
||||
};
|
||||
use crate::ecstore_validation_blackbox::make_local_set_disks;
|
||||
use crate::layout::{
|
||||
endpoints::{Endpoints, PoolEndpoints},
|
||||
format::FormatV3,
|
||||
};
|
||||
use crate::object_api::{
|
||||
GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource, clear_get_object_body_cache_hook,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
use bytes::Bytes;
|
||||
use std::io::Cursor;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
struct CountingMissHook {
|
||||
calls: AtomicUsize,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl GetObjectBodyCacheHook for CountingMissHook {
|
||||
async fn lookup(&self, _bucket: &str, _object: &str, _info: &ObjectInfo) -> Option<Bytes> {
|
||||
self.calls.fetch_add(1, Ordering::Relaxed);
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
struct BodyCacheHookGuard;
|
||||
|
||||
impl Drop for BodyCacheHookGuard {
|
||||
fn drop(&mut self) {
|
||||
clear_get_object_body_cache_hook();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_marker_data_movement_falls_back_when_only_source_pool_has_object() {
|
||||
let target = select_data_movement_target_pool(Ok(1), 1, true).unwrap();
|
||||
@@ -2095,6 +2228,244 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn new_prepared_reader_test_store(set_disks: &[Arc<SetDisks>]) -> ECStore {
|
||||
let mut pool_configs = Vec::with_capacity(set_disks.len());
|
||||
let mut pools = Vec::with_capacity(set_disks.len());
|
||||
|
||||
for (pool_idx, set_disks) in set_disks.iter().enumerate() {
|
||||
let mut endpoints = Endpoints::from(set_disks.set_endpoints.clone());
|
||||
for endpoint in endpoints.as_mut() {
|
||||
endpoint.set_pool_index(pool_idx);
|
||||
}
|
||||
let pool_config = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: set_disks.set_drive_count,
|
||||
endpoints,
|
||||
cmd_line: format!("prepared-reader-test-pool-{pool_idx}"),
|
||||
platform: "test".to_string(),
|
||||
};
|
||||
let disks = set_disks.disks.read().await.clone();
|
||||
let pool = Sets::new(disks, &pool_config, &set_disks.format, pool_idx, set_disks.default_parity_count)
|
||||
.await
|
||||
.expect("prepared-reader test pool should be created from local disks");
|
||||
pool_configs.push(pool_config);
|
||||
pools.push(pool);
|
||||
}
|
||||
|
||||
let endpoint_pools = EndpointServerPools::from(pool_configs);
|
||||
ECStore {
|
||||
id: Uuid::new_v4(),
|
||||
disk_map: HashMap::new(),
|
||||
pools,
|
||||
peer_sys: S3PeerSys::new(&endpoint_pools),
|
||||
pool_meta: RwLock::new(PoolMeta::default()),
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::new(()),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_prepared_reader_blocks_writer(store: &ECStore, bucket: &str, object: &str) {
|
||||
let manager = Arc::clone(store.pools[0].disk_set[0].local_lock_manager_for_test());
|
||||
let lock = rustfs_lock::NamespaceLock::with_local_manager("prepared-reader-writer".to_string(), manager);
|
||||
let err = lock
|
||||
.get_write_lock(rustfs_lock::ObjectKey::new(bucket, object), "competing-writer", Duration::from_millis(50))
|
||||
.await
|
||||
.expect_err("prepared read lock should block the writer");
|
||||
assert!(matches!(err, rustfs_lock::LockError::Timeout { .. }));
|
||||
}
|
||||
|
||||
async fn acquire_prepared_reader_writer(store: &ECStore, bucket: &str, object: &str) -> rustfs_lock::NamespaceLockGuard {
|
||||
let manager = Arc::clone(store.pools[0].disk_set[0].local_lock_manager_for_test());
|
||||
let lock = rustfs_lock::NamespaceLock::with_local_manager("prepared-reader-writer".to_string(), manager);
|
||||
lock.get_write_lock(rustfs_lock::ObjectKey::new(bucket, object), "competing-writer", Duration::from_secs(1))
|
||||
.await
|
||||
.expect("prepared read lock should have been released")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn prepared_reader_uses_authoritative_hook_miss_once_and_streams_full_body() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let store = new_prepared_reader_test_store(&[set_disks]).await;
|
||||
let bucket = "prepared-reader-hook-miss";
|
||||
let object = "object.bin";
|
||||
let payload = b"prepared-reader-hook-miss-payload-".repeat(40_000);
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
store.pools[0]
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
store.pools[0]
|
||||
.put_object(bucket, object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
clear_get_object_body_cache_hook();
|
||||
let hook = Arc::new(CountingMissHook {
|
||||
calls: AtomicUsize::new(0),
|
||||
});
|
||||
register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
|
||||
let _hook_guard = BodyCacheHookGuard;
|
||||
|
||||
let prepared = store
|
||||
.prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("prepared reader metadata should resolve");
|
||||
assert!(matches!(
|
||||
lookup_get_object_body_cache_hook(bucket, object, &None, &opts, prepared.object_info()).await,
|
||||
GetObjectBodyCacheHookLookup::Miss
|
||||
));
|
||||
assert_eq!(hook.calls.load(Ordering::Relaxed), 1, "the authoritative probe should call the hook once");
|
||||
|
||||
let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
|
||||
assert_eq!(reader.body_source, GetObjectBodySource::HookMissed);
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("prepared body should stream");
|
||||
|
||||
assert_eq!(restored, payload);
|
||||
assert_eq!(hook.calls.load(Ordering::Relaxed), 1, "reader construction must not probe the hook again");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn prepared_reader_holds_namespace_lock_until_eof_or_drop() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let store = new_prepared_reader_test_store(&[set_disks]).await;
|
||||
let bucket = "prepared-reader-lock-lifetime";
|
||||
let object = "object.bin";
|
||||
let payload = b"prepared-reader-lock-lifetime-payload-".repeat(40_000);
|
||||
let put_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
store.pools[0]
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
store.pools[0]
|
||||
.put_object(bucket, object, &mut put_reader, &put_opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
let prepared = store
|
||||
.prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("prepared reader metadata should resolve");
|
||||
assert!(prepared.read_lock_guard.is_some());
|
||||
assert_prepared_reader_blocks_writer(&store, bucket, object).await;
|
||||
|
||||
let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
|
||||
assert_prepared_reader_blocks_writer(&store, bucket, object).await;
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("prepared body should stream");
|
||||
assert_eq!(restored, payload);
|
||||
drop(acquire_prepared_reader_writer(&store, bucket, object).await);
|
||||
|
||||
let prepared = store
|
||||
.prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("second prepared reader metadata should resolve");
|
||||
let reader = prepared.into_reader().await.expect("second prepared body reader should open");
|
||||
assert_prepared_reader_blocks_writer(&store, bucket, object).await;
|
||||
drop(reader);
|
||||
drop(acquire_prepared_reader_writer(&store, bucket, object).await);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn prepared_object_info_releases_namespace_lock_immediately() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let store = new_prepared_reader_test_store(&[set_disks]).await;
|
||||
let bucket = "prepared-object-info-lock-release";
|
||||
let object = "object.bin";
|
||||
let payload = b"prepared-object-info-lock-release".to_vec();
|
||||
let put_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
store.pools[0]
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
store.pools[0]
|
||||
.put_object(bucket, object, &mut put_reader, &put_opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
let prepared = store
|
||||
.prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("prepared reader metadata should resolve");
|
||||
assert_prepared_reader_blocks_writer(&store, bucket, object).await;
|
||||
assert_eq!(prepared.into_object_info().size, payload.len() as i64);
|
||||
|
||||
drop(acquire_prepared_reader_writer(&store, bucket, object).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn prepared_reader_resolves_object_from_second_pool() {
|
||||
let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
|
||||
let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
|
||||
let store = new_prepared_reader_test_store(&[first_set, second_set]).await;
|
||||
let bucket = "prepared-reader-second-pool";
|
||||
let object = "object.bin";
|
||||
let payload = b"prepared-reader-second-pool-payload-".repeat(40_000);
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for pool in &store.pools {
|
||||
pool.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created in each pool");
|
||||
}
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
store.pools[1]
|
||||
.put_object(bucket, object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("object should be written only to the second pool");
|
||||
|
||||
let prepared = store
|
||||
.prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("prepared reader should resolve the second-pool object");
|
||||
assert_eq!(prepared.object_info().size, payload.len() as i64);
|
||||
let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("prepared body should stream");
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
|
||||
// Phase 5 Slice 2 (backlog#939): the instance context flows down the whole
|
||||
// object graph — ECStore, its Sets, and their SetDisks must all carry the
|
||||
// same `Arc<InstanceContext>` in a single-instance deployment.
|
||||
|
||||
@@ -14,8 +14,10 @@
|
||||
|
||||
use crate::backend::ObjectDataCacheBackendKind;
|
||||
use crate::config::ObjectDataCacheConfig;
|
||||
use crate::entry::projected_weight;
|
||||
use crate::error::ObjectDataCacheConfigError;
|
||||
use crate::key::{ObjectDataCacheBodyVariant, ObjectDataCacheIdentity, ObjectDataCacheKey};
|
||||
use crate::memory::ObjectDataCacheMemoryReservation;
|
||||
use crate::metrics::{
|
||||
describe_metrics_once, publish_cache_state, record_fill_result, record_hit_bytes, record_invalidation, record_lookup_result,
|
||||
record_plan_decision,
|
||||
@@ -27,6 +29,52 @@ use bytes::Bytes;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
use tokio::sync::OwnedSemaphorePermit;
|
||||
|
||||
/// Admission token for one body allocation performed before a cold cache fill.
|
||||
#[derive(Debug)]
|
||||
pub struct ObjectDataCacheBodyReservation {
|
||||
pub(crate) memory: ObjectDataCacheMemoryReservation,
|
||||
pub(crate) permit: OwnedSemaphorePermit,
|
||||
pub(crate) fill_generation: crate::moka_backend::FillGenerationGuard,
|
||||
pub(crate) key: ObjectDataCacheKey,
|
||||
pub(crate) expected_size: u64,
|
||||
}
|
||||
|
||||
/// A materialized body whose allocation owns its memory claim until the last
|
||||
/// `Bytes` clone is dropped.
|
||||
#[derive(Debug)]
|
||||
pub struct ObjectDataCacheReservedBody {
|
||||
pub(crate) bytes: Bytes,
|
||||
pub(crate) permit: OwnedSemaphorePermit,
|
||||
pub(crate) fill_generation: crate::moka_backend::FillGenerationGuard,
|
||||
pub(crate) key: ObjectDataCacheKey,
|
||||
pub(crate) expected_size: u64,
|
||||
}
|
||||
|
||||
impl ObjectDataCacheBodyReservation {
|
||||
/// Attaches this reservation to a newly materialized body.
|
||||
pub fn wrap_bytes(self, bytes: Bytes) -> ObjectDataCacheReservedBody {
|
||||
ObjectDataCacheReservedBody {
|
||||
bytes: self.memory.wrap_bytes(bytes),
|
||||
permit: self.permit,
|
||||
fill_generation: self.fill_generation,
|
||||
key: self.key,
|
||||
expected_size: self.expected_size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ObjectDataCacheReservedBody {
|
||||
/// Returns a clone that shares the reservation-owning allocation.
|
||||
pub fn bytes(&self) -> Bytes {
|
||||
self.bytes.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn into_parts(self) -> (Bytes, OwnedSemaphorePermit, crate::moka_backend::FillGenerationGuard) {
|
||||
(self.bytes, self.permit, self.fill_generation)
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimum spacing between cache-state gauge publishes. Moka's `entry_count`
|
||||
/// and `weighted_size` are cross-segment approximations that only settle after
|
||||
@@ -45,6 +93,9 @@ pub struct ObjectDataCache {
|
||||
backend: ObjectDataCacheBackendKind,
|
||||
config: Arc<ObjectDataCacheConfig>,
|
||||
stats: Arc<ObjectDataCacheStats>,
|
||||
/// Resolved Moka weighted capacity used by the planner's exact key-aware
|
||||
/// admission check. Zero for the disabled backend.
|
||||
max_capacity: u64,
|
||||
/// Effective fill ceiling in bytes: a body larger than this is planned
|
||||
/// `SkipTooLarge` even when it fits `max_entry_bytes`. The app layer sets it
|
||||
/// to `min(max_entry_bytes, seek-support threshold, 64 MiB buffer cap)` so a
|
||||
@@ -72,6 +123,7 @@ impl ObjectDataCache {
|
||||
backend: ObjectDataCacheBackendKind::Noop(NoopBackend),
|
||||
config,
|
||||
stats,
|
||||
max_capacity: 0,
|
||||
fill_ceiling_bytes: 0,
|
||||
created_at: Instant::now(),
|
||||
last_entry_publish_ms: AtomicU64::new(0),
|
||||
@@ -83,16 +135,19 @@ impl ObjectDataCache {
|
||||
describe_metrics_once();
|
||||
config.validate()?;
|
||||
let stats = Arc::new(ObjectDataCacheStats::default());
|
||||
let backend = if config.is_disabled() {
|
||||
ObjectDataCacheBackendKind::Noop(NoopBackend)
|
||||
let (backend, max_capacity) = if config.is_disabled() {
|
||||
(ObjectDataCacheBackendKind::Noop(NoopBackend), 0)
|
||||
} else {
|
||||
ObjectDataCacheBackendKind::Moka(Box::new(MokaBackend::new(&config, Arc::clone(&stats))?))
|
||||
let backend = MokaBackend::new(&config, Arc::clone(&stats))?;
|
||||
let max_capacity = backend.max_capacity();
|
||||
(ObjectDataCacheBackendKind::Moka(Box::new(backend)), max_capacity)
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
backend,
|
||||
config: Arc::new(config),
|
||||
stats,
|
||||
max_capacity,
|
||||
fill_ceiling_bytes: 0,
|
||||
created_at: Instant::now(),
|
||||
last_entry_publish_ms: AtomicU64::new(0),
|
||||
@@ -121,14 +176,26 @@ impl ObjectDataCache {
|
||||
|
||||
/// Produces a lightweight GET plan from request metadata.
|
||||
pub fn plan_get(&self, request: ObjectDataCacheGetRequest<'_>) -> ObjectDataCacheGetPlan {
|
||||
self.plan_get_inner(request, true)
|
||||
}
|
||||
|
||||
/// Rebuilds a plan for identity revalidation without counting another GET.
|
||||
#[doc(hidden)]
|
||||
pub fn plan_get_untracked(&self, request: ObjectDataCacheGetRequest<'_>) -> ObjectDataCacheGetPlan {
|
||||
self.plan_get_inner(request, false)
|
||||
}
|
||||
|
||||
fn plan_get_inner(&self, request: ObjectDataCacheGetRequest<'_>, record_metric: bool) -> ObjectDataCacheGetPlan {
|
||||
if self.config.is_disabled() {
|
||||
record_plan_decision(
|
||||
self.backend.as_metric_label(),
|
||||
self.config.mode,
|
||||
"disabled",
|
||||
"mode_disabled",
|
||||
request.size,
|
||||
);
|
||||
if record_metric {
|
||||
record_plan_decision(
|
||||
self.backend.as_metric_label(),
|
||||
self.config.mode,
|
||||
"disabled",
|
||||
"mode_disabled",
|
||||
request.size,
|
||||
);
|
||||
}
|
||||
return ObjectDataCacheGetPlan::Disabled;
|
||||
}
|
||||
|
||||
@@ -137,24 +204,34 @@ impl ObjectDataCache {
|
||||
// in-memory GET fill limits could never fill, so admitting it here would
|
||||
// report it "eligible" while it kept a permanent 0% hit rate.
|
||||
if request.size > self.effective_size_ceiling() {
|
||||
record_plan_decision(self.backend.as_metric_label(), self.config.mode, "skip", "too_large", request.size);
|
||||
if record_metric {
|
||||
record_plan_decision(self.backend.as_metric_label(), self.config.mode, "skip", "too_large", request.size);
|
||||
}
|
||||
return ObjectDataCacheGetPlan::SkipTooLarge;
|
||||
}
|
||||
|
||||
record_plan_decision(self.backend.as_metric_label(), self.config.mode, "cacheable", "eligible", request.size);
|
||||
|
||||
ObjectDataCacheGetPlan::Cacheable {
|
||||
key: ObjectDataCacheKey::with_write_anchors(
|
||||
request.bucket,
|
||||
request.object,
|
||||
request.version_id.as_deref(),
|
||||
request.etag,
|
||||
request.size,
|
||||
request.data_dir_u128,
|
||||
request.mod_time_unix_nanos,
|
||||
request.body_variant,
|
||||
),
|
||||
let key = ObjectDataCacheKey::with_write_anchors(
|
||||
request.bucket,
|
||||
request.object,
|
||||
request.version_id.as_deref(),
|
||||
request.etag,
|
||||
request.size,
|
||||
request.data_dir_u128,
|
||||
request.mod_time_unix_nanos,
|
||||
request.body_variant,
|
||||
);
|
||||
if projected_weight(&key, request.size) > self.max_capacity {
|
||||
if record_metric {
|
||||
record_plan_decision(self.backend.as_metric_label(), self.config.mode, "skip", "too_large", request.size);
|
||||
}
|
||||
return ObjectDataCacheGetPlan::SkipTooLarge;
|
||||
}
|
||||
|
||||
if record_metric {
|
||||
record_plan_decision(self.backend.as_metric_label(), self.config.mode, "cacheable", "eligible", request.size);
|
||||
}
|
||||
|
||||
ObjectDataCacheGetPlan::Cacheable { key }
|
||||
}
|
||||
|
||||
/// Looks up an object body from the configured backend.
|
||||
@@ -192,6 +269,61 @@ impl ObjectDataCache {
|
||||
lookup
|
||||
}
|
||||
|
||||
/// Performs an internal second-chance lookup without recording another
|
||||
/// request lookup. Callers must have already performed the authoritative
|
||||
/// lookup for the current GET.
|
||||
#[doc(hidden)]
|
||||
pub async fn peek_body_untracked(&self, plan: &ObjectDataCacheGetPlan) -> ObjectDataCacheLookup {
|
||||
match &self.backend {
|
||||
ObjectDataCacheBackendKind::Noop(backend) => backend.lookup_body(plan).await,
|
||||
ObjectDataCacheBackendKind::Moka(backend) => backend.lookup_body(plan).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Reserves memory and a fill slot before allocating a cold-fill body.
|
||||
pub fn reserve_body(&self, plan: &ObjectDataCacheGetPlan) -> Option<ObjectDataCacheBodyReservation> {
|
||||
if !self.config.fill_enabled() {
|
||||
return None;
|
||||
}
|
||||
match &self.backend {
|
||||
ObjectDataCacheBackendKind::Noop(_) => None,
|
||||
ObjectDataCacheBackendKind::Moka(backend) => backend.reserve_body(plan),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fills from a body admitted before allocation.
|
||||
pub async fn fill_reserved_body(
|
||||
&self,
|
||||
plan: &ObjectDataCacheGetPlan,
|
||||
body: ObjectDataCacheReservedBody,
|
||||
) -> ObjectDataCacheFillResult {
|
||||
let fill_bytes = u64::try_from(body.bytes.len()).unwrap_or(u64::MAX);
|
||||
let fill_start = Instant::now();
|
||||
let result = match &self.backend {
|
||||
ObjectDataCacheBackendKind::Noop(_) => ObjectDataCacheFillResult::SkippedDisabled,
|
||||
ObjectDataCacheBackendKind::Moka(backend) => backend.fill_reserved_body(plan, body).await,
|
||||
};
|
||||
let (recorded_bytes, duration) = match result {
|
||||
ObjectDataCacheFillResult::Inserted => {
|
||||
self.stats.record_fill();
|
||||
self.refresh_entry_count();
|
||||
(fill_bytes, Some(fill_start.elapsed().as_secs_f64()))
|
||||
}
|
||||
ObjectDataCacheFillResult::SkippedInvalidationRace | ObjectDataCacheFillResult::SkippedIdentityOverflow => {
|
||||
(fill_bytes, Some(fill_start.elapsed().as_secs_f64()))
|
||||
}
|
||||
_ => (0, None),
|
||||
};
|
||||
record_fill_result(
|
||||
self.backend.as_metric_label(),
|
||||
self.config.mode,
|
||||
result.as_metric_label(),
|
||||
recorded_bytes,
|
||||
duration,
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
/// Attempts to fill the cache body for the current plan.
|
||||
pub async fn fill_body(&self, plan: &ObjectDataCacheGetPlan, bytes: Bytes) -> ObjectDataCacheFillResult {
|
||||
let fill_bytes = u64::try_from(bytes.len()).unwrap_or(u64::MAX);
|
||||
@@ -434,6 +566,16 @@ pub enum ObjectDataCacheGetPlan {
|
||||
},
|
||||
}
|
||||
|
||||
impl ObjectDataCacheGetPlan {
|
||||
/// Returns the stable cache key for a cacheable plan.
|
||||
pub fn key(&self) -> Option<&ObjectDataCacheKey> {
|
||||
match self {
|
||||
Self::Cacheable { key } => Some(key),
|
||||
Self::Disabled | Self::SkipTooLarge => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of a cache lookup attempt.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ObjectDataCacheLookup {
|
||||
@@ -863,6 +1005,80 @@ mod tests {
|
||||
assert!(matches!(lookup, ObjectDataCacheLookup::Miss));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserved_body_is_bound_to_origin_cache_and_plan() {
|
||||
let cache_a = fill_enabled_cache();
|
||||
let cache_b = fill_enabled_cache();
|
||||
let plan = cache_a.plan_get(plain_request("bucket", "object", "etag", 5));
|
||||
let wrong_plan = cache_a.plan_get(plain_request("bucket", "other", "etag", 5));
|
||||
|
||||
let wrong_plan_body = cache_a
|
||||
.reserve_body(&plan)
|
||||
.expect("the original plan should be admitted")
|
||||
.wrap_bytes(Bytes::from_static(b"hello"));
|
||||
assert_eq!(
|
||||
cache_a.fill_reserved_body(&wrong_plan, wrong_plan_body).await,
|
||||
ObjectDataCacheFillResult::SkippedNotCacheable
|
||||
);
|
||||
|
||||
let cross_cache_body = cache_a
|
||||
.reserve_body(&plan)
|
||||
.expect("the original cache should admit another reservation")
|
||||
.wrap_bytes(Bytes::from_static(b"hello"));
|
||||
assert_eq!(
|
||||
cache_b.fill_reserved_body(&plan, cross_cache_body).await,
|
||||
ObjectDataCacheFillResult::SkippedNotCacheable
|
||||
);
|
||||
|
||||
let wrong_size_body = cache_a
|
||||
.reserve_body(&plan)
|
||||
.expect("the original plan should still be admitted")
|
||||
.wrap_bytes(Bytes::from_static(b"oops"));
|
||||
assert_eq!(
|
||||
cache_a.fill_reserved_body(&plan, wrong_size_body).await,
|
||||
ObjectDataCacheFillResult::SkippedSizeMismatch
|
||||
);
|
||||
assert_eq!(cache_a.stats().fills, 0, "rejected reservations must not count as successful fills");
|
||||
assert!(matches!(cache_a.lookup_body(&plan).await, ObjectDataCacheLookup::Miss));
|
||||
assert!(matches!(cache_a.lookup_body(&wrong_plan).await, ObjectDataCacheLookup::Miss));
|
||||
assert!(matches!(cache_b.lookup_body(&plan).await, ObjectDataCacheLookup::Miss));
|
||||
|
||||
let body = cache_a
|
||||
.reserve_body(&plan)
|
||||
.expect("the matching reservation should be admitted")
|
||||
.wrap_bytes(Bytes::from_static(b"hello"));
|
||||
assert_eq!(cache_a.fill_reserved_body(&plan, body).await, ObjectDataCacheFillResult::Inserted);
|
||||
assert_eq!(cache_a.stats().fills, 1);
|
||||
assert_eq!(cache_a.lookup_body(&plan).await, ObjectDataCacheLookup::Hit(Bytes::from_static(b"hello")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reserved_size_mismatch_records_fill_outcome_without_bytes() {
|
||||
let cache = fill_enabled_cache();
|
||||
let metrics = capture_metrics(|| async {
|
||||
let plan = cache.plan_get(plain_request("bucket", "object", "etag", 5));
|
||||
let body = cache
|
||||
.reserve_body(&plan)
|
||||
.expect("the plan should be admitted before materialization")
|
||||
.wrap_bytes(Bytes::from_static(b"oops"));
|
||||
assert_eq!(
|
||||
cache.fill_reserved_body(&plan, body).await,
|
||||
ObjectDataCacheFillResult::SkippedSizeMismatch
|
||||
);
|
||||
});
|
||||
|
||||
assert!(has_counter_with_label(
|
||||
&metrics,
|
||||
"rustfs_object_data_cache_fill_total",
|
||||
("result", "skipped_size_mismatch")
|
||||
));
|
||||
assert_eq!(
|
||||
counter_total(&metrics, "rustfs_object_data_cache_fill_bytes_total"),
|
||||
None,
|
||||
"a rejected reserved body must not record filled bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plan_clamps_size_eligibility_to_fill_ceiling() {
|
||||
// ODC-24 (backlog#1129): a body in the gap between `max_entry_bytes` and
|
||||
@@ -895,6 +1111,34 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn planner_rejects_key_whose_projected_weight_exceeds_capacity() {
|
||||
let config = ObjectDataCacheConfig {
|
||||
mode: ObjectDataCacheMode::HitOnly,
|
||||
max_bytes: 8 * 1024,
|
||||
max_memory_percent: 0,
|
||||
max_entry_bytes: 4 * 1024,
|
||||
..ObjectDataCacheConfig::default()
|
||||
};
|
||||
let cache = ObjectDataCache::new(config).expect("capacity test config should initialize");
|
||||
// Body (4096) + fixed fields (bucket=1, version=null=4, etag=1) +
|
||||
// object (4026) + entry overhead (64) = 8192, exactly capacity.
|
||||
let boundary_object = "o".repeat(4026);
|
||||
assert!(matches!(
|
||||
cache.plan_get(plain_request("b", &boundary_object, "e", 4 * 1024)),
|
||||
ObjectDataCacheGetPlan::Cacheable { .. }
|
||||
));
|
||||
|
||||
// Body (4096) + fixed fields (bucket=1, version=null=4, etag=1) +
|
||||
// object (4027) + entry overhead (64) = 8193, one byte over capacity.
|
||||
let overweight_object = "o".repeat(4027);
|
||||
|
||||
assert_eq!(
|
||||
cache.plan_get(plain_request("b", &overweight_object, "e", 4 * 1024)),
|
||||
ObjectDataCacheGetPlan::SkipTooLarge
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plan_carries_mod_time_into_key() {
|
||||
// ODC-06: the resolved modification time must reach the key so two
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::error::ObjectDataCacheConfigError;
|
||||
use crate::memory::{MemoryBasis, resolve_effective_memory};
|
||||
use crate::memory::{EffectiveMemory, MemoryBasis, resolve_effective_memory};
|
||||
use std::sync::Once;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -142,13 +142,21 @@ impl ObjectDataCacheConfig {
|
||||
/// Resolves the effective max capacity in bytes for the cache.
|
||||
pub fn resolved_max_bytes(&self) -> Result<u64, ObjectDataCacheConfigError> {
|
||||
if self.max_bytes > 0 {
|
||||
validate_entry_fits_capacity(
|
||||
self.max_bytes,
|
||||
self.max_entry_bytes,
|
||||
ObjectDataCacheConfigError::MaxEntryBytesExceedsMaxBytes,
|
||||
)?;
|
||||
return Ok(self.max_bytes);
|
||||
}
|
||||
|
||||
// Resolve capacity from the effective (container-aware) total memory so
|
||||
// a pod with a cgroup limit far below the node RAM does not size the
|
||||
// cache to the node.
|
||||
let effective = resolve_effective_memory();
|
||||
self.resolved_max_bytes_for_effective_memory(resolve_effective_memory())
|
||||
}
|
||||
|
||||
fn resolved_max_bytes_for_effective_memory(&self, effective: EffectiveMemory) -> Result<u64, ObjectDataCacheConfigError> {
|
||||
let total_memory = effective.total_bytes;
|
||||
let derived = total_memory.saturating_mul(u64::from(self.max_memory_percent)) / 100;
|
||||
let resolved = clamp_derived_max_bytes(derived, total_memory);
|
||||
@@ -159,10 +167,9 @@ impl ObjectDataCacheConfig {
|
||||
|
||||
// The derived capacity is no longer floored by `max_entry_bytes` (that
|
||||
// used to silently inflate the cache above the safety clamp). If a
|
||||
// single entry cannot fit, reject rather than inflate.
|
||||
if self.max_entry_bytes > resolved {
|
||||
return Err(ObjectDataCacheConfigError::MaxEntryBytesExceedsCapacity);
|
||||
}
|
||||
// single entry plus its weight overhead cannot fit, reject rather than
|
||||
// inflate.
|
||||
validate_entry_fits_capacity(resolved, self.max_entry_bytes, ObjectDataCacheConfigError::MaxEntryBytesExceedsCapacity)?;
|
||||
|
||||
log_resolved_capacity_once(resolved, total_memory, effective.basis);
|
||||
|
||||
@@ -186,8 +193,12 @@ impl ObjectDataCacheConfig {
|
||||
// An explicit capacity must leave room for a full entry plus the
|
||||
// weigher overhead, otherwise moka can never retain the entry while
|
||||
// fills still report success.
|
||||
if self.max_bytes > 0 && self.max_bytes < self.max_entry_bytes.saturating_add(ENTRY_WEIGHT_OVERHEAD_BYTES) {
|
||||
return Err(ObjectDataCacheConfigError::MaxEntryBytesExceedsMaxBytes);
|
||||
if self.max_bytes > 0 {
|
||||
validate_entry_fits_capacity(
|
||||
self.max_bytes,
|
||||
self.max_entry_bytes,
|
||||
ObjectDataCacheConfigError::MaxEntryBytesExceedsMaxBytes,
|
||||
)?;
|
||||
}
|
||||
|
||||
if self.ttl.is_zero() {
|
||||
@@ -249,6 +260,17 @@ impl ObjectDataCacheConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_entry_fits_capacity(
|
||||
capacity: u64,
|
||||
max_entry_bytes: u64,
|
||||
error: ObjectDataCacheConfigError,
|
||||
) -> Result<(), ObjectDataCacheConfigError> {
|
||||
match max_entry_bytes.checked_add(ENTRY_WEIGHT_OVERHEAD_BYTES) {
|
||||
Some(required_capacity) if required_capacity <= capacity => Ok(()),
|
||||
_ => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
fn clamp_derived_max_bytes(derived: u64, total_memory: u64) -> u64 {
|
||||
let percent_cap = total_memory.saturating_mul(DEFAULT_DERIVED_MAX_MEMORY_PERCENT_CAP) / 100;
|
||||
let safe_cap = percent_cap.min(DEFAULT_DERIVED_MAX_BYTES_CAP);
|
||||
@@ -269,8 +291,12 @@ fn log_resolved_capacity_once(resolved_max_bytes: u64, effective_total_bytes: u6
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{DEFAULT_DERIVED_MAX_BYTES_CAP, ObjectDataCacheConfig, ObjectDataCacheMode, clamp_derived_max_bytes};
|
||||
use super::{
|
||||
DEFAULT_DERIVED_MAX_BYTES_CAP, ENTRY_WEIGHT_OVERHEAD_BYTES, ObjectDataCacheConfig, ObjectDataCacheMode,
|
||||
clamp_derived_max_bytes,
|
||||
};
|
||||
use crate::error::ObjectDataCacheConfigError;
|
||||
use crate::memory::{EffectiveMemory, MemoryBasis};
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
@@ -459,6 +485,87 @@ mod tests {
|
||||
assert_eq!(err, ObjectDataCacheConfigError::MaxEntryBytesExceedsCapacity);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derived_capacity_rejects_entry_equal_to_capacity() {
|
||||
const CAPACITY: u64 = 8 * 1024;
|
||||
let config = ObjectDataCacheConfig {
|
||||
max_bytes: 0,
|
||||
max_memory_percent: 10,
|
||||
max_entry_bytes: CAPACITY,
|
||||
..ObjectDataCacheConfig::default()
|
||||
};
|
||||
|
||||
let err = config
|
||||
.resolved_max_bytes_for_effective_memory(EffectiveMemory {
|
||||
total_bytes: CAPACITY * 10,
|
||||
available_bytes: CAPACITY * 10,
|
||||
basis: MemoryBasis::Host,
|
||||
})
|
||||
.expect_err("an entry equal to derived capacity leaves no room for its weight overhead");
|
||||
|
||||
assert_eq!(err, ObjectDataCacheConfigError::MaxEntryBytesExceedsCapacity);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_and_derived_capacity_share_entry_margin_matrix() {
|
||||
const CAPACITY: u64 = 8 * 1024;
|
||||
let effective_memory = EffectiveMemory {
|
||||
total_bytes: CAPACITY * 10,
|
||||
available_bytes: CAPACITY * 10,
|
||||
basis: MemoryBasis::Host,
|
||||
};
|
||||
|
||||
for (max_entry_bytes, fits) in [
|
||||
(CAPACITY - ENTRY_WEIGHT_OVERHEAD_BYTES, true),
|
||||
(CAPACITY - ENTRY_WEIGHT_OVERHEAD_BYTES + 1, false),
|
||||
(CAPACITY - 1, false),
|
||||
(CAPACITY, false),
|
||||
] {
|
||||
let explicit = ObjectDataCacheConfig {
|
||||
max_bytes: CAPACITY,
|
||||
max_memory_percent: 0,
|
||||
max_entry_bytes,
|
||||
..ObjectDataCacheConfig::default()
|
||||
};
|
||||
let derived = ObjectDataCacheConfig {
|
||||
max_bytes: 0,
|
||||
max_memory_percent: 10,
|
||||
max_entry_bytes,
|
||||
..ObjectDataCacheConfig::default()
|
||||
};
|
||||
|
||||
if fits {
|
||||
assert_eq!(explicit.validate(), Ok(()), "explicit max_entry_bytes={max_entry_bytes}");
|
||||
assert_eq!(
|
||||
explicit.resolved_max_bytes(),
|
||||
Ok(CAPACITY),
|
||||
"explicit resolved max_entry_bytes={max_entry_bytes}"
|
||||
);
|
||||
assert_eq!(
|
||||
derived.resolved_max_bytes_for_effective_memory(effective_memory),
|
||||
Ok(CAPACITY),
|
||||
"derived max_entry_bytes={max_entry_bytes}"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(
|
||||
explicit.validate(),
|
||||
Err(ObjectDataCacheConfigError::MaxEntryBytesExceedsMaxBytes),
|
||||
"explicit max_entry_bytes={max_entry_bytes}"
|
||||
);
|
||||
assert_eq!(
|
||||
explicit.resolved_max_bytes(),
|
||||
Err(ObjectDataCacheConfigError::MaxEntryBytesExceedsMaxBytes),
|
||||
"explicit resolved max_entry_bytes={max_entry_bytes}"
|
||||
);
|
||||
assert_eq!(
|
||||
derived.resolved_max_bytes_for_effective_memory(effective_memory),
|
||||
Err(ObjectDataCacheConfigError::MaxEntryBytesExceedsCapacity),
|
||||
"derived max_entry_bytes={max_entry_bytes}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derived_max_bytes_clamps_to_v3_safe_cap() {
|
||||
let one_tib = 1024_u64 * 1024 * 1024 * 1024;
|
||||
|
||||
@@ -13,11 +13,26 @@
|
||||
// limitations under the License.
|
||||
|
||||
use bytes::Bytes;
|
||||
use std::cmp;
|
||||
|
||||
use crate::index::ObjectDataCacheGeneration;
|
||||
use crate::key::ObjectDataCacheKey;
|
||||
|
||||
const ENTRY_OVERHEAD_BYTES: usize = 64;
|
||||
const ENTRY_OVERHEAD_BYTES: u64 = 64;
|
||||
|
||||
/// Estimates the weighted capacity charged for a key and planned body.
|
||||
pub(crate) fn projected_weight(key: &ObjectDataCacheKey, body_bytes: u64) -> u64 {
|
||||
let key_bytes = key
|
||||
.bucket
|
||||
.len()
|
||||
.saturating_add(key.object.len())
|
||||
.saturating_add(key.version_id.len())
|
||||
.saturating_add(key.etag.len());
|
||||
|
||||
u64::try_from(key_bytes)
|
||||
.unwrap_or(u64::MAX)
|
||||
.saturating_add(body_bytes)
|
||||
.saturating_add(ENTRY_OVERHEAD_BYTES)
|
||||
}
|
||||
|
||||
/// Cached object body entry.
|
||||
///
|
||||
@@ -29,12 +44,17 @@ const ENTRY_OVERHEAD_BYTES: usize = 64;
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ObjectDataCacheEntry {
|
||||
bytes: Bytes,
|
||||
generation: ObjectDataCacheGeneration,
|
||||
}
|
||||
|
||||
impl ObjectDataCacheEntry {
|
||||
/// Creates a new cached entry.
|
||||
pub fn new(bytes: Bytes) -> Self {
|
||||
Self { bytes }
|
||||
Self { bytes, generation: 0 }
|
||||
}
|
||||
|
||||
pub(crate) fn with_generation(bytes: Bytes, generation: ObjectDataCacheGeneration) -> Self {
|
||||
Self { bytes, generation }
|
||||
}
|
||||
|
||||
/// Returns a clone of the cached body bytes.
|
||||
@@ -42,14 +62,14 @@ impl ObjectDataCacheEntry {
|
||||
self.bytes.clone()
|
||||
}
|
||||
|
||||
pub(crate) const fn generation(&self) -> ObjectDataCacheGeneration {
|
||||
self.generation
|
||||
}
|
||||
|
||||
/// Returns the estimated weighted size for capacity accounting.
|
||||
pub fn estimated_weight(&self, key: &ObjectDataCacheKey) -> u32 {
|
||||
let key_bytes = key.bucket.len() + key.object.len() + key.version_id.len() + key.etag.len();
|
||||
let body_bytes = self.bytes.len();
|
||||
let total = key_bytes.saturating_add(body_bytes).saturating_add(ENTRY_OVERHEAD_BYTES);
|
||||
let clamped = cmp::min(total, u32::MAX as usize);
|
||||
|
||||
u32::try_from(clamped).unwrap_or(u32::MAX)
|
||||
let body_bytes = u64::try_from(self.bytes.len()).unwrap_or(u64::MAX);
|
||||
u32::try_from(projected_weight(key, body_bytes)).unwrap_or(u32::MAX)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,13 +15,24 @@
|
||||
use crate::key::ObjectDataCacheKey;
|
||||
use crate::starshard_index::StarshardIdentityIndex;
|
||||
|
||||
/// Per-entry generation token used to tell a freshly refilled body apart from a
|
||||
/// superseded one under the same key. The token is the cache entry's `Arc`
|
||||
/// pointer captured at fill time: the eviction listener receives the evicted
|
||||
/// value and can compare its pointer against the token currently tracked, so a
|
||||
/// deferred or inline eviction of an old generation cannot remove the index key
|
||||
/// registered for the current generation.
|
||||
/// Legacy public identity-index token type.
|
||||
///
|
||||
/// The backend uses an internal monotonic `u64` generation instead; keeping this
|
||||
/// alias preserves the existing public `StarshardIdentityIndex` method shapes.
|
||||
pub(crate) type ObjectDataCacheKeyToken = usize;
|
||||
pub(crate) type ObjectDataCacheGeneration = u64;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct ObjectDataCacheEvictedGeneration {
|
||||
pub(crate) key: ObjectDataCacheKey,
|
||||
pub(crate) generation: ObjectDataCacheGeneration,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) enum ObjectDataCacheGenerationalInsertResult {
|
||||
Inserted { evicted: Vec<ObjectDataCacheEvictedGeneration> },
|
||||
Duplicate,
|
||||
}
|
||||
|
||||
/// Result of inserting a cache key into the identity index.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -41,7 +52,7 @@ pub enum ObjectDataCacheIndexInsertResult {
|
||||
#[derive(Debug, Clone)]
|
||||
struct TrackedKey {
|
||||
key: ObjectDataCacheKey,
|
||||
token: ObjectDataCacheKeyToken,
|
||||
generation: ObjectDataCacheGeneration,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
@@ -50,37 +61,41 @@ pub(crate) struct ObjectDataCacheKeySet {
|
||||
}
|
||||
|
||||
impl ObjectDataCacheKeySet {
|
||||
pub(crate) fn insert(
|
||||
pub(crate) fn insert_generation(
|
||||
&mut self,
|
||||
key: ObjectDataCacheKey,
|
||||
token: ObjectDataCacheKeyToken,
|
||||
generation: ObjectDataCacheGeneration,
|
||||
max_keys: usize,
|
||||
) -> ObjectDataCacheIndexInsertResult {
|
||||
) -> ObjectDataCacheGenerationalInsertResult {
|
||||
if let Some(existing) = self.keys.iter_mut().find(|existing| existing.key == key) {
|
||||
// Refresh the generation token so a later eviction of the superseded
|
||||
// Refresh the generation so a later eviction of the superseded
|
||||
// body cannot remove the key registered for this new body.
|
||||
existing.token = token;
|
||||
return ObjectDataCacheIndexInsertResult::Duplicate;
|
||||
existing.generation = generation;
|
||||
return ObjectDataCacheGenerationalInsertResult::Duplicate;
|
||||
}
|
||||
|
||||
// Bounded eviction: the Vec preserves insertion order, so evicting from
|
||||
// the front drops the oldest keys and keeps hot (recently filled) ones,
|
||||
// rather than clearing the whole identity and rejecting the new key.
|
||||
let mut evicted_keys = Vec::new();
|
||||
let mut evicted = Vec::new();
|
||||
while self.keys.len() >= max_keys {
|
||||
evicted_keys.push(self.keys.remove(0).key);
|
||||
let tracked = self.keys.remove(0);
|
||||
evicted.push(ObjectDataCacheEvictedGeneration {
|
||||
key: tracked.key,
|
||||
generation: tracked.generation,
|
||||
});
|
||||
}
|
||||
|
||||
self.keys.push(TrackedKey { key, token });
|
||||
ObjectDataCacheIndexInsertResult::Inserted { evicted_keys }
|
||||
self.keys.push(TrackedKey { key, generation });
|
||||
ObjectDataCacheGenerationalInsertResult::Inserted { evicted }
|
||||
}
|
||||
|
||||
/// Removes the key only when its tracked generation token matches, so an
|
||||
/// eviction notification for an old generation leaves a refreshed key intact.
|
||||
pub(crate) fn remove_evicted_key(&mut self, key: &ObjectDataCacheKey, token: ObjectDataCacheKeyToken) -> bool {
|
||||
pub(crate) fn remove_generation(&mut self, key: &ObjectDataCacheKey, generation: ObjectDataCacheGeneration) -> bool {
|
||||
let original_len = self.keys.len();
|
||||
self.keys
|
||||
.retain(|existing| !(existing.key == *key && existing.token == token));
|
||||
.retain(|existing| !(existing.key == *key && existing.generation == generation));
|
||||
original_len != self.keys.len()
|
||||
}
|
||||
|
||||
@@ -99,6 +114,12 @@ impl ObjectDataCacheKeySet {
|
||||
self.keys.iter().any(|existing| &existing.key == key)
|
||||
}
|
||||
|
||||
pub(crate) fn contains_generation(&self, key: &ObjectDataCacheKey, generation: ObjectDataCacheGeneration) -> bool {
|
||||
self.keys
|
||||
.iter()
|
||||
.any(|existing| &existing.key == key && existing.generation == generation)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn len(&self) -> usize {
|
||||
self.keys.len()
|
||||
@@ -114,7 +135,7 @@ pub type ObjectDataCacheIdentityIndex = StarshardIdentityIndex;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ObjectDataCacheIndexInsertResult, ObjectDataCacheKeySet};
|
||||
use super::{ObjectDataCacheGenerationalInsertResult, ObjectDataCacheKeySet};
|
||||
use crate::key::{ObjectDataCacheBodyVariant, ObjectDataCacheKey};
|
||||
|
||||
fn make_key(id: &str) -> ObjectDataCacheKey {
|
||||
@@ -126,16 +147,11 @@ mod tests {
|
||||
let mut set = ObjectDataCacheKeySet::default();
|
||||
let key = make_key("v1");
|
||||
|
||||
let first = set.insert(key.clone(), 1, 4);
|
||||
let second = set.insert(key, 2, 4);
|
||||
let first = set.insert_generation(key.clone(), 1, 4);
|
||||
let second = set.insert_generation(key, 2, 4);
|
||||
|
||||
assert_eq!(
|
||||
first,
|
||||
ObjectDataCacheIndexInsertResult::Inserted {
|
||||
evicted_keys: Vec::new()
|
||||
}
|
||||
);
|
||||
assert_eq!(second, ObjectDataCacheIndexInsertResult::Duplicate);
|
||||
assert_eq!(first, ObjectDataCacheGenerationalInsertResult::Inserted { evicted: Vec::new() });
|
||||
assert_eq!(second, ObjectDataCacheGenerationalInsertResult::Duplicate);
|
||||
assert_eq!(set.len(), 1);
|
||||
}
|
||||
|
||||
@@ -145,12 +161,13 @@ mod tests {
|
||||
let key_a = make_key("v1");
|
||||
let key_b = make_key("v2");
|
||||
|
||||
let _ = set.insert(key_a.clone(), 1, 1);
|
||||
let result = set.insert(key_b.clone(), 2, 1);
|
||||
let _ = set.insert_generation(key_a.clone(), 1, 1);
|
||||
let result = set.insert_generation(key_b.clone(), 2, 1);
|
||||
|
||||
assert!(matches!(
|
||||
result,
|
||||
ObjectDataCacheIndexInsertResult::Inserted { evicted_keys } if evicted_keys == vec![key_a]
|
||||
ObjectDataCacheGenerationalInsertResult::Inserted { evicted }
|
||||
if evicted.len() == 1 && evicted[0].key == key_a && evicted[0].generation == 1
|
||||
));
|
||||
// The new key replaces the evicted one instead of the identity being cleared.
|
||||
assert!(set.contains(&key_b));
|
||||
@@ -162,14 +179,14 @@ mod tests {
|
||||
let mut set = ObjectDataCacheKeySet::default();
|
||||
let key = make_key("v1");
|
||||
|
||||
let _ = set.insert(key.clone(), 1, 4);
|
||||
let _ = set.insert_generation(key.clone(), 1, 4);
|
||||
|
||||
// A stale eviction notification carrying the old token must not remove the key.
|
||||
assert!(!set.remove_evicted_key(&key, 2));
|
||||
assert!(!set.remove_generation(&key, 2));
|
||||
assert!(set.contains(&key));
|
||||
|
||||
// The matching token removes the key.
|
||||
assert!(set.remove_evicted_key(&key, 1));
|
||||
assert!(set.remove_generation(&key, 1));
|
||||
assert!(!set.contains(&key));
|
||||
}
|
||||
|
||||
@@ -178,11 +195,11 @@ mod tests {
|
||||
let mut set = ObjectDataCacheKeySet::default();
|
||||
let key = make_key("v1");
|
||||
|
||||
let _ = set.insert(key.clone(), 1, 4);
|
||||
let _ = set.insert(key.clone(), 2, 4);
|
||||
let _ = set.insert_generation(key.clone(), 1, 4);
|
||||
let _ = set.insert_generation(key.clone(), 2, 4);
|
||||
|
||||
// After the refresh the old token no longer matches.
|
||||
assert!(!set.remove_evicted_key(&key, 1));
|
||||
assert!(set.remove_evicted_key(&key, 2));
|
||||
assert!(!set.remove_generation(&key, 1));
|
||||
assert!(set.remove_generation(&key, 2));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,8 +57,9 @@ pub mod starshard_index;
|
||||
pub mod stats;
|
||||
|
||||
pub use cache::{
|
||||
ObjectDataCache, ObjectDataCacheFillResult, ObjectDataCacheGetPlan, ObjectDataCacheGetRequest,
|
||||
ObjectDataCacheInvalidationReason, ObjectDataCacheInvalidationResult, ObjectDataCacheLookup,
|
||||
ObjectDataCache, ObjectDataCacheBodyReservation, ObjectDataCacheFillResult, ObjectDataCacheGetPlan,
|
||||
ObjectDataCacheGetRequest, ObjectDataCacheInvalidationReason, ObjectDataCacheInvalidationResult, ObjectDataCacheLookup,
|
||||
ObjectDataCacheReservedBody,
|
||||
};
|
||||
pub use config::{ObjectDataCacheConfig, ObjectDataCacheMode};
|
||||
pub use error::ObjectDataCacheConfigError;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -33,7 +33,7 @@ fn lock_fills(fills: &FillSet) -> std::sync::MutexGuard<'_, HashSet<ObjectDataCa
|
||||
/// reports [`Busy`](ObjectDataCacheSingleflightAcquire::Busy) to everyone else.
|
||||
#[derive(Debug)]
|
||||
pub struct ObjectDataCacheSingleflight {
|
||||
fills: FillSet,
|
||||
fills: Arc<FillSet>,
|
||||
stats: Arc<ObjectDataCacheStats>,
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ impl ObjectDataCacheSingleflight {
|
||||
/// Creates a new singleflight controller.
|
||||
pub fn new(stats: Arc<ObjectDataCacheStats>) -> Self {
|
||||
Self {
|
||||
fills: Mutex::new(HashSet::new()),
|
||||
fills: Arc::new(Mutex::new(HashSet::new())),
|
||||
stats,
|
||||
}
|
||||
}
|
||||
@@ -53,7 +53,7 @@ impl ObjectDataCacheSingleflight {
|
||||
/// [`Busy`](ObjectDataCacheSingleflightAcquire::Busy). The caller already
|
||||
/// owns the body, so a `Busy` outcome skips the redundant fill rather than
|
||||
/// waiting for another request's leader to finish.
|
||||
pub fn try_acquire(&self, key: ObjectDataCacheKey) -> ObjectDataCacheSingleflightAcquire<'_> {
|
||||
pub fn try_acquire(&self, key: ObjectDataCacheKey) -> ObjectDataCacheSingleflightAcquire<'static> {
|
||||
// Keep the critical section to the map mutation only; emit metrics after
|
||||
// dropping the guard so the recorder round-trip never serializes fills.
|
||||
let inflight_len = {
|
||||
@@ -75,9 +75,10 @@ impl ObjectDataCacheSingleflight {
|
||||
set_inflight_fills(&self.stats, "moka", len);
|
||||
ObjectDataCacheSingleflightAcquire::Leader(ObjectDataCacheSingleflightLeader {
|
||||
key,
|
||||
fills: &self.fills,
|
||||
fills: Arc::clone(&self.fills),
|
||||
stats: Arc::clone(&self.stats),
|
||||
finished: false,
|
||||
lifetime: std::marker::PhantomData,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -104,12 +105,13 @@ pub enum ObjectDataCacheSingleflightAcquire<'a> {
|
||||
/// Leader handle for a singleflight fill operation.
|
||||
pub struct ObjectDataCacheSingleflightLeader<'a> {
|
||||
key: ObjectDataCacheKey,
|
||||
fills: &'a FillSet,
|
||||
fills: Arc<FillSet>,
|
||||
stats: Arc<ObjectDataCacheStats>,
|
||||
finished: bool,
|
||||
lifetime: std::marker::PhantomData<&'a ()>,
|
||||
}
|
||||
|
||||
impl<'a> ObjectDataCacheSingleflightLeader<'a> {
|
||||
impl ObjectDataCacheSingleflightLeader<'_> {
|
||||
/// Completes the leader operation and releases the key.
|
||||
pub fn finish(mut self, result: ObjectDataCacheFillResult) -> ObjectDataCacheFillResult {
|
||||
self.remove_entry();
|
||||
@@ -121,7 +123,7 @@ impl<'a> ObjectDataCacheSingleflightLeader<'a> {
|
||||
// Capture the length under the guard, then emit the gauge after dropping
|
||||
// it so the recorder round-trip stays out of the critical section.
|
||||
let len = {
|
||||
let mut fills = lock_fills(self.fills);
|
||||
let mut fills = lock_fills(&self.fills);
|
||||
fills.remove(&self.key);
|
||||
fills.len()
|
||||
};
|
||||
@@ -129,7 +131,7 @@ impl<'a> ObjectDataCacheSingleflightLeader<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Drop for ObjectDataCacheSingleflightLeader<'a> {
|
||||
impl Drop for ObjectDataCacheSingleflightLeader<'_> {
|
||||
fn drop(&mut self) {
|
||||
// A leader dropped without finish() was cancelled mid-fill (e.g. the
|
||||
// fill task was aborted). Release the key so a later fill can become the
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::index::{ObjectDataCacheIndexInsertResult, ObjectDataCacheKeySet, ObjectDataCacheKeyToken};
|
||||
use crate::index::{
|
||||
ObjectDataCacheGeneration, ObjectDataCacheGenerationalInsertResult, ObjectDataCacheIndexInsertResult, ObjectDataCacheKeySet,
|
||||
ObjectDataCacheKeyToken,
|
||||
};
|
||||
use crate::key::{ObjectDataCacheIdentity, ObjectDataCacheKey};
|
||||
use starshard::{AsyncShardedHashMap, DEFAULT_SHARDS, SnapshotMode};
|
||||
use std::collections::hash_map::RandomState;
|
||||
@@ -56,6 +59,21 @@ impl StarshardIdentityIndex {
|
||||
key: ObjectDataCacheKey,
|
||||
token: ObjectDataCacheKeyToken,
|
||||
) -> ObjectDataCacheIndexInsertResult {
|
||||
let generation = u64::try_from(token).unwrap_or(u64::MAX);
|
||||
match self.insert_generation(identity, key, generation).await {
|
||||
ObjectDataCacheGenerationalInsertResult::Inserted { evicted } => ObjectDataCacheIndexInsertResult::Inserted {
|
||||
evicted_keys: evicted.into_iter().map(|entry| entry.key).collect(),
|
||||
},
|
||||
ObjectDataCacheGenerationalInsertResult::Duplicate => ObjectDataCacheIndexInsertResult::Duplicate,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn insert_generation(
|
||||
&self,
|
||||
identity: ObjectDataCacheIdentity,
|
||||
key: ObjectDataCacheKey,
|
||||
generation: ObjectDataCacheGeneration,
|
||||
) -> ObjectDataCacheGenerationalInsertResult {
|
||||
let max_keys = self.max_keys_per_identity;
|
||||
loop {
|
||||
let mut outcome = None;
|
||||
@@ -65,7 +83,7 @@ impl StarshardIdentityIndex {
|
||||
let _ = self
|
||||
.by_object
|
||||
.compute_if_present(&identity, move |mut key_set| {
|
||||
let result = key_set.insert(key, token, max_keys);
|
||||
let result = key_set.insert_generation(key, generation, max_keys);
|
||||
// Insert never empties the set (it either dedups or
|
||||
// bounded-evicts and adds the new key), but keep the
|
||||
// guard so an already-empty entry is not republished.
|
||||
@@ -80,7 +98,7 @@ impl StarshardIdentityIndex {
|
||||
|
||||
// Identity not tracked yet: publish a fresh single-key set.
|
||||
let mut fresh = ObjectDataCacheKeySet::default();
|
||||
let result = fresh.insert(key.clone(), token, max_keys);
|
||||
let result = fresh.insert_generation(key.clone(), generation, max_keys);
|
||||
let final_set = self.by_object.compute_if_absent(identity.clone(), move || fresh).await;
|
||||
if final_set.contains(&key) {
|
||||
return result;
|
||||
@@ -146,6 +164,16 @@ impl StarshardIdentityIndex {
|
||||
identity: &ObjectDataCacheIdentity,
|
||||
key: &ObjectDataCacheKey,
|
||||
token: ObjectDataCacheKeyToken,
|
||||
) -> bool {
|
||||
self.remove_generation(identity, key, u64::try_from(token).unwrap_or(u64::MAX))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn remove_generation(
|
||||
&self,
|
||||
identity: &ObjectDataCacheIdentity,
|
||||
key: &ObjectDataCacheKey,
|
||||
generation: ObjectDataCacheGeneration,
|
||||
) -> bool {
|
||||
let mut removed = false;
|
||||
{
|
||||
@@ -153,7 +181,7 @@ impl StarshardIdentityIndex {
|
||||
let _ = self
|
||||
.by_object
|
||||
.compute_if_present(identity, move |mut key_set| {
|
||||
*removed = key_set.remove_evicted_key(key, token);
|
||||
*removed = key_set.remove_generation(key, generation);
|
||||
(!key_set.is_empty()).then_some(key_set)
|
||||
})
|
||||
.await;
|
||||
@@ -169,6 +197,18 @@ impl StarshardIdentityIndex {
|
||||
.is_some_and(|key_set| key_set.contains(key))
|
||||
}
|
||||
|
||||
pub(crate) async fn contains_generation(
|
||||
&self,
|
||||
identity: &ObjectDataCacheIdentity,
|
||||
key: &ObjectDataCacheKey,
|
||||
generation: ObjectDataCacheGeneration,
|
||||
) -> bool {
|
||||
self.by_object
|
||||
.get(identity)
|
||||
.await
|
||||
.is_some_and(|key_set| key_set.contains_generation(key, generation))
|
||||
}
|
||||
|
||||
/// Returns the number of tracked identities.
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn identity_count(&self) -> usize {
|
||||
|
||||
Reference in New Issue
Block a user