From 25193ee6cf5950003a7990ba22589b7be3d943e2 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 7 Jul 2026 01:38:52 +0800 Subject: [PATCH] feat(listobjects): add guarded metadata-fast listing (#4311) * feat(listobjects): add phase 0 observability metrics * perf(listobjects): reduce gather metadata decoding * test(listobjects): add adversarial listing coverage * feat(listobjects): add source-aware cursor fields * feat(listobjects): define index source modes * feat(listobjects): model index rebuild health * feat(listobjects): add opt-in index fallback gate * feat(listobjects): verify index candidate pages * docs(listobjects): add rollout runbook * docs(listobjects): untrack baseline fixtures * feat(listobjects): add opt-in key-only provider * feat(listobjects): add index verification metrics * perf(listobjects): gate list metrics on get stage switch * feat(listobjects): bind provider health generation * feat(listobjects): add persistent key-only provider * feat(listobjects): bind persistent provider lifecycle * feat(listobjects): track persistent index mutations * feat(listobjects): persist index mutation checkpoints * fix(listobjects): aggregate benchmark metric snapshots * feat(listobjects): store journal in system namespace * chore(listobjects): report fallback reasons in bench * test(listobjects): verify metadata-fast stale fallbacks * feat(listobjects): serve metadata-fast snapshots behind guardrails * test(listobjects): add metadata-fast chaos bench * fix(listobjects): attribute metadata-fast fallback metrics * test(listobjects): add metadata-fast fallback chaos probes * test(listobjects): add quorum journal chaos guardrails * fix(listobjects): satisfy pre-pr clippy gate * docs(listobjects): untrack rollout runbook * fix(ecstore): remove redundant heal error conversion * test(filemeta): satisfy replication info clippy ---- Co-Authored-By: heihutu --- crates/ecstore/src/store/list_objects.rs | 4047 ++++++++++++++++- crates/ecstore/src/store/mod.rs | 46 +- crates/filemeta/src/fileinfo.rs | 22 +- crates/io-metrics/src/lib.rs | 47 + crates/io-metrics/src/list_objects_metrics.rs | 376 ++ rustfs/src/memory_observability.rs | 145 +- rustfs/src/storage/ecfs.rs | 1 + scripts/run_listobjects_verified_bench.sh | 1324 ++++++ ...te_issue_841_list_objects_observability.sh | 102 + 9 files changed, 6033 insertions(+), 77 deletions(-) create mode 100644 crates/io-metrics/src/list_objects_metrics.rs create mode 100755 scripts/run_listobjects_verified_bench.sh create mode 100755 scripts/validate_issue_841_list_objects_observability.sh diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 5692f2364..e6695fb3c 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -18,7 +18,7 @@ use crate::bucket::versioning::VersioningApi; use crate::cache_value::metacache_set::{FallbackClaimTracker, ListPathRawOptions, list_path_raw_with_claim_tracker}; use crate::core::sets::Sets; use crate::disk::error::DiskError; -use crate::disk::{DiskAPI, DiskInfo, DiskStore, WalkDirOptions}; +use crate::disk::{DiskAPI, DiskInfo, DiskStore, RUSTFS_META_BUCKET, WalkDirOptions}; use crate::error::{ Error, Result, StorageError, is_all_not_found, is_all_volume_not_found, is_err_bucket_not_found, to_object_err, }; @@ -33,19 +33,31 @@ use crate::storage_api_contracts::{ }; use crate::store::ECStore; use crate::store::utils::is_reserved_or_invalid_bucket; +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; +use bytes::Bytes; use futures::future::join_all; use rand::seq::SliceRandom; use rustfs_filemeta::{ FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetacacheReader, MetadataResolutionParams, is_io_eof, merge_file_meta_versions, }; +use rustfs_io_metrics::{ + LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED, LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED, LIST_OBJECTS_SOURCE_WALKER, + ListObjectsGatherObservation, ListObjectsIndexPageObservation, +}; use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix}; use std::collections::{HashMap, HashSet}; -use std::sync::Arc; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; +use std::time::{SystemTime, UNIX_EPOCH}; use tokio::io::duplex; -use tokio::sync::OnceCell; use tokio::sync::broadcast::{self}; use tokio::sync::mpsc::{self, Receiver, Sender}; +use tokio::sync::{OnceCell, RwLock}; use tokio_util::sync::CancellationToken; use tracing::{Instrument, debug, error, info, warn}; use uuid::Uuid; @@ -237,14 +249,1557 @@ pub struct ListPathOptions { pub pool_idx: Option, pub set_idx: Option, + pub cursor_source: Option, + pub cursor_generation: Option, } const MARKER_TAG_VERSION: &str = "v2"; const LEGACY_MARKER_TAG_VERSIONS: &[&str] = &["v1", MARKER_TAG_VERSION]; +const LIST_CURSOR_SOURCE_WALKER: &str = "walker"; +const LIST_CURSOR_SOURCE_INDEX_KEY_ONLY: &str = "index_key_only"; +const LIST_CURSOR_SOURCE_INDEX_VERIFIED_PAGE: &str = "index_verified_page"; +const LIST_CURSOR_SOURCE_INDEX_METADATA_FAST: &str = "index_metadata_fast"; +const LIST_CURSOR_GENERATION_LIVE: &str = "live"; const ENV_API_LIST_QUORUM: &str = "RUSTFS_API_LIST_QUORUM"; const DEFAULT_API_LIST_QUORUM: &str = "strict"; const ENV_API_LIST_OBJECTS_QUORUM: &str = "RUSTFS_LIST_OBJECTS_QUORUM"; const DEFAULT_API_LIST_OBJECTS_QUORUM: &str = "optimal"; +const ENV_API_LIST_OBJECTS_INDEX_MODE: &str = "RUSTFS_LIST_OBJECTS_INDEX_MODE"; +const ENV_API_LIST_OBJECTS_INDEX_PROVIDER: &str = "RUSTFS_LIST_OBJECTS_INDEX_PROVIDER"; +const ENV_API_LIST_OBJECTS_INDEX_PROVIDER_PATH: &str = "RUSTFS_LIST_OBJECTS_INDEX_PROVIDER_PATH"; +const ENV_API_LIST_OBJECTS_INDEX_PROVIDER_GENERATION: &str = "RUSTFS_LIST_OBJECTS_INDEX_PROVIDER_GENERATION"; +const ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_PATH: &str = "RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_PATH"; +const ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED: &str = "RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED"; +const ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET: &str = "RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET"; +const ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_SEQUENCE: &str = "RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_SEQUENCE"; +const ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS: &str = "RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS"; +const ENV_API_LIST_OBJECTS_METADATA_FAST_ENABLED: &str = "RUSTFS_LIST_OBJECTS_METADATA_FAST_ENABLED"; +const ENV_API_LIST_OBJECTS_METADATA_FAST_STALENESS_MS: &str = "RUSTFS_LIST_OBJECTS_METADATA_FAST_STALENESS_MS"; +const MAX_LIST_OBJECTS_METADATA_FAST_STALENESS_MS: u64 = 60_000; +const LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY: &str = "walker_key_only"; +const LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY: &str = "persistent_key_only"; +const LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY_DEFAULT_GENERATION: &str = "persistent-key-only"; +const PERSISTENT_KEY_ONLY_INDEX_HEADER: &str = "# rustfs-listobjects-key-only-v1"; +const PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER: &str = "# bucket="; +const PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER: &str = "# generation="; +const PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER: &str = "# checkpoint_high_water_mark="; +const PERSISTENT_KEY_ONLY_INDEX_OBJECT_PREFIX: &str = "# object\t"; +const LIST_OBJECTS_NAMESPACE_JOURNAL_HEADER: &str = "# rustfs-listobjects-namespace-journal-v1"; +const LIST_OBJECTS_NAMESPACE_JOURNAL_BUCKET_HEADER: &str = "# bucket="; +const LIST_OBJECTS_NAMESPACE_JOURNAL_HIGH_WATER_MARK_HEADER: &str = "# high_water_mark="; +const LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_HEADER: &str = "# status="; +const LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_HEALTHY: &str = "healthy"; +const LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_DEGRADED: &str = "degraded"; +const LIST_OBJECTS_NAMESPACE_JOURNAL_DEFAULT_DIR: &str = "namespace-mutation-journal"; +const LIST_OBJECTS_NAMESPACE_JOURNAL_SYSTEM_PREFIX: &str = "listobjects/ns-journal/v1"; + +/// Identifies the source that produced a ListObjects page. +/// +/// `Walker` is the current live xl.meta-backed path. Index modes are future +/// integration points only: key-only and verified-page modes still require live +/// metadata validation before they can satisfy a strong listing, while +/// metadata-fast is explicitly eventually consistent. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ListSourceMode { + Walker, + IndexKeyOnly, + IndexVerifiedPage, + IndexMetadataFast, +} + +impl ListSourceMode { + fn cursor_value(self) -> &'static str { + match self { + Self::Walker => LIST_CURSOR_SOURCE_WALKER, + Self::IndexKeyOnly => LIST_CURSOR_SOURCE_INDEX_KEY_ONLY, + Self::IndexVerifiedPage => LIST_CURSOR_SOURCE_INDEX_VERIFIED_PAGE, + Self::IndexMetadataFast => LIST_CURSOR_SOURCE_INDEX_METADATA_FAST, + } + } + + fn from_cursor_value(source: &str) -> Option { + match source { + LIST_CURSOR_SOURCE_WALKER => Some(Self::Walker), + LIST_CURSOR_SOURCE_INDEX_KEY_ONLY => Some(Self::IndexKeyOnly), + LIST_CURSOR_SOURCE_INDEX_VERIFIED_PAGE => Some(Self::IndexVerifiedPage), + LIST_CURSOR_SOURCE_INDEX_METADATA_FAST => Some(Self::IndexMetadataFast), + _ => None, + } + } + + fn metadata_authority(self) -> ListMetadataAuthority { + match self { + Self::Walker => ListMetadataAuthority::LiveXlMeta, + Self::IndexKeyOnly | Self::IndexVerifiedPage => ListMetadataAuthority::LiveVerifiedIndexCandidate, + Self::IndexMetadataFast => ListMetadataAuthority::IndexSnapshotEventuallyConsistent, + } + } + + fn requires_live_metadata_verification(self) -> bool { + matches!(self, Self::IndexKeyOnly | Self::IndexVerifiedPage) + } + + fn can_satisfy_strong_listing(self) -> bool { + !matches!(self.metadata_authority(), ListMetadataAuthority::IndexSnapshotEventuallyConsistent) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ListMetadataAuthority { + /// Object metadata was read from live xl.meta and can be authoritative for + /// strong ListObjects results. + LiveXlMeta, + /// The index provided candidate keys/pages; live xl.meta validation still + /// owns authoritative object metadata. + LiveVerifiedIndexCandidate, + /// Metadata came from an index snapshot and must not be treated as strong + /// consistency-equivalent to live xl.meta. + IndexSnapshotEventuallyConsistent, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ListIndexFallbackReason { + Disabled, + Rebuilding, + Unhealthy, + Lagging, + Degraded, + Corrupt, + MissingGeneration, + GenerationMismatch, + UnsupportedRequest, + MetadataFastUnavailable, +} + +impl ListIndexFallbackReason { + fn metric_label(self) -> &'static str { + match self { + Self::Disabled => "disabled", + Self::Rebuilding => "rebuilding", + Self::Unhealthy => "unhealthy", + Self::Lagging => "lagging", + Self::Degraded => "degraded", + Self::Corrupt => "corrupt", + Self::MissingGeneration => "missing_generation", + Self::GenerationMismatch => "generation_mismatch", + Self::UnsupportedRequest => "unsupported_request", + Self::MetadataFastUnavailable => "metadata_fast_unavailable", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ListObjectsIndexProviderKind { + WalkerKeyOnly, + PersistentKeyOnly, +} + +impl ListObjectsIndexProviderKind { + fn metric_label(self) -> &'static str { + match self { + Self::WalkerKeyOnly => LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY, + Self::PersistentKeyOnly => LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY, + } + } +} + +fn list_objects_index_provider_metric_label(provider: Option) -> &'static str { + provider.map(ListObjectsIndexProviderKind::metric_label).unwrap_or("none") +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ListObjectsIndexProviderState { + kind: ListObjectsIndexProviderKind, + lifecycle: ListIndexLifecycle, + persistent_path: Option, + persistent_generation: Option, +} + +impl ListObjectsIndexProviderState { + fn walker_key_only() -> Self { + let mut lifecycle = ListIndexLifecycle::disabled(0); + lifecycle.begin_rebuild(LIST_CURSOR_GENERATION_LIVE, 0); + lifecycle.checkpoint_rebuild(0); + let _ = lifecycle.publish_rebuild(); + Self { + kind: ListObjectsIndexProviderKind::WalkerKeyOnly, + lifecycle, + persistent_path: None, + persistent_generation: None, + } + } + + fn persistent_key_only(path: Option, generation: Option) -> Self { + let mut lifecycle = ListIndexLifecycle::disabled(0); + if path.as_ref().is_some_and(|path| !path.as_os_str().is_empty()) { + lifecycle.begin_rebuild( + generation + .as_deref() + .unwrap_or(LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY_DEFAULT_GENERATION), + 0, + ); + } else { + lifecycle.mark_degraded(); + } + + Self { + kind: ListObjectsIndexProviderKind::PersistentKeyOnly, + lifecycle, + persistent_path: path, + persistent_generation: generation, + } + } + + fn from_kind(kind: ListObjectsIndexProviderKind) -> Self { + match kind { + ListObjectsIndexProviderKind::WalkerKeyOnly => Self::walker_key_only(), + ListObjectsIndexProviderKind::PersistentKeyOnly => Self::persistent_key_only(None, None), + } + } + + fn health_snapshot(&self) -> ListIndexHealthSnapshot { + self.lifecycle.health_snapshot() + } +} + +#[derive(Debug, Clone)] +struct PersistentKeyOnlyIndexCache { + path: PathBuf, + len: u64, + modified: Option, + index: Arc, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct PersistentKeyOnlyIndex { + bucket: Option, + generation: String, + checkpoint_high_water_mark: u64, + keys: Arc>, + objects: Arc>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct PersistentListMetadataObject { + name: String, + size: i64, + mod_time: Option, + etag: Option, + storage_class: Option, +} + +impl PersistentListMetadataObject { + fn from_object_info(object: &ObjectInfo) -> Self { + Self { + name: object.name.clone(), + size: object.size, + mod_time: object.mod_time, + etag: object.etag.clone(), + storage_class: object.storage_class.clone(), + } + } + + fn to_object_info(&self, bucket: &str) -> ObjectInfo { + ObjectInfo { + bucket: bucket.to_owned(), + name: self.name.clone(), + size: self.size, + mod_time: self.mod_time, + etag: self.etag.clone(), + storage_class: self.storage_class.clone(), + is_latest: true, + ..Default::default() + } + } +} + +static PERSISTENT_KEY_ONLY_INDEX_CACHE: OnceCell>> = OnceCell::const_new(); +static LIST_OBJECTS_NAMESPACE_JOURNAL_LOCK: OnceCell> = OnceCell::const_new(); +static LIST_OBJECTS_MUTATION_SEQUENCE: AtomicU64 = AtomicU64::new(0); +static LIST_OBJECTS_BUCKET_MUTATION_SEQUENCE: OnceCell>> = OnceCell::const_new(); +static LIST_OBJECTS_NAMESPACE_JOURNAL_DEGRADED_BUCKETS: OnceCell>> = OnceCell::const_new(); +static LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_CONFIG: OnceCell> = OnceCell::const_new(); +static LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_APPLIED: OnceCell>> = OnceCell::const_new(); + +async fn persistent_key_only_index_cache() -> &'static RwLock> { + PERSISTENT_KEY_ONLY_INDEX_CACHE + .get_or_init(|| async { RwLock::new(None) }) + .await +} + +async fn list_objects_namespace_journal_lock() -> &'static RwLock<()> { + LIST_OBJECTS_NAMESPACE_JOURNAL_LOCK + .get_or_init(|| async { RwLock::new(()) }) + .await +} + +async fn list_objects_bucket_mutation_sequence() -> &'static RwLock> { + LIST_OBJECTS_BUCKET_MUTATION_SEQUENCE + .get_or_init(|| async { RwLock::new(HashMap::new()) }) + .await +} + +async fn list_objects_namespace_journal_degraded_buckets() -> &'static RwLock> { + LIST_OBJECTS_NAMESPACE_JOURNAL_DEGRADED_BUCKETS + .get_or_init(|| async { RwLock::new(HashSet::new()) }) + .await +} + +async fn list_objects_namespace_journal_chaos_config() -> Option<&'static NamespaceMutationJournalChaosConfig> { + LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_CONFIG + .get_or_init(|| async { namespace_mutation_journal_chaos_config_from_env() }) + .await + .as_ref() +} + +async fn list_objects_namespace_journal_chaos_applied() -> &'static RwLock> { + LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_APPLIED + .get_or_init(|| async { RwLock::new(HashSet::new()) }) + .await +} + +fn advance_global_list_objects_mutation_sequence(sequence: u64) -> u64 { + let mut current = LIST_OBJECTS_MUTATION_SEQUENCE.load(Ordering::Acquire); + loop { + if current >= sequence { + return current; + } + match LIST_OBJECTS_MUTATION_SEQUENCE.compare_exchange(current, sequence, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => return sequence, + Err(observed) => current = observed, + } + } +} + +async fn advance_list_objects_mutation_sequence(bucket: &str, sequence: u64) -> u64 { + let sequence = advance_global_list_objects_mutation_sequence(sequence); + let sequences = list_objects_bucket_mutation_sequence().await; + let mut sequences = sequences.write().await; + sequences + .entry(bucket.to_owned()) + .and_modify(|current| *current = (*current).max(sequence)) + .or_insert(sequence); + sequence +} + +pub(super) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 { + observe_list_objects_mutations(store, bucket, 1).await.unwrap_or_default() +} + +pub(super) async fn observe_list_objects_mutations(store: &ECStore, bucket: &str, count: usize) -> Option { + observe_list_objects_mutations_with_store(Some(store), bucket, count).await +} + +async fn observe_list_objects_mutations_with_store(store: Option<&ECStore>, bucket: &str, count: usize) -> Option { + if count == 0 { + return None; + } + + let delta = u64::try_from(count).unwrap_or(u64::MAX); + let next = LIST_OBJECTS_MUTATION_SEQUENCE + .fetch_add(delta, Ordering::AcqRel) + .saturating_add(delta); + let sequences = list_objects_bucket_mutation_sequence().await; + let mut sequences = sequences.write().await; + sequences + .entry(bucket.to_owned()) + .and_modify(|current| *current = (*current).max(next)) + .or_insert(next); + persist_observed_list_objects_mutation(store, bucket, next).await; + Some(next) +} + +async fn current_list_objects_mutation_sequence(bucket: &str) -> u64 { + current_list_objects_mutation_snapshot(None, bucket).await.high_water_mark +} + +#[cfg(test)] +async fn reset_list_objects_mutation_sequences_for_test() { + LIST_OBJECTS_MUTATION_SEQUENCE.store(0, Ordering::Release); + let sequences = list_objects_bucket_mutation_sequence().await; + sequences.write().await.clear(); + let degraded = list_objects_namespace_journal_degraded_buckets().await; + degraded.write().await.clear(); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum NamespaceMutationJournalStatus { + Healthy, + Degraded, +} + +impl NamespaceMutationJournalStatus { + fn from_env_value(value: &str) -> Option { + if value.eq_ignore_ascii_case(LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_HEALTHY) { + Some(Self::Healthy) + } else if value.eq_ignore_ascii_case(LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_DEGRADED) { + Some(Self::Degraded) + } else { + None + } + } + + fn env_value(self) -> &'static str { + match self { + Self::Healthy => LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_HEALTHY, + Self::Degraded => LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_DEGRADED, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct NamespaceMutationJournalState { + bucket: Option, + high_water_mark: u64, + status: NamespaceMutationJournalStatus, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct NamespaceMutationJournalSnapshot { + high_water_mark: u64, + degraded: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct NamespaceMutationJournalChaosConfig { + bucket: String, + status: NamespaceMutationJournalStatus, + sequence: Option, +} + +#[derive(Clone)] +enum NamespaceMutationJournalBackend<'a> { + System(&'a ECStore), + LocalPath(PathBuf), +} + +fn parse_namespace_mutation_journal_state(contents: &str) -> Option { + let mut bucket = None; + let mut high_water_mark = None; + let mut status = NamespaceMutationJournalStatus::Healthy; + + for line in contents.lines() { + let line = line.trim_end_matches('\r'); + if line.is_empty() || line == LIST_OBJECTS_NAMESPACE_JOURNAL_HEADER { + continue; + } + if let Some(value) = line.strip_prefix(LIST_OBJECTS_NAMESPACE_JOURNAL_BUCKET_HEADER) { + if !value.is_empty() { + bucket = Some(value.to_owned()); + } + continue; + } + if let Some(value) = line.strip_prefix(LIST_OBJECTS_NAMESPACE_JOURNAL_HIGH_WATER_MARK_HEADER) { + high_water_mark = value.parse::().ok(); + continue; + } + if let Some(value) = line.strip_prefix(LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_HEADER) { + status = match value { + LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_HEALTHY => NamespaceMutationJournalStatus::Healthy, + LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_DEGRADED => NamespaceMutationJournalStatus::Degraded, + _ => return None, + }; + } + } + + high_water_mark.map(|high_water_mark| NamespaceMutationJournalState { + bucket, + high_water_mark, + status, + }) +} + +fn encode_namespace_mutation_journal_state(bucket: &str, sequence: u64, degraded: bool) -> String { + let mut contents = String::new(); + contents.push_str(LIST_OBJECTS_NAMESPACE_JOURNAL_HEADER); + contents.push('\n'); + contents.push_str(LIST_OBJECTS_NAMESPACE_JOURNAL_BUCKET_HEADER); + contents.push_str(bucket); + contents.push('\n'); + contents.push_str(LIST_OBJECTS_NAMESPACE_JOURNAL_HIGH_WATER_MARK_HEADER); + contents.push_str(&sequence.to_string()); + contents.push('\n'); + contents.push_str(LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_HEADER); + contents.push_str(if degraded { + LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_DEGRADED + } else { + LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_HEALTHY + }); + contents.push('\n'); + contents +} + +fn namespace_mutation_journal_path(root: &Path, bucket: &str) -> PathBuf { + root.join(format!("{bucket}.state")) +} + +fn namespace_mutation_journal_system_path(bucket: &str) -> String { + format!("{LIST_OBJECTS_NAMESPACE_JOURNAL_SYSTEM_PREFIX}/{bucket}/state") +} + +fn list_objects_namespace_journal_root_from_env() -> Option { + std::env::var_os(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_PATH) + .map(PathBuf::from) + .filter(|path| !path.as_os_str().is_empty()) +} + +fn namespace_mutation_journal_chaos_enabled_from_env() -> bool { + std::env::var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED) + .ok() + .is_some_and(|value| value == "1" || value.eq_ignore_ascii_case("on") || value.eq_ignore_ascii_case("true")) +} + +fn namespace_mutation_journal_chaos_bucket_from_env() -> Option { + std::env::var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET) + .ok() + .filter(|bucket| !bucket.is_empty()) +} + +fn namespace_mutation_journal_chaos_sequence_from_env() -> Option { + std::env::var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_SEQUENCE) + .ok() + .and_then(|value| value.parse::().ok()) +} + +fn namespace_mutation_journal_chaos_status_from_env() -> Option { + std::env::var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS) + .ok() + .and_then(|value| NamespaceMutationJournalStatus::from_env_value(&value)) +} + +fn namespace_mutation_journal_chaos_config_from_env() -> Option { + if !namespace_mutation_journal_chaos_enabled_from_env() { + return None; + } + + let Some(bucket) = namespace_mutation_journal_chaos_bucket_from_env() else { + warn!( + bucket_env = ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET, + "list objects namespace journal chaos is enabled without a target bucket; skipping injection" + ); + return None; + }; + let Some(status) = namespace_mutation_journal_chaos_status_from_env() else { + warn!( + status_env = ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS, + "list objects namespace journal chaos is enabled with an invalid status; skipping injection" + ); + return None; + }; + + Some(NamespaceMutationJournalChaosConfig { + bucket, + status, + sequence: namespace_mutation_journal_chaos_sequence_from_env(), + }) +} + +fn namespace_mutation_journal_chaos_applied_key(bucket: &str, status: NamespaceMutationJournalStatus) -> String { + let mut key = String::with_capacity(bucket.len() + 1 + status.env_value().len()); + key.push_str(bucket); + key.push(':'); + key.push_str(status.env_value()); + key +} + +async fn maybe_apply_system_namespace_mutation_journal_chaos(store: &ECStore, bucket: &str, default_sequence: u64) { + let Some(config) = list_objects_namespace_journal_chaos_config().await else { + return; + }; + if config.bucket != bucket { + return; + }; + + if list_objects_namespace_journal_root_from_env().is_some() { + warn!( + bucket = %bucket, + status = config.status.env_value(), + "list objects namespace journal chaos requires the quorum-backed system journal; skipping local journal override" + ); + return; + } + + let applied_key = namespace_mutation_journal_chaos_applied_key(bucket, config.status); + let applied = list_objects_namespace_journal_chaos_applied().await; + { + let applied = applied.read().await; + if applied.contains(&applied_key) { + return; + } + } + + let sequence = config.sequence.unwrap_or(default_sequence); + let result = match config.status { + NamespaceMutationJournalStatus::Healthy => { + write_namespace_mutation_journal_state(NamespaceMutationJournalBackend::System(store), bucket, sequence, true) + .await + .map(|_| ()) + } + NamespaceMutationJournalStatus::Degraded => { + write_namespace_mutation_journal_degraded_state(NamespaceMutationJournalBackend::System(store), bucket, sequence) + .await + } + }; + + match result { + Ok(()) => { + applied.write().await.insert(applied_key); + warn!( + bucket = %bucket, + sequence, + status = config.status.env_value(), + "applied controlled list objects namespace journal chaos state" + ); + } + Err(err) => { + warn!( + bucket = %bucket, + sequence, + status = config.status.env_value(), + error = %err, + "failed to apply controlled list objects namespace journal chaos state" + ); + } + } +} + +fn list_objects_namespace_journal_root_from_provider_path(provider_path: &Path) -> Option { + provider_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .map(|parent| parent.join(LIST_OBJECTS_NAMESPACE_JOURNAL_DEFAULT_DIR)) +} + +fn list_objects_namespace_journal_backend<'a>( + store: Option<&'a ECStore>, + provider_path: Option<&Path>, +) -> Option> { + if let Some(root) = list_objects_namespace_journal_root_from_env() { + return Some(NamespaceMutationJournalBackend::LocalPath(root)); + } + if let Some(store) = store { + return Some(NamespaceMutationJournalBackend::System(store)); + } + provider_path + .and_then(list_objects_namespace_journal_root_from_provider_path) + .map(NamespaceMutationJournalBackend::LocalPath) +} + +fn list_objects_namespace_journal_write_quorum(total_disks: usize) -> usize { + (total_disks / 2) + 1 +} + +fn validate_namespace_mutation_journal_state(bucket: &str, contents: &str) -> Result { + let Some(state) = parse_namespace_mutation_journal_state(contents) else { + return Err(Error::other("list objects namespace mutation journal state is corrupt")); + }; + if state + .bucket + .as_deref() + .is_some_and(|persisted_bucket| persisted_bucket != bucket) + { + return Err(Error::other("list objects namespace mutation journal bucket mismatch")); + } + Ok(state) +} + +async fn system_namespace_mutation_journal_state_candidates( + store: &ECStore, + bucket: &str, +) -> Result>> { + if store.pools.is_empty() { + return Err(Error::other("list objects namespace mutation journal has no storage pools")); + } + + let path = namespace_mutation_journal_system_path(bucket); + let mut states = Vec::with_capacity(store.pools.len()); + + for pool in &store.pools { + let set = pool.get_disks_by_key(bucket); + let disks = set.disk_inventory().await; + let quorum = list_objects_namespace_journal_write_quorum(set.set_drive_count); + let mut successes = 0usize; + let mut pool_state: Option = None; + let mut futures = Vec::with_capacity(disks.len()); + + for disk in disks.into_iter().flatten() { + let path = path.clone(); + futures.push(async move { disk.read_all(RUSTFS_META_BUCKET, &path).await }); + } + + for result in join_all(futures).await { + match result { + Ok(bytes) => { + successes += 1; + let contents = String::from_utf8_lossy(&bytes); + let state = validate_namespace_mutation_journal_state(bucket, contents.as_ref())?; + match pool_state.as_ref() { + Some(current) + if current.high_water_mark > state.high_water_mark + || (current.high_water_mark == state.high_water_mark + && current.status == NamespaceMutationJournalStatus::Degraded) => {} + _ => { + pool_state = Some(state); + } + } + } + Err(DiskError::FileNotFound) => { + successes += 1; + } + Err(err) => { + debug!( + bucket = %bucket, + path = %path, + error = %err, + "failed to read namespace mutation journal state from disk" + ); + } + } + } + + if successes < quorum { + return Err(Error::other("list objects namespace mutation journal read quorum was not met")); + } + states.push(pool_state); + } + + Ok(states) +} + +async fn load_system_namespace_mutation_journal_state( + store: &ECStore, + bucket: &str, +) -> Result> { + let mut merged: Option = None; + let mut saw_state = false; + + for state in system_namespace_mutation_journal_state_candidates(store, bucket).await? { + let Some(state) = state else { + continue; + }; + saw_state = true; + match merged.as_ref() { + Some(current) + if current.high_water_mark > state.high_water_mark + || (current.high_water_mark == state.high_water_mark + && current.status == NamespaceMutationJournalStatus::Degraded) => {} + _ => merged = Some(state), + } + } + + Ok(saw_state.then_some(merged).flatten()) +} + +async fn write_system_namespace_mutation_journal_state( + store: &ECStore, + bucket: &str, + sequence: u64, + degraded: bool, +) -> Result<()> { + if store.pools.is_empty() { + return Err(Error::other("list objects namespace mutation journal has no storage pools")); + } + + let path = namespace_mutation_journal_system_path(bucket); + let contents = Bytes::from(encode_namespace_mutation_journal_state(bucket, sequence, degraded)); + + for pool in &store.pools { + let set = pool.get_disks_by_key(bucket); + let disks = set.disk_inventory().await; + let quorum = list_objects_namespace_journal_write_quorum(set.set_drive_count); + let mut futures = Vec::with_capacity(disks.len()); + + for disk in disks.into_iter().flatten() { + let path = path.clone(); + let contents = contents.clone(); + futures.push(async move { disk.write_all(RUSTFS_META_BUCKET, &path, contents).await }); + } + + let successes = join_all(futures).await.into_iter().filter(|result| result.is_ok()).count(); + if successes < quorum { + return Err(Error::other("list objects namespace mutation journal write quorum was not met")); + } + } + + Ok(()) +} + +async fn load_local_namespace_mutation_journal_state(root: &Path, bucket: &str) -> Result> { + let journal_path = namespace_mutation_journal_path(root, bucket); + let lock = list_objects_namespace_journal_lock().await; + // Journal IO is serialized and does not take provider/index locks, preserving high-water monotonicity. + let _guard = lock.read().await; + let contents = match tokio::fs::read_to_string(&journal_path).await { + Ok(contents) => contents, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(Error::Io(err)), + }; + validate_namespace_mutation_journal_state(bucket, &contents).map(Some) +} + +async fn load_namespace_mutation_journal_state( + backend: NamespaceMutationJournalBackend<'_>, + bucket: &str, +) -> Result> { + match backend { + NamespaceMutationJournalBackend::System(store) => load_system_namespace_mutation_journal_state(store, bucket).await, + NamespaceMutationJournalBackend::LocalPath(root) => load_local_namespace_mutation_journal_state(&root, bucket).await, + } +} + +async fn write_local_namespace_mutation_journal_state( + root: &Path, + bucket: &str, + sequence: u64, + clear_degraded: bool, +) -> Result { + let journal_path = namespace_mutation_journal_path(root, bucket); + let lock = list_objects_namespace_journal_lock().await; + // Journal IO is serialized and does not take provider/index locks, preserving high-water monotonicity. + let guard = lock.write().await; + + let persisted_state = match tokio::fs::read_to_string(&journal_path).await { + Ok(contents) => { + let state = validate_namespace_mutation_journal_state(bucket, &contents)?; + Some(state) + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => None, + Err(err) => return Err(Error::Io(err)), + }; + let persisted_sequence = persisted_state.as_ref().map(|state| state.high_water_mark).unwrap_or(0); + let degraded = persisted_state + .as_ref() + .is_some_and(|state| state.status == NamespaceMutationJournalStatus::Degraded) + && !clear_degraded; + let sequence = sequence.max(persisted_sequence); + + if let Some(parent) = journal_path.parent() + && !parent.as_os_str().is_empty() + { + tokio::fs::create_dir_all(parent).await.map_err(Error::Io)?; + } + + let contents = encode_namespace_mutation_journal_state(bucket, sequence, degraded); + + let tmp_path = journal_path.with_extension("tmp"); + tokio::fs::write(&tmp_path, contents).await.map_err(Error::Io)?; + tokio::fs::rename(&tmp_path, &journal_path).await.map_err(Error::Io)?; + drop(guard); + if clear_degraded { + let degraded = list_objects_namespace_journal_degraded_buckets().await; + degraded.write().await.remove(bucket); + } + Ok(NamespaceMutationJournalSnapshot { + high_water_mark: sequence, + degraded, + }) +} + +async fn write_namespace_mutation_journal_state( + backend: NamespaceMutationJournalBackend<'_>, + bucket: &str, + sequence: u64, + clear_degraded: bool, +) -> Result { + let persisted_state = load_namespace_mutation_journal_state(backend.clone(), bucket).await?; + let persisted_sequence = persisted_state.as_ref().map(|state| state.high_water_mark).unwrap_or(0); + let degraded = persisted_state + .as_ref() + .is_some_and(|state| state.status == NamespaceMutationJournalStatus::Degraded) + && !clear_degraded; + let sequence = sequence.max(persisted_sequence); + + match backend { + NamespaceMutationJournalBackend::System(store) => { + write_system_namespace_mutation_journal_state(store, bucket, sequence, degraded).await?; + } + NamespaceMutationJournalBackend::LocalPath(root) => { + return write_local_namespace_mutation_journal_state(&root, bucket, sequence, clear_degraded).await; + } + } + + if clear_degraded { + let degraded = list_objects_namespace_journal_degraded_buckets().await; + degraded.write().await.remove(bucket); + } + + Ok(NamespaceMutationJournalSnapshot { + high_water_mark: sequence, + degraded, + }) +} + +async fn mark_namespace_mutation_journal_degraded(store: Option<&ECStore>, bucket: &str, sequence: u64) { + let degraded = list_objects_namespace_journal_degraded_buckets().await; + degraded.write().await.insert(bucket.to_owned()); + + if let Some(backend) = list_objects_namespace_journal_backend(store, None) + && let Err(err) = write_namespace_mutation_journal_degraded_state(backend, bucket, sequence).await + { + warn!( + bucket = %bucket, + sequence, + error = %err, + "failed to persist degraded namespace mutation journal state" + ); + } +} + +async fn write_local_namespace_mutation_journal_degraded_state(root: &Path, bucket: &str, sequence: u64) -> Result<()> { + let journal_path = namespace_mutation_journal_path(root, bucket); + let lock = list_objects_namespace_journal_lock().await; + // Journal IO is serialized and does not take provider/index locks, preserving high-water monotonicity. + let _guard = lock.write().await; + + let persisted_sequence = match tokio::fs::read_to_string(&journal_path).await { + Ok(contents) => parse_namespace_mutation_journal_state(&contents) + .filter(|state| { + state + .bucket + .as_deref() + .is_none_or(|persisted_bucket| persisted_bucket == bucket) + }) + .map(|state| state.high_water_mark) + .unwrap_or(0), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => 0, + Err(err) => return Err(Error::Io(err)), + }; + let sequence = sequence.max(persisted_sequence); + + if let Some(parent) = journal_path.parent() + && !parent.as_os_str().is_empty() + { + tokio::fs::create_dir_all(parent).await.map_err(Error::Io)?; + } + + let contents = encode_namespace_mutation_journal_state(bucket, sequence, true); + + let tmp_path = journal_path.with_extension("tmp"); + tokio::fs::write(&tmp_path, contents).await.map_err(Error::Io)?; + tokio::fs::rename(&tmp_path, &journal_path).await.map_err(Error::Io)?; + Ok(()) +} + +async fn write_namespace_mutation_journal_degraded_state( + backend: NamespaceMutationJournalBackend<'_>, + bucket: &str, + sequence: u64, +) -> Result<()> { + let persisted_state = load_namespace_mutation_journal_state(backend.clone(), bucket).await?; + let sequence = sequence.max(persisted_state.map(|state| state.high_water_mark).unwrap_or(0)); + + match backend { + NamespaceMutationJournalBackend::System(store) => { + write_system_namespace_mutation_journal_state(store, bucket, sequence, true).await + } + NamespaceMutationJournalBackend::LocalPath(root) => { + write_local_namespace_mutation_journal_degraded_state(&root, bucket, sequence).await + } + } +} + +async fn current_list_objects_mutation_snapshot(store: Option<&ECStore>, bucket: &str) -> NamespaceMutationJournalSnapshot { + let sequences = list_objects_bucket_mutation_sequence().await; + let in_memory_sequence = { + let sequences = sequences.read().await; + sequences.get(bucket).copied().unwrap_or(0) + }; + + let degraded = list_objects_namespace_journal_degraded_buckets().await; + let in_memory_degraded = { + let degraded = degraded.read().await; + degraded.contains(bucket) + }; + + let Some(backend) = list_objects_namespace_journal_backend(store, None) else { + return NamespaceMutationJournalSnapshot { + high_water_mark: in_memory_sequence, + degraded: in_memory_degraded, + }; + }; + + match load_namespace_mutation_journal_state(backend, bucket).await { + Ok(Some(state)) => { + advance_list_objects_mutation_sequence(bucket, state.high_water_mark).await; + NamespaceMutationJournalSnapshot { + high_water_mark: in_memory_sequence.max(state.high_water_mark), + degraded: in_memory_degraded || state.status == NamespaceMutationJournalStatus::Degraded, + } + } + Ok(None) => NamespaceMutationJournalSnapshot { + high_water_mark: in_memory_sequence, + degraded: in_memory_degraded, + }, + Err(err) => { + warn!( + bucket = %bucket, + error = %err, + "list objects namespace mutation journal could not be loaded; falling back to walker" + ); + let degraded = list_objects_namespace_journal_degraded_buckets().await; + degraded.write().await.insert(bucket.to_owned()); + NamespaceMutationJournalSnapshot { + high_water_mark: in_memory_sequence, + degraded: true, + } + } + } +} + +async fn invalidate_persistent_key_only_index(path: &Path) { + match tokio::fs::remove_file(path).await { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => { + warn!( + index_path = %path.display(), + error = %err, + "failed to remove persistent key-only index after mutation checkpoint persist failure" + ); + } + } + + let cache = persistent_key_only_index_cache().await; + let mut cached = cache.write().await; + if cached.as_ref().is_some_and(|cached| cached.path == path) { + *cached = None; + } +} + +async fn persist_observed_list_objects_mutation(store: Option<&ECStore>, bucket: &str, sequence: u64) { + if is_reserved_or_invalid_bucket(bucket, false) { + return; + } + + let Some(path) = list_objects_index_provider_path_from_env().filter(|_| { + matches!( + list_objects_index_provider_from_env(), + Some(ListObjectsIndexProviderKind::PersistentKeyOnly) + ) + }) else { + return; + }; + + let Some(backend) = list_objects_namespace_journal_backend(store, Some(&path)) else { + return; + }; + + if let Err(err) = write_namespace_mutation_journal_state(backend, bucket, sequence, false).await { + warn!( + bucket = %bucket, + sequence, + error = %err, + "namespace mutation journal commit failed; degrading persistent key-only provider" + ); + mark_namespace_mutation_journal_degraded(store, bucket, sequence).await; + invalidate_persistent_key_only_index(&path).await; + } +} + +fn encode_persistent_list_metadata_string(value: &str) -> String { + BASE64_STANDARD.encode(value.as_bytes()) +} + +fn decode_persistent_list_metadata_string(value: &str) -> Option { + let bytes = BASE64_STANDARD.decode(value).ok()?; + String::from_utf8(bytes).ok() +} + +fn encode_optional_persistent_list_metadata_string(value: Option<&str>) -> String { + value + .map(encode_persistent_list_metadata_string) + .unwrap_or_else(|| "-".to_owned()) +} + +fn decode_optional_persistent_list_metadata_string(value: &str) -> Option> { + if value == "-" { + Some(None) + } else { + decode_persistent_list_metadata_string(value).map(Some) + } +} + +fn encode_persistent_list_metadata_object(object: &PersistentListMetadataObject) -> String { + let mut line = String::new(); + line.push_str(PERSISTENT_KEY_ONLY_INDEX_OBJECT_PREFIX); + line.push_str(&encode_persistent_list_metadata_string(&object.name)); + line.push('\t'); + line.push_str(&object.size.to_string()); + line.push('\t'); + line.push_str( + &object + .mod_time + .map(|mod_time| mod_time.unix_timestamp_nanos().to_string()) + .unwrap_or_else(|| "-".to_owned()), + ); + line.push('\t'); + line.push_str(&encode_optional_persistent_list_metadata_string(object.etag.as_deref())); + line.push('\t'); + line.push_str(&encode_optional_persistent_list_metadata_string(object.storage_class.as_deref())); + line +} + +fn parse_persistent_list_metadata_object(line: &str) -> Option { + let value = line.strip_prefix(PERSISTENT_KEY_ONLY_INDEX_OBJECT_PREFIX)?; + let mut fields = value.split('\t'); + let name = decode_persistent_list_metadata_string(fields.next()?)?; + let size = fields.next()?.parse::().ok()?; + let mod_time = match fields.next()? { + "-" => None, + value => Some(time::OffsetDateTime::from_unix_timestamp_nanos(value.parse::().ok()?).ok()?), + }; + let etag = decode_optional_persistent_list_metadata_string(fields.next()?)?; + let storage_class = decode_optional_persistent_list_metadata_string(fields.next()?)?; + if fields.next().is_some() { + return None; + } + + Some(PersistentListMetadataObject { + name, + size, + mod_time, + etag, + storage_class, + }) +} + +fn parse_persistent_key_only_index(contents: &str) -> PersistentKeyOnlyIndex { + let mut bucket = None; + let mut generation = None; + let mut checkpoint_high_water_mark = None; + let mut keys = Vec::new(); + let mut objects = Vec::new(); + + for line in contents.lines() { + let line = line.trim_end_matches('\r'); + if line.is_empty() { + continue; + } + if let Some(object) = parse_persistent_list_metadata_object(line) { + keys.push(object.name.clone()); + objects.push(object); + continue; + } + if let Some(value) = line.strip_prefix(PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER) { + if !value.is_empty() { + bucket = Some(value.to_owned()); + } + continue; + } + if let Some(value) = line.strip_prefix(PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER) { + if is_valid_list_objects_index_generation(value) { + generation = Some(value.to_owned()); + } + continue; + } + if let Some(value) = line.strip_prefix(PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER) { + checkpoint_high_water_mark = value.parse::().ok(); + continue; + } + if line.starts_with('#') { + continue; + } + keys.push(line.to_owned()); + } + + keys.sort_unstable(); + keys.dedup(); + objects.sort_unstable_by(|left, right| left.name.cmp(&right.name)); + objects.dedup_by(|left, right| left.name == right.name); + let checkpoint_high_water_mark = checkpoint_high_water_mark.unwrap_or_else(|| u64::try_from(keys.len()).unwrap_or(u64::MAX)); + + PersistentKeyOnlyIndex { + bucket, + generation: generation.unwrap_or_else(|| LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY_DEFAULT_GENERATION.to_owned()), + checkpoint_high_water_mark, + keys: Arc::new(keys), + objects: Arc::new(objects), + } +} + +fn persistent_key_only_index_health( + index: &PersistentKeyOnlyIndex, + mutation_snapshot: NamespaceMutationJournalSnapshot, +) -> ListIndexHealthSnapshot { + let mut lifecycle = ListIndexLifecycle::disabled(0); + let mutation_high_water_mark = mutation_snapshot.high_water_mark.max(index.checkpoint_high_water_mark); + lifecycle.begin_rebuild(&index.generation, mutation_high_water_mark); + lifecycle.checkpoint_rebuild(index.checkpoint_high_water_mark); + let _ = lifecycle.publish_rebuild(); + if mutation_snapshot.degraded { + lifecycle.mark_degraded(); + return lifecycle.health_snapshot(); + } + lifecycle.observe_mutation_high_water_mark(mutation_high_water_mark); + lifecycle.health_snapshot() +} + +fn persistent_key_only_index_matches_provider( + index: &PersistentKeyOnlyIndex, + bucket: &str, + provider_state: &ListObjectsIndexProviderState, +) -> bool { + if index.bucket.as_deref().is_some_and(|index_bucket| index_bucket != bucket) { + return false; + } + provider_state + .persistent_generation + .as_deref() + .is_none_or(|generation| index.generation == generation) +} + +fn persistent_key_only_index_has_complete_metadata_snapshot(index: &PersistentKeyOnlyIndex) -> bool { + index.keys.len() == index.objects.len() + && index + .keys + .iter() + .zip(index.objects.iter()) + .all(|(key, object)| key == &object.name) +} + +fn generated_persistent_key_only_generation(configured: Option<&str>) -> String { + if let Some(configured) = configured + && is_valid_list_objects_index_generation(configured) + { + return configured.to_owned(); + } + + let millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_millis()) + .unwrap_or(0); + format!("{LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY_DEFAULT_GENERATION}-{millis}") +} + +async fn write_persistent_key_only_index( + store: Option<&ECStore>, + path: &Path, + bucket: &str, + generation: &str, + checkpoint_high_water_mark: u64, + keys: &[String], +) -> Result { + write_persistent_key_only_index_with_metadata(store, path, bucket, generation, checkpoint_high_water_mark, keys, &[]).await +} + +async fn write_persistent_key_only_index_with_metadata( + store: Option<&ECStore>, + path: &Path, + bucket: &str, + generation: &str, + checkpoint_high_water_mark: u64, + keys: &[String], + objects: &[PersistentListMetadataObject], +) -> Result { + if let Some(parent) = path.parent() + && !parent.as_os_str().is_empty() + { + tokio::fs::create_dir_all(parent).await.map_err(Error::Io)?; + } + + let mut keys = keys.to_vec(); + keys.sort_unstable(); + keys.dedup(); + let mut objects = objects.to_vec(); + objects.sort_unstable_by(|left, right| left.name.cmp(&right.name)); + objects.dedup_by(|left, right| left.name == right.name); + let mut contents = String::new(); + contents.push_str(PERSISTENT_KEY_ONLY_INDEX_HEADER); + contents.push('\n'); + contents.push_str(PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER); + contents.push_str(bucket); + contents.push('\n'); + contents.push_str(PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER); + contents.push_str(generation); + contents.push('\n'); + contents.push_str(PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER); + contents.push_str(&checkpoint_high_water_mark.to_string()); + contents.push('\n'); + for key in &keys { + contents.push_str(key); + contents.push('\n'); + } + for object in &objects { + contents.push_str(&encode_persistent_list_metadata_object(object)); + contents.push('\n'); + } + + let Some(journal_backend) = list_objects_namespace_journal_backend(store, Some(path)) else { + return Err(Error::other("list objects namespace mutation journal path is not configured")); + }; + let tmp_path = path.with_extension("tmp"); + tokio::fs::write(&tmp_path, contents).await.map_err(Error::Io)?; + write_namespace_mutation_journal_state(journal_backend, bucket, checkpoint_high_water_mark, true).await?; + tokio::fs::rename(&tmp_path, path).await.map_err(Error::Io)?; + + Ok(PersistentKeyOnlyIndex { + bucket: Some(bucket.to_owned()), + generation: generation.to_owned(), + checkpoint_high_water_mark, + keys: Arc::new(keys), + objects: Arc::new(objects), + }) +} + +async fn load_persistent_key_only_index(store: Option<&ECStore>, path: &Path) -> Result> { + let metadata = tokio::fs::metadata(path).await.map_err(Error::Io)?; + let len = metadata.len(); + let modified = metadata.modified().ok(); + let cache = persistent_key_only_index_cache().await; + + { + let cached = cache.read().await; + if let Some(cached) = cached.as_ref() + && cached.path == path + && cached.len == len + && cached.modified == modified + { + return Ok(cached.index.clone()); + } + } + + let contents = tokio::fs::read_to_string(path).await.map_err(Error::Io)?; + let index = Arc::new(parse_persistent_key_only_index(&contents)); + if let Some(bucket) = index.bucket.as_deref() { + let restored_sequence = match list_objects_namespace_journal_backend(store, Some(path)) { + Some(journal_backend) => load_namespace_mutation_journal_state(journal_backend, bucket) + .await? + .map(|state| state.high_water_mark) + .unwrap_or(index.checkpoint_high_water_mark) + .max(index.checkpoint_high_water_mark), + None => index.checkpoint_high_water_mark, + }; + advance_list_objects_mutation_sequence(bucket, restored_sequence).await; + } + let mut cached = cache.write().await; + *cached = Some(PersistentKeyOnlyIndexCache { + path: path.to_path_buf(), + len, + modified, + index: index.clone(), + }); + Ok(index) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ListIndexSourceDecision { + UseIndex(ListSourceMode), + FallbackToWalker(ListIndexFallbackReason), +} + +trait ListMetadataIndexGeneration { + fn generation_id(&self) -> &str; +} + +trait ListMetadataIndexHealth { + fn fallback_reason(&self) -> Option; + + fn is_healthy(&self) -> bool { + self.fallback_reason().is_none() + } +} + +trait ListMetadataIndexPage { + fn source_mode(&self) -> ListSourceMode; + fn generation(&self) -> &dyn ListMetadataIndexGeneration; + fn keys(&self) -> &[String]; + + fn metadata_authority(&self) -> ListMetadataAuthority { + self.source_mode().metadata_authority() + } +} + +trait ListMetadataIndexPageIterator { + type Page: ListMetadataIndexPage; + + fn next_page(&mut self) -> Option; +} + +trait ListMetadataIndexKeyLookup { + type Page: ListMetadataIndexPage; + + fn lookup_page(&self, opts: &ListPathOptions) -> ListIndexSourceDecision; +} + +fn select_list_index_provider_source_mode( + opts: &ListPathOptions, + requested: ListSourceMode, + health: &ListIndexHealthSnapshot, +) -> ListIndexSourceDecision { + let mut parsed_opts = opts.clone(); + parsed_opts.parse_marker(); + + if parsed_opts + .cursor_source + .is_some_and(|source| source != requested && source != ListSourceMode::Walker) + { + return ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::UnsupportedRequest); + } + + if let Some(cursor_generation) = parsed_opts.cursor_generation.as_deref() + && health + .active_generation + .as_deref() + .is_some_and(|active_generation| cursor_generation != active_generation) + { + return ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::GenerationMismatch); + } + + select_list_index_source_mode(Some(requested), health.active_generation.as_deref(), health) +} + +fn select_list_index_source_mode( + requested: Option, + generation: Option<&str>, + health: &dyn ListMetadataIndexHealth, +) -> ListIndexSourceDecision { + let Some(mode) = requested else { + return ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::Disabled); + }; + + if mode == ListSourceMode::Walker { + return ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::Disabled); + } + + if generation.is_none_or(str::is_empty) { + return ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::MissingGeneration); + } + + if let Some(reason) = health.fallback_reason() { + return ListIndexSourceDecision::FallbackToWalker(reason); + } + + ListIndexSourceDecision::UseIndex(mode) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ListIndexLifecycleState { + Disabled, + Rebuilding, + Healthy, + Lagging, + Degraded, + Corrupt, +} + +impl ListIndexLifecycleState { + fn fallback_reason(self) -> Option { + match self { + Self::Disabled => Some(ListIndexFallbackReason::Disabled), + Self::Rebuilding => Some(ListIndexFallbackReason::Rebuilding), + Self::Healthy => None, + Self::Lagging => Some(ListIndexFallbackReason::Lagging), + Self::Degraded => Some(ListIndexFallbackReason::Degraded), + Self::Corrupt => Some(ListIndexFallbackReason::Corrupt), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ListIndexHealthSnapshot { + state: ListIndexLifecycleState, + active_generation: Option, + staging_generation: Option, + checkpoint_high_water_mark: u64, + mutation_high_water_mark: u64, + mutation_lag: u64, + fallback_reason: Option, +} + +impl ListMetadataIndexHealth for ListIndexHealthSnapshot { + fn fallback_reason(&self) -> Option { + self.fallback_reason + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ListIndexLifecycle { + state: ListIndexLifecycleState, + active_generation: Option, + staging_generation: Option, + checkpoint_high_water_mark: u64, + mutation_high_water_mark: u64, + max_allowed_lag: u64, +} + +impl ListIndexLifecycle { + fn disabled(max_allowed_lag: u64) -> Self { + Self { + state: ListIndexLifecycleState::Disabled, + active_generation: None, + staging_generation: None, + checkpoint_high_water_mark: 0, + mutation_high_water_mark: 0, + max_allowed_lag, + } + } + + fn begin_rebuild(&mut self, generation: impl Into, current_high_water_mark: u64) { + self.state = ListIndexLifecycleState::Rebuilding; + self.staging_generation = Some(generation.into()); + self.mutation_high_water_mark = current_high_water_mark; + } + + fn checkpoint_rebuild(&mut self, checkpoint_high_water_mark: u64) { + if self.state == ListIndexLifecycleState::Rebuilding && self.staging_generation.is_some() { + self.checkpoint_high_water_mark = checkpoint_high_water_mark; + } + } + + fn publish_rebuild(&mut self) -> bool { + let Some(generation) = self.staging_generation.take() else { + return false; + }; + + self.active_generation = Some(generation); + self.state = ListIndexLifecycleState::Healthy; + self.mutation_high_water_mark = self.checkpoint_high_water_mark; + true + } + + fn recover_after_restart(mut self) -> Self { + self.staging_generation = None; + self.state = if self.active_generation.is_some() { + ListIndexLifecycleState::Healthy + } else { + ListIndexLifecycleState::Disabled + }; + self + } + + fn observe_mutation_high_water_mark(&mut self, mutation_high_water_mark: u64) { + self.mutation_high_water_mark = self.mutation_high_water_mark.max(mutation_high_water_mark); + if matches!(self.state, ListIndexLifecycleState::Healthy | ListIndexLifecycleState::Lagging) + && self.mutation_lag() > self.max_allowed_lag + { + self.state = ListIndexLifecycleState::Lagging; + } + } + + fn mark_degraded(&mut self) { + self.state = ListIndexLifecycleState::Degraded; + } + + fn mark_corrupt(&mut self) { + self.state = ListIndexLifecycleState::Corrupt; + } + + fn mutation_lag(&self) -> u64 { + self.mutation_high_water_mark.saturating_sub(self.checkpoint_high_water_mark) + } + + fn health_snapshot(&self) -> ListIndexHealthSnapshot { + ListIndexHealthSnapshot { + state: self.state, + active_generation: self.active_generation.clone(), + staging_generation: self.staging_generation.clone(), + checkpoint_high_water_mark: self.checkpoint_high_water_mark, + mutation_high_water_mark: self.mutation_high_water_mark, + mutation_lag: self.mutation_lag(), + fallback_reason: self.state.fallback_reason(), + } + } +} #[derive(Debug, Clone, PartialEq, Eq)] struct ListContinuationV2 { @@ -252,6 +1807,8 @@ struct ListContinuationV2 { id: Option, pool_idx: Option, set_idx: Option, + source: Option, + generation: Option, } impl ListContinuationV2 { @@ -275,6 +1832,14 @@ impl ListContinuationV2 { marker_tag.push_str(",s:"); marker_tag.push_str(&set_idx.to_string()); } + if let Some(source) = &self.source { + marker_tag.push_str(",src:"); + marker_tag.push_str(source.cursor_value()); + } + if let Some(generation) = &self.generation { + marker_tag.push_str(",gen:"); + marker_tag.push_str(generation); + } marker_tag.push(']'); marker_tag } @@ -285,6 +1850,8 @@ impl ListContinuationV2 { id: None, pool_idx: None, set_idx: None, + source: None, + generation: None, }; let mut has_list_cache = false; let mut should_create = false; @@ -324,6 +1891,15 @@ impl ListContinuationV2 { should_create = true; } }, + "src" => { + parsed.source = Some(ListSourceMode::from_cursor_value(value)?); + } + "gen" => { + if value.is_empty() { + return None; + } + parsed.generation = Some(value.to_owned()); + } _ => (), } } @@ -361,20 +1937,125 @@ fn list_objects_quorum_from_env() -> String { normalize_list_quorum(&value).to_owned() } +fn list_objects_index_mode_from_env() -> Option { + let value = rustfs_utils::get_env_str(ENV_API_LIST_OBJECTS_INDEX_MODE, ""); + let value = value.trim(); + if value.eq_ignore_ascii_case(LIST_CURSOR_SOURCE_INDEX_KEY_ONLY) || value.eq_ignore_ascii_case("key_only") { + Some(ListSourceMode::IndexKeyOnly) + } else if value.eq_ignore_ascii_case(LIST_CURSOR_SOURCE_INDEX_VERIFIED_PAGE) || value.eq_ignore_ascii_case("verified_page") { + Some(ListSourceMode::IndexVerifiedPage) + } else if value.eq_ignore_ascii_case(LIST_CURSOR_SOURCE_INDEX_METADATA_FAST) || value.eq_ignore_ascii_case("metadata_fast") { + Some(ListSourceMode::IndexMetadataFast) + } else { + None + } +} + +fn list_objects_metadata_fast_guardrails_from_env() -> Option { + let enabled = rustfs_utils::get_env_str(ENV_API_LIST_OBJECTS_METADATA_FAST_ENABLED, ""); + let enabled = enabled.trim(); + if !(enabled == "1" + || enabled.eq_ignore_ascii_case("true") + || enabled.eq_ignore_ascii_case("yes") + || enabled.eq_ignore_ascii_case("on")) + { + return None; + } + + let staleness = rustfs_utils::get_env_str(ENV_API_LIST_OBJECTS_METADATA_FAST_STALENESS_MS, ""); + let staleness_ms = staleness.trim().parse::().ok()?; + (1..=MAX_LIST_OBJECTS_METADATA_FAST_STALENESS_MS) + .contains(&staleness_ms) + .then_some(staleness_ms) +} + +fn list_objects_index_provider_from_env() -> Option { + let value = rustfs_utils::get_env_str(ENV_API_LIST_OBJECTS_INDEX_PROVIDER, ""); + let value = value.trim(); + if value.eq_ignore_ascii_case(LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY) { + Some(ListObjectsIndexProviderKind::WalkerKeyOnly) + } else if value.eq_ignore_ascii_case(LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY) + || value.eq_ignore_ascii_case("persisted_key_only") + { + Some(ListObjectsIndexProviderKind::PersistentKeyOnly) + } else { + None + } +} + +fn list_objects_index_provider_path_from_env() -> Option { + let value = rustfs_utils::get_env_str(ENV_API_LIST_OBJECTS_INDEX_PROVIDER_PATH, ""); + let value = value.trim(); + if value.is_empty() { None } else { Some(PathBuf::from(value)) } +} + +fn is_valid_list_objects_index_generation(value: &str) -> bool { + let value = value.trim(); + !value.is_empty() && !value.contains([',', ':', '[', ']']) +} + +fn list_objects_index_provider_generation_from_env() -> Option { + let value = rustfs_utils::get_env_str(ENV_API_LIST_OBJECTS_INDEX_PROVIDER_GENERATION, ""); + let value = value.trim(); + is_valid_list_objects_index_generation(value).then(|| value.to_owned()) +} + +fn list_objects_index_provider_state_from_env() -> Option { + list_objects_index_provider_from_env().map(|kind| match kind { + ListObjectsIndexProviderKind::WalkerKeyOnly => ListObjectsIndexProviderState::walker_key_only(), + ListObjectsIndexProviderKind::PersistentKeyOnly => ListObjectsIndexProviderState::persistent_key_only( + list_objects_index_provider_path_from_env(), + list_objects_index_provider_generation_from_env(), + ), + }) +} + +fn list_objects_key_only_provider_health(provider: Option) -> ListIndexHealthSnapshot { + provider + .map(ListObjectsIndexProviderState::from_kind) + .map(|state| state.health_snapshot()) + .unwrap_or_else(|| ListIndexLifecycle::disabled(0).health_snapshot()) +} + +fn record_list_objects_index_fallback(mode: ListSourceMode, reason: ListIndexFallbackReason) { + rustfs_io_metrics::record_list_objects_index_fallback(mode.cursor_value(), reason.metric_label()); +} + +fn record_list_objects_index_opt_in_fallback(opts: &ListPathOptions, mode: ListSourceMode) { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + + let health = list_objects_key_only_provider_health(None); + let reason = match select_list_index_provider_source_mode(opts, mode, &health) { + ListIndexSourceDecision::FallbackToWalker(reason) => reason, + ListIndexSourceDecision::UseIndex(_) => ListIndexFallbackReason::UnsupportedRequest, + }; + + record_list_objects_index_fallback(mode, reason); + debug!( + bucket = %opts.bucket, + prefix = %opts.prefix, + source = mode.cursor_value(), + reason = reason.metric_label(), + "list_objects opt-in index path fell back to walker" + ); +} + fn append_list_cache_id_to_marker(marker: String, cache_id: Option<&str>) -> String { let Some(id) = cache_id else { return marker; }; - let mut marker_tag = String::with_capacity(marker.len() + 24 + id.len()); - marker_tag.push_str(&marker); - marker_tag.push_str("[rustfs_cache:"); - marker_tag.push_str(MARKER_TAG_VERSION); - marker_tag.push_str(",id:"); - marker_tag.push_str(id); - marker_tag.push(']'); - - marker_tag + ListContinuationV2 { + version: MARKER_TAG_VERSION, + id: Some(id.to_owned()), + pool_idx: None, + set_idx: None, + source: Some(ListSourceMode::Walker), + generation: Some(LIST_CURSOR_GENERATION_LIVE.to_owned()), + } + .encode_marker(&marker) } fn build_list_next_marker(objects: &[ObjectInfo], prefixes: &[String], cache_id: Option<&str>) -> Option { @@ -469,6 +2150,282 @@ fn list_objects_paginate( (objects, prefixes, is_truncated, next_marker, next_version_idmarker) } +enum VerifiedIndexVisibleEntry { + Object(Box), + Prefix(String), +} + +impl VerifiedIndexVisibleEntry { + fn marker(&self) -> &str { + match self { + Self::Object(object) => &object.name, + Self::Prefix(prefix) => prefix, + } + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +struct VerifiedIndexCandidateStats { + candidate_keys: usize, + skipped_keys: usize, + common_prefixes: usize, + live_verify_attempts: usize, + live_verify_hits: usize, + live_verify_misses: usize, +} + +struct VerifiedIndexCandidateResult { + info: ListObjectsInfo, + stats: VerifiedIndexCandidateStats, +} + +async fn list_objects_from_verified_index_candidates( + prefix: &str, + marker: Option<&str>, + delimiter: &Option, + max_keys: i32, + candidate_keys: &[String], + live_verify: F, +) -> Result +where + F: FnMut(String) -> Fut, + Fut: Future>>, +{ + Ok(list_objects_from_verified_index_candidates_with_optional_stats( + prefix, + marker, + delimiter, + max_keys, + candidate_keys, + false, + live_verify, + ) + .await? + .info) +} + +async fn list_objects_from_verified_index_candidates_with_stats( + prefix: &str, + marker: Option<&str>, + delimiter: &Option, + max_keys: i32, + candidate_keys: &[String], + live_verify: F, +) -> Result +where + F: FnMut(String) -> Fut, + Fut: Future>>, +{ + list_objects_from_verified_index_candidates_with_optional_stats( + prefix, + marker, + delimiter, + max_keys, + candidate_keys, + true, + live_verify, + ) + .await +} + +async fn list_objects_from_verified_index_candidates_with_optional_stats( + prefix: &str, + marker: Option<&str>, + delimiter: &Option, + max_keys: i32, + candidate_keys: &[String], + collect_stats: bool, + mut live_verify: F, +) -> Result +where + F: FnMut(String) -> Fut, + Fut: Future>>, +{ + let max_keys = normalize_max_keys(max_keys); + let mut stats = if collect_stats { + VerifiedIndexCandidateStats { + candidate_keys: candidate_keys.len(), + ..Default::default() + } + } else { + VerifiedIndexCandidateStats::default() + }; + if max_keys <= 0 { + return Ok(VerifiedIndexCandidateResult { + info: ListObjectsInfo::default(), + stats, + }); + } + + let page_limit = usize::try_from(max_keys_plus_one(max_keys, true)).unwrap_or(usize::MAX); + let mut visible_entries = Vec::new(); + let mut prefix_set = HashSet::new(); + + for key in candidate_keys { + if marker.is_some_and(|marker| key.as_str() <= marker) || !key.starts_with(prefix) { + if collect_stats { + stats.skipped_keys += 1; + } + continue; + } + + if let Some(separator) = delimiter + && !separator.is_empty() + { + let suffix = key.trim_start_matches(prefix); + if let Some((common_prefix, _)) = suffix.split_once(separator) { + let common_prefix = format!("{prefix}{common_prefix}{separator}"); + if prefix_set.insert(common_prefix.clone()) { + if collect_stats { + stats.common_prefixes += 1; + } + visible_entries.push(VerifiedIndexVisibleEntry::Prefix(common_prefix)); + } + if visible_entries.len() >= page_limit { + break; + } + continue; + } + } + + if collect_stats { + stats.live_verify_attempts += 1; + } + match live_verify(key.clone()).await? { + Some(object) => { + if collect_stats { + stats.live_verify_hits += 1; + } + visible_entries.push(VerifiedIndexVisibleEntry::Object(Box::new(object))); + if visible_entries.len() >= page_limit { + break; + } + } + None => { + if collect_stats { + stats.live_verify_misses += 1; + } + } + } + } + + let max_keys_len = usize::try_from(max_keys).unwrap_or(usize::MAX); + let is_truncated = visible_entries.len() > max_keys_len; + if is_truncated { + visible_entries.truncate(max_keys_len); + } + + let next_marker = if is_truncated { + visible_entries.last().map(|entry| entry.marker().to_owned()) + } else { + None + }; + + let mut objects = Vec::new(); + let mut prefixes = Vec::new(); + for entry in visible_entries { + match entry { + VerifiedIndexVisibleEntry::Object(object) => objects.push(*object), + VerifiedIndexVisibleEntry::Prefix(prefix) => prefixes.push(prefix), + } + } + + Ok(VerifiedIndexCandidateResult { + info: ListObjectsInfo { + is_truncated, + next_marker, + objects, + prefixes, + }, + stats, + }) +} + +fn list_objects_from_metadata_snapshot_candidates( + bucket: &str, + prefix: &str, + marker: Option<&str>, + delimiter: &Option, + max_keys: i32, + candidate_objects: &[PersistentListMetadataObject], +) -> VerifiedIndexCandidateResult { + let max_keys = normalize_max_keys(max_keys); + let mut stats = VerifiedIndexCandidateStats { + candidate_keys: candidate_objects.len(), + ..Default::default() + }; + if max_keys <= 0 { + return VerifiedIndexCandidateResult { + info: ListObjectsInfo::default(), + stats, + }; + } + + let page_limit = usize::try_from(max_keys_plus_one(max_keys, true)).unwrap_or(usize::MAX); + let mut visible_entries = Vec::new(); + let mut prefix_set = HashSet::new(); + + for object in candidate_objects { + if marker.is_some_and(|marker| object.name.as_str() <= marker) || !object.name.starts_with(prefix) { + stats.skipped_keys += 1; + continue; + } + + if let Some(separator) = delimiter + && !separator.is_empty() + { + let suffix = object.name.trim_start_matches(prefix); + if let Some((common_prefix, _)) = suffix.split_once(separator) { + let common_prefix = format!("{prefix}{common_prefix}{separator}"); + if prefix_set.insert(common_prefix.clone()) { + stats.common_prefixes += 1; + visible_entries.push(VerifiedIndexVisibleEntry::Prefix(common_prefix)); + } + if visible_entries.len() >= page_limit { + break; + } + continue; + } + } + + visible_entries.push(VerifiedIndexVisibleEntry::Object(Box::new(object.to_object_info(bucket)))); + if visible_entries.len() >= page_limit { + break; + } + } + + let max_keys_len = usize::try_from(max_keys).unwrap_or(usize::MAX); + let is_truncated = visible_entries.len() > max_keys_len; + if is_truncated { + visible_entries.truncate(max_keys_len); + } + + let next_marker = if is_truncated { + visible_entries.last().map(|entry| entry.marker().to_owned()) + } else { + None + }; + + let mut objects = Vec::new(); + let mut prefixes = Vec::new(); + for entry in visible_entries { + match entry { + VerifiedIndexVisibleEntry::Object(object) => objects.push(*object), + VerifiedIndexVisibleEntry::Prefix(prefix) => prefixes.push(prefix), + } + } + + VerifiedIndexCandidateResult { + info: ListObjectsInfo { + is_truncated, + next_marker, + objects, + prefixes, + }, + stats, + } +} + fn list_metadata_resolution_params( bucket: String, listing_quorum: usize, @@ -1097,6 +3054,8 @@ impl ListPathOptions { self.id = continuation.id; self.pool_idx = continuation.pool_idx; self.set_idx = continuation.set_idx; + self.cursor_source = continuation.source; + self.cursor_generation = continuation.generation; if should_create { self.create = true; } @@ -1108,12 +3067,564 @@ impl ListPathOptions { id: self.id.clone(), pool_idx: self.pool_idx, set_idx: self.set_idx, + source: Some(ListSourceMode::Walker), + generation: Some(LIST_CURSOR_GENERATION_LIVE.to_owned()), } .encode_marker(marker) } } impl ECStore { + async fn collect_persistent_key_only_index_objects( + self: Arc, + opts: &ListPathOptions, + ) -> Result> { + let mut marker = None; + let mut objects = Vec::new(); + + loop { + let previous_marker = marker.clone(); + let page_opts = ListPathOptions { + bucket: opts.bucket.clone(), + prefix: String::new(), + marker: marker.clone(), + separator: None, + limit: max_keys_plus_one(MAX_OBJECT_LIST, true), + ask_disks: list_objects_quorum_from_env(), + incl_deleted: false, + cursor_source: Some(ListSourceMode::Walker), + cursor_generation: Some(LIST_CURSOR_GENERATION_LIVE.to_owned()), + ..Default::default() + }; + let mut list_result = self + .clone() + .list_path(&page_opts) + .await + .unwrap_or_else(|err| MetaCacheEntriesSortedResult { + err: Some(err.into()), + ..Default::default() + }); + let reached_end = match list_result.err.take() { + Some(rustfs_filemeta::Error::Unexpected) => true, + Some(err) => return Err(Error::from(err)), + None => false, + }; + + if let Some(result) = list_result.entries.as_mut() { + result.forward_past(marker.clone()); + } + + let page_keys = list_result + .entries + .as_ref() + .map(|entries| { + entries + .entries() + .into_iter() + .map(|entry| entry.name.clone()) + .collect::>() + }) + .unwrap_or_default(); + + if page_keys.is_empty() { + break; + } + if let Some(entries) = list_result.entries.as_ref() { + let page_objects = ObjectInfo::from_meta_cache_entries_sorted_infos(entries, &opts.bucket, "", None).await; + objects.extend(page_objects.iter().map(PersistentListMetadataObject::from_object_info)); + } + + marker = page_keys.last().cloned(); + if marker == previous_marker { + return Err(Error::other("persistent key-only index rebuild marker did not advance")); + } + + if reached_end { + break; + } + } + + Ok(objects) + } + + async fn rebuild_persistent_key_only_index( + self: Arc, + opts: &ListPathOptions, + path: &Path, + configured_generation: Option<&str>, + ) -> Result> { + let generation = generated_persistent_key_only_generation(configured_generation); + const MAX_REBUILD_ATTEMPTS: usize = 3; + + for attempt in 1..=MAX_REBUILD_ATTEMPTS { + let start_sequence = current_list_objects_mutation_snapshot(Some(self.as_ref()), &opts.bucket).await; + if start_sequence.degraded { + return Err(Error::other("list objects namespace mutation journal is degraded")); + } + let objects = self.clone().collect_persistent_key_only_index_objects(opts).await?; + let keys = objects.iter().map(|object| object.name.clone()).collect::>(); + let end_sequence = current_list_objects_mutation_snapshot(Some(self.as_ref()), &opts.bucket).await; + if end_sequence.degraded { + return Err(Error::other("list objects namespace mutation journal is degraded")); + } + if start_sequence.high_water_mark == end_sequence.high_water_mark { + write_persistent_key_only_index_with_metadata( + Some(self.as_ref()), + path, + &opts.bucket, + &generation, + end_sequence.high_water_mark, + &keys, + &objects, + ) + .await?; + return load_persistent_key_only_index(Some(self.as_ref()), path).await; + } + + debug!( + bucket = %opts.bucket, + attempt, + start_sequence = start_sequence.high_water_mark, + end_sequence = end_sequence.high_water_mark, + "persistent key-only index rebuild raced with object mutations" + ); + } + + Err(Error::other( + "persistent key-only index rebuild could not reach a stable mutation checkpoint", + )) + } + + async fn prepare_persistent_key_only_index( + self: Arc, + opts: &ListPathOptions, + provider_state: &ListObjectsIndexProviderState, + ) -> Result> { + let Some(path) = provider_state.persistent_path.as_ref() else { + return Err(Error::other("persistent key-only provider path is not configured")); + }; + + match load_persistent_key_only_index(Some(self.as_ref()), path).await { + Ok(index) if persistent_key_only_index_matches_provider(&index, &opts.bucket, provider_state) => { + let current_sequence = current_list_objects_mutation_snapshot(Some(self.as_ref()), &opts.bucket).await; + if !current_sequence.degraded && current_sequence.high_water_mark <= index.checkpoint_high_water_mark { + Ok(index) + } else { + self.rebuild_persistent_key_only_index(opts, path, provider_state.persistent_generation.as_deref()) + .await + } + } + Ok(_) => { + self.rebuild_persistent_key_only_index(opts, path, provider_state.persistent_generation.as_deref()) + .await + } + Err(Error::Io(err)) if err.kind() == std::io::ErrorKind::NotFound => { + self.rebuild_persistent_key_only_index(opts, path, provider_state.persistent_generation.as_deref()) + .await + } + Err(err) => Err(err), + } + } + + async fn list_objects_from_metadata_fast_provider( + self: Arc, + provider_opts: &ListPathOptions, + provider_state: &ListObjectsIndexProviderState, + provider_label: &'static str, + max_keys: i32, + list_metrics_enabled: bool, + ) -> Result> { + let staleness_ms = match list_objects_metadata_fast_guardrails_from_env() { + Some(staleness_ms) => staleness_ms, + None => { + if list_metrics_enabled { + record_list_objects_index_fallback(ListSourceMode::IndexMetadataFast, ListIndexFallbackReason::Disabled); + } + return Ok(None); + } + }; + + if provider_state.kind != ListObjectsIndexProviderKind::PersistentKeyOnly { + if list_metrics_enabled { + record_list_objects_index_fallback( + ListSourceMode::IndexMetadataFast, + ListIndexFallbackReason::UnsupportedRequest, + ); + } + return Ok(None); + } + + let Some(path) = provider_state.persistent_path.as_ref() else { + if list_metrics_enabled { + record_list_objects_index_fallback(ListSourceMode::IndexMetadataFast, ListIndexFallbackReason::Degraded); + } + return Ok(None); + }; + + let index = match self + .clone() + .prepare_persistent_key_only_index(provider_opts, provider_state) + .await + { + Ok(index) => index, + Err(err) => { + if list_metrics_enabled { + record_list_objects_index_fallback(ListSourceMode::IndexMetadataFast, ListIndexFallbackReason::Unhealthy); + } + debug!( + bucket = %provider_opts.bucket, + prefix = %provider_opts.prefix, + source = ListSourceMode::IndexMetadataFast.cursor_value(), + provider = provider_state.kind.metric_label(), + path = %path.display(), + error = %err, + "list_objects metadata-fast provider failed to prepare snapshot" + ); + return Ok(None); + } + }; + + if !persistent_key_only_index_has_complete_metadata_snapshot(&index) { + if list_metrics_enabled { + record_list_objects_index_fallback( + ListSourceMode::IndexMetadataFast, + ListIndexFallbackReason::MetadataFastUnavailable, + ); + } + debug!( + bucket = %provider_opts.bucket, + prefix = %provider_opts.prefix, + source = ListSourceMode::IndexMetadataFast.cursor_value(), + provider = provider_state.kind.metric_label(), + indexed_keys = index.keys.len(), + snapshot_objects = index.objects.len(), + "list_objects metadata-fast provider has no complete metadata snapshot" + ); + return Ok(None); + } + + maybe_apply_system_namespace_mutation_journal_chaos( + self.as_ref(), + &provider_opts.bucket, + index.checkpoint_high_water_mark, + ) + .await; + + let current_sequence = current_list_objects_mutation_snapshot(Some(self.as_ref()), &provider_opts.bucket).await; + let health = persistent_key_only_index_health(&index, current_sequence); + match select_list_index_provider_source_mode(provider_opts, ListSourceMode::IndexMetadataFast, &health) { + ListIndexSourceDecision::FallbackToWalker(reason) => { + if list_metrics_enabled { + record_list_objects_index_fallback(ListSourceMode::IndexMetadataFast, reason); + } + debug!( + bucket = %provider_opts.bucket, + prefix = %provider_opts.prefix, + source = ListSourceMode::IndexMetadataFast.cursor_value(), + provider = provider_state.kind.metric_label(), + staleness_ms, + reason = reason.metric_label(), + "list_objects metadata-fast provider fell back to walker" + ); + return Ok(None); + } + ListIndexSourceDecision::UseIndex(_) => {} + } + + let verified = list_objects_from_metadata_snapshot_candidates( + &provider_opts.bucket, + &provider_opts.prefix, + provider_opts.marker.as_deref(), + &provider_opts.separator, + max_keys, + index.objects.as_slice(), + ); + let stats = verified.stats; + let mut result = verified.info; + + if list_metrics_enabled { + rustfs_io_metrics::record_list_objects_index_served(ListObjectsIndexPageObservation { + source: ListSourceMode::IndexMetadataFast.cursor_value(), + provider: provider_label, + candidate_keys: stats.candidate_keys, + live_verify_attempts: 0, + live_verify_hits: 0, + live_verify_misses: 0, + returned_objects: result.objects.len(), + returned_prefixes: result.prefixes.len(), + is_truncated: result.is_truncated, + }); + } + + if let Some(next_marker) = result.next_marker.take() { + result.next_marker = Some( + ListContinuationV2 { + version: MARKER_TAG_VERSION, + id: None, + pool_idx: None, + set_idx: None, + source: Some(ListSourceMode::IndexMetadataFast), + generation: health.active_generation, + } + .encode_marker(&next_marker), + ); + } + + Ok(Some(result)) + } + + async fn list_objects_from_opt_in_key_only_provider( + self: Arc, + opts: &ListPathOptions, + mode: ListSourceMode, + max_keys: i32, + incl_deleted: bool, + ) -> Result> { + let list_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + let provider_state = list_objects_index_provider_state_from_env(); + let provider = provider_state.as_ref().map(|state| state.kind); + let provider_label = if list_metrics_enabled { + list_objects_index_provider_metric_label(provider) + } else { + "none" + }; + if list_metrics_enabled { + rustfs_io_metrics::record_list_objects_index_attempt( + mode.cursor_value(), + provider_label, + !opts.prefix.is_empty(), + opts.separator.as_ref().is_some_and(|separator| !separator.is_empty()), + opts.marker.is_some(), + ); + } + + if incl_deleted + || (mode != ListSourceMode::IndexMetadataFast + && (!mode.can_satisfy_strong_listing() || !mode.requires_live_metadata_verification())) + { + if list_metrics_enabled { + record_list_objects_index_fallback(mode, ListIndexFallbackReason::UnsupportedRequest); + } + return Ok(None); + } + + if is_reserved_or_invalid_bucket(&opts.bucket, false) { + if list_metrics_enabled { + record_list_objects_index_fallback(mode, ListIndexFallbackReason::UnsupportedRequest); + } + return Ok(None); + } + + let Some(provider_state) = provider_state else { + if list_metrics_enabled { + record_list_objects_index_fallback(mode, ListIndexFallbackReason::Disabled); + } + return Ok(None); + }; + + let mut provider_opts = opts.clone(); + provider_opts.parse_marker(); + + if mode == ListSourceMode::IndexMetadataFast { + return self + .list_objects_from_metadata_fast_provider( + &provider_opts, + &provider_state, + provider_label, + max_keys, + list_metrics_enabled, + ) + .await; + } + + let (candidate_keys, health) = match provider_state.kind { + ListObjectsIndexProviderKind::WalkerKeyOnly => { + let health = provider_state.health_snapshot(); + match select_list_index_provider_source_mode(&provider_opts, mode, &health) { + ListIndexSourceDecision::FallbackToWalker(reason) => { + if list_metrics_enabled { + record_list_objects_index_fallback(mode, reason); + } + debug!( + bucket = %opts.bucket, + prefix = %opts.prefix, + source = mode.cursor_value(), + reason = reason.metric_label(), + "list_objects opt-in index path fell back to walker" + ); + return Ok(None); + } + ListIndexSourceDecision::UseIndex(_) => {} + } + provider_opts.cursor_source = Some(mode); + provider_opts.cursor_generation = health.active_generation.clone(); + + let mut list_result = + self.clone() + .list_path(&provider_opts) + .await + .unwrap_or_else(|err| MetaCacheEntriesSortedResult { + err: Some(err.into()), + ..Default::default() + }); + + if let Some(err) = list_result.err.take() + && err != rustfs_filemeta::Error::Unexpected + { + if list_metrics_enabled { + record_list_objects_index_fallback(mode, ListIndexFallbackReason::Unhealthy); + } + return Ok(None); + } + + if let Some(result) = list_result.entries.as_mut() { + result.forward_past(provider_opts.marker.clone()); + } + + let candidate_keys = Arc::new( + list_result + .entries + .as_ref() + .map(|entries| { + entries + .entries() + .into_iter() + .map(|entry| entry.name.clone()) + .collect::>() + }) + .unwrap_or_default(), + ); + (candidate_keys, health) + } + ListObjectsIndexProviderKind::PersistentKeyOnly => { + let Some(path) = provider_state.persistent_path.as_ref() else { + if list_metrics_enabled { + record_list_objects_index_fallback(mode, ListIndexFallbackReason::Degraded); + } + return Ok(None); + }; + match self + .clone() + .prepare_persistent_key_only_index(&provider_opts, &provider_state) + .await + { + Ok(index) => { + let current_sequence = + current_list_objects_mutation_snapshot(Some(self.as_ref()), &provider_opts.bucket).await; + let health = persistent_key_only_index_health(&index, current_sequence); + match select_list_index_provider_source_mode(&provider_opts, mode, &health) { + ListIndexSourceDecision::FallbackToWalker(reason) => { + if list_metrics_enabled { + record_list_objects_index_fallback(mode, reason); + } + debug!( + bucket = %opts.bucket, + prefix = %opts.prefix, + source = mode.cursor_value(), + provider = provider_state.kind.metric_label(), + reason = reason.metric_label(), + "list_objects persistent key-only provider fell back to walker" + ); + return Ok(None); + } + ListIndexSourceDecision::UseIndex(_) => {} + } + provider_opts.cursor_source = Some(mode); + provider_opts.cursor_generation = health.active_generation.clone(); + (index.keys.clone(), health) + } + Err(err) => { + if list_metrics_enabled { + record_list_objects_index_fallback(mode, ListIndexFallbackReason::Unhealthy); + } + debug!( + bucket = %opts.bucket, + prefix = %opts.prefix, + source = mode.cursor_value(), + provider = provider_state.kind.metric_label(), + path = %path.display(), + error = %err, + "list_objects persistent key-only provider failed to load candidates" + ); + return Ok(None); + } + } + } + }; + + let bucket = provider_opts.bucket.clone(); + let verified = list_objects_from_verified_index_candidates_with_optional_stats( + &provider_opts.prefix, + provider_opts.marker.as_deref(), + &provider_opts.separator, + max_keys, + candidate_keys.as_slice(), + list_metrics_enabled, + |key| { + let store = self.clone(); + let bucket = bucket.clone(); + async move { + match store + .get_object_info( + &bucket, + &key, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(object) => Ok(Some(object)), + Err(err) if err.is_not_found() => Ok(None), + Err(err) => { + if list_metrics_enabled { + rustfs_io_metrics::record_list_objects_index_live_verify_failure( + mode.cursor_value(), + "read_error", + ); + } + Err(err) + } + } + } + }, + ) + .await?; + let stats = verified.stats; + let mut result = verified.info; + + if list_metrics_enabled { + rustfs_io_metrics::record_list_objects_index_served(ListObjectsIndexPageObservation { + source: mode.cursor_value(), + provider: provider_label, + candidate_keys: stats.candidate_keys, + live_verify_attempts: stats.live_verify_attempts, + live_verify_hits: stats.live_verify_hits, + live_verify_misses: stats.live_verify_misses, + returned_objects: result.objects.len(), + returned_prefixes: result.prefixes.len(), + is_truncated: result.is_truncated, + }); + } + + if let Some(next_marker) = result.next_marker.take() { + result.next_marker = Some( + ListContinuationV2 { + version: MARKER_TAG_VERSION, + id: None, + pool_idx: None, + set_idx: None, + source: Some(mode), + generation: health.active_generation, + } + .encode_marker(&next_marker), + ); + } + + Ok(Some(result)) + } + #[allow(clippy::too_many_arguments)] // @continuation_token marker // @start_after as marker when continuation_token empty @@ -1172,6 +3683,14 @@ impl ECStore { ask_disks: list_objects_quorum_from_env(), ..Default::default() }; + if let Some(mode) = list_objects_index_mode_from_env() + && let Some(result) = self + .clone() + .list_objects_from_opt_in_key_only_provider(&opts, mode, max_keys, incl_deleted) + .await? + { + return Ok(result); + } // Optimization: use get for single object lookup with exact prefix if !opts.prefix.is_empty() && max_keys == 1 && opts.marker.is_none() { @@ -1988,7 +4507,8 @@ async fn gather_results( ) -> Result { let mut recv = recv; let mut entries = Vec::new(); - let gather_started = std::time::Instant::now(); + let list_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + let gather_started = list_metrics_enabled.then(std::time::Instant::now); let mut scanned_entries = 0usize; let mut candidate_entries = 0usize; @@ -2000,11 +4520,7 @@ async fn gather_results( entry.name = entry.name.replace("\\", "/"); } - if !opts.include_directories - && (entry.is_dir() || (!opts.versioned && entry.is_object() && entry.is_latest_delete_marker())) - { - continue; - } + // TODO: rx.recv() if let Some(marker) = &opts.marker && ((!opts.include_marker && &entry.name <= marker) || (opts.include_marker && &entry.name < marker)) @@ -2023,7 +4539,14 @@ async fn gather_results( continue; } - if !opts.incl_deleted && entry.is_object() && entry.is_latest_delete_marker() { + let is_object = entry.is_object(); + let is_latest_delete_marker = is_object && entry.is_latest_delete_marker(); + + if !opts.include_directories && (entry.is_dir() || (!opts.versioned && is_latest_delete_marker)) { + continue; + } + + if !opts.incl_deleted && is_latest_delete_marker { continue; } @@ -2035,6 +4558,21 @@ async fn gather_results( if opts.limit > 0 && entries.len() >= opts.limit as usize { rx.cancel(); let filtered = scanned_entries.saturating_sub(candidate_entries); + if let Some(started) = gather_started { + let duration_ms = started.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_list_objects_gather(ListObjectsGatherObservation { + source: LIST_OBJECTS_SOURCE_WALKER, + outcome: LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED, + limit: opts.limit, + scanned_entries, + returned_entries: candidate_entries, + duration_ms, + has_prefix: !opts.prefix.is_empty(), + has_delimiter: opts.separator.as_ref().is_some_and(|separator| !separator.is_empty()), + has_marker: opts.marker.is_some(), + }); + rustfs_io_metrics::record_stage_duration("store_list_objects_gather", duration_ms); + } debug!( bucket = %opts.bucket, prefix = %opts.prefix, @@ -2045,10 +4583,6 @@ async fn gather_results( marker = %opts.marker.as_deref().unwrap_or(""), "list_objects gather_results reached page limit and cancelled upstream listing" ); - rustfs_io_metrics::record_stage_duration( - "store_list_objects_gather", - gather_started.elapsed().as_secs_f64() * 1000.0, - ); results_tx .send(MetaCacheEntriesSortedResult { @@ -2066,6 +4600,21 @@ async fn gather_results( // finish not full, return eof let filtered = scanned_entries.saturating_sub(candidate_entries); + if let Some(started) = gather_started { + let duration_ms = started.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_list_objects_gather(ListObjectsGatherObservation { + source: LIST_OBJECTS_SOURCE_WALKER, + outcome: LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED, + limit: opts.limit, + scanned_entries, + returned_entries: candidate_entries, + duration_ms, + has_prefix: !opts.prefix.is_empty(), + has_delimiter: opts.separator.as_ref().is_some_and(|separator| !separator.is_empty()), + has_marker: opts.marker.is_some(), + }); + rustfs_io_metrics::record_stage_duration("store_list_objects_gather", duration_ms); + } debug!( bucket = %opts.bucket, prefix = %opts.prefix, @@ -2075,7 +4624,6 @@ async fn gather_results( marker = %opts.marker.as_deref().unwrap_or(""), "list_objects gather_results drained all candidates without hitting limit" ); - rustfs_io_metrics::record_stage_duration("store_list_objects_gather", gather_started.elapsed().as_secs_f64() * 1000.0); results_tx .send(MetaCacheEntriesSortedResult { @@ -2135,6 +4683,8 @@ async fn merge_entry_channels( return Ok(()); } + rustfs_io_metrics::record_list_objects_merge(LIST_OBJECTS_SOURCE_WALKER, in_channels.len(), read_quorum); + let mut in_channels = in_channels; if in_channels.len() == 1 { loop { @@ -3863,20 +6413,50 @@ fn calc_common_counter(infos: &[DiskInfo], read_quorum: usize) -> u64 { #[cfg(test)] mod test { use super::{ + ENV_API_LIST_OBJECTS_INDEX_MODE, ENV_API_LIST_OBJECTS_INDEX_PROVIDER, ENV_API_LIST_OBJECTS_INDEX_PROVIDER_GENERATION, + ENV_API_LIST_OBJECTS_INDEX_PROVIDER_PATH, ENV_API_LIST_OBJECTS_METADATA_FAST_ENABLED, + ENV_API_LIST_OBJECTS_METADATA_FAST_STALENESS_MS, ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET, + ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED, ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_SEQUENCE, + ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS, ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_PATH, ENV_API_LIST_OBJECTS_QUORUM, ENV_API_LIST_QUORUM, FallbackListingEntries, FallbackListingEntry, GatherResultsState, - ListPathOptions, ListPathRawOptions, ListingEntryResolution, ListingSupplement, ListingSupplementOptions, - MAX_OBJECT_LIST, VersionMarker, enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum, - fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects, latest_listing_object_quorum, - latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_metadata_resolution_params, - list_objects_quorum_from_env, list_quorum_from_env, max_keys_plus_one, merge_entry_channels, normalize_list_quorum, - parse_version_marker, resolve_agreed_listing_entry, resolve_listing_entries, send_or_cancel, version_marker_for_entries, - walk_result_from_set_errors, + LIST_CURSOR_GENERATION_LIVE, LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY, + LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY, ListIndexFallbackReason, ListIndexLifecycle, ListIndexLifecycleState, + ListIndexSourceDecision, ListMetadataAuthority, ListMetadataIndexGeneration, ListMetadataIndexHealth, + ListMetadataIndexKeyLookup, ListMetadataIndexPage, ListMetadataIndexPageIterator, ListObjectsIndexProviderKind, + ListObjectsIndexProviderState, ListPathOptions, ListPathRawOptions, ListSourceMode, ListingEntryResolution, + ListingSupplement, ListingSupplementOptions, MAX_OBJECT_LIST, NamespaceMutationJournalBackend, + NamespaceMutationJournalSnapshot, NamespaceMutationJournalStatus, PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER, + PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER, PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER, + PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex, PersistentListMetadataObject, VerifiedIndexCandidateStats, + VersionMarker, current_list_objects_mutation_sequence, encode_persistent_list_metadata_object, + enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results, + latest_listing_allow_agreed_objects, latest_listing_object_quorum, latest_listing_raw_min_disks, + latest_listing_required_object_quorum, list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates, + list_objects_from_verified_index_candidates, list_objects_from_verified_index_candidates_with_optional_stats, + list_objects_from_verified_index_candidates_with_stats, list_objects_index_mode_from_env, + list_objects_index_provider_from_env, list_objects_index_provider_state_from_env, list_objects_key_only_provider_health, + list_objects_metadata_fast_guardrails_from_env, list_objects_quorum_from_env, list_quorum_from_env, + load_namespace_mutation_journal_state, load_persistent_key_only_index, max_keys_plus_one, merge_entry_channels, + namespace_mutation_journal_chaos_bucket_from_env, namespace_mutation_journal_chaos_config_from_env, + namespace_mutation_journal_chaos_enabled_from_env, namespace_mutation_journal_chaos_sequence_from_env, + namespace_mutation_journal_chaos_status_from_env, normalize_list_quorum, observe_list_objects_mutations_with_store, + parse_namespace_mutation_journal_state, parse_persistent_key_only_index, parse_persistent_list_metadata_object, + parse_version_marker, persist_observed_list_objects_mutation, persistent_key_only_index_has_complete_metadata_snapshot, + persistent_key_only_index_health, persistent_key_only_index_matches_provider, record_list_objects_index_opt_in_fallback, + reset_list_objects_mutation_sequences_for_test, resolve_agreed_listing_entry, resolve_listing_entries, + select_list_index_provider_source_mode, select_list_index_source_mode, send_or_cancel, version_marker_for_entries, + walk_result_from_set_errors, write_namespace_mutation_journal_state, write_persistent_key_only_index, }; use crate::cache_value::metacache_set::{FallbackClaimTracker, TestReaderBehavior, list_path_raw}; use crate::disk::{DiskAPI, DiskOption, endpoint::Endpoint, error::DiskError, new_disk}; use crate::error::StorageError; - use rustfs_filemeta::{FileInfo, FileMeta, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntry}; + use crate::object_api::ObjectInfo; + use rustfs_filemeta::{ + FileInfo, FileMeta, FileMetaVersion, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntry, MetaDeleteMarker, + VersionType, + }; use std::collections::{HashMap, HashSet}; + use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::Duration; use tokio::sync::mpsc; @@ -3884,6 +6464,70 @@ mod test { use tokio_util::sync::CancellationToken; use uuid::Uuid; + struct TestIndexGeneration { + id: String, + } + + impl ListMetadataIndexGeneration for TestIndexGeneration { + fn generation_id(&self) -> &str { + &self.id + } + } + + struct TestIndexHealth { + reason: Option, + } + + impl ListMetadataIndexHealth for TestIndexHealth { + fn fallback_reason(&self) -> Option { + self.reason + } + } + + struct TestIndexPage { + mode: ListSourceMode, + generation: TestIndexGeneration, + keys: Vec, + } + + impl ListMetadataIndexPage for TestIndexPage { + fn source_mode(&self) -> ListSourceMode { + self.mode + } + + fn generation(&self) -> &dyn ListMetadataIndexGeneration { + &self.generation + } + + fn keys(&self) -> &[String] { + &self.keys + } + } + + struct TestIndexIterator { + pages: std::vec::IntoIter, + } + + impl ListMetadataIndexPageIterator for TestIndexIterator { + type Page = TestIndexPage; + + fn next_page(&mut self) -> Option { + self.pages.next() + } + } + + struct TestIndexLookup { + decision: ListIndexSourceDecision, + } + + impl ListMetadataIndexKeyLookup for TestIndexLookup { + type Page = TestIndexPage; + + fn lookup_page(&self, _opts: &ListPathOptions) -> ListIndexSourceDecision { + self.decision + } + } + fn test_meta_entry(name: &str) -> MetaCacheEntry { MetaCacheEntry { name: name.to_owned(), @@ -3891,6 +6535,17 @@ mod test { } } + fn test_live_object_info(name: &str, etag: &str) -> ObjectInfo { + let mut fi = FileInfo { + name: name.to_owned(), + size: 1, + mod_time: Some(time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")), + ..Default::default() + }; + fi.metadata.insert("etag".to_string(), etag.to_string()); + ObjectInfo::from_file_info(&fi, "bucket", name, false) + } + fn test_object_meta_entry(name: &str) -> MetaCacheEntry { let mut meta = FileMeta::new(); meta.add_version(FileInfo { @@ -3911,27 +6566,6 @@ mod test { } } - fn test_delete_marker_meta_entry(name: &str) -> MetaCacheEntry { - let mut meta = FileMeta::new(); - meta.add_version(FileInfo { - volume: "bucket".to_owned(), - name: name.to_owned(), - deleted: true, - version_id: Some(Uuid::from_u128(1)), - mod_time: Some(time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")), - ..Default::default() - }) - .expect("test metadata should accept delete marker version"); - let metadata = meta.marshal_msg().expect("test metadata should marshal"); - - MetaCacheEntry { - name: name.to_owned(), - metadata, - cached: Some(meta), - reusable: false, - } - } - #[test] fn fallback_entries_for_object_filters_claimed_physical_disks() { let mut entries = FallbackListingEntries::new(); @@ -4079,6 +6713,35 @@ mod test { } } + fn test_delete_marker_meta_entry(name: &str, mod_time: time::OffsetDateTime) -> MetaCacheEntry { + let mut meta = FileMeta::new(); + let delete_marker = FileMetaVersion { + version_type: VersionType::Delete, + object: None, + delete_marker: Some(MetaDeleteMarker { + version_id: Some(Uuid::from_u128(0x44444444555566667777888888888888)), + mod_time: Some(mod_time), + meta_sys: HashMap::new(), + }), + legacy_object: None, + write_version: 1, + uses_legacy_checksum: false, + }; + meta.versions.push( + delete_marker + .try_into() + .expect("test delete marker should convert to shallow version"), + ); + let metadata = meta.marshal_msg().expect("test delete marker metadata should marshal"); + + MetaCacheEntry { + name: name.to_owned(), + metadata, + cached: Some(meta), + reusable: false, + } + } + fn test_dir_meta_entry(name: &str) -> MetaCacheEntry { MetaCacheEntry { name: name.to_owned(), @@ -4252,14 +6915,67 @@ mod test { assert!(!cancel.is_cancelled()); } + #[tokio::test] + async fn list_path_gather_results_filters_delete_marker_after_name_filters() { + let (entry_tx, entry_rx) = mpsc::channel(4); + let (result_tx, mut result_rx) = mpsc::channel(1); + let cancel = CancellationToken::new(); + let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_500).expect("valid timestamp"); + + entry_tx + .send(test_delete_marker_meta_entry("obj-a", mod_time)) + .await + .expect("delete marker should be queued"); + entry_tx + .send(test_object_meta_entry("obj-b")) + .await + .expect("object should be queued"); + drop(entry_tx); + + let handle = tokio::spawn(gather_results( + cancel.clone(), + ListPathOptions { + bucket: "bucket".to_owned(), + marker: Some("obj-a".to_owned()), + limit: 2, + incl_deleted: false, + ..Default::default() + }, + entry_rx, + result_tx, + )); + + let result = timeout(Duration::from_secs(1), result_rx.recv()) + .await + .expect("result should be sent promptly") + .expect("result should be present"); + let entries = result.entries.unwrap(); + let names = entries + .entries() + .into_iter() + .map(|entry| entry.name.as_str()) + .collect::>(); + + assert_eq!(names, ["obj-b"]); + + let state = timeout(Duration::from_secs(1), handle) + .await + .expect("gather_results should finish") + .expect("gather_results task should not panic") + .expect("gather_results should succeed"); + assert_eq!(state, GatherResultsState::InputClosed); + assert!(!cancel.is_cancelled()); + } + #[tokio::test] async fn list_path_gather_results_skips_directory_delete_marker_by_default() { let (entry_tx, entry_rx) = mpsc::channel(4); let (result_tx, mut result_rx) = mpsc::channel(1); let cancel = CancellationToken::new(); + let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_500).expect("valid timestamp"); entry_tx - .send(test_delete_marker_meta_entry("folder/")) + .send(test_delete_marker_meta_entry("folder/", mod_time)) .await .expect("directory delete marker should be queued"); entry_tx @@ -4327,6 +7043,20 @@ mod test { assert_eq!(first_page_names, vec!["obj-0003".to_string(), "obj-0004".to_string()]); } + #[test] + fn list_path_forward_past_keeps_source_switch_page_boundary() { + let mut walker_page_after_index = sorted_entries(&["obj-0001", "obj-0002", "obj-0003", "obj-0004"]); + walker_page_after_index.forward_past(Some("obj-0002".to_string())); + + let page_names = walker_page_after_index + .entries() + .into_iter() + .map(|entry| entry.name.clone()) + .collect::>(); + + assert_eq!(page_names, vec!["obj-0003".to_string(), "obj-0004".to_string()]); + } + #[test] fn list_path_parse_marker_replay_still_stable() { let marker = "photos/2026/image.jpg[rustfs_cache:v1,id:list-cache-id,p:3,s:7]".to_string(); @@ -4421,6 +7151,948 @@ mod test { }); } + #[test] + #[serial_test::serial] + fn list_objects_index_mode_from_env_defaults_to_walker() { + temp_env::with_var_unset(ENV_API_LIST_OBJECTS_INDEX_MODE, || { + assert_eq!(list_objects_index_mode_from_env(), None); + }); + } + + #[test] + #[serial_test::serial] + fn list_objects_index_mode_from_env_accepts_key_only_aliases() { + temp_env::with_var(ENV_API_LIST_OBJECTS_INDEX_MODE, Some("key_only"), || { + assert_eq!(list_objects_index_mode_from_env(), Some(ListSourceMode::IndexKeyOnly)); + }); + temp_env::with_var(ENV_API_LIST_OBJECTS_INDEX_MODE, Some("index_key_only"), || { + assert_eq!(list_objects_index_mode_from_env(), Some(ListSourceMode::IndexKeyOnly)); + }); + } + + #[test] + #[serial_test::serial] + fn list_objects_index_mode_from_env_accepts_verified_page_aliases() { + temp_env::with_var(ENV_API_LIST_OBJECTS_INDEX_MODE, Some("verified_page"), || { + assert_eq!(list_objects_index_mode_from_env(), Some(ListSourceMode::IndexVerifiedPage)); + }); + temp_env::with_var(ENV_API_LIST_OBJECTS_INDEX_MODE, Some("index_verified_page"), || { + assert_eq!(list_objects_index_mode_from_env(), Some(ListSourceMode::IndexVerifiedPage)); + }); + } + + #[test] + #[serial_test::serial] + fn list_objects_index_mode_from_env_accepts_metadata_fast_without_guardrails() { + temp_env::with_var(ENV_API_LIST_OBJECTS_INDEX_MODE, Some("index_metadata_fast"), || { + temp_env::with_var_unset(ENV_API_LIST_OBJECTS_METADATA_FAST_ENABLED, || { + temp_env::with_var_unset(ENV_API_LIST_OBJECTS_METADATA_FAST_STALENESS_MS, || { + assert_eq!(list_objects_metadata_fast_guardrails_from_env(), None); + assert_eq!(list_objects_index_mode_from_env(), Some(ListSourceMode::IndexMetadataFast)); + }); + }); + }); + } + + #[test] + #[serial_test::serial] + fn list_objects_index_mode_from_env_accepts_metadata_fast_without_sla() { + temp_env::with_var(ENV_API_LIST_OBJECTS_INDEX_MODE, Some("metadata_fast"), || { + temp_env::with_var(ENV_API_LIST_OBJECTS_METADATA_FAST_ENABLED, Some("true"), || { + temp_env::with_var_unset(ENV_API_LIST_OBJECTS_METADATA_FAST_STALENESS_MS, || { + assert_eq!(list_objects_metadata_fast_guardrails_from_env(), None); + assert_eq!(list_objects_index_mode_from_env(), Some(ListSourceMode::IndexMetadataFast)); + }); + }); + }); + } + + #[test] + #[serial_test::serial] + fn list_objects_index_mode_from_env_accepts_metadata_fast_with_sla_guardrails() { + temp_env::with_var(ENV_API_LIST_OBJECTS_INDEX_MODE, Some("index_metadata_fast"), || { + temp_env::with_var(ENV_API_LIST_OBJECTS_METADATA_FAST_ENABLED, Some("true"), || { + temp_env::with_var(ENV_API_LIST_OBJECTS_METADATA_FAST_STALENESS_MS, Some("5000"), || { + assert_eq!(list_objects_metadata_fast_guardrails_from_env(), Some(5000)); + assert_eq!(list_objects_index_mode_from_env(), Some(ListSourceMode::IndexMetadataFast)); + }); + }); + }); + } + + #[test] + #[serial_test::serial] + fn list_objects_index_mode_from_env_accepts_metadata_fast_over_sla_budget() { + temp_env::with_var(ENV_API_LIST_OBJECTS_INDEX_MODE, Some("index_metadata_fast"), || { + temp_env::with_var(ENV_API_LIST_OBJECTS_METADATA_FAST_ENABLED, Some("true"), || { + temp_env::with_var(ENV_API_LIST_OBJECTS_METADATA_FAST_STALENESS_MS, Some("60001"), || { + assert_eq!(list_objects_metadata_fast_guardrails_from_env(), None); + assert_eq!(list_objects_index_mode_from_env(), Some(ListSourceMode::IndexMetadataFast)); + }); + }); + }); + } + + #[test] + fn persistent_list_metadata_object_round_trips_encoded_snapshot() { + let object = PersistentListMetadataObject { + name: "photos/2026/img\t01.jpg".to_string(), + size: 42, + mod_time: Some(time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")), + etag: Some("etag-42".to_string()), + storage_class: Some("STANDARD".to_string()), + }; + + let encoded = encode_persistent_list_metadata_object(&object); + let parsed = parse_persistent_list_metadata_object(&encoded).expect("metadata object should parse"); + + assert_eq!(parsed, object); + } + + #[test] + fn persistent_key_only_index_parses_legacy_keys_and_metadata_snapshots() { + let object = PersistentListMetadataObject { + name: "object-b".to_string(), + size: 7, + mod_time: None, + etag: Some("etag-b".to_string()), + storage_class: None, + }; + let contents = format!( + "{PERSISTENT_KEY_ONLY_INDEX_HEADER}\n\ + {PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER}bucket\n\ + {PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER}generation-42\n\ + {PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER}9\n\ + object-a\n\ + {}\n", + encode_persistent_list_metadata_object(&object) + ); + + let index = parse_persistent_key_only_index(&contents); + + assert_eq!(index.bucket.as_deref(), Some("bucket")); + assert_eq!(index.generation, "generation-42"); + assert_eq!(index.checkpoint_high_water_mark, 9); + assert_eq!(index.keys.as_slice(), &["object-a".to_string(), "object-b".to_string()]); + assert_eq!(index.objects.as_slice(), &[object]); + assert!(!persistent_key_only_index_has_complete_metadata_snapshot(&index)); + } + + #[test] + fn metadata_snapshot_candidates_page_without_live_verification() { + let objects = vec![ + PersistentListMetadataObject { + name: "photos/2026/a.jpg".to_string(), + size: 1, + mod_time: None, + etag: Some("etag-a".to_string()), + storage_class: Some("STANDARD".to_string()), + }, + PersistentListMetadataObject { + name: "photos/2026/nested/b.jpg".to_string(), + size: 2, + mod_time: None, + etag: Some("etag-b".to_string()), + storage_class: Some("STANDARD".to_string()), + }, + PersistentListMetadataObject { + name: "photos/2026/z.jpg".to_string(), + size: 3, + mod_time: None, + etag: Some("etag-z".to_string()), + storage_class: Some("STANDARD".to_string()), + }, + ]; + + let result = list_objects_from_metadata_snapshot_candidates( + "bucket", + "photos/2026/", + Some("photos/2026/a.jpg"), + &Some("/".to_string()), + 2, + &objects, + ); + + assert_eq!(result.stats.candidate_keys, 3); + assert_eq!(result.stats.live_verify_attempts, 0); + assert_eq!( + result + .info + .objects + .iter() + .map(|object| object.name.as_str()) + .collect::>(), + ["photos/2026/z.jpg"] + ); + assert_eq!(result.info.objects[0].etag.as_deref(), Some("etag-z")); + assert_eq!(result.info.prefixes, vec!["photos/2026/nested/".to_string()]); + assert!(!result.info.is_truncated); + } + + #[test] + fn metadata_fast_requires_complete_metadata_snapshot() { + let index = PersistentKeyOnlyIndex { + bucket: Some("bucket".to_string()), + generation: "generation-42".to_string(), + checkpoint_high_water_mark: 42, + keys: Arc::new(vec!["a".to_string(), "b".to_string()]), + objects: Arc::new(vec![PersistentListMetadataObject { + name: "a".to_string(), + size: 1, + mod_time: None, + etag: None, + storage_class: None, + }]), + }; + + assert!(!persistent_key_only_index_has_complete_metadata_snapshot(&index)); + + let complete = PersistentKeyOnlyIndex { + bucket: Some("bucket".to_string()), + generation: "generation-42".to_string(), + checkpoint_high_water_mark: 42, + keys: Arc::new(vec!["a".to_string()]), + objects: Arc::new(vec![PersistentListMetadataObject { + name: "a".to_string(), + size: 1, + mod_time: None, + etag: None, + storage_class: None, + }]), + }; + + assert!(persistent_key_only_index_has_complete_metadata_snapshot(&complete)); + } + + #[test] + #[serial_test::serial] + fn list_objects_index_provider_from_env_is_default_off() { + temp_env::with_var_unset(ENV_API_LIST_OBJECTS_INDEX_PROVIDER, || { + assert_eq!(list_objects_index_provider_from_env(), None); + let health = list_objects_key_only_provider_health(list_objects_index_provider_from_env()); + assert_eq!(health.state, ListIndexLifecycleState::Disabled); + assert_eq!(health.fallback_reason, Some(ListIndexFallbackReason::Disabled)); + }); + } + + #[test] + #[serial_test::serial] + fn list_objects_index_provider_from_env_publishes_live_generation_when_enabled() { + temp_env::with_var( + ENV_API_LIST_OBJECTS_INDEX_PROVIDER, + Some(LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY), + || { + assert_eq!(list_objects_index_provider_from_env(), Some(ListObjectsIndexProviderKind::WalkerKeyOnly)); + let health = list_objects_key_only_provider_health(list_objects_index_provider_from_env()); + assert_eq!(health.state, ListIndexLifecycleState::Healthy); + assert_eq!(health.active_generation.as_deref(), Some(LIST_CURSOR_GENERATION_LIVE)); + assert_eq!(health.fallback_reason, None); + }, + ); + } + + #[test] + #[serial_test::serial] + fn list_objects_index_provider_from_env_accepts_persistent_key_only() { + temp_env::with_var( + ENV_API_LIST_OBJECTS_INDEX_PROVIDER, + Some(LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY), + || { + assert_eq!( + list_objects_index_provider_from_env(), + Some(ListObjectsIndexProviderKind::PersistentKeyOnly) + ); + }, + ); + } + + #[test] + #[serial_test::serial] + fn list_objects_index_provider_state_degrades_persistent_without_path() { + temp_env::with_var( + ENV_API_LIST_OBJECTS_INDEX_PROVIDER, + Some(LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY), + || { + temp_env::with_var_unset(ENV_API_LIST_OBJECTS_INDEX_PROVIDER_PATH, || { + let provider = list_objects_index_provider_state_from_env().expect("persistent provider should be selected"); + let health = provider.health_snapshot(); + + assert_eq!(provider.kind, ListObjectsIndexProviderKind::PersistentKeyOnly); + assert_eq!(provider.persistent_path, None); + assert_eq!(health.state, ListIndexLifecycleState::Degraded); + assert_eq!(health.fallback_reason, Some(ListIndexFallbackReason::Degraded)); + }); + }, + ); + } + + #[test] + #[serial_test::serial] + fn list_objects_index_provider_state_rebuilds_persistent_until_index_is_loaded() { + let index_path = "/tmp/rustfs-listobjects-key-only.index"; + temp_env::with_var( + ENV_API_LIST_OBJECTS_INDEX_PROVIDER, + Some(LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY), + || { + temp_env::with_var(ENV_API_LIST_OBJECTS_INDEX_PROVIDER_PATH, Some(index_path), || { + temp_env::with_var(ENV_API_LIST_OBJECTS_INDEX_PROVIDER_GENERATION, Some("bench-20260706"), || { + let provider = + list_objects_index_provider_state_from_env().expect("persistent provider should be selected"); + let health = provider.health_snapshot(); + + assert_eq!(provider.kind, ListObjectsIndexProviderKind::PersistentKeyOnly); + assert_eq!(provider.persistent_path.as_deref(), Some(std::path::Path::new(index_path))); + assert_eq!(provider.persistent_generation.as_deref(), Some("bench-20260706")); + assert_eq!(health.state, ListIndexLifecycleState::Rebuilding); + assert_eq!(health.staging_generation.as_deref(), Some("bench-20260706")); + assert_eq!(health.fallback_reason, Some(ListIndexFallbackReason::Rebuilding)); + }); + }); + }, + ); + } + + #[test] + fn parse_persistent_key_only_index_sorts_and_deduplicates_keys() { + let index = parse_persistent_key_only_index( + "# rustfs-listobjects-key-only-v1\n\ + # bucket=photos\n\ + # generation=bench-20260706\n\ + # checkpoint_high_water_mark=99\n\ + photos/c.jpg\n\ + photos/a.jpg\r\n\ + \n\ + photos/b.jpg\n\ + photos/a.jpg\n", + ); + + assert_eq!(index.bucket.as_deref(), Some("photos")); + assert_eq!(index.generation, "bench-20260706"); + assert_eq!(index.checkpoint_high_water_mark, 99); + assert_eq!( + index.keys.as_slice(), + vec![ + "photos/a.jpg".to_string(), + "photos/b.jpg".to_string(), + "photos/c.jpg".to_string() + ] + ); + } + + #[test] + fn parse_namespace_mutation_journal_state_reads_bucket_and_high_water_mark() { + let state = parse_namespace_mutation_journal_state( + "# rustfs-listobjects-namespace-journal-v1\n\ + # bucket=photos\n\ + # high_water_mark=123\n\ + # status=healthy\n", + ) + .expect("journal state should parse"); + + assert_eq!(state.bucket.as_deref(), Some("photos")); + assert_eq!(state.high_water_mark, 123); + assert_eq!(state.status, NamespaceMutationJournalStatus::Healthy); + } + + #[test] + fn namespace_mutation_journal_chaos_env_accepts_controlled_status() { + temp_env::with_var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS, Some("degraded"), || { + assert_eq!( + namespace_mutation_journal_chaos_status_from_env(), + Some(NamespaceMutationJournalStatus::Degraded) + ); + }); + temp_env::with_var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS, Some("HEALTHY"), || { + assert_eq!( + namespace_mutation_journal_chaos_status_from_env(), + Some(NamespaceMutationJournalStatus::Healthy) + ); + }); + temp_env::with_var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS, Some("unknown"), || { + assert_eq!(namespace_mutation_journal_chaos_status_from_env(), None); + }); + } + + #[test] + fn namespace_mutation_journal_chaos_env_requires_explicit_enable() { + temp_env::with_var_unset(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED, || { + assert!(!namespace_mutation_journal_chaos_enabled_from_env()); + }); + temp_env::with_var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED, Some("true"), || { + assert!(namespace_mutation_journal_chaos_enabled_from_env()); + }); + temp_env::with_var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED, Some("1"), || { + assert!(namespace_mutation_journal_chaos_enabled_from_env()); + }); + temp_env::with_var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED, Some("on"), || { + assert!(namespace_mutation_journal_chaos_enabled_from_env()); + }); + temp_env::with_var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED, Some("false"), || { + assert!(!namespace_mutation_journal_chaos_enabled_from_env()); + }); + } + + #[test] + fn namespace_mutation_journal_chaos_env_reads_bucket_and_sequence() { + temp_env::with_vars( + [ + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET, Some("bench-bucket")), + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_SEQUENCE, Some("42")), + ], + || { + assert_eq!(namespace_mutation_journal_chaos_bucket_from_env().as_deref(), Some("bench-bucket")); + assert_eq!(namespace_mutation_journal_chaos_sequence_from_env(), Some(42)); + }, + ); + temp_env::with_var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_SEQUENCE, Some("not-a-number"), || { + assert_eq!(namespace_mutation_journal_chaos_sequence_from_env(), None); + }); + } + + #[test] + fn namespace_mutation_journal_chaos_config_requires_enable_bucket_and_status() { + temp_env::with_vars( + [ + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED, Some("true")), + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET, Some("bench-bucket")), + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS, Some("degraded")), + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_SEQUENCE, Some("42")), + ], + || { + let config = namespace_mutation_journal_chaos_config_from_env().expect("enabled chaos config should parse"); + assert_eq!(config.bucket, "bench-bucket"); + assert_eq!(config.status, NamespaceMutationJournalStatus::Degraded); + assert_eq!(config.sequence, Some(42)); + }, + ); + temp_env::with_vars( + [ + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED, Some("false")), + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET, Some("bench-bucket")), + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS, Some("degraded")), + ], + || { + assert_eq!(namespace_mutation_journal_chaos_config_from_env(), None); + }, + ); + temp_env::with_vars( + [ + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED, Some("true")), + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET, Some("bench-bucket")), + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS, Some("invalid")), + ], + || { + assert_eq!(namespace_mutation_journal_chaos_config_from_env(), None); + }, + ); + temp_env::with_vars( + [ + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED, Some("true")), + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET, None::<&str>), + (ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS, Some("degraded")), + ], + || { + assert_eq!(namespace_mutation_journal_chaos_config_from_env(), None); + }, + ); + } + + #[tokio::test] + async fn namespace_mutation_journal_write_never_regresses() { + let tempdir = tempfile::tempdir().expect("tempdir should be created"); + let journal_root = tempdir.path().join("namespace-mutation-journal"); + + assert_eq!( + write_namespace_mutation_journal_state( + NamespaceMutationJournalBackend::LocalPath(journal_root.clone()), + "bucket", + 7, + false, + ) + .await + .expect("journal sequence should be written") + .high_water_mark, + 7 + ); + assert_eq!( + write_namespace_mutation_journal_state( + NamespaceMutationJournalBackend::LocalPath(journal_root.clone()), + "bucket", + 3, + false, + ) + .await + .expect("journal sequence should not regress") + .high_water_mark, + 7 + ); + } + + #[test] + #[serial_test::serial] + fn observed_mutation_persists_namespace_journal_high_water() { + let tempdir = tempfile::tempdir().expect("tempdir should be created"); + let index_path = tempdir.path().join("persistent-key-only.index"); + let journal_root = tempdir.path().join("namespace-mutation-journal"); + let index_path = index_path.to_str().expect("index path should be utf8"); + let journal_root = journal_root.to_str().expect("journal path should be utf8"); + + temp_env::with_var( + ENV_API_LIST_OBJECTS_INDEX_PROVIDER, + Some(LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY), + || { + temp_env::with_var(ENV_API_LIST_OBJECTS_INDEX_PROVIDER_PATH, Some(index_path), || { + temp_env::with_var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_PATH, Some(journal_root), || { + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime should start"); + runtime.block_on(async { + reset_list_objects_mutation_sequences_for_test().await; + + persist_observed_list_objects_mutation(None, "bucket", 11).await; + let state = load_namespace_mutation_journal_state( + NamespaceMutationJournalBackend::LocalPath(std::path::PathBuf::from(journal_root)), + "bucket", + ) + .await + .expect("journal state should load") + .expect("journal state should exist"); + + assert_eq!(state.high_water_mark, 11); + assert_eq!(state.status, NamespaceMutationJournalStatus::Healthy); + assert_eq!(current_list_objects_mutation_sequence("bucket").await, 11); + }); + }); + }); + }, + ); + } + + #[tokio::test] + async fn persistent_key_only_index_load_recovers_mutation_sequence_from_journal() { + let tempdir = tempfile::tempdir().expect("tempdir should be created"); + let index_path = tempdir.path().join("persistent-key-only.index"); + let journal_root = tempdir.path().join("namespace-mutation-journal"); + let keys = vec!["object-a".to_string(), "object-b".to_string()]; + + write_persistent_key_only_index(None, &index_path, "bucket", "generation-42", 5, &keys) + .await + .expect("index should be written"); + write_namespace_mutation_journal_state(NamespaceMutationJournalBackend::LocalPath(journal_root), "bucket", 9, false) + .await + .expect("journal sequence should be written"); + reset_list_objects_mutation_sequences_for_test().await; + + let index = load_persistent_key_only_index(None, &index_path) + .await + .expect("index should load"); + + assert_eq!(index.checkpoint_high_water_mark, 5); + assert_eq!(current_list_objects_mutation_sequence("bucket").await, 9); + } + + #[test] + fn persistent_key_only_index_provider_match_rejects_configured_generation_mismatch() { + let index = PersistentKeyOnlyIndex { + bucket: Some("bucket".to_string()), + generation: "generation-old".to_string(), + checkpoint_high_water_mark: 42, + keys: Arc::new(vec!["a".to_string(), "b".to_string()]), + objects: Arc::new(Vec::new()), + }; + let matching_provider = ListObjectsIndexProviderState::persistent_key_only( + Some(PathBuf::from("/tmp/persistent-key-only.index")), + Some("generation-old".to_string()), + ); + let mismatching_provider = ListObjectsIndexProviderState::persistent_key_only( + Some(PathBuf::from("/tmp/persistent-key-only.index")), + Some("generation-new".to_string()), + ); + + assert!(persistent_key_only_index_matches_provider(&index, "bucket", &matching_provider)); + assert!(!persistent_key_only_index_matches_provider(&index, "bucket", &mismatching_provider)); + assert!(!persistent_key_only_index_matches_provider(&index, "other-bucket", &matching_provider)); + } + + #[test] + fn persistent_key_only_index_health_uses_snapshot_generation_and_checkpoint() { + let index = PersistentKeyOnlyIndex { + bucket: Some("bucket".to_string()), + generation: "generation-42".to_string(), + checkpoint_high_water_mark: 42, + keys: Arc::new(vec!["a".to_string(), "b".to_string()]), + objects: Arc::new(Vec::new()), + }; + + let health = persistent_key_only_index_health( + &index, + NamespaceMutationJournalSnapshot { + high_water_mark: 42, + degraded: false, + }, + ); + + assert_eq!(health.state, ListIndexLifecycleState::Healthy); + assert_eq!(health.active_generation.as_deref(), Some("generation-42")); + assert_eq!(health.checkpoint_high_water_mark, 42); + assert_eq!(health.mutation_high_water_mark, 42); + assert_eq!(health.fallback_reason, None); + } + + #[test] + fn persistent_key_only_index_health_reports_lagging_mutation_checkpoint() { + let index = PersistentKeyOnlyIndex { + bucket: Some("bucket".to_string()), + generation: "generation-42".to_string(), + checkpoint_high_water_mark: 42, + keys: Arc::new(vec!["a".to_string(), "b".to_string()]), + objects: Arc::new(Vec::new()), + }; + + let health = persistent_key_only_index_health( + &index, + NamespaceMutationJournalSnapshot { + high_water_mark: 43, + degraded: false, + }, + ); + + assert_eq!(health.state, ListIndexLifecycleState::Lagging); + assert_eq!(health.active_generation.as_deref(), Some("generation-42")); + assert_eq!(health.checkpoint_high_water_mark, 42); + assert_eq!(health.mutation_high_water_mark, 43); + assert_eq!(health.mutation_lag, 1); + assert_eq!(health.fallback_reason, Some(ListIndexFallbackReason::Lagging)); + } + + #[test] + fn persistent_key_only_index_health_reports_degraded_journal() { + let index = PersistentKeyOnlyIndex { + bucket: Some("bucket".to_string()), + generation: "generation-42".to_string(), + checkpoint_high_water_mark: 42, + keys: Arc::new(vec!["a".to_string(), "b".to_string()]), + objects: Arc::new(Vec::new()), + }; + + let health = persistent_key_only_index_health( + &index, + NamespaceMutationJournalSnapshot { + high_water_mark: 42, + degraded: true, + }, + ); + + assert_eq!(health.state, ListIndexLifecycleState::Degraded); + assert_eq!(health.fallback_reason, Some(ListIndexFallbackReason::Degraded)); + } + + #[tokio::test] + async fn list_objects_mutation_sequence_advances_per_bucket() { + reset_list_objects_mutation_sequences_for_test().await; + + assert_eq!(current_list_objects_mutation_sequence("bucket-a").await, 0); + assert_eq!(observe_list_objects_mutations_with_store(None, "bucket-a", 1).await, Some(1)); + assert_eq!(observe_list_objects_mutations_with_store(None, "bucket-a", 2).await, Some(3)); + assert_eq!(current_list_objects_mutation_sequence("bucket-a").await, 3); + assert_eq!(current_list_objects_mutation_sequence("bucket-b").await, 0); + assert_eq!(observe_list_objects_mutations_with_store(None, "bucket-b", 0).await, None); + assert_eq!(current_list_objects_mutation_sequence("bucket-b").await, 0); + } + + #[test] + fn list_objects_index_provider_state_uses_lifecycle_active_generation() { + let provider = ListObjectsIndexProviderState::from_kind(ListObjectsIndexProviderKind::WalkerKeyOnly); + let health = provider.health_snapshot(); + + assert_eq!(provider.kind, ListObjectsIndexProviderKind::WalkerKeyOnly); + assert_eq!(health.state, ListIndexLifecycleState::Healthy); + assert_eq!(health.staging_generation, None); + assert_eq!(health.active_generation.as_deref(), Some(LIST_CURSOR_GENERATION_LIVE)); + assert_eq!( + select_list_index_source_mode(Some(ListSourceMode::IndexKeyOnly), health.active_generation.as_deref(), &health), + ListIndexSourceDecision::UseIndex(ListSourceMode::IndexKeyOnly) + ); + } + + #[test] + fn list_objects_index_opt_in_fallback_accepts_incompatible_cursor_without_panicking() { + record_list_objects_index_opt_in_fallback( + &ListPathOptions { + bucket: "bucket".to_string(), + prefix: "photos/".to_string(), + marker: Some( + "photos/2026/[rustfs_cache:v2,id:list-cache-id,src:index_verified_page,gen:generation-1]".to_string(), + ), + ..Default::default() + }, + ListSourceMode::IndexKeyOnly, + ); + } + + #[tokio::test] + async fn list_objects_verified_index_candidates_use_live_metadata_only() { + let candidates = vec![ + "photos/2026/a.jpg".to_string(), + "photos/2026/b.jpg".to_string(), + "photos/2026/c.jpg".to_string(), + ]; + + let result = + list_objects_from_verified_index_candidates("photos/2026/", None, &None, 100, &candidates, |key| async move { + Ok(Some(test_live_object_info(&key, "live-etag"))) + }) + .await + .expect("verified candidate listing should succeed"); + + assert_eq!(result.objects.len(), 3); + assert!( + result + .objects + .iter() + .all(|object| object.etag.as_deref() == Some("live-etag")) + ); + assert!(result.prefixes.is_empty()); + assert!(!result.is_truncated); + } + + #[tokio::test] + async fn list_objects_verified_index_candidates_apply_marker_and_delimiter_boundaries() { + let candidates = vec![ + "photos/2025/old.jpg".to_string(), + "photos/2026/a.jpg".to_string(), + "photos/2026/archive/b.jpg".to_string(), + "photos/2026/archive/c.jpg".to_string(), + "photos/2026/d.jpg".to_string(), + ]; + + let result = list_objects_from_verified_index_candidates( + "photos/2026/", + Some("photos/2026/a.jpg"), + &Some("/".to_string()), + 100, + &candidates, + |key| async move { Ok(Some(test_live_object_info(&key, "live-etag"))) }, + ) + .await + .expect("verified candidate listing should succeed"); + + assert_eq!( + result.objects.iter().map(|object| object.name.as_str()).collect::>(), + vec!["photos/2026/d.jpg"] + ); + assert_eq!(result.prefixes, vec!["photos/2026/archive/".to_string()]); + assert!(!result.is_truncated); + } + + #[tokio::test] + async fn list_objects_verified_index_candidates_use_common_prefix_as_page_marker() { + let candidates = vec![ + "photos/2026/archive/b.jpg".to_string(), + "photos/2026/archive/c.jpg".to_string(), + "photos/2026/d.jpg".to_string(), + ]; + + let result = list_objects_from_verified_index_candidates( + "photos/2026/", + None, + &Some("/".to_string()), + 1, + &candidates, + |key| async move { Ok(Some(test_live_object_info(&key, "live-etag"))) }, + ) + .await + .expect("verified candidate listing should succeed"); + + assert!(result.objects.is_empty()); + assert_eq!(result.prefixes, vec!["photos/2026/archive/".to_string()]); + assert!(result.is_truncated); + assert_eq!(result.next_marker.as_deref(), Some("photos/2026/archive/")); + } + + #[tokio::test] + async fn list_objects_verified_index_candidates_drop_deleted_or_failed_live_verifications() { + let candidates = vec!["photos/2026/a.jpg".to_string(), "photos/2026/deleted.jpg".to_string()]; + + let result = + list_objects_from_verified_index_candidates("photos/2026/", None, &None, 100, &candidates, |key| async move { + if key.ends_with("deleted.jpg") { + Ok(None) + } else { + Ok(Some(test_live_object_info(&key, "live-etag"))) + } + }) + .await + .expect("verified candidate listing should succeed"); + + assert_eq!(result.objects.len(), 1); + assert_eq!(result.objects[0].name, "photos/2026/a.jpg"); + } + + #[tokio::test] + async fn list_objects_verified_index_candidates_ignore_stale_delete_markers() { + let candidates = vec![ + "photos/2026/live.jpg".to_string(), + "photos/2026/stale-delete-marker.jpg".to_string(), + ]; + + let result = + list_objects_from_verified_index_candidates("photos/2026/", None, &None, 100, &candidates, |key| async move { + if key.ends_with("stale-delete-marker.jpg") { + Ok(None) + } else { + Ok(Some(test_live_object_info(&key, "live-etag"))) + } + }) + .await + .expect("verified candidate listing should succeed"); + + assert_eq!( + result.objects.iter().map(|object| object.name.as_str()).collect::>(), + vec!["photos/2026/live.jpg"] + ); + } + + #[tokio::test] + async fn list_objects_verified_index_candidates_return_overwritten_live_metadata() { + let candidates = vec!["photos/2026/overwritten.jpg".to_string()]; + + let result = + list_objects_from_verified_index_candidates("photos/2026/", None, &None, 100, &candidates, |key| async move { + Ok(Some(test_live_object_info(&key, "new-live-etag"))) + }) + .await + .expect("verified candidate listing should succeed"); + + assert_eq!(result.objects.len(), 1); + assert_eq!(result.objects[0].etag.as_deref(), Some("new-live-etag")); + } + + #[tokio::test] + async fn list_objects_verified_index_candidates_include_completed_multipart_after_live_verify() { + let candidates = vec!["photos/2026/multipart-complete.bin".to_string()]; + + let result = + list_objects_from_verified_index_candidates("photos/2026/", None, &None, 100, &candidates, |key| async move { + Ok(Some(test_live_object_info(&key, "completed-multipart-etag"))) + }) + .await + .expect("verified candidate listing should succeed"); + + assert_eq!(result.objects.len(), 1); + assert_eq!(result.objects[0].name, "photos/2026/multipart-complete.bin"); + assert_eq!(result.objects[0].etag.as_deref(), Some("completed-multipart-etag")); + } + + #[tokio::test] + async fn list_objects_verified_index_candidates_report_verification_stats() { + let candidates = vec![ + "photos/2025/old.jpg".to_string(), + "photos/2026/a.jpg".to_string(), + "photos/2026/archive/b.jpg".to_string(), + "photos/2026/deleted.jpg".to_string(), + ]; + + let result = list_objects_from_verified_index_candidates_with_stats( + "photos/2026/", + Some("photos/2026/a.jpg"), + &Some("/".to_string()), + 100, + &candidates, + |key| async move { + if key.ends_with("deleted.jpg") { + Ok(None) + } else { + Ok(Some(test_live_object_info(&key, "live-etag"))) + } + }, + ) + .await + .expect("verified candidate listing should succeed"); + + assert!(result.info.objects.is_empty()); + assert_eq!(result.info.prefixes, vec!["photos/2026/archive/".to_string()]); + assert_eq!(result.stats.candidate_keys, 4); + assert_eq!(result.stats.skipped_keys, 2); + assert_eq!(result.stats.common_prefixes, 1); + assert_eq!(result.stats.live_verify_attempts, 1); + assert_eq!(result.stats.live_verify_hits, 0); + assert_eq!(result.stats.live_verify_misses, 1); + } + + #[tokio::test] + async fn list_objects_verified_index_candidates_skip_stats_when_disabled() { + let candidates = vec!["photos/2026/a.jpg".to_string(), "photos/2026/deleted.jpg".to_string()]; + + let result = list_objects_from_verified_index_candidates_with_optional_stats( + "photos/2026/", + None, + &None, + 100, + &candidates, + false, + |key| async move { + if key.ends_with("deleted.jpg") { + Ok(None) + } else { + Ok(Some(test_live_object_info(&key, "live-etag"))) + } + }, + ) + .await + .expect("verified candidate listing should succeed"); + + assert_eq!(result.info.objects.len(), 1); + assert_eq!(result.stats, VerifiedIndexCandidateStats::default()); + } + + #[test] + fn list_index_provider_selection_rejects_generation_mismatch() { + let mut lifecycle = ListIndexLifecycle::disabled(10); + lifecycle.begin_rebuild("generation-2", 100); + lifecycle.checkpoint_rebuild(100); + assert!(lifecycle.publish_rebuild()); + let health = lifecycle.health_snapshot(); + + let opts = ListPathOptions { + marker: Some("photos/2026/a.jpg[rustfs_cache:v2,id:list-cache-id,src:index_key_only,gen:generation-1]".to_string()), + ..Default::default() + }; + + assert_eq!( + select_list_index_provider_source_mode(&opts, ListSourceMode::IndexKeyOnly, &health), + ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::GenerationMismatch) + ); + } + + #[test] + fn list_index_provider_selection_falls_back_when_degraded() { + let mut lifecycle = ListIndexLifecycle::disabled(10); + lifecycle.begin_rebuild("generation-1", 100); + lifecycle.checkpoint_rebuild(100); + assert!(lifecycle.publish_rebuild()); + lifecycle.mark_degraded(); + let health = lifecycle.health_snapshot(); + + assert_eq!( + select_list_index_provider_source_mode(&ListPathOptions::default(), ListSourceMode::IndexKeyOnly, &health), + ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::Degraded) + ); + } + + #[test] + fn metadata_fast_provider_selection_falls_back_when_checkpoint_lags() { + let mut lifecycle = ListIndexLifecycle::disabled(0); + lifecycle.begin_rebuild("generation-1", 100); + lifecycle.checkpoint_rebuild(100); + assert!(lifecycle.publish_rebuild()); + lifecycle.observe_mutation_high_water_mark(101); + let health = lifecycle.health_snapshot(); + + assert_eq!( + select_list_index_provider_source_mode(&ListPathOptions::default(), ListSourceMode::IndexMetadataFast, &health), + ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::Lagging) + ); + } + #[test] fn list_metadata_resolution_params_limits_plain_listing_to_latest_version() { let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 3, false); @@ -4780,7 +8452,7 @@ mod test { let marker = opts.encode_marker("photos/2026/image.jpg"); let expected_marker = format!( - "photos/2026/image.jpg[rustfs_cache:{},id:list-cache-id,p:3,s:7]", + "photos/2026/image.jpg[rustfs_cache:{},id:list-cache-id,p:3,s:7,src:walker,gen:live]", super::MARKER_TAG_VERSION ); assert_eq!(marker, expected_marker); @@ -4795,6 +8467,225 @@ mod test { assert_eq!(parsed.id.as_deref(), Some("list-cache-id")); assert_eq!(parsed.pool_idx, Some(3)); assert_eq!(parsed.set_idx, Some(7)); + assert_eq!(parsed.cursor_source, Some(ListSourceMode::Walker)); + assert_eq!(parsed.cursor_generation.as_deref(), Some("live")); + assert!(!parsed.create); + } + + #[test] + fn list_path_marker_parser_preserves_source_aware_cursor_fields() { + let mut parsed = ListPathOptions { + marker: Some( + "photos/2026/image.jpg[rustfs_cache:v2,id:list-cache-id,p:3,s:7,src:index_key_only,gen:generation-42]" + .to_string(), + ), + ..Default::default() + }; + + parsed.parse_marker(); + + assert_eq!(parsed.marker.as_deref(), Some("photos/2026/image.jpg")); + assert_eq!(parsed.id.as_deref(), Some("list-cache-id")); + assert_eq!(parsed.pool_idx, Some(3)); + assert_eq!(parsed.set_idx, Some(7)); + assert_eq!(parsed.cursor_source, Some(ListSourceMode::IndexKeyOnly)); + assert_eq!(parsed.cursor_generation.as_deref(), Some("generation-42")); + assert!(!parsed.create); + } + + #[test] + fn list_source_modes_expose_metadata_authority_boundaries() { + assert_eq!(ListSourceMode::Walker.metadata_authority(), ListMetadataAuthority::LiveXlMeta); + assert_eq!( + ListSourceMode::IndexKeyOnly.metadata_authority(), + ListMetadataAuthority::LiveVerifiedIndexCandidate + ); + assert_eq!( + ListSourceMode::IndexVerifiedPage.metadata_authority(), + ListMetadataAuthority::LiveVerifiedIndexCandidate + ); + assert_eq!( + ListSourceMode::IndexMetadataFast.metadata_authority(), + ListMetadataAuthority::IndexSnapshotEventuallyConsistent + ); + + assert!(ListSourceMode::IndexKeyOnly.requires_live_metadata_verification()); + assert!(ListSourceMode::IndexVerifiedPage.requires_live_metadata_verification()); + assert!(!ListSourceMode::IndexMetadataFast.requires_live_metadata_verification()); + assert!(!ListSourceMode::IndexMetadataFast.can_satisfy_strong_listing()); + } + + #[test] + fn list_index_source_selection_falls_back_until_request_is_healthy_and_versioned() { + let healthy = TestIndexHealth { reason: None }; + let unhealthy = TestIndexHealth { + reason: Some(ListIndexFallbackReason::Unhealthy), + }; + + assert_eq!( + select_list_index_source_mode(None, Some("generation-1"), &healthy), + ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::Disabled) + ); + assert_eq!( + select_list_index_source_mode(Some(ListSourceMode::Walker), Some("generation-1"), &healthy), + ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::Disabled) + ); + assert_eq!( + select_list_index_source_mode(Some(ListSourceMode::IndexKeyOnly), None, &healthy), + ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::MissingGeneration) + ); + assert_eq!( + select_list_index_source_mode(Some(ListSourceMode::IndexKeyOnly), Some(""), &healthy), + ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::MissingGeneration) + ); + assert_eq!( + select_list_index_source_mode(Some(ListSourceMode::IndexKeyOnly), Some("generation-1"), &unhealthy), + ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::Unhealthy) + ); + assert_eq!( + select_list_index_source_mode(Some(ListSourceMode::IndexVerifiedPage), Some("generation-1"), &healthy), + ListIndexSourceDecision::UseIndex(ListSourceMode::IndexVerifiedPage) + ); + } + + #[test] + fn list_index_traits_keep_page_iteration_and_metadata_authority_explicit() { + let page = TestIndexPage { + mode: ListSourceMode::IndexMetadataFast, + generation: TestIndexGeneration { + id: "snapshot-7".to_string(), + }, + keys: vec!["photos/2026/image.jpg".to_string()], + }; + let mut iterator = TestIndexIterator { + pages: vec![page].into_iter(), + }; + let page = iterator.next_page().expect("test iterator should produce one page"); + + assert_eq!(page.keys(), &["photos/2026/image.jpg".to_string()]); + assert_eq!(page.generation().generation_id(), "snapshot-7"); + assert_eq!(page.metadata_authority(), ListMetadataAuthority::IndexSnapshotEventuallyConsistent); + assert!(!page.source_mode().can_satisfy_strong_listing()); + } + + #[test] + fn list_index_key_lookup_trait_returns_explicit_fallback_decision() { + let lookup = TestIndexLookup { + decision: ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::UnsupportedRequest), + }; + + assert_eq!( + lookup.lookup_page(&ListPathOptions::default()), + ListIndexSourceDecision::FallbackToWalker(ListIndexFallbackReason::UnsupportedRequest) + ); + } + + #[test] + fn list_index_lifecycle_rebuild_requires_publish_before_health() { + let mut lifecycle = ListIndexLifecycle::disabled(10); + + lifecycle.begin_rebuild("generation-1", 100); + lifecycle.checkpoint_rebuild(80); + + let snapshot = lifecycle.health_snapshot(); + assert_eq!(snapshot.state, ListIndexLifecycleState::Rebuilding); + assert_eq!(snapshot.active_generation, None); + assert_eq!(snapshot.staging_generation.as_deref(), Some("generation-1")); + assert_eq!(snapshot.checkpoint_high_water_mark, 80); + assert_eq!(snapshot.fallback_reason, Some(ListIndexFallbackReason::Rebuilding)); + + assert!(lifecycle.publish_rebuild()); + let snapshot = lifecycle.health_snapshot(); + assert_eq!(snapshot.state, ListIndexLifecycleState::Healthy); + assert_eq!(snapshot.active_generation.as_deref(), Some("generation-1")); + assert_eq!(snapshot.staging_generation, None); + assert_eq!(snapshot.fallback_reason, None); + } + + #[test] + fn list_index_lifecycle_restart_never_trusts_unpublished_staging_generation() { + let mut lifecycle = ListIndexLifecycle::disabled(10); + lifecycle.begin_rebuild("generation-1", 100); + lifecycle.checkpoint_rebuild(90); + + let recovered = lifecycle.recover_after_restart(); + let snapshot = recovered.health_snapshot(); + + assert_eq!(snapshot.state, ListIndexLifecycleState::Disabled); + assert_eq!(snapshot.active_generation, None); + assert_eq!(snapshot.staging_generation, None); + assert_eq!(snapshot.fallback_reason, Some(ListIndexFallbackReason::Disabled)); + } + + #[test] + fn list_index_lifecycle_restart_keeps_only_last_published_generation() { + let mut lifecycle = ListIndexLifecycle::disabled(10); + lifecycle.begin_rebuild("generation-1", 100); + lifecycle.checkpoint_rebuild(100); + assert!(lifecycle.publish_rebuild()); + + lifecycle.begin_rebuild("generation-2", 125); + lifecycle.checkpoint_rebuild(120); + + let recovered = lifecycle.recover_after_restart(); + let snapshot = recovered.health_snapshot(); + + assert_eq!(snapshot.state, ListIndexLifecycleState::Healthy); + assert_eq!(snapshot.active_generation.as_deref(), Some("generation-1")); + assert_eq!(snapshot.staging_generation, None); + assert_eq!(snapshot.fallback_reason, None); + } + + #[test] + fn list_index_lifecycle_reports_lagging_and_fallback_reason() { + let mut lifecycle = ListIndexLifecycle::disabled(10); + lifecycle.begin_rebuild("generation-1", 100); + lifecycle.checkpoint_rebuild(100); + assert!(lifecycle.publish_rebuild()); + + lifecycle.observe_mutation_high_water_mark(115); + let snapshot = lifecycle.health_snapshot(); + + assert_eq!(snapshot.state, ListIndexLifecycleState::Lagging); + assert_eq!(snapshot.mutation_lag, 15); + assert_eq!(snapshot.fallback_reason, Some(ListIndexFallbackReason::Lagging)); + assert!(!snapshot.is_healthy()); + } + + #[test] + fn list_index_lifecycle_reports_degraded_and_corrupt_as_unhealthy() { + let mut lifecycle = ListIndexLifecycle::disabled(10); + lifecycle.begin_rebuild("generation-1", 100); + lifecycle.checkpoint_rebuild(100); + assert!(lifecycle.publish_rebuild()); + + lifecycle.mark_degraded(); + let degraded = lifecycle.health_snapshot(); + assert_eq!(degraded.state, ListIndexLifecycleState::Degraded); + assert_eq!(degraded.fallback_reason, Some(ListIndexFallbackReason::Degraded)); + assert!(!degraded.is_healthy()); + + lifecycle.mark_corrupt(); + let corrupt = lifecycle.health_snapshot(); + assert_eq!(corrupt.state, ListIndexLifecycleState::Corrupt); + assert_eq!(corrupt.fallback_reason, Some(ListIndexFallbackReason::Corrupt)); + assert!(!corrupt.is_healthy()); + } + + #[test] + fn list_path_marker_parser_rejects_unknown_cursor_source() { + let original = "photos/2026/image.jpg[rustfs_cache:v2,id:list-cache-id,src:unknown,gen:generation-42]"; + let mut parsed = ListPathOptions { + marker: Some(original.to_string()), + ..Default::default() + }; + + parsed.parse_marker(); + + assert_eq!(parsed.marker.as_deref(), Some(original)); + assert!(parsed.id.is_none()); + assert!(parsed.cursor_source.is_none()); + assert!(parsed.cursor_generation.is_none()); assert!(!parsed.create); } @@ -4862,6 +8753,8 @@ mod test { assert_eq!(parsed.id.as_deref(), Some("legacy-cache-id")); assert_eq!(parsed.pool_idx, Some(4)); assert_eq!(parsed.set_idx, Some(1)); + assert!(parsed.cursor_source.is_none()); + assert!(parsed.cursor_generation.is_none()); assert!(!parsed.create); } @@ -5269,6 +9162,50 @@ mod test { assert_eq!(results, vec!["obj-a", "obj-b", "obj-c"]); } + #[tokio::test] + async fn merge_entry_channels_documents_candidate_metadata_authority_risk() { + let (tx_a, rx_a) = mpsc::channel(4); + let (tx_b, rx_b) = mpsc::channel(4); + let (tx_c, rx_c) = mpsc::channel(4); + let (out_tx, mut out_rx) = mpsc::channel(4); + let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_500).expect("valid timestamp"); + let stale_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); + let stale_object = test_object_meta_entry_with_erasure_versions("obj-a", &[(stale_mod_time, "stale-etag", 4, 2)]); + + tx_a.send(test_delete_marker_meta_entry("obj-a", mod_time)) + .await + .expect("first delete marker should queue"); + tx_b.send(test_delete_marker_meta_entry("obj-a", mod_time)) + .await + .expect("second delete marker should queue"); + tx_c.send(stale_object).await.expect("stale object should queue"); + drop(tx_a); + drop(tx_b); + drop(tx_c); + + let cancel = CancellationToken::new(); + let handle = tokio::spawn(merge_entry_channels(cancel, vec![rx_a, rx_b, rx_c], out_tx, 2)); + + let mut merged = timeout(Duration::from_secs(1), out_rx.recv()) + .await + .expect("merged entry should be emitted promptly") + .expect("merged entry should be present"); + assert_eq!(merged.name, "obj-a"); + assert!( + !merged.is_latest_delete_marker(), + "current merge consumes candidate metadata bytes; future index-backed strong modes must live-verify metadata instead" + ); + assert!( + matches!(timeout(Duration::from_secs(1), out_rx.recv()).await, Ok(None)), + "merge should not emit a duplicate entry for the same key" + ); + + handle + .await + .expect("merge task should not panic") + .expect("merge task should succeed"); + } + #[tokio::test] async fn merge_entry_channels_handles_single_channel() { let (tx, rx) = mpsc::channel(4); diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 414ac7cde..fc5d80d3e 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -352,7 +352,13 @@ impl crate::storage_api_contracts::object::ObjectIO for ECStore { } #[instrument(level = "debug", skip(self, data))] async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result { - enqueue_transition_after_write(self.handle_put_object(bucket, object, data, opts).await, LcEventSrc::S3PutObject).await + let result = + enqueue_transition_after_write(self.handle_put_object(bucket, object, data, opts).await, LcEventSrc::S3PutObject) + .await; + if result.is_ok() { + list_objects::observe_list_objects_mutation(self, bucket).await; + } + result } } @@ -418,22 +424,34 @@ impl crate::storage_api_contracts::object::ObjectOperations for ECStore { src_opts: &ObjectOptions, dst_opts: &ObjectOptions, ) -> Result { - enqueue_transition_after_write( + let result = enqueue_transition_after_write( self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts) .await, LcEventSrc::S3CopyObject, ) - .await + .await; + if result.is_ok() { + list_objects::observe_list_objects_mutation(self, dst_bucket).await; + } + result } #[instrument(skip(self))] async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> { - self.handle_delete_object_version(bucket, object, fi, force_del_marker).await + let result = self.handle_delete_object_version(bucket, object, fi, force_del_marker).await; + if result.is_ok() { + list_objects::observe_list_objects_mutation(self, bucket).await; + } + result } #[instrument(skip(self))] async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result { - self.handle_delete_object(bucket, object, opts).await + let result = self.handle_delete_object(bucket, object, opts).await; + if result.is_ok() { + list_objects::observe_list_objects_mutation(self, bucket).await; + } + result } #[instrument(skip(self))] @@ -443,7 +461,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for ECStore { objects: Vec, opts: ObjectOptions, ) -> (Vec, Vec>) { - self.handle_delete_objects(bucket, objects, opts).await + let result = self.handle_delete_objects(bucket, objects, opts).await; + let success_count = result.1.iter().filter(|err| err.is_none()).count(); + if success_count > 0 { + list_objects::observe_list_objects_mutations(self, bucket, success_count).await; + } + result } #[instrument(skip(self))] @@ -660,12 +683,17 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for ECStore { uploaded_parts: Vec, opts: &ObjectOptions, ) -> Result { - enqueue_transition_after_write( - self.handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts) + let result = enqueue_transition_after_write( + self.clone() + .handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts) .await, LcEventSrc::S3CompleteMultipartUpload, ) - .await + .await; + if result.is_ok() { + list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await; + } + result } } diff --git a/crates/filemeta/src/fileinfo.rs b/crates/filemeta/src/fileinfo.rs index cf562be8d..a02597753 100644 --- a/crates/filemeta/src/fileinfo.rs +++ b/crates/filemeta/src/fileinfo.rs @@ -896,22 +896,28 @@ mod tests { assert!(base.replication_info_equals(&FileInfo::default())); // Differing mark_deleted breaks equality. - let mut marked = FileInfo::default(); - marked.mark_deleted = true; + let marked = FileInfo { + mark_deleted: true, + ..Default::default() + }; assert!(!base.replication_info_equals(&marked)); // Differing replication_state_internal breaks equality (regression guard: // this field used to be ignored by replication_info_equals). - let mut with_state = FileInfo::default(); - with_state.replication_state_internal = Some(ReplicationState { - replicate_decision_str: "arn:aws:s3:::dest".to_string(), + let with_state = FileInfo { + replication_state_internal: Some(ReplicationState { + replicate_decision_str: "arn:aws:s3:::dest".to_string(), + ..Default::default() + }), ..Default::default() - }); + }; assert!(!base.replication_info_equals(&with_state)); // Equal replication states remain equal. - let mut with_state_clone = FileInfo::default(); - with_state_clone.replication_state_internal = with_state.replication_state_internal.clone(); + let with_state_clone = FileInfo { + replication_state_internal: with_state.replication_state_internal.clone(), + ..Default::default() + }; assert!(with_state.replication_info_equals(&with_state_clone)); } } diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index e49d85fe7..4148b1b7e 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -95,6 +95,7 @@ pub mod config; pub mod deadlock_metrics; pub mod internode_metrics; pub mod io_metrics; +pub mod list_objects_metrics; pub mod lock_metrics; pub mod performance; pub mod process_lock_metrics; @@ -129,6 +130,12 @@ pub use io_metrics::{ IoSchedulerStats, record_bandwidth_observation, record_buffer_size_adjustment, record_io_priority_decision, record_io_scheduler_decision, record_load_level_change, record_queue_operation, record_starvation_event, }; +pub use list_objects_metrics::{ + LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED, LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED, LIST_OBJECTS_SOURCE_WALKER, + ListObjectsGatherObservation, ListObjectsIndexPageObservation, init_list_objects_metrics, record_list_objects_gather, + record_list_objects_index_attempt, record_list_objects_index_fallback, record_list_objects_index_live_verify_failure, + record_list_objects_index_served, record_list_objects_merge, +}; // Backpressure metrics exports pub use backpressure_metrics::{ @@ -1941,6 +1948,46 @@ pub fn record_cgroup_memory_split( } } +/// Allocator memory stats captured from the active process allocator. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct AllocatorMemoryObservation { + pub reserved_bytes: Option, + pub committed_bytes: Option, + pub page_committed_bytes: Option, + pub malloc_requested_bytes: Option, + pub malloc_requested_peak_bytes: Option, + pub malloc_requested_total_bytes: Option, + pub heap_count: Option, +} + +/// Record allocator-level memory attribution when supported by the allocator. +#[inline(always)] +pub fn record_allocator_memory_observation(backend: &'static str, observation: AllocatorMemoryObservation) { + if let Some(reserved_bytes) = observation.reserved_bytes { + gauge!("rustfs_memory_allocator_reserved_bytes", "backend" => backend).set(reserved_bytes as f64); + } + if let Some(committed_bytes) = observation.committed_bytes { + gauge!("rustfs_memory_allocator_committed_bytes", "backend" => backend).set(committed_bytes as f64); + } + if let Some(page_committed_bytes) = observation.page_committed_bytes { + gauge!("rustfs_memory_allocator_page_committed_bytes", "backend" => backend).set(page_committed_bytes as f64); + } + if let Some(malloc_requested_bytes) = observation.malloc_requested_bytes { + gauge!("rustfs_memory_allocator_malloc_requested_bytes", "backend" => backend).set(malloc_requested_bytes as f64); + } + if let Some(malloc_requested_peak_bytes) = observation.malloc_requested_peak_bytes { + gauge!("rustfs_memory_allocator_malloc_requested_peak_bytes", "backend" => backend) + .set(malloc_requested_peak_bytes as f64); + } + if let Some(malloc_requested_total_bytes) = observation.malloc_requested_total_bytes { + gauge!("rustfs_memory_allocator_malloc_requested_total_bytes", "backend" => backend) + .set(malloc_requested_total_bytes as f64); + } + if let Some(heap_count) = observation.heap_count { + gauge!("rustfs_memory_allocator_heap_count", "backend" => backend).set(heap_count as f64); + } +} + /// Track encoded bytes currently queued between erasure encode and disk writers. #[inline(always)] pub fn add_ec_encode_inflight_bytes(bytes: usize) { diff --git a/crates/io-metrics/src/list_objects_metrics.rs b/crates/io-metrics/src/list_objects_metrics.rs new file mode 100644 index 000000000..ce90fd017 --- /dev/null +++ b/crates/io-metrics/src/list_objects_metrics.rs @@ -0,0 +1,376 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use metrics::{counter, describe_counter, describe_histogram, histogram}; +use std::sync::OnceLock; + +use crate::get_stage_metrics_enabled; + +pub const LIST_OBJECTS_SOURCE_WALKER: &str = "walker"; +pub const LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED: &str = "limit_reached"; +pub const LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED: &str = "input_closed"; +pub const LIST_OBJECTS_MERGE_OUTCOME_STARTED: &str = "started"; + +const LIST_OBJECTS_GATHER_TOTAL: &str = "rustfs_s3_list_objects_gather_total"; +const LIST_OBJECTS_GATHER_DURATION_MS: &str = "rustfs_s3_list_objects_gather_duration_ms"; +const LIST_OBJECTS_GATHER_SCANNED_ENTRIES: &str = "rustfs_s3_list_objects_gather_scanned_entries"; +const LIST_OBJECTS_GATHER_RETURNED_ENTRIES: &str = "rustfs_s3_list_objects_gather_returned_entries"; +const LIST_OBJECTS_GATHER_FILTERED_ENTRIES: &str = "rustfs_s3_list_objects_gather_filtered_entries"; +const LIST_OBJECTS_GATHER_SCAN_AMPLIFICATION: &str = "rustfs_s3_list_objects_gather_scan_amplification"; +const LIST_OBJECTS_GATHER_LIMIT: &str = "rustfs_s3_list_objects_gather_limit"; +const LIST_OBJECTS_MERGE_FAN_IN: &str = "rustfs_s3_list_objects_merge_fan_in"; +const LIST_OBJECTS_MERGE_READ_QUORUM: &str = "rustfs_s3_list_objects_merge_read_quorum"; +const LIST_OBJECTS_INDEX_ATTEMPT_TOTAL: &str = "rustfs_s3_list_objects_index_attempt_total"; +const LIST_OBJECTS_INDEX_FALLBACK_TOTAL: &str = "rustfs_s3_list_objects_index_fallback_total"; +const LIST_OBJECTS_INDEX_SERVED_TOTAL: &str = "rustfs_s3_list_objects_index_served_total"; +const LIST_OBJECTS_INDEX_CANDIDATE_KEYS: &str = "rustfs_s3_list_objects_index_candidate_keys"; +const LIST_OBJECTS_INDEX_LIVE_VERIFY_ATTEMPTS: &str = "rustfs_s3_list_objects_index_live_verify_attempts"; +const LIST_OBJECTS_INDEX_LIVE_VERIFY_HITS: &str = "rustfs_s3_list_objects_index_live_verify_hits"; +const LIST_OBJECTS_INDEX_LIVE_VERIFY_MISSES: &str = "rustfs_s3_list_objects_index_live_verify_misses"; +const LIST_OBJECTS_INDEX_LIVE_VERIFY_FAILURE_TOTAL: &str = "rustfs_s3_list_objects_index_live_verify_failure_total"; +const LIST_OBJECTS_INDEX_RETURNED_OBJECTS: &str = "rustfs_s3_list_objects_index_returned_objects"; +const LIST_OBJECTS_INDEX_RETURNED_PREFIXES: &str = "rustfs_s3_list_objects_index_returned_prefixes"; +const LIST_OBJECTS_INDEX_VERIFICATION_IO_AMPLIFICATION: &str = "rustfs_s3_list_objects_index_verification_io_amplification"; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ListObjectsGatherObservation { + pub source: &'static str, + pub outcome: &'static str, + pub limit: i32, + pub scanned_entries: usize, + pub returned_entries: usize, + pub duration_ms: f64, + pub has_prefix: bool, + pub has_delimiter: bool, + pub has_marker: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ListObjectsIndexPageObservation { + pub source: &'static str, + pub provider: &'static str, + pub candidate_keys: usize, + pub live_verify_attempts: usize, + pub live_verify_hits: usize, + pub live_verify_misses: usize, + pub returned_objects: usize, + pub returned_prefixes: usize, + pub is_truncated: bool, +} + +#[inline(always)] +fn bool_label(value: bool) -> &'static str { + if value { "true" } else { "false" } +} + +#[inline(always)] +fn count_as_f64(value: usize) -> f64 { + value as f64 +} + +#[inline(always)] +fn limit_as_f64(value: i32) -> f64 { + value.max(0) as f64 +} + +pub fn init_list_objects_metrics() { + static METRICS_DESC_INIT: OnceLock<()> = OnceLock::new(); + METRICS_DESC_INIT.get_or_init(|| { + describe_counter!( + LIST_OBJECTS_GATHER_TOTAL, + "Total number of ListObjects gather phases by source, outcome, and request shape." + ); + describe_histogram!(LIST_OBJECTS_GATHER_DURATION_MS, "ListObjects gather phase duration in milliseconds."); + describe_histogram!( + LIST_OBJECTS_GATHER_SCANNED_ENTRIES, + "Number of entries consumed by ListObjects gather before producing a page." + ); + describe_histogram!( + LIST_OBJECTS_GATHER_RETURNED_ENTRIES, + "Number of entries returned by ListObjects gather before pagination trimming." + ); + describe_histogram!( + LIST_OBJECTS_GATHER_FILTERED_ENTRIES, + "Number of entries filtered by ListObjects gather before producing a page." + ); + describe_histogram!( + LIST_OBJECTS_GATHER_SCAN_AMPLIFICATION, + "Ratio of scanned entries to returned entries in ListObjects gather." + ); + describe_histogram!(LIST_OBJECTS_GATHER_LIMIT, "Requested internal ListObjects gather page limit."); + describe_histogram!(LIST_OBJECTS_MERGE_FAN_IN, "Number of input streams merged for ListObjects pages."); + describe_histogram!(LIST_OBJECTS_MERGE_READ_QUORUM, "Read quorum used while merging ListObjects entries."); + describe_counter!( + LIST_OBJECTS_INDEX_ATTEMPT_TOTAL, + "Total number of opt-in ListObjects index serving attempts by source, provider, and request shape." + ); + describe_counter!( + LIST_OBJECTS_INDEX_FALLBACK_TOTAL, + "Total number of opt-in ListObjects index attempts that fell back to the live walker." + ); + describe_counter!( + LIST_OBJECTS_INDEX_SERVED_TOTAL, + "Total number of ListObjects pages served by an opt-in index provider." + ); + describe_histogram!( + LIST_OBJECTS_INDEX_CANDIDATE_KEYS, + "Number of candidate keys proposed by the opt-in ListObjects index provider per page." + ); + describe_histogram!( + LIST_OBJECTS_INDEX_LIVE_VERIFY_ATTEMPTS, + "Number of live xl.meta verification attempts made by opt-in ListObjects index serving per page." + ); + describe_histogram!( + LIST_OBJECTS_INDEX_LIVE_VERIFY_HITS, + "Number of opt-in ListObjects index candidates accepted by live xl.meta verification per page." + ); + describe_histogram!( + LIST_OBJECTS_INDEX_LIVE_VERIFY_MISSES, + "Number of opt-in ListObjects index candidates rejected as stale or missing by live xl.meta verification per page." + ); + describe_counter!( + LIST_OBJECTS_INDEX_LIVE_VERIFY_FAILURE_TOTAL, + "Total number of opt-in ListObjects index live xl.meta verification failures." + ); + describe_histogram!( + LIST_OBJECTS_INDEX_RETURNED_OBJECTS, + "Number of objects returned by opt-in ListObjects index serving per page." + ); + describe_histogram!( + LIST_OBJECTS_INDEX_RETURNED_PREFIXES, + "Number of common prefixes returned by opt-in ListObjects index serving per page." + ); + describe_histogram!( + LIST_OBJECTS_INDEX_VERIFICATION_IO_AMPLIFICATION, + "Ratio of live verification attempts to returned objects for opt-in ListObjects index serving." + ); + }); +} + +pub fn record_list_objects_gather(observation: ListObjectsGatherObservation) { + if !get_stage_metrics_enabled() { + return; + } + + let filtered_entries = observation.scanned_entries.saturating_sub(observation.returned_entries); + let scan_amplification = if observation.returned_entries == 0 { + count_as_f64(observation.scanned_entries) + } else { + count_as_f64(observation.scanned_entries) / count_as_f64(observation.returned_entries) + }; + + counter!( + LIST_OBJECTS_GATHER_TOTAL, + "source" => observation.source, + "outcome" => observation.outcome, + "has_prefix" => bool_label(observation.has_prefix), + "has_delimiter" => bool_label(observation.has_delimiter), + "has_marker" => bool_label(observation.has_marker), + ) + .increment(1); + histogram!( + LIST_OBJECTS_GATHER_DURATION_MS, + "source" => observation.source, + "outcome" => observation.outcome + ) + .record(observation.duration_ms); + histogram!(LIST_OBJECTS_GATHER_SCANNED_ENTRIES, "source" => observation.source) + .record(count_as_f64(observation.scanned_entries)); + histogram!(LIST_OBJECTS_GATHER_RETURNED_ENTRIES, "source" => observation.source) + .record(count_as_f64(observation.returned_entries)); + histogram!(LIST_OBJECTS_GATHER_FILTERED_ENTRIES, "source" => observation.source).record(count_as_f64(filtered_entries)); + histogram!(LIST_OBJECTS_GATHER_SCAN_AMPLIFICATION, "source" => observation.source).record(scan_amplification); + histogram!(LIST_OBJECTS_GATHER_LIMIT, "source" => observation.source).record(limit_as_f64(observation.limit)); +} + +pub fn record_list_objects_merge(source: &'static str, input_channels: usize, read_quorum: usize) { + if !get_stage_metrics_enabled() { + return; + } + + histogram!( + LIST_OBJECTS_MERGE_FAN_IN, + "source" => source, + "outcome" => LIST_OBJECTS_MERGE_OUTCOME_STARTED + ) + .record(count_as_f64(input_channels)); + histogram!( + LIST_OBJECTS_MERGE_READ_QUORUM, + "source" => source, + "outcome" => LIST_OBJECTS_MERGE_OUTCOME_STARTED + ) + .record(count_as_f64(read_quorum)); +} + +pub fn record_list_objects_index_fallback(source: &'static str, reason: &'static str) { + if !get_stage_metrics_enabled() { + return; + } + + counter!( + LIST_OBJECTS_INDEX_FALLBACK_TOTAL, + "source" => source, + "reason" => reason + ) + .increment(1); +} + +pub fn record_list_objects_index_attempt( + source: &'static str, + provider: &'static str, + has_prefix: bool, + has_delimiter: bool, + has_marker: bool, +) { + if !get_stage_metrics_enabled() { + return; + } + + counter!( + LIST_OBJECTS_INDEX_ATTEMPT_TOTAL, + "source" => source, + "provider" => provider, + "has_prefix" => bool_label(has_prefix), + "has_delimiter" => bool_label(has_delimiter), + "has_marker" => bool_label(has_marker), + ) + .increment(1); +} + +pub fn record_list_objects_index_live_verify_failure(source: &'static str, reason: &'static str) { + if !get_stage_metrics_enabled() { + return; + } + + counter!( + LIST_OBJECTS_INDEX_LIVE_VERIFY_FAILURE_TOTAL, + "source" => source, + "reason" => reason, + ) + .increment(1); +} + +pub fn record_list_objects_index_served(observation: ListObjectsIndexPageObservation) { + if !get_stage_metrics_enabled() { + return; + } + + let returned_objects = count_as_f64(observation.returned_objects); + let verification_io_amplification = if observation.returned_objects == 0 { + count_as_f64(observation.live_verify_attempts) + } else { + count_as_f64(observation.live_verify_attempts) / returned_objects + }; + + counter!( + LIST_OBJECTS_INDEX_SERVED_TOTAL, + "source" => observation.source, + "provider" => observation.provider, + "is_truncated" => bool_label(observation.is_truncated), + ) + .increment(1); + histogram!( + LIST_OBJECTS_INDEX_CANDIDATE_KEYS, + "source" => observation.source, + "provider" => observation.provider, + ) + .record(count_as_f64(observation.candidate_keys)); + histogram!( + LIST_OBJECTS_INDEX_LIVE_VERIFY_ATTEMPTS, + "source" => observation.source, + "provider" => observation.provider, + ) + .record(count_as_f64(observation.live_verify_attempts)); + histogram!( + LIST_OBJECTS_INDEX_LIVE_VERIFY_HITS, + "source" => observation.source, + "provider" => observation.provider, + ) + .record(count_as_f64(observation.live_verify_hits)); + histogram!( + LIST_OBJECTS_INDEX_LIVE_VERIFY_MISSES, + "source" => observation.source, + "provider" => observation.provider, + ) + .record(count_as_f64(observation.live_verify_misses)); + histogram!( + LIST_OBJECTS_INDEX_RETURNED_OBJECTS, + "source" => observation.source, + "provider" => observation.provider, + ) + .record(returned_objects); + histogram!( + LIST_OBJECTS_INDEX_RETURNED_PREFIXES, + "source" => observation.source, + "provider" => observation.provider, + ) + .record(count_as_f64(observation.returned_prefixes)); + histogram!( + LIST_OBJECTS_INDEX_VERIFICATION_IO_AMPLIFICATION, + "source" => observation.source, + "provider" => observation.provider, + ) + .record(verification_io_amplification); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn record_gather_observation_handles_empty_page() { + init_list_objects_metrics(); + record_list_objects_gather(ListObjectsGatherObservation { + source: LIST_OBJECTS_SOURCE_WALKER, + outcome: LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED, + limit: 1001, + scanned_entries: 42, + returned_entries: 0, + duration_ms: 3.5, + has_prefix: true, + has_delimiter: false, + has_marker: true, + }); + } + + #[test] + fn record_merge_observation_accepts_zero_quorum() { + init_list_objects_metrics(); + record_list_objects_merge(LIST_OBJECTS_SOURCE_WALKER, 4, 0); + } + + #[test] + fn record_index_fallback_observation_accepts_reason() { + init_list_objects_metrics(); + record_list_objects_index_fallback("index_key_only", "unsupported_request"); + } + + #[test] + fn record_index_attempt_and_served_observations_accept_counts() { + init_list_objects_metrics(); + record_list_objects_index_attempt("index_key_only", "walker_key_only", true, true, false); + record_list_objects_index_served(ListObjectsIndexPageObservation { + source: "index_key_only", + provider: "walker_key_only", + candidate_keys: 1000, + live_verify_attempts: 700, + live_verify_hits: 650, + live_verify_misses: 50, + returned_objects: 600, + returned_prefixes: 10, + is_truncated: true, + }); + record_list_objects_index_live_verify_failure("index_key_only", "read_error"); + } +} diff --git a/rustfs/src/memory_observability.rs b/rustfs/src/memory_observability.rs index 01336027c..38193e862 100644 --- a/rustfs/src/memory_observability.rs +++ b/rustfs/src/memory_observability.rs @@ -13,11 +13,14 @@ // limitations under the License. use rustfs_io_metrics::{ - record_cgroup_memory_split, record_cpu_usage, record_memory_usage, record_process_memory_split, - snapshot_process_resource_and_system, + AllocatorMemoryObservation, record_allocator_memory_observation, record_cgroup_memory_split, record_cpu_usage, + record_memory_usage, record_process_memory_split, snapshot_process_resource_and_system, }; use serde::Serialize; +use serde_json::Value; use std::collections::HashMap; +#[cfg(not(target_os = "windows"))] +use std::ffi::CStr; use std::path::Path; use std::sync::{Mutex, OnceLock}; use std::time::Duration; @@ -127,6 +130,12 @@ struct CgroupMemorySnapshot { inactive_file_bytes: Option, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct AllocatorMemorySnapshot { + backend: &'static str, + observation: AllocatorMemoryObservation, +} + fn memory_system() -> &'static Mutex { MEMORY_SYSTEM.get_or_init(|| Mutex::new(System::new())) } @@ -204,6 +213,94 @@ fn read_cgroup_memory_snapshot() -> Option { read_cgroup_v2().or_else(read_cgroup_v1) } +fn numeric_json_value(value: &Value) -> Option { + match value { + Value::Number(number) => number + .as_u64() + .or_else(|| number.as_i64().and_then(|value| u64::try_from(value).ok())), + Value::String(value) => value.parse::().ok(), + _ => None, + } +} + +fn numeric_json_field(value: &Value, field: &str) -> Option { + match value { + Value::Object(fields) => fields + .get(field) + .and_then(numeric_json_value) + .or_else(|| fields.values().find_map(|value| numeric_json_field(value, field))), + Value::Array(values) => values.iter().find_map(|value| numeric_json_field(value, field)), + _ => None, + } +} + +fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option { + match value { + Value::Object(fields) => { + if let Some(metric_value) = fields.get(metric) + && let Some(value) = numeric_json_value(metric_value).or_else(|| numeric_json_field(metric_value, field)) + { + return Some(value); + } + + fields.values().find_map(|value| mimalloc_stat_field(value, metric, field)) + } + Value::Array(values) => values.iter().find_map(|value| mimalloc_stat_field(value, metric, field)), + _ => None, + } +} + +fn mimalloc_stat_current(value: &Value, metric: &str) -> Option { + mimalloc_stat_field(value, metric, "current") +} + +fn parse_mimalloc_stats_json(stats_json: &str) -> Option { + let value = serde_json::from_str::(stats_json).ok()?; + let observation = AllocatorMemoryObservation { + reserved_bytes: mimalloc_stat_current(&value, "reserved"), + committed_bytes: mimalloc_stat_current(&value, "committed"), + page_committed_bytes: mimalloc_stat_current(&value, "page_committed"), + malloc_requested_bytes: mimalloc_stat_current(&value, "malloc_requested"), + malloc_requested_peak_bytes: mimalloc_stat_field(&value, "malloc_requested", "peak"), + malloc_requested_total_bytes: mimalloc_stat_field(&value, "malloc_requested", "total"), + heap_count: mimalloc_stat_current(&value, "heaps").or_else(|| mimalloc_stat_current(&value, "heap_count")), + }; + + if observation == AllocatorMemoryObservation::default() { + None + } else { + Some(observation) + } +} + +#[cfg(not(target_os = "windows"))] +#[allow(unsafe_code)] +fn read_allocator_memory_snapshot() -> Option { + // SAFETY: `mi_stats_get_json` returns a null-terminated JSON buffer owned by + // mimalloc when called with a null input buffer. The mimalloc API requires + // freeing that buffer with `mi_free`, which is done before returning. + let stats = unsafe { + let stats_ptr = libmimalloc_sys::mi_stats_get_json(0, std::ptr::null_mut()); + if stats_ptr.is_null() { + return None; + } + + let stats = CStr::from_ptr(stats_ptr).to_str().ok().map(str::to_owned); + libmimalloc_sys::mi_free(stats_ptr.cast()); + stats? + }; + let observation = parse_mimalloc_stats_json(&stats)?; + Some(AllocatorMemorySnapshot { + backend: crate::allocator_reclaim::allocator_backend(), + observation, + }) +} + +#[cfg(target_os = "windows")] +fn read_allocator_memory_snapshot() -> Option { + None +} + fn configured_memory_observability_interval_secs() -> u64 { rustfs_utils::get_env_u64(ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS, DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS).max(1) } @@ -276,11 +373,12 @@ async fn record_memory_snapshot() { let (resource, process) = snapshot_process_resource_and_system(); let total_memory = refresh_total_memory(); let cgroup = read_cgroup_memory_snapshot(); - (resource, process, total_memory, cgroup) + let allocator = read_allocator_memory_snapshot(); + (resource, process, total_memory, cgroup, allocator) }) .await { - Ok((resource, process, total_memory, cgroup)) => { + Ok((resource, process, total_memory, cgroup, allocator)) => { record_memory_usage(process.resident_memory_bytes, total_memory); record_cpu_usage(resource.cpu_percent); record_process_memory_split(process.resident_memory_bytes, process.virtual_memory_bytes); @@ -295,6 +393,10 @@ async fn record_memory_snapshot() { cgroup.inactive_file_bytes, ); } + + if let Some(allocator) = allocator { + record_allocator_memory_observation(allocator.backend, allocator.observation); + } } Err(err) => { debug!(error = ?err, "memory observability sampler task failed"); @@ -330,7 +432,7 @@ mod tests { CgroupMemorySnapshot, MEMORY_OBSERVABILITY_SERVICE_NAME, MemoryObservabilityCancellationSource, MemoryObservabilityController, MemoryObservabilityDesiredState, MemoryObservabilityServiceState, MemoryObservabilityShutdownHandle, MemoryObservabilityWorkerMutation, build_memory_observability_controller_snapshot, - build_memory_observability_status_snapshot, parse_kv_stats, read_optional_u64, + build_memory_observability_status_snapshot, parse_kv_stats, parse_mimalloc_stats_json, read_optional_u64, }; use std::fs; use std::path::PathBuf; @@ -366,6 +468,39 @@ mod tests { assert_eq!(snapshot.inactive_file_bytes, None); } + #[test] + fn parse_mimalloc_stats_json_extracts_allocator_attribution() { + let parsed = parse_mimalloc_stats_json( + r#"{ + "process": { + "reserved": { "current": 1048576 }, + "committed": { "current": "524288" }, + "page_committed": { "current": 262144 }, + "malloc_requested": { + "current": 131072, + "peak": 196608, + "total": 10485760 + }, + "heaps": { "current": 8 } + } + }"#, + ) + .expect("mimalloc stats should parse"); + + assert_eq!(parsed.reserved_bytes, Some(1_048_576)); + assert_eq!(parsed.committed_bytes, Some(524_288)); + assert_eq!(parsed.page_committed_bytes, Some(262_144)); + assert_eq!(parsed.malloc_requested_bytes, Some(131_072)); + assert_eq!(parsed.malloc_requested_peak_bytes, Some(196_608)); + assert_eq!(parsed.malloc_requested_total_bytes, Some(10_485_760)); + assert_eq!(parsed.heap_count, Some(8)); + } + + #[test] + fn parse_mimalloc_stats_json_rejects_unrecognized_payload() { + assert_eq!(parse_mimalloc_stats_json(r#"{ "allocator": "unknown" }"#), None); + } + #[test] fn memory_observability_snapshot_reports_disabled_when_metrics_are_disabled() { let snapshot = build_memory_observability_status_snapshot(false, 15, false); diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 92b6798e7..d68745331 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -78,6 +78,7 @@ impl Default for FS { impl FS { pub fn new() -> Self { rustfs_io_metrics::init_s3_metrics(); + rustfs_io_metrics::init_list_objects_metrics(); Self {} } diff --git a/scripts/run_listobjects_verified_bench.sh b/scripts/run_listobjects_verified_bench.sh new file mode 100755 index 000000000..582fe4e1a --- /dev/null +++ b/scripts/run_listobjects_verified_bench.sh @@ -0,0 +1,1324 @@ +#!/usr/bin/env bash +set -euo pipefail + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/bench/listobjects-verified-$(date +%Y%m%d-%H%M%S)}" +DATA_ROOT="${DATA_ROOT:-/private/tmp/listobjects-verified-$(date +%Y%m%d-%H%M%S)}" +BUILD_PROFILE="${BUILD_PROFILE:-debug}" +RUSTFS_BIN_USER_SET="false" +if [[ -n "${RUSTFS_BIN:-}" ]]; then + RUSTFS_BIN_USER_SET="true" +else + RUSTFS_BIN="${PROJECT_ROOT}/target/${BUILD_PROFILE}/rustfs" +fi +WARP_BIN="${WARP_BIN:-warp}" +ADDRESS="${ADDRESS:-127.0.0.1:9000}" +ENDPOINT="${ENDPOINT:-http://${ADDRESS}}" +ACCESS_KEY="${ACCESS_KEY:-minioadmin}" +SECRET_KEY="${SECRET_KEY:-minioadmin}" +REGION="${REGION:-us-east-1}" +OBJECTS="${OBJECTS:-10000}" +OBJECT_SIZE="${OBJECT_SIZE:-1KB}" +CONCURRENCY="${CONCURRENCY:-16}" +MAX_KEYS="${MAX_KEYS:-1000}" +DURATION="${DURATION:-20s}" +WARMUP_DURATION="${WARMUP_DURATION:-5s}" +METER_INTERVAL="${METER_INTERVAL:-1}" +RESOURCE_SAMPLE_INTERVAL="${RESOURCE_SAMPLE_INTERVAL:-1}" +SKIP_BUILD="${SKIP_BUILD:-false}" +INDEX_PROVIDER="${INDEX_PROVIDER:-persistent_key_only}" +INDEX_GENERATION="${INDEX_GENERATION:-bench-$(date +%Y%m%d%H%M%S)}" +KEY_INDEX_PATH="${KEY_INDEX_PATH:-${OUT_DIR}/persistent-key-only.index}" +NAMESPACE_JOURNAL_PATH="${NAMESPACE_JOURNAL_PATH:-}" +PREBUILD_KEY_INDEX="${PREBUILD_KEY_INDEX:-false}" +PREPARE_ONLY="${PREPARE_ONLY:-false}" +REUSE_PREPARED_DATA="${REUSE_PREPARED_DATA:-false}" +RUN_METADATA_FAST="${RUN_METADATA_FAST:-true}" +RUN_CHAOS="${RUN_CHAOS:-true}" +RUN_QUORUM_JOURNAL_CHAOS="${RUN_QUORUM_JOURNAL_CHAOS:-true}" +CHAOS_MAX_KEYS="${CHAOS_MAX_KEYS:-3}" +METADATA_FAST_STALENESS_MS="${METADATA_FAST_STALENESS_MS:-5000}" +PROMETHEUS_QUERY_URL="${PROMETHEUS_QUERY_URL:-http://localhost:9090/api/v1/query}" +PROMETHEUS_JOB="${PROMETHEUS_JOB:-rustfs-app-metrics}" +JOURNAL_CHAOS_ENABLED="" +JOURNAL_CHAOS_BUCKET="" +JOURNAL_CHAOS_SEQUENCE="" +JOURNAL_CHAOS_STATUS="" + +SERVER_PID="" +RESOURCE_SAMPLER_PID="" + +log_info() { printf '[INFO] %s\n' "$*"; } +log_warn() { printf '[WARN] %s\n' "$*" >&2; } +die() { + printf '[ERROR] %s\n' "$*" >&2 + exit 1 +} + +usage() { + cat <<'USAGE' +Usage: + scripts/run_listobjects_verified_bench.sh [options] + +Options: + --out-dir Output directory. + --data-root RustFS data root for d1..d4. + --rustfs-bin RustFS binary path. + --warp-bin warp binary path. + --objects Objects prepared by warp list (default: 10000). + --duration Timed benchmark duration (default: 20s). + --warmup-duration Warmup duration before timed run (default: 5s). + --concurrency warp --concurrent value (default: 16). + --max-keys warp --max-keys value (default: 1000). + --release Build and run target/release/rustfs. + --skip-build Use existing RustFS binary. + --index-provider Opt-in provider (default: persistent_key_only). + --key-index-path

Persistent key-only index path. + --journal-path

Optional local namespace mutation journal override path. + --prebuild-key-index Build persistent key-only index from the bench script. + --prepare-only Prepare bucket data and exit without running modes. + --reuse-prepared-data Reuse --data-root and skip warp data preparation. + --skip-metadata-fast Skip metadata-fast benchmark mode. + --skip-chaos Skip metadata-fast stale/fallback chaos probes. + --skip-quorum-journal-chaos + Skip quorum-backed .rustfs.sys degraded/restore probe. + --chaos-max-keys Metadata-fast chaos page size (default: 3). + --metadata-fast-staleness-ms + Metadata-fast staleness SLA in milliseconds (default: 5000). + --prometheus-query-url + Prometheus API query URL or UI /query URL. + -h, --help Show this help. + +Environment: + OUT_DIR, DATA_ROOT, RUSTFS_BIN, WARP_BIN, ADDRESS, ACCESS_KEY, SECRET_KEY, + REGION, OBJECTS, OBJECT_SIZE, CONCURRENCY, MAX_KEYS, DURATION, + WARMUP_DURATION, METER_INTERVAL, RESOURCE_SAMPLE_INTERVAL, SKIP_BUILD, + BUILD_PROFILE, INDEX_PROVIDER, INDEX_GENERATION, KEY_INDEX_PATH, + NAMESPACE_JOURNAL_PATH, PREBUILD_KEY_INDEX, PREPARE_ONLY, REUSE_PREPARED_DATA, + RUN_METADATA_FAST, RUN_CHAOS, RUN_QUORUM_JOURNAL_CHAOS, CHAOS_MAX_KEYS, + METADATA_FAST_STALENESS_MS, PROMETHEUS_QUERY_URL, PROMETHEUS_JOB. +USAGE +} + +arg_value() { + local flag="$1" + local value="${2:-}" + [[ -n "$value" ]] || die "missing value for $flag" + printf '%s\n' "$value" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --data-root) DATA_ROOT="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --rustfs-bin) RUSTFS_BIN="$(arg_value "$1" "${2:-}")"; RUSTFS_BIN_USER_SET="true"; shift 2 ;; + --warp-bin) WARP_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --objects) OBJECTS="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --duration) DURATION="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --warmup-duration) WARMUP_DURATION="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --concurrency) CONCURRENCY="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --max-keys) MAX_KEYS="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --release) + BUILD_PROFILE="release" + if [[ "$RUSTFS_BIN_USER_SET" == "false" ]]; then + RUSTFS_BIN="${PROJECT_ROOT}/target/release/rustfs" + fi + shift + ;; + --skip-build) SKIP_BUILD="true"; shift ;; + --index-provider) INDEX_PROVIDER="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --key-index-path) KEY_INDEX_PATH="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --journal-path) NAMESPACE_JOURNAL_PATH="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --prebuild-key-index) PREBUILD_KEY_INDEX="true"; shift ;; + --prepare-only) PREPARE_ONLY="true"; shift ;; + --reuse-prepared-data) REUSE_PREPARED_DATA="true"; shift ;; + --skip-metadata-fast) RUN_METADATA_FAST="false"; shift ;; + --skip-chaos) RUN_CHAOS="false"; shift ;; + --skip-quorum-journal-chaos) RUN_QUORUM_JOURNAL_CHAOS="false"; shift ;; + --chaos-max-keys) CHAOS_MAX_KEYS="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --metadata-fast-staleness-ms) METADATA_FAST_STALENESS_MS="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --prometheus-query-url) PROMETHEUS_QUERY_URL="$(arg_value "$1" "${2:-}")"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) die "unknown arg: $1" ;; + esac + done +} + +require_cmds() { + command -v "$WARP_BIN" >/dev/null 2>&1 || die "warp binary not found: $WARP_BIN" + command -v curl >/dev/null 2>&1 || die "curl not found" + command -v python3 >/dev/null 2>&1 || die "python3 not found" + command -v awk >/dev/null 2>&1 || die "awk not found" + command -v sed >/dev/null 2>&1 || die "sed not found" +} + +build_rustfs() { + if [[ "$SKIP_BUILD" == "true" ]]; then + [[ -x "$RUSTFS_BIN" ]] || die "RustFS binary is not executable: $RUSTFS_BIN" + return + fi + if [[ "$BUILD_PROFILE" == "release" ]]; then + (cd "$PROJECT_ROOT" && cargo build -p rustfs --release) + else + (cd "$PROJECT_ROOT" && cargo build -p rustfs) + fi + [[ -x "$RUSTFS_BIN" ]] || die "RustFS binary is not executable after build: $RUSTFS_BIN" +} + +stop_server() { + stop_resource_sampler + if [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" >/dev/null 2>&1; then + kill "$SERVER_PID" >/dev/null 2>&1 || true + wait "$SERVER_PID" >/dev/null 2>&1 || true + fi + SERVER_PID="" +} + +stop_resource_sampler() { + if [[ -n "$RESOURCE_SAMPLER_PID" ]] && kill -0 "$RESOURCE_SAMPLER_PID" >/dev/null 2>&1; then + kill "$RESOURCE_SAMPLER_PID" >/dev/null 2>&1 || true + wait "$RESOURCE_SAMPLER_PID" >/dev/null 2>&1 || true + fi + RESOURCE_SAMPLER_PID="" +} + +start_resource_sampler() { + local out_file="$1" + local pid="$SERVER_PID" + echo "unix_ts,pid,cpu_percent,rss_kib" >"$out_file" + ( + while kill -0 "$pid" >/dev/null 2>&1; do + ps -p "$pid" -o pid=,%cpu=,rss= | awk -v ts="$(date +%s)" '{printf "%s,%s,%s,%s\n", ts, $1, $2, $3}' + sleep "$RESOURCE_SAMPLE_INTERVAL" + done + ) >>"$out_file" 2>/dev/null & + RESOURCE_SAMPLER_PID="$!" +} + +wait_for_health() { + local log_file="$1" + local deadline=$((SECONDS + 60)) + while (( SECONDS < deadline )); do + if curl -fsS "${ENDPOINT}/health" >/dev/null 2>&1 || curl -fsS "${ENDPOINT}/health/ready" >/dev/null 2>&1; then + return 0 + fi + if [[ -n "$SERVER_PID" ]] && ! kill -0 "$SERVER_PID" >/dev/null 2>&1; then + tail -n 80 "$log_file" >&2 || true + die "RustFS exited before health check passed" + fi + sleep 1 + done + tail -n 80 "$log_file" >&2 || true + die "RustFS health check timed out" +} + +start_server() { + local mode="$1" + local log_file="$2" + local reset_data="${3:-false}" + + stop_server + if [[ "$reset_data" == "true" ]]; then + rm -rf "$DATA_ROOT" + fi + mkdir -p "$DATA_ROOT"/d{1,2,3,4} + + ( + export RUST_LOG=off + export RUSTFS_OBS_LOGGER_LEVEL=off + export RUSTFS_OBS_TRACES_EXPORT_ENABLED=false + export RUSTFS_OBS_METRICS_EXPORT_ENABLED=true + export RUSTFS_OBS_LOGS_EXPORT_ENABLED=false + export RUSTFS_OBS_PROFILING_EXPORT_ENABLED=false + export RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318 + export RUSTFS_OBS_METRIC_ENDPOINT=http://127.0.0.1:4318/v1/metrics + export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://127.0.0.1:4318/v1/metrics + export RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS=200 + export RUSTFS_OBS_USE_STDOUT=true + export RUSTFS_OBS_METER_INTERVAL="$METER_INTERVAL" + export RUSTFS_OBS_LOG_STDOUT_ENABLED=false + export RUSTFS_SCANNER_ENABLED=false + export RUSTFS_CONSOLE_ENABLE=false + export RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true + export RUSTFS_ADDRESS="$ADDRESS" + export RUSTFS_ACCESS_KEY="$ACCESS_KEY" + export RUSTFS_SECRET_KEY="$SECRET_KEY" + export RUSTFS_RPC_SECRET="rustfs-listobjects-verified-rpc-secret" + export RUSTFS_REGION="$REGION" + unset RUSTFS_LIST_OBJECTS_INDEX_MODE + unset RUSTFS_LIST_OBJECTS_INDEX_PROVIDER + unset RUSTFS_LIST_OBJECTS_INDEX_PROVIDER_PATH + unset RUSTFS_LIST_OBJECTS_INDEX_PROVIDER_GENERATION + unset RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_PATH + unset RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED + unset RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET + unset RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_SEQUENCE + unset RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS + unset RUSTFS_LIST_OBJECTS_METADATA_FAST_ENABLED + unset RUSTFS_LIST_OBJECTS_METADATA_FAST_STALENESS_MS + if [[ "$mode" == "opt_in_verified" || "$mode" == "metadata_fast" ]]; then + export RUSTFS_LIST_OBJECTS_INDEX_MODE=index_key_only + if [[ "$mode" == "metadata_fast" ]]; then + export RUSTFS_LIST_OBJECTS_INDEX_MODE=index_metadata_fast + export RUSTFS_LIST_OBJECTS_METADATA_FAST_ENABLED=true + export RUSTFS_LIST_OBJECTS_METADATA_FAST_STALENESS_MS="$METADATA_FAST_STALENESS_MS" + fi + export RUSTFS_LIST_OBJECTS_INDEX_PROVIDER="$INDEX_PROVIDER" + if [[ "$INDEX_PROVIDER" == "persistent_key_only" || "$INDEX_PROVIDER" == "persisted_key_only" ]]; then + export RUSTFS_LIST_OBJECTS_INDEX_PROVIDER_PATH="$KEY_INDEX_PATH" + export RUSTFS_LIST_OBJECTS_INDEX_PROVIDER_GENERATION="$INDEX_GENERATION" + if [[ -n "$NAMESPACE_JOURNAL_PATH" ]]; then + export RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_PATH="$NAMESPACE_JOURNAL_PATH" + fi + if [[ -n "$JOURNAL_CHAOS_BUCKET" && -n "$JOURNAL_CHAOS_STATUS" ]]; then + if [[ -n "$JOURNAL_CHAOS_ENABLED" ]]; then + export RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED="$JOURNAL_CHAOS_ENABLED" + fi + export RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET="$JOURNAL_CHAOS_BUCKET" + export RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS="$JOURNAL_CHAOS_STATUS" + if [[ -n "$JOURNAL_CHAOS_SEQUENCE" ]]; then + export RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_SEQUENCE="$JOURNAL_CHAOS_SEQUENCE" + fi + fi + fi + fi + exec "$RUSTFS_BIN" server \ + "${DATA_ROOT}/d1" \ + "${DATA_ROOT}/d2" \ + "${DATA_ROOT}/d3" \ + "${DATA_ROOT}/d4" + ) >"$log_file" 2>&1 & + + SERVER_PID="$!" + wait_for_health "$log_file" +} + +prepare_bucket() { + local duration="$2" + local log_file="$3" + local bucket="$1" + + "$WARP_BIN" list \ + --host "$ADDRESS" \ + --access-key "$ACCESS_KEY" \ + --secret-key "$SECRET_KEY" \ + --region "$REGION" \ + --bucket "$bucket" \ + --objects "$OBJECTS" \ + --obj.size "$OBJECT_SIZE" \ + --concurrent "$CONCURRENCY" \ + --max-keys "$MAX_KEYS" \ + --duration "$duration" \ + --noclear \ + --no-color \ + >"$log_file" 2>&1 +} + +write_key_index() { + local bucket="$1" + local index_path="$2" + local index_dir tmp_keys tmp_objects sorted_keys tmp_index token page response_file next_token key_count + index_dir="$(dirname "$index_path")" + tmp_keys="${index_path}.keys.tmp" + tmp_objects="${index_path}.objects.tmp" + sorted_keys="${index_path}.keys.sorted.tmp" + tmp_index="${index_path}.tmp" + mkdir -p "$index_dir" + : >"$tmp_keys" + : >"$tmp_objects" + + token="" + page=0 + while true; do + local url="${ENDPOINT}/${bucket}?list-type=2&max-keys=${MAX_KEYS}" + if [[ -n "$token" ]]; then + local encoded_token + encoded_token="$(printf '%s' "$token" | urlencode)" + url="${url}&continuation-token=${encoded_token}" + fi + + response_file="${index_path}.page-${page}.xml" + curl -fsS \ + --user "${ACCESS_KEY}:${SECRET_KEY}" \ + --aws-sigv4 "aws:amz:${REGION}:s3" \ + -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ + -o "$response_file" \ + "$url" + extract_xml_keys <"$response_file" >>"$tmp_keys" || true + extract_metadata_object_rows <"$response_file" >>"$tmp_objects" || true + next_token="$(extract_next_token <"$response_file" || true)" + rm -f "$response_file" + if [[ -z "$next_token" ]]; then + break + fi + token="$next_token" + page=$((page + 1)) + done + + LC_ALL=C sort -u "$tmp_keys" >"$sorted_keys" + key_count="$(awk 'END{print NR + 0}' "$sorted_keys")" + { + echo "# rustfs-listobjects-key-only-v1" + echo "# bucket=${bucket}" + echo "# generation=${INDEX_GENERATION}" + echo "# checkpoint_high_water_mark=${key_count}" + cat "$sorted_keys" + cat "$tmp_objects" + } >"$tmp_index" + mv "$tmp_index" "$index_path" + rm -f "$tmp_keys" "$tmp_objects" "$sorted_keys" +} + +extract_xml_keys() { + python3 -c 'import sys, xml.etree.ElementTree as ET +root = ET.fromstring(sys.stdin.read()) +for elem in root.iter(): + if elem.tag.rsplit("}", 1)[-1] == "Key" and elem.text: + print(elem.text)' +} + +extract_next_token() { + python3 -c 'import sys, xml.etree.ElementTree as ET +root = ET.fromstring(sys.stdin.read()) +for elem in root.iter(): + if elem.tag.rsplit("}", 1)[-1] == "NextContinuationToken" and elem.text: + print(elem.text) + break' +} + +xml_tag_text() { + local tag="$1" + python3 -c 'import sys, xml.etree.ElementTree as ET +tag = sys.argv[1] +root = ET.fromstring(sys.stdin.read()) +for elem in root.iter(): + if elem.tag.rsplit("}", 1)[-1] == tag and elem.text: + print(elem.text) + break' "$tag" +} + +extract_metadata_object_rows() { + python3 -c 'import base64, datetime, sys, xml.etree.ElementTree as ET +def local_name(tag): + return tag.rsplit("}", 1)[-1] +def text(parent, name): + for child in parent: + if local_name(child.tag) == name: + return child.text or "" + return "" +def enc(value): + if not value: + return "-" + return base64.b64encode(value.encode("utf-8")).decode("ascii") +def nanos(value): + if not value: + return "-" + try: + dt = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + return str(int(dt.timestamp() * 1_000_000_000)) + except ValueError: + return "-" +root = ET.fromstring(sys.stdin.read()) +for contents in root.iter(): + if local_name(contents.tag) != "Contents": + continue + key = text(contents, "Key") + if not key: + continue + size = text(contents, "Size") or "0" + etag = text(contents, "ETag").strip("\"") + storage_class = text(contents, "StorageClass") + print("# object\t{}\t{}\t{}\t{}\t{}".format( + enc(key), + size, + nanos(text(contents, "LastModified")), + enc(etag), + enc(storage_class), + ))' +} + +signed_list_once() { + local bucket="$1" + local response_file="$2" + local max_keys="${3:-1}" + signed_list_page "$bucket" "$response_file" "$max_keys" "" +} + +signed_list_page() { + local bucket="$1" + local response_file="$2" + local max_keys="${3:-1}" + local token="${4:-}" + local url="${ENDPOINT}/${bucket}?list-type=2&max-keys=${max_keys}" + if [[ -n "$token" ]]; then + local encoded_token + encoded_token="$(printf '%s' "$token" | urlencode)" + url="${url}&continuation-token=${encoded_token}" + fi + curl -fsS \ + --user "${ACCESS_KEY}:${SECRET_KEY}" \ + --aws-sigv4 "aws:amz:${REGION}:s3" \ + -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ + -o "$response_file" \ + "$url" +} + +urlencode_path() { + python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe="/-_.~"))' "$1" +} + +signed_put_object() { + local bucket="$1" + local key="$2" + local body_file="$3" + local out_file="$4" + local encoded_key + encoded_key="$(urlencode_path "$key")" + curl -fsS \ + --request PUT \ + --user "${ACCESS_KEY}:${SECRET_KEY}" \ + --aws-sigv4 "aws:amz:${REGION}:s3" \ + -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ + --upload-file "$body_file" \ + -o "$out_file" \ + "${ENDPOINT}/${bucket}/${encoded_key}" +} + +signed_delete_object() { + local bucket="$1" + local key="$2" + local out_file="$3" + local encoded_key + encoded_key="$(urlencode_path "$key")" + curl -fsS \ + --request DELETE \ + --user "${ACCESS_KEY}:${SECRET_KEY}" \ + --aws-sigv4 "aws:amz:${REGION}:s3" \ + -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ + -o "$out_file" \ + "${ENDPOINT}/${bucket}/${encoded_key}" +} + +response_etag() { + awk 'BEGIN{IGNORECASE=1} /^etag:/ {gsub("\r", "", $0); sub("^[^:]*:[[:space:]]*", "", $0); print $0; exit}' "$1" +} + +signed_multipart_complete() { + local bucket="$1" + local key="$2" + local work_dir="$3" + local encoded_key upload_xml upload_id encoded_upload_id part1_file part2_file part1_headers part2_headers etag1 etag2 complete_xml + encoded_key="$(urlencode_path "$key")" + upload_xml="$work_dir/multipart-init.xml" + + curl -fsS \ + --request POST \ + --user "${ACCESS_KEY}:${SECRET_KEY}" \ + --aws-sigv4 "aws:amz:${REGION}:s3" \ + -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ + -o "$upload_xml" \ + "${ENDPOINT}/${bucket}/${encoded_key}?uploads" + upload_id="$(xml_tag_text UploadId <"$upload_xml")" + [[ -n "$upload_id" ]] || die "multipart initiate did not return UploadId" + encoded_upload_id="$(printf '%s' "$upload_id" | urlencode)" + + part1_file="$work_dir/multipart-part-1.bin" + part2_file="$work_dir/multipart-part-2.bin" + part1_headers="$work_dir/multipart-part-1.headers" + part2_headers="$work_dir/multipart-part-2.headers" + python3 -c 'from pathlib import Path; Path(__import__("sys").argv[1]).write_bytes(b"a" * (5 * 1024 * 1024)); Path(__import__("sys").argv[2]).write_bytes(b"b")' "$part1_file" "$part2_file" + + curl -fsS \ + --request PUT \ + --user "${ACCESS_KEY}:${SECRET_KEY}" \ + --aws-sigv4 "aws:amz:${REGION}:s3" \ + -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ + --upload-file "$part1_file" \ + -D "$part1_headers" \ + -o "$work_dir/multipart-part-1.out" \ + "${ENDPOINT}/${bucket}/${encoded_key}?partNumber=1&uploadId=${encoded_upload_id}" + curl -fsS \ + --request PUT \ + --user "${ACCESS_KEY}:${SECRET_KEY}" \ + --aws-sigv4 "aws:amz:${REGION}:s3" \ + -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ + --upload-file "$part2_file" \ + -D "$part2_headers" \ + -o "$work_dir/multipart-part-2.out" \ + "${ENDPOINT}/${bucket}/${encoded_key}?partNumber=2&uploadId=${encoded_upload_id}" + + etag1="$(response_etag "$part1_headers")" + etag2="$(response_etag "$part2_headers")" + [[ -n "$etag1" && -n "$etag2" ]] || die "multipart part upload did not return ETags" + complete_xml="$work_dir/multipart-complete.xml" + { + echo "" + echo " 1${etag1}" + echo " 2${etag2}" + echo "" + } >"$complete_xml" + + curl -fsS \ + --request POST \ + --user "${ACCESS_KEY}:${SECRET_KEY}" \ + --aws-sigv4 "aws:amz:${REGION}:s3" \ + -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ + -H "Content-Type: application/xml" \ + --data-binary "@${complete_xml}" \ + -o "$work_dir/multipart-complete.out" \ + "${ENDPOINT}/${bucket}/${encoded_key}?uploadId=${encoded_upload_id}" +} + +urlencode() { + python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.stdin.read().strip(), safe=""))' +} + +base64_token() { + python3 -c 'import base64, sys; print(base64.b64encode(sys.stdin.read().encode("utf-8")).decode("ascii"))' +} + +percentile_summary() { + local values_file="$1" + python3 - "$values_file" <<'PY' +import math +import statistics +import sys + +path = sys.argv[1] +values = [] +with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + values.append(float(line)) + +if not values: + print("0,N/A,N/A,N/A,N/A") + raise SystemExit(0) + +values.sort() +def percentile(p: float) -> float: + if len(values) == 1: + return values[0] + rank = math.ceil((p / 100.0) * len(values)) - 1 + rank = min(max(rank, 0), len(values) - 1) + return values[rank] + +print( + f"{len(values)}," + f"{statistics.fmean(values):.6f}," + f"{percentile(50):.6f}," + f"{percentile(95):.6f}," + f"{percentile(99):.6f}" +) +PY +} + +run_signed_list_bench() { + local mode="$1" + local bucket="$2" + local duration="$3" + local out_dir="$4" + local deadline=$((SECONDS + ${duration%s})) + local latencies_file="$out_dir/list-latencies-ms.txt" + local request_log="$out_dir/list-requests.csv" + local total_keys=0 + local total_requests=0 + local iterations=0 + + : >"$latencies_file" + echo "iteration,request,keys,latency_ms,truncated" >"$request_log" + + while (( SECONDS < deadline )); do + local token="" + local request_in_iteration=0 + iterations=$((iterations + 1)) + + while (( SECONDS < deadline )); do + local url="${ENDPOINT}/${bucket}?list-type=2&max-keys=${MAX_KEYS}" + if [[ -n "$token" ]]; then + local encoded_token + encoded_token="$(printf '%s' "$token" | urlencode)" + url="${url}&continuation-token=${encoded_token}" + fi + + local response_file time_total status key_count next_token truncated + response_file="$out_dir/response-${mode}-${iterations}-${request_in_iteration}.xml" + time_total="$(curl -fsS \ + --user "${ACCESS_KEY}:${SECRET_KEY}" \ + --aws-sigv4 "aws:amz:${REGION}:s3" \ + -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ + -w '%{time_total}' \ + -o "$response_file" \ + "$url")" + status=$? + if [[ "$status" -ne 0 ]]; then + log_warn "curl list failed for mode=$mode status=$status url=$url" + return 1 + fi + + key_count="$(extract_xml_keys <"$response_file" | grep -cv '^$' || true)" + next_token="$(extract_next_token <"$response_file" || true)" + truncated="false" + [[ -n "$next_token" ]] && truncated="true" + + awk -v sec="$time_total" 'BEGIN{printf "%.6f\n", sec * 1000.0}' >>"$latencies_file" + echo "$iterations,$request_in_iteration,$key_count,$(awk -v sec="$time_total" 'BEGIN{printf "%.6f", sec * 1000.0}'),$truncated" >>"$request_log" + + total_keys=$((total_keys + key_count)) + total_requests=$((total_requests + 1)) + request_in_iteration=$((request_in_iteration + 1)) + + if [[ -z "$next_token" ]]; then + break + fi + token="$next_token" + done + done + + local percentile_line req_count avg_ms p50_ms p95_ms p99_ms objps + percentile_line="$(percentile_summary "$latencies_file")" + IFS=',' read -r req_count avg_ms p50_ms p95_ms p99_ms <<<"$percentile_line" + objps="$(awk -v keys="$total_keys" -v duration="${duration%s}" 'BEGIN{if(duration<=0) print "N/A"; else printf "%.6f", keys/duration}')" + + { + echo "mode,objects,object_size,max_keys,duration,requests,scan_iterations,total_keys,objps,latency_avg_ms,p50_ms,p95_ms,p99_ms" + echo "$mode,$OBJECTS,$OBJECT_SIZE,$MAX_KEYS,$duration,$req_count,$iterations,$total_keys,$objps,$avg_ms,$p50_ms,$p95_ms,$p99_ms" + } >"$out_dir/list-summary.csv" +} + +last_metric_field_sum() { + local metric="$1" + local field="$2" + local log_file="$3" + awk -v metric="$metric" -v field="$field" ' + function finish_block() { + if (found) { + snapshot_total += sum + snapshot_seen = 1 + found=0 + sum=0 + } + } + function finish_snapshot() { + finish_block() + if (snapshot_seen) { + value=snapshot_total + } + snapshot_total=0 + snapshot_seen=0 + } + /^Metric #0$/ { + finish_snapshot() + } + /^Metric #[0-9]+/ { + finish_block() + } + $0 ~ "Name[[:space:]]*:[[:space:]]*" metric "$" { found=1; sum=0; next } + found && $0 ~ "^[[:space:]]*" field "[[:space:]]*:" { + line=$0 + sub("^[[:space:]]*" field "[[:space:]]*:[[:space:]]*", "", line) + gsub(/[^0-9.eE+-]/, "", line) + sum += line + 0 + } + END { + finish_snapshot() + if (value == "") print "0"; else print value + } + ' "$log_file" +} + +last_metric_value() { + local metric="$1" + local log_file="$2" + last_metric_field_sum "$metric" "Value" "$log_file" +} + +last_metric_sum() { + local metric="$1" + local log_file="$2" + last_metric_field_sum "$metric" "Sum" "$log_file" +} + +last_metric_count() { + local metric="$1" + local log_file="$2" + last_metric_field_sum "$metric" "Count" "$log_file" +} + +write_ratio_summary() { + local mode="$1" + local log_file="$2" + local csv_file="$3" + local attempt fallback served verify_attempts verify_misses returned_objects amplification_sum amplification_count amplification stale_ratio fallback_ratio + + attempt="$(last_metric_value rustfs_s3_list_objects_index_attempt_total "$log_file")" + fallback="$(last_metric_value rustfs_s3_list_objects_index_fallback_total "$log_file")" + served="$(last_metric_value rustfs_s3_list_objects_index_served_total "$log_file")" + verify_attempts="$(last_metric_sum rustfs_s3_list_objects_index_live_verify_attempts "$log_file")" + verify_misses="$(last_metric_sum rustfs_s3_list_objects_index_live_verify_misses "$log_file")" + returned_objects="$(last_metric_sum rustfs_s3_list_objects_index_returned_objects "$log_file")" + amplification_sum="$(last_metric_sum rustfs_s3_list_objects_index_verification_io_amplification "$log_file")" + amplification_count="$(last_metric_count rustfs_s3_list_objects_index_verification_io_amplification "$log_file")" + amplification="$(awk -v sum="$amplification_sum" -v count="$amplification_count" 'BEGIN{if(count+0==0) print "0"; else printf "%.6f", (sum+0)/(count+0)}')" + + fallback_ratio="$(awk -v f="$fallback" -v a="$attempt" 'BEGIN{if(a+0==0) print "N/A"; else printf "%.6f", (f+0)/(a+0)}')" + stale_ratio="$(awk -v m="$verify_misses" -v a="$verify_attempts" 'BEGIN{if(a+0==0) print "N/A"; else printf "%.6f", (m+0)/(a+0)}')" + + { + echo "mode,index_attempt_total,index_fallback_total,index_served_total,live_verify_attempts,live_verify_misses,returned_objects,verification_io_amplification,fallback_ratio,stale_missing_candidate_ratio" + echo "$mode,$attempt,$fallback,$served,$verify_attempts,$verify_misses,$returned_objects,$amplification,$fallback_ratio,$stale_ratio" + } >"$csv_file" +} + +write_fallback_reason_summary() { + local mode="$1" + local log_file="$2" + local csv_file="$3" + + awk -v mode="$mode" ' + BEGIN { + print "mode,reason,index_fallback_total" + } + /Name[[:space:]]*: rustfs_s3_list_objects_index_fallback_total/ { + in_metric = 1 + value = "" + next + } + in_metric && /Name[[:space:]]*:/ && $0 !~ /rustfs_s3_list_objects_index_fallback_total/ { + in_metric = 0 + value = "" + next + } + in_metric && /Value[[:space:]]*:/ { + line = $0 + sub(/^.*Value[[:space:]]*:[[:space:]]*/, "", line) + split(line, parts, /[[:space:]]+/) + value = parts[1] + next + } + in_metric && /->[[:space:]]*reason:/ { + line = $0 + sub(/^.*->[[:space:]]*reason:[[:space:]]*/, "", line) + split(line, parts, /[[:space:]]+/) + reason = parts[1] + last[reason] = value + 0 + seen[reason] = 1 + value = "" + next + } + END { + count = 0 + for (reason in seen) { + printf "%s,%s,%.0f\n", mode, reason, last[reason] + count += 1 + } + if (count == 0) { + printf "%s,N/A,0\n", mode + } + } + ' "$log_file" >"$csv_file" +} + +prometheus_api_query_url() { + local url="$PROMETHEUS_QUERY_URL" + if [[ "$url" == */api/v1/query ]]; then + printf '%s\n' "$url" + elif [[ "$url" == */query ]]; then + printf '%s/api/v1/query\n' "${url%/query}" + else + printf '%s\n' "$url" + fi +} + +query_prometheus_expr() { + local expr="$1" + local out_file="$2" + local api_url + api_url="$(prometheus_api_query_url)" + curl -fsS --get --data-urlencode "query=${expr}" "$api_url" -o "$out_file" +} + +write_prometheus_snapshot() { + local mode="$1" + local out_dir="$2" + local prom_dir="$out_dir/prometheus" + mkdir -p "$prom_dir" + + query_prometheus_expr "sum(rustfs_s3_list_objects_index_attempt_total)" "$prom_dir/index_attempt_total.json" || { + log_warn "Prometheus query failed; skipping Prometheus snapshot for mode=$mode" + return 0 + } + query_prometheus_expr "sum(rustfs_s3_list_objects_index_fallback_total)" "$prom_dir/index_fallback_total.json" || true + query_prometheus_expr "sum(rustfs_s3_list_objects_index_served_total)" "$prom_dir/index_served_total.json" || true + query_prometheus_expr "sum(rustfs_s3_list_objects_index_live_verify_attempts_sum)" "$prom_dir/live_verify_attempts_sum.json" || true + query_prometheus_expr "sum(rustfs_s3_list_objects_index_live_verify_misses_sum)" "$prom_dir/live_verify_misses_sum.json" || true + query_prometheus_expr "sum(rustfs_s3_list_objects_index_returned_objects_sum)" "$prom_dir/returned_objects_sum.json" || true + query_prometheus_expr "sum(rustfs_s3_list_objects_index_verification_io_amplification_sum) / sum(rustfs_s3_list_objects_index_verification_io_amplification_count)" "$prom_dir/verification_io_amplification_avg.json" || true + query_prometheus_expr "rate(process_cpu_seconds_total{job=\"${PROMETHEUS_JOB}\"}[1m])" "$prom_dir/process_cpu_rate_1m.json" || true + query_prometheus_expr "process_resident_memory_bytes{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/process_resident_memory_bytes.json" || true + query_prometheus_expr "rustfs_memory_allocator_reserved_bytes{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/allocator_reserved_bytes.json" || true + query_prometheus_expr "rustfs_memory_allocator_committed_bytes{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/allocator_committed_bytes.json" || true + query_prometheus_expr "rustfs_memory_allocator_page_committed_bytes{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/allocator_page_committed_bytes.json" || true + query_prometheus_expr "rustfs_memory_allocator_malloc_requested_bytes{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/allocator_malloc_requested_bytes.json" || true + query_prometheus_expr "rustfs_memory_allocator_malloc_requested_peak_bytes{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/allocator_malloc_requested_peak_bytes.json" || true + query_prometheus_expr "rustfs_memory_allocator_malloc_requested_total_bytes{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/allocator_malloc_requested_total_bytes.json" || true + query_prometheus_expr "rustfs_memory_allocator_heap_count{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/allocator_heap_count.json" || true +} + +write_resource_summary() { + local samples_file="$1" + local summary_file="$2" + awk -F',' ' + NR == 1 { next } + NF >= 4 { + cpu += $3 + rss += $4 + if ($3 > max_cpu) max_cpu = $3 + if ($4 > max_rss) max_rss = $4 + count += 1 + } + END { + print "samples,avg_cpu_percent,max_cpu_percent,avg_rss_mib,max_rss_mib" + if (count == 0) { + print "0,N/A,N/A,N/A,N/A" + } else { + printf "%d,%.6f,%.6f,%.6f,%.6f\n", count, cpu / count, max_cpu, (rss / count) / 1024.0, max_rss / 1024.0 + } + } + ' "$samples_file" >"$summary_file" +} + +run_metadata_fast_chaos() { + local bucket="$1" + local out_dir="$2" + mkdir -p "$out_dir" + + local summary="$out_dir/chaos-summary.csv" + local fallback_summary="$out_dir/fallback-probes.csv" + local page1="$out_dir/page1-before-mutation.xml" + local page2="$out_dir/page2-after-mutation.xml" + local fresh_after="$out_dir/fresh-after-mutation.xml" + local generation_mismatch="$out_dir/generation-mismatch.xml" + local degraded_response="$out_dir/degraded-fallback.xml" + local page1_keys="$out_dir/page1-before-mutation.keys" + local page2_keys="$out_dir/page2-after-mutation.keys" + local fresh_after_keys="$out_dir/fresh-after-mutation.keys" + local expected_page2_keys="$out_dir/expected-page2-after-mutation.keys" + local body_file="$out_dir/overwrite-body.bin" + local delete_out="$out_dir/delete.out" + local overwrite_out="$out_dir/overwrite.out" + local token first_key second_key multipart_key overlap last_key monotonic_status no_skip_status page1_count page2_count fresh_max_keys forged_token journal_sequence + + echo "probe,status,detail" >"$summary" + echo "probe,status,detail" >"$fallback_summary" + + signed_list_page "$bucket" "$page1" "$CHAOS_MAX_KEYS" "" + extract_xml_keys <"$page1" >"$page1_keys" + token="$(extract_next_token <"$page1" || true)" + page1_count="$(awk 'END{print NR + 0}' "$page1_keys")" + if [[ -z "$token" || "$page1_count" -lt 2 ]]; then + echo "initial_page,skipped,requires at least two keys and a continuation token" >>"$summary" + return 0 + fi + echo "initial_page,ok,keys=${page1_count}; continuation_token=yes" >>"$summary" + + first_key="$(sed -n '1p' "$page1_keys")" + second_key="$(sed -n '2p' "$page1_keys")" + printf 'metadata-fast overwrite probe %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$body_file" + + signed_put_object "$bucket" "$first_key" "$body_file" "$overwrite_out" + echo "overwrite,ok,key=${first_key}" >>"$summary" + + signed_delete_object "$bucket" "$second_key" "$delete_out" + echo "delete,ok,key=${second_key}" >>"$summary" + + multipart_key="zzzz-metadata-fast-chaos-multipart-$(date +%s).bin" + signed_multipart_complete "$bucket" "$multipart_key" "$out_dir" + echo "multipart_complete,ok,key=${multipart_key}" >>"$summary" + + signed_list_page "$bucket" "$page2" "$CHAOS_MAX_KEYS" "$token" + extract_xml_keys <"$page2" >"$page2_keys" + page2_count="$(awk 'END{print NR + 0}' "$page2_keys")" + + overlap="$(comm -12 <(LC_ALL=C sort "$page1_keys") <(LC_ALL=C sort "$page2_keys") | awk 'END{print NR + 0}')" + if [[ "$overlap" -eq 0 ]]; then + echo "continuation_no_duplicate,ok,overlap=0" >>"$summary" + else + echo "continuation_no_duplicate,failed,overlap=${overlap}" >>"$summary" + return 1 + fi + + last_key="$(tail -n 1 "$page1_keys")" + monotonic_status="ok" + if [[ -n "$last_key" ]] && ! awk -v last="$last_key" 'NF && $0 <= last { bad = 1 } END { exit bad }' "$page2_keys"; then + monotonic_status="failed" + fi + echo "continuation_monotonic,${monotonic_status},page2_keys=${page2_count}; last_page1=${last_key}" >>"$summary" + [[ "$monotonic_status" == "ok" ]] || return 1 + + fresh_max_keys=$((page1_count + page2_count + CHAOS_MAX_KEYS)) + signed_list_page "$bucket" "$fresh_after" "$fresh_max_keys" "" + extract_xml_keys <"$fresh_after" >"$fresh_after_keys" + awk -v last="$last_key" 'NF && $0 > last { print }' "$fresh_after_keys" | head -n "$page2_count" >"$expected_page2_keys" + no_skip_status="ok" + if ! cmp -s "$expected_page2_keys" "$page2_keys"; then + no_skip_status="failed" + fi + echo "continuation_no_skip,${no_skip_status},expected_next_keys=$(awk 'END{print NR + 0}' "$expected_page2_keys"); actual_page2_keys=${page2_count}" >>"$summary" + [[ "$no_skip_status" == "ok" ]] || return 1 + + forged_token="$(printf '%s[rustfs_cache:v2,return:,src:index_metadata_fast,gen:%s-mismatch]' "$last_key" "$INDEX_GENERATION" | base64_token)" + if signed_list_page "$bucket" "$generation_mismatch" "$CHAOS_MAX_KEYS" "$forged_token"; then + echo "generation_mismatch,ok,forged_generation=${INDEX_GENERATION}-mismatch" >>"$fallback_summary" + else + echo "generation_mismatch,failed,forged_generation=${INDEX_GENERATION}-mismatch" >>"$fallback_summary" + fi + + if [[ -z "$NAMESPACE_JOURNAL_PATH" ]]; then + echo "degraded,skipped,requires --journal-path to avoid mutating quorum-backed journal state" >>"$fallback_summary" + return 0 + fi + + journal_sequence="$(journal_high_water_mark "$NAMESPACE_JOURNAL_PATH" "$bucket")" + write_local_journal_state "$NAMESPACE_JOURNAL_PATH" "$bucket" "$journal_sequence" "degraded" + if signed_list_page "$bucket" "$degraded_response" "$CHAOS_MAX_KEYS" ""; then + echo "degraded,ok,journal_path=${NAMESPACE_JOURNAL_PATH}; high_water_mark=${journal_sequence}" >>"$fallback_summary" + else + echo "degraded,failed,journal_path=${NAMESPACE_JOURNAL_PATH}; high_water_mark=${journal_sequence}" >>"$fallback_summary" + fi + write_local_journal_state "$NAMESPACE_JOURNAL_PATH" "$bucket" "$journal_sequence" "healthy" + echo "degraded_restore,ok,journal_path=${NAMESPACE_JOURNAL_PATH}; high_water_mark=${journal_sequence}" >>"$fallback_summary" +} + +journal_high_water_mark() { + local root="$1" + local bucket="$2" + local state_file="${root}/${bucket}.state" + if [[ -f "$state_file" ]]; then + awk -F= '/^# high_water_mark=/ {print $2; found=1; exit} END {if(!found) print "0"}' "$state_file" + else + echo "0" + fi +} + +write_local_journal_state() { + local root="$1" + local bucket="$2" + local high_water_mark="$3" + local status="$4" + local state_file="${root}/${bucket}.state" + local tmp_file="${state_file}.tmp" + mkdir -p "$root" + { + echo "# rustfs-listobjects-namespace-journal-v1" + echo "# bucket=${bucket}" + echo "# high_water_mark=${high_water_mark}" + echo "# status=${status}" + } >"$tmp_file" + mv "$tmp_file" "$state_file" +} + +run_metadata_fast_quorum_journal_chaos() { + local bucket="$1" + local out_dir="$2" + mkdir -p "$out_dir" + + local summary="$out_dir/quorum-journal-probes.csv" + local degraded_response="$out_dir/quorum-degraded-fallback.xml" + local disabled_gate_rebuild_response="$out_dir/quorum-disabled-gate-rebuild.xml" + local disabled_gate_response="$out_dir/quorum-disabled-gate.xml" + local restore_state_response="$out_dir/quorum-restore-state.xml" + local restore_rebuild_response="$out_dir/quorum-restore-rebuild.xml" + local restore_response="$out_dir/quorum-restore.xml" + local disabled_gate_fallbacks + + echo "probe,status,detail" >"$summary" + if [[ "$RUN_QUORUM_JOURNAL_CHAOS" != "true" ]]; then + echo "quorum_degraded,skipped,RUN_QUORUM_JOURNAL_CHAOS=false" >>"$summary" + return 0 + fi + if [[ -n "$NAMESPACE_JOURNAL_PATH" ]]; then + echo "quorum_degraded,skipped,local namespace journal override is active" >>"$summary" + return 0 + fi + + stop_server + + JOURNAL_CHAOS_ENABLED="" + JOURNAL_CHAOS_BUCKET="" + JOURNAL_CHAOS_SEQUENCE="" + JOURNAL_CHAOS_STATUS="" + start_server opt_in_verified "$out_dir/quorum-disabled-gate-rebuild-rustfs.log" false + if signed_list_once "$bucket" "$disabled_gate_rebuild_response" 1; then + echo "quorum_disabled_gate_rebuild,ok,bucket=${bucket}; provider=rebuild" >>"$summary" + else + echo "quorum_disabled_gate_rebuild,failed,bucket=${bucket}; provider=rebuild" >>"$summary" + fi + stop_server + + JOURNAL_CHAOS_ENABLED="" + JOURNAL_CHAOS_BUCKET="$bucket" + JOURNAL_CHAOS_SEQUENCE="" + JOURNAL_CHAOS_STATUS="degraded" + start_server metadata_fast "$out_dir/quorum-disabled-gate-rustfs.log" false + signed_list_page "$bucket" "$disabled_gate_response" "$CHAOS_MAX_KEYS" "" + sleep "$((METER_INTERVAL + 2))" + write_ratio_summary "metadata_fast_quorum_disabled_gate" "$out_dir/quorum-disabled-gate-rustfs.log" "$out_dir/quorum-disabled-gate-index-ratios.csv" + write_fallback_reason_summary \ + "metadata_fast_quorum_disabled_gate" \ + "$out_dir/quorum-disabled-gate-rustfs.log" \ + "$out_dir/quorum-disabled-gate-index-fallback-reasons.csv" + disabled_gate_fallbacks="$(awk -F',' 'NR == 2 {print $3 + 0}' "$out_dir/quorum-disabled-gate-index-ratios.csv")" + if [[ "$disabled_gate_fallbacks" -eq 0 ]]; then + echo "quorum_disabled_gate,ok,bucket=${bucket}; status=degraded; enabled=unset" >>"$summary" + else + echo "quorum_disabled_gate,failed,bucket=${bucket}; status=degraded; enabled=unset; fallbacks=${disabled_gate_fallbacks}" >>"$summary" + fi + stop_server + + JOURNAL_CHAOS_ENABLED="true" + JOURNAL_CHAOS_BUCKET="$bucket" + JOURNAL_CHAOS_SEQUENCE="" + JOURNAL_CHAOS_STATUS="degraded" + start_server metadata_fast "$out_dir/quorum-degraded-rustfs.log" false + if signed_list_page "$bucket" "$degraded_response" "$CHAOS_MAX_KEYS" ""; then + echo "quorum_degraded,ok,bucket=${bucket}; status=degraded" >>"$summary" + else + echo "quorum_degraded,failed,bucket=${bucket}; status=degraded" >>"$summary" + fi + sleep "$((METER_INTERVAL + 2))" + write_ratio_summary "metadata_fast_quorum_degraded" "$out_dir/quorum-degraded-rustfs.log" "$out_dir/quorum-degraded-index-ratios.csv" + write_fallback_reason_summary \ + "metadata_fast_quorum_degraded" \ + "$out_dir/quorum-degraded-rustfs.log" \ + "$out_dir/quorum-degraded-index-fallback-reasons.csv" + stop_server + + JOURNAL_CHAOS_ENABLED="true" + JOURNAL_CHAOS_STATUS="healthy" + start_server metadata_fast "$out_dir/quorum-restore-state-rustfs.log" false + if signed_list_page "$bucket" "$restore_state_response" "$CHAOS_MAX_KEYS" ""; then + echo "quorum_restore_state,ok,bucket=${bucket}; status=healthy" >>"$summary" + else + echo "quorum_restore_state,failed,bucket=${bucket}; status=healthy" >>"$summary" + fi + stop_server + + JOURNAL_CHAOS_ENABLED="" + JOURNAL_CHAOS_BUCKET="" + JOURNAL_CHAOS_SEQUENCE="" + JOURNAL_CHAOS_STATUS="" + + start_server opt_in_verified "$out_dir/quorum-restore-rebuild-rustfs.log" false + if signed_list_once "$bucket" "$restore_rebuild_response" 1; then + echo "quorum_restore_rebuild,ok,bucket=${bucket}; provider=rebuild" >>"$summary" + else + echo "quorum_restore_rebuild,failed,bucket=${bucket}; provider=rebuild" >>"$summary" + fi + stop_server + + start_server metadata_fast "$out_dir/quorum-restore-rustfs.log" false + if signed_list_page "$bucket" "$restore_response" "$CHAOS_MAX_KEYS" ""; then + echo "quorum_restore,ok,bucket=${bucket}; status=healthy; provider=metadata_fast" >>"$summary" + else + echo "quorum_restore,failed,bucket=${bucket}; status=healthy; provider=metadata_fast" >>"$summary" + fi + sleep "$((METER_INTERVAL + 2))" + write_ratio_summary "metadata_fast_quorum_restore" "$out_dir/quorum-restore-rustfs.log" "$out_dir/quorum-restore-index-ratios.csv" + write_fallback_reason_summary \ + "metadata_fast_quorum_restore" \ + "$out_dir/quorum-restore-rustfs.log" \ + "$out_dir/quorum-restore-index-fallback-reasons.csv" + stop_server +} + +run_mode() { + local mode="$1" + local bucket="$2" + local reset_data="${3:-false}" + local mode_dir="$OUT_DIR/$mode" + mkdir -p "$mode_dir" + + log_info "Starting RustFS for mode=$mode" + start_server "$mode" "$mode_dir/rustfs.log" "$reset_data" + start_resource_sampler "$mode_dir/resource-samples.csv" + + if [[ "$mode" == "opt_in_verified" && "$PREBUILD_KEY_INDEX" != "true" ]] \ + && [[ "$INDEX_PROVIDER" == "persistent_key_only" || "$INDEX_PROVIDER" == "persisted_key_only" ]]; then + log_info "Warming persistent key-only provider through production rebuild path" + signed_list_once "$bucket" "$mode_dir/provider-warmup.xml" 1 + fi + + log_info "Signed ListObjectsV2 bench for mode=$mode duration=$DURATION" + run_signed_list_bench "$mode" "$bucket" "$DURATION" "$mode_dir" + stop_resource_sampler + sleep "$((METER_INTERVAL + 2))" + write_ratio_summary "$mode" "$mode_dir/rustfs.log" "$mode_dir/index-ratios.csv" + write_fallback_reason_summary "$mode" "$mode_dir/rustfs.log" "$mode_dir/index-fallback-reasons.csv" + write_resource_summary "$mode_dir/resource-samples.csv" "$mode_dir/resource-summary.csv" + write_prometheus_snapshot "$mode" "$mode_dir" + + if [[ "$mode" == "metadata_fast" && "$RUN_CHAOS" == "true" ]]; then + log_info "Running metadata-fast chaos probes" + run_metadata_fast_chaos "$bucket" "$mode_dir/chaos" + sleep "$((METER_INTERVAL + 2))" + write_ratio_summary "${mode}_after_chaos" "$mode_dir/rustfs.log" "$mode_dir/chaos/index-ratios-after-chaos.csv" + write_fallback_reason_summary "${mode}_after_chaos" "$mode_dir/rustfs.log" "$mode_dir/chaos/index-fallback-reasons-after-chaos.csv" + log_info "Running metadata-fast quorum-backed namespace journal degraded probe" + run_metadata_fast_quorum_journal_chaos "$bucket" "$mode_dir/chaos" + fi + + stop_server +} + +write_manifest() { + mkdir -p "$OUT_DIR" + { + echo "out_dir=$OUT_DIR" + echo "data_root=$DATA_ROOT" + echo "build_profile=$BUILD_PROFILE" + echo "rustfs_bin=$RUSTFS_BIN" + echo "warp_bin=$WARP_BIN" + echo "address=$ADDRESS" + echo "objects=$OBJECTS" + echo "object_size=$OBJECT_SIZE" + echo "concurrency=$CONCURRENCY" + echo "max_keys=$MAX_KEYS" + echo "duration=$DURATION" + echo "warmup_duration=$WARMUP_DURATION" + echo "meter_interval=$METER_INTERVAL" + echo "resource_sample_interval=$RESOURCE_SAMPLE_INTERVAL" + echo "index_provider=$INDEX_PROVIDER" + echo "index_generation=$INDEX_GENERATION" + echo "key_index_path=$KEY_INDEX_PATH" + echo "namespace_journal_path=${NAMESPACE_JOURNAL_PATH:-.rustfs.sys/listobjects/ns-journal/v1}" + echo "prebuild_key_index=$PREBUILD_KEY_INDEX" + echo "prepare_only=$PREPARE_ONLY" + echo "reuse_prepared_data=$REUSE_PREPARED_DATA" + echo "run_metadata_fast=$RUN_METADATA_FAST" + echo "run_chaos=$RUN_CHAOS" + echo "run_quorum_journal_chaos=$RUN_QUORUM_JOURNAL_CHAOS" + echo "chaos_max_keys=$CHAOS_MAX_KEYS" + echo "metadata_fast_staleness_ms=$METADATA_FAST_STALENESS_MS" + echo "prometheus_query_url=$PROMETHEUS_QUERY_URL" + echo "prometheus_job=$PROMETHEUS_JOB" + } >"$OUT_DIR/manifest.env" +} + +summary_csv_section() { + local title="$1" + local file="$2" + if [[ -f "$file" ]]; then + echo "### $title" + echo + cat "$file" + echo + fi +} + +write_combined_summary() { + local summary="$OUT_DIR/summary.md" + { + echo "# ListObjects Verified Benchmark Summary" + echo + echo "## Manifest" + echo + sed 's/^/- /' "$OUT_DIR/manifest.env" + echo + echo "## Warp" + echo + summary_csv_section "Walker" "$OUT_DIR/walker/list-summary.csv" + summary_csv_section "Opt-in verified" "$OUT_DIR/opt_in_verified/list-summary.csv" + summary_csv_section "Metadata-fast" "$OUT_DIR/metadata_fast/list-summary.csv" + echo + echo "## Index Ratios" + echo + summary_csv_section "Walker" "$OUT_DIR/walker/index-ratios.csv" + summary_csv_section "Opt-in verified" "$OUT_DIR/opt_in_verified/index-ratios.csv" + summary_csv_section "Metadata-fast" "$OUT_DIR/metadata_fast/index-ratios.csv" + echo + echo "## Fallback Reasons" + echo + summary_csv_section "Walker" "$OUT_DIR/walker/index-fallback-reasons.csv" + summary_csv_section "Opt-in verified" "$OUT_DIR/opt_in_verified/index-fallback-reasons.csv" + summary_csv_section "Metadata-fast" "$OUT_DIR/metadata_fast/index-fallback-reasons.csv" + echo + echo "## Resource Samples" + echo + summary_csv_section "Walker" "$OUT_DIR/walker/resource-summary.csv" + summary_csv_section "Opt-in verified" "$OUT_DIR/opt_in_verified/resource-summary.csv" + summary_csv_section "Metadata-fast" "$OUT_DIR/metadata_fast/resource-summary.csv" + if [[ -f "$OUT_DIR/metadata_fast/chaos/chaos-summary.csv" ]]; then + echo + echo "## Metadata-fast Chaos" + echo + summary_csv_section "Chaos probes" "$OUT_DIR/metadata_fast/chaos/chaos-summary.csv" + summary_csv_section "Fallback probes" "$OUT_DIR/metadata_fast/chaos/fallback-probes.csv" + summary_csv_section "Ratios after chaos" "$OUT_DIR/metadata_fast/chaos/index-ratios-after-chaos.csv" + summary_csv_section "Fallback reasons after chaos" "$OUT_DIR/metadata_fast/chaos/index-fallback-reasons-after-chaos.csv" + summary_csv_section "Quorum journal probes" "$OUT_DIR/metadata_fast/chaos/quorum-journal-probes.csv" + summary_csv_section "Quorum disabled-gate ratios" "$OUT_DIR/metadata_fast/chaos/quorum-disabled-gate-index-ratios.csv" + summary_csv_section "Quorum disabled-gate fallback reasons" "$OUT_DIR/metadata_fast/chaos/quorum-disabled-gate-index-fallback-reasons.csv" + summary_csv_section "Quorum degraded ratios" "$OUT_DIR/metadata_fast/chaos/quorum-degraded-index-ratios.csv" + summary_csv_section "Quorum degraded fallback reasons" "$OUT_DIR/metadata_fast/chaos/quorum-degraded-index-fallback-reasons.csv" + summary_csv_section "Quorum restore ratios" "$OUT_DIR/metadata_fast/chaos/quorum-restore-index-ratios.csv" + summary_csv_section "Quorum restore fallback reasons" "$OUT_DIR/metadata_fast/chaos/quorum-restore-index-fallback-reasons.csv" + fi + } >"$summary" +} + +main() { + parse_args "$@" + require_cmds + write_manifest + build_rustfs + trap stop_server EXIT INT TERM + + local bucket="rustfs-listobjects-bench" + local prep_dir="$OUT_DIR/prepare" + mkdir -p "$prep_dir" + + if [[ "$REUSE_PREPARED_DATA" == "true" ]]; then + log_info "Reusing prepared data root: $DATA_ROOT" + else + log_info "Starting RustFS for data preparation" + start_server walker "$prep_dir/rustfs.log" true + log_info "Preparing bucket=$bucket objects=$OBJECTS with warp list" + prepare_bucket "$bucket" "$WARMUP_DURATION" "$prep_dir/warp-prepare.log" || { + tail -n 80 "$prep_dir/warp-prepare.log" >&2 || true + return 1 + } + fi + + if [[ "$PREBUILD_KEY_INDEX" == "true" ]]; then + if [[ "$REUSE_PREPARED_DATA" == "true" ]]; then + log_info "Starting RustFS to build persistent key-only index from reused data" + start_server walker "$prep_dir/rustfs.log" false + fi + log_info "Writing persistent key-only index from bench script: $KEY_INDEX_PATH" + write_key_index "$bucket" "$KEY_INDEX_PATH" + stop_server + else + log_info "Skipping bench-side key index generation; opt-in mode will rebuild through RustFS lifecycle" + stop_server + fi + + if [[ "$PREPARE_ONLY" == "true" ]]; then + log_info "Prepare-only mode complete: $DATA_ROOT" + return 0 + fi + + run_mode walker "$bucket" false + run_mode opt_in_verified "$bucket" false + if [[ "$RUN_METADATA_FAST" == "true" ]]; then + run_mode metadata_fast "$bucket" false + fi + write_combined_summary + + log_info "Summary: $OUT_DIR/summary.md" +} + +main "$@" diff --git a/scripts/validate_issue_841_list_objects_observability.sh b/scripts/validate_issue_841_list_objects_observability.sh new file mode 100755 index 000000000..837a52856 --- /dev/null +++ b/scripts/validate_issue_841_list_objects_observability.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Acceptance runner for rustfs/backlog#841. +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/issue-841-acceptance-$(date +%Y%m%d-%H%M%S)}" + +log_info() { printf '[INFO] %s\n' "$*"; } +log_error() { printf '[ERROR] %s\n' "$*" >&2; } + +usage() { + cat <<'USAGE' +Usage: + scripts/validate_issue_841_list_objects_observability.sh + +Environment: + OUT_DIR output directory (default: target/issue-841-acceptance-) +USAGE +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + log_error "command not found: $1" + exit 1 + fi +} + +run_unit_checks() { + log_info "Running ListObjects observability unit checks" + local test_log="$OUT_DIR/issue-841-tests.log" + : > "$test_log" + + ( + cd "$PROJECT_ROOT" + cargo test -p rustfs-io-metrics list_objects_metrics -- --nocapture + cargo test -p rustfs-ecstore list_objects -- --nocapture + ) 2>&1 | tee "$test_log" + + log_info "Unit test log: $test_log" +} + +run_static_checks() { + log_info "Running ListObjects observability static checks" + local static_log="$OUT_DIR/static-checks.log" + : > "$static_log" + + local required_patterns=( + "rustfs_s3_list_objects_gather_total" + "rustfs_s3_list_objects_gather_scan_amplification" + "rustfs_s3_list_objects_merge_fan_in" + "record_list_objects_gather" + "record_list_objects_merge" + "init_list_objects_metrics" + "listobjects-v2-baseline-fixtures" + ) + + local pattern + for pattern in "${required_patterns[@]}"; do + if rg --no-ignore -n "$pattern" "$PROJECT_ROOT/crates" "$PROJECT_ROOT/rustfs" "$PROJECT_ROOT/docs" "$PROJECT_ROOT/scripts" >> "$static_log"; then + log_info "Found required symbol: $pattern" + else + log_error "Missing required symbol: $pattern" + log_error "Static check log: $static_log" + return 1 + fi + done + + log_info "Static check log: $static_log" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + *) + log_error "unknown arg: $1" + usage + exit 1 + ;; + esac + done +} + +main() { + parse_args "$@" + require_cmd cargo + require_cmd rg + require_cmd tee + + mkdir -p "$OUT_DIR" + run_unit_checks + run_static_checks + + log_info "Validation summary:" + log_info " - logs: $OUT_DIR" + log_info " - status: pass" +} + +main "$@"