perf(ecstore): support per-bucket durability tier overrides (#4407)

perf(ecstore): per-bucket durability tier overrides (HP-5 phase 2)

Let a bucket override the process-wide RUSTFS_DURABILITY_MODE with its own
strict/relaxed/none tier, stored as a durability.json extension entry in the
bucket metadata file and resolved at commit points via effective_durability.
System-critical buckets (.rustfs.sys, .minio.sys) can never carry an override
and stay pinned to strict; the legacy full-off switch keeps its historical
semantics and per-bucket overrides do not apply under it. Overrides are
published and cleared through the existing bucket metadata cache-invalidation
path, and an admin GET/PUT handler exposes the configuration. Default behavior
is unchanged: with no override a bucket follows the global mode, which defaults
to strict and stays byte-for-byte identical to before.

Refs: https://github.com/rustfs/backlog/issues/938, https://github.com/rustfs/backlog/issues/936

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-07-08 16:33:49 +08:00
committed by GitHub
parent 09e6983175
commit 13e48d93aa
14 changed files with 904 additions and 31 deletions
+12 -5
View File
@@ -82,6 +82,7 @@ pub mod bucket {
}
pub mod metadata {
pub use crate::bucket::metadata::BUCKET_DURABILITY_CONFIG;
pub use crate::bucket::metadata::{
BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG,
BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_QUOTA_CONFIG_FILE,
@@ -92,14 +93,20 @@ pub mod bucket {
};
}
pub mod durability {
pub use crate::bucket::durability::{
BUCKET_DURABILITY_MODE_NONE, BUCKET_DURABILITY_MODE_RELAXED, BUCKET_DURABILITY_MODE_STRICT, BucketDurabilityConfig,
};
}
pub mod metadata_sys {
pub use crate::bucket::metadata_sys::{
BucketMetadataSys, delete, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw,
get_bucket_targets_config, get_config_from_disk, get_cors_config, get_global_bucket_metadata_sys,
get_lifecycle_config, get_logging_config, get_notification_config, get_object_lock_config,
get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config,
get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets,
remove_bucket_metadata, set_bucket_metadata, update,
get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
init_bucket_metadata_sys, list_bucket_targets, remove_bucket_metadata, set_bucket_metadata, update,
};
}
+102
View File
@@ -0,0 +1,102 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Per-bucket durability tier configuration (HP-5 phase 2).
//!
//! A bucket can override the process-wide durability mode
//! (`RUSTFS_DURABILITY_MODE`) with its own tier. The override is stored as a
//! RustFS extension entry (`durability.json`) in the bucket metadata file and
//! is resolved by the disk layer at commit points via
//! `effective_durability(volume)`. System buckets (`.rustfs.sys`,
//! `.minio.sys`) can never carry an override; they stay pinned to `strict`.
//! See docs/operations/durability-modes.md for the semantics.
use serde::{Deserialize, Serialize};
/// Valid per-bucket durability tier names. `legacy-off` is deliberately not
/// representable per bucket: it exists only for the process-wide legacy
/// switch (`RUSTFS_DRIVE_SYNC_ENABLE=false`).
pub const BUCKET_DURABILITY_MODE_STRICT: &str = "strict";
pub const BUCKET_DURABILITY_MODE_RELAXED: &str = "relaxed";
pub const BUCKET_DURABILITY_MODE_NONE: &str = "none";
/// JSON payload stored under the `durability.json` bucket metadata entry.
///
/// An absent entry, an empty payload, or an empty `mode` string all mean
/// "no override": the bucket follows the process-wide durability mode.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct BucketDurabilityConfig {
/// Requested tier: `strict` | `relaxed` | `none` (case-insensitive,
/// whitespace-tolerant). Empty means "inherit the global mode".
#[serde(default)]
pub mode: String,
}
impl BucketDurabilityConfig {
pub fn new(mode: &str) -> Self {
Self { mode: mode.to_string() }
}
/// Whether `mode` names a valid per-bucket tier.
pub fn is_valid_mode(mode: &str) -> bool {
matches!(
mode.trim().to_ascii_lowercase().as_str(),
BUCKET_DURABILITY_MODE_STRICT | BUCKET_DURABILITY_MODE_RELAXED | BUCKET_DURABILITY_MODE_NONE
)
}
/// The canonical (trimmed, lowercase) tier name, or `None` when the
/// config does not carry a valid override.
pub fn normalized_mode(&self) -> Option<String> {
let mode = self.mode.trim().to_ascii_lowercase();
if Self::is_valid_mode(&mode) { Some(mode) } else { None }
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn valid_modes_are_recognized() {
assert!(BucketDurabilityConfig::is_valid_mode("strict"));
assert!(BucketDurabilityConfig::is_valid_mode("relaxed"));
assert!(BucketDurabilityConfig::is_valid_mode("none"));
assert!(BucketDurabilityConfig::is_valid_mode(" RELAXED "));
assert!(!BucketDurabilityConfig::is_valid_mode(""));
assert!(!BucketDurabilityConfig::is_valid_mode("bogus"));
// The legacy full-off switch is process-wide only.
assert!(!BucketDurabilityConfig::is_valid_mode("legacy-off"));
}
#[test]
fn normalized_mode_canonicalizes_or_rejects() {
assert_eq!(BucketDurabilityConfig::new(" Relaxed ").normalized_mode().as_deref(), Some("relaxed"));
assert_eq!(BucketDurabilityConfig::new("strict").normalized_mode().as_deref(), Some("strict"));
assert_eq!(BucketDurabilityConfig::default().normalized_mode(), None);
assert_eq!(BucketDurabilityConfig::new("bogus").normalized_mode(), None);
}
#[test]
fn json_round_trip() {
let cfg = BucketDurabilityConfig::new("relaxed");
let json = serde_json::to_vec(&cfg).expect("serialize");
let back: BucketDurabilityConfig = serde_json::from_slice(&json).expect("deserialize");
assert_eq!(back, cfg);
// Empty object deserializes to the "inherit" default.
let empty: BucketDurabilityConfig = serde_json::from_slice(b"{}").expect("deserialize empty");
assert_eq!(empty.normalized_mode(), None);
}
}
+74 -2
View File
@@ -246,6 +246,7 @@ pub const BUCKET_REQUEST_PAYMENT_CONFIG: &str = "request-payment.xml";
pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
pub const BUCKET_TABLE_CONFIG: &str = "table-bucket.json";
pub const BUCKET_DURABILITY_CONFIG: &str = "durability.json";
pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table";
pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog";
pub const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str = "table-buckets";
@@ -294,6 +295,7 @@ pub struct BucketMetadata {
pub public_access_block_config_xml: Vec<u8>,
pub bucket_acl_config_json: Vec<u8>,
pub table_bucket_config_json: Vec<u8>,
pub durability_config_json: Vec<u8>,
pub policy_config_updated_at: OffsetDateTime,
pub object_lock_config_updated_at: OffsetDateTime,
@@ -314,6 +316,7 @@ pub struct BucketMetadata {
pub public_access_block_config_updated_at: OffsetDateTime,
pub bucket_acl_config_updated_at: OffsetDateTime,
pub table_bucket_config_updated_at: OffsetDateTime,
pub durability_config_updated_at: OffsetDateTime,
pub new_field_updated_at: OffsetDateTime,
@@ -362,6 +365,7 @@ impl Default for BucketMetadata {
public_access_block_config_xml: Default::default(),
bucket_acl_config_json: Default::default(),
table_bucket_config_json: Default::default(),
durability_config_json: Default::default(),
policy_config_updated_at: OffsetDateTime::UNIX_EPOCH,
object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH,
encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH,
@@ -381,6 +385,7 @@ impl Default for BucketMetadata {
public_access_block_config_updated_at: OffsetDateTime::UNIX_EPOCH,
bucket_acl_config_updated_at: OffsetDateTime::UNIX_EPOCH,
table_bucket_config_updated_at: OffsetDateTime::UNIX_EPOCH,
durability_config_updated_at: OffsetDateTime::UNIX_EPOCH,
new_field_updated_at: OffsetDateTime::UNIX_EPOCH,
policy_config: Default::default(),
notification_config: Default::default(),
@@ -430,6 +435,32 @@ impl BucketMetadata {
!self.table_bucket_config_json.is_empty()
}
/// Parsed per-bucket durability override, if a valid one is stored.
///
/// Absent/empty/unparsable payloads all mean "no override" (the bucket
/// follows the global durability mode); a parse failure is logged so a
/// corrupted entry cannot silently change fsync behavior.
pub fn durability_config(&self) -> Option<super::durability::BucketDurabilityConfig> {
if self.durability_config_json.is_empty() {
return None;
}
match serde_json::from_slice(&self.durability_config_json) {
Ok(cfg) => Some(cfg),
Err(e) => {
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "durability",
error = %e,
"Failed to parse bucket metadata config"
);
None
}
}
}
/// Decode from msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
pub fn decode_from<R: Read>(&mut self, rd: &mut R) -> Result<()> {
let mut fields = rmp::decode::read_map_len(rd)?;
@@ -481,6 +512,7 @@ impl BucketMetadata {
}
"BucketAclConfigJSON" | "BucketAclConfigJson" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
"TableBucketConfigJSON" | "TableBucketConfigJson" => self.table_bucket_config_json = read_msgp_bin(rd)?,
"DurabilityConfigJSON" | "DurabilityConfigJson" => self.durability_config_json = read_msgp_bin(rd)?,
"CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
"LoggingConfigUpdatedAt" => self.logging_config_updated_at = read_msgp_time_value(rd)?,
"WebsiteConfigUpdatedAt" => self.website_config_updated_at = read_msgp_time_value(rd)?,
@@ -489,6 +521,7 @@ impl BucketMetadata {
"PublicAccessBlockConfigUpdatedAt" => self.public_access_block_config_updated_at = read_msgp_time_value(rd)?,
"BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?,
"TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
"DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
other => {
tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
skip_msgp_value(rd)?;
@@ -501,8 +534,8 @@ impl BucketMetadata {
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
// Map size: MinIO fields (25) + RustFS extensions (16)
let map_len: u32 = 41;
// Map size: MinIO fields (25) + RustFS extensions (18)
let map_len: u32 = 43;
rmp::encode::write_map_len(wr, map_len)?;
// MinIO field order (same as Go struct)
@@ -559,6 +592,7 @@ impl BucketMetadata {
write_bin_field(wr, "PublicAccessBlockConfigXML", &self.public_access_block_config_xml)?;
write_bin_field(wr, "BucketAclConfigJSON", &self.bucket_acl_config_json)?;
write_bin_field(wr, "TableBucketConfigJSON", &self.table_bucket_config_json)?;
write_bin_field(wr, "DurabilityConfigJSON", &self.durability_config_json)?;
rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?;
write_msgp_time(wr, self.cors_config_updated_at)?;
rmp::encode::write_str(wr, "LoggingConfigUpdatedAt")?;
@@ -575,6 +609,8 @@ impl BucketMetadata {
write_msgp_time(wr, self.bucket_acl_config_updated_at)?;
rmp::encode::write_str(wr, "TableBucketConfigUpdatedAt")?;
write_msgp_time(wr, self.table_bucket_config_updated_at)?;
rmp::encode::write_str(wr, "DurabilityConfigUpdatedAt")?;
write_msgp_time(wr, self.durability_config_updated_at)?;
Ok(())
}
@@ -673,6 +709,9 @@ impl BucketMetadata {
if self.table_bucket_config_updated_at == OffsetDateTime::UNIX_EPOCH {
self.table_bucket_config_updated_at = self.created
}
if self.durability_config_updated_at == OffsetDateTime::UNIX_EPOCH {
self.durability_config_updated_at = self.created
}
}
pub fn update_config(&mut self, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
@@ -755,6 +794,10 @@ impl BucketMetadata {
self.table_bucket_config_json = data;
self.table_bucket_config_updated_at = updated;
}
BUCKET_DURABILITY_CONFIG => {
self.durability_config_json = data;
self.durability_config_updated_at = updated;
}
_ => return Err(Error::other(format!("config file not found : {config_file}"))),
}
@@ -1444,6 +1487,35 @@ mod test {
assert!(!bm.table_bucket_enabled());
}
/// HP-5b (rustfs/backlog#938): the durability override is a RustFS
/// extension entry and must survive an encode/decode round trip.
#[test]
fn durability_config_round_trips_and_tracks_updates() {
let mut bm = BucketMetadata::new("durability-bucket");
assert!(bm.durability_config().is_none(), "fresh metadata carries no override");
bm.update_config(BUCKET_DURABILITY_CONFIG, br#"{"mode":"relaxed"}"#.to_vec())
.unwrap();
assert_ne!(bm.durability_config_updated_at, OffsetDateTime::UNIX_EPOCH);
let buf = bm.marshal_msg().unwrap();
let back = BucketMetadata::unmarshal(&buf).unwrap();
assert_eq!(back.durability_config_json, bm.durability_config_json);
assert_eq!(
back.durability_config_updated_at.unix_timestamp(),
bm.durability_config_updated_at.unix_timestamp()
);
assert_eq!(back.durability_config().and_then(|c| c.normalized_mode()).as_deref(), Some("relaxed"));
// Clearing the entry removes the override.
bm.update_config(BUCKET_DURABILITY_CONFIG, Vec::new()).unwrap();
assert!(bm.durability_config().is_none());
// Corrupted payloads must degrade to "no override", never to a tier.
bm.durability_config_json = b"not-json".to_vec();
assert!(bm.durability_config().is_none());
}
/// After policy deletion (policy_config_json cleared), parse_policy_config sets policy_config to None.
#[test]
fn test_parse_policy_config_clears_cache_when_json_empty() {
+71
View File
@@ -153,6 +153,26 @@ async fn sync_bucket_target_sys(bucket: &str, bm: &BucketMetadata) {
.await;
}
/// Publish the bucket's durability override (or its absence) to the disk
/// layer registry consulted by `effective_durability`.
///
/// Called from every path that installs a bucket's metadata into the cache
/// (initial load, config update, peer reload notification, refresh loop,
/// lazy load), so the override propagates with exactly the bucket-metadata
/// cache invalidation semantics and never through a channel of its own.
fn sync_bucket_durability(bucket: &str, bm: &BucketMetadata) {
let mode = bm
.durability_config()
.and_then(|cfg| cfg.normalized_mode())
.and_then(|mode| crate::disk::local::DurabilityMode::parse(&mode));
crate::disk::local::bucket_durability::set(bucket, mode);
}
/// Drop a bucket's durability override when its metadata leaves the cache.
fn clear_bucket_durability(bucket: &str) {
crate::disk::local::bucket_durability::set(bucket, None);
}
pub async fn get(bucket: &str) -> Result<Arc<BucketMetadata>> {
let sys = get_bucket_metadata_sys()?;
let lock = sys.read().await;
@@ -196,6 +216,20 @@ pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTi
bucket_meta_sys.get_bucket_acl_config(bucket).await
}
/// The bucket's durability override config (if any) with its update time.
///
/// `Ok((None, ..))` means the bucket has no override and follows the global
/// durability mode.
pub async fn get_durability_config(
bucket: &str,
) -> Result<(Option<crate::bucket::durability::BucketDurabilityConfig>, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
let (bm, _) = bucket_meta_sys.get_config(bucket).await?;
Ok((bm.durability_config(), bm.durability_config_updated_at))
}
pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -425,6 +459,7 @@ impl BucketMetadataSys {
map.insert(bucket.clone(), bm.clone());
drop(map);
sync_bucket_target_sys(&bucket, &bm).await;
sync_bucket_durability(&bucket, &bm);
}
}
@@ -440,6 +475,7 @@ impl BucketMetadataSys {
drop(map);
if removed {
BucketTargetSys::get().delete(bucket).await;
clear_bucket_durability(bucket);
}
removed
}
@@ -540,6 +576,7 @@ impl BucketMetadataSys {
map.insert(bucket.to_string(), bm.clone());
drop(map);
sync_bucket_target_sys(bucket, &bm).await;
sync_bucket_durability(bucket, &bm);
Ok((bm, true))
}
@@ -826,6 +863,40 @@ mod tests {
target_sys.delete(bucket).await;
}
/// HP-5b (rustfs/backlog#938): installing bucket metadata publishes the
/// durability override to the disk-layer registry, and clearing the
/// config (or an invalid payload) withdraws it.
#[test]
fn metadata_sync_publishes_and_clears_durability_override() {
use crate::disk::local::{DurabilityMode, bucket_durability};
let bucket = "metadata-sync-durability";
let mut bm = BucketMetadata::new(bucket);
bm.durability_config_json = br#"{"mode":"relaxed"}"#.to_vec();
sync_bucket_durability(bucket, &bm);
assert_eq!(bucket_durability::lookup(bucket), Some(DurabilityMode::Relaxed));
// Metadata without the config entry clears the override.
let bm = BucketMetadata::new(bucket);
sync_bucket_durability(bucket, &bm);
assert_eq!(bucket_durability::lookup(bucket), None);
// Invalid payloads degrade to "no override", never to a tier.
let mut bm = BucketMetadata::new(bucket);
bm.durability_config_json = br#"{"mode":"bogus"}"#.to_vec();
sync_bucket_durability(bucket, &bm);
assert_eq!(bucket_durability::lookup(bucket), None);
// Cache removal clears the override too.
let mut bm = BucketMetadata::new(bucket);
bm.durability_config_json = br#"{"mode":"none"}"#.to_vec();
sync_bucket_durability(bucket, &bm);
assert_eq!(bucket_durability::lookup(bucket), Some(DurabilityMode::None));
clear_bucket_durability(bucket);
assert_eq!(bucket_durability::lookup(bucket), None);
}
#[tokio::test]
async fn refresh_wait_exits_when_cancelled() {
let cancel_token = CancellationToken::new();
+1
View File
@@ -17,6 +17,7 @@
pub mod bandwidth;
pub mod bucket_target_sys;
pub mod durability;
pub mod error;
pub mod lifecycle;
pub mod metadata;
+248 -14
View File
@@ -288,7 +288,7 @@ pub(crate) enum DurabilityMode {
}
impl DurabilityMode {
fn parse(value: &str) -> Option<Self> {
pub(crate) fn parse(value: &str) -> Option<Self> {
match value.trim().to_ascii_lowercase().as_str() {
"strict" => Some(Self::Strict),
"relaxed" => Some(Self::Relaxed),
@@ -406,6 +406,96 @@ pub(crate) mod durability_mode_override {
}
}
/// Per-bucket durability overrides (HP-5 phase 2, rustfs/backlog#938).
///
/// The disk layer never loads bucket metadata itself: the bucket metadata
/// subsystem publishes the parsed override here whenever a bucket's cached
/// metadata is set, refreshed, or removed, so this registry follows exactly
/// the existing bucket-metadata cache invalidation semantics (immediate on
/// the node applying a config change, peer reload notification plus the
/// periodic refresh loop elsewhere). Lookups sit on the commit hot path, so
/// the empty-registry case (no bucket overrides configured anywhere — the
/// default) is a single relaxed atomic load and the phase 1 behavior is
/// preserved bit for bit.
pub(crate) mod bucket_durability {
use super::{
DurabilityMode, EVENT_DISK_LOCAL_DURABILITY_MODE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_DISK_LOCAL, is_scratch_volume,
is_system_critical_volume,
};
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{OnceLock, PoisonError, RwLock};
use tracing::{info, warn};
static OVERRIDES: OnceLock<RwLock<HashMap<String, DurabilityMode>>> = OnceLock::new();
/// Fast-path gate: false means "no override registered anywhere", which
/// keeps default deployments off the map lookup entirely.
static NON_EMPTY: AtomicBool = AtomicBool::new(false);
fn overrides() -> &'static RwLock<HashMap<String, DurabilityMode>> {
OVERRIDES.get_or_init(|| RwLock::new(HashMap::new()))
}
/// Publish (or clear, with `None`) the durability override for `bucket`.
///
/// System namespaces can never carry an override: they are pinned to
/// `strict` by [`super::effective_durability`], and any attempt to
/// register one is rejected here as defense in depth.
pub(crate) fn set(bucket: &str, mode: Option<DurabilityMode>) {
if bucket.is_empty() {
return;
}
if is_system_critical_volume(bucket) || is_scratch_volume(bucket) {
if mode.is_some() {
warn!(
event = EVENT_DISK_LOCAL_DURABILITY_MODE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
bucket = %bucket,
"Rejected per-bucket durability override for a system namespace; it stays pinned to strict"
);
}
return;
}
// The legacy full-off switch is process-wide only and deliberately
// unreachable per bucket (`DurabilityMode::parse` never returns it).
let mode = mode.filter(|m| *m != DurabilityMode::LegacyOff);
let mut map = overrides().write().unwrap_or_else(PoisonError::into_inner);
let changed = match mode {
Some(mode) => map.insert(bucket.to_string(), mode) != Some(mode),
None => map.remove(bucket).is_some(),
};
NON_EMPTY.store(!map.is_empty(), Ordering::Release);
drop(map);
if changed {
info!(
event = EVENT_DISK_LOCAL_DURABILITY_MODE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
bucket = %bucket,
mode = mode.map_or("inherit", |m| m.as_str()),
"Per-bucket durability override updated"
);
}
}
/// The override registered for `volume`, if any. `volume` is the commit
/// destination, so user buckets resolve by name while scratch and system
/// namespaces never match (they are refused by [`set`]).
pub(crate) fn lookup(volume: &str) -> Option<DurabilityMode> {
if !NON_EMPTY.load(Ordering::Acquire) {
return None;
}
overrides()
.read()
.unwrap_or_else(PoisonError::into_inner)
.get(volume)
.copied()
}
}
/// Whether `volume` stages in-flight user object data (`.rustfs.sys/tmp`,
/// `.rustfs.sys/multipart`, and their subtrees). These namespaces follow the
/// configured durability mode: their contents commit into user buckets and
@@ -438,21 +528,21 @@ fn is_system_critical_volume(volume: &str) -> bool {
/// Effective durability for writes that commit into `volume`.
///
/// System-critical volumes are pinned to `Strict` regardless of the
/// configured tier. The legacy full-off switch keeps its historical
/// semantics and is never pinned.
/// Resolution order: system-critical volumes are pinned to `Strict`
/// regardless of any configuration; otherwise a per-bucket override
/// (published by the bucket metadata subsystem, see [`bucket_durability`])
/// wins over the process-wide mode; otherwise the process-wide mode applies.
/// The legacy full-off switch keeps its historical semantics: it is never
/// pinned and per-bucket overrides do not apply under it.
pub(crate) fn effective_durability(volume: &str) -> DurabilityMode {
let mode = durability_mode();
match mode {
DurabilityMode::Strict | DurabilityMode::LegacyOff => mode,
DurabilityMode::Relaxed | DurabilityMode::None => {
if is_system_critical_volume(volume) {
DurabilityMode::Strict
} else {
mode
}
}
let global = durability_mode();
if global == DurabilityMode::LegacyOff {
return global;
}
if is_system_critical_volume(volume) {
return DurabilityMode::Strict;
}
bucket_durability::lookup(volume).unwrap_or(global)
}
/// Get the O_DIRECT read threshold size.
@@ -5627,6 +5717,150 @@ mod test {
}
}
/// Removes the bucket's durability override when dropped so a test can
/// never leak its override into another test's lookup.
struct BucketOverrideGuard(&'static str);
impl BucketOverrideGuard {
fn set(bucket: &'static str, mode: DurabilityMode) -> Self {
bucket_durability::set(bucket, Some(mode));
Self(bucket)
}
}
impl Drop for BucketOverrideGuard {
fn drop(&mut self) {
bucket_durability::set(self.0, None);
}
}
#[test]
fn test_effective_durability_bucket_override() {
// Global strict + per-bucket relaxed: only the named bucket drops.
{
let _mode = durability_mode_override::set(DurabilityMode::Strict);
let _guard = BucketOverrideGuard::set("hp5b-override-relaxed", DurabilityMode::Relaxed);
assert_eq!(effective_durability("hp5b-override-relaxed"), DurabilityMode::Relaxed);
assert_eq!(effective_durability("hp5b-other-bucket"), DurabilityMode::Strict);
assert_eq!(effective_durability(RUSTFS_META_BUCKET), DurabilityMode::Strict);
}
// Override cleared: the bucket follows the global mode again (a new
// PUT after a config change resolves the new tier).
{
let _mode = durability_mode_override::set(DurabilityMode::Strict);
assert_eq!(effective_durability("hp5b-override-relaxed"), DurabilityMode::Strict);
}
// Global relaxed + per-bucket strict: overrides can raise durability.
{
let _mode = durability_mode_override::set(DurabilityMode::Relaxed);
let _guard = BucketOverrideGuard::set("hp5b-override-strict", DurabilityMode::Strict);
assert_eq!(effective_durability("hp5b-override-strict"), DurabilityMode::Strict);
assert_eq!(effective_durability("hp5b-other-bucket"), DurabilityMode::Relaxed);
}
}
#[test]
fn test_bucket_durability_refuses_system_namespaces() {
let _mode = durability_mode_override::set(DurabilityMode::Strict);
// System-critical and scratch namespaces can never carry an override.
bucket_durability::set(RUSTFS_META_BUCKET, Some(DurabilityMode::Relaxed));
bucket_durability::set(&format!("{RUSTFS_META_BUCKET}/buckets"), Some(DurabilityMode::None));
bucket_durability::set(RUSTFS_META_TMP_BUCKET, Some(DurabilityMode::Relaxed));
bucket_durability::set(super::super::RUSTFS_META_MULTIPART_BUCKET, Some(DurabilityMode::Relaxed));
bucket_durability::set("", Some(DurabilityMode::Relaxed));
assert_eq!(bucket_durability::lookup(RUSTFS_META_BUCKET), Option::None);
assert_eq!(bucket_durability::lookup(RUSTFS_META_TMP_BUCKET), Option::None);
assert_eq!(effective_durability(RUSTFS_META_BUCKET), DurabilityMode::Strict);
// The legacy full-off tier is process-wide only: registering it per
// bucket is dropped, not stored.
bucket_durability::set("hp5b-legacy-refused", Some(DurabilityMode::LegacyOff));
assert_eq!(bucket_durability::lookup("hp5b-legacy-refused"), Option::None);
}
#[test]
fn test_effective_durability_legacy_off_ignores_bucket_overrides() {
let _mode = durability_mode_override::set(DurabilityMode::LegacyOff);
let _guard = BucketOverrideGuard::set("hp5b-legacy-bucket", DurabilityMode::Strict);
// The legacy switch keeps its historical semantics bit for bit.
assert_eq!(effective_durability("hp5b-legacy-bucket"), DurabilityMode::LegacyOff);
}
/// HP-5b behavior regression: with the global mode at strict (the
/// default), a bucket override to relaxed must skip the metadata-commit
/// dir fsync for that bucket only, and clearing the override must restore
/// the strict behavior for the next write.
#[tokio::test]
#[allow(clippy::await_holding_lock)]
async fn test_write_all_meta_bucket_override_relaxed_then_cleared() {
use tempfile::tempdir;
let _mode = durability_mode_override::set(DurabilityMode::Strict);
let dir = tempdir().expect("temp dir should be created");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
let overridden = "hp5b-relaxed-write-bucket";
let untouched = "hp5b-strict-write-bucket";
ensure_test_volume(&disk, overridden).await;
ensure_test_volume(&disk, untouched).await;
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
let meta_path = format!("dir/object/{STORAGE_FORMAT_FILE}");
{
let _guard = BucketOverrideGuard::set("hp5b-relaxed-write-bucket", DurabilityMode::Relaxed);
disk.write_all_meta(overridden, &meta_path, b"payload", true)
.await
.expect("write_all_meta should succeed");
let overridden_parent = disk
.get_object_path(overridden, &meta_path)
.expect("dst path should resolve")
.parent()
.expect("dst file should have a parent")
.to_path_buf();
assert!(
!os::fsync_dir_recorder::was_fsynced(&overridden_parent),
"bucket override to relaxed must skip the metadata-commit dir fsync"
);
// A bucket without an override keeps the strict default.
disk.write_all_meta(untouched, &meta_path, b"payload", true)
.await
.expect("write_all_meta should succeed");
let untouched_parent = disk
.get_object_path(untouched, &meta_path)
.expect("dst path should resolve")
.parent()
.expect("dst file should have a parent")
.to_path_buf();
assert!(
os::fsync_dir_recorder::was_fsynced(&untouched_parent),
"buckets without an override must keep the strict commit fsyncs"
);
}
// Override cleared (guard dropped): the next write is strict again.
let second_meta_path = format!("dir/object-after-clear/{STORAGE_FORMAT_FILE}");
disk.write_all_meta(overridden, &second_meta_path, b"payload", true)
.await
.expect("write_all_meta should succeed");
let after_clear_parent = disk
.get_object_path(overridden, &second_meta_path)
.expect("dst path should resolve")
.parent()
.expect("dst file should have a parent")
.to_path_buf();
assert!(
os::fsync_dir_recorder::was_fsynced(&after_clear_parent),
"clearing the override must restore strict fsyncs for new writes"
);
}
#[tokio::test]
#[allow(clippy::await_holding_lock)]
async fn test_write_all_meta_relaxed_skips_dst_parent_dir_fsync() {