mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-13 08:36:54 +00:00
perf(ecstore): support per-bucket durability tier overrides (#4407)
perf(ecstore): per-bucket durability tier overrides (HP-5 phase 2) Let a bucket override the process-wide RUSTFS_DURABILITY_MODE with its own strict/relaxed/none tier, stored as a durability.json extension entry in the bucket metadata file and resolved at commit points via effective_durability. System-critical buckets (.rustfs.sys, .minio.sys) can never carry an override and stay pinned to strict; the legacy full-off switch keeps its historical semantics and per-bucket overrides do not apply under it. Overrides are published and cleared through the existing bucket metadata cache-invalidation path, and an admin GET/PUT handler exposes the configuration. Default behavior is unchanged: with no override a bucket follows the global mode, which defaults to strict and stays byte-for-byte identical to before. Refs: https://github.com/rustfs/backlog/issues/938, https://github.com/rustfs/backlog/issues/936 Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
@@ -82,6 +82,7 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod metadata {
|
||||
pub use crate::bucket::metadata::BUCKET_DURABILITY_CONFIG;
|
||||
pub use crate::bucket::metadata::{
|
||||
BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG,
|
||||
BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_QUOTA_CONFIG_FILE,
|
||||
@@ -92,14 +93,20 @@ pub mod bucket {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod durability {
|
||||
pub use crate::bucket::durability::{
|
||||
BUCKET_DURABILITY_MODE_NONE, BUCKET_DURABILITY_MODE_RELAXED, BUCKET_DURABILITY_MODE_STRICT, BucketDurabilityConfig,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod metadata_sys {
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BucketMetadataSys, delete, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw,
|
||||
get_bucket_targets_config, get_config_from_disk, get_cors_config, get_global_bucket_metadata_sys,
|
||||
get_lifecycle_config, get_logging_config, get_notification_config, get_object_lock_config,
|
||||
get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config,
|
||||
get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets,
|
||||
remove_bucket_metadata, set_bucket_metadata, update,
|
||||
get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
|
||||
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
|
||||
init_bucket_metadata_sys, list_bucket_targets, remove_bucket_metadata, set_bucket_metadata, update,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-bucket durability tier configuration (HP-5 phase 2).
|
||||
//!
|
||||
//! A bucket can override the process-wide durability mode
|
||||
//! (`RUSTFS_DURABILITY_MODE`) with its own tier. The override is stored as a
|
||||
//! RustFS extension entry (`durability.json`) in the bucket metadata file and
|
||||
//! is resolved by the disk layer at commit points via
|
||||
//! `effective_durability(volume)`. System buckets (`.rustfs.sys`,
|
||||
//! `.minio.sys`) can never carry an override; they stay pinned to `strict`.
|
||||
//! See docs/operations/durability-modes.md for the semantics.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Valid per-bucket durability tier names. `legacy-off` is deliberately not
|
||||
/// representable per bucket: it exists only for the process-wide legacy
|
||||
/// switch (`RUSTFS_DRIVE_SYNC_ENABLE=false`).
|
||||
pub const BUCKET_DURABILITY_MODE_STRICT: &str = "strict";
|
||||
pub const BUCKET_DURABILITY_MODE_RELAXED: &str = "relaxed";
|
||||
pub const BUCKET_DURABILITY_MODE_NONE: &str = "none";
|
||||
|
||||
/// JSON payload stored under the `durability.json` bucket metadata entry.
|
||||
///
|
||||
/// An absent entry, an empty payload, or an empty `mode` string all mean
|
||||
/// "no override": the bucket follows the process-wide durability mode.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct BucketDurabilityConfig {
|
||||
/// Requested tier: `strict` | `relaxed` | `none` (case-insensitive,
|
||||
/// whitespace-tolerant). Empty means "inherit the global mode".
|
||||
#[serde(default)]
|
||||
pub mode: String,
|
||||
}
|
||||
|
||||
impl BucketDurabilityConfig {
|
||||
pub fn new(mode: &str) -> Self {
|
||||
Self { mode: mode.to_string() }
|
||||
}
|
||||
|
||||
/// Whether `mode` names a valid per-bucket tier.
|
||||
pub fn is_valid_mode(mode: &str) -> bool {
|
||||
matches!(
|
||||
mode.trim().to_ascii_lowercase().as_str(),
|
||||
BUCKET_DURABILITY_MODE_STRICT | BUCKET_DURABILITY_MODE_RELAXED | BUCKET_DURABILITY_MODE_NONE
|
||||
)
|
||||
}
|
||||
|
||||
/// The canonical (trimmed, lowercase) tier name, or `None` when the
|
||||
/// config does not carry a valid override.
|
||||
pub fn normalized_mode(&self) -> Option<String> {
|
||||
let mode = self.mode.trim().to_ascii_lowercase();
|
||||
if Self::is_valid_mode(&mode) { Some(mode) } else { None }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn valid_modes_are_recognized() {
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("strict"));
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("relaxed"));
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("none"));
|
||||
assert!(BucketDurabilityConfig::is_valid_mode(" RELAXED "));
|
||||
assert!(!BucketDurabilityConfig::is_valid_mode(""));
|
||||
assert!(!BucketDurabilityConfig::is_valid_mode("bogus"));
|
||||
// The legacy full-off switch is process-wide only.
|
||||
assert!(!BucketDurabilityConfig::is_valid_mode("legacy-off"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalized_mode_canonicalizes_or_rejects() {
|
||||
assert_eq!(BucketDurabilityConfig::new(" Relaxed ").normalized_mode().as_deref(), Some("relaxed"));
|
||||
assert_eq!(BucketDurabilityConfig::new("strict").normalized_mode().as_deref(), Some("strict"));
|
||||
assert_eq!(BucketDurabilityConfig::default().normalized_mode(), None);
|
||||
assert_eq!(BucketDurabilityConfig::new("bogus").normalized_mode(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_round_trip() {
|
||||
let cfg = BucketDurabilityConfig::new("relaxed");
|
||||
let json = serde_json::to_vec(&cfg).expect("serialize");
|
||||
let back: BucketDurabilityConfig = serde_json::from_slice(&json).expect("deserialize");
|
||||
assert_eq!(back, cfg);
|
||||
|
||||
// Empty object deserializes to the "inherit" default.
|
||||
let empty: BucketDurabilityConfig = serde_json::from_slice(b"{}").expect("deserialize empty");
|
||||
assert_eq!(empty.normalized_mode(), None);
|
||||
}
|
||||
}
|
||||
@@ -246,6 +246,7 @@ pub const BUCKET_REQUEST_PAYMENT_CONFIG: &str = "request-payment.xml";
|
||||
pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
|
||||
pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
|
||||
pub const BUCKET_TABLE_CONFIG: &str = "table-bucket.json";
|
||||
pub const BUCKET_DURABILITY_CONFIG: &str = "durability.json";
|
||||
pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table";
|
||||
pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog";
|
||||
pub const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str = "table-buckets";
|
||||
@@ -294,6 +295,7 @@ pub struct BucketMetadata {
|
||||
pub public_access_block_config_xml: Vec<u8>,
|
||||
pub bucket_acl_config_json: Vec<u8>,
|
||||
pub table_bucket_config_json: Vec<u8>,
|
||||
pub durability_config_json: Vec<u8>,
|
||||
|
||||
pub policy_config_updated_at: OffsetDateTime,
|
||||
pub object_lock_config_updated_at: OffsetDateTime,
|
||||
@@ -314,6 +316,7 @@ pub struct BucketMetadata {
|
||||
pub public_access_block_config_updated_at: OffsetDateTime,
|
||||
pub bucket_acl_config_updated_at: OffsetDateTime,
|
||||
pub table_bucket_config_updated_at: OffsetDateTime,
|
||||
pub durability_config_updated_at: OffsetDateTime,
|
||||
|
||||
pub new_field_updated_at: OffsetDateTime,
|
||||
|
||||
@@ -362,6 +365,7 @@ impl Default for BucketMetadata {
|
||||
public_access_block_config_xml: Default::default(),
|
||||
bucket_acl_config_json: Default::default(),
|
||||
table_bucket_config_json: Default::default(),
|
||||
durability_config_json: Default::default(),
|
||||
policy_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
@@ -381,6 +385,7 @@ impl Default for BucketMetadata {
|
||||
public_access_block_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
bucket_acl_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
table_bucket_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
durability_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
new_field_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
policy_config: Default::default(),
|
||||
notification_config: Default::default(),
|
||||
@@ -430,6 +435,32 @@ impl BucketMetadata {
|
||||
!self.table_bucket_config_json.is_empty()
|
||||
}
|
||||
|
||||
/// Parsed per-bucket durability override, if a valid one is stored.
|
||||
///
|
||||
/// Absent/empty/unparsable payloads all mean "no override" (the bucket
|
||||
/// follows the global durability mode); a parse failure is logged so a
|
||||
/// corrupted entry cannot silently change fsync behavior.
|
||||
pub fn durability_config(&self) -> Option<super::durability::BucketDurabilityConfig> {
|
||||
if self.durability_config_json.is_empty() {
|
||||
return None;
|
||||
}
|
||||
match serde_json::from_slice(&self.durability_config_json) {
|
||||
Ok(cfg) => Some(cfg),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %self.name,
|
||||
config = "durability",
|
||||
error = %e,
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode from msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn decode_from<R: Read>(&mut self, rd: &mut R) -> Result<()> {
|
||||
let mut fields = rmp::decode::read_map_len(rd)?;
|
||||
@@ -481,6 +512,7 @@ impl BucketMetadata {
|
||||
}
|
||||
"BucketAclConfigJSON" | "BucketAclConfigJson" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
|
||||
"TableBucketConfigJSON" | "TableBucketConfigJson" => self.table_bucket_config_json = read_msgp_bin(rd)?,
|
||||
"DurabilityConfigJSON" | "DurabilityConfigJson" => self.durability_config_json = read_msgp_bin(rd)?,
|
||||
"CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"LoggingConfigUpdatedAt" => self.logging_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"WebsiteConfigUpdatedAt" => self.website_config_updated_at = read_msgp_time_value(rd)?,
|
||||
@@ -489,6 +521,7 @@ impl BucketMetadata {
|
||||
"PublicAccessBlockConfigUpdatedAt" => self.public_access_block_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
|
||||
other => {
|
||||
tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
|
||||
skip_msgp_value(rd)?;
|
||||
@@ -501,8 +534,8 @@ impl BucketMetadata {
|
||||
|
||||
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
|
||||
// Map size: MinIO fields (25) + RustFS extensions (16)
|
||||
let map_len: u32 = 41;
|
||||
// Map size: MinIO fields (25) + RustFS extensions (18)
|
||||
let map_len: u32 = 43;
|
||||
rmp::encode::write_map_len(wr, map_len)?;
|
||||
|
||||
// MinIO field order (same as Go struct)
|
||||
@@ -559,6 +592,7 @@ impl BucketMetadata {
|
||||
write_bin_field(wr, "PublicAccessBlockConfigXML", &self.public_access_block_config_xml)?;
|
||||
write_bin_field(wr, "BucketAclConfigJSON", &self.bucket_acl_config_json)?;
|
||||
write_bin_field(wr, "TableBucketConfigJSON", &self.table_bucket_config_json)?;
|
||||
write_bin_field(wr, "DurabilityConfigJSON", &self.durability_config_json)?;
|
||||
rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.cors_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "LoggingConfigUpdatedAt")?;
|
||||
@@ -575,6 +609,8 @@ impl BucketMetadata {
|
||||
write_msgp_time(wr, self.bucket_acl_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "TableBucketConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.table_bucket_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "DurabilityConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.durability_config_updated_at)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -673,6 +709,9 @@ impl BucketMetadata {
|
||||
if self.table_bucket_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.table_bucket_config_updated_at = self.created
|
||||
}
|
||||
if self.durability_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.durability_config_updated_at = self.created
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_config(&mut self, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
|
||||
@@ -755,6 +794,10 @@ impl BucketMetadata {
|
||||
self.table_bucket_config_json = data;
|
||||
self.table_bucket_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_DURABILITY_CONFIG => {
|
||||
self.durability_config_json = data;
|
||||
self.durability_config_updated_at = updated;
|
||||
}
|
||||
_ => return Err(Error::other(format!("config file not found : {config_file}"))),
|
||||
}
|
||||
|
||||
@@ -1444,6 +1487,35 @@ mod test {
|
||||
assert!(!bm.table_bucket_enabled());
|
||||
}
|
||||
|
||||
/// HP-5b (rustfs/backlog#938): the durability override is a RustFS
|
||||
/// extension entry and must survive an encode/decode round trip.
|
||||
#[test]
|
||||
fn durability_config_round_trips_and_tracks_updates() {
|
||||
let mut bm = BucketMetadata::new("durability-bucket");
|
||||
assert!(bm.durability_config().is_none(), "fresh metadata carries no override");
|
||||
|
||||
bm.update_config(BUCKET_DURABILITY_CONFIG, br#"{"mode":"relaxed"}"#.to_vec())
|
||||
.unwrap();
|
||||
assert_ne!(bm.durability_config_updated_at, OffsetDateTime::UNIX_EPOCH);
|
||||
|
||||
let buf = bm.marshal_msg().unwrap();
|
||||
let back = BucketMetadata::unmarshal(&buf).unwrap();
|
||||
assert_eq!(back.durability_config_json, bm.durability_config_json);
|
||||
assert_eq!(
|
||||
back.durability_config_updated_at.unix_timestamp(),
|
||||
bm.durability_config_updated_at.unix_timestamp()
|
||||
);
|
||||
assert_eq!(back.durability_config().and_then(|c| c.normalized_mode()).as_deref(), Some("relaxed"));
|
||||
|
||||
// Clearing the entry removes the override.
|
||||
bm.update_config(BUCKET_DURABILITY_CONFIG, Vec::new()).unwrap();
|
||||
assert!(bm.durability_config().is_none());
|
||||
|
||||
// Corrupted payloads must degrade to "no override", never to a tier.
|
||||
bm.durability_config_json = b"not-json".to_vec();
|
||||
assert!(bm.durability_config().is_none());
|
||||
}
|
||||
|
||||
/// After policy deletion (policy_config_json cleared), parse_policy_config sets policy_config to None.
|
||||
#[test]
|
||||
fn test_parse_policy_config_clears_cache_when_json_empty() {
|
||||
|
||||
@@ -153,6 +153,26 @@ async fn sync_bucket_target_sys(bucket: &str, bm: &BucketMetadata) {
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Publish the bucket's durability override (or its absence) to the disk
|
||||
/// layer registry consulted by `effective_durability`.
|
||||
///
|
||||
/// Called from every path that installs a bucket's metadata into the cache
|
||||
/// (initial load, config update, peer reload notification, refresh loop,
|
||||
/// lazy load), so the override propagates with exactly the bucket-metadata
|
||||
/// cache invalidation semantics and never through a channel of its own.
|
||||
fn sync_bucket_durability(bucket: &str, bm: &BucketMetadata) {
|
||||
let mode = bm
|
||||
.durability_config()
|
||||
.and_then(|cfg| cfg.normalized_mode())
|
||||
.and_then(|mode| crate::disk::local::DurabilityMode::parse(&mode));
|
||||
crate::disk::local::bucket_durability::set(bucket, mode);
|
||||
}
|
||||
|
||||
/// Drop a bucket's durability override when its metadata leaves the cache.
|
||||
fn clear_bucket_durability(bucket: &str) {
|
||||
crate::disk::local::bucket_durability::set(bucket, None);
|
||||
}
|
||||
|
||||
pub async fn get(bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
let lock = sys.read().await;
|
||||
@@ -196,6 +216,20 @@ pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTi
|
||||
bucket_meta_sys.get_bucket_acl_config(bucket).await
|
||||
}
|
||||
|
||||
/// The bucket's durability override config (if any) with its update time.
|
||||
///
|
||||
/// `Ok((None, ..))` means the bucket has no override and follows the global
|
||||
/// durability mode.
|
||||
pub async fn get_durability_config(
|
||||
bucket: &str,
|
||||
) -> Result<(Option<crate::bucket::durability::BucketDurabilityConfig>, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
let (bm, _) = bucket_meta_sys.get_config(bucket).await?;
|
||||
Ok((bm.durability_config(), bm.durability_config_updated_at))
|
||||
}
|
||||
|
||||
pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -425,6 +459,7 @@ impl BucketMetadataSys {
|
||||
map.insert(bucket.clone(), bm.clone());
|
||||
drop(map);
|
||||
sync_bucket_target_sys(&bucket, &bm).await;
|
||||
sync_bucket_durability(&bucket, &bm);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -440,6 +475,7 @@ impl BucketMetadataSys {
|
||||
drop(map);
|
||||
if removed {
|
||||
BucketTargetSys::get().delete(bucket).await;
|
||||
clear_bucket_durability(bucket);
|
||||
}
|
||||
removed
|
||||
}
|
||||
@@ -540,6 +576,7 @@ impl BucketMetadataSys {
|
||||
map.insert(bucket.to_string(), bm.clone());
|
||||
drop(map);
|
||||
sync_bucket_target_sys(bucket, &bm).await;
|
||||
sync_bucket_durability(bucket, &bm);
|
||||
|
||||
Ok((bm, true))
|
||||
}
|
||||
@@ -826,6 +863,40 @@ mod tests {
|
||||
target_sys.delete(bucket).await;
|
||||
}
|
||||
|
||||
/// HP-5b (rustfs/backlog#938): installing bucket metadata publishes the
|
||||
/// durability override to the disk-layer registry, and clearing the
|
||||
/// config (or an invalid payload) withdraws it.
|
||||
#[test]
|
||||
fn metadata_sync_publishes_and_clears_durability_override() {
|
||||
use crate::disk::local::{DurabilityMode, bucket_durability};
|
||||
|
||||
let bucket = "metadata-sync-durability";
|
||||
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.durability_config_json = br#"{"mode":"relaxed"}"#.to_vec();
|
||||
sync_bucket_durability(bucket, &bm);
|
||||
assert_eq!(bucket_durability::lookup(bucket), Some(DurabilityMode::Relaxed));
|
||||
|
||||
// Metadata without the config entry clears the override.
|
||||
let bm = BucketMetadata::new(bucket);
|
||||
sync_bucket_durability(bucket, &bm);
|
||||
assert_eq!(bucket_durability::lookup(bucket), None);
|
||||
|
||||
// Invalid payloads degrade to "no override", never to a tier.
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.durability_config_json = br#"{"mode":"bogus"}"#.to_vec();
|
||||
sync_bucket_durability(bucket, &bm);
|
||||
assert_eq!(bucket_durability::lookup(bucket), None);
|
||||
|
||||
// Cache removal clears the override too.
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.durability_config_json = br#"{"mode":"none"}"#.to_vec();
|
||||
sync_bucket_durability(bucket, &bm);
|
||||
assert_eq!(bucket_durability::lookup(bucket), Some(DurabilityMode::None));
|
||||
clear_bucket_durability(bucket);
|
||||
assert_eq!(bucket_durability::lookup(bucket), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_wait_exits_when_cancelled() {
|
||||
let cancel_token = CancellationToken::new();
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
|
||||
pub mod bandwidth;
|
||||
pub mod bucket_target_sys;
|
||||
pub mod durability;
|
||||
pub mod error;
|
||||
pub mod lifecycle;
|
||||
pub mod metadata;
|
||||
|
||||
@@ -288,7 +288,7 @@ pub(crate) enum DurabilityMode {
|
||||
}
|
||||
|
||||
impl DurabilityMode {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
pub(crate) fn parse(value: &str) -> Option<Self> {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"strict" => Some(Self::Strict),
|
||||
"relaxed" => Some(Self::Relaxed),
|
||||
@@ -406,6 +406,96 @@ pub(crate) mod durability_mode_override {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-bucket durability overrides (HP-5 phase 2, rustfs/backlog#938).
|
||||
///
|
||||
/// The disk layer never loads bucket metadata itself: the bucket metadata
|
||||
/// subsystem publishes the parsed override here whenever a bucket's cached
|
||||
/// metadata is set, refreshed, or removed, so this registry follows exactly
|
||||
/// the existing bucket-metadata cache invalidation semantics (immediate on
|
||||
/// the node applying a config change, peer reload notification plus the
|
||||
/// periodic refresh loop elsewhere). Lookups sit on the commit hot path, so
|
||||
/// the empty-registry case (no bucket overrides configured anywhere — the
|
||||
/// default) is a single relaxed atomic load and the phase 1 behavior is
|
||||
/// preserved bit for bit.
|
||||
pub(crate) mod bucket_durability {
|
||||
use super::{
|
||||
DurabilityMode, EVENT_DISK_LOCAL_DURABILITY_MODE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_DISK_LOCAL, is_scratch_volume,
|
||||
is_system_critical_volume,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{OnceLock, PoisonError, RwLock};
|
||||
use tracing::{info, warn};
|
||||
|
||||
static OVERRIDES: OnceLock<RwLock<HashMap<String, DurabilityMode>>> = OnceLock::new();
|
||||
/// Fast-path gate: false means "no override registered anywhere", which
|
||||
/// keeps default deployments off the map lookup entirely.
|
||||
static NON_EMPTY: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
fn overrides() -> &'static RwLock<HashMap<String, DurabilityMode>> {
|
||||
OVERRIDES.get_or_init(|| RwLock::new(HashMap::new()))
|
||||
}
|
||||
|
||||
/// Publish (or clear, with `None`) the durability override for `bucket`.
|
||||
///
|
||||
/// System namespaces can never carry an override: they are pinned to
|
||||
/// `strict` by [`super::effective_durability`], and any attempt to
|
||||
/// register one is rejected here as defense in depth.
|
||||
pub(crate) fn set(bucket: &str, mode: Option<DurabilityMode>) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
if is_system_critical_volume(bucket) || is_scratch_volume(bucket) {
|
||||
if mode.is_some() {
|
||||
warn!(
|
||||
event = EVENT_DISK_LOCAL_DURABILITY_MODE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
bucket = %bucket,
|
||||
"Rejected per-bucket durability override for a system namespace; it stays pinned to strict"
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// The legacy full-off switch is process-wide only and deliberately
|
||||
// unreachable per bucket (`DurabilityMode::parse` never returns it).
|
||||
let mode = mode.filter(|m| *m != DurabilityMode::LegacyOff);
|
||||
|
||||
let mut map = overrides().write().unwrap_or_else(PoisonError::into_inner);
|
||||
let changed = match mode {
|
||||
Some(mode) => map.insert(bucket.to_string(), mode) != Some(mode),
|
||||
None => map.remove(bucket).is_some(),
|
||||
};
|
||||
NON_EMPTY.store(!map.is_empty(), Ordering::Release);
|
||||
drop(map);
|
||||
|
||||
if changed {
|
||||
info!(
|
||||
event = EVENT_DISK_LOCAL_DURABILITY_MODE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
bucket = %bucket,
|
||||
mode = mode.map_or("inherit", |m| m.as_str()),
|
||||
"Per-bucket durability override updated"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The override registered for `volume`, if any. `volume` is the commit
|
||||
/// destination, so user buckets resolve by name while scratch and system
|
||||
/// namespaces never match (they are refused by [`set`]).
|
||||
pub(crate) fn lookup(volume: &str) -> Option<DurabilityMode> {
|
||||
if !NON_EMPTY.load(Ordering::Acquire) {
|
||||
return None;
|
||||
}
|
||||
overrides()
|
||||
.read()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.get(volume)
|
||||
.copied()
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `volume` stages in-flight user object data (`.rustfs.sys/tmp`,
|
||||
/// `.rustfs.sys/multipart`, and their subtrees). These namespaces follow the
|
||||
/// configured durability mode: their contents commit into user buckets and
|
||||
@@ -438,21 +528,21 @@ fn is_system_critical_volume(volume: &str) -> bool {
|
||||
|
||||
/// Effective durability for writes that commit into `volume`.
|
||||
///
|
||||
/// System-critical volumes are pinned to `Strict` regardless of the
|
||||
/// configured tier. The legacy full-off switch keeps its historical
|
||||
/// semantics and is never pinned.
|
||||
/// Resolution order: system-critical volumes are pinned to `Strict`
|
||||
/// regardless of any configuration; otherwise a per-bucket override
|
||||
/// (published by the bucket metadata subsystem, see [`bucket_durability`])
|
||||
/// wins over the process-wide mode; otherwise the process-wide mode applies.
|
||||
/// The legacy full-off switch keeps its historical semantics: it is never
|
||||
/// pinned and per-bucket overrides do not apply under it.
|
||||
pub(crate) fn effective_durability(volume: &str) -> DurabilityMode {
|
||||
let mode = durability_mode();
|
||||
match mode {
|
||||
DurabilityMode::Strict | DurabilityMode::LegacyOff => mode,
|
||||
DurabilityMode::Relaxed | DurabilityMode::None => {
|
||||
if is_system_critical_volume(volume) {
|
||||
DurabilityMode::Strict
|
||||
} else {
|
||||
mode
|
||||
}
|
||||
}
|
||||
let global = durability_mode();
|
||||
if global == DurabilityMode::LegacyOff {
|
||||
return global;
|
||||
}
|
||||
if is_system_critical_volume(volume) {
|
||||
return DurabilityMode::Strict;
|
||||
}
|
||||
bucket_durability::lookup(volume).unwrap_or(global)
|
||||
}
|
||||
|
||||
/// Get the O_DIRECT read threshold size.
|
||||
@@ -5627,6 +5717,150 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
/// Removes the bucket's durability override when dropped so a test can
|
||||
/// never leak its override into another test's lookup.
|
||||
struct BucketOverrideGuard(&'static str);
|
||||
|
||||
impl BucketOverrideGuard {
|
||||
fn set(bucket: &'static str, mode: DurabilityMode) -> Self {
|
||||
bucket_durability::set(bucket, Some(mode));
|
||||
Self(bucket)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BucketOverrideGuard {
|
||||
fn drop(&mut self) {
|
||||
bucket_durability::set(self.0, None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_effective_durability_bucket_override() {
|
||||
// Global strict + per-bucket relaxed: only the named bucket drops.
|
||||
{
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
let _guard = BucketOverrideGuard::set("hp5b-override-relaxed", DurabilityMode::Relaxed);
|
||||
assert_eq!(effective_durability("hp5b-override-relaxed"), DurabilityMode::Relaxed);
|
||||
assert_eq!(effective_durability("hp5b-other-bucket"), DurabilityMode::Strict);
|
||||
assert_eq!(effective_durability(RUSTFS_META_BUCKET), DurabilityMode::Strict);
|
||||
}
|
||||
// Override cleared: the bucket follows the global mode again (a new
|
||||
// PUT after a config change resolves the new tier).
|
||||
{
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
assert_eq!(effective_durability("hp5b-override-relaxed"), DurabilityMode::Strict);
|
||||
}
|
||||
// Global relaxed + per-bucket strict: overrides can raise durability.
|
||||
{
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Relaxed);
|
||||
let _guard = BucketOverrideGuard::set("hp5b-override-strict", DurabilityMode::Strict);
|
||||
assert_eq!(effective_durability("hp5b-override-strict"), DurabilityMode::Strict);
|
||||
assert_eq!(effective_durability("hp5b-other-bucket"), DurabilityMode::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_durability_refuses_system_namespaces() {
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
|
||||
// System-critical and scratch namespaces can never carry an override.
|
||||
bucket_durability::set(RUSTFS_META_BUCKET, Some(DurabilityMode::Relaxed));
|
||||
bucket_durability::set(&format!("{RUSTFS_META_BUCKET}/buckets"), Some(DurabilityMode::None));
|
||||
bucket_durability::set(RUSTFS_META_TMP_BUCKET, Some(DurabilityMode::Relaxed));
|
||||
bucket_durability::set(super::super::RUSTFS_META_MULTIPART_BUCKET, Some(DurabilityMode::Relaxed));
|
||||
bucket_durability::set("", Some(DurabilityMode::Relaxed));
|
||||
|
||||
assert_eq!(bucket_durability::lookup(RUSTFS_META_BUCKET), Option::None);
|
||||
assert_eq!(bucket_durability::lookup(RUSTFS_META_TMP_BUCKET), Option::None);
|
||||
assert_eq!(effective_durability(RUSTFS_META_BUCKET), DurabilityMode::Strict);
|
||||
|
||||
// The legacy full-off tier is process-wide only: registering it per
|
||||
// bucket is dropped, not stored.
|
||||
bucket_durability::set("hp5b-legacy-refused", Some(DurabilityMode::LegacyOff));
|
||||
assert_eq!(bucket_durability::lookup("hp5b-legacy-refused"), Option::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_effective_durability_legacy_off_ignores_bucket_overrides() {
|
||||
let _mode = durability_mode_override::set(DurabilityMode::LegacyOff);
|
||||
let _guard = BucketOverrideGuard::set("hp5b-legacy-bucket", DurabilityMode::Strict);
|
||||
// The legacy switch keeps its historical semantics bit for bit.
|
||||
assert_eq!(effective_durability("hp5b-legacy-bucket"), DurabilityMode::LegacyOff);
|
||||
}
|
||||
|
||||
/// HP-5b behavior regression: with the global mode at strict (the
|
||||
/// default), a bucket override to relaxed must skip the metadata-commit
|
||||
/// dir fsync for that bucket only, and clearing the override must restore
|
||||
/// the strict behavior for the next write.
|
||||
#[tokio::test]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn test_write_all_meta_bucket_override_relaxed_then_cleared() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
|
||||
let overridden = "hp5b-relaxed-write-bucket";
|
||||
let untouched = "hp5b-strict-write-bucket";
|
||||
ensure_test_volume(&disk, overridden).await;
|
||||
ensure_test_volume(&disk, untouched).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
|
||||
let meta_path = format!("dir/object/{STORAGE_FORMAT_FILE}");
|
||||
|
||||
{
|
||||
let _guard = BucketOverrideGuard::set("hp5b-relaxed-write-bucket", DurabilityMode::Relaxed);
|
||||
|
||||
disk.write_all_meta(overridden, &meta_path, b"payload", true)
|
||||
.await
|
||||
.expect("write_all_meta should succeed");
|
||||
let overridden_parent = disk
|
||||
.get_object_path(overridden, &meta_path)
|
||||
.expect("dst path should resolve")
|
||||
.parent()
|
||||
.expect("dst file should have a parent")
|
||||
.to_path_buf();
|
||||
assert!(
|
||||
!os::fsync_dir_recorder::was_fsynced(&overridden_parent),
|
||||
"bucket override to relaxed must skip the metadata-commit dir fsync"
|
||||
);
|
||||
|
||||
// A bucket without an override keeps the strict default.
|
||||
disk.write_all_meta(untouched, &meta_path, b"payload", true)
|
||||
.await
|
||||
.expect("write_all_meta should succeed");
|
||||
let untouched_parent = disk
|
||||
.get_object_path(untouched, &meta_path)
|
||||
.expect("dst path should resolve")
|
||||
.parent()
|
||||
.expect("dst file should have a parent")
|
||||
.to_path_buf();
|
||||
assert!(
|
||||
os::fsync_dir_recorder::was_fsynced(&untouched_parent),
|
||||
"buckets without an override must keep the strict commit fsyncs"
|
||||
);
|
||||
}
|
||||
|
||||
// Override cleared (guard dropped): the next write is strict again.
|
||||
let second_meta_path = format!("dir/object-after-clear/{STORAGE_FORMAT_FILE}");
|
||||
disk.write_all_meta(overridden, &second_meta_path, b"payload", true)
|
||||
.await
|
||||
.expect("write_all_meta should succeed");
|
||||
let after_clear_parent = disk
|
||||
.get_object_path(overridden, &second_meta_path)
|
||||
.expect("dst path should resolve")
|
||||
.parent()
|
||||
.expect("dst file should have a parent")
|
||||
.to_path_buf();
|
||||
assert!(
|
||||
os::fsync_dir_recorder::was_fsynced(&after_clear_parent),
|
||||
"clearing the override must restore strict fsyncs for new writes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn test_write_all_meta_relaxed_skips_dst_parent_dir_fsync() {
|
||||
|
||||
Reference in New Issue
Block a user