From 13e48d93aaf7f6b0e63837cc809e345a52dbc8e3 Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 8 Jul 2026 16:33:49 +0800 Subject: [PATCH] perf(ecstore): support per-bucket durability tier overrides (#4407) perf(ecstore): per-bucket durability tier overrides (HP-5 phase 2) Let a bucket override the process-wide RUSTFS_DURABILITY_MODE with its own strict/relaxed/none tier, stored as a durability.json extension entry in the bucket metadata file and resolved at commit points via effective_durability. System-critical buckets (.rustfs.sys, .minio.sys) can never carry an override and stay pinned to strict; the legacy full-off switch keeps its historical semantics and per-bucket overrides do not apply under it. Overrides are published and cleared through the existing bucket metadata cache-invalidation path, and an admin GET/PUT handler exposes the configuration. Default behavior is unchanged: with no override a bucket follows the global mode, which defaults to strict and stays byte-for-byte identical to before. Refs: https://github.com/rustfs/backlog/issues/938, https://github.com/rustfs/backlog/issues/936 Co-authored-by: heihutu --- crates/ecstore/src/api/mod.rs | 17 +- crates/ecstore/src/bucket/durability.rs | 102 ++++++++ crates/ecstore/src/bucket/metadata.rs | 76 +++++- crates/ecstore/src/bucket/metadata_sys.rs | 71 +++++ crates/ecstore/src/bucket/mod.rs | 1 + crates/ecstore/src/disk/local.rs | 262 ++++++++++++++++++- docs/operations/durability-modes.md | 80 +++++- rustfs/src/admin/handlers/durability.rs | 273 ++++++++++++++++++++ rustfs/src/admin/handlers/mod.rs | 1 + rustfs/src/admin/mod.rs | 8 +- rustfs/src/admin/route_policy.rs | 18 ++ rustfs/src/admin/route_registration_test.rs | 6 + rustfs/src/admin/storage_api.rs | 16 +- rustfs/src/storage/storage_api.rs | 4 +- 14 files changed, 904 insertions(+), 31 deletions(-) create mode 100644 crates/ecstore/src/bucket/durability.rs create mode 100644 rustfs/src/admin/handlers/durability.rs diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 8e99ac21f..d461184c4 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -82,6 +82,7 @@ pub mod bucket { } pub mod metadata { + pub use crate::bucket::metadata::BUCKET_DURABILITY_CONFIG; pub use crate::bucket::metadata::{ BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_QUOTA_CONFIG_FILE, @@ -92,14 +93,20 @@ pub mod bucket { }; } + pub mod durability { + pub use crate::bucket::durability::{ + BUCKET_DURABILITY_MODE_NONE, BUCKET_DURABILITY_MODE_RELAXED, BUCKET_DURABILITY_MODE_STRICT, BucketDurabilityConfig, + }; + } + pub mod metadata_sys { pub use crate::bucket::metadata_sys::{ BucketMetadataSys, delete, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, - get_bucket_targets_config, get_config_from_disk, get_cors_config, get_global_bucket_metadata_sys, - get_lifecycle_config, get_logging_config, get_notification_config, get_object_lock_config, - get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, - get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, - remove_bucket_metadata, set_bucket_metadata, update, + get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config, + get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config, + get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config, + get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config, + init_bucket_metadata_sys, list_bucket_targets, remove_bucket_metadata, set_bucket_metadata, update, }; } diff --git a/crates/ecstore/src/bucket/durability.rs b/crates/ecstore/src/bucket/durability.rs new file mode 100644 index 000000000..404c046de --- /dev/null +++ b/crates/ecstore/src/bucket/durability.rs @@ -0,0 +1,102 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Per-bucket durability tier configuration (HP-5 phase 2). +//! +//! A bucket can override the process-wide durability mode +//! (`RUSTFS_DURABILITY_MODE`) with its own tier. The override is stored as a +//! RustFS extension entry (`durability.json`) in the bucket metadata file and +//! is resolved by the disk layer at commit points via +//! `effective_durability(volume)`. System buckets (`.rustfs.sys`, +//! `.minio.sys`) can never carry an override; they stay pinned to `strict`. +//! See docs/operations/durability-modes.md for the semantics. + +use serde::{Deserialize, Serialize}; + +/// Valid per-bucket durability tier names. `legacy-off` is deliberately not +/// representable per bucket: it exists only for the process-wide legacy +/// switch (`RUSTFS_DRIVE_SYNC_ENABLE=false`). +pub const BUCKET_DURABILITY_MODE_STRICT: &str = "strict"; +pub const BUCKET_DURABILITY_MODE_RELAXED: &str = "relaxed"; +pub const BUCKET_DURABILITY_MODE_NONE: &str = "none"; + +/// JSON payload stored under the `durability.json` bucket metadata entry. +/// +/// An absent entry, an empty payload, or an empty `mode` string all mean +/// "no override": the bucket follows the process-wide durability mode. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct BucketDurabilityConfig { + /// Requested tier: `strict` | `relaxed` | `none` (case-insensitive, + /// whitespace-tolerant). Empty means "inherit the global mode". + #[serde(default)] + pub mode: String, +} + +impl BucketDurabilityConfig { + pub fn new(mode: &str) -> Self { + Self { mode: mode.to_string() } + } + + /// Whether `mode` names a valid per-bucket tier. + pub fn is_valid_mode(mode: &str) -> bool { + matches!( + mode.trim().to_ascii_lowercase().as_str(), + BUCKET_DURABILITY_MODE_STRICT | BUCKET_DURABILITY_MODE_RELAXED | BUCKET_DURABILITY_MODE_NONE + ) + } + + /// The canonical (trimmed, lowercase) tier name, or `None` when the + /// config does not carry a valid override. + pub fn normalized_mode(&self) -> Option { + let mode = self.mode.trim().to_ascii_lowercase(); + if Self::is_valid_mode(&mode) { Some(mode) } else { None } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn valid_modes_are_recognized() { + assert!(BucketDurabilityConfig::is_valid_mode("strict")); + assert!(BucketDurabilityConfig::is_valid_mode("relaxed")); + assert!(BucketDurabilityConfig::is_valid_mode("none")); + assert!(BucketDurabilityConfig::is_valid_mode(" RELAXED ")); + assert!(!BucketDurabilityConfig::is_valid_mode("")); + assert!(!BucketDurabilityConfig::is_valid_mode("bogus")); + // The legacy full-off switch is process-wide only. + assert!(!BucketDurabilityConfig::is_valid_mode("legacy-off")); + } + + #[test] + fn normalized_mode_canonicalizes_or_rejects() { + assert_eq!(BucketDurabilityConfig::new(" Relaxed ").normalized_mode().as_deref(), Some("relaxed")); + assert_eq!(BucketDurabilityConfig::new("strict").normalized_mode().as_deref(), Some("strict")); + assert_eq!(BucketDurabilityConfig::default().normalized_mode(), None); + assert_eq!(BucketDurabilityConfig::new("bogus").normalized_mode(), None); + } + + #[test] + fn json_round_trip() { + let cfg = BucketDurabilityConfig::new("relaxed"); + let json = serde_json::to_vec(&cfg).expect("serialize"); + let back: BucketDurabilityConfig = serde_json::from_slice(&json).expect("deserialize"); + assert_eq!(back, cfg); + + // Empty object deserializes to the "inherit" default. + let empty: BucketDurabilityConfig = serde_json::from_slice(b"{}").expect("deserialize empty"); + assert_eq!(empty.normalized_mode(), None); + } +} diff --git a/crates/ecstore/src/bucket/metadata.rs b/crates/ecstore/src/bucket/metadata.rs index 41b9063d1..31ac19520 100644 --- a/crates/ecstore/src/bucket/metadata.rs +++ b/crates/ecstore/src/bucket/metadata.rs @@ -246,6 +246,7 @@ pub const BUCKET_REQUEST_PAYMENT_CONFIG: &str = "request-payment.xml"; pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml"; pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json"; pub const BUCKET_TABLE_CONFIG: &str = "table-bucket.json"; +pub const BUCKET_DURABILITY_CONFIG: &str = "durability.json"; pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table"; pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog"; pub const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str = "table-buckets"; @@ -294,6 +295,7 @@ pub struct BucketMetadata { pub public_access_block_config_xml: Vec, pub bucket_acl_config_json: Vec, pub table_bucket_config_json: Vec, + pub durability_config_json: Vec, pub policy_config_updated_at: OffsetDateTime, pub object_lock_config_updated_at: OffsetDateTime, @@ -314,6 +316,7 @@ pub struct BucketMetadata { pub public_access_block_config_updated_at: OffsetDateTime, pub bucket_acl_config_updated_at: OffsetDateTime, pub table_bucket_config_updated_at: OffsetDateTime, + pub durability_config_updated_at: OffsetDateTime, pub new_field_updated_at: OffsetDateTime, @@ -362,6 +365,7 @@ impl Default for BucketMetadata { public_access_block_config_xml: Default::default(), bucket_acl_config_json: Default::default(), table_bucket_config_json: Default::default(), + durability_config_json: Default::default(), policy_config_updated_at: OffsetDateTime::UNIX_EPOCH, object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH, encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH, @@ -381,6 +385,7 @@ impl Default for BucketMetadata { public_access_block_config_updated_at: OffsetDateTime::UNIX_EPOCH, bucket_acl_config_updated_at: OffsetDateTime::UNIX_EPOCH, table_bucket_config_updated_at: OffsetDateTime::UNIX_EPOCH, + durability_config_updated_at: OffsetDateTime::UNIX_EPOCH, new_field_updated_at: OffsetDateTime::UNIX_EPOCH, policy_config: Default::default(), notification_config: Default::default(), @@ -430,6 +435,32 @@ impl BucketMetadata { !self.table_bucket_config_json.is_empty() } + /// Parsed per-bucket durability override, if a valid one is stored. + /// + /// Absent/empty/unparsable payloads all mean "no override" (the bucket + /// follows the global durability mode); a parse failure is logged so a + /// corrupted entry cannot silently change fsync behavior. + pub fn durability_config(&self) -> Option { + if self.durability_config_json.is_empty() { + return None; + } + match serde_json::from_slice(&self.durability_config_json) { + Ok(cfg) => Some(cfg), + Err(e) => { + tracing::warn!( + event = "bucket_metadata_parse_failed", + component = "ecstore", + subsystem = "bucket_metadata", + bucket = %self.name, + config = "durability", + error = %e, + "Failed to parse bucket metadata config" + ); + None + } + } + } + /// Decode from msgp bytes. Field order follows MinIO BucketMetadata for compatibility. pub fn decode_from(&mut self, rd: &mut R) -> Result<()> { let mut fields = rmp::decode::read_map_len(rd)?; @@ -481,6 +512,7 @@ impl BucketMetadata { } "BucketAclConfigJSON" | "BucketAclConfigJson" => self.bucket_acl_config_json = read_msgp_bin(rd)?, "TableBucketConfigJSON" | "TableBucketConfigJson" => self.table_bucket_config_json = read_msgp_bin(rd)?, + "DurabilityConfigJSON" | "DurabilityConfigJson" => self.durability_config_json = read_msgp_bin(rd)?, "CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?, "LoggingConfigUpdatedAt" => self.logging_config_updated_at = read_msgp_time_value(rd)?, "WebsiteConfigUpdatedAt" => self.website_config_updated_at = read_msgp_time_value(rd)?, @@ -489,6 +521,7 @@ impl BucketMetadata { "PublicAccessBlockConfigUpdatedAt" => self.public_access_block_config_updated_at = read_msgp_time_value(rd)?, "BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?, "TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?, + "DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?, other => { tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field"); skip_msgp_value(rd)?; @@ -501,8 +534,8 @@ impl BucketMetadata { /// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility. pub fn encode_to(&self, wr: &mut W) -> Result<()> { - // Map size: MinIO fields (25) + RustFS extensions (16) - let map_len: u32 = 41; + // Map size: MinIO fields (25) + RustFS extensions (18) + let map_len: u32 = 43; rmp::encode::write_map_len(wr, map_len)?; // MinIO field order (same as Go struct) @@ -559,6 +592,7 @@ impl BucketMetadata { write_bin_field(wr, "PublicAccessBlockConfigXML", &self.public_access_block_config_xml)?; write_bin_field(wr, "BucketAclConfigJSON", &self.bucket_acl_config_json)?; write_bin_field(wr, "TableBucketConfigJSON", &self.table_bucket_config_json)?; + write_bin_field(wr, "DurabilityConfigJSON", &self.durability_config_json)?; rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?; write_msgp_time(wr, self.cors_config_updated_at)?; rmp::encode::write_str(wr, "LoggingConfigUpdatedAt")?; @@ -575,6 +609,8 @@ impl BucketMetadata { write_msgp_time(wr, self.bucket_acl_config_updated_at)?; rmp::encode::write_str(wr, "TableBucketConfigUpdatedAt")?; write_msgp_time(wr, self.table_bucket_config_updated_at)?; + rmp::encode::write_str(wr, "DurabilityConfigUpdatedAt")?; + write_msgp_time(wr, self.durability_config_updated_at)?; Ok(()) } @@ -673,6 +709,9 @@ impl BucketMetadata { if self.table_bucket_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.table_bucket_config_updated_at = self.created } + if self.durability_config_updated_at == OffsetDateTime::UNIX_EPOCH { + self.durability_config_updated_at = self.created + } } pub fn update_config(&mut self, config_file: &str, data: Vec) -> Result { @@ -755,6 +794,10 @@ impl BucketMetadata { self.table_bucket_config_json = data; self.table_bucket_config_updated_at = updated; } + BUCKET_DURABILITY_CONFIG => { + self.durability_config_json = data; + self.durability_config_updated_at = updated; + } _ => return Err(Error::other(format!("config file not found : {config_file}"))), } @@ -1444,6 +1487,35 @@ mod test { assert!(!bm.table_bucket_enabled()); } + /// HP-5b (rustfs/backlog#938): the durability override is a RustFS + /// extension entry and must survive an encode/decode round trip. + #[test] + fn durability_config_round_trips_and_tracks_updates() { + let mut bm = BucketMetadata::new("durability-bucket"); + assert!(bm.durability_config().is_none(), "fresh metadata carries no override"); + + bm.update_config(BUCKET_DURABILITY_CONFIG, br#"{"mode":"relaxed"}"#.to_vec()) + .unwrap(); + assert_ne!(bm.durability_config_updated_at, OffsetDateTime::UNIX_EPOCH); + + let buf = bm.marshal_msg().unwrap(); + let back = BucketMetadata::unmarshal(&buf).unwrap(); + assert_eq!(back.durability_config_json, bm.durability_config_json); + assert_eq!( + back.durability_config_updated_at.unix_timestamp(), + bm.durability_config_updated_at.unix_timestamp() + ); + assert_eq!(back.durability_config().and_then(|c| c.normalized_mode()).as_deref(), Some("relaxed")); + + // Clearing the entry removes the override. + bm.update_config(BUCKET_DURABILITY_CONFIG, Vec::new()).unwrap(); + assert!(bm.durability_config().is_none()); + + // Corrupted payloads must degrade to "no override", never to a tier. + bm.durability_config_json = b"not-json".to_vec(); + assert!(bm.durability_config().is_none()); + } + /// After policy deletion (policy_config_json cleared), parse_policy_config sets policy_config to None. #[test] fn test_parse_policy_config_clears_cache_when_json_empty() { diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index c3ce22a1c..073059069 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -153,6 +153,26 @@ async fn sync_bucket_target_sys(bucket: &str, bm: &BucketMetadata) { .await; } +/// Publish the bucket's durability override (or its absence) to the disk +/// layer registry consulted by `effective_durability`. +/// +/// Called from every path that installs a bucket's metadata into the cache +/// (initial load, config update, peer reload notification, refresh loop, +/// lazy load), so the override propagates with exactly the bucket-metadata +/// cache invalidation semantics and never through a channel of its own. +fn sync_bucket_durability(bucket: &str, bm: &BucketMetadata) { + let mode = bm + .durability_config() + .and_then(|cfg| cfg.normalized_mode()) + .and_then(|mode| crate::disk::local::DurabilityMode::parse(&mode)); + crate::disk::local::bucket_durability::set(bucket, mode); +} + +/// Drop a bucket's durability override when its metadata leaves the cache. +fn clear_bucket_durability(bucket: &str) { + crate::disk::local::bucket_durability::set(bucket, None); +} + pub async fn get(bucket: &str) -> Result> { let sys = get_bucket_metadata_sys()?; let lock = sys.read().await; @@ -196,6 +216,20 @@ pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTi bucket_meta_sys.get_bucket_acl_config(bucket).await } +/// The bucket's durability override config (if any) with its update time. +/// +/// `Ok((None, ..))` means the bucket has no override and follows the global +/// durability mode. +pub async fn get_durability_config( + bucket: &str, +) -> Result<(Option, OffsetDateTime)> { + let bucket_meta_sys_lock = get_bucket_metadata_sys()?; + let bucket_meta_sys = bucket_meta_sys_lock.read().await; + + let (bm, _) = bucket_meta_sys.get_config(bucket).await?; + Ok((bm.durability_config(), bm.durability_config_updated_at)) +} + pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> { let bucket_meta_sys_lock = get_bucket_metadata_sys()?; let bucket_meta_sys = bucket_meta_sys_lock.read().await; @@ -425,6 +459,7 @@ impl BucketMetadataSys { map.insert(bucket.clone(), bm.clone()); drop(map); sync_bucket_target_sys(&bucket, &bm).await; + sync_bucket_durability(&bucket, &bm); } } @@ -440,6 +475,7 @@ impl BucketMetadataSys { drop(map); if removed { BucketTargetSys::get().delete(bucket).await; + clear_bucket_durability(bucket); } removed } @@ -540,6 +576,7 @@ impl BucketMetadataSys { map.insert(bucket.to_string(), bm.clone()); drop(map); sync_bucket_target_sys(bucket, &bm).await; + sync_bucket_durability(bucket, &bm); Ok((bm, true)) } @@ -826,6 +863,40 @@ mod tests { target_sys.delete(bucket).await; } + /// HP-5b (rustfs/backlog#938): installing bucket metadata publishes the + /// durability override to the disk-layer registry, and clearing the + /// config (or an invalid payload) withdraws it. + #[test] + fn metadata_sync_publishes_and_clears_durability_override() { + use crate::disk::local::{DurabilityMode, bucket_durability}; + + let bucket = "metadata-sync-durability"; + + let mut bm = BucketMetadata::new(bucket); + bm.durability_config_json = br#"{"mode":"relaxed"}"#.to_vec(); + sync_bucket_durability(bucket, &bm); + assert_eq!(bucket_durability::lookup(bucket), Some(DurabilityMode::Relaxed)); + + // Metadata without the config entry clears the override. + let bm = BucketMetadata::new(bucket); + sync_bucket_durability(bucket, &bm); + assert_eq!(bucket_durability::lookup(bucket), None); + + // Invalid payloads degrade to "no override", never to a tier. + let mut bm = BucketMetadata::new(bucket); + bm.durability_config_json = br#"{"mode":"bogus"}"#.to_vec(); + sync_bucket_durability(bucket, &bm); + assert_eq!(bucket_durability::lookup(bucket), None); + + // Cache removal clears the override too. + let mut bm = BucketMetadata::new(bucket); + bm.durability_config_json = br#"{"mode":"none"}"#.to_vec(); + sync_bucket_durability(bucket, &bm); + assert_eq!(bucket_durability::lookup(bucket), Some(DurabilityMode::None)); + clear_bucket_durability(bucket); + assert_eq!(bucket_durability::lookup(bucket), None); + } + #[tokio::test] async fn refresh_wait_exits_when_cancelled() { let cancel_token = CancellationToken::new(); diff --git a/crates/ecstore/src/bucket/mod.rs b/crates/ecstore/src/bucket/mod.rs index 9fa11cd39..4f04755e1 100644 --- a/crates/ecstore/src/bucket/mod.rs +++ b/crates/ecstore/src/bucket/mod.rs @@ -17,6 +17,7 @@ pub mod bandwidth; pub mod bucket_target_sys; +pub mod durability; pub mod error; pub mod lifecycle; pub mod metadata; diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 39f1cac21..b0dcd04db 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -288,7 +288,7 @@ pub(crate) enum DurabilityMode { } impl DurabilityMode { - fn parse(value: &str) -> Option { + pub(crate) fn parse(value: &str) -> Option { match value.trim().to_ascii_lowercase().as_str() { "strict" => Some(Self::Strict), "relaxed" => Some(Self::Relaxed), @@ -406,6 +406,96 @@ pub(crate) mod durability_mode_override { } } +/// Per-bucket durability overrides (HP-5 phase 2, rustfs/backlog#938). +/// +/// The disk layer never loads bucket metadata itself: the bucket metadata +/// subsystem publishes the parsed override here whenever a bucket's cached +/// metadata is set, refreshed, or removed, so this registry follows exactly +/// the existing bucket-metadata cache invalidation semantics (immediate on +/// the node applying a config change, peer reload notification plus the +/// periodic refresh loop elsewhere). Lookups sit on the commit hot path, so +/// the empty-registry case (no bucket overrides configured anywhere — the +/// default) is a single relaxed atomic load and the phase 1 behavior is +/// preserved bit for bit. +pub(crate) mod bucket_durability { + use super::{ + DurabilityMode, EVENT_DISK_LOCAL_DURABILITY_MODE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_DISK_LOCAL, is_scratch_volume, + is_system_critical_volume, + }; + use std::collections::HashMap; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{OnceLock, PoisonError, RwLock}; + use tracing::{info, warn}; + + static OVERRIDES: OnceLock>> = OnceLock::new(); + /// Fast-path gate: false means "no override registered anywhere", which + /// keeps default deployments off the map lookup entirely. + static NON_EMPTY: AtomicBool = AtomicBool::new(false); + + fn overrides() -> &'static RwLock> { + OVERRIDES.get_or_init(|| RwLock::new(HashMap::new())) + } + + /// Publish (or clear, with `None`) the durability override for `bucket`. + /// + /// System namespaces can never carry an override: they are pinned to + /// `strict` by [`super::effective_durability`], and any attempt to + /// register one is rejected here as defense in depth. + pub(crate) fn set(bucket: &str, mode: Option) { + if bucket.is_empty() { + return; + } + if is_system_critical_volume(bucket) || is_scratch_volume(bucket) { + if mode.is_some() { + warn!( + event = EVENT_DISK_LOCAL_DURABILITY_MODE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + bucket = %bucket, + "Rejected per-bucket durability override for a system namespace; it stays pinned to strict" + ); + } + return; + } + // The legacy full-off switch is process-wide only and deliberately + // unreachable per bucket (`DurabilityMode::parse` never returns it). + let mode = mode.filter(|m| *m != DurabilityMode::LegacyOff); + + let mut map = overrides().write().unwrap_or_else(PoisonError::into_inner); + let changed = match mode { + Some(mode) => map.insert(bucket.to_string(), mode) != Some(mode), + None => map.remove(bucket).is_some(), + }; + NON_EMPTY.store(!map.is_empty(), Ordering::Release); + drop(map); + + if changed { + info!( + event = EVENT_DISK_LOCAL_DURABILITY_MODE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + bucket = %bucket, + mode = mode.map_or("inherit", |m| m.as_str()), + "Per-bucket durability override updated" + ); + } + } + + /// The override registered for `volume`, if any. `volume` is the commit + /// destination, so user buckets resolve by name while scratch and system + /// namespaces never match (they are refused by [`set`]). + pub(crate) fn lookup(volume: &str) -> Option { + if !NON_EMPTY.load(Ordering::Acquire) { + return None; + } + overrides() + .read() + .unwrap_or_else(PoisonError::into_inner) + .get(volume) + .copied() + } +} + /// Whether `volume` stages in-flight user object data (`.rustfs.sys/tmp`, /// `.rustfs.sys/multipart`, and their subtrees). These namespaces follow the /// configured durability mode: their contents commit into user buckets and @@ -438,21 +528,21 @@ fn is_system_critical_volume(volume: &str) -> bool { /// Effective durability for writes that commit into `volume`. /// -/// System-critical volumes are pinned to `Strict` regardless of the -/// configured tier. The legacy full-off switch keeps its historical -/// semantics and is never pinned. +/// Resolution order: system-critical volumes are pinned to `Strict` +/// regardless of any configuration; otherwise a per-bucket override +/// (published by the bucket metadata subsystem, see [`bucket_durability`]) +/// wins over the process-wide mode; otherwise the process-wide mode applies. +/// The legacy full-off switch keeps its historical semantics: it is never +/// pinned and per-bucket overrides do not apply under it. pub(crate) fn effective_durability(volume: &str) -> DurabilityMode { - let mode = durability_mode(); - match mode { - DurabilityMode::Strict | DurabilityMode::LegacyOff => mode, - DurabilityMode::Relaxed | DurabilityMode::None => { - if is_system_critical_volume(volume) { - DurabilityMode::Strict - } else { - mode - } - } + let global = durability_mode(); + if global == DurabilityMode::LegacyOff { + return global; } + if is_system_critical_volume(volume) { + return DurabilityMode::Strict; + } + bucket_durability::lookup(volume).unwrap_or(global) } /// Get the O_DIRECT read threshold size. @@ -5627,6 +5717,150 @@ mod test { } } + /// Removes the bucket's durability override when dropped so a test can + /// never leak its override into another test's lookup. + struct BucketOverrideGuard(&'static str); + + impl BucketOverrideGuard { + fn set(bucket: &'static str, mode: DurabilityMode) -> Self { + bucket_durability::set(bucket, Some(mode)); + Self(bucket) + } + } + + impl Drop for BucketOverrideGuard { + fn drop(&mut self) { + bucket_durability::set(self.0, None); + } + } + + #[test] + fn test_effective_durability_bucket_override() { + // Global strict + per-bucket relaxed: only the named bucket drops. + { + let _mode = durability_mode_override::set(DurabilityMode::Strict); + let _guard = BucketOverrideGuard::set("hp5b-override-relaxed", DurabilityMode::Relaxed); + assert_eq!(effective_durability("hp5b-override-relaxed"), DurabilityMode::Relaxed); + assert_eq!(effective_durability("hp5b-other-bucket"), DurabilityMode::Strict); + assert_eq!(effective_durability(RUSTFS_META_BUCKET), DurabilityMode::Strict); + } + // Override cleared: the bucket follows the global mode again (a new + // PUT after a config change resolves the new tier). + { + let _mode = durability_mode_override::set(DurabilityMode::Strict); + assert_eq!(effective_durability("hp5b-override-relaxed"), DurabilityMode::Strict); + } + // Global relaxed + per-bucket strict: overrides can raise durability. + { + let _mode = durability_mode_override::set(DurabilityMode::Relaxed); + let _guard = BucketOverrideGuard::set("hp5b-override-strict", DurabilityMode::Strict); + assert_eq!(effective_durability("hp5b-override-strict"), DurabilityMode::Strict); + assert_eq!(effective_durability("hp5b-other-bucket"), DurabilityMode::Relaxed); + } + } + + #[test] + fn test_bucket_durability_refuses_system_namespaces() { + let _mode = durability_mode_override::set(DurabilityMode::Strict); + + // System-critical and scratch namespaces can never carry an override. + bucket_durability::set(RUSTFS_META_BUCKET, Some(DurabilityMode::Relaxed)); + bucket_durability::set(&format!("{RUSTFS_META_BUCKET}/buckets"), Some(DurabilityMode::None)); + bucket_durability::set(RUSTFS_META_TMP_BUCKET, Some(DurabilityMode::Relaxed)); + bucket_durability::set(super::super::RUSTFS_META_MULTIPART_BUCKET, Some(DurabilityMode::Relaxed)); + bucket_durability::set("", Some(DurabilityMode::Relaxed)); + + assert_eq!(bucket_durability::lookup(RUSTFS_META_BUCKET), Option::None); + assert_eq!(bucket_durability::lookup(RUSTFS_META_TMP_BUCKET), Option::None); + assert_eq!(effective_durability(RUSTFS_META_BUCKET), DurabilityMode::Strict); + + // The legacy full-off tier is process-wide only: registering it per + // bucket is dropped, not stored. + bucket_durability::set("hp5b-legacy-refused", Some(DurabilityMode::LegacyOff)); + assert_eq!(bucket_durability::lookup("hp5b-legacy-refused"), Option::None); + } + + #[test] + fn test_effective_durability_legacy_off_ignores_bucket_overrides() { + let _mode = durability_mode_override::set(DurabilityMode::LegacyOff); + let _guard = BucketOverrideGuard::set("hp5b-legacy-bucket", DurabilityMode::Strict); + // The legacy switch keeps its historical semantics bit for bit. + assert_eq!(effective_durability("hp5b-legacy-bucket"), DurabilityMode::LegacyOff); + } + + /// HP-5b behavior regression: with the global mode at strict (the + /// default), a bucket override to relaxed must skip the metadata-commit + /// dir fsync for that bucket only, and clearing the override must restore + /// the strict behavior for the next write. + #[tokio::test] + #[allow(clippy::await_holding_lock)] + async fn test_write_all_meta_bucket_override_relaxed_then_cleared() { + use tempfile::tempdir; + + let _mode = durability_mode_override::set(DurabilityMode::Strict); + + let dir = tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + + let overridden = "hp5b-relaxed-write-bucket"; + let untouched = "hp5b-strict-write-bucket"; + ensure_test_volume(&disk, overridden).await; + ensure_test_volume(&disk, untouched).await; + ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await; + + let meta_path = format!("dir/object/{STORAGE_FORMAT_FILE}"); + + { + let _guard = BucketOverrideGuard::set("hp5b-relaxed-write-bucket", DurabilityMode::Relaxed); + + disk.write_all_meta(overridden, &meta_path, b"payload", true) + .await + .expect("write_all_meta should succeed"); + let overridden_parent = disk + .get_object_path(overridden, &meta_path) + .expect("dst path should resolve") + .parent() + .expect("dst file should have a parent") + .to_path_buf(); + assert!( + !os::fsync_dir_recorder::was_fsynced(&overridden_parent), + "bucket override to relaxed must skip the metadata-commit dir fsync" + ); + + // A bucket without an override keeps the strict default. + disk.write_all_meta(untouched, &meta_path, b"payload", true) + .await + .expect("write_all_meta should succeed"); + let untouched_parent = disk + .get_object_path(untouched, &meta_path) + .expect("dst path should resolve") + .parent() + .expect("dst file should have a parent") + .to_path_buf(); + assert!( + os::fsync_dir_recorder::was_fsynced(&untouched_parent), + "buckets without an override must keep the strict commit fsyncs" + ); + } + + // Override cleared (guard dropped): the next write is strict again. + let second_meta_path = format!("dir/object-after-clear/{STORAGE_FORMAT_FILE}"); + disk.write_all_meta(overridden, &second_meta_path, b"payload", true) + .await + .expect("write_all_meta should succeed"); + let after_clear_parent = disk + .get_object_path(overridden, &second_meta_path) + .expect("dst path should resolve") + .parent() + .expect("dst file should have a parent") + .to_path_buf(); + assert!( + os::fsync_dir_recorder::was_fsynced(&after_clear_parent), + "clearing the override must restore strict fsyncs for new writes" + ); + } + #[tokio::test] #[allow(clippy::await_holding_lock)] async fn test_write_all_meta_relaxed_skips_dst_parent_dir_fsync() { diff --git a/docs/operations/durability-modes.md b/docs/operations/durability-modes.md index f8409e39e..5e870ee40 100644 --- a/docs/operations/durability-modes.md +++ b/docs/operations/durability-modes.md @@ -113,6 +113,78 @@ staged in tmp still commits with full `strict` durability. The durability mode is server-side configuration only; it cannot be raised or lowered by any request header. +## Per-bucket durability (phase 2) + +A bucket can override the process-wide mode with its own tier. The override +is stored in the bucket's metadata (a `durability.json` entry in +`.rustfs.sys/buckets//.metadata.bin`, written as a RustFS extension +field that MinIO's decoder skips) and resolved per write at the commit +points, so no restart is needed. + +### Configuration + +```bash +# Set an override (admin credentials, ConfigUpdateAdminAction) +curl -X PUT "http:///rustfs/admin/v3/bucket-durability/" \ + -d '{"mode":"relaxed"}' # strict | relaxed | none + +# Read it back ("mode": null means the bucket inherits the global mode) +curl "http:///rustfs/admin/v3/bucket-durability/" + +# Clear it (the bucket inherits the global mode again) +curl -X DELETE "http:///rustfs/admin/v3/bucket-durability/" +``` + +`mc` integration is a follow-up; for now the admin API above is the +configuration plane. + +### Resolution order + +For a write committing into volume `V`: + +1. system-critical namespaces (`.rustfs.sys`, `.minio.sys` outside the + scratch dirs) are always `strict` — a bucket override can never be + attached to them, and any attempt is rejected and logged; +2. otherwise, if the destination bucket has an override, the override wins — + in **both** directions (a bucket can be `relaxed` under a `strict` global + default, or pinned `strict` under a `relaxed` global default); +3. otherwise the process-wide mode applies. + +Under `legacy-off` (`RUSTFS_DRIVE_SYNC_ENABLE=false`) per-bucket overrides +do not apply at all: the legacy switch keeps its historical semantics bit +for bit. `legacy-off` is also not a valid per-bucket tier. + +Because scratch-staged data commits under the destination volume, an object +staged in `.rustfs.sys/tmp` follows the override of the bucket it commits +into, exactly like the global tier. + +### Propagation and effect latency + +The override rides the existing bucket-metadata cache, with no invalidation +channel of its own: + +- on the node that applies the config change, the new tier is effective for + writes that resolve their durability after the update completes; +- other nodes are told to reload the bucket's metadata right away (the same + peer notification used for every bucket config change); if a peer misses + the notification, the periodic bucket-metadata refresh loop (15 minutes) + converges it; +- an in-flight operation keeps the tier it resolved at its start — a single + commit is never half-old-tier, half-new-tier; +- deleting the bucket drops the override with the rest of its metadata. + +Until a peer has reloaded the metadata, its writes use the previous tier. +This is the same eventual-consistency window every other bucket config +(policy, quota, versioning) already has. + +### Power-loss guarantees + +Identical to the corresponding global tier, scoped to the bucket. In +particular the `relaxed` deployment rule (multi-node clusters in independent +power domains only) applies per bucket: overriding a bucket to `relaxed` on +a single-node deployment can lose recently acknowledged objects in that +bucket on power failure. + ## Performance expectations The often-quoted 26x PUT throughput delta was measured on macOS with the old @@ -124,6 +196,8 @@ numbers to size `relaxed`. ## Scope -This is phase 1 of rustfs/backlog#926: a global, per-process tier configured -by environment variable. Per-bucket durability tiers (bucket metadata + -admin API) are a separate follow-up phase. +Phase 1 (rustfs/backlog#926) shipped the global, per-process tier configured +by environment variable. Phase 2 (rustfs/backlog#938) adds the per-bucket +override described above, configured through the admin API and stored in +bucket metadata. `mc admin` integration for the per-bucket tier is a +follow-up. diff --git a/rustfs/src/admin/handlers/durability.rs b/rustfs/src/admin/handlers/durability.rs new file mode 100644 index 000000000..98350f7cb --- /dev/null +++ b/rustfs/src/admin/handlers/durability.rs @@ -0,0 +1,273 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Per-bucket durability tier admin handlers (HP-5 phase 2, rustfs/backlog#938). +//! +//! A bucket can override the process-wide durability mode +//! (`RUSTFS_DURABILITY_MODE`) with its own `strict` | `relaxed` | `none` +//! tier. The override is stored in the bucket metadata (`durability.json` +//! entry) and consumed by the disk layer at commit points. System buckets +//! can never carry an override. See docs/operations/durability-modes.md. + +use crate::admin::auth::validate_admin_request; +use crate::admin::router::{AdminOperation, Operation, S3Router}; +use crate::admin::runtime_sources::current_notification_system; +use crate::admin::storage_api::bucket::durability::BucketDurabilityConfig; +use crate::admin::storage_api::bucket::metadata::BUCKET_DURABILITY_CONFIG; +use crate::admin::storage_api::bucket::metadata_sys; +use crate::auth::{check_key_valid, get_session_token}; +use crate::server::ADMIN_PREFIX; +use hyper::{Method, StatusCode}; +use matchit::Params; +use rustfs_policy::policy::action::{Action, AdminAction}; +use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; +use serde::{Deserialize, Serialize}; +use tracing::{info, warn}; + +const LOG_COMPONENT_ADMIN: &str = "admin"; +const LOG_SUBSYSTEM_DURABILITY: &str = "bucket_durability"; +const EVENT_ADMIN_BUCKET_DURABILITY: &str = "admin_bucket_durability_state"; + +#[derive(Debug, Deserialize)] +struct SetBucketDurabilityRequest { + mode: String, +} + +#[derive(Debug, Serialize)] +struct BucketDurabilityResponse { + bucket: String, + /// The bucket's own override, or `null` when the bucket inherits the + /// process-wide durability mode. + mode: Option, +} + +pub struct SetBucketDurabilityHandler; +pub struct GetBucketDurabilityHandler; +pub struct DeleteBucketDurabilityHandler; + +pub fn register_durability_route(r: &mut S3Router) -> std::io::Result<()> { + r.insert( + Method::PUT, + format!("{}{}", ADMIN_PREFIX, "/v3/bucket-durability/{bucket}").as_str(), + AdminOperation(&SetBucketDurabilityHandler {}), + )?; + + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/bucket-durability/{bucket}").as_str(), + AdminOperation(&GetBucketDurabilityHandler {}), + )?; + + r.insert( + Method::DELETE, + format!("{}{}", ADMIN_PREFIX, "/v3/bucket-durability/{bucket}").as_str(), + AdminOperation(&DeleteBucketDurabilityHandler {}), + )?; + + Ok(()) +} + +fn parse_set_bucket_durability_request(body: &[u8]) -> Result { + if body.is_empty() { + return Err(s3_error!(InvalidRequest, "request body is required, e.g. {{\"mode\":\"relaxed\"}}")); + } + serde_json::from_slice(body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e)) +} + +/// Validates and canonicalizes the requested tier name. +fn normalize_mode(mode: &str) -> Result { + let normalized = mode.trim().to_ascii_lowercase(); + if BucketDurabilityConfig::is_valid_mode(&normalized) { + Ok(normalized) + } else { + Err(s3_error!( + InvalidArgument, + "invalid durability mode {:?}: expected strict|relaxed|none", + mode + )) + } +} + +/// Kick peers to reload the bucket's metadata so the override takes effect +/// cluster-wide without waiting for the periodic refresh loop. Failures are +/// logged, not surfaced: the refresh loop converges peers eventually. +fn notify_peers_reload(bucket: String, operation: &'static str) { + tokio::spawn(async move { + if let Some(notification_sys) = current_notification_system() + && let Err(err) = notification_sys.load_bucket_metadata(&bucket).await + { + warn!( + event = EVENT_ADMIN_BUCKET_DURABILITY, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_DURABILITY, + bucket = %bucket, + error = %err, + "failed to notify peers after {operation}" + ); + } + }); +} + +async fn authenticate_admin(req: &S3Request) -> S3Result<()> { + let Some(ref cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "authentication required")); + }; + + let (cred, owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?; + + validate_admin_request( + &req.headers, + &cred, + owner, + false, + vec![Action::AdminAction(AdminAction::ConfigUpdateAdminAction)], + None, + ) + .await?; + + Ok(()) +} + +fn bucket_from_params(params: &Params<'_, '_>) -> S3Result { + let bucket = params.get("bucket").unwrap_or("").to_string(); + if bucket.is_empty() { + return Err(s3_error!(InvalidRequest, "bucket name is required")); + } + Ok(bucket) +} + +fn durability_response(bucket: String, mode: Option) -> S3Result> { + let response = BucketDurabilityResponse { bucket, mode }; + let json = serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?; + Ok(S3Response::new((StatusCode::OK, Body::from(json)))) +} + +#[async_trait::async_trait] +impl Operation for SetBucketDurabilityHandler { + #[tracing::instrument(skip_all)] + async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { + authenticate_admin(&req).await?; + + let bucket = bucket_from_params(¶ms)?; + + let body = req + .input + .store_all_limited(rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE) + .await + .map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?; + + let request = parse_set_bucket_durability_request(&body)?; + let mode = normalize_mode(&request.mode)?; + + let config = BucketDurabilityConfig::new(&mode); + let json = serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "failed to encode config: {}", e))?; + + // System buckets are rejected by the metadata layer (and pinned to + // strict by the disk layer regardless). + metadata_sys::update(&bucket, BUCKET_DURABILITY_CONFIG, json) + .await + .map_err(|e| s3_error!(InternalError, "failed to set bucket durability: {}", e))?; + + info!( + event = EVENT_ADMIN_BUCKET_DURABILITY, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_DURABILITY, + bucket = %bucket, + mode = %mode, + "bucket durability override set" + ); + + notify_peers_reload(bucket.clone(), "set bucket durability"); + + durability_response(bucket, Some(mode)) + } +} + +#[async_trait::async_trait] +impl Operation for GetBucketDurabilityHandler { + #[tracing::instrument(skip_all)] + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + authenticate_admin(&req).await?; + + let bucket = bucket_from_params(¶ms)?; + + let (config, _updated_at) = metadata_sys::get_durability_config(&bucket) + .await + .map_err(|e| s3_error!(NoSuchBucket, "failed to read bucket durability: {}", e))?; + + let mode = config.and_then(|c| c.normalized_mode()); + durability_response(bucket, mode) + } +} + +#[async_trait::async_trait] +impl Operation for DeleteBucketDurabilityHandler { + #[tracing::instrument(skip_all)] + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + authenticate_admin(&req).await?; + + let bucket = bucket_from_params(¶ms)?; + + metadata_sys::delete(&bucket, BUCKET_DURABILITY_CONFIG) + .await + .map_err(|e| s3_error!(InternalError, "failed to clear bucket durability: {}", e))?; + + info!( + event = EVENT_ADMIN_BUCKET_DURABILITY, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_DURABILITY, + bucket = %bucket, + "bucket durability override cleared" + ); + + notify_peers_reload(bucket.clone(), "clear bucket durability"); + + durability_response(bucket, None) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_set_request_requires_body_and_valid_json() { + assert!(parse_set_bucket_durability_request(b"").is_err()); + assert!(parse_set_bucket_durability_request(b"not-json").is_err()); + + let req = parse_set_bucket_durability_request(br#"{"mode":"relaxed"}"#).expect("parse"); + assert_eq!(req.mode, "relaxed"); + } + + #[test] + fn normalize_mode_accepts_tiers_and_rejects_everything_else() { + assert_eq!(normalize_mode("strict").unwrap(), "strict"); + assert_eq!(normalize_mode(" RELAXED ").unwrap(), "relaxed"); + assert_eq!(normalize_mode("none").unwrap(), "none"); + assert!(normalize_mode("").is_err()); + assert!(normalize_mode("bogus").is_err()); + // The legacy full-off switch is process-wide only. + assert!(normalize_mode("legacy-off").is_err()); + } + + #[test] + fn response_serializes_inherit_as_null() { + let json = serde_json::to_string(&BucketDurabilityResponse { + bucket: "b".to_string(), + mode: None, + }) + .expect("serialize"); + assert_eq!(json, r#"{"bucket":"b","mode":null}"#); + } +} diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index 7b47cb9b6..e0cc3d447 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -20,6 +20,7 @@ pub mod bucket_meta; pub mod cluster_snapshot; pub mod config_admin; pub mod diagnostics; +pub mod durability; pub mod event; pub mod extensions; pub mod group; diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index df294f673..46dd2ee45 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -33,9 +33,10 @@ mod console_test; mod route_registration_test; use handlers::{ - audit, batch_job, bucket_meta, cluster_snapshot, config_admin, diagnostics, extensions, heal, health, idp_compat, kms, - module_switch, object_zip_download, oidc, plugins_catalog, plugins_instances, pools, profile_admin, quota as quota_handler, - rebalance, replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, user, + audit, batch_job, bucket_meta, cluster_snapshot, config_admin, diagnostics, durability as durability_handler, extensions, + heal, health, idp_compat, kms, module_switch, object_zip_download, oidc, plugins_catalog, plugins_instances, pools, + profile_admin, quota as quota_handler, rebalance, replication as replication_handler, scanner, site_replication, sts, system, + table_catalog, tier, tls_debug, user, }; use router::{AdminOperation, S3Router}; use s3s::route::S3Route; @@ -67,6 +68,7 @@ fn register_admin_routes(r: &mut S3Router) -> std::io::Result<() tier::register_tier_route(r)?; quota_handler::register_quota_route(r)?; + durability_handler::register_durability_route(r)?; bucket_meta::register_bucket_meta_route(r)?; config_admin::register_config_route(r)?; scanner::register_scanner_route(r)?; diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 2f5a9a1ce..711ac9714 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -332,6 +332,24 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ SET_BUCKET_QUOTA, RouteRiskLevel::High, ), + admin( + HttpMethod::Put, + "/rustfs/admin/v3/bucket-durability/{bucket}", + CONFIG_UPDATE, + RouteRiskLevel::High, + ), + admin( + HttpMethod::Get, + "/rustfs/admin/v3/bucket-durability/{bucket}", + CONFIG_UPDATE, + RouteRiskLevel::Sensitive, + ), + admin( + HttpMethod::Delete, + "/rustfs/admin/v3/bucket-durability/{bucket}", + CONFIG_UPDATE, + RouteRiskLevel::High, + ), admin( HttpMethod::Get, "/rustfs/admin/export-bucket-metadata", diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index 6dbc624b9..e94157b6c 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -199,6 +199,9 @@ fn expected_admin_route_matrix() -> Vec { admin_route_sample(Method::DELETE, "/v3/quota/{bucket}", "/v3/quota/test-bucket"), admin_route_sample(Method::GET, "/v3/quota-stats/{bucket}", "/v3/quota-stats/test-bucket"), admin_route_sample(Method::POST, "/v3/quota-check/{bucket}", "/v3/quota-check/test-bucket"), + admin_route_sample(Method::PUT, "/v3/bucket-durability/{bucket}", "/v3/bucket-durability/test-bucket"), + admin_route_sample(Method::GET, "/v3/bucket-durability/{bucket}", "/v3/bucket-durability/test-bucket"), + admin_route_sample(Method::DELETE, "/v3/bucket-durability/{bucket}", "/v3/bucket-durability/test-bucket"), admin_route(Method::GET, "/export-bucket-metadata"), admin_route(Method::GET, "/v3/export-bucket-metadata"), admin_route(Method::PUT, "/import-bucket-metadata"), @@ -1155,6 +1158,9 @@ fn test_register_routes_cover_representative_admin_paths() { assert_route(&router, Method::GET, &admin_path("/v3/get-bucket-quota")); assert_route(&router, Method::PUT, &admin_path("/v3/quota/test-bucket")); assert_route(&router, Method::GET, &admin_path("/v3/quota-stats/test-bucket")); + assert_route(&router, Method::PUT, &admin_path("/v3/bucket-durability/test-bucket")); + assert_route(&router, Method::GET, &admin_path("/v3/bucket-durability/test-bucket")); + assert_route(&router, Method::DELETE, &admin_path("/v3/bucket-durability/test-bucket")); assert_route(&router, Method::GET, &admin_path("/export-bucket-metadata")); assert_route(&router, Method::GET, &admin_path("/v3/export-bucket-metadata")); diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index f0c3fa3db..5a500d8cd 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -20,8 +20,8 @@ use time::OffsetDateTime; mod ecstore_bucket { pub(crate) use crate::storage::storage_api::ecstore_bucket::{ - bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning, - versioning_sys, + bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, quota, replication, target, utils, + versioning, versioning_sys, }; } @@ -209,6 +209,7 @@ pub(crate) mod metadata { pub(crate) const BUCKET_TAGGING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_TAGGING_CONFIG; pub(crate) const BUCKET_TARGETS_FILE: &str = super::ecstore_bucket::metadata::BUCKET_TARGETS_FILE; pub(crate) const BUCKET_VERSIONING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG; + pub(crate) const BUCKET_DURABILITY_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_DURABILITY_CONFIG; pub(crate) const OBJECT_LOCK_CONFIG: &str = super::ecstore_bucket::metadata::OBJECT_LOCK_CONFIG; pub(crate) type BucketMetadata = super::ecstore_bucket::metadata::BucketMetadata; @@ -218,6 +219,10 @@ pub(crate) mod metadata { } } +pub(crate) mod durability { + pub(crate) type BucketDurabilityConfig = super::ecstore_bucket::durability::BucketDurabilityConfig; +} + pub(crate) mod metadata_sys { use std::sync::Arc; @@ -271,6 +276,12 @@ pub(crate) mod metadata_sys { super::ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await } + pub(crate) async fn get_durability_config( + bucket: &str, + ) -> Result<(Option, OffsetDateTime)> { + super::ecstore_bucket::metadata_sys::get_durability_config(bucket).await + } + pub(crate) async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> { super::ecstore_bucket::metadata_sys::get_quota_config(bucket).await } @@ -457,6 +468,7 @@ pub(crate) mod access { pub(crate) mod bucket { pub(crate) use super::bandwidth; pub(crate) use super::bucket_target_sys as target_sys; + pub(crate) use super::durability; #[cfg(test)] pub(crate) use super::lifecycle; pub(crate) use super::metadata; diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 03ed6d7af..80982651b 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -318,8 +318,8 @@ pub(crate) mod ecstore_admin { pub(crate) mod ecstore_bucket { pub(crate) use rustfs_ecstore::api::bucket::{ - bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys, replication, - tagging, target, utils, + bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys, + replication, tagging, target, utils, }; pub(crate) use rustfs_ecstore::api::bucket::{quota, versioning, versioning_sys}; }