mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
perf(ecstore): support per-bucket durability tier overrides (#4407)
perf(ecstore): per-bucket durability tier overrides (HP-5 phase 2) Let a bucket override the process-wide RUSTFS_DURABILITY_MODE with its own strict/relaxed/none tier, stored as a durability.json extension entry in the bucket metadata file and resolved at commit points via effective_durability. System-critical buckets (.rustfs.sys, .minio.sys) can never carry an override and stay pinned to strict; the legacy full-off switch keeps its historical semantics and per-bucket overrides do not apply under it. Overrides are published and cleared through the existing bucket metadata cache-invalidation path, and an admin GET/PUT handler exposes the configuration. Default behavior is unchanged: with no override a bucket follows the global mode, which defaults to strict and stays byte-for-byte identical to before. Refs: https://github.com/rustfs/backlog/issues/938, https://github.com/rustfs/backlog/issues/936 Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
@@ -82,6 +82,7 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod metadata {
|
||||
pub use crate::bucket::metadata::BUCKET_DURABILITY_CONFIG;
|
||||
pub use crate::bucket::metadata::{
|
||||
BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG,
|
||||
BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_QUOTA_CONFIG_FILE,
|
||||
@@ -92,14 +93,20 @@ pub mod bucket {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod durability {
|
||||
pub use crate::bucket::durability::{
|
||||
BUCKET_DURABILITY_MODE_NONE, BUCKET_DURABILITY_MODE_RELAXED, BUCKET_DURABILITY_MODE_STRICT, BucketDurabilityConfig,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod metadata_sys {
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BucketMetadataSys, delete, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw,
|
||||
get_bucket_targets_config, get_config_from_disk, get_cors_config, get_global_bucket_metadata_sys,
|
||||
get_lifecycle_config, get_logging_config, get_notification_config, get_object_lock_config,
|
||||
get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config,
|
||||
get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets,
|
||||
remove_bucket_metadata, set_bucket_metadata, update,
|
||||
get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
|
||||
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
|
||||
init_bucket_metadata_sys, list_bucket_targets, remove_bucket_metadata, set_bucket_metadata, update,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-bucket durability tier configuration (HP-5 phase 2).
|
||||
//!
|
||||
//! A bucket can override the process-wide durability mode
|
||||
//! (`RUSTFS_DURABILITY_MODE`) with its own tier. The override is stored as a
|
||||
//! RustFS extension entry (`durability.json`) in the bucket metadata file and
|
||||
//! is resolved by the disk layer at commit points via
|
||||
//! `effective_durability(volume)`. System buckets (`.rustfs.sys`,
|
||||
//! `.minio.sys`) can never carry an override; they stay pinned to `strict`.
|
||||
//! See docs/operations/durability-modes.md for the semantics.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Valid per-bucket durability tier names. `legacy-off` is deliberately not
|
||||
/// representable per bucket: it exists only for the process-wide legacy
|
||||
/// switch (`RUSTFS_DRIVE_SYNC_ENABLE=false`).
|
||||
pub const BUCKET_DURABILITY_MODE_STRICT: &str = "strict";
|
||||
pub const BUCKET_DURABILITY_MODE_RELAXED: &str = "relaxed";
|
||||
pub const BUCKET_DURABILITY_MODE_NONE: &str = "none";
|
||||
|
||||
/// JSON payload stored under the `durability.json` bucket metadata entry.
|
||||
///
|
||||
/// An absent entry, an empty payload, or an empty `mode` string all mean
|
||||
/// "no override": the bucket follows the process-wide durability mode.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct BucketDurabilityConfig {
|
||||
/// Requested tier: `strict` | `relaxed` | `none` (case-insensitive,
|
||||
/// whitespace-tolerant). Empty means "inherit the global mode".
|
||||
#[serde(default)]
|
||||
pub mode: String,
|
||||
}
|
||||
|
||||
impl BucketDurabilityConfig {
|
||||
pub fn new(mode: &str) -> Self {
|
||||
Self { mode: mode.to_string() }
|
||||
}
|
||||
|
||||
/// Whether `mode` names a valid per-bucket tier.
|
||||
pub fn is_valid_mode(mode: &str) -> bool {
|
||||
matches!(
|
||||
mode.trim().to_ascii_lowercase().as_str(),
|
||||
BUCKET_DURABILITY_MODE_STRICT | BUCKET_DURABILITY_MODE_RELAXED | BUCKET_DURABILITY_MODE_NONE
|
||||
)
|
||||
}
|
||||
|
||||
/// The canonical (trimmed, lowercase) tier name, or `None` when the
|
||||
/// config does not carry a valid override.
|
||||
pub fn normalized_mode(&self) -> Option<String> {
|
||||
let mode = self.mode.trim().to_ascii_lowercase();
|
||||
if Self::is_valid_mode(&mode) { Some(mode) } else { None }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn valid_modes_are_recognized() {
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("strict"));
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("relaxed"));
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("none"));
|
||||
assert!(BucketDurabilityConfig::is_valid_mode(" RELAXED "));
|
||||
assert!(!BucketDurabilityConfig::is_valid_mode(""));
|
||||
assert!(!BucketDurabilityConfig::is_valid_mode("bogus"));
|
||||
// The legacy full-off switch is process-wide only.
|
||||
assert!(!BucketDurabilityConfig::is_valid_mode("legacy-off"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalized_mode_canonicalizes_or_rejects() {
|
||||
assert_eq!(BucketDurabilityConfig::new(" Relaxed ").normalized_mode().as_deref(), Some("relaxed"));
|
||||
assert_eq!(BucketDurabilityConfig::new("strict").normalized_mode().as_deref(), Some("strict"));
|
||||
assert_eq!(BucketDurabilityConfig::default().normalized_mode(), None);
|
||||
assert_eq!(BucketDurabilityConfig::new("bogus").normalized_mode(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_round_trip() {
|
||||
let cfg = BucketDurabilityConfig::new("relaxed");
|
||||
let json = serde_json::to_vec(&cfg).expect("serialize");
|
||||
let back: BucketDurabilityConfig = serde_json::from_slice(&json).expect("deserialize");
|
||||
assert_eq!(back, cfg);
|
||||
|
||||
// Empty object deserializes to the "inherit" default.
|
||||
let empty: BucketDurabilityConfig = serde_json::from_slice(b"{}").expect("deserialize empty");
|
||||
assert_eq!(empty.normalized_mode(), None);
|
||||
}
|
||||
}
|
||||
@@ -246,6 +246,7 @@ pub const BUCKET_REQUEST_PAYMENT_CONFIG: &str = "request-payment.xml";
|
||||
pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
|
||||
pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
|
||||
pub const BUCKET_TABLE_CONFIG: &str = "table-bucket.json";
|
||||
pub const BUCKET_DURABILITY_CONFIG: &str = "durability.json";
|
||||
pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table";
|
||||
pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog";
|
||||
pub const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str = "table-buckets";
|
||||
@@ -294,6 +295,7 @@ pub struct BucketMetadata {
|
||||
pub public_access_block_config_xml: Vec<u8>,
|
||||
pub bucket_acl_config_json: Vec<u8>,
|
||||
pub table_bucket_config_json: Vec<u8>,
|
||||
pub durability_config_json: Vec<u8>,
|
||||
|
||||
pub policy_config_updated_at: OffsetDateTime,
|
||||
pub object_lock_config_updated_at: OffsetDateTime,
|
||||
@@ -314,6 +316,7 @@ pub struct BucketMetadata {
|
||||
pub public_access_block_config_updated_at: OffsetDateTime,
|
||||
pub bucket_acl_config_updated_at: OffsetDateTime,
|
||||
pub table_bucket_config_updated_at: OffsetDateTime,
|
||||
pub durability_config_updated_at: OffsetDateTime,
|
||||
|
||||
pub new_field_updated_at: OffsetDateTime,
|
||||
|
||||
@@ -362,6 +365,7 @@ impl Default for BucketMetadata {
|
||||
public_access_block_config_xml: Default::default(),
|
||||
bucket_acl_config_json: Default::default(),
|
||||
table_bucket_config_json: Default::default(),
|
||||
durability_config_json: Default::default(),
|
||||
policy_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
@@ -381,6 +385,7 @@ impl Default for BucketMetadata {
|
||||
public_access_block_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
bucket_acl_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
table_bucket_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
durability_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
new_field_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
policy_config: Default::default(),
|
||||
notification_config: Default::default(),
|
||||
@@ -430,6 +435,32 @@ impl BucketMetadata {
|
||||
!self.table_bucket_config_json.is_empty()
|
||||
}
|
||||
|
||||
/// Parsed per-bucket durability override, if a valid one is stored.
|
||||
///
|
||||
/// Absent/empty/unparsable payloads all mean "no override" (the bucket
|
||||
/// follows the global durability mode); a parse failure is logged so a
|
||||
/// corrupted entry cannot silently change fsync behavior.
|
||||
pub fn durability_config(&self) -> Option<super::durability::BucketDurabilityConfig> {
|
||||
if self.durability_config_json.is_empty() {
|
||||
return None;
|
||||
}
|
||||
match serde_json::from_slice(&self.durability_config_json) {
|
||||
Ok(cfg) => Some(cfg),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %self.name,
|
||||
config = "durability",
|
||||
error = %e,
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode from msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn decode_from<R: Read>(&mut self, rd: &mut R) -> Result<()> {
|
||||
let mut fields = rmp::decode::read_map_len(rd)?;
|
||||
@@ -481,6 +512,7 @@ impl BucketMetadata {
|
||||
}
|
||||
"BucketAclConfigJSON" | "BucketAclConfigJson" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
|
||||
"TableBucketConfigJSON" | "TableBucketConfigJson" => self.table_bucket_config_json = read_msgp_bin(rd)?,
|
||||
"DurabilityConfigJSON" | "DurabilityConfigJson" => self.durability_config_json = read_msgp_bin(rd)?,
|
||||
"CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"LoggingConfigUpdatedAt" => self.logging_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"WebsiteConfigUpdatedAt" => self.website_config_updated_at = read_msgp_time_value(rd)?,
|
||||
@@ -489,6 +521,7 @@ impl BucketMetadata {
|
||||
"PublicAccessBlockConfigUpdatedAt" => self.public_access_block_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
|
||||
other => {
|
||||
tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
|
||||
skip_msgp_value(rd)?;
|
||||
@@ -501,8 +534,8 @@ impl BucketMetadata {
|
||||
|
||||
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
|
||||
// Map size: MinIO fields (25) + RustFS extensions (16)
|
||||
let map_len: u32 = 41;
|
||||
// Map size: MinIO fields (25) + RustFS extensions (18)
|
||||
let map_len: u32 = 43;
|
||||
rmp::encode::write_map_len(wr, map_len)?;
|
||||
|
||||
// MinIO field order (same as Go struct)
|
||||
@@ -559,6 +592,7 @@ impl BucketMetadata {
|
||||
write_bin_field(wr, "PublicAccessBlockConfigXML", &self.public_access_block_config_xml)?;
|
||||
write_bin_field(wr, "BucketAclConfigJSON", &self.bucket_acl_config_json)?;
|
||||
write_bin_field(wr, "TableBucketConfigJSON", &self.table_bucket_config_json)?;
|
||||
write_bin_field(wr, "DurabilityConfigJSON", &self.durability_config_json)?;
|
||||
rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.cors_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "LoggingConfigUpdatedAt")?;
|
||||
@@ -575,6 +609,8 @@ impl BucketMetadata {
|
||||
write_msgp_time(wr, self.bucket_acl_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "TableBucketConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.table_bucket_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "DurabilityConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.durability_config_updated_at)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -673,6 +709,9 @@ impl BucketMetadata {
|
||||
if self.table_bucket_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.table_bucket_config_updated_at = self.created
|
||||
}
|
||||
if self.durability_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.durability_config_updated_at = self.created
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_config(&mut self, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
|
||||
@@ -755,6 +794,10 @@ impl BucketMetadata {
|
||||
self.table_bucket_config_json = data;
|
||||
self.table_bucket_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_DURABILITY_CONFIG => {
|
||||
self.durability_config_json = data;
|
||||
self.durability_config_updated_at = updated;
|
||||
}
|
||||
_ => return Err(Error::other(format!("config file not found : {config_file}"))),
|
||||
}
|
||||
|
||||
@@ -1444,6 +1487,35 @@ mod test {
|
||||
assert!(!bm.table_bucket_enabled());
|
||||
}
|
||||
|
||||
/// HP-5b (rustfs/backlog#938): the durability override is a RustFS
|
||||
/// extension entry and must survive an encode/decode round trip.
|
||||
#[test]
|
||||
fn durability_config_round_trips_and_tracks_updates() {
|
||||
let mut bm = BucketMetadata::new("durability-bucket");
|
||||
assert!(bm.durability_config().is_none(), "fresh metadata carries no override");
|
||||
|
||||
bm.update_config(BUCKET_DURABILITY_CONFIG, br#"{"mode":"relaxed"}"#.to_vec())
|
||||
.unwrap();
|
||||
assert_ne!(bm.durability_config_updated_at, OffsetDateTime::UNIX_EPOCH);
|
||||
|
||||
let buf = bm.marshal_msg().unwrap();
|
||||
let back = BucketMetadata::unmarshal(&buf).unwrap();
|
||||
assert_eq!(back.durability_config_json, bm.durability_config_json);
|
||||
assert_eq!(
|
||||
back.durability_config_updated_at.unix_timestamp(),
|
||||
bm.durability_config_updated_at.unix_timestamp()
|
||||
);
|
||||
assert_eq!(back.durability_config().and_then(|c| c.normalized_mode()).as_deref(), Some("relaxed"));
|
||||
|
||||
// Clearing the entry removes the override.
|
||||
bm.update_config(BUCKET_DURABILITY_CONFIG, Vec::new()).unwrap();
|
||||
assert!(bm.durability_config().is_none());
|
||||
|
||||
// Corrupted payloads must degrade to "no override", never to a tier.
|
||||
bm.durability_config_json = b"not-json".to_vec();
|
||||
assert!(bm.durability_config().is_none());
|
||||
}
|
||||
|
||||
/// After policy deletion (policy_config_json cleared), parse_policy_config sets policy_config to None.
|
||||
#[test]
|
||||
fn test_parse_policy_config_clears_cache_when_json_empty() {
|
||||
|
||||
@@ -153,6 +153,26 @@ async fn sync_bucket_target_sys(bucket: &str, bm: &BucketMetadata) {
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Publish the bucket's durability override (or its absence) to the disk
|
||||
/// layer registry consulted by `effective_durability`.
|
||||
///
|
||||
/// Called from every path that installs a bucket's metadata into the cache
|
||||
/// (initial load, config update, peer reload notification, refresh loop,
|
||||
/// lazy load), so the override propagates with exactly the bucket-metadata
|
||||
/// cache invalidation semantics and never through a channel of its own.
|
||||
fn sync_bucket_durability(bucket: &str, bm: &BucketMetadata) {
|
||||
let mode = bm
|
||||
.durability_config()
|
||||
.and_then(|cfg| cfg.normalized_mode())
|
||||
.and_then(|mode| crate::disk::local::DurabilityMode::parse(&mode));
|
||||
crate::disk::local::bucket_durability::set(bucket, mode);
|
||||
}
|
||||
|
||||
/// Drop a bucket's durability override when its metadata leaves the cache.
|
||||
fn clear_bucket_durability(bucket: &str) {
|
||||
crate::disk::local::bucket_durability::set(bucket, None);
|
||||
}
|
||||
|
||||
pub async fn get(bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
let lock = sys.read().await;
|
||||
@@ -196,6 +216,20 @@ pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTi
|
||||
bucket_meta_sys.get_bucket_acl_config(bucket).await
|
||||
}
|
||||
|
||||
/// The bucket's durability override config (if any) with its update time.
|
||||
///
|
||||
/// `Ok((None, ..))` means the bucket has no override and follows the global
|
||||
/// durability mode.
|
||||
pub async fn get_durability_config(
|
||||
bucket: &str,
|
||||
) -> Result<(Option<crate::bucket::durability::BucketDurabilityConfig>, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
let (bm, _) = bucket_meta_sys.get_config(bucket).await?;
|
||||
Ok((bm.durability_config(), bm.durability_config_updated_at))
|
||||
}
|
||||
|
||||
pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -425,6 +459,7 @@ impl BucketMetadataSys {
|
||||
map.insert(bucket.clone(), bm.clone());
|
||||
drop(map);
|
||||
sync_bucket_target_sys(&bucket, &bm).await;
|
||||
sync_bucket_durability(&bucket, &bm);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -440,6 +475,7 @@ impl BucketMetadataSys {
|
||||
drop(map);
|
||||
if removed {
|
||||
BucketTargetSys::get().delete(bucket).await;
|
||||
clear_bucket_durability(bucket);
|
||||
}
|
||||
removed
|
||||
}
|
||||
@@ -540,6 +576,7 @@ impl BucketMetadataSys {
|
||||
map.insert(bucket.to_string(), bm.clone());
|
||||
drop(map);
|
||||
sync_bucket_target_sys(bucket, &bm).await;
|
||||
sync_bucket_durability(bucket, &bm);
|
||||
|
||||
Ok((bm, true))
|
||||
}
|
||||
@@ -826,6 +863,40 @@ mod tests {
|
||||
target_sys.delete(bucket).await;
|
||||
}
|
||||
|
||||
/// HP-5b (rustfs/backlog#938): installing bucket metadata publishes the
|
||||
/// durability override to the disk-layer registry, and clearing the
|
||||
/// config (or an invalid payload) withdraws it.
|
||||
#[test]
|
||||
fn metadata_sync_publishes_and_clears_durability_override() {
|
||||
use crate::disk::local::{DurabilityMode, bucket_durability};
|
||||
|
||||
let bucket = "metadata-sync-durability";
|
||||
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.durability_config_json = br#"{"mode":"relaxed"}"#.to_vec();
|
||||
sync_bucket_durability(bucket, &bm);
|
||||
assert_eq!(bucket_durability::lookup(bucket), Some(DurabilityMode::Relaxed));
|
||||
|
||||
// Metadata without the config entry clears the override.
|
||||
let bm = BucketMetadata::new(bucket);
|
||||
sync_bucket_durability(bucket, &bm);
|
||||
assert_eq!(bucket_durability::lookup(bucket), None);
|
||||
|
||||
// Invalid payloads degrade to "no override", never to a tier.
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.durability_config_json = br#"{"mode":"bogus"}"#.to_vec();
|
||||
sync_bucket_durability(bucket, &bm);
|
||||
assert_eq!(bucket_durability::lookup(bucket), None);
|
||||
|
||||
// Cache removal clears the override too.
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.durability_config_json = br#"{"mode":"none"}"#.to_vec();
|
||||
sync_bucket_durability(bucket, &bm);
|
||||
assert_eq!(bucket_durability::lookup(bucket), Some(DurabilityMode::None));
|
||||
clear_bucket_durability(bucket);
|
||||
assert_eq!(bucket_durability::lookup(bucket), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_wait_exits_when_cancelled() {
|
||||
let cancel_token = CancellationToken::new();
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
|
||||
pub mod bandwidth;
|
||||
pub mod bucket_target_sys;
|
||||
pub mod durability;
|
||||
pub mod error;
|
||||
pub mod lifecycle;
|
||||
pub mod metadata;
|
||||
|
||||
@@ -288,7 +288,7 @@ pub(crate) enum DurabilityMode {
|
||||
}
|
||||
|
||||
impl DurabilityMode {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
pub(crate) fn parse(value: &str) -> Option<Self> {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"strict" => Some(Self::Strict),
|
||||
"relaxed" => Some(Self::Relaxed),
|
||||
@@ -406,6 +406,96 @@ pub(crate) mod durability_mode_override {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-bucket durability overrides (HP-5 phase 2, rustfs/backlog#938).
|
||||
///
|
||||
/// The disk layer never loads bucket metadata itself: the bucket metadata
|
||||
/// subsystem publishes the parsed override here whenever a bucket's cached
|
||||
/// metadata is set, refreshed, or removed, so this registry follows exactly
|
||||
/// the existing bucket-metadata cache invalidation semantics (immediate on
|
||||
/// the node applying a config change, peer reload notification plus the
|
||||
/// periodic refresh loop elsewhere). Lookups sit on the commit hot path, so
|
||||
/// the empty-registry case (no bucket overrides configured anywhere — the
|
||||
/// default) is a single relaxed atomic load and the phase 1 behavior is
|
||||
/// preserved bit for bit.
|
||||
pub(crate) mod bucket_durability {
|
||||
use super::{
|
||||
DurabilityMode, EVENT_DISK_LOCAL_DURABILITY_MODE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_DISK_LOCAL, is_scratch_volume,
|
||||
is_system_critical_volume,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{OnceLock, PoisonError, RwLock};
|
||||
use tracing::{info, warn};
|
||||
|
||||
static OVERRIDES: OnceLock<RwLock<HashMap<String, DurabilityMode>>> = OnceLock::new();
|
||||
/// Fast-path gate: false means "no override registered anywhere", which
|
||||
/// keeps default deployments off the map lookup entirely.
|
||||
static NON_EMPTY: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
fn overrides() -> &'static RwLock<HashMap<String, DurabilityMode>> {
|
||||
OVERRIDES.get_or_init(|| RwLock::new(HashMap::new()))
|
||||
}
|
||||
|
||||
/// Publish (or clear, with `None`) the durability override for `bucket`.
|
||||
///
|
||||
/// System namespaces can never carry an override: they are pinned to
|
||||
/// `strict` by [`super::effective_durability`], and any attempt to
|
||||
/// register one is rejected here as defense in depth.
|
||||
pub(crate) fn set(bucket: &str, mode: Option<DurabilityMode>) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
if is_system_critical_volume(bucket) || is_scratch_volume(bucket) {
|
||||
if mode.is_some() {
|
||||
warn!(
|
||||
event = EVENT_DISK_LOCAL_DURABILITY_MODE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
bucket = %bucket,
|
||||
"Rejected per-bucket durability override for a system namespace; it stays pinned to strict"
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// The legacy full-off switch is process-wide only and deliberately
|
||||
// unreachable per bucket (`DurabilityMode::parse` never returns it).
|
||||
let mode = mode.filter(|m| *m != DurabilityMode::LegacyOff);
|
||||
|
||||
let mut map = overrides().write().unwrap_or_else(PoisonError::into_inner);
|
||||
let changed = match mode {
|
||||
Some(mode) => map.insert(bucket.to_string(), mode) != Some(mode),
|
||||
None => map.remove(bucket).is_some(),
|
||||
};
|
||||
NON_EMPTY.store(!map.is_empty(), Ordering::Release);
|
||||
drop(map);
|
||||
|
||||
if changed {
|
||||
info!(
|
||||
event = EVENT_DISK_LOCAL_DURABILITY_MODE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
bucket = %bucket,
|
||||
mode = mode.map_or("inherit", |m| m.as_str()),
|
||||
"Per-bucket durability override updated"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The override registered for `volume`, if any. `volume` is the commit
|
||||
/// destination, so user buckets resolve by name while scratch and system
|
||||
/// namespaces never match (they are refused by [`set`]).
|
||||
pub(crate) fn lookup(volume: &str) -> Option<DurabilityMode> {
|
||||
if !NON_EMPTY.load(Ordering::Acquire) {
|
||||
return None;
|
||||
}
|
||||
overrides()
|
||||
.read()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.get(volume)
|
||||
.copied()
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `volume` stages in-flight user object data (`.rustfs.sys/tmp`,
|
||||
/// `.rustfs.sys/multipart`, and their subtrees). These namespaces follow the
|
||||
/// configured durability mode: their contents commit into user buckets and
|
||||
@@ -438,21 +528,21 @@ fn is_system_critical_volume(volume: &str) -> bool {
|
||||
|
||||
/// Effective durability for writes that commit into `volume`.
|
||||
///
|
||||
/// System-critical volumes are pinned to `Strict` regardless of the
|
||||
/// configured tier. The legacy full-off switch keeps its historical
|
||||
/// semantics and is never pinned.
|
||||
/// Resolution order: system-critical volumes are pinned to `Strict`
|
||||
/// regardless of any configuration; otherwise a per-bucket override
|
||||
/// (published by the bucket metadata subsystem, see [`bucket_durability`])
|
||||
/// wins over the process-wide mode; otherwise the process-wide mode applies.
|
||||
/// The legacy full-off switch keeps its historical semantics: it is never
|
||||
/// pinned and per-bucket overrides do not apply under it.
|
||||
pub(crate) fn effective_durability(volume: &str) -> DurabilityMode {
|
||||
let mode = durability_mode();
|
||||
match mode {
|
||||
DurabilityMode::Strict | DurabilityMode::LegacyOff => mode,
|
||||
DurabilityMode::Relaxed | DurabilityMode::None => {
|
||||
if is_system_critical_volume(volume) {
|
||||
DurabilityMode::Strict
|
||||
} else {
|
||||
mode
|
||||
}
|
||||
}
|
||||
let global = durability_mode();
|
||||
if global == DurabilityMode::LegacyOff {
|
||||
return global;
|
||||
}
|
||||
if is_system_critical_volume(volume) {
|
||||
return DurabilityMode::Strict;
|
||||
}
|
||||
bucket_durability::lookup(volume).unwrap_or(global)
|
||||
}
|
||||
|
||||
/// Get the O_DIRECT read threshold size.
|
||||
@@ -5627,6 +5717,150 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
/// Removes the bucket's durability override when dropped so a test can
|
||||
/// never leak its override into another test's lookup.
|
||||
struct BucketOverrideGuard(&'static str);
|
||||
|
||||
impl BucketOverrideGuard {
|
||||
fn set(bucket: &'static str, mode: DurabilityMode) -> Self {
|
||||
bucket_durability::set(bucket, Some(mode));
|
||||
Self(bucket)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BucketOverrideGuard {
|
||||
fn drop(&mut self) {
|
||||
bucket_durability::set(self.0, None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_effective_durability_bucket_override() {
|
||||
// Global strict + per-bucket relaxed: only the named bucket drops.
|
||||
{
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
let _guard = BucketOverrideGuard::set("hp5b-override-relaxed", DurabilityMode::Relaxed);
|
||||
assert_eq!(effective_durability("hp5b-override-relaxed"), DurabilityMode::Relaxed);
|
||||
assert_eq!(effective_durability("hp5b-other-bucket"), DurabilityMode::Strict);
|
||||
assert_eq!(effective_durability(RUSTFS_META_BUCKET), DurabilityMode::Strict);
|
||||
}
|
||||
// Override cleared: the bucket follows the global mode again (a new
|
||||
// PUT after a config change resolves the new tier).
|
||||
{
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
assert_eq!(effective_durability("hp5b-override-relaxed"), DurabilityMode::Strict);
|
||||
}
|
||||
// Global relaxed + per-bucket strict: overrides can raise durability.
|
||||
{
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Relaxed);
|
||||
let _guard = BucketOverrideGuard::set("hp5b-override-strict", DurabilityMode::Strict);
|
||||
assert_eq!(effective_durability("hp5b-override-strict"), DurabilityMode::Strict);
|
||||
assert_eq!(effective_durability("hp5b-other-bucket"), DurabilityMode::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_durability_refuses_system_namespaces() {
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
|
||||
// System-critical and scratch namespaces can never carry an override.
|
||||
bucket_durability::set(RUSTFS_META_BUCKET, Some(DurabilityMode::Relaxed));
|
||||
bucket_durability::set(&format!("{RUSTFS_META_BUCKET}/buckets"), Some(DurabilityMode::None));
|
||||
bucket_durability::set(RUSTFS_META_TMP_BUCKET, Some(DurabilityMode::Relaxed));
|
||||
bucket_durability::set(super::super::RUSTFS_META_MULTIPART_BUCKET, Some(DurabilityMode::Relaxed));
|
||||
bucket_durability::set("", Some(DurabilityMode::Relaxed));
|
||||
|
||||
assert_eq!(bucket_durability::lookup(RUSTFS_META_BUCKET), Option::None);
|
||||
assert_eq!(bucket_durability::lookup(RUSTFS_META_TMP_BUCKET), Option::None);
|
||||
assert_eq!(effective_durability(RUSTFS_META_BUCKET), DurabilityMode::Strict);
|
||||
|
||||
// The legacy full-off tier is process-wide only: registering it per
|
||||
// bucket is dropped, not stored.
|
||||
bucket_durability::set("hp5b-legacy-refused", Some(DurabilityMode::LegacyOff));
|
||||
assert_eq!(bucket_durability::lookup("hp5b-legacy-refused"), Option::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_effective_durability_legacy_off_ignores_bucket_overrides() {
|
||||
let _mode = durability_mode_override::set(DurabilityMode::LegacyOff);
|
||||
let _guard = BucketOverrideGuard::set("hp5b-legacy-bucket", DurabilityMode::Strict);
|
||||
// The legacy switch keeps its historical semantics bit for bit.
|
||||
assert_eq!(effective_durability("hp5b-legacy-bucket"), DurabilityMode::LegacyOff);
|
||||
}
|
||||
|
||||
/// HP-5b behavior regression: with the global mode at strict (the
|
||||
/// default), a bucket override to relaxed must skip the metadata-commit
|
||||
/// dir fsync for that bucket only, and clearing the override must restore
|
||||
/// the strict behavior for the next write.
|
||||
#[tokio::test]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn test_write_all_meta_bucket_override_relaxed_then_cleared() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
|
||||
let overridden = "hp5b-relaxed-write-bucket";
|
||||
let untouched = "hp5b-strict-write-bucket";
|
||||
ensure_test_volume(&disk, overridden).await;
|
||||
ensure_test_volume(&disk, untouched).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
|
||||
let meta_path = format!("dir/object/{STORAGE_FORMAT_FILE}");
|
||||
|
||||
{
|
||||
let _guard = BucketOverrideGuard::set("hp5b-relaxed-write-bucket", DurabilityMode::Relaxed);
|
||||
|
||||
disk.write_all_meta(overridden, &meta_path, b"payload", true)
|
||||
.await
|
||||
.expect("write_all_meta should succeed");
|
||||
let overridden_parent = disk
|
||||
.get_object_path(overridden, &meta_path)
|
||||
.expect("dst path should resolve")
|
||||
.parent()
|
||||
.expect("dst file should have a parent")
|
||||
.to_path_buf();
|
||||
assert!(
|
||||
!os::fsync_dir_recorder::was_fsynced(&overridden_parent),
|
||||
"bucket override to relaxed must skip the metadata-commit dir fsync"
|
||||
);
|
||||
|
||||
// A bucket without an override keeps the strict default.
|
||||
disk.write_all_meta(untouched, &meta_path, b"payload", true)
|
||||
.await
|
||||
.expect("write_all_meta should succeed");
|
||||
let untouched_parent = disk
|
||||
.get_object_path(untouched, &meta_path)
|
||||
.expect("dst path should resolve")
|
||||
.parent()
|
||||
.expect("dst file should have a parent")
|
||||
.to_path_buf();
|
||||
assert!(
|
||||
os::fsync_dir_recorder::was_fsynced(&untouched_parent),
|
||||
"buckets without an override must keep the strict commit fsyncs"
|
||||
);
|
||||
}
|
||||
|
||||
// Override cleared (guard dropped): the next write is strict again.
|
||||
let second_meta_path = format!("dir/object-after-clear/{STORAGE_FORMAT_FILE}");
|
||||
disk.write_all_meta(overridden, &second_meta_path, b"payload", true)
|
||||
.await
|
||||
.expect("write_all_meta should succeed");
|
||||
let after_clear_parent = disk
|
||||
.get_object_path(overridden, &second_meta_path)
|
||||
.expect("dst path should resolve")
|
||||
.parent()
|
||||
.expect("dst file should have a parent")
|
||||
.to_path_buf();
|
||||
assert!(
|
||||
os::fsync_dir_recorder::was_fsynced(&after_clear_parent),
|
||||
"clearing the override must restore strict fsyncs for new writes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn test_write_all_meta_relaxed_skips_dst_parent_dir_fsync() {
|
||||
|
||||
@@ -113,6 +113,78 @@ staged in tmp still commits with full `strict` durability.
|
||||
The durability mode is server-side configuration only; it cannot be raised or
|
||||
lowered by any request header.
|
||||
|
||||
## Per-bucket durability (phase 2)
|
||||
|
||||
A bucket can override the process-wide mode with its own tier. The override
|
||||
is stored in the bucket's metadata (a `durability.json` entry in
|
||||
`.rustfs.sys/buckets/<bucket>/.metadata.bin`, written as a RustFS extension
|
||||
field that MinIO's decoder skips) and resolved per write at the commit
|
||||
points, so no restart is needed.
|
||||
|
||||
### Configuration
|
||||
|
||||
```bash
|
||||
# Set an override (admin credentials, ConfigUpdateAdminAction)
|
||||
curl -X PUT "http://<host>/rustfs/admin/v3/bucket-durability/<bucket>" \
|
||||
-d '{"mode":"relaxed"}' # strict | relaxed | none
|
||||
|
||||
# Read it back ("mode": null means the bucket inherits the global mode)
|
||||
curl "http://<host>/rustfs/admin/v3/bucket-durability/<bucket>"
|
||||
|
||||
# Clear it (the bucket inherits the global mode again)
|
||||
curl -X DELETE "http://<host>/rustfs/admin/v3/bucket-durability/<bucket>"
|
||||
```
|
||||
|
||||
`mc` integration is a follow-up; for now the admin API above is the
|
||||
configuration plane.
|
||||
|
||||
### Resolution order
|
||||
|
||||
For a write committing into volume `V`:
|
||||
|
||||
1. system-critical namespaces (`.rustfs.sys`, `.minio.sys` outside the
|
||||
scratch dirs) are always `strict` — a bucket override can never be
|
||||
attached to them, and any attempt is rejected and logged;
|
||||
2. otherwise, if the destination bucket has an override, the override wins —
|
||||
in **both** directions (a bucket can be `relaxed` under a `strict` global
|
||||
default, or pinned `strict` under a `relaxed` global default);
|
||||
3. otherwise the process-wide mode applies.
|
||||
|
||||
Under `legacy-off` (`RUSTFS_DRIVE_SYNC_ENABLE=false`) per-bucket overrides
|
||||
do not apply at all: the legacy switch keeps its historical semantics bit
|
||||
for bit. `legacy-off` is also not a valid per-bucket tier.
|
||||
|
||||
Because scratch-staged data commits under the destination volume, an object
|
||||
staged in `.rustfs.sys/tmp` follows the override of the bucket it commits
|
||||
into, exactly like the global tier.
|
||||
|
||||
### Propagation and effect latency
|
||||
|
||||
The override rides the existing bucket-metadata cache, with no invalidation
|
||||
channel of its own:
|
||||
|
||||
- on the node that applies the config change, the new tier is effective for
|
||||
writes that resolve their durability after the update completes;
|
||||
- other nodes are told to reload the bucket's metadata right away (the same
|
||||
peer notification used for every bucket config change); if a peer misses
|
||||
the notification, the periodic bucket-metadata refresh loop (15 minutes)
|
||||
converges it;
|
||||
- an in-flight operation keeps the tier it resolved at its start — a single
|
||||
commit is never half-old-tier, half-new-tier;
|
||||
- deleting the bucket drops the override with the rest of its metadata.
|
||||
|
||||
Until a peer has reloaded the metadata, its writes use the previous tier.
|
||||
This is the same eventual-consistency window every other bucket config
|
||||
(policy, quota, versioning) already has.
|
||||
|
||||
### Power-loss guarantees
|
||||
|
||||
Identical to the corresponding global tier, scoped to the bucket. In
|
||||
particular the `relaxed` deployment rule (multi-node clusters in independent
|
||||
power domains only) applies per bucket: overriding a bucket to `relaxed` on
|
||||
a single-node deployment can lose recently acknowledged objects in that
|
||||
bucket on power failure.
|
||||
|
||||
## Performance expectations
|
||||
|
||||
The often-quoted 26x PUT throughput delta was measured on macOS with the old
|
||||
@@ -124,6 +196,8 @@ numbers to size `relaxed`.
|
||||
|
||||
## Scope
|
||||
|
||||
This is phase 1 of rustfs/backlog#926: a global, per-process tier configured
|
||||
by environment variable. Per-bucket durability tiers (bucket metadata +
|
||||
admin API) are a separate follow-up phase.
|
||||
Phase 1 (rustfs/backlog#926) shipped the global, per-process tier configured
|
||||
by environment variable. Phase 2 (rustfs/backlog#938) adds the per-bucket
|
||||
override described above, configured through the admin API and stored in
|
||||
bucket metadata. `mc admin` integration for the per-bucket tier is a
|
||||
follow-up.
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-bucket durability tier admin handlers (HP-5 phase 2, rustfs/backlog#938).
|
||||
//!
|
||||
//! A bucket can override the process-wide durability mode
|
||||
//! (`RUSTFS_DURABILITY_MODE`) with its own `strict` | `relaxed` | `none`
|
||||
//! tier. The override is stored in the bucket metadata (`durability.json`
|
||||
//! entry) and consumed by the disk layer at commit points. System buckets
|
||||
//! can never carry an override. See docs/operations/durability-modes.md.
|
||||
|
||||
use crate::admin::auth::validate_admin_request;
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::runtime_sources::current_notification_system;
|
||||
use crate::admin::storage_api::bucket::durability::BucketDurabilityConfig;
|
||||
use crate::admin::storage_api::bucket::metadata::BUCKET_DURABILITY_CONFIG;
|
||||
use crate::admin::storage_api::bucket::metadata_sys;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::server::ADMIN_PREFIX;
|
||||
use hyper::{Method, StatusCode};
|
||||
use matchit::Params;
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::{info, warn};
|
||||
|
||||
const LOG_COMPONENT_ADMIN: &str = "admin";
|
||||
const LOG_SUBSYSTEM_DURABILITY: &str = "bucket_durability";
|
||||
const EVENT_ADMIN_BUCKET_DURABILITY: &str = "admin_bucket_durability_state";
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SetBucketDurabilityRequest {
|
||||
mode: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct BucketDurabilityResponse {
|
||||
bucket: String,
|
||||
/// The bucket's own override, or `null` when the bucket inherits the
|
||||
/// process-wide durability mode.
|
||||
mode: Option<String>,
|
||||
}
|
||||
|
||||
pub struct SetBucketDurabilityHandler;
|
||||
pub struct GetBucketDurabilityHandler;
|
||||
pub struct DeleteBucketDurabilityHandler;
|
||||
|
||||
pub fn register_durability_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
|
||||
r.insert(
|
||||
Method::PUT,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/bucket-durability/{bucket}").as_str(),
|
||||
AdminOperation(&SetBucketDurabilityHandler {}),
|
||||
)?;
|
||||
|
||||
r.insert(
|
||||
Method::GET,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/bucket-durability/{bucket}").as_str(),
|
||||
AdminOperation(&GetBucketDurabilityHandler {}),
|
||||
)?;
|
||||
|
||||
r.insert(
|
||||
Method::DELETE,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/bucket-durability/{bucket}").as_str(),
|
||||
AdminOperation(&DeleteBucketDurabilityHandler {}),
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_set_bucket_durability_request(body: &[u8]) -> Result<SetBucketDurabilityRequest, s3s::S3Error> {
|
||||
if body.is_empty() {
|
||||
return Err(s3_error!(InvalidRequest, "request body is required, e.g. {{\"mode\":\"relaxed\"}}"));
|
||||
}
|
||||
serde_json::from_slice(body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e))
|
||||
}
|
||||
|
||||
/// Validates and canonicalizes the requested tier name.
|
||||
fn normalize_mode(mode: &str) -> Result<String, s3s::S3Error> {
|
||||
let normalized = mode.trim().to_ascii_lowercase();
|
||||
if BucketDurabilityConfig::is_valid_mode(&normalized) {
|
||||
Ok(normalized)
|
||||
} else {
|
||||
Err(s3_error!(
|
||||
InvalidArgument,
|
||||
"invalid durability mode {:?}: expected strict|relaxed|none",
|
||||
mode
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Kick peers to reload the bucket's metadata so the override takes effect
|
||||
/// cluster-wide without waiting for the periodic refresh loop. Failures are
|
||||
/// logged, not surfaced: the refresh loop converges peers eventually.
|
||||
fn notify_peers_reload(bucket: String, operation: &'static str) {
|
||||
tokio::spawn(async move {
|
||||
if let Some(notification_sys) = current_notification_system()
|
||||
&& let Err(err) = notification_sys.load_bucket_metadata(&bucket).await
|
||||
{
|
||||
warn!(
|
||||
event = EVENT_ADMIN_BUCKET_DURABILITY,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_DURABILITY,
|
||||
bucket = %bucket,
|
||||
error = %err,
|
||||
"failed to notify peers after {operation}"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn authenticate_admin(req: &S3Request<Body>) -> S3Result<()> {
|
||||
let Some(ref cred) = req.credentials else {
|
||||
return Err(s3_error!(InvalidRequest, "authentication required"));
|
||||
};
|
||||
|
||||
let (cred, owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?;
|
||||
|
||||
validate_admin_request(
|
||||
&req.headers,
|
||||
&cred,
|
||||
owner,
|
||||
false,
|
||||
vec![Action::AdminAction(AdminAction::ConfigUpdateAdminAction)],
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn bucket_from_params(params: &Params<'_, '_>) -> S3Result<String> {
|
||||
let bucket = params.get("bucket").unwrap_or("").to_string();
|
||||
if bucket.is_empty() {
|
||||
return Err(s3_error!(InvalidRequest, "bucket name is required"));
|
||||
}
|
||||
Ok(bucket)
|
||||
}
|
||||
|
||||
fn durability_response(bucket: String, mode: Option<String>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let response = BucketDurabilityResponse { bucket, mode };
|
||||
let json = serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
|
||||
Ok(S3Response::new((StatusCode::OK, Body::from(json))))
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for SetBucketDurabilityHandler {
|
||||
#[tracing::instrument(skip_all)]
|
||||
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
authenticate_admin(&req).await?;
|
||||
|
||||
let bucket = bucket_from_params(¶ms)?;
|
||||
|
||||
let body = req
|
||||
.input
|
||||
.store_all_limited(rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?;
|
||||
|
||||
let request = parse_set_bucket_durability_request(&body)?;
|
||||
let mode = normalize_mode(&request.mode)?;
|
||||
|
||||
let config = BucketDurabilityConfig::new(&mode);
|
||||
let json = serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "failed to encode config: {}", e))?;
|
||||
|
||||
// System buckets are rejected by the metadata layer (and pinned to
|
||||
// strict by the disk layer regardless).
|
||||
metadata_sys::update(&bucket, BUCKET_DURABILITY_CONFIG, json)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to set bucket durability: {}", e))?;
|
||||
|
||||
info!(
|
||||
event = EVENT_ADMIN_BUCKET_DURABILITY,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_DURABILITY,
|
||||
bucket = %bucket,
|
||||
mode = %mode,
|
||||
"bucket durability override set"
|
||||
);
|
||||
|
||||
notify_peers_reload(bucket.clone(), "set bucket durability");
|
||||
|
||||
durability_response(bucket, Some(mode))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for GetBucketDurabilityHandler {
|
||||
#[tracing::instrument(skip_all)]
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
authenticate_admin(&req).await?;
|
||||
|
||||
let bucket = bucket_from_params(¶ms)?;
|
||||
|
||||
let (config, _updated_at) = metadata_sys::get_durability_config(&bucket)
|
||||
.await
|
||||
.map_err(|e| s3_error!(NoSuchBucket, "failed to read bucket durability: {}", e))?;
|
||||
|
||||
let mode = config.and_then(|c| c.normalized_mode());
|
||||
durability_response(bucket, mode)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for DeleteBucketDurabilityHandler {
|
||||
#[tracing::instrument(skip_all)]
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
authenticate_admin(&req).await?;
|
||||
|
||||
let bucket = bucket_from_params(¶ms)?;
|
||||
|
||||
metadata_sys::delete(&bucket, BUCKET_DURABILITY_CONFIG)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to clear bucket durability: {}", e))?;
|
||||
|
||||
info!(
|
||||
event = EVENT_ADMIN_BUCKET_DURABILITY,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_DURABILITY,
|
||||
bucket = %bucket,
|
||||
"bucket durability override cleared"
|
||||
);
|
||||
|
||||
notify_peers_reload(bucket.clone(), "clear bucket durability");
|
||||
|
||||
durability_response(bucket, None)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_set_request_requires_body_and_valid_json() {
|
||||
assert!(parse_set_bucket_durability_request(b"").is_err());
|
||||
assert!(parse_set_bucket_durability_request(b"not-json").is_err());
|
||||
|
||||
let req = parse_set_bucket_durability_request(br#"{"mode":"relaxed"}"#).expect("parse");
|
||||
assert_eq!(req.mode, "relaxed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_mode_accepts_tiers_and_rejects_everything_else() {
|
||||
assert_eq!(normalize_mode("strict").unwrap(), "strict");
|
||||
assert_eq!(normalize_mode(" RELAXED ").unwrap(), "relaxed");
|
||||
assert_eq!(normalize_mode("none").unwrap(), "none");
|
||||
assert!(normalize_mode("").is_err());
|
||||
assert!(normalize_mode("bogus").is_err());
|
||||
// The legacy full-off switch is process-wide only.
|
||||
assert!(normalize_mode("legacy-off").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_serializes_inherit_as_null() {
|
||||
let json = serde_json::to_string(&BucketDurabilityResponse {
|
||||
bucket: "b".to_string(),
|
||||
mode: None,
|
||||
})
|
||||
.expect("serialize");
|
||||
assert_eq!(json, r#"{"bucket":"b","mode":null}"#);
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ pub mod bucket_meta;
|
||||
pub mod cluster_snapshot;
|
||||
pub mod config_admin;
|
||||
pub mod diagnostics;
|
||||
pub mod durability;
|
||||
pub mod event;
|
||||
pub mod extensions;
|
||||
pub mod group;
|
||||
|
||||
@@ -33,9 +33,10 @@ mod console_test;
|
||||
mod route_registration_test;
|
||||
|
||||
use handlers::{
|
||||
audit, batch_job, bucket_meta, cluster_snapshot, config_admin, diagnostics, extensions, heal, health, idp_compat, kms,
|
||||
module_switch, object_zip_download, oidc, plugins_catalog, plugins_instances, pools, profile_admin, quota as quota_handler,
|
||||
rebalance, replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, user,
|
||||
audit, batch_job, bucket_meta, cluster_snapshot, config_admin, diagnostics, durability as durability_handler, extensions,
|
||||
heal, health, idp_compat, kms, module_switch, object_zip_download, oidc, plugins_catalog, plugins_instances, pools,
|
||||
profile_admin, quota as quota_handler, rebalance, replication as replication_handler, scanner, site_replication, sts, system,
|
||||
table_catalog, tier, tls_debug, user,
|
||||
};
|
||||
use router::{AdminOperation, S3Router};
|
||||
use s3s::route::S3Route;
|
||||
@@ -67,6 +68,7 @@ fn register_admin_routes(r: &mut S3Router<AdminOperation>) -> std::io::Result<()
|
||||
tier::register_tier_route(r)?;
|
||||
|
||||
quota_handler::register_quota_route(r)?;
|
||||
durability_handler::register_durability_route(r)?;
|
||||
bucket_meta::register_bucket_meta_route(r)?;
|
||||
config_admin::register_config_route(r)?;
|
||||
scanner::register_scanner_route(r)?;
|
||||
|
||||
@@ -332,6 +332,24 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
SET_BUCKET_QUOTA,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Put,
|
||||
"/rustfs/admin/v3/bucket-durability/{bucket}",
|
||||
CONFIG_UPDATE,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/rustfs/admin/v3/bucket-durability/{bucket}",
|
||||
CONFIG_UPDATE,
|
||||
RouteRiskLevel::Sensitive,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Delete,
|
||||
"/rustfs/admin/v3/bucket-durability/{bucket}",
|
||||
CONFIG_UPDATE,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/rustfs/admin/export-bucket-metadata",
|
||||
|
||||
@@ -199,6 +199,9 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
||||
admin_route_sample(Method::DELETE, "/v3/quota/{bucket}", "/v3/quota/test-bucket"),
|
||||
admin_route_sample(Method::GET, "/v3/quota-stats/{bucket}", "/v3/quota-stats/test-bucket"),
|
||||
admin_route_sample(Method::POST, "/v3/quota-check/{bucket}", "/v3/quota-check/test-bucket"),
|
||||
admin_route_sample(Method::PUT, "/v3/bucket-durability/{bucket}", "/v3/bucket-durability/test-bucket"),
|
||||
admin_route_sample(Method::GET, "/v3/bucket-durability/{bucket}", "/v3/bucket-durability/test-bucket"),
|
||||
admin_route_sample(Method::DELETE, "/v3/bucket-durability/{bucket}", "/v3/bucket-durability/test-bucket"),
|
||||
admin_route(Method::GET, "/export-bucket-metadata"),
|
||||
admin_route(Method::GET, "/v3/export-bucket-metadata"),
|
||||
admin_route(Method::PUT, "/import-bucket-metadata"),
|
||||
@@ -1155,6 +1158,9 @@ fn test_register_routes_cover_representative_admin_paths() {
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/get-bucket-quota"));
|
||||
assert_route(&router, Method::PUT, &admin_path("/v3/quota/test-bucket"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/quota-stats/test-bucket"));
|
||||
assert_route(&router, Method::PUT, &admin_path("/v3/bucket-durability/test-bucket"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/bucket-durability/test-bucket"));
|
||||
assert_route(&router, Method::DELETE, &admin_path("/v3/bucket-durability/test-bucket"));
|
||||
|
||||
assert_route(&router, Method::GET, &admin_path("/export-bucket-metadata"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/export-bucket-metadata"));
|
||||
|
||||
@@ -20,8 +20,8 @@ use time::OffsetDateTime;
|
||||
|
||||
mod ecstore_bucket {
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::{
|
||||
bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning,
|
||||
versioning_sys,
|
||||
bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, quota, replication, target, utils,
|
||||
versioning, versioning_sys,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -209,6 +209,7 @@ pub(crate) mod metadata {
|
||||
pub(crate) const BUCKET_TAGGING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
pub(crate) const BUCKET_TARGETS_FILE: &str = super::ecstore_bucket::metadata::BUCKET_TARGETS_FILE;
|
||||
pub(crate) const BUCKET_VERSIONING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG;
|
||||
pub(crate) const BUCKET_DURABILITY_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_DURABILITY_CONFIG;
|
||||
pub(crate) const OBJECT_LOCK_CONFIG: &str = super::ecstore_bucket::metadata::OBJECT_LOCK_CONFIG;
|
||||
|
||||
pub(crate) type BucketMetadata = super::ecstore_bucket::metadata::BucketMetadata;
|
||||
@@ -218,6 +219,10 @@ pub(crate) mod metadata {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) mod durability {
|
||||
pub(crate) type BucketDurabilityConfig = super::ecstore_bucket::durability::BucketDurabilityConfig;
|
||||
}
|
||||
|
||||
pub(crate) mod metadata_sys {
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -271,6 +276,12 @@ pub(crate) mod metadata_sys {
|
||||
super::ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_durability_config(
|
||||
bucket: &str,
|
||||
) -> Result<(Option<super::durability::BucketDurabilityConfig>, OffsetDateTime)> {
|
||||
super::ecstore_bucket::metadata_sys::get_durability_config(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
|
||||
super::ecstore_bucket::metadata_sys::get_quota_config(bucket).await
|
||||
}
|
||||
@@ -457,6 +468,7 @@ pub(crate) mod access {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use super::bandwidth;
|
||||
pub(crate) use super::bucket_target_sys as target_sys;
|
||||
pub(crate) use super::durability;
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::lifecycle;
|
||||
pub(crate) use super::metadata;
|
||||
|
||||
@@ -318,8 +318,8 @@ pub(crate) mod ecstore_admin {
|
||||
|
||||
pub(crate) mod ecstore_bucket {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::{
|
||||
bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys, replication,
|
||||
tagging, target, utils,
|
||||
bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys,
|
||||
replication, tagging, target, utils,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::{quota, versioning, versioning_sys};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user