perf(ecstore): support per-bucket durability tier overrides (#4407)

perf(ecstore): per-bucket durability tier overrides (HP-5 phase 2)

Let a bucket override the process-wide RUSTFS_DURABILITY_MODE with its own
strict/relaxed/none tier, stored as a durability.json extension entry in the
bucket metadata file and resolved at commit points via effective_durability.
System-critical buckets (.rustfs.sys, .minio.sys) can never carry an override
and stay pinned to strict; the legacy full-off switch keeps its historical
semantics and per-bucket overrides do not apply under it. Overrides are
published and cleared through the existing bucket metadata cache-invalidation
path, and an admin GET/PUT handler exposes the configuration. Default behavior
is unchanged: with no override a bucket follows the global mode, which defaults
to strict and stays byte-for-byte identical to before.

Refs: https://github.com/rustfs/backlog/issues/938, https://github.com/rustfs/backlog/issues/936

Co-authored-by: heihutu <heihutu@gmail.com>

rustfs/src/admin/handlers/durability.rs

@@ -0,0 +1,273 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Per-bucket durability tier admin handlers (HP-5 phase 2, rustfs/backlog#938).
+//!
+//! A bucket can override the process-wide durability mode
+//! (`RUSTFS_DURABILITY_MODE`) with its own `strict` | `relaxed` | `none`
+//! tier. The override is stored in the bucket metadata (`durability.json`
+//! entry) and consumed by the disk layer at commit points. System buckets
+//! can never carry an override. See docs/operations/durability-modes.md.
+
+use crate::admin::auth::validate_admin_request;
+use crate::admin::router::{AdminOperation, Operation, S3Router};
+use crate::admin::runtime_sources::current_notification_system;
+use crate::admin::storage_api::bucket::durability::BucketDurabilityConfig;
+use crate::admin::storage_api::bucket::metadata::BUCKET_DURABILITY_CONFIG;
+use crate::admin::storage_api::bucket::metadata_sys;
+use crate::auth::{check_key_valid, get_session_token};
+use crate::server::ADMIN_PREFIX;
+use hyper::{Method, StatusCode};
+use matchit::Params;
+use rustfs_policy::policy::action::{Action, AdminAction};
+use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
+use serde::{Deserialize, Serialize};
+use tracing::{info, warn};
+
+const LOG_COMPONENT_ADMIN: &str = "admin";
+const LOG_SUBSYSTEM_DURABILITY: &str = "bucket_durability";
+const EVENT_ADMIN_BUCKET_DURABILITY: &str = "admin_bucket_durability_state";
+
+#[derive(Debug, Deserialize)]
+struct SetBucketDurabilityRequest {
+    mode: String,
+}
+
+#[derive(Debug, Serialize)]
+struct BucketDurabilityResponse {
+    bucket: String,
+    /// The bucket's own override, or `null` when the bucket inherits the
+    /// process-wide durability mode.
+    mode: Option<String>,
+}
+
+pub struct SetBucketDurabilityHandler;
+pub struct GetBucketDurabilityHandler;
+pub struct DeleteBucketDurabilityHandler;
+
+pub fn register_durability_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
+    r.insert(
+        Method::PUT,
+        format!("{}{}", ADMIN_PREFIX, "/v3/bucket-durability/{bucket}").as_str(),
+        AdminOperation(&SetBucketDurabilityHandler {}),
+    )?;
+
+    r.insert(
+        Method::GET,
+        format!("{}{}", ADMIN_PREFIX, "/v3/bucket-durability/{bucket}").as_str(),
+        AdminOperation(&GetBucketDurabilityHandler {}),
+    )?;
+
+    r.insert(
+        Method::DELETE,
+        format!("{}{}", ADMIN_PREFIX, "/v3/bucket-durability/{bucket}").as_str(),
+        AdminOperation(&DeleteBucketDurabilityHandler {}),
+    )?;
+
+    Ok(())
+}
+
+fn parse_set_bucket_durability_request(body: &[u8]) -> Result<SetBucketDurabilityRequest, s3s::S3Error> {
+    if body.is_empty() {
+        return Err(s3_error!(InvalidRequest, "request body is required, e.g. {{\"mode\":\"relaxed\"}}"));
+    }
+    serde_json::from_slice(body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e))
+}
+
+/// Validates and canonicalizes the requested tier name.
+fn normalize_mode(mode: &str) -> Result<String, s3s::S3Error> {
+    let normalized = mode.trim().to_ascii_lowercase();
+    if BucketDurabilityConfig::is_valid_mode(&normalized) {
+        Ok(normalized)
+    } else {
+        Err(s3_error!(
+            InvalidArgument,
+            "invalid durability mode {:?}: expected strict|relaxed|none",
+            mode
+        ))
+    }
+}
+
+/// Kick peers to reload the bucket's metadata so the override takes effect
+/// cluster-wide without waiting for the periodic refresh loop. Failures are
+/// logged, not surfaced: the refresh loop converges peers eventually.
+fn notify_peers_reload(bucket: String, operation: &'static str) {
+    tokio::spawn(async move {
+        if let Some(notification_sys) = current_notification_system()
+            && let Err(err) = notification_sys.load_bucket_metadata(&bucket).await
+        {
+            warn!(
+                event = EVENT_ADMIN_BUCKET_DURABILITY,
+                component = LOG_COMPONENT_ADMIN,
+                subsystem = LOG_SUBSYSTEM_DURABILITY,
+                bucket = %bucket,
+                error = %err,
+                "failed to notify peers after {operation}"
+            );
+        }
+    });
+}
+
+async fn authenticate_admin(req: &S3Request<Body>) -> S3Result<()> {
+    let Some(ref cred) = req.credentials else {
+        return Err(s3_error!(InvalidRequest, "authentication required"));
+    };
+
+    let (cred, owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?;
+
+    validate_admin_request(
+        &req.headers,
+        &cred,
+        owner,
+        false,
+        vec![Action::AdminAction(AdminAction::ConfigUpdateAdminAction)],
+        None,
+    )
+    .await?;
+
+    Ok(())
+}
+
+fn bucket_from_params(params: &Params<'_, '_>) -> S3Result<String> {
+    let bucket = params.get("bucket").unwrap_or("").to_string();
+    if bucket.is_empty() {
+        return Err(s3_error!(InvalidRequest, "bucket name is required"));
+    }
+    Ok(bucket)
+}
+
+fn durability_response(bucket: String, mode: Option<String>) -> S3Result<S3Response<(StatusCode, Body)>> {
+    let response = BucketDurabilityResponse { bucket, mode };
+    let json = serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
+    Ok(S3Response::new((StatusCode::OK, Body::from(json))))
+}
+
+#[async_trait::async_trait]
+impl Operation for SetBucketDurabilityHandler {
+    #[tracing::instrument(skip_all)]
+    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        authenticate_admin(&req).await?;
+
+        let bucket = bucket_from_params(&params)?;
+
+        let body = req
+            .input
+            .store_all_limited(rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE)
+            .await
+            .map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?;
+
+        let request = parse_set_bucket_durability_request(&body)?;
+        let mode = normalize_mode(&request.mode)?;
+
+        let config = BucketDurabilityConfig::new(&mode);
+        let json = serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "failed to encode config: {}", e))?;
+
+        // System buckets are rejected by the metadata layer (and pinned to
+        // strict by the disk layer regardless).
+        metadata_sys::update(&bucket, BUCKET_DURABILITY_CONFIG, json)
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to set bucket durability: {}", e))?;
+
+        info!(
+            event = EVENT_ADMIN_BUCKET_DURABILITY,
+            component = LOG_COMPONENT_ADMIN,
+            subsystem = LOG_SUBSYSTEM_DURABILITY,
+            bucket = %bucket,
+            mode = %mode,
+            "bucket durability override set"
+        );
+
+        notify_peers_reload(bucket.clone(), "set bucket durability");
+
+        durability_response(bucket, Some(mode))
+    }
+}
+
+#[async_trait::async_trait]
+impl Operation for GetBucketDurabilityHandler {
+    #[tracing::instrument(skip_all)]
+    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        authenticate_admin(&req).await?;
+
+        let bucket = bucket_from_params(&params)?;
+
+        let (config, _updated_at) = metadata_sys::get_durability_config(&bucket)
+            .await
+            .map_err(|e| s3_error!(NoSuchBucket, "failed to read bucket durability: {}", e))?;
+
+        let mode = config.and_then(|c| c.normalized_mode());
+        durability_response(bucket, mode)
+    }
+}
+
+#[async_trait::async_trait]
+impl Operation for DeleteBucketDurabilityHandler {
+    #[tracing::instrument(skip_all)]
+    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        authenticate_admin(&req).await?;
+
+        let bucket = bucket_from_params(&params)?;
+
+        metadata_sys::delete(&bucket, BUCKET_DURABILITY_CONFIG)
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to clear bucket durability: {}", e))?;
+
+        info!(
+            event = EVENT_ADMIN_BUCKET_DURABILITY,
+            component = LOG_COMPONENT_ADMIN,
+            subsystem = LOG_SUBSYSTEM_DURABILITY,
+            bucket = %bucket,
+            "bucket durability override cleared"
+        );
+
+        notify_peers_reload(bucket.clone(), "clear bucket durability");
+
+        durability_response(bucket, None)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parse_set_request_requires_body_and_valid_json() {
+        assert!(parse_set_bucket_durability_request(b"").is_err());
+        assert!(parse_set_bucket_durability_request(b"not-json").is_err());
+
+        let req = parse_set_bucket_durability_request(br#"{"mode":"relaxed"}"#).expect("parse");
+        assert_eq!(req.mode, "relaxed");
+    }
+
+    #[test]
+    fn normalize_mode_accepts_tiers_and_rejects_everything_else() {
+        assert_eq!(normalize_mode("strict").unwrap(), "strict");
+        assert_eq!(normalize_mode(" RELAXED ").unwrap(), "relaxed");
+        assert_eq!(normalize_mode("none").unwrap(), "none");
+        assert!(normalize_mode("").is_err());
+        assert!(normalize_mode("bogus").is_err());
+        // The legacy full-off switch is process-wide only.
+        assert!(normalize_mode("legacy-off").is_err());
+    }
+
+    #[test]
+    fn response_serializes_inherit_as_null() {
+        let json = serde_json::to_string(&BucketDurabilityResponse {
+            bucket: "b".to_string(),
+            mode: None,
+        })
+        .expect("serialize");
+        assert_eq!(json, r#"{"bucket":"b","mode":null}"#);
+    }
+}

rustfs/src/admin/handlers/mod.rs

@@ -20,6 +20,7 @@ pub mod bucket_meta;
 pub mod cluster_snapshot;
 pub mod config_admin;
 pub mod diagnostics;
+pub mod durability;
 pub mod event;
 pub mod extensions;
 pub mod group;

rustfs/src/admin/mod.rs

@@ -33,9 +33,10 @@ mod console_test;
 mod route_registration_test;
 
 use handlers::{
-    audit, batch_job, bucket_meta, cluster_snapshot, config_admin, diagnostics, extensions, heal, health, idp_compat, kms,
-    module_switch, object_zip_download, oidc, plugins_catalog, plugins_instances, pools, profile_admin, quota as quota_handler,
-    rebalance, replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, user,
+    audit, batch_job, bucket_meta, cluster_snapshot, config_admin, diagnostics, durability as durability_handler, extensions,
+    heal, health, idp_compat, kms, module_switch, object_zip_download, oidc, plugins_catalog, plugins_instances, pools,
+    profile_admin, quota as quota_handler, rebalance, replication as replication_handler, scanner, site_replication, sts, system,
+    table_catalog, tier, tls_debug, user,
 };
 use router::{AdminOperation, S3Router};
 use s3s::route::S3Route;
@@ -67,6 +68,7 @@ fn register_admin_routes(r: &mut S3Router<AdminOperation>) -> std::io::Result<()
     tier::register_tier_route(r)?;
 
     quota_handler::register_quota_route(r)?;
+    durability_handler::register_durability_route(r)?;
     bucket_meta::register_bucket_meta_route(r)?;
     config_admin::register_config_route(r)?;
     scanner::register_scanner_route(r)?;

rustfs/src/admin/route_policy.rs

@@ -332,6 +332,24 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
         SET_BUCKET_QUOTA,
         RouteRiskLevel::High,
     ),
+    admin(
+        HttpMethod::Put,
+        "/rustfs/admin/v3/bucket-durability/{bucket}",
+        CONFIG_UPDATE,
+        RouteRiskLevel::High,
+    ),
+    admin(
+        HttpMethod::Get,
+        "/rustfs/admin/v3/bucket-durability/{bucket}",
+        CONFIG_UPDATE,
+        RouteRiskLevel::Sensitive,
+    ),
+    admin(
+        HttpMethod::Delete,
+        "/rustfs/admin/v3/bucket-durability/{bucket}",
+        CONFIG_UPDATE,
+        RouteRiskLevel::High,
+    ),
     admin(
         HttpMethod::Get,
         "/rustfs/admin/export-bucket-metadata",

rustfs/src/admin/route_registration_test.rs

@@ -199,6 +199,9 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
         admin_route_sample(Method::DELETE, "/v3/quota/{bucket}", "/v3/quota/test-bucket"),
         admin_route_sample(Method::GET, "/v3/quota-stats/{bucket}", "/v3/quota-stats/test-bucket"),
         admin_route_sample(Method::POST, "/v3/quota-check/{bucket}", "/v3/quota-check/test-bucket"),
+        admin_route_sample(Method::PUT, "/v3/bucket-durability/{bucket}", "/v3/bucket-durability/test-bucket"),
+        admin_route_sample(Method::GET, "/v3/bucket-durability/{bucket}", "/v3/bucket-durability/test-bucket"),
+        admin_route_sample(Method::DELETE, "/v3/bucket-durability/{bucket}", "/v3/bucket-durability/test-bucket"),
         admin_route(Method::GET, "/export-bucket-metadata"),
         admin_route(Method::GET, "/v3/export-bucket-metadata"),
         admin_route(Method::PUT, "/import-bucket-metadata"),
@@ -1155,6 +1158,9 @@ fn test_register_routes_cover_representative_admin_paths() {
     assert_route(&router, Method::GET, &admin_path("/v3/get-bucket-quota"));
     assert_route(&router, Method::PUT, &admin_path("/v3/quota/test-bucket"));
     assert_route(&router, Method::GET, &admin_path("/v3/quota-stats/test-bucket"));
+    assert_route(&router, Method::PUT, &admin_path("/v3/bucket-durability/test-bucket"));
+    assert_route(&router, Method::GET, &admin_path("/v3/bucket-durability/test-bucket"));
+    assert_route(&router, Method::DELETE, &admin_path("/v3/bucket-durability/test-bucket"));
 
     assert_route(&router, Method::GET, &admin_path("/export-bucket-metadata"));
     assert_route(&router, Method::GET, &admin_path("/v3/export-bucket-metadata"));

rustfs/src/admin/storage_api.rs

@@ -20,8 +20,8 @@ use time::OffsetDateTime;
 
 mod ecstore_bucket {
     pub(crate) use crate::storage::storage_api::ecstore_bucket::{
-        bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning,
-        versioning_sys,
+        bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, quota, replication, target, utils,
+        versioning, versioning_sys,
     };
 }
 
@@ -209,6 +209,7 @@ pub(crate) mod metadata {
     pub(crate) const BUCKET_TAGGING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_TAGGING_CONFIG;
     pub(crate) const BUCKET_TARGETS_FILE: &str = super::ecstore_bucket::metadata::BUCKET_TARGETS_FILE;
     pub(crate) const BUCKET_VERSIONING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG;
+    pub(crate) const BUCKET_DURABILITY_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_DURABILITY_CONFIG;
     pub(crate) const OBJECT_LOCK_CONFIG: &str = super::ecstore_bucket::metadata::OBJECT_LOCK_CONFIG;
 
     pub(crate) type BucketMetadata = super::ecstore_bucket::metadata::BucketMetadata;
@@ -218,6 +219,10 @@ pub(crate) mod metadata {
     }
 }
 
+pub(crate) mod durability {
+    pub(crate) type BucketDurabilityConfig = super::ecstore_bucket::durability::BucketDurabilityConfig;
+}
+
 pub(crate) mod metadata_sys {
     use std::sync::Arc;
 
@@ -271,6 +276,12 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await
     }
 
+    pub(crate) async fn get_durability_config(
+        bucket: &str,
+    ) -> Result<(Option<super::durability::BucketDurabilityConfig>, OffsetDateTime)> {
+        super::ecstore_bucket::metadata_sys::get_durability_config(bucket).await
+    }
+
     pub(crate) async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
         super::ecstore_bucket::metadata_sys::get_quota_config(bucket).await
     }
@@ -457,6 +468,7 @@ pub(crate) mod access {
 pub(crate) mod bucket {
     pub(crate) use super::bandwidth;
     pub(crate) use super::bucket_target_sys as target_sys;
+    pub(crate) use super::durability;
     #[cfg(test)]
     pub(crate) use super::lifecycle;
     pub(crate) use super::metadata;

rustfs/src/storage/storage_api.rs

@@ -318,8 +318,8 @@ pub(crate) mod ecstore_admin {
 
 pub(crate) mod ecstore_bucket {
     pub(crate) use rustfs_ecstore::api::bucket::{
-        bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys, replication,
-        tagging, target, utils,
+        bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys,
+        replication, tagging, target, utils,
     };
     pub(crate) use rustfs_ecstore::api::bucket::{quota, versioning, versioning_sys};
 }