refactor(storage): move the bucket-generation fences into storage/authorization

The generation guards, snapshots and their loaders return the owned S3 error
type; legacy-typed callers convert at the call site.
This commit is contained in:
overtrue
2026-11-18 17:36:59 +08:00
parent 1d3137e52a
commit 10cb203c76
10 changed files with 397 additions and 332 deletions
+1 -1
View File
@@ -4454,7 +4454,7 @@ impl DefaultObjectUsecase {
let expected_incarnation = match odm_read_generation(envelope, bucket) {
Ok(Some(incarnation)) => incarnation,
Ok(None) => return None,
Err(err) => return Some(OdmGetOutcome::Respond(Err(err))),
Err(err) => return Some(OdmGetOutcome::Respond(Err(err.into()))),
};
match store.bucket_incarnation_id(bucket).await {
Ok(current) if current == expected_incarnation => {}
+1 -1
View File
@@ -984,7 +984,7 @@ impl PutObjectOrigin<'_> {
fn apply_bucket_generation_guard(&self, bucket: &str, opts: &mut ObjectOptions) -> S3Result<()> {
match self {
Self::S3 { req, .. } => apply_bucket_generation_guard(*req, bucket, opts),
Self::S3 { req, .. } => apply_bucket_generation_guard(*req, bucket, opts).map_err(Into::into),
Self::Internal {
expected_bucket_incarnation_id,
..
+7 -5
View File
@@ -1903,19 +1903,21 @@ async fn owned_and_legacy_bucket_config_mutations_hand_on_the_same_decision_and_
};
let legacy_outcome = outcome(
legacy_result.map_err(|err| err.code().clone()),
bucket_config_mutation_incarnation(&legacy, bucket).map_err(|err| err.code().clone()),
bucket_config_mutation_incarnation(&legacy, bucket).map_err(|err| err.code().clone().into()),
);
let owned_outcome = outcome(
owned_result.map_err(|err| err.code().clone()),
bucket_config_mutation_incarnation(&owned, bucket).map_err(|err| err.code().clone()),
bucket_config_mutation_incarnation(&owned, bucket).map_err(|err| err.code().clone().into()),
);
// A refused request hands no snapshot on: the use case's reader refuses to run without one.
let refused_without_snapshot = |snapshot: Result<Option<uuid::Uuid>, S3ErrorCode>| {
matches!(expected, MutationOutcome::Allowed(_)) || snapshot == Err(S3ErrorCode::InternalError)
};
let both_refused_without_snapshot =
refused_without_snapshot(bucket_config_mutation_incarnation(&legacy, bucket).map_err(|err| err.code().clone()))
&& refused_without_snapshot(bucket_config_mutation_incarnation(&owned, bucket).map_err(|err| err.code().clone()));
let both_refused_without_snapshot = refused_without_snapshot(
bucket_config_mutation_incarnation(&legacy, bucket).map_err(|err| err.code().clone().into()),
) && refused_without_snapshot(
bucket_config_mutation_incarnation(&owned, bucket).map_err(|err| err.code().clone().into()),
);
let bucket_recorded = [legacy.extensions.get::<ReqInfo>(), owned.req_info()]
.iter()
.all(|info| info.and_then(|info| info.bucket.as_deref()) == Some(bucket));
+17 -305
View File
@@ -18,7 +18,7 @@ use self::access_owned::listing_multipart::{
use self::access_owned::object_misc::{authorize_delete_object, authorize_delete_objects};
use super::ObjectOptions;
use super::ecfs::FS;
use super::{ECStore, PolicySys, StorageError, get_lock_acquire_timeout, is_err_bucket_not_found};
use super::{ECStore, PolicySys, StorageError, is_err_bucket_not_found};
use crate::auth::facts::{
AuthzFacts, AuthzPrincipal, AuthzTarget, base_conditions, build_args, build_conditions, forward_secret_headers,
};
@@ -41,6 +41,20 @@ use crate::storage::authorization::{
recursive_force_delete_has_authenticated_caller, req_info_ref, request_iam_store, request_object_store, resolve_req_info,
set_requested_object_lock_retain_until,
};
use crate::storage::authorization::{
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, OdmReadGenerationGuard,
PendingDeleteBucketGenerationGuard, capture_odm_read_generation, load_bucket_config_mutation_snapshot,
load_bucket_generation,
};
#[cfg(test)]
use crate::storage::authorization::{
PUT_GENERATION_READ_QUORUM_FAILURE, install_object_generation_test_hook, wait_for_object_generation_test_hook,
};
#[cfg(test)]
use crate::storage::authorization::{
apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, bucket_config_mutation_incarnation,
odm_read_generation, retained_table_data_plane_publication_guards,
};
use crate::storage::denial_audit::DenialAuditor;
use crate::storage::helper::DenialScope;
pub(crate) use crate::storage::req_info::ReqInfo;
@@ -50,8 +64,7 @@ pub(crate) use crate::storage::request_markers::{
MultipartStagingOperation, PostObjectPolicyHeaders, PostObjectRequestMarker, RequestedCopySource, RequestedObjectLegalHold,
RequestedObjectLockRetainUntil, RequestedObjectLockRetention, RequestedObjectTags, StagedMultipartPartAuthorization,
};
use crate::storage::storage_api::contract::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, BucketOperations, BucketOptions};
use crate::storage::storage_api::contract::namespace::NamespaceLocking as _;
use crate::storage::storage_api::contract::bucket::{BucketOperations, BucketOptions};
use crate::storage::storage_api::ecstore_bucket::lockless_cache::is_lockless_cache_bucket;
use crate::storage::storage_api::runtime_sources_consumer::ServerContextSlot;
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
@@ -84,8 +97,6 @@ use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::{BTreeSet, HashMap};
use std::sync::Arc;
#[cfg(test)]
use std::sync::OnceLock;
use time::OffsetDateTime;
use url::Url;
@@ -388,67 +399,6 @@ fn invalid_table_data_plane_list_cursor() -> S3Error {
S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid table listing continuation token".to_string())
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct BucketGenerationGuard {
bucket: String,
incarnation_id: uuid::Uuid,
}
#[derive(Clone, Debug)]
pub(crate) struct BucketConfigMutationSnapshot {
bucket: String,
incarnation_id: uuid::Uuid,
}
pub(crate) fn bucket_config_mutation_incarnation<R: AuthorizationRequest + ?Sized>(
req: &R,
bucket: &str,
) -> S3Result<Option<uuid::Uuid>> {
let Some(snapshot) = req.bucket_config_mutation() else {
if req.server_context().is_some() {
return Err(s3_error!(InternalError, "bucket config mutation snapshot is missing"));
}
return Ok(None);
};
if snapshot.bucket != bucket {
return Err(s3_error!(InternalError, "bucket config mutation snapshot does not match request bucket"));
}
Ok(Some(snapshot.incarnation_id))
}
async fn load_bucket_config_mutation_snapshot<R: AuthorizationRequest + ?Sized>(
server_ctx: &ServerContextSlot,
req: &R,
bucket: &str,
) -> S3Result<BucketConfigMutationSnapshot> {
let store = server_ctx
.object_store()
.ok_or_else(|| s3_error!(InternalError, "object store is not initialized"))?;
let lock = store
.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT)
.await
.map_err(ApiError::from)?;
let _lifecycle_guard = lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(|err| {
ApiError::from(match err {
rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable {
mode: "bucket_config_mutation",
bucket: bucket.to_string(),
object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(),
required,
achieved,
},
other => StorageError::Lock(other),
})
})?;
let incarnation_id = load_bucket_generation_from_store(store.as_ref(), req, bucket)
.await?
.incarnation_id;
Ok(BucketConfigMutationSnapshot {
bucket: bucket.to_string(),
incarnation_id,
})
}
/// Authorizes a bucket-configuration mutation for either edge: `action` on `bucket`, with the
/// bucket incarnation snapshotted under the lifecycle namespace read lock BEFORE the decision, so
/// a bucket recreated during authorization is refused by the use case's conditional write. A
@@ -489,174 +439,6 @@ pub(crate) async fn authorize_bucket_tag_deletion<R: AuthorizationRequest + ?Siz
.await
}
#[derive(Clone, Debug)]
pub(crate) struct CopySourceBucketGenerationGuard {
bucket: String,
incarnation_id: uuid::Uuid,
}
#[cfg(test)]
type ObjectGenerationTestHook = (String, tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
#[cfg(test)]
static OBJECT_GENERATION_TEST_HOOK: OnceLock<std::sync::Mutex<Option<ObjectGenerationTestHook>>> = OnceLock::new();
#[cfg(test)]
tokio::task_local! {
static PUT_GENERATION_READ_QUORUM_FAILURE: ();
}
#[cfg(test)]
fn install_object_generation_test_hook(
bucket: String,
loaded: tokio::sync::oneshot::Sender<()>,
resume: tokio::sync::oneshot::Receiver<()>,
) {
*OBJECT_GENERATION_TEST_HOOK
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("object generation test hook lock should not be poisoned") = Some((bucket, loaded, resume));
}
#[cfg(test)]
async fn wait_for_object_generation_test_hook(bucket: &str) {
let hook = {
let mut slot = OBJECT_GENERATION_TEST_HOOK
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("object generation test hook lock should not be poisoned");
if slot.as_ref().is_some_and(|(expected_bucket, _, _)| expected_bucket == bucket) {
slot.take()
} else {
None
}
};
if let Some((_bucket, loaded, resume)) = hook {
let _ = loaded.send(());
let _ = resume.await;
}
}
#[derive(Clone, Debug)]
pub(crate) enum PendingDeleteBucketGenerationGuard {
Ready(BucketGenerationGuard),
Failed { code: S3ErrorCode, message: String },
}
impl PendingDeleteBucketGenerationGuard {
fn from_result(result: S3Result<BucketGenerationGuard>) -> Self {
match result {
Ok(guard) => Self::Ready(guard),
Err(err) => Self::Failed {
code: err.code().clone(),
message: err.message().unwrap_or_else(|| err.code().as_str()).to_string(),
},
}
}
}
pub(crate) async fn load_bucket_generation_from_store<R: AuthorizationRequest + ?Sized>(
store: &crate::storage::storage_api::ECStore,
req: &R,
bucket: &str,
) -> S3Result<BucketGenerationGuard> {
if let Some(existing) = req.bucket_generation() {
return if existing.bucket == bucket {
Ok(existing.clone())
} else {
Err(s3_error!(InternalError, "bucket generation guard does not match request bucket"))
};
}
let incarnation_id = store.bucket_incarnation_id(bucket).await.map_err(ApiError::from)?;
Ok(BucketGenerationGuard {
bucket: bucket.to_string(),
incarnation_id,
})
}
/// Loads the bucket incarnation through `server_ctx`'s object store, or returns the guard the
/// request already carries. The legacy hooks pass `FS::server_ctx`; the gateway edge passes the
/// envelope's server context.
pub(crate) async fn load_bucket_generation<R: AuthorizationRequest + ?Sized>(
server_ctx: &ServerContextSlot,
req: &R,
bucket: &str,
) -> S3Result<BucketGenerationGuard> {
let store = server_ctx
.object_store()
.ok_or_else(|| s3_error!(InternalError, "object store is not initialized"))?;
load_bucket_generation_from_store(store.as_ref(), req, bucket).await
}
/// A read may consult only the source admitted before its authorization.
/// Capture failures are deferred until a local miss actually needs that source.
#[derive(Clone, Debug)]
pub(crate) enum OdmReadGenerationGuard {
Unavailable,
Ready(BucketGenerationGuard),
Failed { code: S3ErrorCode, message: String },
}
impl OdmReadGenerationGuard {
fn from_result(result: S3Result<BucketGenerationGuard>) -> Self {
match result {
Ok(guard) => Self::Ready(guard),
Err(err) => Self::Failed {
code: err.code().clone(),
message: err.message().unwrap_or_else(|| err.code().as_str()).to_string(),
},
}
}
}
fn odm_read_source_configured(bucket: &str) -> bool {
let sys = crate::on_demand_migration::OnDemandMigrationSys::get();
sys.is_module_enabled() && sys.state(bucket).is_some()
}
/// The on-demand-migration source generation a read may consult, captured before its
/// authorization. A capture failure is kept in the guard and surfaces only on a source miss.
pub(crate) async fn capture_odm_read_generation<R: AuthorizationRequest + ?Sized>(
server_ctx: &ServerContextSlot,
req: &R,
bucket: &str,
) -> OdmReadGenerationGuard {
if !odm_read_source_configured(bucket) {
return OdmReadGenerationGuard::Unavailable;
}
OdmReadGenerationGuard::from_result(load_bucket_generation(server_ctx, req, bucket).await)
}
/// Direct usecase callers have no access middleware; capture at their entry.
/// A server request without the access marker must never bind to a later source.
pub(crate) async fn prepare_odm_read_generation<R: AuthorizationRequest + ?Sized>(
store: &crate::storage::storage_api::ECStore,
req: &mut R,
bucket: &str,
) {
if req.odm_read_generation().is_some() {
return;
}
let guard = if req.server_context().is_some() || !odm_read_source_configured(bucket) {
OdmReadGenerationGuard::Unavailable
} else {
OdmReadGenerationGuard::from_result(load_bucket_generation_from_store(store, req, bucket).await)
};
req.install_odm_read_generation(guard);
}
pub(crate) fn odm_read_generation<R: AuthorizationRequest + ?Sized>(req: &R, bucket: &str) -> S3Result<Option<uuid::Uuid>> {
match req.odm_read_generation() {
Some(OdmReadGenerationGuard::Ready(guard)) if guard.bucket == bucket => Ok(Some(guard.incarnation_id)),
Some(OdmReadGenerationGuard::Ready(_)) => {
Err(s3_error!(InternalError, "source generation guard does not match request bucket"))
}
Some(OdmReadGenerationGuard::Failed { code, message }) => Err(S3Error::with_message(code.clone(), message.clone())),
Some(OdmReadGenerationGuard::Unavailable) | None => Ok(None),
}
}
/// The legacy decoder's copy source as owned parts; `None` for an access-point or Outposts ARN.
pub(crate) fn legacy_copy_source(source: &CopySource) -> Option<(String, String, Option<String>)> {
match source {
@@ -675,77 +457,6 @@ fn copy_source_object<'a>(
access_owned::object_commit::CopySourceObject { bucket, key, version_id }
}
pub(crate) fn apply_bucket_generation_guard<R: AuthorizationRequest + ?Sized>(
req: &R,
bucket: &str,
opts: &mut ObjectOptions,
) -> S3Result<()> {
let guard = match req.pending_delete_bucket_generation() {
Some(PendingDeleteBucketGenerationGuard::Ready(guard)) => Some(guard),
Some(PendingDeleteBucketGenerationGuard::Failed { code, message }) => {
return Err(S3Error::with_message(code.clone(), message.clone()));
}
None => req.bucket_generation(),
};
apply_generation_fence(
req,
guard.map(|guard| (guard.bucket.as_str(), guard.incarnation_id)),
bucket,
opts,
"bucket generation guard",
)
}
pub(crate) fn retained_table_data_plane_publication_guards<R: AuthorizationRequest + ?Sized>(
req: &R,
) -> Option<TableDataPlanePublicationGuards> {
req.publication_guards().cloned()
}
pub(crate) fn apply_copy_source_bucket_generation_guard<R: AuthorizationRequest + ?Sized>(
req: &R,
bucket: &str,
opts: &mut ObjectOptions,
) -> S3Result<()> {
let guard = req.copy_source_bucket_generation();
apply_generation_fence(
req,
guard.map(|guard| (guard.bucket.as_str(), guard.incarnation_id)),
bucket,
opts,
"copy source bucket generation guard",
)
}
/// Binds `opts` to the captured incarnation of `bucket` and to the retained table publication
/// locks. A server request without a captured guard is refused: only direct use-case callers,
/// which carry no server context, run unfenced.
fn apply_generation_fence<R: AuthorizationRequest + ?Sized>(
req: &R,
guard: Option<(&str, uuid::Uuid)>,
bucket: &str,
opts: &mut ObjectOptions,
guard_name: &str,
) -> S3Result<()> {
match guard {
Some((guarded, _)) if guarded != bucket => {
return Err(S3Error::with_message(
S3ErrorCode::InternalError,
format!("{guard_name} does not match request bucket"),
));
}
Some((_, incarnation_id)) => opts.expected_bucket_incarnation_id = Some(incarnation_id),
None if req.server_context().is_some() => {
return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{guard_name} is missing")));
}
None => {}
}
if let Some(publication_guards) = req.publication_guards() {
publication_guards.add_lock_loss_fences(opts);
}
Ok(())
}
#[derive(Clone, Debug)]
pub(crate) struct ObjectTagConditions {
bucket: String,
@@ -8050,6 +7761,7 @@ mod tests {
let mut opts = crate::storage::ObjectOptions::default();
let err = apply_bucket_generation_guard(&req, "generation-guard-bucket", &mut opts)
.map_err(s3s::S3Error::from)
.expect_err("a dispatched PutObject must not commit without its authorization fence");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
@@ -80,7 +80,7 @@ pub(crate) async fn authorize_delete_object<R: AuthorizationRequest + ?Sized>(
let bucket_generation = load_bucket_generation(server_ctx, req, bucket).await;
#[cfg(test)]
wait_for_object_generation_test_hook(bucket).await;
let guard = decide_delete_object(req, bucket, key, version_id, bucket_generation).await?;
let guard = decide_delete_object(req, bucket, key, version_id, bucket_generation.map_err(Into::into)).await?;
req.install_pending_delete_bucket_generation(guard);
Ok(())
}
@@ -150,7 +150,7 @@ async fn decide_delete_object<R: AuthorizationRequest + ?Sized>(
authorize_request(req, Action::S3Action(S3Action::BypassGovernanceRetentionAction)).await?;
}
Ok(PendingDeleteBucketGenerationGuard::from_result(bucket_generation))
Ok(PendingDeleteBucketGenerationGuard::from_result(bucket_generation.map_err(Into::into)))
}
/// Select reads the current object and needs the same GetObject permission on either edge.
+6 -6
View File
@@ -23,7 +23,7 @@
use super::{
Action, AuthorizationRequest, BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, HashMap,
HeaderMap, ObjectOptions, OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, RemoteAddr, ReqInfo, RequestContext,
S3Action, S3ErrorCode, S3Request, S3Result, TableDataPlanePublicationGuards, apply_bucket_generation_guard,
S3Action, S3ErrorCode, S3Request, TableDataPlanePublicationGuards, apply_bucket_generation_guard,
apply_copy_source_bucket_generation_guard, authorize_object_request, bucket_config_mutation_incarnation,
capture_odm_read_generation, odm_read_generation, replication_request_authorized, request_context_from_req,
resource_free_condition_values, retained_table_data_plane_publication_guards,
@@ -69,10 +69,10 @@ fn server_request(extensions: &mut http::Extensions) {
/// The incarnation a reader bound, or the refusal's code and message.
type Answer = Result<Option<uuid::Uuid>, (S3ErrorCode, String)>;
fn answer<T>(result: S3Result<T>, bound: impl FnOnce(T) -> Option<uuid::Uuid>) -> Answer {
fn answer<T>(result: rustfs_s3_types::S3Result<T>, bound: impl FnOnce(T) -> Option<uuid::Uuid>) -> Answer {
result
.map(bound)
.map_err(|err| (err.code().clone(), err.message().unwrap_or_default().to_owned()))
.map_err(|err| (err.code().clone().into(), err.message().unwrap_or_default().to_owned()))
}
fn refused(code: S3ErrorCode, message: &str) -> Answer {
@@ -147,7 +147,7 @@ fn bucket_generation_guard_reads_the_same_through_both_forms() {
legacy(|ext| {
ext.insert(guard(BUCKET));
ext.insert(PendingDeleteBucketGenerationGuard::Failed {
code: S3ErrorCode::NoSuchBucket,
code: S3ErrorCode::NoSuchBucket.into(),
message: "bucket is gone".to_owned(),
});
}),
@@ -227,7 +227,7 @@ fn odm_read_generation_reads_the_same_through_both_forms() {
"failed capture surfaces its error",
legacy(|ext| {
ext.insert(OdmReadGenerationGuard::Failed {
code: S3ErrorCode::SlowDown,
code: S3ErrorCode::SlowDown.into(),
message: "capture failed".to_owned(),
});
}),
@@ -292,7 +292,7 @@ fn installed_state_reads_back_through_the_envelope() {
envelope.install_bucket_generation(guard(BUCKET));
assert_eq!(bucket_fence(&envelope), Ok(Some(incarnation())));
envelope.install_pending_delete_bucket_generation(PendingDeleteBucketGenerationGuard::Failed {
code: S3ErrorCode::NoSuchBucket,
code: S3ErrorCode::NoSuchBucket.into(),
message: "bucket is gone".to_owned(),
});
assert_eq!(bucket_fence(&envelope), refused(S3ErrorCode::NoSuchBucket, "bucket is gone"));
+335
View File
@@ -0,0 +1,335 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The bucket-generation fences an authorized request carries to its use case.
//!
//! Responsible for: the guards and snapshots the access phase captures before a decision
//! (bucket incarnation, pending-delete, copy-source, on-demand-migration source, bucket-config
//! mutation), loading them from the object store, and applying them to `ObjectOptions` so a
//! bucket recreated after authorization is refused by the conditional write.
//! Not responsible for: the policy decision itself (`decision`) or the table publication locks
//! the fence also binds (`table_data_plane`).
//! Upstream: the access hooks of both edges and `storage::access_owned`. Downstream: the object
//! store's bucket metadata and the use cases' `ObjectOptions`.
use super::AuthorizationRequest;
use crate::error::ApiError;
use crate::storage::access::TableDataPlanePublicationGuards;
use crate::storage::storage_api::contract::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT;
use crate::storage::storage_api::contract::namespace::NamespaceLocking as _;
use crate::storage::storage_api::runtime_sources_consumer::ServerContextSlot;
use crate::storage::{ObjectOptions, StorageError, get_lock_acquire_timeout};
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
#[cfg(test)]
use std::sync::OnceLock;
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct BucketGenerationGuard {
pub(crate) bucket: String,
pub(crate) incarnation_id: uuid::Uuid,
}
#[derive(Clone, Debug)]
pub(crate) struct BucketConfigMutationSnapshot {
pub(crate) bucket: String,
pub(crate) incarnation_id: uuid::Uuid,
}
pub(crate) fn bucket_config_mutation_incarnation<R: AuthorizationRequest + ?Sized>(
req: &R,
bucket: &str,
) -> S3Result<Option<uuid::Uuid>> {
let Some(snapshot) = req.bucket_config_mutation() else {
if req.server_context().is_some() {
return Err(s3_error!(InternalError, "bucket config mutation snapshot is missing"));
}
return Ok(None);
};
if snapshot.bucket != bucket {
return Err(s3_error!(InternalError, "bucket config mutation snapshot does not match request bucket"));
}
Ok(Some(snapshot.incarnation_id))
}
pub(crate) async fn load_bucket_config_mutation_snapshot<R: AuthorizationRequest + ?Sized>(
server_ctx: &ServerContextSlot,
req: &R,
bucket: &str,
) -> S3Result<BucketConfigMutationSnapshot> {
let store = server_ctx
.object_store()
.ok_or_else(|| s3_error!(InternalError, "object store is not initialized"))?;
let lock = store
.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT)
.await
.map_err(ApiError::from)?;
let _lifecycle_guard = lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(|err| {
ApiError::from(match err {
rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable {
mode: "bucket_config_mutation",
bucket: bucket.to_string(),
object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(),
required,
achieved,
},
other => StorageError::Lock(other),
})
})?;
let incarnation_id = load_bucket_generation_from_store(store.as_ref(), req, bucket)
.await?
.incarnation_id;
Ok(BucketConfigMutationSnapshot {
bucket: bucket.to_string(),
incarnation_id,
})
}
#[derive(Clone, Debug)]
pub(crate) struct CopySourceBucketGenerationGuard {
pub(crate) bucket: String,
pub(crate) incarnation_id: uuid::Uuid,
}
#[cfg(test)]
type ObjectGenerationTestHook = (String, tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
#[cfg(test)]
static OBJECT_GENERATION_TEST_HOOK: OnceLock<std::sync::Mutex<Option<ObjectGenerationTestHook>>> = OnceLock::new();
#[cfg(test)]
tokio::task_local! {
pub(crate) static PUT_GENERATION_READ_QUORUM_FAILURE: ();
}
#[cfg(test)]
pub(crate) fn install_object_generation_test_hook(
bucket: String,
loaded: tokio::sync::oneshot::Sender<()>,
resume: tokio::sync::oneshot::Receiver<()>,
) {
*OBJECT_GENERATION_TEST_HOOK
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("object generation test hook lock should not be poisoned") = Some((bucket, loaded, resume));
}
#[cfg(test)]
pub(crate) async fn wait_for_object_generation_test_hook(bucket: &str) {
let hook = {
let mut slot = OBJECT_GENERATION_TEST_HOOK
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("object generation test hook lock should not be poisoned");
if slot.as_ref().is_some_and(|(expected_bucket, _, _)| expected_bucket == bucket) {
slot.take()
} else {
None
}
};
if let Some((_bucket, loaded, resume)) = hook {
let _ = loaded.send(());
let _ = resume.await;
}
}
#[derive(Clone, Debug)]
pub(crate) enum PendingDeleteBucketGenerationGuard {
Ready(BucketGenerationGuard),
Failed { code: S3ErrorCode, message: String },
}
impl PendingDeleteBucketGenerationGuard {
pub(crate) fn from_result(result: S3Result<BucketGenerationGuard>) -> Self {
match result {
Ok(guard) => Self::Ready(guard),
Err(err) => Self::Failed {
code: err.code().clone(),
message: err.message().unwrap_or_else(|| err.code().as_str()).to_string(),
},
}
}
}
pub(crate) async fn load_bucket_generation_from_store<R: AuthorizationRequest + ?Sized>(
store: &crate::storage::storage_api::ECStore,
req: &R,
bucket: &str,
) -> S3Result<BucketGenerationGuard> {
if let Some(existing) = req.bucket_generation() {
return if existing.bucket == bucket {
Ok(existing.clone())
} else {
Err(s3_error!(InternalError, "bucket generation guard does not match request bucket"))
};
}
let incarnation_id = store.bucket_incarnation_id(bucket).await.map_err(ApiError::from)?;
Ok(BucketGenerationGuard {
bucket: bucket.to_string(),
incarnation_id,
})
}
/// Loads the bucket incarnation through `server_ctx`'s object store, or returns the guard the
/// request already carries. The legacy hooks pass `FS::server_ctx`; the gateway edge passes the
/// envelope's server context.
pub(crate) async fn load_bucket_generation<R: AuthorizationRequest + ?Sized>(
server_ctx: &ServerContextSlot,
req: &R,
bucket: &str,
) -> S3Result<BucketGenerationGuard> {
let store = server_ctx
.object_store()
.ok_or_else(|| s3_error!(InternalError, "object store is not initialized"))?;
load_bucket_generation_from_store(store.as_ref(), req, bucket).await
}
/// A read may consult only the source admitted before its authorization.
/// Capture failures are deferred until a local miss actually needs that source.
#[derive(Clone, Debug)]
pub(crate) enum OdmReadGenerationGuard {
Unavailable,
Ready(BucketGenerationGuard),
Failed { code: S3ErrorCode, message: String },
}
impl OdmReadGenerationGuard {
fn from_result(result: S3Result<BucketGenerationGuard>) -> Self {
match result {
Ok(guard) => Self::Ready(guard),
Err(err) => Self::Failed {
code: err.code().clone(),
message: err.message().unwrap_or_else(|| err.code().as_str()).to_string(),
},
}
}
}
fn odm_read_source_configured(bucket: &str) -> bool {
let sys = crate::on_demand_migration::OnDemandMigrationSys::get();
sys.is_module_enabled() && sys.state(bucket).is_some()
}
/// The on-demand-migration source generation a read may consult, captured before its
/// authorization. A capture failure is kept in the guard and surfaces only on a source miss.
pub(crate) async fn capture_odm_read_generation<R: AuthorizationRequest + ?Sized>(
server_ctx: &ServerContextSlot,
req: &R,
bucket: &str,
) -> OdmReadGenerationGuard {
if !odm_read_source_configured(bucket) {
return OdmReadGenerationGuard::Unavailable;
}
OdmReadGenerationGuard::from_result(load_bucket_generation(server_ctx, req, bucket).await)
}
/// Direct usecase callers have no access middleware; capture at their entry.
/// A server request without the access marker must never bind to a later source.
pub(crate) async fn prepare_odm_read_generation<R: AuthorizationRequest + ?Sized>(
store: &crate::storage::storage_api::ECStore,
req: &mut R,
bucket: &str,
) {
if req.odm_read_generation().is_some() {
return;
}
let guard = if req.server_context().is_some() || !odm_read_source_configured(bucket) {
OdmReadGenerationGuard::Unavailable
} else {
OdmReadGenerationGuard::from_result(load_bucket_generation_from_store(store, req, bucket).await)
};
req.install_odm_read_generation(guard);
}
pub(crate) fn odm_read_generation<R: AuthorizationRequest + ?Sized>(req: &R, bucket: &str) -> S3Result<Option<uuid::Uuid>> {
match req.odm_read_generation() {
Some(OdmReadGenerationGuard::Ready(guard)) if guard.bucket == bucket => Ok(Some(guard.incarnation_id)),
Some(OdmReadGenerationGuard::Ready(_)) => {
Err(s3_error!(InternalError, "source generation guard does not match request bucket"))
}
Some(OdmReadGenerationGuard::Failed { code, message }) => Err(S3Error::with_message(code.clone(), message.clone())),
Some(OdmReadGenerationGuard::Unavailable) | None => Ok(None),
}
}
pub(crate) fn apply_bucket_generation_guard<R: AuthorizationRequest + ?Sized>(
req: &R,
bucket: &str,
opts: &mut ObjectOptions,
) -> S3Result<()> {
let guard = match req.pending_delete_bucket_generation() {
Some(PendingDeleteBucketGenerationGuard::Ready(guard)) => Some(guard),
Some(PendingDeleteBucketGenerationGuard::Failed { code, message }) => {
return Err(S3Error::with_message(code.clone(), message.clone()));
}
None => req.bucket_generation(),
};
apply_generation_fence(
req,
guard.map(|guard| (guard.bucket.as_str(), guard.incarnation_id)),
bucket,
opts,
"bucket generation guard",
)
}
pub(crate) fn retained_table_data_plane_publication_guards<R: AuthorizationRequest + ?Sized>(
req: &R,
) -> Option<TableDataPlanePublicationGuards> {
req.publication_guards().cloned()
}
pub(crate) fn apply_copy_source_bucket_generation_guard<R: AuthorizationRequest + ?Sized>(
req: &R,
bucket: &str,
opts: &mut ObjectOptions,
) -> S3Result<()> {
let guard = req.copy_source_bucket_generation();
apply_generation_fence(
req,
guard.map(|guard| (guard.bucket.as_str(), guard.incarnation_id)),
bucket,
opts,
"copy source bucket generation guard",
)
}
/// Binds `opts` to the captured incarnation of `bucket` and to the retained table publication
/// locks. A server request without a captured guard is refused: only direct use-case callers,
/// which carry no server context, run unfenced.
fn apply_generation_fence<R: AuthorizationRequest + ?Sized>(
req: &R,
guard: Option<(&str, uuid::Uuid)>,
bucket: &str,
opts: &mut ObjectOptions,
guard_name: &str,
) -> S3Result<()> {
match guard {
Some((guarded, _)) if guarded != bucket => {
return Err(S3Error::with_message(
S3ErrorCode::InternalError,
format!("{guard_name} does not match request bucket"),
));
}
Some((_, incarnation_id)) => opts.expected_bucket_incarnation_id = Some(incarnation_id),
None if req.server_context().is_some() => {
return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{guard_name} is missing")));
}
None => {}
}
if let Some(publication_guards) = req.publication_guards() {
publication_guards.add_lock_loss_fences(opts);
}
Ok(())
}
+12
View File
@@ -24,9 +24,21 @@
//! policy, IAM and table-catalog systems.
mod admission;
mod guards;
mod request;
pub(crate) use admission::{AccessMarkers, VerifiedSignature, admit_operation, resolve_req_info};
pub(crate) use guards::{
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, OdmReadGenerationGuard,
PendingDeleteBucketGenerationGuard, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard,
bucket_config_mutation_incarnation, capture_odm_read_generation, load_bucket_config_mutation_snapshot,
load_bucket_generation, load_bucket_generation_from_store, odm_read_generation, prepare_odm_read_generation,
retained_table_data_plane_publication_guards,
};
#[cfg(test)]
pub(crate) use guards::{
PUT_GENERATION_READ_QUORUM_FAILURE, install_object_generation_test_hook, wait_for_object_generation_test_hook,
};
#[cfg(test)]
pub(crate) use request::AMZ_WRITE_OFFSET_BYTES_HEADER;
pub(crate) use request::{
+5 -4
View File
@@ -23,11 +23,12 @@
//! Upstream: both edges' access phases. Downstream: every module of `storage::authorization`,
//! `storage::access_owned`, and the use cases that read authorization state.
use crate::storage::ECStore;
use crate::storage::access::{
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, ObjectTagConditions,
OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, TableDataPlaneListAccess, TableDataPlanePublicationGuards,
use super::{
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, OdmReadGenerationGuard,
PendingDeleteBucketGenerationGuard,
};
use crate::storage::ECStore;
use crate::storage::access::{ObjectTagConditions, TableDataPlaneListAccess, TableDataPlanePublicationGuards};
use crate::storage::denial_audit::DenialAuditor;
use crate::storage::req_info::ReqInfo;
use crate::storage::request_context::RequestContext;
+11 -8
View File
@@ -157,13 +157,9 @@ pub(crate) mod access_consumer {
pub(crate) use super::super::access::access_owned::object_commit::CopySourceObject;
pub(crate) use super::super::access::access_owned::object_read::authorize_get_object;
pub(crate) use super::super::access::{
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, ObjectTagConditions,
OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, StagedMultipartPartAuthorization,
TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess, TableDataPlaneListCursorPosition,
TableDataPlanePublicationGuards, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard,
authorize_copy_object_tags, authorize_request, authorize_request_core, bucket_config_mutation_incarnation,
load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, retained_table_data_plane_publication_guards,
ObjectTagConditions, StagedMultipartPartAuthorization, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess,
TableDataPlaneListCursorPosition, TableDataPlanePublicationGuards, authorize_copy_object_tags, authorize_request,
authorize_request_core, log_list_buckets_iam_implicit_deny, prepare_list_buckets_iam_authorization,
};
#[cfg(feature = "gateway-stack")]
pub(crate) use super::super::access::{
@@ -175,6 +171,12 @@ pub(crate) mod access_consumer {
AuthorizationRequest, has_bypass_governance_header, recursive_force_delete_has_authenticated_caller,
replication_request_authorized, req_info_mut, req_info_ref, set_requested_object_lock_retain_until,
};
pub(crate) use super::super::authorization::{
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, OdmReadGenerationGuard,
PendingDeleteBucketGenerationGuard, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard,
bucket_config_mutation_incarnation, load_bucket_generation_from_store, odm_read_generation, prepare_odm_read_generation,
retained_table_data_plane_publication_guards,
};
pub(crate) use super::super::authorization_actions::delete_object_authorize_action;
pub(crate) use super::super::denial_audit::DenialAuditor;
pub(crate) use super::super::helper::{AuditRequestView, DenialScope};
@@ -191,8 +193,9 @@ pub(crate) mod owned_parity_consumer {
pub(crate) use super::super::access::{
ObjectTagConditions, TableDataPlanePublicationGuards, authorize_bucket_config_mutation, authorize_object_request,
authorize_replication_only_put_headers, authorize_request, authorize_staged_multipart_part,
authorize_write_etag_condition, bucket_config_mutation_incarnation, parity_test_support,
authorize_write_etag_condition, parity_test_support,
};
pub(crate) use super::super::authorization::bucket_config_mutation_incarnation;
pub(crate) use super::super::authorization::{AuthorizationRequest, replication_request_authorized};
pub(crate) use super::super::denial_audit::{DenialAuditor, DenialRateLimiter, SUPPRESSED_DENIALS_TAG};
pub(crate) use super::super::ecfs::FS;