diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs index ff7838cda7..244808adc0 100644 --- a/rustfs/src/app/object/get.rs +++ b/rustfs/src/app/object/get.rs @@ -4454,7 +4454,7 @@ impl DefaultObjectUsecase { let expected_incarnation = match odm_read_generation(envelope, bucket) { Ok(Some(incarnation)) => incarnation, Ok(None) => return None, - Err(err) => return Some(OdmGetOutcome::Respond(Err(err))), + Err(err) => return Some(OdmGetOutcome::Respond(Err(err.into()))), }; match store.bucket_incarnation_id(bucket).await { Ok(current) if current == expected_incarnation => {} diff --git a/rustfs/src/app/object/put.rs b/rustfs/src/app/object/put.rs index 0ff56a193c..30c6e920a9 100644 --- a/rustfs/src/app/object/put.rs +++ b/rustfs/src/app/object/put.rs @@ -984,7 +984,7 @@ impl PutObjectOrigin<'_> { fn apply_bucket_generation_guard(&self, bucket: &str, opts: &mut ObjectOptions) -> S3Result<()> { match self { - Self::S3 { req, .. } => apply_bucket_generation_guard(*req, bucket, opts), + Self::S3 { req, .. } => apply_bucket_generation_guard(*req, bucket, opts).map_err(Into::into), Self::Internal { expected_bucket_incarnation_id, .. diff --git a/rustfs/src/app/owned_authorization_test.rs b/rustfs/src/app/owned_authorization_test.rs index cd4cad29d9..7642ae842b 100644 --- a/rustfs/src/app/owned_authorization_test.rs +++ b/rustfs/src/app/owned_authorization_test.rs @@ -1903,19 +1903,21 @@ async fn owned_and_legacy_bucket_config_mutations_hand_on_the_same_decision_and_ }; let legacy_outcome = outcome( legacy_result.map_err(|err| err.code().clone()), - bucket_config_mutation_incarnation(&legacy, bucket).map_err(|err| err.code().clone()), + bucket_config_mutation_incarnation(&legacy, bucket).map_err(|err| err.code().clone().into()), ); let owned_outcome = outcome( owned_result.map_err(|err| err.code().clone()), - bucket_config_mutation_incarnation(&owned, bucket).map_err(|err| err.code().clone()), + bucket_config_mutation_incarnation(&owned, bucket).map_err(|err| err.code().clone().into()), ); // A refused request hands no snapshot on: the use case's reader refuses to run without one. let refused_without_snapshot = |snapshot: Result, S3ErrorCode>| { matches!(expected, MutationOutcome::Allowed(_)) || snapshot == Err(S3ErrorCode::InternalError) }; - let both_refused_without_snapshot = - refused_without_snapshot(bucket_config_mutation_incarnation(&legacy, bucket).map_err(|err| err.code().clone())) - && refused_without_snapshot(bucket_config_mutation_incarnation(&owned, bucket).map_err(|err| err.code().clone())); + let both_refused_without_snapshot = refused_without_snapshot( + bucket_config_mutation_incarnation(&legacy, bucket).map_err(|err| err.code().clone().into()), + ) && refused_without_snapshot( + bucket_config_mutation_incarnation(&owned, bucket).map_err(|err| err.code().clone().into()), + ); let bucket_recorded = [legacy.extensions.get::(), owned.req_info()] .iter() .all(|info| info.and_then(|info| info.bucket.as_deref()) == Some(bucket)); diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index 0a85fe5eeb..25b1332e86 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -18,7 +18,7 @@ use self::access_owned::listing_multipart::{ use self::access_owned::object_misc::{authorize_delete_object, authorize_delete_objects}; use super::ObjectOptions; use super::ecfs::FS; -use super::{ECStore, PolicySys, StorageError, get_lock_acquire_timeout, is_err_bucket_not_found}; +use super::{ECStore, PolicySys, StorageError, is_err_bucket_not_found}; use crate::auth::facts::{ AuthzFacts, AuthzPrincipal, AuthzTarget, base_conditions, build_args, build_conditions, forward_secret_headers, }; @@ -41,6 +41,20 @@ use crate::storage::authorization::{ recursive_force_delete_has_authenticated_caller, req_info_ref, request_iam_store, request_object_store, resolve_req_info, set_requested_object_lock_retain_until, }; +use crate::storage::authorization::{ + BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, OdmReadGenerationGuard, + PendingDeleteBucketGenerationGuard, capture_odm_read_generation, load_bucket_config_mutation_snapshot, + load_bucket_generation, +}; +#[cfg(test)] +use crate::storage::authorization::{ + PUT_GENERATION_READ_QUORUM_FAILURE, install_object_generation_test_hook, wait_for_object_generation_test_hook, +}; +#[cfg(test)] +use crate::storage::authorization::{ + apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, bucket_config_mutation_incarnation, + odm_read_generation, retained_table_data_plane_publication_guards, +}; use crate::storage::denial_audit::DenialAuditor; use crate::storage::helper::DenialScope; pub(crate) use crate::storage::req_info::ReqInfo; @@ -50,8 +64,7 @@ pub(crate) use crate::storage::request_markers::{ MultipartStagingOperation, PostObjectPolicyHeaders, PostObjectRequestMarker, RequestedCopySource, RequestedObjectLegalHold, RequestedObjectLockRetainUntil, RequestedObjectLockRetention, RequestedObjectTags, StagedMultipartPartAuthorization, }; -use crate::storage::storage_api::contract::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, BucketOperations, BucketOptions}; -use crate::storage::storage_api::contract::namespace::NamespaceLocking as _; +use crate::storage::storage_api::contract::bucket::{BucketOperations, BucketOptions}; use crate::storage::storage_api::ecstore_bucket::lockless_cache::is_lockless_cache_bucket; use crate::storage::storage_api::runtime_sources_consumer::ServerContextSlot; use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; @@ -84,8 +97,6 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::collections::{BTreeSet, HashMap}; use std::sync::Arc; -#[cfg(test)] -use std::sync::OnceLock; use time::OffsetDateTime; use url::Url; @@ -388,67 +399,6 @@ fn invalid_table_data_plane_list_cursor() -> S3Error { S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid table listing continuation token".to_string()) } -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct BucketGenerationGuard { - bucket: String, - incarnation_id: uuid::Uuid, -} - -#[derive(Clone, Debug)] -pub(crate) struct BucketConfigMutationSnapshot { - bucket: String, - incarnation_id: uuid::Uuid, -} - -pub(crate) fn bucket_config_mutation_incarnation( - req: &R, - bucket: &str, -) -> S3Result> { - let Some(snapshot) = req.bucket_config_mutation() else { - if req.server_context().is_some() { - return Err(s3_error!(InternalError, "bucket config mutation snapshot is missing")); - } - return Ok(None); - }; - if snapshot.bucket != bucket { - return Err(s3_error!(InternalError, "bucket config mutation snapshot does not match request bucket")); - } - Ok(Some(snapshot.incarnation_id)) -} - -async fn load_bucket_config_mutation_snapshot( - server_ctx: &ServerContextSlot, - req: &R, - bucket: &str, -) -> S3Result { - let store = server_ctx - .object_store() - .ok_or_else(|| s3_error!(InternalError, "object store is not initialized"))?; - let lock = store - .new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT) - .await - .map_err(ApiError::from)?; - let _lifecycle_guard = lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(|err| { - ApiError::from(match err { - rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable { - mode: "bucket_config_mutation", - bucket: bucket.to_string(), - object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(), - required, - achieved, - }, - other => StorageError::Lock(other), - }) - })?; - let incarnation_id = load_bucket_generation_from_store(store.as_ref(), req, bucket) - .await? - .incarnation_id; - Ok(BucketConfigMutationSnapshot { - bucket: bucket.to_string(), - incarnation_id, - }) -} - /// Authorizes a bucket-configuration mutation for either edge: `action` on `bucket`, with the /// bucket incarnation snapshotted under the lifecycle namespace read lock BEFORE the decision, so /// a bucket recreated during authorization is refused by the use case's conditional write. A @@ -489,174 +439,6 @@ pub(crate) async fn authorize_bucket_tag_deletion, tokio::sync::oneshot::Receiver<()>); - -#[cfg(test)] -static OBJECT_GENERATION_TEST_HOOK: OnceLock>> = OnceLock::new(); - -#[cfg(test)] -tokio::task_local! { - static PUT_GENERATION_READ_QUORUM_FAILURE: (); -} - -#[cfg(test)] -fn install_object_generation_test_hook( - bucket: String, - loaded: tokio::sync::oneshot::Sender<()>, - resume: tokio::sync::oneshot::Receiver<()>, -) { - *OBJECT_GENERATION_TEST_HOOK - .get_or_init(|| std::sync::Mutex::new(None)) - .lock() - .expect("object generation test hook lock should not be poisoned") = Some((bucket, loaded, resume)); -} - -#[cfg(test)] -async fn wait_for_object_generation_test_hook(bucket: &str) { - let hook = { - let mut slot = OBJECT_GENERATION_TEST_HOOK - .get_or_init(|| std::sync::Mutex::new(None)) - .lock() - .expect("object generation test hook lock should not be poisoned"); - if slot.as_ref().is_some_and(|(expected_bucket, _, _)| expected_bucket == bucket) { - slot.take() - } else { - None - } - }; - if let Some((_bucket, loaded, resume)) = hook { - let _ = loaded.send(()); - let _ = resume.await; - } -} - -#[derive(Clone, Debug)] -pub(crate) enum PendingDeleteBucketGenerationGuard { - Ready(BucketGenerationGuard), - Failed { code: S3ErrorCode, message: String }, -} - -impl PendingDeleteBucketGenerationGuard { - fn from_result(result: S3Result) -> Self { - match result { - Ok(guard) => Self::Ready(guard), - Err(err) => Self::Failed { - code: err.code().clone(), - message: err.message().unwrap_or_else(|| err.code().as_str()).to_string(), - }, - } - } -} - -pub(crate) async fn load_bucket_generation_from_store( - store: &crate::storage::storage_api::ECStore, - req: &R, - bucket: &str, -) -> S3Result { - if let Some(existing) = req.bucket_generation() { - return if existing.bucket == bucket { - Ok(existing.clone()) - } else { - Err(s3_error!(InternalError, "bucket generation guard does not match request bucket")) - }; - } - - let incarnation_id = store.bucket_incarnation_id(bucket).await.map_err(ApiError::from)?; - Ok(BucketGenerationGuard { - bucket: bucket.to_string(), - incarnation_id, - }) -} - -/// Loads the bucket incarnation through `server_ctx`'s object store, or returns the guard the -/// request already carries. The legacy hooks pass `FS::server_ctx`; the gateway edge passes the -/// envelope's server context. -pub(crate) async fn load_bucket_generation( - server_ctx: &ServerContextSlot, - req: &R, - bucket: &str, -) -> S3Result { - let store = server_ctx - .object_store() - .ok_or_else(|| s3_error!(InternalError, "object store is not initialized"))?; - load_bucket_generation_from_store(store.as_ref(), req, bucket).await -} - -/// A read may consult only the source admitted before its authorization. -/// Capture failures are deferred until a local miss actually needs that source. -#[derive(Clone, Debug)] -pub(crate) enum OdmReadGenerationGuard { - Unavailable, - Ready(BucketGenerationGuard), - Failed { code: S3ErrorCode, message: String }, -} - -impl OdmReadGenerationGuard { - fn from_result(result: S3Result) -> Self { - match result { - Ok(guard) => Self::Ready(guard), - Err(err) => Self::Failed { - code: err.code().clone(), - message: err.message().unwrap_or_else(|| err.code().as_str()).to_string(), - }, - } - } -} - -fn odm_read_source_configured(bucket: &str) -> bool { - let sys = crate::on_demand_migration::OnDemandMigrationSys::get(); - sys.is_module_enabled() && sys.state(bucket).is_some() -} - -/// The on-demand-migration source generation a read may consult, captured before its -/// authorization. A capture failure is kept in the guard and surfaces only on a source miss. -pub(crate) async fn capture_odm_read_generation( - server_ctx: &ServerContextSlot, - req: &R, - bucket: &str, -) -> OdmReadGenerationGuard { - if !odm_read_source_configured(bucket) { - return OdmReadGenerationGuard::Unavailable; - } - OdmReadGenerationGuard::from_result(load_bucket_generation(server_ctx, req, bucket).await) -} - -/// Direct usecase callers have no access middleware; capture at their entry. -/// A server request without the access marker must never bind to a later source. -pub(crate) async fn prepare_odm_read_generation( - store: &crate::storage::storage_api::ECStore, - req: &mut R, - bucket: &str, -) { - if req.odm_read_generation().is_some() { - return; - } - let guard = if req.server_context().is_some() || !odm_read_source_configured(bucket) { - OdmReadGenerationGuard::Unavailable - } else { - OdmReadGenerationGuard::from_result(load_bucket_generation_from_store(store, req, bucket).await) - }; - req.install_odm_read_generation(guard); -} - -pub(crate) fn odm_read_generation(req: &R, bucket: &str) -> S3Result> { - match req.odm_read_generation() { - Some(OdmReadGenerationGuard::Ready(guard)) if guard.bucket == bucket => Ok(Some(guard.incarnation_id)), - Some(OdmReadGenerationGuard::Ready(_)) => { - Err(s3_error!(InternalError, "source generation guard does not match request bucket")) - } - Some(OdmReadGenerationGuard::Failed { code, message }) => Err(S3Error::with_message(code.clone(), message.clone())), - Some(OdmReadGenerationGuard::Unavailable) | None => Ok(None), - } -} - /// The legacy decoder's copy source as owned parts; `None` for an access-point or Outposts ARN. pub(crate) fn legacy_copy_source(source: &CopySource) -> Option<(String, String, Option)> { match source { @@ -675,77 +457,6 @@ fn copy_source_object<'a>( access_owned::object_commit::CopySourceObject { bucket, key, version_id } } -pub(crate) fn apply_bucket_generation_guard( - req: &R, - bucket: &str, - opts: &mut ObjectOptions, -) -> S3Result<()> { - let guard = match req.pending_delete_bucket_generation() { - Some(PendingDeleteBucketGenerationGuard::Ready(guard)) => Some(guard), - Some(PendingDeleteBucketGenerationGuard::Failed { code, message }) => { - return Err(S3Error::with_message(code.clone(), message.clone())); - } - None => req.bucket_generation(), - }; - apply_generation_fence( - req, - guard.map(|guard| (guard.bucket.as_str(), guard.incarnation_id)), - bucket, - opts, - "bucket generation guard", - ) -} - -pub(crate) fn retained_table_data_plane_publication_guards( - req: &R, -) -> Option { - req.publication_guards().cloned() -} - -pub(crate) fn apply_copy_source_bucket_generation_guard( - req: &R, - bucket: &str, - opts: &mut ObjectOptions, -) -> S3Result<()> { - let guard = req.copy_source_bucket_generation(); - apply_generation_fence( - req, - guard.map(|guard| (guard.bucket.as_str(), guard.incarnation_id)), - bucket, - opts, - "copy source bucket generation guard", - ) -} - -/// Binds `opts` to the captured incarnation of `bucket` and to the retained table publication -/// locks. A server request without a captured guard is refused: only direct use-case callers, -/// which carry no server context, run unfenced. -fn apply_generation_fence( - req: &R, - guard: Option<(&str, uuid::Uuid)>, - bucket: &str, - opts: &mut ObjectOptions, - guard_name: &str, -) -> S3Result<()> { - match guard { - Some((guarded, _)) if guarded != bucket => { - return Err(S3Error::with_message( - S3ErrorCode::InternalError, - format!("{guard_name} does not match request bucket"), - )); - } - Some((_, incarnation_id)) => opts.expected_bucket_incarnation_id = Some(incarnation_id), - None if req.server_context().is_some() => { - return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{guard_name} is missing"))); - } - None => {} - } - if let Some(publication_guards) = req.publication_guards() { - publication_guards.add_lock_loss_fences(opts); - } - Ok(()) -} - #[derive(Clone, Debug)] pub(crate) struct ObjectTagConditions { bucket: String, @@ -8050,6 +7761,7 @@ mod tests { let mut opts = crate::storage::ObjectOptions::default(); let err = apply_bucket_generation_guard(&req, "generation-guard-bucket", &mut opts) + .map_err(s3s::S3Error::from) .expect_err("a dispatched PutObject must not commit without its authorization fence"); assert_eq!(err.code(), &S3ErrorCode::InternalError); } diff --git a/rustfs/src/storage/access_owned/object_misc.rs b/rustfs/src/storage/access_owned/object_misc.rs index 9a9a4dc1de..54c642f2a8 100644 --- a/rustfs/src/storage/access_owned/object_misc.rs +++ b/rustfs/src/storage/access_owned/object_misc.rs @@ -80,7 +80,7 @@ pub(crate) async fn authorize_delete_object( let bucket_generation = load_bucket_generation(server_ctx, req, bucket).await; #[cfg(test)] wait_for_object_generation_test_hook(bucket).await; - let guard = decide_delete_object(req, bucket, key, version_id, bucket_generation).await?; + let guard = decide_delete_object(req, bucket, key, version_id, bucket_generation.map_err(Into::into)).await?; req.install_pending_delete_bucket_generation(guard); Ok(()) } @@ -150,7 +150,7 @@ async fn decide_delete_object( authorize_request(req, Action::S3Action(S3Action::BypassGovernanceRetentionAction)).await?; } - Ok(PendingDeleteBucketGenerationGuard::from_result(bucket_generation)) + Ok(PendingDeleteBucketGenerationGuard::from_result(bucket_generation.map_err(Into::into))) } /// Select reads the current object and needs the same GetObject permission on either edge. diff --git a/rustfs/src/storage/access_readers_test.rs b/rustfs/src/storage/access_readers_test.rs index 2f37085e28..578cd75e01 100644 --- a/rustfs/src/storage/access_readers_test.rs +++ b/rustfs/src/storage/access_readers_test.rs @@ -23,7 +23,7 @@ use super::{ Action, AuthorizationRequest, BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, HashMap, HeaderMap, ObjectOptions, OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, RemoteAddr, ReqInfo, RequestContext, - S3Action, S3ErrorCode, S3Request, S3Result, TableDataPlanePublicationGuards, apply_bucket_generation_guard, + S3Action, S3ErrorCode, S3Request, TableDataPlanePublicationGuards, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorize_object_request, bucket_config_mutation_incarnation, capture_odm_read_generation, odm_read_generation, replication_request_authorized, request_context_from_req, resource_free_condition_values, retained_table_data_plane_publication_guards, @@ -69,10 +69,10 @@ fn server_request(extensions: &mut http::Extensions) { /// The incarnation a reader bound, or the refusal's code and message. type Answer = Result, (S3ErrorCode, String)>; -fn answer(result: S3Result, bound: impl FnOnce(T) -> Option) -> Answer { +fn answer(result: rustfs_s3_types::S3Result, bound: impl FnOnce(T) -> Option) -> Answer { result .map(bound) - .map_err(|err| (err.code().clone(), err.message().unwrap_or_default().to_owned())) + .map_err(|err| (err.code().clone().into(), err.message().unwrap_or_default().to_owned())) } fn refused(code: S3ErrorCode, message: &str) -> Answer { @@ -147,7 +147,7 @@ fn bucket_generation_guard_reads_the_same_through_both_forms() { legacy(|ext| { ext.insert(guard(BUCKET)); ext.insert(PendingDeleteBucketGenerationGuard::Failed { - code: S3ErrorCode::NoSuchBucket, + code: S3ErrorCode::NoSuchBucket.into(), message: "bucket is gone".to_owned(), }); }), @@ -227,7 +227,7 @@ fn odm_read_generation_reads_the_same_through_both_forms() { "failed capture surfaces its error", legacy(|ext| { ext.insert(OdmReadGenerationGuard::Failed { - code: S3ErrorCode::SlowDown, + code: S3ErrorCode::SlowDown.into(), message: "capture failed".to_owned(), }); }), @@ -292,7 +292,7 @@ fn installed_state_reads_back_through_the_envelope() { envelope.install_bucket_generation(guard(BUCKET)); assert_eq!(bucket_fence(&envelope), Ok(Some(incarnation()))); envelope.install_pending_delete_bucket_generation(PendingDeleteBucketGenerationGuard::Failed { - code: S3ErrorCode::NoSuchBucket, + code: S3ErrorCode::NoSuchBucket.into(), message: "bucket is gone".to_owned(), }); assert_eq!(bucket_fence(&envelope), refused(S3ErrorCode::NoSuchBucket, "bucket is gone")); diff --git a/rustfs/src/storage/authorization/guards.rs b/rustfs/src/storage/authorization/guards.rs new file mode 100644 index 0000000000..fc5f4c7f7d --- /dev/null +++ b/rustfs/src/storage/authorization/guards.rs @@ -0,0 +1,335 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The bucket-generation fences an authorized request carries to its use case. +//! +//! Responsible for: the guards and snapshots the access phase captures before a decision +//! (bucket incarnation, pending-delete, copy-source, on-demand-migration source, bucket-config +//! mutation), loading them from the object store, and applying them to `ObjectOptions` so a +//! bucket recreated after authorization is refused by the conditional write. +//! Not responsible for: the policy decision itself (`decision`) or the table publication locks +//! the fence also binds (`table_data_plane`). +//! Upstream: the access hooks of both edges and `storage::access_owned`. Downstream: the object +//! store's bucket metadata and the use cases' `ObjectOptions`. + +use super::AuthorizationRequest; +use crate::error::ApiError; +use crate::storage::access::TableDataPlanePublicationGuards; +use crate::storage::storage_api::contract::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT; +use crate::storage::storage_api::contract::namespace::NamespaceLocking as _; +use crate::storage::storage_api::runtime_sources_consumer::ServerContextSlot; +use crate::storage::{ObjectOptions, StorageError, get_lock_acquire_timeout}; +use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error}; +#[cfg(test)] +use std::sync::OnceLock; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct BucketGenerationGuard { + pub(crate) bucket: String, + pub(crate) incarnation_id: uuid::Uuid, +} + +#[derive(Clone, Debug)] +pub(crate) struct BucketConfigMutationSnapshot { + pub(crate) bucket: String, + pub(crate) incarnation_id: uuid::Uuid, +} + +pub(crate) fn bucket_config_mutation_incarnation( + req: &R, + bucket: &str, +) -> S3Result> { + let Some(snapshot) = req.bucket_config_mutation() else { + if req.server_context().is_some() { + return Err(s3_error!(InternalError, "bucket config mutation snapshot is missing")); + } + return Ok(None); + }; + if snapshot.bucket != bucket { + return Err(s3_error!(InternalError, "bucket config mutation snapshot does not match request bucket")); + } + Ok(Some(snapshot.incarnation_id)) +} + +pub(crate) async fn load_bucket_config_mutation_snapshot( + server_ctx: &ServerContextSlot, + req: &R, + bucket: &str, +) -> S3Result { + let store = server_ctx + .object_store() + .ok_or_else(|| s3_error!(InternalError, "object store is not initialized"))?; + let lock = store + .new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT) + .await + .map_err(ApiError::from)?; + let _lifecycle_guard = lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(|err| { + ApiError::from(match err { + rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable { + mode: "bucket_config_mutation", + bucket: bucket.to_string(), + object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(), + required, + achieved, + }, + other => StorageError::Lock(other), + }) + })?; + let incarnation_id = load_bucket_generation_from_store(store.as_ref(), req, bucket) + .await? + .incarnation_id; + Ok(BucketConfigMutationSnapshot { + bucket: bucket.to_string(), + incarnation_id, + }) +} + +#[derive(Clone, Debug)] +pub(crate) struct CopySourceBucketGenerationGuard { + pub(crate) bucket: String, + pub(crate) incarnation_id: uuid::Uuid, +} + +#[cfg(test)] +type ObjectGenerationTestHook = (String, tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>); + +#[cfg(test)] +static OBJECT_GENERATION_TEST_HOOK: OnceLock>> = OnceLock::new(); + +#[cfg(test)] +tokio::task_local! { + pub(crate) static PUT_GENERATION_READ_QUORUM_FAILURE: (); +} + +#[cfg(test)] +pub(crate) fn install_object_generation_test_hook( + bucket: String, + loaded: tokio::sync::oneshot::Sender<()>, + resume: tokio::sync::oneshot::Receiver<()>, +) { + *OBJECT_GENERATION_TEST_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("object generation test hook lock should not be poisoned") = Some((bucket, loaded, resume)); +} + +#[cfg(test)] +pub(crate) async fn wait_for_object_generation_test_hook(bucket: &str) { + let hook = { + let mut slot = OBJECT_GENERATION_TEST_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("object generation test hook lock should not be poisoned"); + if slot.as_ref().is_some_and(|(expected_bucket, _, _)| expected_bucket == bucket) { + slot.take() + } else { + None + } + }; + if let Some((_bucket, loaded, resume)) = hook { + let _ = loaded.send(()); + let _ = resume.await; + } +} + +#[derive(Clone, Debug)] +pub(crate) enum PendingDeleteBucketGenerationGuard { + Ready(BucketGenerationGuard), + Failed { code: S3ErrorCode, message: String }, +} + +impl PendingDeleteBucketGenerationGuard { + pub(crate) fn from_result(result: S3Result) -> Self { + match result { + Ok(guard) => Self::Ready(guard), + Err(err) => Self::Failed { + code: err.code().clone(), + message: err.message().unwrap_or_else(|| err.code().as_str()).to_string(), + }, + } + } +} + +pub(crate) async fn load_bucket_generation_from_store( + store: &crate::storage::storage_api::ECStore, + req: &R, + bucket: &str, +) -> S3Result { + if let Some(existing) = req.bucket_generation() { + return if existing.bucket == bucket { + Ok(existing.clone()) + } else { + Err(s3_error!(InternalError, "bucket generation guard does not match request bucket")) + }; + } + + let incarnation_id = store.bucket_incarnation_id(bucket).await.map_err(ApiError::from)?; + Ok(BucketGenerationGuard { + bucket: bucket.to_string(), + incarnation_id, + }) +} + +/// Loads the bucket incarnation through `server_ctx`'s object store, or returns the guard the +/// request already carries. The legacy hooks pass `FS::server_ctx`; the gateway edge passes the +/// envelope's server context. +pub(crate) async fn load_bucket_generation( + server_ctx: &ServerContextSlot, + req: &R, + bucket: &str, +) -> S3Result { + let store = server_ctx + .object_store() + .ok_or_else(|| s3_error!(InternalError, "object store is not initialized"))?; + load_bucket_generation_from_store(store.as_ref(), req, bucket).await +} + +/// A read may consult only the source admitted before its authorization. +/// Capture failures are deferred until a local miss actually needs that source. +#[derive(Clone, Debug)] +pub(crate) enum OdmReadGenerationGuard { + Unavailable, + Ready(BucketGenerationGuard), + Failed { code: S3ErrorCode, message: String }, +} + +impl OdmReadGenerationGuard { + fn from_result(result: S3Result) -> Self { + match result { + Ok(guard) => Self::Ready(guard), + Err(err) => Self::Failed { + code: err.code().clone(), + message: err.message().unwrap_or_else(|| err.code().as_str()).to_string(), + }, + } + } +} + +fn odm_read_source_configured(bucket: &str) -> bool { + let sys = crate::on_demand_migration::OnDemandMigrationSys::get(); + sys.is_module_enabled() && sys.state(bucket).is_some() +} + +/// The on-demand-migration source generation a read may consult, captured before its +/// authorization. A capture failure is kept in the guard and surfaces only on a source miss. +pub(crate) async fn capture_odm_read_generation( + server_ctx: &ServerContextSlot, + req: &R, + bucket: &str, +) -> OdmReadGenerationGuard { + if !odm_read_source_configured(bucket) { + return OdmReadGenerationGuard::Unavailable; + } + OdmReadGenerationGuard::from_result(load_bucket_generation(server_ctx, req, bucket).await) +} + +/// Direct usecase callers have no access middleware; capture at their entry. +/// A server request without the access marker must never bind to a later source. +pub(crate) async fn prepare_odm_read_generation( + store: &crate::storage::storage_api::ECStore, + req: &mut R, + bucket: &str, +) { + if req.odm_read_generation().is_some() { + return; + } + let guard = if req.server_context().is_some() || !odm_read_source_configured(bucket) { + OdmReadGenerationGuard::Unavailable + } else { + OdmReadGenerationGuard::from_result(load_bucket_generation_from_store(store, req, bucket).await) + }; + req.install_odm_read_generation(guard); +} + +pub(crate) fn odm_read_generation(req: &R, bucket: &str) -> S3Result> { + match req.odm_read_generation() { + Some(OdmReadGenerationGuard::Ready(guard)) if guard.bucket == bucket => Ok(Some(guard.incarnation_id)), + Some(OdmReadGenerationGuard::Ready(_)) => { + Err(s3_error!(InternalError, "source generation guard does not match request bucket")) + } + Some(OdmReadGenerationGuard::Failed { code, message }) => Err(S3Error::with_message(code.clone(), message.clone())), + Some(OdmReadGenerationGuard::Unavailable) | None => Ok(None), + } +} + +pub(crate) fn apply_bucket_generation_guard( + req: &R, + bucket: &str, + opts: &mut ObjectOptions, +) -> S3Result<()> { + let guard = match req.pending_delete_bucket_generation() { + Some(PendingDeleteBucketGenerationGuard::Ready(guard)) => Some(guard), + Some(PendingDeleteBucketGenerationGuard::Failed { code, message }) => { + return Err(S3Error::with_message(code.clone(), message.clone())); + } + None => req.bucket_generation(), + }; + apply_generation_fence( + req, + guard.map(|guard| (guard.bucket.as_str(), guard.incarnation_id)), + bucket, + opts, + "bucket generation guard", + ) +} + +pub(crate) fn retained_table_data_plane_publication_guards( + req: &R, +) -> Option { + req.publication_guards().cloned() +} + +pub(crate) fn apply_copy_source_bucket_generation_guard( + req: &R, + bucket: &str, + opts: &mut ObjectOptions, +) -> S3Result<()> { + let guard = req.copy_source_bucket_generation(); + apply_generation_fence( + req, + guard.map(|guard| (guard.bucket.as_str(), guard.incarnation_id)), + bucket, + opts, + "copy source bucket generation guard", + ) +} + +/// Binds `opts` to the captured incarnation of `bucket` and to the retained table publication +/// locks. A server request without a captured guard is refused: only direct use-case callers, +/// which carry no server context, run unfenced. +fn apply_generation_fence( + req: &R, + guard: Option<(&str, uuid::Uuid)>, + bucket: &str, + opts: &mut ObjectOptions, + guard_name: &str, +) -> S3Result<()> { + match guard { + Some((guarded, _)) if guarded != bucket => { + return Err(S3Error::with_message( + S3ErrorCode::InternalError, + format!("{guard_name} does not match request bucket"), + )); + } + Some((_, incarnation_id)) => opts.expected_bucket_incarnation_id = Some(incarnation_id), + None if req.server_context().is_some() => { + return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{guard_name} is missing"))); + } + None => {} + } + if let Some(publication_guards) = req.publication_guards() { + publication_guards.add_lock_loss_fences(opts); + } + Ok(()) +} diff --git a/rustfs/src/storage/authorization/mod.rs b/rustfs/src/storage/authorization/mod.rs index 14346c1c72..0f31d69e2d 100644 --- a/rustfs/src/storage/authorization/mod.rs +++ b/rustfs/src/storage/authorization/mod.rs @@ -24,9 +24,21 @@ //! policy, IAM and table-catalog systems. mod admission; +mod guards; mod request; pub(crate) use admission::{AccessMarkers, VerifiedSignature, admit_operation, resolve_req_info}; +pub(crate) use guards::{ + BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, OdmReadGenerationGuard, + PendingDeleteBucketGenerationGuard, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, + bucket_config_mutation_incarnation, capture_odm_read_generation, load_bucket_config_mutation_snapshot, + load_bucket_generation, load_bucket_generation_from_store, odm_read_generation, prepare_odm_read_generation, + retained_table_data_plane_publication_guards, +}; +#[cfg(test)] +pub(crate) use guards::{ + PUT_GENERATION_READ_QUORUM_FAILURE, install_object_generation_test_hook, wait_for_object_generation_test_hook, +}; #[cfg(test)] pub(crate) use request::AMZ_WRITE_OFFSET_BYTES_HEADER; pub(crate) use request::{ diff --git a/rustfs/src/storage/authorization/request.rs b/rustfs/src/storage/authorization/request.rs index e004c5ece0..75970e3d0b 100644 --- a/rustfs/src/storage/authorization/request.rs +++ b/rustfs/src/storage/authorization/request.rs @@ -23,11 +23,12 @@ //! Upstream: both edges' access phases. Downstream: every module of `storage::authorization`, //! `storage::access_owned`, and the use cases that read authorization state. -use crate::storage::ECStore; -use crate::storage::access::{ - BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, ObjectTagConditions, - OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, TableDataPlaneListAccess, TableDataPlanePublicationGuards, +use super::{ + BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, OdmReadGenerationGuard, + PendingDeleteBucketGenerationGuard, }; +use crate::storage::ECStore; +use crate::storage::access::{ObjectTagConditions, TableDataPlaneListAccess, TableDataPlanePublicationGuards}; use crate::storage::denial_audit::DenialAuditor; use crate::storage::req_info::ReqInfo; use crate::storage::request_context::RequestContext; diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index e593adedbe..0d9d38fc12 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -157,13 +157,9 @@ pub(crate) mod access_consumer { pub(crate) use super::super::access::access_owned::object_commit::CopySourceObject; pub(crate) use super::super::access::access_owned::object_read::authorize_get_object; pub(crate) use super::super::access::{ - BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, ObjectTagConditions, - OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, StagedMultipartPartAuthorization, - TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess, TableDataPlaneListCursorPosition, - TableDataPlanePublicationGuards, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, - authorize_copy_object_tags, authorize_request, authorize_request_core, bucket_config_mutation_incarnation, - load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation, - prepare_list_buckets_iam_authorization, prepare_odm_read_generation, retained_table_data_plane_publication_guards, + ObjectTagConditions, StagedMultipartPartAuthorization, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess, + TableDataPlaneListCursorPosition, TableDataPlanePublicationGuards, authorize_copy_object_tags, authorize_request, + authorize_request_core, log_list_buckets_iam_implicit_deny, prepare_list_buckets_iam_authorization, }; #[cfg(feature = "gateway-stack")] pub(crate) use super::super::access::{ @@ -175,6 +171,12 @@ pub(crate) mod access_consumer { AuthorizationRequest, has_bypass_governance_header, recursive_force_delete_has_authenticated_caller, replication_request_authorized, req_info_mut, req_info_ref, set_requested_object_lock_retain_until, }; + pub(crate) use super::super::authorization::{ + BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, OdmReadGenerationGuard, + PendingDeleteBucketGenerationGuard, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, + bucket_config_mutation_incarnation, load_bucket_generation_from_store, odm_read_generation, prepare_odm_read_generation, + retained_table_data_plane_publication_guards, + }; pub(crate) use super::super::authorization_actions::delete_object_authorize_action; pub(crate) use super::super::denial_audit::DenialAuditor; pub(crate) use super::super::helper::{AuditRequestView, DenialScope}; @@ -191,8 +193,9 @@ pub(crate) mod owned_parity_consumer { pub(crate) use super::super::access::{ ObjectTagConditions, TableDataPlanePublicationGuards, authorize_bucket_config_mutation, authorize_object_request, authorize_replication_only_put_headers, authorize_request, authorize_staged_multipart_part, - authorize_write_etag_condition, bucket_config_mutation_incarnation, parity_test_support, + authorize_write_etag_condition, parity_test_support, }; + pub(crate) use super::super::authorization::bucket_config_mutation_incarnation; pub(crate) use super::super::authorization::{AuthorizationRequest, replication_request_authorized}; pub(crate) use super::super::denial_audit::{DenialAuditor, DenialRateLimiter, SUPPRESSED_DENIALS_TAG}; pub(crate) use super::super::ecfs::FS;