diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index bab6931e8..b5f8d7064 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -19,6 +19,7 @@ pub mod heal_channel; pub mod last_minute; pub mod metrics; mod readiness; +pub mod table_catalog; pub use globals::*; pub use readiness::{GlobalReadiness, SystemStage}; diff --git a/crates/common/src/table_catalog.rs b/crates/common/src/table_catalog.rs new file mode 100644 index 000000000..ecc694e87 --- /dev/null +++ b/crates/common/src/table_catalog.rs @@ -0,0 +1,17 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Cross-crate lock identity used to fence table-bucket publication against +/// object mutations that bypass the S3 request authorization layer. +pub const TABLE_BUCKET_PUBLICATION_LOCK_PATH: &str = ".rustfs-table/warehouses/default/publication.lock"; diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 7b04c0710..10f176965 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -3318,6 +3318,9 @@ pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) { return; } }; + if configs.table_bucket_enabled { + return; + } let Some(lifecycle) = configs.lifecycle else { return; }; @@ -3978,6 +3981,9 @@ async fn enqueue_expiry_for_existing_object_group( pub async fn enqueue_expiry_for_existing_objects(api: Arc, bucket: &str) -> Result<(), Error> { let configs = metadata_boundary::get_expiry_configs(&api, bucket).await?; + if configs.table_bucket_enabled { + return Ok(()); + } let Some(lc) = configs.lifecycle else { return Ok(()); }; @@ -4194,12 +4200,16 @@ pub async fn expire_transitioned_object( _src: &LcEventSrc, bucket_incarnation_id: Uuid, ) -> Result { + let publication_guard = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id) + .await + .ok_or_else(|| std::io::Error::other("lifecycle expiry is not allowed for this bucket"))?; let snapshot = lifecycle_delete_config_snapshot(&api, oi) .await .map_err(std::io::Error::other)?; let (versioned, version_suspended) = snapshot.versioning_config().delete_state(&oi.name); let mut opts = transitioned_object_delete_opts(oi, lc_event.action, versioned, version_suspended, bucket_incarnation_id) .map_err(std::io::Error::other)?; + opts.add_namespace_lock_guard(&publication_guard); opts.delete_replication_config_snapshot = Some(Arc::new(snapshot)); //let tags = LcAuditEvent::new(src, lcEvent).Tags(); if lc_event.action.delete_restored() { @@ -4789,6 +4799,43 @@ pub async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, o .await } +async fn lifecycle_expiry_publication_guard( + api: &ECStore, + oi: &ObjectInfo, + bucket_incarnation_id: Uuid, +) -> Option { + let result = async { + let lock = api + .new_ns_lock(&oi.bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await?; + let guard = lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(Error::other)?; + if guard.is_lock_lost() { + return Err(Error::other("table-bucket publication lock was lost before lifecycle delete admission")); + } + if !metadata_boundary::lifecycle_expiry_allowed(api, &oi.bucket, bucket_incarnation_id).await? { + return Ok(None); + } + Ok(Some(guard)) + } + .await; + match result { + Ok(guard) => guard, + Err(err) => { + warn!( + event = EVENT_LIFECYCLE_DELETE_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + bucket = %oi.bucket, + object = %oi.name, + operation = "authorize_lifecycle_expiry", + error = %err, + "Lifecycle delete admission failed" + ); + None + } + } +} + pub async fn apply_expiry_on_transitioned_object( api: Arc, oi: &ObjectInfo, @@ -4812,6 +4859,9 @@ pub async fn apply_expiry_on_non_transitioned_objects( _src: &LcEventSrc, bucket_incarnation_id: Uuid, ) -> bool { + let Some(publication_guard) = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id).await else { + return false; + }; let snapshot = match lifecycle_delete_config_snapshot(&api, oi).await { Ok(snapshot) => snapshot, Err(err) => { @@ -4837,6 +4887,7 @@ pub async fn apply_expiry_on_non_transitioned_objects( expected_bucket_incarnation_id: Some(bucket_incarnation_id), ..Default::default() }; + opts.add_namespace_lock_guard(&publication_guard); if lc_event.action.delete_versioned() { opts.version_id = oi.version_id.map(|v| v.to_string()); @@ -5033,7 +5084,7 @@ mod tests { cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at, enqueue_recovered_free_version_with_state, enqueue_transition_for_existing_objects_scoped, enqueue_transition_with_lifecycle, enqueue_transition_with_lifecycle_report, eval_action_from_lifecycle, - jitter_tier_free_version_recovery_delay, lifecycle_action_blocked_by_replication, + get_lock_acquire_timeout, jitter_tier_free_version_recovery_delay, lifecycle_action_blocked_by_replication, lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object, lifecycle_replication_blocks_action, lifecycle_rule_has_date_expiration, manual_transition_duration_elapsed, manual_transition_has_more_after_limit, manual_transition_recovery_progress_sink, manual_transition_version_marker, manual_transition_worker_failure_reason, @@ -5090,7 +5141,6 @@ mod tests { #[cfg(feature = "test-util")] use crate::services::tier::warm_backend::WarmBackend as _; use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY}; - #[cfg(feature = "test-util")] use crate::storage_api_contracts::namespace::NamespaceLocking as _; use crate::storage_api_contracts::{ bucket::{BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions}, @@ -10319,6 +10369,85 @@ mod tests { ); } + #[tokio::test] + #[serial] + async fn queued_lifecycle_expiry_does_not_delete_from_table_bucket() { + let (_disk_paths, ecstore) = setup_test_env().await; + let bucket = format!("table-bucket-lifecycle-{}", Uuid::new_v4().simple()); + let object = "tables/table-id/data/part-00001.parquet"; + create_test_bucket(&ecstore, &bucket).await; + + let mut reader = PutObjReader::from_vec(b"referenced table data".to_vec()); + let object_info = ecstore + .put_object(&bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("table data object should be created"); + + let publication_lock = ecstore + .new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await + .expect("table-bucket publication lock should be created"); + let enable_guard = publication_lock + .get_write_lock(get_lock_acquire_timeout()) + .await + .expect("table-bucket enablement should acquire the publication lock"); + let expiry_store = ecstore.clone(); + let expiry_object = object_info.clone(); + let (expiry_started_tx, expiry_started_rx) = tokio::sync::oneshot::channel(); + let mut expiry = tokio::spawn(async move { + let event = crate::bucket::lifecycle::lifecycle::Event { + action: IlmAction::DeleteAction, + ..Default::default() + }; + let bucket_incarnation_id = expiry_store + .bucket_incarnation_id_from_disk(&expiry_object.bucket) + .await + .expect("bucket incarnation should be available"); + expiry_started_tx.send(()).expect("lifecycle expiry start should be observed"); + super::apply_expiry_on_non_transitioned_objects( + expiry_store, + &expiry_object, + &event, + &LcEventSrc::Scanner, + bucket_incarnation_id, + ) + .await + }); + expiry_started_rx.await.expect("lifecycle expiry should start"); + assert!( + tokio::time::timeout(StdDuration::from_millis(100), &mut expiry) + .await + .is_err(), + "queued lifecycle expiry must wait for table-bucket enablement" + ); + + let sys = metadata_sys::bucket_metadata_sys_of(&ecstore.ctx).expect("metadata system should be initialized"); + let sys = sys.read().await.clone(); + let mut metadata = (*sys.get(&bucket).await.expect("bucket metadata should exist")).clone(); + metadata.table_bucket_config_json = br#"{"enabled":true}"#.to_vec(); + sys.persist_and_set(metadata) + .await + .expect("table bucket marker should be persisted"); + sys.reload_from_store(&bucket) + .await + .expect("table bucket marker should become authoritative"); + drop(enable_guard); + assert!( + !tokio::time::timeout(StdDuration::from_secs(2), expiry) + .await + .expect("queued lifecycle expiry should resume after enablement") + .expect("queued lifecycle expiry task should join"), + "a queued lifecycle task must be rejected after the bucket becomes table-enabled" + ); + assert!( + ecstore + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .is_ok(), + "table data must remain readable after lifecycle admission rejects the delete" + ); + } + #[tokio::test] async fn existing_object_lifecycle_skips_current_expiration_for_explicit_legal_hold() { let lc = latest_expiration_lifecycle(); diff --git a/crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs b/crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs index 22e3b52a0..2bb4941fb 100644 --- a/crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs +++ b/crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs @@ -18,6 +18,7 @@ use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration}; use time::OffsetDateTime; use uuid::Uuid; +use crate::bucket::metadata::BucketMetadata; use crate::bucket::metadata_sys::{self, ObjectLockConfigState}; use crate::error::{Error, Result}; @@ -26,16 +27,37 @@ pub(crate) struct LifecycleExpiryConfigs { pub(crate) lifecycle: Option>, pub(crate) object_lock: Option>, pub(crate) bucket_incarnation_id: Uuid, + pub(crate) table_bucket_enabled: bool, } -pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result { - let bucket_incarnation_id = api.bucket_incarnation_id_from_disk(bucket).await?; +async fn get_authoritative_metadata( + api: &crate::store::ECStore, + bucket: &str, + bucket_incarnation_id: Uuid, +) -> Result> { let sys = metadata_sys::bucket_metadata_sys_of(&api.ctx)?; let sys = sys.read().await.clone(); let metadata = sys.get_authoritative_metadata(bucket).await?; if !metadata.bucket_incarnation_sidecar || metadata.bucket_incarnation_id != bucket_incarnation_id { return Err(Error::other(format!("bucket lifecycle metadata is not authoritative: {bucket}"))); } + Ok(metadata) +} + +pub(crate) async fn lifecycle_expiry_allowed( + api: &crate::store::ECStore, + bucket: &str, + bucket_incarnation_id: Uuid, +) -> Result { + Ok(!get_authoritative_metadata(api, bucket, bucket_incarnation_id) + .await? + .table_bucket_enabled()) +} + +pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result { + let bucket_incarnation_id = api.bucket_incarnation_id_from_disk(bucket).await?; + let metadata = get_authoritative_metadata(api, bucket, bucket_incarnation_id).await?; + let table_bucket_enabled = metadata.table_bucket_enabled(); let lifecycle = if metadata.lifecycle_config.is_none() && !metadata.lifecycle_config_xml.is_empty() { return Err(Error::other("persisted bucket lifecycle configuration is invalid")); @@ -51,6 +73,7 @@ pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str lifecycle: None, object_lock: None, bucket_incarnation_id, + table_bucket_enabled, }); } let object_lock = match metadata_sys::object_lock_config_state_from_authoritative_metadata(&metadata)? { @@ -65,6 +88,7 @@ pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str lifecycle, object_lock, bucket_incarnation_id, + table_bucket_enabled, }) } @@ -125,6 +149,7 @@ mod tests { let lifecycle = lifecycle_config(); metadata.lifecycle_config_xml = crate::bucket::utils::serialize(&lifecycle).unwrap(); metadata.lifecycle_config = Some(lifecycle); + metadata.table_bucket_config_json = br#"{"enabled":true}"#.to_vec(); metadata_sys::set_new_bucket_metadata_in(&store_a.ctx, metadata) .await .unwrap(); @@ -132,7 +157,14 @@ mod tests { .await .unwrap(); - assert!(get_expiry_configs(&store_a, bucket).await.unwrap().lifecycle.is_some()); + let configs = get_expiry_configs(&store_a, bucket).await.unwrap(); + assert!(configs.lifecycle.is_some()); + assert!(configs.table_bucket_enabled); + assert!( + !lifecycle_expiry_allowed(&store_a, bucket, configs.bucket_incarnation_id) + .await + .unwrap() + ); assert!(get_expiry_configs(&store_b, bucket).await.unwrap().lifecycle.is_none()); } } diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 442a5d55d..59f5aae37 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -288,6 +288,13 @@ pub(crate) fn bucket_metadata_sys_of(ctx: &crate::runtime::instance::InstanceCon get_bucket_metadata_sys() } +pub(crate) fn require_bucket_metadata_sys_in( + ctx: &crate::runtime::instance::InstanceContext, +) -> Result>> { + ctx.bucket_metadata_sys() + .ok_or_else(|| Error::other("bucket metadata sys not initialized for this instance")) +} + pub(crate) async fn object_store_in(ctx: &crate::runtime::instance::InstanceContext) -> Result> { let sys = bucket_metadata_sys_of(ctx)?; Ok(sys.read().await.api.clone()) @@ -376,6 +383,15 @@ pub async fn update(bucket: &str, config_file: &str, data: Vec) -> Result, +) -> Result { + Box::pin(update_with_sys(require_bucket_metadata_sys_in(ctx)?, bucket, config_file, data)).await +} + pub async fn delete(bucket: &str, config_file: &str) -> Result { delete_with_sys(get_bucket_metadata_sys()?, bucket, config_file).await } diff --git a/crates/ecstore/src/bucket/policy_sys.rs b/crates/ecstore/src/bucket/policy_sys.rs index 75a9a4473..68bac09fe 100644 --- a/crates/ecstore/src/bucket/policy_sys.rs +++ b/crates/ecstore/src/bucket/policy_sys.rs @@ -14,6 +14,7 @@ use super::metadata_sys::get_bucket_metadata_sys; use crate::error::{Result, StorageError}; +use crate::store::ECStore; use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs}; pub struct PolicySys {} @@ -27,6 +28,10 @@ impl PolicySys { Self::is_allowed_with_policy(args, Self::get(args.bucket).await).await } + pub async fn try_is_allowed_for_store(store: &ECStore, args: &BucketPolicyArgs<'_>) -> Result { + Self::is_allowed_with_policy(args, store.get_bucket_policy(args.bucket).await.map(|(policy, _)| policy)).await + } + async fn is_allowed_with_policy(args: &BucketPolicyArgs<'_>, policy: Result) -> Result { match policy { Ok(policy) => Ok(policy.is_allowed(args).await), diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 8dc2e8a13..5ab25fef6 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -23,6 +23,7 @@ use crate::set_disk::get_lock_acquire_timeout; use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp}; use crate::storage_api_contracts::namespace::NamespaceLocking as _; use futures::stream::{self, StreamExt}; +use rustfs_policy::policy::BucketPolicy; use std::collections::BTreeMap; use std::future::Future; @@ -153,6 +154,31 @@ where } impl ECStore { + pub async fn get_bucket_metadata(&self, bucket: &str) -> Result> { + let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?; + sys.read().await.get(bucket).await + } + + pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> { + let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?; + sys.read().await.get_bucket_policy(bucket).await + } + + pub async fn get_bucket_policy_raw(&self, bucket: &str) -> Result<(String, OffsetDateTime)> { + let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?; + sys.read().await.get_bucket_policy_raw(bucket).await + } + + pub async fn restricts_public_bucket_access(&self, bucket: &str) -> Result { + let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?; + let (config, _) = sys.read().await.get_public_access_block_config(bucket).await?; + Ok(config.restrict_public_buckets.unwrap_or(false)) + } + + pub async fn update_bucket_metadata_config(&self, bucket: &str, config_file: &str, data: Vec) -> Result { + metadata_sys::update_in(&self.ctx, bucket, config_file, data).await + } + pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result { metadata_sys::get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await } @@ -1077,6 +1103,26 @@ mod tests { (temp_dir, ecstore) } + #[tokio::test] + async fn request_metadata_methods_fail_closed_before_instance_initialization() { + let (_temp_dir, store) = setup_multi_pool_scanner_listing_test_env().await; + + let expected = "bucket metadata sys not initialized for this instance"; + let errors = [ + store.get_bucket_metadata("bucket").await.unwrap_err(), + store.get_bucket_policy("bucket").await.unwrap_err(), + store.get_bucket_policy_raw("bucket").await.unwrap_err(), + store.restricts_public_bucket_access("bucket").await.unwrap_err(), + store + .update_bucket_metadata_config("bucket", crate::bucket::metadata::BUCKET_POLICY_CONFIG, Vec::new()) + .await + .unwrap_err(), + ]; + for error in errors { + assert_eq!(error.to_string(), format!("Io error: {expected}")); + } + } + async fn create_bucket_with_object(ecstore: &Arc, bucket: &str, object: &str) { let generation_before_make = ecstore.scanner_namespace_mutation_generation(); ecstore diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index d1926f823..e985a3441 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -12,6 +12,7 @@ for later deletion. ## Open Items +- `table-publication-fence-v1` S3 Tables publication fencing: nodes that predate table and table-bucket publication fences can mutate live files while a new node is publishing a catalog pointer. New nodes retain exact object guards until the operator confirms that every serving node uses the new fences. Fleet confirmation also requires non-overlapping active warehouse prefixes and lifecycle workers that exclude table buckets. Remove the exact live-file fallback and the fleet-confirmation gate after the minimum supported RustFS release acquires table fences for registered-table mutations and table-bucket fences for unresolved-prefix mutations. - `cross-pool-fence-v1` authenticated unsupported advertisement: predeployment servers recognize the versioned cross-pool fence capability probe but report support version 0, allowing a later all-peer probe to distinguish predeployment nodes without activating a second lock domain. Replace the unsupported advertisement only when composite lock acquisition, a cluster-wide activation fence, complete fleet proof, commit-time proof revalidation, and fail-closed revocation ship together. - `table-catalog-dotted-namespace` Iceberg REST namespace path compatibility: existing RustFS clients use dotted namespace paths, while the standard multi-level contract uses the URL-encoded unit separator `%1F`. New servers accept both forms so a rolling upgrade does not invalidate existing catalog configuration. Remove the dotted fallback after the minimum supported RustFS release advertises `%1F` and all supported clients have refreshed their catalog configuration. - `rustfs-5509` FileInfo positional MessagePack decoding: beta.11 serialized 28 fields, while beta.12 inserted transition-version fields in the middle and serialized an incompatible 30-field array. New releases write named maps and retain readers for both shipped array layouts so direct and rolling upgrades can read either release. Remove the positional-array readers after every supported direct-upgrade release writes named maps and no retained RPC payload can contain a pre-map FileInfo array. diff --git a/docs/architecture/s3-tables-support-matrix.md b/docs/architecture/s3-tables-support-matrix.md index 138b0aca0..1a619c921 100644 --- a/docs/architecture/s3-tables-support-matrix.md +++ b/docs/architecture/s3-tables-support-matrix.md @@ -92,6 +92,7 @@ catalog extension. |---|---|---| | Metadata retention dry-run | Supported | Reports retained metadata and deletion candidates without moving the table pointer. | | Metadata cleanup delete | Supported | Deletes only candidates that pass the safety window and current-pointer checks. | +| Ordinary bucket lifecycle expiry | Disabled for table buckets | Table bucket objects are excluded from ordinary lifecycle expiration, including already queued expiry work. Snapshot expiration and orphan cleanup remain catalog maintenance operations so referenced Iceberg files cannot be deleted outside publication fencing. | | Snapshot expiration planning | Supported | Produces expiration plans with retained and candidate snapshots. | | Snapshot expiration commit | Preview / controlled | Can manually commit safe snapshot expiration through the catalog. Stale plans fail closed. | | Manifest/data/delete reachability cleanup | Supported | Reads manifest-list and manifest Avro references, reports reachable objects, and deletes only unreferenced table objects that pass the safety window. | @@ -112,6 +113,7 @@ catalog extension. |---|---|---| | Single-table CAS | Supported | The table pointer advances only through expected-token and expected-metadata-location validation. | | Idempotent retry | Supported | Repeated commit IDs can return the already finalized result or surface recoverable finalization gaps. | +| Commit publication fencing | Supported with rolling-upgrade gate | Existing deployments retain exact object guards so older writers cannot mutate referenced files during publication. Set `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` only after every serving node supports table and table-bucket publication fences. In scalable mode, active table warehouse prefixes must not overlap, ordinary lifecycle expiry remains disabled for table buckets, and first enablement, first publication, drop, and warehouse relocation are serialized by the table-bucket fence. | | Post-CAS finalization recovery | Supported | Diagnostics and recovery can repair stale or missing idempotency indexes without changing the current table pointer. | | Catalog export | Supported | Exposes table state, commit recovery state, and backing migration information for operator inspection. | | Strong backing state transfer | Supported | Object-backed table bucket, namespace, table, view, commit-log, and idempotency state can be materialized into the durable strong snapshot. The transfer is deterministic, ETag-CAS protected, idempotent after an interrupted finalization, and fails closed when a table or view has no owning namespace entry. | diff --git a/rustfs/src/admin/auth.rs b/rustfs/src/admin/auth.rs index ee1f3e227..41bb6155b 100644 --- a/rustfs/src/admin/auth.rs +++ b/rustfs/src/admin/auth.rs @@ -188,6 +188,25 @@ pub async fn validate_admin_request_with_bucket_object( evaluate_admin_actions(iam_store, &ctx, &actions, resource.bucket, resource.object).await } +pub(crate) async fn validate_admin_action_with_bucket_object_for_iam( + iam_store: Arc>, + headers: &HeaderMap, + cred: &Credentials, + is_owner: bool, + action: Action, + remote_addr: Option, + resource: AdminResourceScope<'_>, +) -> S3Result<()> { + let ctx = AuthContext { + headers, + cred, + is_owner, + deny_only: false, + remote_addr, + }; + evaluate_admin_actions(iam_store, &ctx, &[action], resource.bucket, resource.object).await +} + /// Admin gate for KMS endpoints that act on one key. /// /// `key_id` is the identifier as requested (before any alias resolution), so a diff --git a/rustfs/src/admin/handlers/table_catalog/config.rs b/rustfs/src/admin/handlers/table_catalog/config.rs index 53d520edf..46959a717 100644 --- a/rustfs/src/admin/handlers/table_catalog/config.rs +++ b/rustfs/src/admin/handlers/table_catalog/config.rs @@ -33,8 +33,12 @@ impl Operation for EnableTableBucketHandler { let warehouse = warehouse_from_params(¶ms)?; let resource = TableCatalogResource::warehouse(&warehouse); authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableBucketAction).await?; - let store = table_catalog_store()?; - let response = enable_table_bucket_response(&store, &warehouse).await?; + let backend = table_catalog_backend_from_extensions(&req.extensions)?; + let object_store = runtime_sources::object_store_from_req(&req) + .ok_or_else(|| table_catalog_internal_error("request object store is not initialized"))?; + let store = table_catalog_store_from_backend(backend.clone())?; + let publication = TableCommitObjectBackend::preauthorized(backend); + let response = enable_table_bucket_response(&store, &publication, object_store.as_ref(), &warehouse).await?; build_json_response(StatusCode::OK, &response) } } @@ -47,8 +51,8 @@ impl Operation for GetTableBucketHandler { let warehouse = warehouse_from_params(¶ms)?; let resource = TableCatalogResource::warehouse(&warehouse); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableBucketAction).await?; - let store = table_catalog_store()?; - let enabled = table_bucket_enabled_from_metadata(&warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; + let enabled = table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let response = table_bucket_response(&store, &warehouse, enabled).await?; build_json_response(StatusCode::OK, &response) } @@ -62,8 +66,8 @@ impl Operation for GetTableCatalogMigrationHandler { let warehouse = warehouse_from_params(¶ms)?; let resource = TableCatalogResource::warehouse(&warehouse); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableCatalogAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_object_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_object_store_from_extensions(&req.extensions)?; let started = Instant::now(); let result = store .plan_durable_strong_backing_migration(&warehouse) @@ -82,8 +86,8 @@ impl Operation for MaterializeTableCatalogMigrationHandler { async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { let warehouse = warehouse_from_params(¶ms)?; authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_object_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_object_store_from_extensions(&req.extensions)?; let started = Instant::now(); let result = store .materialize_durable_strong_backing_migration(&warehouse) @@ -102,8 +106,8 @@ impl Operation for CancelTableCatalogMigrationHandler { async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { let warehouse = warehouse_from_params(¶ms)?; authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_object_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_object_store_from_extensions(&req.extensions)?; let started = Instant::now(); let result = store .cancel_durable_strong_backing_migration(&warehouse) @@ -125,8 +129,8 @@ impl Operation for ExternalCatalogBridgeHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_object_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_object_store_from_extensions(&req.extensions)?; let response = external_catalog_bridge_response(&store, &warehouse, &namespace, &table).await?; build_json_response(StatusCode::OK, &response) } @@ -142,9 +146,9 @@ impl Operation for PutExternalCatalogBridgeHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let request = read_json_body::(req.input).await?; - let store = table_catalog_object_store()?; + let store = table_catalog_object_store_from_extensions(&req.extensions)?; let response = put_external_catalog_bridge_response(&store, &warehouse, &namespace, &table, request).await?; build_json_response(StatusCode::OK, &response) } @@ -154,15 +158,16 @@ pub struct SyncExternalCatalogBridgeHandler {} #[async_trait::async_trait] impl Operation for SyncExternalCatalogBridgeHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { let warehouse = warehouse_from_params(¶ms)?; let namespace = namespace_from_params(¶ms)?; let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableMetadataLocationAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_object_store()?; + let principal = + authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableMetadataLocationAction).await?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; + let store = table_catalog_object_store_from_extensions(&req.extensions)?; if store .load_table(&warehouse, &namespace.public_name(), &table) .await @@ -171,18 +176,21 @@ impl Operation for SyncExternalCatalogBridgeHandler { { authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?; } - let request = read_json_body::(req.input).await?; - let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; - let response = sync_external_catalog_bridge_response( + install_table_catalog_s3_request_info(&mut req, &principal)?; + let request = read_json_body::(std::mem::take(&mut req.input)).await?; + let table_bucket_enabled = table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let commit_backend = TableCommitObjectBackend::for_request(metadata_backend, req); + let result = sync_external_catalog_bridge_response( &store, - &metadata_backend, + &commit_backend, &warehouse, &namespace, &table, request, table_bucket_enabled, ) - .await?; + .await; + let response = commit_backend.finish(result).await?; build_json_response(StatusCode::OK, &response) } } diff --git a/rustfs/src/admin/handlers/table_catalog/credentials.rs b/rustfs/src/admin/handlers/table_catalog/credentials.rs index 2c07f459a..e997e39d7 100644 --- a/rustfs/src/admin/handlers/table_catalog/credentials.rs +++ b/rustfs/src/admin/handlers/table_catalog/credentials.rs @@ -23,12 +23,12 @@ impl Operation for RestLoadCredentialsHandler { let namespace = namespace_from_params(¶ms)?; let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableCredentialsAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let principal = table_catalog_request_principal(&req).await?; - let store = table_catalog_store()?; - let issuer = IamTableCredentialIssuer::from_env(); - let response = load_credentials_response(&store, &warehouse, &namespace, &table, &issuer, Some(&principal)).await?; + let principal = authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableCredentialsAction).await?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; + let issuer = IamTableCredentialIssuer::from_request(&req)?; + let response = + load_credentials_response(&store, &warehouse, &namespace, &table, &issuer, Some(&principal.credentials)).await?; build_json_response(StatusCode::OK, &response) } } diff --git a/rustfs/src/admin/handlers/table_catalog/maintenance.rs b/rustfs/src/admin/handlers/table_catalog/maintenance.rs index d576fc7dd..c0dc5fb61 100644 --- a/rustfs/src/admin/handlers/table_catalog/maintenance.rs +++ b/rustfs/src/admin/handlers/table_catalog/maintenance.rs @@ -24,10 +24,10 @@ impl Operation for RestTableMetadataMaintenanceHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; - let store = table_catalog_object_store()?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; + let store = table_catalog_object_store_from_extensions(&req.extensions)?; let response = table_metadata_maintenance_response(&store, &metadata_backend, &warehouse, &namespace, &table, request).await?; build_json_response(StatusCode::OK, &response) @@ -44,8 +44,8 @@ impl Operation for GetTableMaintenanceConfigHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableLifecycleAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let response = store .get_table_maintenance_config(&warehouse, &namespace.public_name(), &table) .await @@ -64,9 +64,9 @@ impl Operation for PutTableMaintenanceConfigHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableLifecycleAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let request = read_json_body::(req.input).await?; - let store = table_catalog_store()?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let response = store .put_table_maintenance_config(&warehouse, &namespace.public_name(), &table, request) .await @@ -86,8 +86,8 @@ impl Operation for GetTableMaintenanceJobHandler { let job = job_id_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableLifecycleAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let Some(response) = store .get_table_metadata_maintenance_report(&warehouse, &namespace.public_name(), &table, &job) .await @@ -109,8 +109,8 @@ impl Operation for GetTableMaintenanceSchedulerHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableLifecycleAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let response = store .get_table_maintenance_scheduler_report(&warehouse, &namespace.public_name(), &table) .await @@ -129,9 +129,9 @@ impl Operation for RunTableMaintenanceSchedulerHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let request = read_json_body_or_default::(req.input).await?; - let store = table_catalog_store()?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let response = store .run_table_maintenance_scheduler_once( &warehouse, @@ -155,9 +155,9 @@ impl Operation for RunTableMaintenanceWorkerHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let request = read_json_body::(req.input).await?; - let store = table_catalog_store()?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let response = store .run_table_metadata_maintenance_worker_once( &warehouse, @@ -182,9 +182,9 @@ impl Operation for HeartbeatTableMaintenanceJobHandler { let job = job_id_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let request = read_json_body::(req.input).await?; - let store = table_catalog_store()?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let response = store .heartbeat_table_metadata_maintenance_job( &warehouse, @@ -211,9 +211,9 @@ impl Operation for TableMaintenanceQuarantineHandler { let job = job_id_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let request = read_json_body::(req.input).await?; - let store = table_catalog_store()?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let response = store .apply_table_maintenance_quarantine_operation(&warehouse, &namespace.public_name(), &table, &job, request) .await diff --git a/rustfs/src/admin/handlers/table_catalog/mod.rs b/rustfs/src/admin/handlers/table_catalog/mod.rs index 68b12628b..b2e90b90c 100644 --- a/rustfs/src/admin/handlers/table_catalog/mod.rs +++ b/rustfs/src/admin/handlers/table_catalog/mod.rs @@ -12,18 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::admin::runtime_sources; use crate::admin::runtime_sources::default_admin_usecase; -use crate::admin::runtime_sources::{current_object_store_handle, current_token_signing_key}; -use crate::admin::storage_api::bucket::{metadata::table_catalog_path_hash, metadata_sys}; +use crate::admin::storage_api::access::{ReqInfo, authorize_internal_object_request}; +use crate::admin::storage_api::bucket::metadata::table_catalog_path_hash; use crate::admin::storage_api::runtime::ECStore; use crate::admin::{ - auth::{AdminResourceScope, validate_admin_request, validate_admin_request_with_bucket_object}, + auth::{AdminResourceScope, validate_admin_action_with_bucket_object_for_iam}, router::{AdminOperation, Operation, S3Router}, }; -use crate::auth::{check_key_valid, get_session_token}; +use crate::auth::{check_key_valid_with_context, get_session_token}; use crate::error::ApiError; use crate::server::{RemoteAddr, TABLE_CATALOG_COMPAT_PREFIX, TABLE_CATALOG_PREFIX}; use crate::table_catalog::{DEFAULT_WAREHOUSE_ID, TableCatalogStore}; +use futures::{StreamExt, TryStreamExt, stream}; use http::{HeaderMap, HeaderValue, StatusCode}; use hyper::Method; use matchit::Params; @@ -35,7 +37,7 @@ use rustfs_policy::{ auth::get_new_credentials_with_metadata, policy::{ Policy, - action::{Action, AdminAction}, + action::{Action, AdminAction, S3Action}, }, }; use rustfs_utils::crypto::{base64_decode_url_safe_no_pad, base64_encode_url_safe_no_pad, hex_sha256}; @@ -43,6 +45,7 @@ use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::C use serde::{Deserialize, Serialize, de::DeserializeOwned}; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::num::NonZeroUsize; +use std::sync::Arc; use std::time::{Duration as StdDuration, Instant}; use time::{Duration, OffsetDateTime}; use uuid::Uuid; @@ -75,6 +78,10 @@ const NAMESPACE_REQUEST_BODY_MAX_SIZE: usize = MAX_ADMIN_REQUEST_BODY_SIZE; const NAMESPACE_REQUEST_BODY_TIMEOUT: StdDuration = StdDuration::from_secs(10); const WAREHOUSE_PROPERTY: &str = "warehouse"; const PREFIX_PROPERTY: &str = "prefix"; + +fn table_catalog_internal_error(message: &'static str) -> S3Error { + S3Error::with_message(S3ErrorCode::InternalError, message) +} const NAMESPACE_SEPARATOR_PROPERTY: &str = "namespace-separator"; const ICEBERG_ERROR_ALREADY_EXISTS: &str = "AlreadyExistsException"; const ICEBERG_ERROR_BAD_REQUEST: &str = "BadRequestException"; @@ -781,14 +788,25 @@ impl TableCredentialIssuer for DisabledTableCredentialIssuer { struct IamTableCredentialIssuer { enabled: bool, ttl_seconds: i64, + iam_store: Arc>, + token_signing_key: Option, } impl IamTableCredentialIssuer { - fn from_env() -> Self { - Self { + fn from_request(req: &S3Request) -> S3Result { + let context = runtime_sources::app_context_from_req(req) + .ok_or_else(|| table_catalog_internal_error("request application context is not initialized"))?; + let iam = context.iam(); + if !iam.is_ready() { + return Err(table_catalog_internal_error("iam not init")); + } + let token_signing_key = context.action_credentials().get().map(|credentials| credentials.secret_key); + Ok(Self { enabled: table_credential_vending_enabled(), ttl_seconds: table_credential_ttl_seconds(), - } + iam_store: iam.handle(), + token_signing_key, + }) } } @@ -840,14 +858,15 @@ impl TableCredentialIssuer for IamTableCredentialIssuer { serde_json::Value::String(request.scope_prefix.clone()), ); - let secret = current_token_signing_key().ok_or_else(|| s3_error!(InternalError, "token signing key not initialized"))?; - let mut credential = get_new_credentials_with_metadata(&claims, &secret) + let secret = self + .token_signing_key + .as_deref() + .ok_or_else(|| table_catalog_internal_error("token signing key not initialized"))?; + let mut credential = get_new_credentials_with_metadata(&claims, secret) .map_err(|err| s3_error!(InternalError, "failed to generate table credentials: {}", err))?; bind_table_credential_parent(&mut credential, principal); - let iam_store = - crate::admin::runtime_sources::current_ready_iam_handle().map_err(|_| s3_error!(InternalError, "iam not init"))?; - iam_store + self.iam_store .set_temp_user(&credential.access_key, &credential, None) .await .map_err(|_| s3_error!(InternalError, "failed to store table credentials"))?; @@ -1017,20 +1036,15 @@ fn exists_status(exists: bool) -> StatusCode { } async fn authorize_table_catalog_request(req: &S3Request, action: AdminAction) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { - return Err(s3_error!(InvalidRequest, "authentication required")); - }; - - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - validate_admin_request( + let principal = table_catalog_request_principal(req).await?; + validate_admin_action_with_bucket_object_for_iam( + principal.iam_store, &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(action)], + &principal.credentials, + principal.owner, + Action::AdminAction(action), req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), + AdminResourceScope::bucket(""), ) .await } @@ -1098,34 +1112,644 @@ async fn authorize_table_catalog_resource_request( req: &S3Request, resource: &TableCatalogResource<'_>, action: AdminAction, -) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { - return Err(s3_error!(InvalidRequest, "authentication required")); - }; - - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; +) -> S3Result { + let principal = table_catalog_request_principal(req).await?; let object_path = resource.object_path(); - validate_admin_request_with_bucket_object( + validate_admin_action_with_bucket_object_for_iam( + principal.iam_store.clone(), &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(action)], + &principal.credentials, + principal.owner, + Action::AdminAction(action), req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), AdminResourceScope::bucket_object(resource.warehouse, object_path.as_deref().unwrap_or("")), ) - .await + .await?; + Ok(principal) } -async fn table_catalog_request_principal(req: &S3Request) -> S3Result { +struct TableCatalogRequestPrincipal { + credentials: rustfs_credentials::Credentials, + owner: bool, + iam_store: Arc>, +} + +fn install_table_catalog_s3_request_info(req: &mut S3Request, principal: &TableCatalogRequestPrincipal) -> S3Result<()> { + if req.extensions.get::().is_none() { + req.extensions.insert(ReqInfo { + region: req.region.clone(), + ..Default::default() + }); + } + let req_info = req + .extensions + .get_mut::() + .ok_or_else(|| s3_error!(InternalError, "failed to install table catalog authorization context"))?; + req_info.cred = Some(principal.credentials.clone()); + req_info.is_owner = principal.owner; + Ok(()) +} + +async fn authorize_table_catalog_s3_actions( + req: &mut S3Request, + bucket: &str, + object: &str, + actions: &[S3Action], +) -> S3Result<()> { + let original = { + let req_info = req + .extensions + .get_mut::() + .ok_or_else(|| s3_error!(AccessDenied, "authentication required"))?; + ( + req_info.bucket.replace(bucket.to_string()), + req_info.object.replace(object.to_string()), + req_info.version_id.take(), + ) + }; + + let mut result = Ok(()); + for action in actions { + if let Err(err) = authorize_internal_object_request(req, Action::S3Action(*action)).await { + result = Err(err); + break; + } + } + if let Some(req_info) = req.extensions.get_mut::() { + (req_info.bucket, req_info.object, req_info.version_id) = original; + } + result +} + +async fn table_catalog_request_principal(req: &S3Request) -> S3Result { let Some(input_cred) = &req.credentials else { return Err(s3_error!(InvalidRequest, "authentication required")); }; - let (cred, _owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - Ok(cred) + let context = runtime_sources::app_context_from_req(req) + .ok_or_else(|| table_catalog_internal_error("request application context is not initialized"))?; + let (credentials, owner) = check_key_valid_with_context( + get_session_token(&req.uri, &req.headers).unwrap_or_default(), + &input_cred.access_key, + Some(context.as_ref()), + ) + .await?; + let iam = context.iam(); + if !iam.is_ready() { + return Err(table_catalog_internal_error("iam not init")); + } + Ok(TableCatalogRequestPrincipal { + credentials, + owner, + iam_store: iam.handle(), + }) +} + +#[derive(Clone)] +enum TableCommitObjectAuthorization { + Request(Arc>>), + Preauthorized, + #[cfg(test)] + Test { + authorized_objects: Arc>>, + denied_object: Option, + }, +} + +#[derive(Default, PartialEq, Eq)] +enum TableCommitPublicationPhase { + #[default] + Discovering, + Preparing, + Prepared, + Complete, +} + +#[derive(Clone, PartialEq, Eq)] +enum TableCommitObjectIdentity { + Missing, + Metadata(crate::table_catalog::TableCatalogObjectMetadata), + ContentSha256(String), +} + +#[derive(Clone, PartialEq, Eq)] +struct TableCommitObservedObject { + identity: TableCommitObjectIdentity, + max_size: Option, +} + +#[derive(Default)] +struct TableCommitPublicationState { + phase: TableCommitPublicationPhase, + bucket_fence: Option, + table_fence: Option<(String, String, String)>, + observed_objects: BTreeMap<(String, String), TableCommitObservedObject>, + guards: Vec>, +} + +#[derive(Clone)] +struct TableCommitObjectBackend { + backend: B, + authorization: TableCommitObjectAuthorization, + authorization_error: Arc>>, + publication: Arc>, + publication_fence_fleet_confirmed: bool, +} + +impl TableCommitObjectBackend +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ + fn new(backend: B, authorization: TableCommitObjectAuthorization, publication_fence_fleet_confirmed: bool) -> Self { + Self { + backend, + authorization, + authorization_error: Arc::new(tokio::sync::Mutex::new(None)), + publication: Arc::new(parking_lot::Mutex::new(TableCommitPublicationState::default())), + publication_fence_fleet_confirmed, + } + } + + fn for_request(backend: B, req: S3Request) -> Self { + Self::new( + backend, + TableCommitObjectAuthorization::Request(Arc::new(tokio::sync::Mutex::new(req))), + rustfs_utils::get_env_bool(crate::table_catalog::ENV_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED, false), + ) + } + + fn preauthorized(backend: B) -> Self { + Self::new( + backend, + TableCommitObjectAuthorization::Preauthorized, + rustfs_utils::get_env_bool(crate::table_catalog::ENV_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED, false), + ) + } + + #[cfg(test)] + fn trusted(backend: B) -> Self { + Self::new(backend, TableCommitObjectAuthorization::Preauthorized, true) + } + + #[cfg(test)] + fn rolling_upgrade(backend: B) -> Self { + Self::new(backend, TableCommitObjectAuthorization::Preauthorized, false) + } + + #[cfg(test)] + fn test( + backend: B, + authorized_objects: Arc>>, + denied_object: Option, + ) -> Self { + Self::new( + backend, + TableCommitObjectAuthorization::Test { + authorized_objects, + denied_object, + }, + true, + ) + } + + async fn authorize(&self, bucket: &str, object: &str, action: S3Action) -> crate::table_catalog::TableCatalogStoreResult<()> { + let result = match &self.authorization { + TableCommitObjectAuthorization::Request(req) => { + let mut req = req.lock().await; + authorize_table_catalog_s3_actions(&mut req, bucket, object, &[action]).await + } + TableCommitObjectAuthorization::Preauthorized => Ok(()), + #[cfg(test)] + TableCommitObjectAuthorization::Test { + authorized_objects, + denied_object, + } => { + authorized_objects.lock().await.push((object.to_string(), action)); + if denied_object + .as_deref() + .is_some_and(|denied| denied == "*" || denied == object) + { + Err(s3_error!(AccessDenied, "test object authorization denied")) + } else { + Ok(()) + } + } + }; + if let Err(err) = result { + let mut authorization_error = self.authorization_error.lock().await; + if authorization_error.is_none() { + *authorization_error = Some(err); + } + return Err(crate::table_catalog::TableCatalogStoreError::Internal( + "table commit object authorization failed".to_string(), + )); + } + Ok(()) + } + + fn ensure_observation_allowed(&self, bucket: &str, object: &str) -> crate::table_catalog::TableCatalogStoreResult<()> { + let publication = self.publication.lock(); + match publication.phase { + TableCommitPublicationPhase::Discovering | TableCommitPublicationPhase::Complete => Ok(()), + TableCommitPublicationPhase::Preparing => Err(crate::table_catalog::TableCatalogStoreError::Internal( + "table commit publication preparation is already in progress".to_string(), + )), + TableCommitPublicationPhase::Prepared => Err(crate::table_catalog::TableCatalogStoreError::Internal(format!( + "table commit accessed an object after publication preparation: {bucket}/{object}" + ))), + } + } + + fn record_observation( + &self, + bucket: &str, + object: &str, + observation: TableCommitObservedObject, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + let key = (bucket.to_string(), object.to_string()); + let mut publication = self.publication.lock(); + if publication.phase == TableCommitPublicationPhase::Complete { + return Ok(()); + } + let Some(existing) = publication.observed_objects.get_mut(&key) else { + if publication.phase != TableCommitPublicationPhase::Discovering { + return Err(crate::table_catalog::TableCatalogStoreError::Internal(format!( + "table commit accessed an unvalidated object after publication preparation: {bucket}/{object}" + ))); + } + publication.observed_objects.insert(key, observation); + return Ok(()); + }; + if existing.identity != observation.identity { + return Err(crate::table_catalog::TableCatalogStoreError::Conflict(format!( + "table commit object changed during validation: {bucket}/{object}" + ))); + } + existing.max_size = match (existing.max_size, observation.max_size) { + (Some(expected), Some(actual)) => Some(expected.min(actual)), + (Some(expected), None) => Some(expected), + (None, actual) => actual, + }; + Ok(()) + } + + fn observed_content( + object: Option<&crate::table_catalog::TableCatalogObject>, + max_size: Option, + ) -> TableCommitObservedObject { + let identity = match object { + None => TableCommitObjectIdentity::Missing, + Some(object) => TableCommitObjectIdentity::ContentSha256(hex_sha256(&object.data, str::to_string)), + }; + TableCommitObservedObject { identity, max_size } + } + + fn observed_metadata( + bucket: &str, + object: &str, + metadata: Option, + ) -> crate::table_catalog::TableCatalogStoreResult { + let identity = match metadata { + None => TableCommitObjectIdentity::Missing, + Some(metadata) if metadata.etag.is_some() || metadata.mod_time.is_some() => { + TableCommitObjectIdentity::Metadata(metadata) + } + Some(_) => { + return Err(crate::table_catalog::TableCatalogStoreError::Internal(format!( + "catalog object {bucket}/{object} does not expose a stable fingerprint" + ))); + } + }; + Ok(TableCommitObservedObject { + identity, + max_size: None, + }) + } + + async fn begin_bucket_publication(&self, table_bucket: &str) -> crate::table_catalog::TableCatalogStoreResult<()> { + { + let mut publication = self.publication.lock(); + if publication.phase != TableCommitPublicationPhase::Discovering { + return Err(crate::table_catalog::TableCatalogStoreError::Internal( + "table-bucket commit publication must begin before publication preparation".to_string(), + )); + } + if publication.bucket_fence.as_deref() == Some(table_bucket) { + return Ok(()); + } + if publication.bucket_fence.is_some() { + return Err(crate::table_catalog::TableCatalogStoreError::Internal( + "table commit publication cannot span table buckets".to_string(), + )); + } + publication.bucket_fence = Some(table_bucket.to_string()); + } + let publication_lock = crate::table_catalog::default_table_bucket_publication_lock_path(); + let guard = match self.backend.acquire_write_lock(table_bucket, &publication_lock).await { + Ok(guard) => guard, + Err(err) => { + self.publication.lock().bucket_fence = None; + return Err(err); + } + }; + self.publication.lock().guards.push(guard); + Ok(()) + } + + async fn prepare_publication( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + let namespace = crate::table_catalog::Namespace::parse(namespace) + .map_err(|err| crate::table_catalog::TableCatalogStoreError::Invalid(format!("invalid namespace: {err}")))?; + let table = crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| crate::table_catalog::TableCatalogStoreError::Invalid(format!("invalid table: {err}")))?; + let table_fence = (table_bucket.to_string(), namespace.public_name(), table.as_str().to_string()); + let publication_lock = crate::table_catalog::default_table_publication_lock_path(&namespace, &table); + let expected = { + let mut publication = self.publication.lock(); + match publication.phase { + TableCommitPublicationPhase::Prepared if publication.table_fence.as_ref() == Some(&table_fence) => { + return Ok(()); + } + TableCommitPublicationPhase::Prepared => { + return Err(crate::table_catalog::TableCatalogStoreError::Internal( + "table commit publication cannot span tables".to_string(), + )); + } + TableCommitPublicationPhase::Discovering => { + publication.phase = TableCommitPublicationPhase::Preparing; + std::mem::take(&mut publication.observed_objects) + } + TableCommitPublicationPhase::Preparing => { + return Err(crate::table_catalog::TableCatalogStoreError::Internal( + "table commit publication preparation is already in progress".to_string(), + )); + } + TableCommitPublicationPhase::Complete => { + return Err(crate::table_catalog::TableCatalogStoreError::Internal( + "table commit publication has already completed".to_string(), + )); + } + } + }; + + let prepared = async { + // RUSTFS_COMPAT_TODO(table-publication-fence-v1): Retain exact live-file guards for old nodes. Remove after the minimum supported release uses table and bucket publication fences. + let retain_legacy_object_guards = !self.publication_fence_fleet_confirmed; + let guarded_object_count = if retain_legacy_object_guards { + expected.len() + } else { + expected + .values() + .filter(|object| matches!(&object.identity, TableCommitObjectIdentity::ContentSha256(_))) + .count() + }; + let mut guards = Vec::with_capacity(guarded_object_count.saturating_add(1)); + guards.push(self.backend.acquire_write_lock(table_bucket, &publication_lock).await?); + for ((bucket, object), expected_object) in &expected { + if retain_legacy_object_guards || matches!(&expected_object.identity, TableCommitObjectIdentity::ContentSha256(_)) + { + guards.push(self.backend.acquire_read_lock(bucket, object).await?); + } + } + for ((bucket, object), expected_object) in &expected { + if !matches!(&expected_object.identity, TableCommitObjectIdentity::ContentSha256(_)) { + continue; + } + let actual_object = match expected_object.max_size { + Some(max_size) => self.backend.read_object_unlocked_limited(bucket, object, max_size).await?, + None => self.backend.read_object_unlocked(bucket, object).await?, + }; + let actual = Self::observed_content(actual_object.as_ref(), expected_object.max_size); + if actual.identity != expected_object.identity { + return Err(crate::table_catalog::TableCatalogStoreError::Conflict(format!( + "table commit object changed before catalog publication: {bucket}/{object}" + ))); + } + } + stream::iter( + expected + .into_iter() + .filter_map(|((bucket, object), expected_object)| match expected_object.identity { + expected_identity @ (TableCommitObjectIdentity::Metadata(_) | TableCommitObjectIdentity::Missing) => { + Some((bucket, object, expected_identity)) + } + TableCommitObjectIdentity::ContentSha256(_) => None, + }), + ) + .map(|(bucket, object, expected_identity)| async move { + let metadata = if retain_legacy_object_guards { + self.backend.object_metadata_unlocked(&bucket, &object).await? + } else { + self.backend.object_metadata(&bucket, &object).await? + }; + let actual = Self::observed_metadata(&bucket, &object, metadata)?; + if actual.identity != expected_identity { + return Err(crate::table_catalog::TableCatalogStoreError::Conflict(format!( + "table commit object changed before catalog publication: {bucket}/{object}" + ))); + } + Ok(()) + }) + .buffer_unordered(crate::table_catalog::TABLE_COMMIT_OBJECT_VALIDATION_CONCURRENCY) + .try_for_each(|()| async { Ok::<(), crate::table_catalog::TableCatalogStoreError>(()) }) + .await?; + Ok(guards) + } + .await; + + let mut publication = self.publication.lock(); + match prepared { + Ok(guards) if publication.phase == TableCommitPublicationPhase::Preparing => { + publication.guards.extend(guards); + publication.table_fence = Some(table_fence); + publication.phase = TableCommitPublicationPhase::Prepared; + Ok(()) + } + Ok(_) => { + publication.phase = TableCommitPublicationPhase::Discovering; + Err(crate::table_catalog::TableCatalogStoreError::Internal( + "table commit publication state changed during preparation".to_string(), + )) + } + Err(err) => { + publication.phase = TableCommitPublicationPhase::Discovering; + Err(err) + } + } + } + + fn complete_publication(&self) { + let mut publication = self.publication.lock(); + publication.guards.clear(); + publication.observed_objects.clear(); + publication.bucket_fence = None; + publication.table_fence = None; + publication.phase = TableCommitPublicationPhase::Complete; + } + + async fn finish(&self, result: S3Result) -> S3Result { + match self.authorization_error.lock().await.take() { + Some(err) => Err(err), + None => result, + } + } +} + +#[async_trait::async_trait] +impl crate::table_catalog::TableCatalogObjectBackend for TableCommitObjectBackend +where + B: crate::table_catalog::TableCatalogObjectBackend, +{ + async fn read_object( + &self, + bucket: &str, + object: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + self.authorize(bucket, object, S3Action::GetObjectAction).await?; + self.ensure_observation_allowed(bucket, object)?; + let result = self.backend.read_object(bucket, object).await?; + self.record_observation(bucket, object, Self::observed_content(result.as_ref(), None))?; + Ok(result) + } + + async fn read_object_limited( + &self, + bucket: &str, + object: &str, + max_size: usize, + ) -> crate::table_catalog::TableCatalogStoreResult> { + self.authorize(bucket, object, S3Action::GetObjectAction).await?; + self.ensure_observation_allowed(bucket, object)?; + let result = self.backend.read_object_limited(bucket, object, max_size).await?; + self.record_observation(bucket, object, Self::observed_content(result.as_ref(), Some(max_size)))?; + Ok(result) + } + + async fn read_object_unlocked( + &self, + bucket: &str, + object: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + self.read_object(bucket, object).await + } + + async fn read_object_unlocked_limited( + &self, + bucket: &str, + object: &str, + max_size: usize, + ) -> crate::table_catalog::TableCatalogStoreResult> { + self.read_object_limited(bucket, object, max_size).await + } + + async fn object_exists(&self, bucket: &str, object: &str) -> crate::table_catalog::TableCatalogStoreResult { + self.authorize(bucket, object, S3Action::GetObjectAction).await?; + self.ensure_observation_allowed(bucket, object)?; + let metadata = self.backend.object_metadata(bucket, object).await?; + let observation = Self::observed_metadata(bucket, object, metadata)?; + let exists = observation.identity != TableCommitObjectIdentity::Missing; + self.record_observation(bucket, object, observation)?; + Ok(exists) + } + + async fn object_exists_unlocked(&self, bucket: &str, object: &str) -> crate::table_catalog::TableCatalogStoreResult { + self.object_exists(bucket, object).await + } + + async fn put_object( + &self, + bucket: &str, + object: &str, + data: Vec, + precondition: crate::table_catalog::TableCatalogPutPrecondition, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.authorize(bucket, object, S3Action::PutObjectAction).await?; + self.ensure_observation_allowed(bucket, object)?; + let observation = TableCommitObservedObject { + identity: TableCommitObjectIdentity::ContentSha256(hex_sha256(&data, str::to_string)), + max_size: None, + }; + self.backend.put_object(bucket, object, data, precondition).await?; + self.record_observation(bucket, object, observation) + } + + async fn put_object_unlocked( + &self, + bucket: &str, + object: &str, + data: Vec, + precondition: crate::table_catalog::TableCatalogPutPrecondition, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.put_object(bucket, object, data, precondition).await + } + + async fn delete_object(&self, bucket: &str, object: &str) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.authorize(bucket, object, S3Action::DeleteObjectAction).await?; + self.ensure_observation_allowed(bucket, object)?; + self.backend.delete_object(bucket, object).await + } + + async fn delete_object_unlocked(&self, bucket: &str, object: &str) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.delete_object(bucket, object).await + } + + async fn list_objects(&self, _bucket: &str, _prefix: &str) -> crate::table_catalog::TableCatalogStoreResult> { + Err(crate::table_catalog::TableCatalogStoreError::Unsupported( + "table commit validation does not list object prefixes".to_string(), + )) + } + + async fn acquire_read_lock( + &self, + bucket: &str, + object: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + self.backend.acquire_read_lock(bucket, object).await + } + + async fn acquire_write_lock( + &self, + bucket: &str, + object: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + self.backend.acquire_write_lock(bucket, object).await + } + + async fn begin_table_bucket_commit_publication( + &self, + table_bucket: &str, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.begin_bucket_publication(table_bucket).await + } + + fn table_bucket_commit_publication_is_held(&self, table_bucket: &str) -> bool { + self.publication.lock().bucket_fence.as_deref() == Some(table_bucket) + } + + async fn prepare_table_commit_publication( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.prepare_publication(table_bucket, namespace, table).await + } + + fn table_commit_publication_is_held(&self, table_bucket: &str, namespace: &str, table: &str) -> bool { + self.publication + .lock() + .table_fence + .as_ref() + .is_some_and(|held| held.0 == table_bucket && held.1 == namespace && held.2 == table) + } + + fn complete_table_commit_publication(&self) { + self.complete_publication(); + } } async fn read_json_body(mut input: Body) -> S3Result { @@ -1440,8 +2064,11 @@ fn job_id_from_params(params: &Params<'_, '_>) -> S3Result { Ok(job.to_string()) } -fn table_catalog_backend() -> S3Result> { - let store = current_object_store_handle().ok_or_else(|| s3_error!(InternalError, "object store not initialized"))?; +fn table_catalog_backend_from_extensions( + extensions: &http::Extensions, +) -> S3Result> { + let store = runtime_sources::object_store_from_extensions(extensions) + .ok_or_else(|| table_catalog_internal_error("request object store is not initialized"))?; Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new(store)) } @@ -1454,17 +2081,17 @@ fn table_catalog_store_from_backend( crate::table_catalog::ConfiguredTableCatalogStore::from_env(backend).map_err(catalog_store_error) } -fn table_catalog_store() -> S3Result> { - let backend = table_catalog_backend()?; - table_catalog_store_from_backend(backend) +fn table_catalog_store_from_extensions( + extensions: &http::Extensions, +) -> S3Result> { + table_catalog_store_from_backend(table_catalog_backend_from_extensions(extensions)?) } -fn table_catalog_object_store() -> S3Result { +fn table_catalog_object_store_from_extensions(extensions: &http::Extensions) -> S3Result { match crate::table_catalog::TableCatalogBackingMode::from_env().map_err(catalog_store_error)? { - crate::table_catalog::TableCatalogBackingMode::ObjectBacked => { - let backend = table_catalog_backend()?; - Ok(crate::table_catalog::ObjectTableCatalogStore::new(backend)) - } + crate::table_catalog::TableCatalogBackingMode::ObjectBacked => Ok(crate::table_catalog::ObjectTableCatalogStore::new( + table_catalog_backend_from_extensions(extensions)?, + )), crate::table_catalog::TableCatalogBackingMode::DurableStrong => Err(s3_error!( InvalidRequest, "operation is not supported with {} table catalog backing", @@ -1473,15 +2100,18 @@ fn table_catalog_object_store() -> S3Result { } } -async fn table_bucket_enabled_from_metadata(bucket: &str) -> S3Result { - let metadata = metadata_sys::get(bucket) +async fn table_bucket_enabled_from_extensions(extensions: &http::Extensions, bucket: &str) -> S3Result { + let store = runtime_sources::object_store_from_extensions(extensions) + .ok_or_else(|| table_catalog_internal_error("request object store is not initialized"))?; + let metadata = store + .get_bucket_metadata(bucket) .await .map_err(|err| s3_error!(InvalidRequest, "failed to load table bucket metadata for {bucket}: {}", err))?; Ok(metadata.table_bucket_enabled()) } -async fn ensure_table_bucket_enabled(bucket: &str) -> S3Result<()> { - if table_bucket_enabled_from_metadata(bucket).await? { +async fn ensure_table_bucket_enabled_from_extensions(extensions: &http::Extensions, bucket: &str) -> S3Result<()> { + if table_bucket_enabled_from_extensions(extensions, bucket).await? { return Ok(()); } Err(s3_error!(InvalidRequest, "bucket {bucket} is not table-enabled")) @@ -1500,10 +2130,11 @@ fn table_bucket_entry_from_metadata_marker(bucket: &str) -> crate::table_catalog } } -async fn enable_table_bucket_marker(bucket: &str) -> S3Result<()> { +async fn enable_table_bucket_marker(store: &ECStore, bucket: &str) -> S3Result<()> { let marker = crate::table_catalog::table_bucket_marker_json() .map_err(|err| s3_error!(InternalError, "failed to serialize table bucket marker: {}", err))?; - metadata_sys::update(bucket, crate::table_catalog::TABLE_BUCKET_MARKER_CONFIG, marker) + store + .update_bucket_metadata_config(bucket, crate::table_catalog::TABLE_BUCKET_MARKER_CONFIG, marker) .await .map(|_| ()) .map_err(|err| s3_error!(InvalidRequest, "failed to enable table bucket {bucket}: {}", err)) @@ -1556,11 +2187,23 @@ where }) } -async fn enable_table_bucket_response(store: &S, bucket: &str) -> S3Result +async fn enable_table_bucket_response( + store: &S, + publication: &impl crate::table_catalog::TableCatalogObjectBackend, + object_store: &ECStore, + bucket: &str, +) -> S3Result where S: crate::table_catalog::TableCatalogStore + ?Sized, { - enable_table_bucket_marker(bucket).await?; + crate::table_catalog::TableCommitPublication::begin_table_bucket(publication, bucket) + .await + .map_err(catalog_store_error)?; + if !crate::table_catalog::TableCommitPublication::holds_table_bucket(publication, bucket) { + return Err(s3_error!(InternalError, "table bucket enablement requires a publication fence")); + } + let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(publication); + enable_table_bucket_marker(object_store, bucket).await?; ensure_table_bucket_entry(store, bucket, true).await?; table_bucket_response(store, bucket, true).await } @@ -1932,7 +2575,10 @@ fn table_commit_request_from_rest_request( table_bucket: bucket.to_string(), namespace: namespace.public_name(), table: table.to_string(), - commit_id: request.commit_id.unwrap_or_else(|| Uuid::new_v4().to_string()), + commit_id: request + .commit_id + .or_else(|| request.idempotency_key.clone()) + .unwrap_or_else(|| Uuid::new_v4().to_string()), idempotency_key: request.idempotency_key, operation: request.operation.unwrap_or_else(|| "commit".to_string()), expected_version_token: request @@ -1970,6 +2616,21 @@ fn validate_metadata_table_location_in_bucket(bucket: &str, metadata: &serde_jso validate_table_location_in_bucket(bucket, location) } +fn table_warehouse_location_changes( + current: &crate::table_catalog::TableEntry, + target_metadata: &serde_json::Value, +) -> S3Result { + Ok(current.warehouse_location != metadata_table_location(target_metadata)?) +} + +fn metadata_digest_requirement(metadata: &serde_json::Value) -> S3Result { + let sha256 = crate::table_catalog::canonical_json_sha256(metadata).map_err(catalog_store_error)?; + Ok(serde_json::json!({ + "type": crate::table_catalog::TABLE_METADATA_DIGEST_REQUIREMENT_TYPE, + "sha256": sha256 + })) +} + fn validate_metadata_view_location_in_bucket(bucket: &str, metadata: &serde_json::Value) -> S3Result<()> { let location = metadata_table_location(metadata)?; validate_view_location_in_bucket(bucket, location) @@ -2331,6 +2992,7 @@ fn initial_view_metadata_json( "format-version": entry.format_version, "view-uuid": entry.view_uuid, "location": entry.warehouse_location, + "last-updated-ms": current_time_millis(), "current-version-id": version_id, "schemas": [schema], "versions": [view_version], @@ -2544,9 +3206,18 @@ fn validate_current_snapshot_requirement(metadata: &serde_json::Value, requireme } fn apply_table_commit_updates( + metadata: serde_json::Value, + updates: &[serde_json::Value], + previous_metadata_location: &str, +) -> S3Result { + apply_table_commit_updates_at(metadata, updates, previous_metadata_location, current_time_millis()) +} + +fn apply_table_commit_updates_at( mut metadata: serde_json::Value, updates: &[serde_json::Value], previous_metadata_location: &str, + commit_timestamp_ms: i64, ) -> S3Result { if !metadata.is_object() { return Err(s3_error!(InvalidRequest, "current table metadata must be a JSON object")); @@ -2581,7 +3252,7 @@ fn apply_table_commit_updates( crate::table_catalog::synchronize_table_metadata_version_fields(&mut metadata).map_err(catalog_store_error)?; } append_previous_metadata_log(&mut metadata, previous_metadata_location)?; - metadata_object_mut(&mut metadata)?.insert("last-updated-ms".to_string(), serde_json::Value::from(current_time_millis())); + metadata_object_mut(&mut metadata)?.insert("last-updated-ms".to_string(), serde_json::Value::from(commit_timestamp_ms)); Ok(metadata) } @@ -3566,8 +4237,12 @@ fn apply_remove_properties_update(metadata: &mut serde_json::Value, update: &ser } fn append_previous_metadata_log(metadata: &mut serde_json::Value, previous_metadata_location: &str) -> S3Result<()> { + let previous_metadata_timestamp = metadata + .get("last-updated-ms") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "current table metadata is missing last-updated-ms"))?; ensure_array_field(metadata, "metadata-log")?.push(serde_json::json!({ - "timestamp-ms": current_time_millis(), + "timestamp-ms": previous_metadata_timestamp, "metadata-file": previous_metadata_location })); Ok(()) @@ -3869,14 +4544,14 @@ where { let mut entry = table_entry_from_register_request(bucket, namespace, request)?; ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; + let table = crate::table_catalog::IdentifierSegment::parse(entry.table.clone()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; validate_metadata_table_location_in_bucket(bucket, &metadata)?; adopt_registered_metadata_identity(&mut entry, &metadata)?; - let table = crate::table_catalog::IdentifierSegment::parse(entry.table.clone()) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; validate_table_metadata_snapshot_graph(metadata_backend, bucket, namespace, &table, &entry, None, &metadata).await?; store - .register_table(entry.clone()) + .register_table_with_publication(entry.clone(), metadata_backend) .await .map_err(catalog_store_already_exists_error)?; Ok(load_table_response_from_entry(entry, metadata)) @@ -3895,6 +4570,15 @@ where { let (entry, metadata) = table_entry_from_create_table_request(bucket, namespace, request)?; ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; + crate::table_catalog::TableCommitPublication::begin_table_bucket(metadata_backend, bucket) + .await + .map_err(catalog_store_error)?; + if !crate::table_catalog::TableCommitPublication::holds_table_bucket(metadata_backend, bucket) { + return Err(catalog_store_error(crate::table_catalog::TableCatalogStoreError::Internal( + "table creation requires a table-bucket publication fence".to_string(), + ))); + } + let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(metadata_backend); let metadata_data = serde_json::to_vec(&metadata) .map_err(|err| s3_error!(InternalError, "failed to serialize initial table metadata: {}", err))?; metadata_backend @@ -3907,7 +4591,7 @@ where .await .map_err(catalog_store_already_exists_error)?; store - .create_table(entry.clone()) + .register_table_with_publication(entry.clone(), metadata_backend) .await .map_err(catalog_store_already_exists_error)?; Ok(load_table_response_from_entry(entry, metadata)) @@ -4243,6 +4927,30 @@ where Ok(table_metadata_location_response_from_entry(entry)) } +async fn publish_table_commit( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + table_bucket_fence_required: bool, + request: crate::table_catalog::TableCommitRequest, +) -> S3Result +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let _bucket_publication_completion = if table_bucket_fence_required { + metadata_backend + .begin_table_bucket_commit_publication(&request.table_bucket) + .await + .map_err(catalog_store_error)?; + Some(crate::table_catalog::TableCommitPublicationCompletion::new(metadata_backend)) + } else { + None + }; + store + .commit_table_with_publication(request, metadata_backend) + .await + .map_err(catalog_store_error) +} + async fn update_table_metadata_location_response( store: &S, metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, @@ -4254,6 +4962,8 @@ async fn update_table_metadata_location_response( where S: crate::table_catalog::TableCatalogStore + ?Sized, { + let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; let Some(current) = store .load_table(bucket, &namespace.public_name(), table) .await @@ -4261,41 +4971,70 @@ where else { return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); }; - let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; let metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?; if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table_name, &metadata_location) { return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); } - let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; - validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; + let existing_commit = table_commit_for_retry_ids( + store, + bucket, + ¤t.table_id, + request.commit_id.as_deref(), + request.idempotency_key.as_deref(), + ) + .await?; + let previous_metadata_location = existing_commit + .as_ref() + .map_or_else(|| current.metadata_location.clone(), |commit| commit.previous_metadata_location.clone()); + let previous_metadata = read_table_metadata_json(metadata_backend, bucket, &previous_metadata_location).await?; + validate_metadata_table_location_in_bucket(bucket, &previous_metadata)?; let target_metadata = read_table_metadata_json(metadata_backend, bucket, &metadata_location).await?; validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; - validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?; + let table_bucket_fence_required = table_warehouse_location_changes(¤t, &target_metadata)?; + validate_metadata_matches_current_metadata(&previous_metadata, &target_metadata)?; validate_table_metadata_snapshot_graph( metadata_backend, bucket, namespace, &table_name, ¤t, - Some(¤t_metadata), + Some(&previous_metadata), &target_metadata, ) .await?; + let requirements = match existing_commit.as_ref() { + Some(existing_commit) => replay_commit_requirements(existing_commit, &[], &target_metadata)?, + None => Vec::new(), + }; + let commit_id = existing_commit + .as_ref() + .map(|commit| commit.commit_id.clone()) + .or(request.commit_id) + .or_else(|| request.idempotency_key.clone()) + .unwrap_or_else(|| Uuid::new_v4().to_string()); let commit_request = crate::table_catalog::TableCommitRequest { table_bucket: bucket.to_string(), namespace: namespace.public_name(), table: table.to_string(), - commit_id: request.commit_id.unwrap_or_else(|| Uuid::new_v4().to_string()), + commit_id, idempotency_key: request.idempotency_key, operation: "update-metadata-location".to_string(), expected_version_token: request.version_token, - expected_metadata_location: current.metadata_location, + expected_metadata_location: previous_metadata_location, new_metadata_location: metadata_location, - requirements: Vec::new(), + requirements, writer: Some("rustfs-metadata-location-api".to_string()), }; - let result = store.commit_table(commit_request).await.map_err(catalog_store_error)?; + if let Some(existing_commit) = existing_commit.as_ref() + && !crate::table_catalog::commit_log_matches_request(existing_commit, &commit_request, ¤t.table_id) + { + return Err(iceberg_rest_error( + ICEBERG_ERROR_COMMIT_FAILED, + StatusCode::CONFLICT, + "commit retry does not match the original request", + )); + } + let result = publish_table_commit(store, metadata_backend, table_bucket_fence_required, commit_request).await?; Ok(table_metadata_location_response_from_entry(result.table)) } @@ -4305,7 +5044,7 @@ async fn commit_table_response( bucket: &str, namespace: &crate::table_catalog::Namespace, table: &str, - request: RestCommitTableRequest, + mut request: RestCommitTableRequest, ) -> S3Result where S: crate::table_catalog::TableCatalogStore + ?Sized, @@ -4314,7 +5053,8 @@ where return standard_commit_table_response(store, metadata_backend, bucket, namespace, table, request).await; } - let request = table_commit_request_from_rest_request(bucket, namespace, table, request)?; + let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; let Some(current) = store .load_table(bucket, &namespace.public_name(), table) .await @@ -4322,8 +5062,14 @@ where else { return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); }; - let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; + let existing_commit = table_commit_for_retry(store, bucket, ¤t.table_id, &request).await?; + if request.commit_id.is_none() + && let Some(existing_commit) = existing_commit.as_ref() + { + request.commit_id = Some(existing_commit.commit_id.clone()); + } + let client_requirements = request.requirements.clone(); + let mut request = table_commit_request_from_rest_request(bucket, namespace, table, request)?; if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table_name, &request.new_metadata_location) { return Err(s3_error!(InvalidRequest, "metadata location must be inside the table metadata directory")); } @@ -4331,7 +5077,46 @@ where validate_metadata_table_location_in_bucket(bucket, ¤t_metadata)?; let target_metadata = read_table_metadata_json(metadata_backend, bucket, &request.new_metadata_location).await?; validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; + let table_bucket_fence_required = table_warehouse_location_changes(¤t, &target_metadata)?; + if let Some(existing_commit) = existing_commit { + request.requirements = replay_commit_requirements(&existing_commit, &client_requirements, &target_metadata)?; + if !crate::table_catalog::commit_log_matches_request(&existing_commit, &request, ¤t.table_id) { + return Err(iceberg_rest_error( + ICEBERG_ERROR_COMMIT_FAILED, + StatusCode::CONFLICT, + "commit retry does not match the original request", + )); + } + let previous_metadata = + read_table_metadata_json(metadata_backend, bucket, &existing_commit.previous_metadata_location).await?; + validate_metadata_table_location_in_bucket(bucket, &previous_metadata)?; + validate_table_commit_requirements(&previous_metadata, &client_requirements)?; + validate_metadata_matches_current_metadata(&previous_metadata, &target_metadata)?; + validate_table_metadata_snapshot_graph( + metadata_backend, + bucket, + namespace, + &table_name, + ¤t, + Some(&previous_metadata), + &target_metadata, + ) + .await?; + let committed_metadata_location = request.new_metadata_location.clone(); + let result = publish_table_commit(store, metadata_backend, table_bucket_fence_required, request).await?; + return commit_table_replay_response( + metadata_backend, + bucket, + namespace, + table, + result, + &committed_metadata_location, + target_metadata, + ) + .await; + } validate_metadata_matches_current_metadata(¤t_metadata, &target_metadata)?; + validate_table_commit_requirements(¤t_metadata, &client_requirements)?; validate_table_metadata_snapshot_graph( metadata_backend, bucket, @@ -4342,7 +5127,7 @@ where &target_metadata, ) .await?; - let result = store.commit_table(request).await.map_err(catalog_store_error)?; + let result = publish_table_commit(store, metadata_backend, table_bucket_fence_required, request).await?; Ok(commit_table_response_from_result(result, target_metadata)) } @@ -4357,6 +5142,8 @@ async fn standard_commit_table_response( where S: crate::table_catalog::TableCatalogStore + ?Sized, { + let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; let Some(current) = store .load_table(bucket, &namespace.public_name(), table) .await @@ -4364,13 +5151,18 @@ where else { return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found")); }; - let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; + if let Some(response) = + replay_standard_table_commit(store, metadata_backend, bucket, namespace, table, ¤t, &request).await? + { + return Ok(response); + } let current_metadata = read_table_metadata_json(metadata_backend, bucket, ¤t.metadata_location).await?; validate_table_commit_requirements(¤t_metadata, &request.requirements)?; let expected_metadata = current_metadata.clone(); let previous_metadata_location = table_metadata_location_for_client(bucket, ¤t.metadata_location); - let next_metadata = apply_table_commit_updates(current_metadata, &request.updates, &previous_metadata_location)?; + let commit_timestamp_ms = current_time_millis(); + let mut next_metadata = + apply_table_commit_updates_at(current_metadata, &request.updates, &previous_metadata_location, commit_timestamp_ms)?; validate_metadata_table_location_in_bucket(bucket, &next_metadata)?; validate_metadata_identity_matches_current_metadata(&expected_metadata, &next_metadata)?; validate_table_metadata_snapshot_graph_result( @@ -4398,7 +5190,7 @@ where ) .await?; validate_metadata_matches_current_metadata(&expected_metadata, &next_metadata)?; - let (commit_id, metadata_file_token) = standard_commit_ids(request.commit_id); + let (commit_id, metadata_file_token) = standard_commit_ids(request.commit_id.or_else(|| request.idempotency_key.clone())); let next_generation = current.generation.saturating_add(1); let next_metadata_location = crate::table_catalog::default_table_metadata_file_path( namespace, @@ -4407,15 +5199,49 @@ where ); let next_metadata_data = serde_json::to_vec(&next_metadata) .map_err(|err| s3_error!(InternalError, "failed to serialize table metadata update: {}", err))?; - metadata_backend + let put_result = metadata_backend .put_object( bucket, &next_metadata_location, next_metadata_data, crate::table_catalog::TableCatalogPutPrecondition::IfAbsent, ) - .await - .map_err(catalog_store_error)?; + .await; + match put_result { + Ok(()) => { + let persisted_metadata = read_table_metadata_json(metadata_backend, bucket, &next_metadata_location).await?; + if persisted_metadata != next_metadata { + return Err(iceberg_rest_error( + ICEBERG_ERROR_COMMIT_FAILED, + StatusCode::CONFLICT, + "generated metadata changed before catalog publication", + )); + } + } + Err(crate::table_catalog::TableCatalogStoreError::Conflict(_)) => { + let existing_metadata = read_table_metadata_json(metadata_backend, bucket, &next_metadata_location).await?; + let persisted_timestamp = existing_metadata + .get("last-updated-ms") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "existing generated metadata is missing last-updated-ms"))?; + let rebuilt_metadata = apply_table_commit_updates_at( + expected_metadata.clone(), + &request.updates, + &previous_metadata_location, + persisted_timestamp, + )?; + if existing_metadata != rebuilt_metadata { + return Err(iceberg_rest_error( + ICEBERG_ERROR_COMMIT_FAILED, + StatusCode::CONFLICT, + "generated metadata location already contains a different commit", + )); + } + next_metadata = existing_metadata; + } + Err(err) => return Err(catalog_store_error(err)), + } + let table_bucket_fence_required = table_warehouse_location_changes(¤t, &next_metadata)?; let commit_request = crate::table_catalog::TableCommitRequest { table_bucket: bucket.to_string(), @@ -4430,10 +5256,231 @@ where requirements: request.requirements, writer: request.writer, }; - let result = store.commit_table(commit_request).await.map_err(catalog_store_error)?; + let result = publish_table_commit(store, metadata_backend, table_bucket_fence_required, commit_request).await?; Ok(commit_table_response_from_result(result, next_metadata)) } +async fn table_commit_for_retry( + store: &S, + bucket: &str, + table_id: &str, + request: &RestCommitTableRequest, +) -> S3Result> +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + table_commit_for_retry_ids(store, bucket, table_id, request.commit_id.as_deref(), request.idempotency_key.as_deref()).await +} + +async fn table_commit_for_retry_ids( + store: &S, + bucket: &str, + table_id: &str, + commit_id: Option<&str>, + idempotency_key: Option<&str>, +) -> S3Result> +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let by_commit_id = match commit_id { + Some(commit_id) => store + .get_commit_by_id(bucket, table_id, commit_id) + .await + .map_err(catalog_store_error)?, + None => None, + }; + let by_idempotency_key = match idempotency_key { + Some(idempotency_key) => store + .get_commit_by_idempotency_key(bucket, table_id, idempotency_key) + .await + .map_err(catalog_store_error)?, + None => None, + }; + if let (Some(by_commit_id), Some(by_idempotency_key)) = (&by_commit_id, &by_idempotency_key) + && !crate::table_catalog::commit_logs_share_recovery_payload(by_commit_id, by_idempotency_key) + { + return Err(iceberg_rest_error( + ICEBERG_ERROR_COMMIT_FAILED, + StatusCode::CONFLICT, + "commit id and idempotency key identify different commit payloads", + )); + } + Ok(by_commit_id.or(by_idempotency_key)) +} + +fn replay_commit_requirements( + commit: &crate::table_catalog::CommitLogEntry, + client_requirements: &[serde_json::Value], + target_metadata: &serde_json::Value, +) -> S3Result> { + if commit.requirements == client_requirements { + return Ok(client_requirements.to_vec()); + } + let mut requirements = client_requirements.to_vec(); + requirements.push(metadata_digest_requirement(target_metadata)?); + if commit.requirements != requirements { + return Err(iceberg_rest_error( + ICEBERG_ERROR_COMMIT_FAILED, + StatusCode::CONFLICT, + "commit retry requirements do not match the original commit", + )); + } + Ok(requirements) +} + +async fn commit_table_replay_response( + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + result: crate::table_catalog::TableCommitResult, + committed_metadata_location: &str, + committed_metadata: serde_json::Value, +) -> S3Result { + let metadata = if result.table.metadata_location == committed_metadata_location { + committed_metadata + } else { + let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; + if !crate::table_catalog::is_valid_table_metadata_location(namespace, &table_name, &result.table.metadata_location) { + return Err(iceberg_rest_error( + ICEBERG_ERROR_REST, + StatusCode::INTERNAL_SERVER_ERROR, + "persisted table metadata location is outside the protected table metadata directory", + )); + } + read_table_metadata_json(metadata_backend, bucket, &result.table.metadata_location).await? + }; + Ok(commit_table_response_from_result(result, metadata)) +} + +async fn replay_standard_table_commit( + store: &S, + metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend, + bucket: &str, + namespace: &crate::table_catalog::Namespace, + table: &str, + current: &crate::table_catalog::TableEntry, + request: &RestCommitTableRequest, +) -> S3Result> +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let Some(commit) = table_commit_for_retry(store, bucket, ¤t.table_id, request).await? else { + return Ok(None); + }; + if request + .commit_id + .as_deref() + .is_some_and(|commit_id| commit_id != commit.commit_id) + || request.idempotency_key != commit.idempotency_key + || request.writer != commit.writer + { + return Err(iceberg_rest_error( + ICEBERG_ERROR_COMMIT_FAILED, + StatusCode::CONFLICT, + "commit retry does not match the original request", + )); + } + + let previous_metadata = read_table_metadata_json(metadata_backend, bucket, &commit.previous_metadata_location).await?; + validate_metadata_table_location_in_bucket(bucket, &previous_metadata)?; + validate_table_commit_requirements(&previous_metadata, &request.requirements)?; + let target_metadata = read_table_metadata_json(metadata_backend, bucket, &commit.new_metadata_location).await?; + validate_metadata_table_location_in_bucket(bucket, &target_metadata)?; + let commit_timestamp_ms = target_metadata + .get("last-updated-ms") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| s3_error!(InvalidRequest, "committed metadata is missing last-updated-ms"))?; + let previous_metadata_location = table_metadata_location_for_client(bucket, &commit.previous_metadata_location); + let rebuilt_metadata = apply_table_commit_updates_at( + previous_metadata.clone(), + &request.updates, + &previous_metadata_location, + commit_timestamp_ms, + )?; + if rebuilt_metadata != target_metadata { + return Err(iceberg_rest_error( + ICEBERG_ERROR_COMMIT_FAILED, + StatusCode::CONFLICT, + "commit retry updates do not match the original commit", + )); + } + let operation = request + .operation + .clone() + .unwrap_or_else(|| table_commit_operation(&target_metadata)); + if operation != commit.operation { + return Err(iceberg_rest_error( + ICEBERG_ERROR_COMMIT_FAILED, + StatusCode::CONFLICT, + "commit retry operation does not match the original commit", + )); + } + if crate::table_catalog::table_matches_staged_base(current, &commit) { + let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; + validate_table_metadata_snapshot_graph_result( + metadata_backend, + bucket, + namespace, + &table_name, + current, + Some(&previous_metadata), + &target_metadata, + ) + .await + .map_err(|err| match err { + crate::table_catalog::TableCatalogStoreError::Invalid(message) => s3_error!(InvalidRequest, "{}", message), + err => catalog_store_error(err), + })?; + validate_table_snapshot_commit_conflicts( + metadata_backend, + bucket, + namespace, + &table_name, + current, + &previous_metadata, + &request.updates, + ) + .await?; + } + let requirements = replay_commit_requirements(&commit, &request.requirements, &target_metadata)?; + let committed_metadata_location = commit.new_metadata_location.clone(); + let table_bucket_fence_required = table_warehouse_location_changes(current, &target_metadata)?; + let result = publish_table_commit( + store, + metadata_backend, + table_bucket_fence_required, + crate::table_catalog::TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.to_string(), + commit_id: commit.commit_id, + idempotency_key: request.idempotency_key.clone(), + operation, + expected_version_token: commit.expected_version_token, + expected_metadata_location: commit.previous_metadata_location, + new_metadata_location: commit.new_metadata_location, + requirements, + writer: request.writer.clone(), + }, + ) + .await?; + Ok(Some( + commit_table_replay_response( + metadata_backend, + bucket, + namespace, + table, + result, + &committed_metadata_location, + target_metadata, + ) + .await?, + )) +} + async fn drop_table_in_store(store: &S, bucket: &str, namespace: &crate::table_catalog::Namespace, table: &str) -> S3Result<()> where S: crate::table_catalog::TableCatalogStore + ?Sized, @@ -4463,7 +5510,7 @@ async fn table_metadata_maintenance_response( request: TableMetadataMaintenanceRequest, ) -> S3Result where - B: crate::table_catalog::TableCatalogObjectBackend, + B: crate::table_catalog::TableCatalogObjectBackend + Clone, { if request.delete && request.commit_snapshot_expiration { return Err(s3_error!( @@ -4489,12 +5536,22 @@ where let compaction_request = request.compaction; let commit_compaction = request.commit_compaction; let compaction = match compaction_request { - Some(config) if commit_compaction => Some( - store - .commit_table_compaction(bucket, &namespace.public_name(), table, config) - .await - .map_err(catalog_store_error)?, - ), + Some(config) if commit_compaction => { + let publication_backend = TableCommitObjectBackend::preauthorized(metadata_backend.clone()); + Some( + store + .commit_table_compaction_with_publication( + &publication_backend, + &publication_backend, + bucket, + &namespace.public_name(), + table, + config, + ) + .await + .map_err(catalog_store_error)?, + ) + } Some(config) => Some( store .plan_table_compaction(bucket, &namespace.public_name(), table, config) @@ -4503,14 +5560,31 @@ where ), None => None, }; - let snapshot_expiration_plan = match snapshot_expiration_request { - Some(config) => Some( - store - .plan_table_snapshot_expiration(bucket, &namespace.public_name(), table, config) + let (snapshot_expiration_plan, snapshot_publication_backend) = match snapshot_expiration_request { + Some(config) if commit_snapshot_expiration => { + let publication_backend = TableCommitObjectBackend::preauthorized(metadata_backend.clone()); + let plan = store + .plan_table_snapshot_expiration_with_backend( + &publication_backend, + bucket, + &namespace.public_name(), + table, + config, + ) .await - .map_err(catalog_store_error)?, + .map_err(catalog_store_error)?; + (Some(plan), Some(publication_backend)) + } + Some(config) => ( + Some( + store + .plan_table_snapshot_expiration(bucket, &namespace.public_name(), table, config) + .await + .map_err(catalog_store_error)?, + ), + None, ), - None => None, + None => (None, None), }; let mut report = store .run_table_metadata_maintenance_with_retention( @@ -4523,12 +5597,15 @@ where ) .await .map_err(catalog_store_error)?; - let snapshot_expiration = match (snapshot_expiration_plan, commit_snapshot_expiration) { - (Some(plan), true) => { - Some(commit_table_snapshot_expiration_response(store, metadata_backend, bucket, namespace, table, plan).await?) + let snapshot_expiration = match (snapshot_expiration_plan, snapshot_publication_backend) { + (Some(plan), Some(publication_backend)) => { + Some(commit_table_snapshot_expiration_response(store, &publication_backend, bucket, namespace, table, plan).await?) + } + (Some(plan), None) => Some(plan), + (None, None) => None, + (None, Some(_)) => { + return Err(s3_error!(InternalError, "snapshot expiration publication state is missing its plan")); } - (Some(plan), false) => Some(plan), - (None, _) => None, }; report.snapshot_expiration = snapshot_expiration; report.compaction = compaction; @@ -4558,9 +5635,9 @@ where Ok(report) } -async fn commit_table_snapshot_expiration_response( +async fn commit_table_snapshot_expiration_response( store: &crate::table_catalog::ObjectTableCatalogStore, - metadata_backend: &B, + metadata_backend: &M, bucket: &str, namespace: &crate::table_catalog::Namespace, table: &str, @@ -4568,6 +5645,7 @@ async fn commit_table_snapshot_expiration_response( ) -> S3Result where B: crate::table_catalog::TableCatalogObjectBackend, + M: crate::table_catalog::TableCatalogObjectBackend, { let Some(current) = store .load_table(bucket, &namespace.public_name(), table) @@ -4631,6 +5709,7 @@ where ) .await .map_err(catalog_store_error)?; + let table_bucket_fence_required = table_warehouse_location_changes(¤t, &next_metadata)?; let commit_request = crate::table_catalog::TableCommitRequest { table_bucket: bucket.to_string(), @@ -4645,7 +5724,7 @@ where requirements: Vec::new(), writer: Some("rustfs-maintenance".to_string()), }; - let result = store.commit_table(commit_request).await.map_err(catalog_store_error)?; + let result = publish_table_commit(store, metadata_backend, table_bucket_fence_required, commit_request).await?; report.expired_snapshot_ids = expired_snapshot_ids; report.committed_metadata_location = Some(result.table.metadata_location); Ok(report) @@ -5081,9 +6160,9 @@ where Ok(external_catalog_bridge_response_from_entry(bucket, namespace, table, Some(entry))) } -async fn sync_external_catalog_bridge_response( +async fn sync_external_catalog_bridge_response( store: &crate::table_catalog::ObjectTableCatalogStore, - metadata_backend: &B, + metadata_backend: &M, bucket: &str, namespace: &crate::table_catalog::Namespace, table: &str, @@ -5092,6 +6171,7 @@ async fn sync_external_catalog_bridge_response( ) -> S3Result where B: crate::table_catalog::TableCatalogObjectBackend, + M: crate::table_catalog::TableCatalogObjectBackend, { let mut request = request; request.metadata_location = table_metadata_location_for_catalog(bucket, &request.metadata_location)?; @@ -5133,8 +6213,12 @@ where &target_metadata, ) .await?; - let result = store - .commit_table(crate::table_catalog::TableCommitRequest { + let table_bucket_fence_required = table_warehouse_location_changes(¤t, &target_metadata)?; + let result = publish_table_commit( + store, + metadata_backend, + table_bucket_fence_required, + crate::table_catalog::TableCommitRequest { table_bucket: bucket.to_string(), namespace: namespace.public_name(), table: table.to_string(), @@ -5146,9 +6230,9 @@ where new_metadata_location: request.metadata_location.clone(), requirements: Vec::new(), writer: Some(EXTERNAL_CATALOG_SYNC_WRITER.to_string()), - }) - .await - .map_err(catalog_store_error)?; + }, + ) + .await?; ( EXTERNAL_CATALOG_ACTION_COMMITTED.to_string(), load_table_response_from_entry(result.table, target_metadata), @@ -5172,7 +6256,10 @@ where adopt_registered_metadata_identity(&mut entry, &target_metadata)?; validate_table_metadata_snapshot_graph(metadata_backend, bucket, namespace, &table_name, &entry, None, &target_metadata) .await?; - store.register_table(entry.clone()).await.map_err(catalog_store_error)?; + store + .register_table_with_publication(entry.clone(), metadata_backend) + .await + .map_err(catalog_store_error)?; ( EXTERNAL_CATALOG_ACTION_REGISTERED.to_string(), load_table_response_from_entry(entry, target_metadata), @@ -5208,11 +6295,11 @@ where let result = async { ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?; let mut entry = table_entry_from_import_request(bucket, namespace, table, request)?; + let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) + .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; let metadata = read_table_metadata_json(metadata_backend, bucket, &entry.metadata_location).await?; validate_metadata_table_location_in_bucket(bucket, &metadata)?; adopt_registered_metadata_identity(&mut entry, &metadata)?; - let table_name = crate::table_catalog::IdentifierSegment::parse(table.to_string()) - .map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?; validate_table_metadata_snapshot_graph(metadata_backend, bucket, namespace, &table_name, &entry, None, &metadata).await?; if let Some(existing) = store .load_table(bucket, &namespace.public_name(), table) @@ -5230,7 +6317,10 @@ where "catalog import target already exists with different table identity or metadata pointer" )); } - store.register_table(entry.clone()).await.map_err(catalog_store_error)?; + store + .register_table_with_publication(entry.clone(), metadata_backend) + .await + .map_err(catalog_store_error)?; Ok(load_table_response_from_entry(entry, metadata)) } .await; @@ -5279,6 +6369,7 @@ where &target_metadata, ) .await?; + let table_bucket_fence_required = table_warehouse_location_changes(¤t, &target_metadata)?; let commit_request = crate::table_catalog::TableCommitRequest { table_bucket: bucket.to_string(), namespace: namespace.public_name(), @@ -5292,7 +6383,7 @@ where requirements: Vec::new(), writer: Some("rustfs-catalog-rollback-api".to_string()), }; - let result = store.commit_table(commit_request).await.map_err(catalog_store_error)?; + let result = publish_table_commit(store, metadata_backend, table_bucket_fence_required, commit_request).await?; Ok(commit_table_response_from_result(result, target_metadata)) } .await; diff --git a/rustfs/src/admin/handlers/table_catalog/namespace.rs b/rustfs/src/admin/handlers/table_catalog/namespace.rs index a956bde20..a7bf17661 100644 --- a/rustfs/src/admin/handlers/table_catalog/namespace.rs +++ b/rustfs/src/admin/handlers/table_catalog/namespace.rs @@ -26,8 +26,8 @@ impl Operation for RestListNamespacesHandler { None => TableCatalogResource::warehouse(&warehouse), }; authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableNamespaceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let response = list_namespaces_response(&store, &warehouse, parent.as_ref(), &req.uri).await?; build_json_response(StatusCode::OK, &response) } @@ -49,8 +49,8 @@ impl Operation for RestCreateNamespaceHandler { "namespace creation", ) .await?; - let store = table_catalog_store()?; - let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; + let table_bucket_enabled = table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let response = create_namespace_response(&store, &warehouse, request, table_bucket_enabled).await?; build_json_response(StatusCode::OK, &response) } @@ -65,8 +65,8 @@ impl Operation for RestGetNamespaceHandler { let namespace = namespace_from_params(¶ms)?; let resource = TableCatalogResource::namespace(&warehouse, &namespace); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableNamespaceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let response = get_namespace_response(&store, &warehouse, &namespace).await?; build_json_response(StatusCode::OK, &response) } @@ -81,8 +81,8 @@ impl Operation for RestDropNamespaceHandler { let namespace = namespace_from_params(¶ms)?; let resource = TableCatalogResource::namespace(&warehouse, &namespace); authorize_table_catalog_resource_request(&req, &resource, AdminAction::DeleteTableNamespaceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; drop_namespace_in_store(&store, &warehouse, &namespace.public_name()).await?; Ok(empty_response(StatusCode::NO_CONTENT)) } @@ -97,7 +97,7 @@ impl Operation for RestUpdateNamespacePropertiesHandler { let namespace = namespace_from_params(¶ms)?; let resource = TableCatalogResource::namespace(&warehouse, &namespace); authorize_table_catalog_resource_request(&req, &resource, AdminAction::UpdateTableNamespacePropertiesAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let request = read_bounded_json_body::( &req.headers, req.input, @@ -106,7 +106,7 @@ impl Operation for RestUpdateNamespacePropertiesHandler { "namespace properties", ) .await?; - let store = table_catalog_store()?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let response = update_namespace_properties_response(&store, &warehouse, &namespace, request).await?; build_json_response(StatusCode::OK, &response) } @@ -121,8 +121,8 @@ impl Operation for RestNamespaceExistsHandler { let namespace = namespace_from_params(¶ms)?; let resource = TableCatalogResource::namespace(&warehouse, &namespace); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableNamespaceAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; Ok(empty_response(namespace_exists_status(&store, &warehouse, &namespace).await?)) } } diff --git a/rustfs/src/admin/handlers/table_catalog/refs.rs b/rustfs/src/admin/handlers/table_catalog/refs.rs index 2d2569abd..db17b6156 100644 --- a/rustfs/src/admin/handlers/table_catalog/refs.rs +++ b/rustfs/src/admin/handlers/table_catalog/refs.rs @@ -24,8 +24,8 @@ impl Operation for ListTableRefsHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let metadata_backend = table_catalog_backend()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; let response = table_refs_response(&store, &metadata_backend, &warehouse, &namespace, &table).await?; build_json_response(StatusCode::OK, &response) @@ -36,19 +36,21 @@ pub struct PutTableRefHandler {} #[async_trait::async_trait] impl Operation for PutTableRefHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { let warehouse = warehouse_from_params(¶ms)?; let namespace = namespace_from_params(¶ms)?; let table = table_name_from_params(¶ms)?; let ref_name = ref_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; + let principal = authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; + install_table_catalog_s3_request_info(&mut req, &principal)?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let request = read_json_body::(std::mem::take(&mut req.input)).await?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = - put_table_ref_response(&store, &metadata_backend, &warehouse, &namespace, &table, &ref_name, request).await?; + let commit_backend = TableCommitObjectBackend::for_request(metadata_backend, req); + let result = put_table_ref_response(&store, &commit_backend, &warehouse, &namespace, &table, &ref_name, request).await; + let response = commit_backend.finish(result).await?; build_json_response(StatusCode::OK, &response) } } @@ -57,19 +59,21 @@ pub struct DeleteTableRefHandler {} #[async_trait::async_trait] impl Operation for DeleteTableRefHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { let warehouse = warehouse_from_params(¶ms)?; let namespace = namespace_from_params(¶ms)?; let table = table_name_from_params(¶ms)?; let ref_name = ref_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body_or_default::(req.input).await?; - let metadata_backend = table_catalog_backend()?; + let principal = authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; + install_table_catalog_s3_request_info(&mut req, &principal)?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let request = read_json_body_or_default::(std::mem::take(&mut req.input)).await?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = - delete_table_ref_response(&store, &metadata_backend, &warehouse, &namespace, &table, &ref_name, request).await?; + let commit_backend = TableCommitObjectBackend::for_request(metadata_backend, req); + let result = delete_table_ref_response(&store, &commit_backend, &warehouse, &namespace, &table, &ref_name, request).await; + let response = commit_backend.finish(result).await?; build_json_response(StatusCode::OK, &response) } } diff --git a/rustfs/src/admin/handlers/table_catalog/table.rs b/rustfs/src/admin/handlers/table_catalog/table.rs index 10471fd94..fd300a2a4 100644 --- a/rustfs/src/admin/handlers/table_catalog/table.rs +++ b/rustfs/src/admin/handlers/table_catalog/table.rs @@ -23,8 +23,8 @@ impl Operation for RestListTablesHandler { let namespace = namespace_from_params(¶ms)?; let resource = TableCatalogResource::namespace(&warehouse, &namespace); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let response = list_tables_response(&store, &warehouse, &namespace, &req.uri).await?; build_json_response(StatusCode::OK, &response) } @@ -40,11 +40,12 @@ impl Operation for RestCreateTableHandler { let resource = TableCatalogResource::namespace(&warehouse, &namespace); authorize_table_catalog_resource_request(&req, &resource, AdminAction::CreateTableAction).await?; let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; + let table_bucket_enabled = table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let commit_backend = TableCommitObjectBackend::preauthorized(metadata_backend); let response = - create_table_response(&store, &metadata_backend, &warehouse, &namespace, request, table_bucket_enabled).await?; + create_table_response(&store, &commit_backend, &warehouse, &namespace, request, table_bucket_enabled).await?; build_json_response(StatusCode::OK, &response) } } @@ -53,17 +54,20 @@ pub struct RestRegisterTableHandler {} #[async_trait::async_trait] impl Operation for RestRegisterTableHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { let warehouse = warehouse_from_params(¶ms)?; let namespace = namespace_from_params(¶ms)?; let resource = TableCatalogResource::namespace(&warehouse, &namespace); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; + let principal = authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?; + install_table_catalog_s3_request_info(&mut req, &principal)?; + let request = read_json_body::(std::mem::take(&mut req.input)).await?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; - let response = - register_table_response(&store, &metadata_backend, &warehouse, &namespace, request, table_bucket_enabled).await?; + let table_bucket_enabled = table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let commit_backend = TableCommitObjectBackend::for_request(metadata_backend, req); + let result = + register_table_response(&store, &commit_backend, &warehouse, &namespace, request, table_bucket_enabled).await; + let response = commit_backend.finish(result).await?; build_json_response(StatusCode::OK, &response) } } @@ -78,8 +82,8 @@ impl Operation for RestLoadTableHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let metadata_backend = table_catalog_backend()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; let response = load_table_response(&store, &metadata_backend, &warehouse, &namespace, &table).await?; build_json_response(StatusCode::OK, &response) @@ -96,8 +100,8 @@ impl Operation for RestTableExistsHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; Ok(empty_response(table_exists_status(&store, &warehouse, &namespace, &table).await?)) } } @@ -106,17 +110,20 @@ pub struct RestCommitTableHandler {} #[async_trait::async_trait] impl Operation for RestCommitTableHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { let warehouse = warehouse_from_params(¶ms)?; let namespace = namespace_from_params(¶ms)?; let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; + let principal = authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; + install_table_catalog_s3_request_info(&mut req, &principal)?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let request = read_json_body::(std::mem::take(&mut req.input)).await?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = commit_table_response(&store, &metadata_backend, &warehouse, &namespace, &table, request).await?; + let commit_backend = TableCommitObjectBackend::for_request(metadata_backend, req); + let result = commit_table_response(&store, &commit_backend, &warehouse, &namespace, &table, request).await; + let response = commit_backend.finish(result).await?; build_json_response(StatusCode::OK, &response) } } @@ -131,8 +138,8 @@ impl Operation for RestDropTableHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::DeleteTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; drop_table_in_store(&store, &warehouse, &namespace, &table).await?; Ok(empty_response(StatusCode::NO_CONTENT)) } @@ -148,8 +155,8 @@ impl Operation for GetTableMetadataLocationHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataLocationAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let response = get_table_metadata_location_response(&store, &warehouse, &namespace, &table).await?; build_json_response(StatusCode::OK, &response) } @@ -159,18 +166,22 @@ pub struct UpdateTableMetadataLocationHandler {} #[async_trait::async_trait] impl Operation for UpdateTableMetadataLocationHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { let warehouse = warehouse_from_params(¶ms)?; let namespace = namespace_from_params(¶ms)?; let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableMetadataLocationAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; + let principal = + authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableMetadataLocationAction).await?; + install_table_catalog_s3_request_info(&mut req, &principal)?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let request = read_json_body::(std::mem::take(&mut req.input)).await?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = - update_table_metadata_location_response(&store, &metadata_backend, &warehouse, &namespace, &table, request).await?; + let commit_backend = TableCommitObjectBackend::for_request(metadata_backend, req); + let result = + update_table_metadata_location_response(&store, &commit_backend, &warehouse, &namespace, &table, request).await; + let response = commit_backend.finish(result).await?; build_json_response(StatusCode::OK, &response) } } @@ -185,8 +196,8 @@ impl Operation for ExportTableCatalogHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let started = Instant::now(); let result = store .export_table_catalog_entry(&warehouse, &namespace.public_name(), &table) @@ -202,19 +213,21 @@ pub struct ImportTableCatalogHandler {} #[async_trait::async_trait] impl Operation for ImportTableCatalogHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { let warehouse = warehouse_from_params(¶ms)?; let namespace = namespace_from_params(¶ms)?; let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; + let principal = authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?; + install_table_catalog_s3_request_info(&mut req, &principal)?; + let request = read_json_body::(std::mem::take(&mut req.input)).await?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; - let response = - catalog_import_response(&store, &metadata_backend, &warehouse, &namespace, &table, request, table_bucket_enabled) - .await?; + let table_bucket_enabled = table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let commit_backend = TableCommitObjectBackend::for_request(metadata_backend, req); + let result = + catalog_import_response(&store, &commit_backend, &warehouse, &namespace, &table, request, table_bucket_enabled).await; + let response = commit_backend.finish(result).await?; build_json_response(StatusCode::OK, &response) } } @@ -229,8 +242,8 @@ impl Operation for GetTableCatalogDiagnosticsHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let config = store .get_table_maintenance_config(&warehouse, &namespace.public_name(), &table) .await @@ -263,8 +276,8 @@ impl Operation for RecoverTableCatalogHandler { let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let started = Instant::now(); let result = store .recover_table_commits(&warehouse, &namespace.public_name(), &table) @@ -280,17 +293,20 @@ pub struct RollbackTableCatalogHandler {} #[async_trait::async_trait] impl Operation for RollbackTableCatalogHandler { - async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { let warehouse = warehouse_from_params(¶ms)?; let namespace = namespace_from_params(¶ms)?; let table = table_name_from_params(¶ms)?; let resource = TableCatalogResource::table(&warehouse, &namespace, &table); - authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; + let principal = authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; + install_table_catalog_s3_request_info(&mut req, &principal)?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let request = read_json_body::(std::mem::take(&mut req.input)).await?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let response = rollback_table_response(&store, &metadata_backend, &warehouse, &namespace, &table, request).await?; + let commit_backend = TableCommitObjectBackend::for_request(metadata_backend, req); + let result = rollback_table_response(&store, &commit_backend, &warehouse, &namespace, &table, request).await; + let response = commit_backend.finish(result).await?; build_json_response(StatusCode::OK, &response) } } diff --git a/rustfs/src/admin/handlers/table_catalog/tests.rs b/rustfs/src/admin/handlers/table_catalog/tests.rs index 8b7ac85e2..da3a685c2 100644 --- a/rustfs/src/admin/handlers/table_catalog/tests.rs +++ b/rustfs/src/admin/handlers/table_catalog/tests.rs @@ -1,7 +1,128 @@ use super::*; +use crate::admin::runtime_sources::{AppContext, IamInterface, KmsInterface, ServerContextSlot}; use crate::table_catalog::{TableCatalogObjectBackend, TableCatalogStore}; +use datafusion::{ + arrow::{ + array::Int32Array, + datatypes::{DataType, Field, Schema, SchemaRef}, + record_batch::RecordBatch, + }, + parquet::arrow::ArrowWriter, +}; use std::sync::Arc; +use rustfs_iam::store::{Store as _, UserType}; +use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq}; + +struct RequestIam { + handle: Arc>, +} + +impl IamInterface for RequestIam { + fn handle(&self) -> Arc> { + self.handle.clone() + } + + fn is_ready(&self) -> bool { + true + } +} + +struct RequestKms; + +impl KmsInterface for RequestKms { + fn handle(&self) -> Arc { + Arc::new(rustfs_kms::KmsServiceManager::new()) + } +} + +#[tokio::test] +async fn table_catalog_authentication_and_credentials_use_the_request_context() { + let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await; + rustfs_iam::store::object::ObjectStore::new(store.clone()) + .save_iam_config( + serde_json::json!({"version": 1}), + format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX), + ) + .await + .expect("request IAM format should be seeded"); + let iam = rustfs_iam::build_iam_sys(store.clone()) + .await + .expect("request IAM should initialize"); + let user_access_key = "request-table-user"; + let user_secret_key = "request-table-user-secret-key"; + let policy_name = "request-table-get"; + iam.create_user( + user_access_key, + &AddOrUpdateUserReq { + secret_key: user_secret_key.to_string(), + policy: None, + status: AccountStatus::Enabled, + }, + ) + .await + .expect("request IAM user should be created"); + iam.set_policy( + policy_name, + Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:GetTable"]}]}"#) + .expect("request table policy should parse"), + ) + .await + .expect("request table policy should be stored"); + iam.policy_db_set(user_access_key, UserType::Reg, false, policy_name) + .await + .expect("request table policy should be attached"); + let context = Arc::new(AppContext::new( + store.clone(), + Arc::new(RequestIam { handle: iam.clone() }), + Arc::new(RequestKms), + )); + let credentials = rustfs_credentials::Credentials { + access_key: "request-root-access-key".to_string(), + secret_key: "request-root-secret-key".to_string(), + status: "on".to_string(), + ..Default::default() + }; + assert!(context.publish_action_credentials(credentials.clone())); + let slot = ServerContextSlot::new(); + assert!(slot.install(context)); + let mut extensions = http::Extensions::new(); + extensions.insert(slot); + let request = S3Request { + input: Body::empty(), + method: Method::GET, + uri: "/table-context".parse().expect("test URI"), + headers: HeaderMap::new(), + extensions, + credentials: Some(s3s::auth::Credentials { + access_key: user_access_key.to_string(), + secret_key: s3s::auth::SecretKey::from(user_secret_key.to_string()), + }), + region: None, + service: None, + trailing_headers: None, + }; + + let principal = table_catalog_request_principal(&request) + .await + .expect("request IAM credentials should authenticate"); + assert!(!principal.owner); + assert!(Arc::ptr_eq(&principal.iam_store, &iam)); + authorize_table_catalog_request(&request, AdminAction::GetTableAction) + .await + .expect("unscoped authorization should use the request IAM"); + let resource = TableCatalogResource::warehouse("request-warehouse"); + authorize_table_catalog_resource_request(&request, &resource, AdminAction::GetTableAction) + .await + .expect("resource authorization should use the request IAM"); + let issuer = IamTableCredentialIssuer::from_request(&request).expect("credential issuer should use the request context"); + assert!(Arc::ptr_eq(&issuer.iam_store, &iam)); + assert_eq!(issuer.token_signing_key.as_deref(), Some("request-root-secret-key")); + let resolved_store = + runtime_sources::object_store_from_extensions(&request.extensions).expect("request object store should resolve"); + assert!(Arc::ptr_eq(&resolved_store, &store)); +} + #[test] #[serial_test::serial] fn catalog_config_response_lists_standard_rest_endpoints() { @@ -228,7 +349,7 @@ fn table_catalog_handlers_require_table_admin_actions() { .contains("authorize_table_catalog_request(&req, AdminAction::GetTableCatalogAction).await?;") ); assert!( - src.contains("validate_admin_request_with_bucket_object("), + src.contains("validate_admin_action_with_bucket_object_for_iam("), "catalog resource auth should pass namespace/table scope into IAM object matching" ); @@ -279,7 +400,7 @@ fn table_catalog_handlers_require_table_admin_actions() { ] { let block = operation_block(&src, handler); assert!( - block.contains(&format!("authorize_table_catalog_resource_request(&req, &resource, {action}).await?;")), + block.contains(&format!("authorize_table_catalog_resource_request(&req, &resource, {action})")), "{handler} should require {action} with catalog resource auth" ); assert!( @@ -353,12 +474,24 @@ fn table_catalog_handlers_require_table_admin_actions() { "{handler} should build a table-aware catalog resource" ); assert!( - block.contains(&format!("authorize_table_catalog_resource_request(&req, &resource, {action}).await?;")), + block.contains(&format!("authorize_table_catalog_resource_request(&req, &resource, {action})")), "{handler} should authorize against the table-aware catalog resource" ); } } +#[test] +fn table_bucket_handlers_resolve_state_from_the_request_context() { + let src = table_catalog_handler_source(); + let enable = operation_block(&src, "EnableTableBucketHandler"); + assert!(enable.contains("table_catalog_backend_from_extensions(&req.extensions)?;")); + assert!(enable.contains("runtime_sources::object_store_from_req(&req)")); + + let get = operation_block(&src, "GetTableBucketHandler"); + assert!(get.contains("table_catalog_store_from_extensions(&req.extensions)?;")); + assert!(get.contains("table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;")); +} + #[test] fn table_catalog_list_handlers_parse_standard_pagination() { let src = table_catalog_handler_source(); @@ -396,6 +529,35 @@ fn namespace_write_handlers_bound_request_bodies() { } } +#[test] +fn table_pointer_write_handlers_install_commit_publication_guard() { + let src = table_catalog_handler_source(); + for handler in [ + "RestRegisterTableHandler", + "RestCommitTableHandler", + "UpdateTableMetadataLocationHandler", + "ImportTableCatalogHandler", + "PutTableRefHandler", + "DeleteTableRefHandler", + "RollbackTableCatalogHandler", + "SyncExternalCatalogBridgeHandler", + ] { + let block = operation_block(&src, handler); + assert!( + block.contains("install_table_catalog_s3_request_info(&mut req, &principal)?;"), + "{handler} should install exact-key S3 authorization context" + ); + assert!( + block.contains("TableCommitObjectBackend::for_request(metadata_backend, req)"), + "{handler} should use the guarded table commit backend" + ); + assert!( + block.contains("commit_backend.finish(result).await?"), + "{handler} should preserve exact-key authorization errors" + ); + } +} + #[test] fn table_catalog_handlers_require_enabled_table_bucket_marker_before_catalog_state() { let src = table_catalog_handler_source(); @@ -449,27 +611,30 @@ fn table_catalog_handlers_require_enabled_table_bucket_marker_before_catalog_sta ] { let block = operation_block(&src, handler); assert!( - block.contains("ensure_table_bucket_enabled(&warehouse).await?;") - || block.contains("table_bucket_enabled_from_metadata(&warehouse).await?;"), + block.contains("ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;") + || block.contains("table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;"), "{handler} should require the table bucket metadata marker before catalog state access" ); } } #[test] -fn enable_table_bucket_response_writes_metadata_marker_before_catalog_entry() { +fn enable_table_bucket_response_fences_before_marker_and_catalog_entry() { let src = table_catalog_handler_source(); let block = function_block(&src, "async fn enable_table_bucket_response"); + let publication_fence = block + .find("TableCommitPublication::begin_table_bucket(publication, bucket)") + .expect("enable should acquire the table-bucket publication fence"); let marker_write = block - .find("enable_table_bucket_marker(bucket).await?;") + .find("enable_table_bucket_marker(object_store, bucket).await?;") .expect("enable should write the metadata marker"); let catalog_entry_write = block .find("ensure_table_bucket_entry(store, bucket, true).await?;") .expect("enable should write the catalog entry"); assert!( - marker_write < catalog_entry_write, - "enable should write the metadata marker before the catalog entry" + publication_fence < marker_write && marker_write < catalog_entry_write, + "enable should fence object mutations before writing the metadata marker and catalog entry" ); } @@ -1787,7 +1952,8 @@ async fn create_table_response_writes_initial_metadata_for_standard_request() { })) .expect("standard create table request should parse"); - let response = create_table_response(&store, &metadata_backend, "warehouse", &namespace, request, true) + let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let response = create_table_response(&store, &commit_backend, "warehouse", &namespace, request, true) .await .expect("table should be created"); @@ -1820,6 +1986,88 @@ async fn create_table_response_writes_initial_metadata_for_standard_request() { ); } +#[tokio::test] +async fn create_table_holds_bucket_fence_from_metadata_write_through_registration() { + let store = Arc::new(TestTableCatalogStore::default()); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let metadata_backend = TestTableCatalogObjectBackend { + put_object_barrier: Some(Arc::clone(&barrier)), + ..Default::default() + }; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(store.as_ref(), "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + store.as_ref(), + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let request = serde_json::from_value::(serde_json::json!({ + "name": "events", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}] + } + })) + .expect("create table request should parse"); + + let create_store = Arc::clone(&store); + let create_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let create_namespace = namespace.clone(); + let create = tokio::spawn(async move { + create_table_response(create_store.as_ref(), &create_backend, "warehouse", &create_namespace, request, true).await + }); + tokio::time::timeout(StdDuration::from_secs(2), async { + while metadata_backend.objects.lock().await.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("initial metadata write should reach its publication pause"); + + let bucket_lock = crate::table_catalog::default_table_bucket_publication_lock_path(); + assert!( + metadata_backend.write_lock_is_held("warehouse", &bucket_lock).await, + "first publication must fence data-plane writers before initial metadata is visible" + ); + metadata_backend.lock_attempts.lock().await.clear(); + let writer_backend = metadata_backend.clone(); + let writer_lock = bucket_lock.clone(); + let writer = tokio::spawn(async move { + crate::table_catalog::TableCatalogObjectBackend::acquire_read_lock(&writer_backend, "warehouse", &writer_lock).await + }); + metadata_backend.wait_for_lock_attempts(1).await; + assert!(!writer.is_finished(), "a first-publication writer must wait until registration finishes"); + + barrier.wait().await; + tokio::time::timeout(StdDuration::from_secs(2), create) + .await + .expect("table creation should complete") + .expect("table creation task should join") + .expect("table creation should succeed"); + tokio::time::timeout(StdDuration::from_secs(2), writer) + .await + .expect("writer should continue after registration") + .expect("writer task should join") + .expect("writer lock acquisition should succeed"); + assert!( + store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .is_some(), + "table must become visible before the bucket fence is released" + ); +} + #[tokio::test] async fn create_table_response_recreates_dropped_identifier_without_overwriting_retained_metadata() { let store = TestTableCatalogStore::default(); @@ -1848,9 +2096,16 @@ async fn create_table_response_recreates_dropped_identifier_without_overwriting_ ] })) .expect("standard commit request should parse"); - standard_commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) - .await - .expect("first table metadata commit should succeed"); + standard_commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + commit_request, + ) + .await + .expect("first table metadata commit should succeed"); let first_committed_entry = store .load_table("warehouse", "analytics", "events") .await @@ -1891,7 +2146,8 @@ async fn create_table_response_recreates_dropped_identifier_without_overwriting_ } })) .expect("recreate table request should parse"); - let second = create_table_response(&store, &metadata_backend, "warehouse", &namespace, create_request, true) + let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let second = create_table_response(&store, &commit_backend, "warehouse", &namespace, create_request, true) .await .expect("dropped table identifier should be reusable"); let second_entry = store @@ -1933,10 +2189,7 @@ async fn create_table_response_recreates_dropped_identifier_without_overwriting_ async fn concurrent_create_table_responses_keep_one_catalog_winner_with_distinct_metadata() { let catalog_backend = TestTableCatalogObjectBackend::default(); let store = crate::table_catalog::ObjectTableCatalogStore::new(catalog_backend); - let metadata_backend = TestTableCatalogObjectBackend { - objects: Arc::new(tokio::sync::Mutex::new(BTreeMap::new())), - put_object_barrier: Some(Arc::new(tokio::sync::Barrier::new(2))), - }; + let metadata_backend = TestTableCatalogObjectBackend::default(); let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); ensure_table_bucket_entry(&store, "warehouse", true) .await @@ -1971,9 +2224,11 @@ async fn concurrent_create_table_responses_keep_one_catalog_winner_with_distinct .expect("concurrent create table request should parse") }; + let first_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let second_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); let (first, second) = tokio::join!( - create_table_response(&store, &metadata_backend, "warehouse", &namespace, create_request(), true,), - create_table_response(&store, &metadata_backend, "warehouse", &namespace, create_request(), true,) + create_table_response(&store, &first_backend, "warehouse", &namespace, create_request(), true,), + create_table_response(&store, &second_backend, "warehouse", &namespace, create_request(), true,) ); assert_ne!(first.is_ok(), second.is_ok(), "exactly one concurrent create should succeed"); let (winner, loser) = match (first, second) { @@ -2082,9 +2337,16 @@ async fn standard_commit_applies_updates_and_writes_next_metadata() { })) .expect("standard commit table request should parse"); - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) - .await - .expect("standard commit should succeed"); + let commit = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + commit_request, + ) + .await + .expect("standard commit should succeed"); let metadata_file_prefix = "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-"; @@ -2101,6 +2363,7 @@ async fn standard_commit_applies_updates_and_writes_next_metadata() { assert_eq!(commit.metadata["last-sequence-number"], 1); assert_eq!(commit.metadata["refs"]["main"]["snapshot-id"], 10); assert_eq!(commit.metadata["metadata-log"][0]["metadata-file"], created.metadata_location); + assert_eq!(commit.metadata["metadata-log"][0]["timestamp-ms"], created.metadata["last-updated-ms"]); let committed = store .load_table("warehouse", "analytics", "events") .await @@ -2138,9 +2401,16 @@ async fn standard_commit_uses_client_uuid_commit_id_in_metadata_file_name() { ] })) .expect("standard commit table request should parse"); - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) - .await - .expect("standard commit should succeed"); + let commit = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + commit_request, + ) + .await + .expect("standard commit should succeed"); assert_eq!(commit.commit_id, commit_id); assert_eq!( @@ -2179,9 +2449,16 @@ async fn standard_commit_accepts_non_uuid_client_commit_id_without_using_it_in_m ] })) .expect("standard commit table request should parse"); - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) - .await - .expect("standard commit should succeed"); + let commit = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + commit_request, + ) + .await + .expect("standard commit should succeed"); let metadata_file_prefix = "s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00002-"; @@ -2196,6 +2473,1141 @@ async fn standard_commit_accepts_non_uuid_client_commit_id_without_using_it_in_m assert_eq!(metadata_file_token, table_catalog_path_hash("commit-1")); } +#[tokio::test] +async fn commit_publication_uses_idempotency_key_as_retry_identity() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + create_standard_events_table(&store, &metadata_backend, &namespace).await; + let idempotency_key = Uuid::now_v7().to_string(); + let request = serde_json::from_value(serde_json::json!({ + "idempotency-key": idempotency_key, + "requirements": [], + "updates": [{ + "action": "set-properties", + "updates": {"owner": "lakehouse"} + }] + })) + .expect("standard commit request should parse"); + + let commit = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + request, + ) + .await + .expect("standard commit should succeed"); + + assert_eq!(commit.commit_id, idempotency_key); + assert!(commit.metadata_location.contains(&idempotency_key)); +} + +#[tokio::test] +async fn commit_publication_replays_historical_standard_commit_across_backings() { + for mode in [ + crate::table_catalog::TableCatalogBackingMode::ObjectBacked, + crate::table_catalog::TableCatalogBackingMode::DurableStrong, + ] { + let metadata_backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ConfiguredTableCatalogStore::new(metadata_backend.clone(), mode); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + create_standard_events_table(&store, &metadata_backend, &namespace).await; + let first_request = serde_json::json!({ + "commit-id": "commit-a", + "idempotency-key": "request-a", + "writer": "test-client", + "requirements": [], + "updates": [{ + "action": "set-properties", + "updates": {"owner": "first"} + }] + }); + let first = standard_commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + serde_json::from_value(first_request.clone()).expect("first commit request should parse"), + ) + .await + .expect("first commit should update the table"); + + if mode == crate::table_catalog::TableCatalogBackingMode::ObjectBacked { + let current = store + .load_table("warehouse", "analytics", "events") + .await + .expect("first committed table lookup should succeed") + .expect("first committed table should exist"); + let mut historical = store + .get_commit_by_id("warehouse", ¤t.table_id, "commit-a") + .await + .expect("historical commit lookup should succeed") + .expect("historical commit should exist"); + historical.requirements = vec![metadata_digest_requirement(&first.metadata).expect("metadata digest should build")]; + let paths = crate::table_catalog::TableCatalogObjectPaths::default(); + let commit_path = paths.commit_log_entry_path("warehouse", ¤t.table_id, "commit-a"); + let idempotency_path = paths.commit_idempotency_entry_path("warehouse", ¤t.table_id, "request-a"); + let historical = serde_json::to_value(historical).expect("historical commit should serialize"); + for path in [commit_path, idempotency_path] { + metadata_backend + .put_json(crate::admin::storage_api::RUSTFS_META_BUCKET, &path, historical.clone()) + .await; + } + } + + let store = crate::table_catalog::ConfiguredTableCatalogStore::new(metadata_backend.clone(), mode); + let second = standard_commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + serde_json::from_value(serde_json::json!({ + "commit-id": "commit-b", + "idempotency-key": "request-b", + "requirements": [], + "updates": [{ + "action": "set-properties", + "updates": {"owner": "current"} + }] + })) + .expect("second commit request should parse"), + ) + .await + .expect("second commit should advance the table"); + let current = store + .load_table("warehouse", "analytics", "events") + .await + .expect("current table lookup should succeed") + .expect("current table should exist"); + + let replay = standard_commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + serde_json::from_value(first_request.clone()).expect("first commit retry should parse"), + ) + .await + .expect("an exact old commit retry should succeed"); + + assert_eq!(replay.commit_id, "commit-a", "{mode:?}"); + assert_eq!(replay.metadata_location, second.metadata_location, "{mode:?}"); + assert_eq!(replay.version_token, second.version_token, "{mode:?}"); + assert_eq!(replay.generation, second.generation, "{mode:?}"); + assert_eq!(replay.metadata["properties"]["owner"], "current", "{mode:?}"); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup after replay should succeed") + .expect("table should remain present"); + assert_eq!(unchanged.metadata_location, current.metadata_location, "{mode:?}"); + assert_eq!(unchanged.version_token, current.version_token, "{mode:?}"); + + let mut mutated_request = first_request; + mutated_request["updates"][0]["updates"]["owner"] = serde_json::Value::String("mutated".to_string()); + let error = standard_commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + serde_json::from_value(mutated_request).expect("mutated retry should parse"), + ) + .await + .expect_err("mutating updates under a persisted idempotency key must conflict"); + assert_eq!(error.status_code(), Some(StatusCode::CONFLICT), "{mode:?}"); + } +} + +#[tokio::test] +async fn staged_standard_commit_retry_revalidates_referenced_objects() { + let metadata_backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(metadata_backend.clone()); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let current = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let table_name = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; + let commit_id = "11111111-1111-4111-8111-111111111111"; + let request: RestCommitTableRequest = serde_json::from_value(serde_json::json!({ + "commit-id": commit_id, + "requirements": [], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("standard commit request should parse"); + let previous_metadata = read_table_metadata_json(&metadata_backend, "warehouse", ¤t.metadata_location) + .await + .expect("current metadata should load"); + let target_metadata = apply_table_commit_updates_at( + previous_metadata, + &request.updates, + &table_metadata_location_for_client("warehouse", ¤t.metadata_location), + 2000, + ) + .expect("target metadata should build"); + let (_, metadata_file_token) = standard_commit_ids(Some(commit_id.to_string())); + let target_location = crate::table_catalog::default_table_metadata_file_path( + &namespace, + &table_name, + &next_metadata_file_name(current.generation.saturating_add(1), &metadata_file_token), + ); + metadata_backend + .put_json("warehouse", &target_location, target_metadata.clone()) + .await; + let staged = crate::table_catalog::CommitLogEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + commit_id: commit_id.to_string(), + idempotency_key: None, + table_id: current.table_id.clone(), + operation: table_commit_operation(&target_metadata), + expected_version_token: current.version_token.clone(), + new_version_token: "staged-token".to_string(), + previous_metadata_location: current.metadata_location.clone(), + new_metadata_location: target_location, + requirements: Vec::new(), + status: crate::table_catalog::CommitLogStatus::Staged, + writer: None, + created_at: None, + updated_at: None, + }; + let commit_path = + crate::table_catalog::TableCatalogObjectPaths::default().commit_log_entry_path("warehouse", ¤t.table_id, commit_id); + metadata_backend + .put_json( + crate::admin::storage_api::RUSTFS_META_BUCKET, + &commit_path, + serde_json::to_value(staged).expect("staged commit should serialize"), + ) + .await; + let data_object = test_snapshot_object_key("warehouse", &data_file); + crate::table_catalog::TableCatalogObjectBackend::delete_object(&metadata_backend, "warehouse", &data_object) + .await + .expect("referenced data object should be removed before retry"); + let authorized = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let commit_backend = TableCommitObjectBackend::test(metadata_backend, Arc::clone(&authorized), None); + + let result = standard_commit_table_response(&store, &commit_backend, "warehouse", &namespace, "events", request).await; + commit_backend + .finish(result) + .await + .expect_err("a staged retry must reject a now-missing referenced object"); + + assert!(authorized.lock().await.contains(&(data_object, S3Action::GetObjectAction))); + let unchanged = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should remain"); + assert_eq!(unchanged.metadata_location, current.metadata_location); + assert_eq!(unchanged.version_token, current.version_token); +} + +#[tokio::test] +async fn commit_publication_denies_generated_metadata_write_before_pointer_advance() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + create_standard_events_table(&store, &metadata_backend, &namespace).await; + let before = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let commit_id = "11111111-1111-4111-8111-111111111111"; + let table_name = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let next_metadata_location = crate::table_catalog::default_table_metadata_file_path( + &namespace, + &table_name, + &next_metadata_file_name(before.generation.saturating_add(1), commit_id), + ); + let request = serde_json::from_value(serde_json::json!({ + "commit-id": commit_id, + "requirements": [], + "updates": [{ + "action": "set-properties", + "updates": {"owner": "lakehouse"} + }] + })) + .expect("commit request should parse"); + let authorized = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let commit_backend = + TableCommitObjectBackend::test(metadata_backend.clone(), Arc::clone(&authorized), Some(next_metadata_location.clone())); + + let result = commit_table_response(&store, &commit_backend, "warehouse", &namespace, "events", request).await; + let error = commit_backend + .finish(result) + .await + .expect_err("denied metadata write should fail the commit"); + + assert_eq!(error.code(), &S3ErrorCode::AccessDenied); + assert!( + authorized + .lock() + .await + .contains(&(next_metadata_location.clone(), S3Action::PutObjectAction)) + ); + let after = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should remain"); + assert_eq!(after.metadata_location, before.metadata_location); + assert_eq!(after.version_token, before.version_token); + assert_eq!(after.generation, before.generation); + assert!( + !metadata_backend + .object_exists("warehouse", &next_metadata_location) + .await + .expect("metadata object lookup should succeed") + ); +} + +#[tokio::test] +async fn commit_publication_authorizes_referenced_objects() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let entry = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let manifest = format!("{table_location}/metadata/manifest-snap-10.avro"); + let data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; + let request = serde_json::from_value(serde_json::json!({ + "commit-id": "22222222-2222-4222-8222-222222222222", + "requirements": [], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("standard commit request should parse"); + let authorized = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let commit_backend = TableCommitObjectBackend::test(metadata_backend.clone(), Arc::clone(&authorized), None); + + let result = commit_table_response(&store, &commit_backend, "warehouse", &namespace, "events", request).await; + commit_backend + .finish(result) + .await + .expect("authorized standard commit should succeed"); + + let authorizations = authorized.lock().await; + let expected_reads = [ + entry.metadata_location, + test_snapshot_object_key("warehouse", &manifest_list), + test_snapshot_object_key("warehouse", &manifest), + test_snapshot_object_key("warehouse", &data_file), + ]; + for object in expected_reads { + assert!( + authorizations + .iter() + .any(|(authorized_object, action)| authorized_object == &object && *action == S3Action::GetObjectAction), + "missing GetObject authorization for {object}" + ); + } +} + +#[tokio::test] +async fn commit_publication_denies_referenced_data_read_before_pointer_advance() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let before = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; + let request = serde_json::from_value(serde_json::json!({ + "commit-id": "33333333-3333-4333-8333-333333333333", + "requirements": [], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("standard commit request should parse"); + let denied_object = test_snapshot_object_key("warehouse", &data_file); + let authorized = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let commit_backend = TableCommitObjectBackend::test(metadata_backend, Arc::clone(&authorized), Some(denied_object.clone())); + + let result = commit_table_response(&store, &commit_backend, "warehouse", &namespace, "events", request).await; + let error = commit_backend + .finish(result) + .await + .expect_err("denied referenced data read should fail the commit"); + + assert_eq!(error.code(), &S3ErrorCode::AccessDenied); + assert!(authorized.lock().await.contains(&(denied_object, S3Action::GetObjectAction))); + let after = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should remain"); + assert_eq!(after.metadata_location, before.metadata_location); + assert_eq!(after.version_token, before.version_token); + assert_eq!(after.generation, before.generation); +} + +#[tokio::test] +async fn commit_publication_holds_referenced_object_locks_until_pointer_publish() { + let pause = TestCatalogPublishPause::default(); + let store = Arc::new(TestTableCatalogStore { + commit_table_pause: Some(pause.clone()), + ..Default::default() + }); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(store.as_ref(), &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should have table location"); + let manifest_list = format!("{table_location}/metadata/snap-10.avro"); + let manifest = format!("{table_location}/metadata/manifest-snap-10.avro"); + let data_file = format!("{table_location}/data/part-10.parquet"); + seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; + let request = serde_json::from_value(serde_json::json!({ + "requirements": [], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 10, + "type": "branch" + } + ] + })) + .expect("standard commit request should parse"); + let commit_store = Arc::clone(&store); + let commit_namespace = namespace.clone(); + let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let commit = tokio::spawn(async move { + let result = + commit_table_response(commit_store.as_ref(), &commit_backend, "warehouse", &commit_namespace, "events", request) + .await; + commit_backend.finish(result).await + }); + + tokio::time::timeout(StdDuration::from_secs(2), pause.wait_started()) + .await + .expect("commit should reach catalog publication"); + let table_name = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let publication_lock = crate::table_catalog::default_table_publication_lock_path(&namespace, &table_name); + assert!( + metadata_backend.write_lock_is_held("warehouse", &publication_lock).await, + "table publication fence must remain held during catalog publication" + ); + assert!( + !metadata_backend + .write_lock_is_held("warehouse", &crate::table_catalog::default_table_bucket_publication_lock_path(),) + .await, + "same-warehouse commits must not serialize unrelated tables through the table-bucket fence" + ); + for location in [&manifest_list, &manifest] { + let object = test_snapshot_object_key("warehouse", location); + assert!( + metadata_backend.write_lock_is_held("warehouse", &object).await, + "snapshot metadata lock must remain held during catalog publication: {location}" + ); + } + let data_object = test_snapshot_object_key("warehouse", &data_file); + assert!( + !metadata_backend.write_lock_is_held("warehouse", &data_object).await, + "live data files must not consume one retained publication lock per object" + ); + metadata_backend.lock_attempts.lock().await.clear(); + let data_plane_backend = metadata_backend.clone(); + let data_plane_publication_lock = publication_lock.clone(); + let data_plane = tokio::spawn(async move { + crate::table_catalog::TableCatalogObjectBackend::acquire_read_lock( + &data_plane_backend, + "warehouse", + &data_plane_publication_lock, + ) + .await + }); + metadata_backend.wait_for_lock_attempts(1).await; + assert!(!data_plane.is_finished(), "a data-plane mutation must wait for catalog publication"); + pause.release(); + tokio::time::timeout(StdDuration::from_secs(2), commit) + .await + .expect("commit task should complete") + .expect("commit task should join") + .expect("standard commit should succeed"); + tokio::time::timeout(StdDuration::from_secs(2), data_plane) + .await + .expect("data-plane publication guard should unblock") + .expect("data-plane guard task should join") + .expect("data-plane guard acquisition should succeed"); + assert!( + !metadata_backend.write_lock_is_held("warehouse", &publication_lock).await, + "table publication fence must be released after catalog publication" + ); + for location in [&manifest_list, &manifest] { + let object = test_snapshot_object_key("warehouse", location); + assert!( + !metadata_backend.write_lock_is_held("warehouse", &object).await, + "snapshot metadata lock must be released after catalog publication: {location}" + ); + } +} + +#[tokio::test] +async fn rolling_upgrade_commit_retains_legacy_data_file_guard_until_publication_completes() { + let metadata_backend = TestTableCatalogObjectBackend::default(); + let data_file = "tables/table-id/data/part-00001.parquet"; + metadata_backend.put_bytes("warehouse", data_file, b"data".to_vec()).await; + let commit_backend = TableCommitObjectBackend::rolling_upgrade(metadata_backend.clone()); + assert!( + crate::table_catalog::TableCatalogObjectBackend::object_exists(&commit_backend, "warehouse", data_file) + .await + .expect("data-file discovery should succeed") + ); + + crate::table_catalog::TableCatalogObjectBackend::prepare_table_commit_publication( + &commit_backend, + "warehouse", + "analytics", + "events", + ) + .await + .expect("rolling-upgrade publication should prepare"); + assert!( + metadata_backend.write_lock_is_held("warehouse", data_file).await, + "old-node object writers must remain fenced until every node uses table publication locks" + ); + + let old_writer_backend = metadata_backend.clone(); + let old_writer = tokio::spawn(async move { + crate::table_catalog::TableCatalogObjectBackend::acquire_write_lock(&old_writer_backend, "warehouse", data_file).await + }); + metadata_backend.wait_for_lock_attempts(3).await; + assert!(!old_writer.is_finished(), "an old-node object writer must wait for catalog publication"); + + crate::table_catalog::TableCatalogObjectBackend::complete_table_commit_publication(&commit_backend); + tokio::time::timeout(StdDuration::from_secs(2), old_writer) + .await + .expect("old-node writer should continue after publication") + .expect("old-node writer task should join") + .expect("old-node writer lock acquisition should succeed"); +} + +#[tokio::test] +async fn rolling_upgrade_initial_publication_fences_old_and_new_data_plane_writers() { + let pause = TestCatalogPublishPause::default(); + let store = Arc::new(TestTableCatalogStore { + register_table_pause: Some(pause.clone()), + ..Default::default() + }); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(store.as_ref(), "warehouse", true) + .await + .expect("table bucket entry should seed"); + create_namespace_response( + store.as_ref(), + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should seed"); + + let data_file = "tables/table-id/data/part-00001.parquet"; + let metadata_backend = TestTableCatalogObjectBackend::default(); + metadata_backend.put_bytes("warehouse", data_file, b"data".to_vec()).await; + let publication_backend = TableCommitObjectBackend::rolling_upgrade(metadata_backend.clone()); + assert!( + crate::table_catalog::TableCatalogObjectBackend::object_exists(&publication_backend, "warehouse", data_file) + .await + .expect("data-file discovery should succeed") + ); + let entry = crate::table_catalog::TableEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: "warehouse".to_string(), + namespace: namespace.public_name(), + table: "events".to_string(), + table_id: "table-id".to_string(), + table_uuid: "table-uuid".to_string(), + format: "ICEBERG".to_string(), + format_version: 2, + warehouse_location: "s3://warehouse/tables/table-id".to_string(), + metadata_location: "s3://warehouse/tables/table-id/metadata/v1.metadata.json".to_string(), + version_token: "token-v1".to_string(), + generation: 1, + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }; + let register_store = Arc::clone(&store); + let register_backend = publication_backend.clone(); + let register = tokio::spawn(async move { register_store.register_table_with_publication(entry, ®ister_backend).await }); + + tokio::time::timeout(StdDuration::from_secs(2), pause.wait_started()) + .await + .expect("registration should reach catalog publication"); + let bucket_publication_lock = crate::table_catalog::default_table_bucket_publication_lock_path(); + assert!( + metadata_backend + .write_lock_is_held("warehouse", &bucket_publication_lock) + .await, + "initial publication must retain the bucket fence before the table entry is visible" + ); + assert!( + metadata_backend.write_lock_is_held("warehouse", data_file).await, + "rolling-upgrade publication must retain the exact object guard for old-node writers" + ); + assert!( + store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .is_none(), + "the table must still be invisible while the registration is paused" + ); + + metadata_backend.lock_attempts.lock().await.clear(); + let new_writer_backend = metadata_backend.clone(); + let new_writer_lock = bucket_publication_lock.clone(); + let new_writer = tokio::spawn(async move { + crate::table_catalog::TableCatalogObjectBackend::acquire_read_lock(&new_writer_backend, "warehouse", &new_writer_lock) + .await + }); + let old_writer_backend = metadata_backend.clone(); + let old_writer = tokio::spawn(async move { + crate::table_catalog::TableCatalogObjectBackend::acquire_write_lock(&old_writer_backend, "warehouse", data_file).await + }); + metadata_backend.wait_for_lock_attempts(2).await; + assert!( + !new_writer.is_finished(), + "a new-node first-publication writer must wait on the table-bucket fence" + ); + assert!(!old_writer.is_finished(), "an old-node writer must wait on the exact object guard"); + + pause.release(); + tokio::time::timeout(StdDuration::from_secs(2), register) + .await + .expect("registration should complete after the pause is released") + .expect("registration task should join") + .expect("registration should succeed"); + tokio::time::timeout(StdDuration::from_secs(2), new_writer) + .await + .expect("new-node writer should continue after registration") + .expect("new-node writer task should join") + .expect("new-node writer guard acquisition should succeed"); + tokio::time::timeout(StdDuration::from_secs(2), old_writer) + .await + .expect("old-node writer should continue after registration") + .expect("old-node writer task should join") + .expect("old-node writer guard acquisition should succeed"); +} + +#[tokio::test] +async fn warehouse_relocation_holds_bucket_fence_before_catalog_publication() { + let pause = TestCatalogPublishPause::default(); + let store = Arc::new(TestTableCatalogStore { + commit_table_pause: Some(pause.clone()), + ..Default::default() + }); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(store.as_ref(), &metadata_backend, &namespace).await; + let current = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should exist"); + let mut target_metadata = created.metadata; + target_metadata["location"] = serde_json::json!("s3://warehouse/tables/relocated-table-id"); + let table_bucket_fence_required = + table_warehouse_location_changes(¤t, &target_metadata).expect("relocated warehouse location should be valid"); + assert!(table_bucket_fence_required); + + let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let commit_store = Arc::clone(&store); + let commit = tokio::spawn(async move { + publish_table_commit( + commit_store.as_ref(), + &commit_backend, + table_bucket_fence_required, + crate::table_catalog::TableCommitRequest { + table_bucket: "warehouse".to_string(), + namespace: "analytics".to_string(), + table: "events".to_string(), + commit_id: "relocate-commit".to_string(), + idempotency_key: None, + operation: "set-location".to_string(), + expected_version_token: current.version_token, + expected_metadata_location: current.metadata_location, + new_metadata_location: "tables/catalog/analytics/events/metadata/00002.metadata.json".to_string(), + requirements: Vec::new(), + writer: Some("test".to_string()), + }, + ) + .await + }); + + tokio::time::timeout(StdDuration::from_secs(2), pause.wait_started()) + .await + .expect("relocation should reach catalog publication"); + let bucket_publication_lock = crate::table_catalog::default_table_bucket_publication_lock_path(); + assert!( + metadata_backend + .write_lock_is_held("warehouse", &bucket_publication_lock) + .await, + "warehouse relocation must retain the table-bucket fence before pointer publication" + ); + metadata_backend.lock_attempts.lock().await.clear(); + let writer_backend = metadata_backend.clone(); + let writer_lock = bucket_publication_lock.clone(); + let writer = tokio::spawn(async move { + crate::table_catalog::TableCatalogObjectBackend::acquire_read_lock(&writer_backend, "warehouse", &writer_lock).await + }); + metadata_backend.wait_for_lock_attempts(1).await; + assert!( + !writer.is_finished(), + "a writer in the new warehouse prefix must wait for relocation publication" + ); + + pause.release(); + tokio::time::timeout(StdDuration::from_secs(2), commit) + .await + .expect("relocation commit should complete") + .expect("relocation task should join") + .expect("relocation commit should succeed"); + tokio::time::timeout(StdDuration::from_secs(2), writer) + .await + .expect("writer should continue after relocation publication") + .expect("writer task should join") + .expect("writer guard acquisition should succeed"); +} + +#[tokio::test] +async fn commit_publication_lock_order_remains_compatible_with_old_maintenance_nodes() { + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let current_metadata = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let store = Arc::new(crate::table_catalog::ObjectTableCatalogStore::new(metadata_backend.clone())); + seed_object_table_for_metadata_maintenance( + store.as_ref(), + &metadata_backend, + "warehouse", + &namespace, + &table, + current_metadata.clone(), + ) + .await; + metadata_backend + .put_json("warehouse", &new_metadata, serde_json::json!({})) + .await; + let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + crate::table_catalog::TableCatalogObjectBackend::read_object(&commit_backend, "warehouse", &new_metadata) + .await + .expect("new metadata discovery should succeed") + .expect("new metadata should exist"); + + let table_path = crate::table_catalog::TableCatalogObjectPaths::default().table_entry_path("warehouse", &namespace, &table); + let old_table_guard = crate::table_catalog::TableCatalogObjectBackend::acquire_write_lock( + &metadata_backend, + crate::admin::storage_api::RUSTFS_META_BUCKET, + &table_path, + ) + .await + .expect("old maintenance node should acquire the table lock first"); + metadata_backend.lock_attempts.lock().await.clear(); + let commit_store = Arc::clone(&store); + let publication_backend = commit_backend.clone(); + let commit_new_metadata = new_metadata.clone(); + let commit = tokio::spawn(async move { + commit_store + .commit_table_with_publication( + crate::table_catalog::TableCommitRequest { + table_bucket: "warehouse".to_string(), + namespace: "analytics".to_string(), + table: "events".to_string(), + commit_id: "mixed-version-lock-order".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: commit_new_metadata, + requirements: Vec::new(), + writer: Some("rolling-upgrade-test".to_string()), + }, + &publication_backend, + ) + .await + }); + metadata_backend.wait_for_lock_attempts(2).await; + let migration_lock = crate::table_catalog::TableCatalogObjectPaths::default().backing_migration_fence_lock_path("warehouse"); + assert_eq!( + metadata_backend.lock_attempts.lock().await.as_slice(), + &[ + (crate::admin::storage_api::RUSTFS_META_BUCKET.to_string(), migration_lock), + (crate::admin::storage_api::RUSTFS_META_BUCKET.to_string(), table_path), + ] + ); + + let old_object_guard = tokio::time::timeout( + StdDuration::from_secs(2), + crate::table_catalog::TableCatalogObjectBackend::acquire_write_lock(&metadata_backend, "warehouse", &new_metadata), + ) + .await + .expect("new commit must not hold object locks while waiting for the old table lock") + .expect("old maintenance node should acquire the object lock"); + drop(old_object_guard); + drop(old_table_guard); + tokio::time::timeout(StdDuration::from_secs(2), commit) + .await + .expect("commit should complete after old maintenance releases its locks") + .expect("commit task should join") + .expect("commit should succeed"); +} + +#[tokio::test] +async fn commit_publication_acquires_discovered_object_locks_in_key_order() { + let metadata_backend = TestTableCatalogObjectBackend::default(); + let first = "metadata/a.json"; + let last = "metadata/z.json"; + metadata_backend.put_bytes("warehouse", first, b"a".to_vec()).await; + metadata_backend.put_bytes("warehouse", last, b"z".to_vec()).await; + let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + + crate::table_catalog::TableCatalogObjectBackend::read_object(&commit_backend, "warehouse", last) + .await + .expect("last object discovery should succeed"); + let first_writer = crate::table_catalog::TableCatalogObjectBackend::acquire_write_lock(&metadata_backend, "warehouse", first) + .await + .expect("first object writer should acquire its lock"); + tokio::time::timeout( + StdDuration::from_secs(2), + crate::table_catalog::TableCatalogObjectBackend::read_object(&commit_backend, "warehouse", first), + ) + .await + .expect("discovery must not retain the last object's lock") + .expect("first object discovery should succeed"); + + metadata_backend.lock_attempts.lock().await.clear(); + let publication_backend = commit_backend.clone(); + let publication = tokio::spawn(async move { + crate::table_catalog::TableCatalogObjectBackend::prepare_table_commit_publication( + &publication_backend, + "warehouse", + "analytics", + "events", + ) + .await + }); + metadata_backend.wait_for_lock_attempts(2).await; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let publication_lock = crate::table_catalog::default_table_publication_lock_path(&namespace, &table); + assert_eq!( + metadata_backend.lock_attempts.lock().await.as_slice(), + &[ + ("warehouse".to_string(), publication_lock.clone()), + ("warehouse".to_string(), first.to_string()), + ] + ); + + let last_writer = tokio::time::timeout( + StdDuration::from_secs(2), + crate::table_catalog::TableCatalogObjectBackend::acquire_write_lock(&metadata_backend, "warehouse", last), + ) + .await + .expect("publication waiting on the first key must not retain the last key") + .expect("last object writer should acquire its lock"); + assert_eq!( + metadata_backend.lock_attempts.lock().await.pop(), + Some(("warehouse".to_string(), last.to_string())) + ); + drop(last_writer); + drop(first_writer); + tokio::time::timeout(StdDuration::from_secs(2), publication) + .await + .expect("ordered publication locking should complete") + .expect("publication task should join") + .expect("publication preparation should succeed"); + assert_eq!( + metadata_backend.lock_attempts.lock().await.as_slice(), + &[ + ("warehouse".to_string(), publication_lock), + ("warehouse".to_string(), first.to_string()), + ("warehouse".to_string(), last.to_string()), + ] + ); + crate::table_catalog::TableCatalogObjectBackend::complete_table_commit_publication(&commit_backend); +} + +#[tokio::test] +async fn commit_publication_revalidates_objects_after_ordered_lock_acquisition() { + let metadata_backend = TestTableCatalogObjectBackend::default(); + let object = "metadata/current.json"; + metadata_backend.put_bytes("warehouse", object, b"before".to_vec()).await; + let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + crate::table_catalog::TableCatalogObjectBackend::read_object(&commit_backend, "warehouse", object) + .await + .expect("object discovery should succeed"); + metadata_backend.put_bytes("warehouse", object, b"after".to_vec()).await; + + let error = crate::table_catalog::TableCatalogObjectBackend::prepare_table_commit_publication( + &commit_backend, + "warehouse", + "analytics", + "events", + ) + .await + .expect_err("changed object must fail publication preparation"); + assert!(matches!(error, crate::table_catalog::TableCatalogStoreError::Conflict(_))); +} + +#[tokio::test] +async fn commit_publication_binds_fingerprint_to_returned_bytes() { + let metadata_backend = TestTableCatalogObjectBackend::default(); + let object = "metadata/current.json"; + let original = b"original".to_vec(); + let replacement = b"replacement".to_vec(); + let original_etag = hex_sha256(&original, str::to_string); + metadata_backend.objects.lock().await.insert( + ("warehouse".to_string(), object.to_string()), + crate::table_catalog::TableCatalogObject { + data: replacement, + etag: Some(original_etag.clone()), + mod_time: None, + }, + ); + let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + crate::table_catalog::TableCatalogObjectBackend::read_object(&commit_backend, "warehouse", object) + .await + .expect("replacement bytes should be discovered") + .expect("replacement object should exist"); + metadata_backend.objects.lock().await.insert( + ("warehouse".to_string(), object.to_string()), + crate::table_catalog::TableCatalogObject { + data: original, + etag: Some(original_etag), + mod_time: None, + }, + ); + + let error = crate::table_catalog::TableCatalogObjectBackend::prepare_table_commit_publication( + &commit_backend, + "warehouse", + "analytics", + "events", + ) + .await + .expect_err("restoring the statted version after reading replacement bytes must fail publication"); + + assert!(matches!(error, crate::table_catalog::TableCatalogStoreError::Conflict(_))); +} + +#[tokio::test] +async fn standard_commit_publishes_more_than_ten_thousand_live_files() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let created = create_standard_events_table(&store, &metadata_backend, &namespace).await; + let table_location = created.metadata["location"] + .as_str() + .expect("created metadata should include the table location"); + let manifest_list = format!("{table_location}/metadata/boundary-list.avro"); + let manifest = format!("{table_location}/metadata/boundary-manifest.avro"); + let data_file_count = 10_001; + let data_files = (0..data_file_count) + .map(|index| format!("{table_location}/data/part-{index:05}.parquet")) + .collect::>(); + let manifest_files = data_files.iter().map(|file| (file.as_str(), 0, 1, 20, 1)).collect::>(); + metadata_backend + .put_bytes( + "warehouse", + &test_snapshot_object_key("warehouse", &manifest_list), + test_manifest_list_avro_bytes(&[&manifest], 1, 20), + ) + .await; + metadata_backend + .put_bytes( + "warehouse", + &test_snapshot_object_key("warehouse", &manifest), + test_manifest_avro_bytes(&manifest_files), + ) + .await; + { + let mut objects = metadata_backend.objects.lock().await; + let data = vec![1]; + let etag = hex_sha256(&data, str::to_string); + for file in &data_files { + objects.insert( + ("warehouse".to_string(), test_snapshot_object_key("warehouse", file)), + crate::table_catalog::TableCatalogObject { + data: data.clone(), + etag: Some(etag.clone()), + mod_time: None, + }, + ); + } + } + let request = serde_json::from_value(serde_json::json!({ + "commit-id": "publication-boundary", + "requirements": [], + "updates": [ + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 20, + "sequence-number": 1, + "timestamp-ms": 1234, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + } + }, + { + "action": "set-snapshot-ref", + "ref-name": "main", + "snapshot-id": 20, + "type": "branch" + } + ] + })) + .expect("boundary commit request should parse"); + let commit_backend = TableCommitObjectBackend::trusted(metadata_backend); + + let result = standard_commit_table_response(&store, &commit_backend, "warehouse", &namespace, "events", request).await; + let response = commit_backend + .finish(result) + .await + .expect("a valid snapshot with more than ten thousand live files should publish"); + + assert_eq!(response.metadata["current-snapshot-id"], 20); + assert_eq!( + store + .load_table("warehouse", "analytics", "events") + .await + .expect("committed table lookup should succeed") + .expect("committed table should exist") + .metadata_location, + table_metadata_location_for_catalog("warehouse", &response.metadata_location) + .expect("response metadata location should map to the catalog object") + ); +} + +#[tokio::test] +async fn commit_publication_rejects_recreated_object_observed_by_exists() { + let metadata_backend = TestTableCatalogObjectBackend::default(); + let object = "data/part-00001.parquet"; + metadata_backend.put_bytes("warehouse", object, b"before".to_vec()).await; + let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + assert!( + crate::table_catalog::TableCatalogObjectBackend::object_exists(&commit_backend, "warehouse", object) + .await + .expect("object discovery should succeed") + ); + crate::table_catalog::TableCatalogObjectBackend::delete_object(&metadata_backend, "warehouse", object) + .await + .expect("object delete should succeed"); + metadata_backend.put_bytes("warehouse", object, b"after".to_vec()).await; + + let error = crate::table_catalog::TableCatalogObjectBackend::prepare_table_commit_publication( + &commit_backend, + "warehouse", + "analytics", + "events", + ) + .await + .expect_err("a recreated object with different bytes must fail publication preparation"); + + assert!(matches!(error, crate::table_catalog::TableCatalogStoreError::Conflict(_))); +} + #[tokio::test] async fn standard_commit_ignores_generation_only_orphan_metadata_file() { let store = TestTableCatalogStore::default(); @@ -2227,9 +3639,16 @@ async fn standard_commit_ignores_generation_only_orphan_metadata_file() { ] })) .expect("standard commit table request should parse"); - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) - .await - .expect("standard commit should not collide with generation-only orphan"); + let commit = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + commit_request, + ) + .await + .expect("standard commit should not collide with generation-only orphan"); assert_eq!( commit.metadata_location, @@ -2249,6 +3668,7 @@ async fn concurrent_standard_commits_write_distinct_metadata_files_before_pointe let metadata_backend = TestTableCatalogObjectBackend { objects: Arc::clone(&metadata_backend.objects), put_object_barrier: Some(barrier), + ..Default::default() }; let first_commit_id = "33333333-3333-4333-8333-333333333333"; let second_commit_id = "44444444-4444-4444-8444-444444444444"; @@ -2277,9 +3697,11 @@ async fn concurrent_standard_commits_write_distinct_metadata_files_before_pointe })) .expect("second standard commit table request should parse"); + let first_backend = trusted_table_commit_backend(&metadata_backend); + let second_backend = trusted_table_commit_backend(&metadata_backend); let (first, second) = tokio::join!( - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", first_request), - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", second_request) + commit_table_response(&store, &first_backend, "warehouse", &namespace, "events", first_request), + commit_table_response(&store, &second_backend, "warehouse", &namespace, "events", second_request) ); let success_count = [first.is_ok(), second.is_ok()].into_iter().filter(|ok| *ok).count(); @@ -2358,9 +3780,16 @@ async fn standard_commit_accepts_legacy_catalog_uuid_when_current_metadata_match ] })) .expect("standard commit table request should parse"); - let committed = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", commit_request) - .await - .expect("legacy catalog uuid should not block standard commit"); + let committed = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + commit_request, + ) + .await + .expect("legacy catalog uuid should not block standard commit"); assert_eq!(committed.metadata["table-uuid"], "metadata-table-uuid"); assert_eq!(committed.metadata["properties"]["owner"], "lakehouse"); @@ -2416,7 +3845,7 @@ async fn metadata_location_api_accepts_legacy_catalog_uuid_when_target_matches_c let updated = update_table_metadata_location_response( &store, - &metadata_backend, + &trusted_table_commit_backend(&metadata_backend), "warehouse", &namespace, "events", @@ -2597,6 +4026,15 @@ async fn table_metadata_maintenance_helper_runs_dry_run_and_delete() { .await .expect("old metadata lookup should succeed after delete") ); + let publication_lock = crate::table_catalog::default_table_publication_lock_path(&namespace, &table); + assert!( + backend + .lock_attempts + .lock() + .await + .contains(&(bucket.to_string(), publication_lock)), + "metadata deletion should enter through the table publication fence" + ); } #[tokio::test] @@ -2669,6 +4107,7 @@ async fn table_metadata_maintenance_helper_commits_snapshot_expiration() { Some(OffsetDateTime::UNIX_EPOCH), ) .await; + backend.lock_attempts.lock().await.clear(); let report = table_metadata_maintenance_response( &store, @@ -2739,6 +4178,111 @@ async fn table_metadata_maintenance_helper_commits_snapshot_expiration() { .await .expect("previous metadata lookup should succeed") ); + let publication_lock = crate::table_catalog::default_table_publication_lock_path(&namespace, &table); + assert!( + backend + .lock_attempts + .lock() + .await + .contains(&(bucket.to_string(), publication_lock)), + "internal snapshot expiration commits should enter through the table publication fence" + ); + assert!( + backend.lock_attempts.lock().await.contains(&(bucket.to_string(), current)), + "snapshot expiration must retain the metadata observation through pointer publication" + ); +} + +#[tokio::test] +async fn table_metadata_maintenance_helper_commits_compaction_through_publication_observer() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let metadata_dir = crate::table_catalog::default_table_metadata_dir_path(&namespace, &table); + let data_dir = crate::table_catalog::default_table_data_dir_path(&namespace, &table); + let current = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let manifest_list = format!("{metadata_dir}/snap-20.avro"); + let manifest = format!("{metadata_dir}/manifest-20.avro"); + let left_data = format!("{data_dir}/part-left.parquet"); + let right_data = format!("{data_dir}/part-right.parquet"); + + seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current.clone()).await; + backend + .put_bytes(bucket, &manifest_list, test_manifest_list_avro_bytes(&[&manifest], 7, 20)) + .await; + backend + .put_bytes( + bucket, + &manifest, + test_manifest_avro_bytes(&[(&left_data, 0, 0, 20, 7), (&right_data, 0, 0, 20, 7)]), + ) + .await; + backend.put_bytes(bucket, &left_data, test_parquet_i32_bytes(&[1, 2])).await; + backend.put_bytes(bucket, &right_data, test_parquet_i32_bytes(&[3, 4])).await; + backend + .put_json( + bucket, + ¤t, + serde_json::json!({ + "format-version": 2, + "table-uuid": "table-uuid", + "location": "s3://warehouse/tables/table-id", + "last-sequence-number": 7, + "last-updated-ms": 2000, + "metadata-log": [], + "snapshots": [{ + "snapshot-id": 20, + "sequence-number": 7, + "timestamp-ms": 2000, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + }], + "current-snapshot-id": 20, + "refs": { + "main": { + "snapshot-id": 20, + "type": "branch" + } + } + }), + ) + .await; + backend.lock_attempts.lock().await.clear(); + + let report = table_metadata_maintenance_response( + &store, + &backend, + bucket, + &namespace, + "events", + TableMetadataMaintenanceRequest { + retain_recent_metadata_files: 0, + delete: false, + snapshot_expiration: None, + commit_snapshot_expiration: false, + compaction: Some(crate::table_catalog::TableCompactionPlanningConfig { + target_file_size_bytes: 64 * 1024, + small_file_threshold_bytes: 64 * 1024, + min_input_files: 2, + max_rewrite_bytes_per_job: 128 * 1024, + }), + commit_compaction: true, + }, + ) + .await + .expect("compaction maintenance commit should succeed"); + + let compaction = report.compaction.expect("maintenance report should include compaction"); + assert_eq!(compaction.status, crate::table_catalog::TableCompactionPlanningStatus::Committed); + let lock_attempts = backend.lock_attempts.lock().await; + for observed_object in [current, manifest_list, manifest, left_data, right_data] { + assert!( + lock_attempts.contains(&(bucket.to_string(), observed_object.clone())), + "compaction must retain its content observation through pointer publication: {observed_object}" + ); + } } #[tokio::test] @@ -3103,10 +4647,11 @@ async fn external_catalog_bridge_sync_registers_missing_table_from_snapshot() { test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"), ) .await; + let commit_backend = trusted_table_commit_backend(&backend); let synced = sync_external_catalog_bridge_response( &store, - &backend, + &commit_backend, bucket, &namespace, "events", @@ -3167,10 +4712,11 @@ async fn external_catalog_bridge_sync_commits_existing_table_pointer() { let mut next_metadata = test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"); next_metadata["last-sequence-number"] = serde_json::Value::from(2); backend.put_json(bucket, &next_location, next_metadata).await; + let commit_backend = trusted_table_commit_backend(&backend); let synced = sync_external_catalog_bridge_response( &store, - &backend, + &commit_backend, bucket, &namespace, "events", @@ -3207,6 +4753,75 @@ async fn external_catalog_bridge_sync_commits_existing_table_pointer() { assert_eq!(current.generation, 2); } +#[tokio::test] +async fn external_catalog_bridge_sync_denies_metadata_reads_before_pointer_publish() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let current_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let next_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current_location.clone()).await; + backend + .put_json( + bucket, + ¤t_location, + test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"), + ) + .await; + backend + .put_json( + bucket, + &next_location, + test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"), + ) + .await; + let authorized = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let commit_backend = TableCommitObjectBackend::test(backend, Arc::clone(&authorized), Some(next_location.clone())); + + let result = sync_external_catalog_bridge_response( + &store, + &commit_backend, + bucket, + &namespace, + "events", + ExternalCatalogBridgeSyncRequest { + catalog: "hive-metastore".to_string(), + external_catalog_id: Some("hms-prod".to_string()), + external_namespace: "sales".to_string(), + external_table: "orders".to_string(), + external_table_uuid: Some("table-uuid".to_string()), + metadata_location: next_location.clone(), + external_version_token: Some("hms-version-2".to_string()), + expected_version_token: Some("token-v1".to_string()), + expected_metadata_location: Some(current_location.clone()), + commit_id: Some("external-sync-denied".to_string()), + idempotency_key: None, + policy_mode: Some("rustfs-authoritative".to_string()), + credential_mode: Some("rustfs-table-credentials".to_string()), + rollback_strategy: Some("retain-current-pointer".to_string()), + properties: BTreeMap::new(), + }, + true, + ) + .await; + let error = commit_backend + .finish(result) + .await + .expect_err("denied external metadata read must fail before pointer publication"); + + assert_eq!(error.code(), &S3ErrorCode::AccessDenied); + assert!(authorized.lock().await.contains(&(next_location, S3Action::GetObjectAction))); + let current = store + .load_table(bucket, "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should remain"); + assert_eq!(current.metadata_location, current_location); + assert_eq!(current.version_token, "token-v1"); +} + #[tokio::test] async fn external_catalog_bridge_sync_conflicts_leave_pointer_unchanged() { let backend = TestTableCatalogObjectBackend::default(); @@ -3231,10 +4846,11 @@ async fn external_catalog_bridge_sync_conflicts_leave_pointer_unchanged() { test_table_metadata_json("different-table-uuid", "s3://warehouse/tables/table-id"), ) .await; + let commit_backend = trusted_table_commit_backend(&backend); let result = sync_external_catalog_bridge_response( &store, - &backend, + &commit_backend, bucket, &namespace, "events", @@ -3439,9 +5055,16 @@ async fn row_level_conflict_allows_overwrite_when_deleted_file_is_current() { ] })) .expect("append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("append commit should succeed"); + commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) + .await + .expect("append commit should succeed"); let overwrite_manifest_list = format!("{table_location}/metadata/snap-11.avro"); let replacement_data_file = format!("{table_location}/data/part-11.parquet"); @@ -3486,9 +5109,16 @@ async fn row_level_conflict_allows_overwrite_when_deleted_file_is_current() { let stale_overwrite_request: RestCommitTableRequest = serde_json::from_value(overwrite_request_json.clone()).expect("stale overwrite request should parse"); - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", stale_overwrite_request) - .await - .expect_err("deleted file sequence must not change"); + let error = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + stale_overwrite_request, + ) + .await + .expect_err("deleted file sequence must not change"); assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); seed_test_snapshot_manifest( @@ -3503,9 +5133,16 @@ async fn row_level_conflict_allows_overwrite_when_deleted_file_is_current() { let overwrite_request: RestCommitTableRequest = serde_json::from_value(overwrite_request_json).expect("overwrite request should parse"); - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) - .await - .expect("overwrite commit should pass manifest conflict validation"); + let commit = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + overwrite_request, + ) + .await + .expect("overwrite commit should pass manifest conflict validation"); assert_eq!(commit.metadata["current-snapshot-id"], 11); assert_eq!(commit.metadata["last-sequence-number"], 2); @@ -3549,9 +5186,16 @@ async fn row_level_conflict_allows_v1_manifest_snapshot() { })) .expect("append request should parse"); - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("v1 manifests snapshot should commit"); + let commit = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) + .await + .expect("v1 manifests snapshot should commit"); assert_eq!(commit.metadata["current-snapshot-id"], 10); assert_eq!(commit.metadata["last-sequence-number"], 1); @@ -3599,9 +5243,16 @@ async fn row_level_conflict_allows_v1_manifest_snapshot() { })) .expect("second append request should parse"); - let upgraded = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", second_append) - .await - .expect("manifest-list snapshot should inherit a legacy manifest with unknown provenance"); + let upgraded = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + second_append, + ) + .await + .expect("manifest-list snapshot should inherit a legacy manifest with unknown provenance"); assert_eq!(upgraded.metadata["current-snapshot-id"], 11); assert_eq!(upgraded.metadata["last-sequence-number"], 2); @@ -3645,9 +5296,16 @@ async fn row_level_conflict_inherits_manifest_list_sequence_numbers() { })) .expect("append request should parse"); - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("manifest entry should inherit sequence numbers from manifest list"); + let commit = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) + .await + .expect("manifest entry should inherit sequence numbers from manifest list"); assert_eq!(commit.metadata["current-snapshot-id"], 10); assert_eq!(commit.metadata["last-sequence-number"], 1); @@ -3690,9 +5348,16 @@ async fn row_level_conflict_allows_inherited_manifests_on_append() { ] })) .expect("first append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", first_append) - .await - .expect("first append should commit"); + commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + first_append, + ) + .await + .expect("first append should commit"); let second_manifest_list = format!("{table_location}/metadata/snap-11.avro"); let second_manifest = format!("{table_location}/metadata/manifest-11.avro"); @@ -3737,9 +5402,16 @@ async fn row_level_conflict_allows_inherited_manifests_on_append() { })) .expect("second append request should parse"); - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", second_append) - .await - .expect("append should preserve inherited manifests"); + let commit = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + second_append, + ) + .await + .expect("append should preserve inherited manifests"); assert_eq!(commit.metadata["current-snapshot-id"], 11); assert_eq!(commit.metadata["last-sequence-number"], 2); @@ -3782,9 +5454,16 @@ async fn row_level_conflict_rejects_changed_inherited_manifest_identity() { ] })) .expect("first append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", first_append) - .await - .expect("first append should commit"); + commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + first_append, + ) + .await + .expect("first append should commit"); let current = store .load_table("warehouse", "analytics", "events") .await @@ -3818,9 +5497,16 @@ async fn row_level_conflict_rejects_changed_inherited_manifest_identity() { })) .expect("second append request should parse"); - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", second_append) - .await - .expect_err("inherited manifest identity must not change"); + let error = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + second_append, + ) + .await + .expect_err("inherited manifest identity must not change"); assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); let unchanged = store @@ -3870,9 +5556,16 @@ async fn row_level_conflict_rejects_stale_new_manifest_sequence() { })) .expect("append request should parse"); - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect_err("new manifest sequence must match the committed snapshot"); + let error = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) + .await + .expect_err("new manifest sequence must match the committed snapshot"); assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); let unchanged = store @@ -3922,9 +5615,16 @@ async fn row_level_conflict_rejects_stale_added_entry_sequence() { })) .expect("append request should parse"); - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect_err("added file sequence must match the new manifest"); + let error = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) + .await + .expect_err("added file sequence must match the new manifest"); assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); let unchanged = store @@ -3974,9 +5674,16 @@ async fn row_level_conflict_rejects_historical_change_in_new_manifest() { })) .expect("append request should parse"); - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect_err("new manifest must not claim a historical changed entry"); + let error = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) + .await + .expect_err("new manifest must not claim a historical changed entry"); assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); let unchanged = store @@ -4032,9 +5739,16 @@ async fn row_level_conflict_allows_add_only_overwrite_snapshot() { ] })) .expect("append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("append commit should succeed"); + commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) + .await + .expect("append commit should succeed"); let overwrite_manifest_list = format!("{table_location}/metadata/snap-11.avro"); let added_data_file = format!("{table_location}/data/part-11.parquet"); @@ -4078,9 +5792,16 @@ async fn row_level_conflict_allows_add_only_overwrite_snapshot() { })) .expect("overwrite request should parse"); - let commit = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) - .await - .expect("add-only overwrite should pass conflict validation"); + let commit = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + overwrite_request, + ) + .await + .expect("add-only overwrite should pass conflict validation"); assert_eq!(commit.metadata["current-snapshot-id"], 11); assert_eq!(commit.metadata["last-sequence-number"], 2); @@ -4129,9 +5850,16 @@ async fn row_level_conflict_rejects_delete_of_non_current_file() { ] })) .expect("append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("append commit should succeed"); + commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) + .await + .expect("append commit should succeed"); let committed = store .load_table("warehouse", "analytics", "events") .await @@ -4176,9 +5904,16 @@ async fn row_level_conflict_rejects_delete_of_non_current_file() { })) .expect("overwrite request should parse"); - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) - .await - .expect_err("stale row-level delete should conflict"); + let error = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + overwrite_request, + ) + .await + .expect_err("stale row-level delete should conflict"); assert_eq!(error.code(), &s3s::S3ErrorCode::PreconditionFailed); let unchanged = store @@ -4226,9 +5961,16 @@ async fn row_level_conflict_rejects_append_with_delete_files() { })) .expect("append request should parse"); - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect_err("append must not add delete files"); + let error = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) + .await + .expect_err("append must not add delete files"); assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); let unchanged = store @@ -4284,9 +6026,16 @@ async fn row_level_conflict_rejects_missing_manifest_before_pointer_update() { ] })) .expect("append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("append commit should succeed"); + commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) + .await + .expect("append commit should succeed"); let committed = store .load_table("warehouse", "analytics", "events") .await @@ -4318,9 +6067,16 @@ async fn row_level_conflict_rejects_missing_manifest_before_pointer_update() { })) .expect("overwrite request should parse"); - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) - .await - .expect_err("missing manifest-list should fail before pointer update"); + let error = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + overwrite_request, + ) + .await + .expect_err("missing manifest-list should fail before pointer update"); assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); let unchanged = store @@ -4376,9 +6132,16 @@ async fn row_level_conflict_rejects_manifest_outside_table_warehouse() { ] })) .expect("append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("append commit should succeed"); + commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) + .await + .expect("append commit should succeed"); let committed = store .load_table("warehouse", "analytics", "events") .await @@ -4410,9 +6173,16 @@ async fn row_level_conflict_rejects_manifest_outside_table_warehouse() { })) .expect("overwrite request should parse"); - let error = commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", overwrite_request) - .await - .expect_err("outside manifest-list should fail before pointer update"); + let error = commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + overwrite_request, + ) + .await + .expect_err("outside manifest-list should fail before pointer update"); assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidRequest); let unchanged = store @@ -4454,6 +6224,7 @@ fn table_updates_reject_unknown_actions() { fn table_location_updates_must_stay_inside_bucket() { let metadata = serde_json::json!({ "location": "s3://warehouse/tables/table-id", + "last-updated-ms": 1, "metadata-log": [] }); let updates = vec![serde_json::json!({ @@ -4716,7 +6487,7 @@ async fn view_catalog_responses_persist_replace_and_drop_view_metadata() { } #[tokio::test] -async fn table_ref_write_responses_commit_retention_refs_and_protect_deletes() { +async fn table_ref_write_responses_use_commit_guard_and_protect_deletes() { let store = TestTableCatalogStore::default(); let metadata_backend = TestTableCatalogObjectBackend::default(); let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); @@ -4751,20 +6522,70 @@ async fn table_ref_write_responses_commit_retention_refs_and_protect_deletes() { ] })) .expect("append request should parse"); - commit_table_response(&store, &metadata_backend, "warehouse", &namespace, "events", append_request) - .await - .expect("append should commit"); + commit_table_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + append_request, + ) + .await + .expect("append should commit"); - let ref_request: PutTableRefRequest = serde_json::from_value(serde_json::json!({ + let ref_request_json = serde_json::json!({ "snapshot-id": 10, "type": "tag", "max-ref-age-ms": 86400000, "expected-snapshot-id": null - })) - .expect("ref put request should parse"); - put_table_ref_response(&store, &metadata_backend, "warehouse", &namespace, "events", "audit", ref_request) + }); + let before = store + .load_table("warehouse", "analytics", "events") .await - .expect("ref put should commit"); + .expect("table lookup should succeed") + .expect("table should exist"); + let denied_authorizations = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let denied_backend = TableCommitObjectBackend::test( + metadata_backend.clone(), + Arc::clone(&denied_authorizations), + Some(before.metadata_location.clone()), + ); + let denied_request: PutTableRefRequest = + serde_json::from_value(ref_request_json.clone()).expect("denied ref put request should parse"); + let denied = + put_table_ref_response(&store, &denied_backend, "warehouse", &namespace, "events", "audit", denied_request).await; + let error = denied_backend + .finish(denied) + .await + .expect_err("ref writes must honor exact object authorization"); + assert_eq!(error.code(), &S3ErrorCode::AccessDenied); + assert!( + denied_authorizations + .lock() + .await + .contains(&(before.metadata_location.clone(), S3Action::GetObjectAction)) + ); + let after_denial = store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should remain"); + assert_eq!(after_denial.metadata_location, before.metadata_location); + assert_eq!(after_denial.version_token, before.version_token); + assert_eq!(after_denial.generation, before.generation); + + let ref_request: PutTableRefRequest = serde_json::from_value(ref_request_json).expect("ref put request should parse"); + let ref_authorizations = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let ref_backend = TableCommitObjectBackend::test(metadata_backend.clone(), Arc::clone(&ref_authorizations), None); + let result = put_table_ref_response(&store, &ref_backend, "warehouse", &namespace, "events", "audit", ref_request).await; + ref_backend.finish(result).await.expect("ref put should commit"); + assert!( + ref_authorizations + .lock() + .await + .iter() + .any(|(_, action)| *action == S3Action::PutObjectAction) + ); let refs = table_refs_response(&store, &metadata_backend, "warehouse", &namespace, "events") .await @@ -4789,9 +6610,18 @@ async fn table_ref_write_responses_commit_retention_refs_and_protect_deletes() { let force_delete: DeleteTableRefRequest = serde_json::from_value(serde_json::json!({ "force": true })).expect("ref force delete should parse"); - delete_table_ref_response(&store, &metadata_backend, "warehouse", &namespace, "events", "audit", force_delete) - .await - .expect("force delete should commit"); + let delete_authorizations = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let delete_backend = TableCommitObjectBackend::test(metadata_backend.clone(), Arc::clone(&delete_authorizations), None); + let result = + delete_table_ref_response(&store, &delete_backend, "warehouse", &namespace, "events", "audit", force_delete).await; + delete_backend.finish(result).await.expect("force delete should commit"); + assert!( + delete_authorizations + .lock() + .await + .iter() + .any(|(_, action)| *action == S3Action::PutObjectAction) + ); let refs = table_refs_response(&store, &metadata_backend, "warehouse", &namespace, "events") .await .expect("refs should load after delete"); @@ -5291,21 +7121,44 @@ struct TestTableCatalogStore { views: tokio::sync::Mutex>, commits: tokio::sync::Mutex>, fail_put_table_bucket: tokio::sync::Mutex, + register_table_pause: Option, + commit_table_pause: Option, } +#[derive(Clone, Default)] +struct TestCatalogPublishPause { + started: Arc, + release: Arc, +} + +impl TestCatalogPublishPause { + async fn wait_started(&self) { + self.started.notified().await; + } + + fn release(&self) { + self.release.notify_one(); + } +} + +type TestTableCatalogObjectLocks = Arc>>>>; + #[derive(Clone, Default)] struct TestTableCatalogObjectBackend { objects: Arc>>, put_object_barrier: Option>, + locks: TestTableCatalogObjectLocks, + lock_attempts: Arc>>, } impl TestTableCatalogObjectBackend { async fn put_bytes(&self, bucket: &str, object: &str, data: Vec) { + let etag = hex_sha256(&data, str::to_string); self.objects.lock().await.insert( (bucket.to_string(), object.to_string()), crate::table_catalog::TableCatalogObject { data, - etag: Some("etag".to_string()), + etag: Some(etag), mod_time: None, }, ); @@ -5333,15 +7186,45 @@ impl TestTableCatalogObjectBackend { mod_time: Option, ) { let data = serde_json::to_vec(&value).expect("metadata JSON should serialize"); + let etag = hex_sha256(&data, str::to_string); self.objects.lock().await.insert( (bucket.to_string(), object.to_string()), crate::table_catalog::TableCatalogObject { data, - etag: Some("etag".to_string()), + etag: Some(etag), mod_time, }, ); } + + async fn write_lock_is_held(&self, bucket: &str, object: &str) -> bool { + let lock = self + .locks + .lock() + .await + .get(&(bucket.to_string(), object.to_string())) + .cloned(); + lock.is_some_and(|lock| lock.try_lock_owned().is_err()) + } + + async fn wait_for_lock_attempts(&self, count: usize) { + tokio::time::timeout(StdDuration::from_secs(2), async { + loop { + if self.lock_attempts.lock().await.len() >= count { + return; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("lock acquisition attempts should be observable"); + } +} + +fn trusted_table_commit_backend( + backend: &TestTableCatalogObjectBackend, +) -> TableCommitObjectBackend { + TableCommitObjectBackend::trusted(backend.clone()) } fn test_table_metadata_json(table_uuid: &str, location: &str) -> serde_json::Value { @@ -5376,6 +7259,19 @@ fn test_snapshot_object_key(bucket: &str, location: &str) -> String { .expect("test snapshot object location should be valid") } +fn test_parquet_i32_bytes(values: &[i32]) -> Vec { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let batch = RecordBatch::try_new(Arc::clone(&schema) as SchemaRef, vec![Arc::new(Int32Array::from(values.to_vec()))]) + .expect("parquet test batch should build"); + let mut bytes = Vec::new(); + { + let mut writer = ArrowWriter::try_new(&mut bytes, schema, None).expect("parquet writer should build"); + writer.write(&batch).expect("parquet batch should write"); + writer.close().expect("parquet writer should close"); + } + bytes +} + fn test_manifest_list_avro_bytes(manifest_paths: &[&str], sequence_number: i64, snapshot_id: i64) -> Vec { let manifests = manifest_paths .iter() @@ -5682,11 +7578,46 @@ where } })) .expect("standard create table request should parse"); - create_table_response(store, metadata_backend, "warehouse", namespace, create_request, true) + let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + create_table_response(store, &commit_backend, "warehouse", namespace, create_request, true) .await .expect("table should be created") } +async fn seed_events_registration_target( + store: &S, + metadata_backend: &TestTableCatalogObjectBackend, +) -> (crate::table_catalog::Namespace, String) +where + S: crate::table_catalog::TableCatalogStore + ?Sized, +{ + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + ensure_table_bucket_entry(store, "warehouse", true) + .await + .expect("table bucket entry should be seeded"); + create_namespace_response( + store, + "warehouse", + CreateNamespaceRequest { + namespace: vec!["analytics".to_string()], + properties: BTreeMap::new(), + }, + true, + ) + .await + .expect("namespace should be created"); + let metadata_location = + ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001.metadata.json".to_string(); + metadata_backend + .put_json( + "warehouse", + &metadata_location, + test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"), + ) + .await; + (namespace, metadata_location) +} + async fn seed_object_table_for_metadata_maintenance( store: &crate::table_catalog::ObjectTableCatalogStore, backend: &TestTableCatalogObjectBackend, @@ -5786,11 +7717,12 @@ impl crate::table_catalog::TableCatalogObjectBackend for TestTableCatalogObjectB "object already exists: {object}" ))) } else { + let etag = hex_sha256(&data, str::to_string); objects.insert( key, crate::table_catalog::TableCatalogObject { data, - etag: Some("etag".to_string()), + etag: Some(etag), mod_time: None, }, ); @@ -5821,10 +7753,18 @@ impl crate::table_catalog::TableCatalogObjectBackend for TestTableCatalogObjectB async fn acquire_write_lock( &self, - _bucket: &str, - _object: &str, + bucket: &str, + object: &str, ) -> crate::table_catalog::TableCatalogStoreResult> { - Ok(Box::new(())) + self.lock_attempts.lock().await.push((bucket.to_string(), object.to_string())); + let lock = { + let mut locks = self.locks.lock().await; + locks + .entry((bucket.to_string(), object.to_string())) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone() + }; + Ok(Box::new(lock.lock_owned().await)) } } @@ -5958,10 +7898,37 @@ impl crate::table_catalog::TableCatalogStore for TestTableCatalogStore { entry.table_bucket, entry.namespace ))); } + if let Some(pause) = &self.register_table_pause { + pause.started.notify_one(); + pause.release.notified().await; + } self.tables.lock().await.push(entry); Ok(()) } + async fn register_table_with_publication( + &self, + entry: crate::table_catalog::TableEntry, + publication: &(dyn crate::table_catalog::TableCommitPublication + Sync), + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + publication.begin_table_bucket(&entry.table_bucket).await?; + if !publication.holds_table_bucket(&entry.table_bucket) { + return Err(crate::table_catalog::TableCatalogStoreError::Internal( + "table registration requires a table-bucket publication fence".to_string(), + )); + } + let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(publication); + publication + .prepare(&entry.table_bucket, &entry.namespace, &entry.table) + .await?; + if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table) { + return Err(crate::table_catalog::TableCatalogStoreError::Internal( + "table registration requires a table publication fence".to_string(), + )); + } + self.register_table(entry).await + } + async fn list_tables( &self, table_bucket: &str, @@ -6031,6 +7998,10 @@ impl crate::table_catalog::TableCatalogStore for TestTableCatalogStore { "current table metadata location does not match expected location".to_string(), )); } + if let Some(pause) = &self.commit_table_pause { + pause.started.notify_one(); + pause.release.notified().await; + } let mut next = current.clone(); next.metadata_location = request.new_metadata_location.clone(); @@ -6060,6 +8031,23 @@ impl crate::table_catalog::TableCatalogStore for TestTableCatalogStore { Ok(crate::table_catalog::TableCommitResult { table: next, commit_log }) } + async fn commit_table_with_publication( + &self, + request: crate::table_catalog::TableCommitRequest, + publication: &(dyn crate::table_catalog::TableCommitPublication + Sync), + ) -> crate::table_catalog::TableCatalogStoreResult { + publication + .prepare(&request.table_bucket, &request.namespace, &request.table) + .await?; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) { + return Err(crate::table_catalog::TableCatalogStoreError::Internal( + "table commit requires a table publication fence".to_string(), + )); + } + let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(publication); + self.commit_table(request).await + } + async fn drop_table( &self, table_bucket: &str, @@ -6322,7 +8310,7 @@ async fn table_helpers_call_catalog_store() { .await; let register = register_table_response( &store, - &metadata_backend, + &trusted_table_commit_backend(&metadata_backend), "warehouse", &namespace, RegisterTableRequest { @@ -6363,9 +8351,13 @@ async fn table_helpers_call_catalog_store() { metadata_backend .put_json("warehouse", next_metadata_location, next_metadata) .await; + let client_requirements = vec![serde_json::json!({ + "type": "assert-table-uuid", + "uuid": "table-uuid" + })]; let commit = commit_table_response( &store, - &metadata_backend, + &trusted_table_commit_backend(&metadata_backend), "warehouse", &namespace, "events", @@ -6376,7 +8368,7 @@ async fn table_helpers_call_catalog_store() { expected_version_token: Some(current.version_token.clone()), expected_metadata_location: Some(table_metadata_location_for_client("warehouse", ¤t.metadata_location)), new_metadata_location: Some(table_metadata_location_for_client("warehouse", next_metadata_location)), - requirements: Vec::new(), + requirements: client_requirements.clone(), updates: Vec::new(), _identifier: None, writer: Some("pyiceberg".to_string()), @@ -6391,6 +8383,17 @@ async fn table_helpers_call_catalog_store() { assert_eq!(commit.version_token, "token-committed"); assert_eq!(commit.generation, current.generation + 1); assert_eq!(commit.commit_id, "commit-1"); + assert_eq!( + store + .commits + .lock() + .await + .last() + .expect("commit log should be recorded") + .requirements, + client_requirements, + "persisted requirements must remain compatible with clients that do not know RustFS-private requirements" + ); let committed = store .load_table("warehouse", "analytics", "events") @@ -6439,7 +8442,7 @@ async fn register_table_response_adopts_metadata_table_uuid() { register_table_response( &store, - &metadata_backend, + &trusted_table_commit_backend(&metadata_backend), "warehouse", &namespace, RegisterTableRequest { @@ -6460,6 +8463,228 @@ async fn register_table_response_adopts_metadata_table_uuid() { assert_eq!(entry.table_uuid, "metadata-table-uuid"); } +#[tokio::test] +async fn register_table_denies_metadata_read_before_catalog_publication() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let (namespace, metadata_location) = seed_events_registration_target(&store, &metadata_backend).await; + let authorized = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let guarded_backend = + TableCommitObjectBackend::test(metadata_backend, Arc::clone(&authorized), Some(metadata_location.clone())); + + let result = register_table_response( + &store, + &guarded_backend, + "warehouse", + &namespace, + RegisterTableRequest { + name: "events".to_string(), + metadata_location: metadata_location.clone(), + overwrite: false, + }, + true, + ) + .await; + let error = guarded_backend + .finish(result) + .await + .expect_err("denied registration metadata read must fail before catalog publication"); + + assert_eq!(error.code(), &S3ErrorCode::AccessDenied); + assert!( + authorized + .lock() + .await + .contains(&(metadata_location, S3Action::GetObjectAction)) + ); + assert!( + store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .is_none() + ); +} + +#[tokio::test] +async fn catalog_import_denies_metadata_read_before_catalog_publication() { + let store = TestTableCatalogStore::default(); + let metadata_backend = TestTableCatalogObjectBackend::default(); + let (namespace, metadata_location) = seed_events_registration_target(&store, &metadata_backend).await; + let authorized = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let guarded_backend = + TableCommitObjectBackend::test(metadata_backend, Arc::clone(&authorized), Some(metadata_location.clone())); + + let result = catalog_import_response( + &store, + &guarded_backend, + "warehouse", + &namespace, + "events", + CatalogImportRequest { + metadata_location: metadata_location.clone(), + properties: BTreeMap::new(), + }, + true, + ) + .await; + let error = guarded_backend + .finish(result) + .await + .expect_err("denied import metadata read must fail before catalog publication"); + + assert_eq!(error.code(), &S3ErrorCode::AccessDenied); + assert!( + authorized + .lock() + .await + .contains(&(metadata_location, S3Action::GetObjectAction)) + ); + assert!( + store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .is_none() + ); +} + +#[tokio::test] +async fn register_table_rejects_metadata_replaced_before_catalog_publication() { + let metadata_backend = TestTableCatalogObjectBackend::default(); + let store = Arc::new(crate::table_catalog::ObjectTableCatalogStore::new(metadata_backend.clone())); + let (namespace, metadata_location) = seed_events_registration_target(store.as_ref(), &metadata_backend).await; + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let table_path = crate::table_catalog::TableCatalogObjectPaths::default().table_entry_path("warehouse", &namespace, &table); + let table_guard = crate::table_catalog::TableCatalogObjectBackend::acquire_write_lock( + &metadata_backend, + crate::admin::storage_api::RUSTFS_META_BUCKET, + &table_path, + ) + .await + .expect("catalog publication gate should acquire its lock"); + metadata_backend.lock_attempts.lock().await.clear(); + let guarded_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let register_store = Arc::clone(&store); + let register_backend = guarded_backend.clone(); + let register_namespace = namespace.clone(); + let register_location = metadata_location.clone(); + let registration = tokio::spawn(async move { + let result = register_table_response( + register_store.as_ref(), + ®ister_backend, + "warehouse", + ®ister_namespace, + RegisterTableRequest { + name: "events".to_string(), + metadata_location: register_location, + overwrite: false, + }, + true, + ) + .await; + register_backend.finish(result).await + }); + metadata_backend.wait_for_lock_attempts(3).await; + let writer_guard = tokio::time::timeout( + StdDuration::from_secs(2), + crate::table_catalog::TableCatalogObjectBackend::acquire_write_lock(&metadata_backend, "warehouse", &metadata_location), + ) + .await + .expect("metadata validation must release its read lock before catalog publication") + .expect("concurrent metadata writer should acquire its lock"); + metadata_backend + .put_json( + "warehouse", + &metadata_location, + test_table_metadata_json("replacement-uuid", "s3://warehouse/tables/table-id"), + ) + .await; + drop(writer_guard); + drop(table_guard); + + tokio::time::timeout(StdDuration::from_secs(2), registration) + .await + .expect("registration should finish after the writer releases its lock") + .expect("registration task should join") + .expect_err("replaced metadata must not be registered"); + assert!( + store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .is_none() + ); +} + +#[tokio::test] +async fn catalog_import_rejects_metadata_replaced_before_catalog_publication() { + let metadata_backend = TestTableCatalogObjectBackend::default(); + let store = Arc::new(crate::table_catalog::ObjectTableCatalogStore::new(metadata_backend.clone())); + let (namespace, metadata_location) = seed_events_registration_target(store.as_ref(), &metadata_backend).await; + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let table_path = crate::table_catalog::TableCatalogObjectPaths::default().table_entry_path("warehouse", &namespace, &table); + let table_guard = crate::table_catalog::TableCatalogObjectBackend::acquire_write_lock( + &metadata_backend, + crate::admin::storage_api::RUSTFS_META_BUCKET, + &table_path, + ) + .await + .expect("catalog publication gate should acquire its lock"); + metadata_backend.lock_attempts.lock().await.clear(); + let guarded_backend = TableCommitObjectBackend::trusted(metadata_backend.clone()); + let import_store = Arc::clone(&store); + let import_backend = guarded_backend.clone(); + let import_namespace = namespace.clone(); + let import_location = metadata_location.clone(); + let import = tokio::spawn(async move { + let result = catalog_import_response( + import_store.as_ref(), + &import_backend, + "warehouse", + &import_namespace, + "events", + CatalogImportRequest { + metadata_location: import_location, + properties: BTreeMap::new(), + }, + true, + ) + .await; + import_backend.finish(result).await + }); + metadata_backend.wait_for_lock_attempts(3).await; + let writer_guard = tokio::time::timeout( + StdDuration::from_secs(2), + crate::table_catalog::TableCatalogObjectBackend::acquire_write_lock(&metadata_backend, "warehouse", &metadata_location), + ) + .await + .expect("metadata validation must release its read lock before catalog publication") + .expect("concurrent metadata writer should acquire its lock"); + metadata_backend + .put_json( + "warehouse", + &metadata_location, + test_table_metadata_json("replacement-uuid", "s3://warehouse/tables/table-id"), + ) + .await; + drop(writer_guard); + drop(table_guard); + + tokio::time::timeout(StdDuration::from_secs(2), import) + .await + .expect("import should finish after the writer releases its lock") + .expect("import task should join") + .expect_err("replaced metadata must not be imported"); + assert!( + store + .load_table("warehouse", "analytics", "events") + .await + .expect("table lookup should succeed") + .is_none() + ); +} + #[tokio::test] async fn register_table_response_rejects_metadata_without_format_version() { let store = TestTableCatalogStore::default(); @@ -6494,7 +8719,7 @@ async fn register_table_response_rejects_metadata_without_format_version() { assert!( register_table_response( &store, - &metadata_backend, + &trusted_table_commit_backend(&metadata_backend), "warehouse", &namespace, RegisterTableRequest { @@ -6573,7 +8798,7 @@ async fn metadata_location_api_loads_and_updates_current_pointer() { let updated = update_table_metadata_location_response( &store, - &metadata_backend, + &trusted_table_commit_backend(&metadata_backend), "warehouse", &namespace, "events", @@ -6611,7 +8836,7 @@ async fn metadata_location_api_accepts_gzip_table_metadata() { let updated = update_table_metadata_location_response( &store, - &metadata_backend, + &trusted_table_commit_backend(&metadata_backend), "warehouse", &namespace, "events", @@ -6667,9 +8892,16 @@ async fn metadata_location_api_validates_snapshot_graph_before_commit() { idempotency_key: Some("graph-replay".to_string()), }; - let error = update_table_metadata_location_response(&store, &metadata_backend, "warehouse", &namespace, "events", request()) - .await - .expect_err("missing manifest-list must fail before pointer publication"); + let error = update_table_metadata_location_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + request(), + ) + .await + .expect_err("missing manifest-list must fail before pointer publication"); assert_eq!(error.message(), Some("snapshot manifest-list object is missing")); let unchanged = store .load_table("warehouse", "analytics", "events") @@ -6679,9 +8911,16 @@ async fn metadata_location_api_validates_snapshot_graph_before_commit() { assert_eq!(unchanged.metadata_location, current.metadata_location); seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await; - update_table_metadata_location_response(&store, &metadata_backend, "warehouse", &namespace, "events", request()) - .await - .expect("complete snapshot graph should commit"); + update_table_metadata_location_response( + &store, + &trusted_table_commit_backend(&metadata_backend), + "warehouse", + &namespace, + "events", + request(), + ) + .await + .expect("complete snapshot graph should commit"); } #[tokio::test] @@ -6720,7 +8959,7 @@ async fn metadata_location_api_validates_relocated_snapshot_graph_under_target_w update_table_metadata_location_response( &store, - &metadata_backend, + &trusted_table_commit_backend(&metadata_backend), "warehouse", &namespace, "events", @@ -6796,7 +9035,7 @@ async fn metadata_location_api_rejects_invalid_target_metadata_before_commit() { assert!( update_table_metadata_location_response( &store, - &metadata_backend, + &trusted_table_commit_backend(&metadata_backend), "warehouse", &namespace, "events", @@ -6849,7 +9088,7 @@ async fn metadata_location_api_rejects_mismatched_table_uuid_before_commit() { .await; register_table_response( &store, - &metadata_backend, + &trusted_table_commit_backend(&metadata_backend), "warehouse", &namespace, RegisterTableRequest { @@ -6876,7 +9115,7 @@ async fn metadata_location_api_rejects_mismatched_table_uuid_before_commit() { assert!( update_table_metadata_location_response( &store, - &metadata_backend, + &trusted_table_commit_backend(&metadata_backend), "warehouse", &namespace, "events", @@ -6932,7 +9171,7 @@ async fn catalog_import_and_rollback_use_register_and_commit_paths() { let imported = catalog_import_response( &store, - &backend, + &trusted_table_commit_backend(&backend), bucket, &namespace, "events", @@ -6954,7 +9193,7 @@ async fn catalog_import_and_rollback_use_register_and_commit_paths() { let imported_again = catalog_import_response( &store, - &backend, + &trusted_table_commit_backend(&backend), bucket, &namespace, "events", @@ -6977,7 +9216,7 @@ async fn catalog_import_and_rollback_use_register_and_commit_paths() { backend.put_json(bucket, &rollback_location, rollback_metadata).await; let rollback = rollback_table_response( &store, - &backend, + &trusted_table_commit_backend(&backend), bucket, &namespace, "events", @@ -6995,6 +9234,68 @@ async fn catalog_import_and_rollback_use_register_and_commit_paths() { assert_eq!(rollback.commit_id, "rollback-1"); } +#[tokio::test] +async fn rollback_denies_metadata_reads_before_pointer_publish() { + let backend = TestTableCatalogObjectBackend::default(); + let store = crate::table_catalog::ObjectTableCatalogStore::new(backend.clone()); + let bucket = "warehouse"; + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table should parse"); + let current_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let rollback_location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + seed_object_table_for_metadata_maintenance(&store, &backend, bucket, &namespace, &table, current_location.clone()).await; + backend + .put_json( + bucket, + ¤t_location, + test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"), + ) + .await; + backend + .put_json( + bucket, + &rollback_location, + test_table_metadata_json("table-uuid", "s3://warehouse/tables/table-id"), + ) + .await; + let authorized = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let commit_backend = TableCommitObjectBackend::test(backend, Arc::clone(&authorized), Some(rollback_location.clone())); + + let result = rollback_table_response( + &store, + &commit_backend, + bucket, + &namespace, + "events", + RollbackTableRequest { + metadata_location: rollback_location.clone(), + version_token: "token-v1".to_string(), + commit_id: Some("rollback-denied".to_string()), + idempotency_key: None, + }, + ) + .await; + let error = commit_backend + .finish(result) + .await + .expect_err("denied rollback metadata read must fail before pointer publication"); + + assert_eq!(error.code(), &S3ErrorCode::AccessDenied); + assert!( + authorized + .lock() + .await + .contains(&(rollback_location, S3Action::GetObjectAction)) + ); + let current = store + .load_table(bucket, "analytics", "events") + .await + .expect("table lookup should succeed") + .expect("table should remain"); + assert_eq!(current.metadata_location, current_location); + assert_eq!(current.version_token, "token-v1"); +} + #[tokio::test] async fn rollback_rejects_invalid_target_metadata_before_commit() { let backend = TestTableCatalogObjectBackend::default(); @@ -7026,7 +9327,7 @@ async fn rollback_rejects_invalid_target_metadata_before_commit() { .await; catalog_import_response( &store, - &backend, + &trusted_table_commit_backend(&backend), bucket, &namespace, "events", @@ -7060,7 +9361,7 @@ async fn rollback_rejects_invalid_target_metadata_before_commit() { assert!( rollback_table_response( &store, - &backend, + &trusted_table_commit_backend(&backend), bucket, &namespace, "events", @@ -7115,7 +9416,7 @@ async fn rollback_rejects_mismatched_table_uuid_before_commit() { .await; catalog_import_response( &store, - &backend, + &trusted_table_commit_backend(&backend), bucket, &namespace, "events", @@ -7145,7 +9446,7 @@ async fn rollback_rejects_mismatched_table_uuid_before_commit() { assert!( rollback_table_response( &store, - &backend, + &trusted_table_commit_backend(&backend), bucket, &namespace, "events", @@ -7197,7 +9498,7 @@ async fn legacy_commit_rejects_mismatched_table_uuid_before_commit() { .await; register_table_response( &store, - &metadata_backend, + &trusted_table_commit_backend(&metadata_backend), "warehouse", &namespace, RegisterTableRequest { @@ -7226,7 +9527,7 @@ async fn legacy_commit_rejects_mismatched_table_uuid_before_commit() { assert!( commit_table_response( &store, - &metadata_backend, + &trusted_table_commit_backend(&metadata_backend), "warehouse", &namespace, "events", diff --git a/rustfs/src/admin/handlers/table_catalog/view.rs b/rustfs/src/admin/handlers/table_catalog/view.rs index 4c16172ef..2b4ca5d31 100644 --- a/rustfs/src/admin/handlers/table_catalog/view.rs +++ b/rustfs/src/admin/handlers/table_catalog/view.rs @@ -23,8 +23,8 @@ impl Operation for RestListViewsHandler { let namespace = namespace_from_params(¶ms)?; let resource = TableCatalogResource::namespace(&warehouse, &namespace); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; let response = list_views_response(&store, &warehouse, &namespace, &req.uri).await?; build_json_response(StatusCode::OK, &response) } @@ -40,9 +40,9 @@ impl Operation for RestCreateViewHandler { let resource = TableCatalogResource::namespace(&warehouse, &namespace); authorize_table_catalog_resource_request(&req, &resource, AdminAction::CreateTableAction).await?; let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; - let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?; + let table_bucket_enabled = table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let response = create_view_response(&store, &metadata_backend, &warehouse, &namespace, request, table_bucket_enabled).await?; build_json_response(StatusCode::OK, &response) @@ -59,8 +59,8 @@ impl Operation for RestLoadViewHandler { let view = view_name_from_params(¶ms)?; let resource = TableCatalogResource::view(&warehouse, &namespace, &view); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableMetadataAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let metadata_backend = table_catalog_backend()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; let response = load_view_response(&store, &metadata_backend, &warehouse, &namespace, &view).await?; build_json_response(StatusCode::OK, &response) @@ -77,8 +77,8 @@ impl Operation for RestViewExistsHandler { let view = view_name_from_params(¶ms)?; let resource = TableCatalogResource::view(&warehouse, &namespace, &view); authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; Ok(empty_response(view_exists_status(&store, &warehouse, &namespace, &view).await?)) } } @@ -93,9 +93,9 @@ impl Operation for RestReplaceViewHandler { let view = view_name_from_params(¶ms)?; let resource = TableCatalogResource::view(&warehouse, &namespace, &view); authorize_table_catalog_resource_request(&req, &resource, AdminAction::CommitTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; let request = read_json_body::(req.input).await?; - let metadata_backend = table_catalog_backend()?; + let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?; let store = table_catalog_store_from_backend(metadata_backend.clone())?; let response = replace_view_response(&store, &metadata_backend, &warehouse, &namespace, &view, request).await?; build_json_response(StatusCode::OK, &response) @@ -112,8 +112,8 @@ impl Operation for RestDropViewHandler { let view = view_name_from_params(¶ms)?; let resource = TableCatalogResource::view(&warehouse, &namespace, &view); authorize_table_catalog_resource_request(&req, &resource, AdminAction::DeleteTableAction).await?; - ensure_table_bucket_enabled(&warehouse).await?; - let store = table_catalog_store()?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; drop_view_in_store(&store, &warehouse, &namespace, &view).await?; Ok(empty_response(StatusCode::NO_CONTENT)) } diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 5dbc7f58d..f0b88002a 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -793,7 +793,9 @@ pub(crate) mod data_usage { } pub(crate) mod access { - pub(crate) use crate::storage::storage_api::access_consumer::{ReqInfo, authorize_request}; + pub(crate) use crate::storage::storage_api::access_consumer::{ + ReqInfo, authorize_internal_object_request, authorize_request, + }; pub(crate) use crate::storage::storage_api::request_context_consumer::{RequestContext, spawn_traced}; } diff --git a/rustfs/src/auth.rs b/rustfs/src/auth.rs index 07acbe6ff..0bacf8933 100644 --- a/rustfs/src/auth.rs +++ b/rustfs/src/auth.rs @@ -451,7 +451,7 @@ pub async fn check_key_valid_with_context( cred = u.credentials; } - let claims = check_claims_from_token(session_token, &cred) + let claims = check_claims_from_token_with_context(session_token, &cred, ctx) .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("check claims failed {e}")))?; cred.claims = if !claims.is_empty() { Some(claims) } else { None }; @@ -461,6 +461,14 @@ pub async fn check_key_valid_with_context( } pub fn check_claims_from_token(token: &str, cred: &Credentials) -> S3Result> { + check_claims_from_token_with_context(token, cred, None) +} + +fn check_claims_from_token_with_context( + token: &str, + cred: &Credentials, + ctx: Option<&AppContext>, +) -> S3Result> { if !token.is_empty() && cred.access_key.is_empty() { return Err(s3_error!(InvalidRequest, "no access key")); } @@ -481,7 +489,11 @@ pub fn check_claims_from_token(token: &str, cred: &Credentials) -> S3Result context.action_credentials().get(), + None => current_action_credentials(), + }; + let Some(sys_cred) = sys_cred else { return Err(s3_error!(InternalError, "action cred not init")); }; @@ -1059,14 +1071,47 @@ pub fn get_query_param<'a>(query: &'a str, param_name: &str) -> Option<&'a str> #[cfg(test)] mod tests { use super::*; + use crate::runtime_sources::{IamInterface, KmsInterface}; use http::{HeaderMap, HeaderValue, Uri}; use rustfs_credentials::Credentials; + use rustfs_iam::{ + store::{ + Store as _, + object::{IAM_CONFIG_PREFIX, ObjectStore}, + }, + sys::IamSys, + }; + use rustfs_kms::KmsServiceManager; + use rustfs_policy::auth::get_new_credentials_with_metadata; use rustfs_trusted_proxies::ValidationMode; use s3s::auth::SecretKey; use serde_json::json; use std::collections::HashMap; + use std::sync::Arc; use time::OffsetDateTime; + struct ContextIam { + handle: Arc>, + } + + impl IamInterface for ContextIam { + fn handle(&self) -> Arc> { + self.handle.clone() + } + + fn is_ready(&self) -> bool { + true + } + } + + struct TestKms; + + impl KmsInterface for TestKms { + fn handle(&self) -> Arc { + Arc::new(KmsServiceManager::new()) + } + } + fn create_test_credentials() -> Credentials { Credentials { access_key: "test-access-key".to_string(), @@ -1268,6 +1313,59 @@ mod tests { } } + #[tokio::test] + async fn check_claims_uses_the_explicit_context_signing_secret() { + let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await; + ObjectStore::new(store.clone()) + .save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX)) + .await + .expect("seed request IAM format"); + let iam = rustfs_iam::build_iam_sys(store.clone()) + .await + .expect("request IAM should initialize"); + let matching = AppContext::new(store.clone(), Arc::new(ContextIam { handle: iam.clone() }), Arc::new(TestKms)); + let mismatching = AppContext::new(store, Arc::new(ContextIam { handle: iam.clone() }), Arc::new(TestKms)); + assert!(matching.publish_action_credentials(Credentials { + access_key: "matching-root".to_string(), + secret_key: "matching-signing-secret".to_string(), + status: "on".to_string(), + ..Default::default() + })); + assert!(mismatching.publish_action_credentials(Credentials { + access_key: "mismatching-root".to_string(), + secret_key: "mismatching-signing-secret".to_string(), + status: "on".to_string(), + ..Default::default() + })); + let claims = HashMap::from([ + ( + "exp".to_string(), + json!((OffsetDateTime::now_utc() + time::Duration::minutes(5)).unix_timestamp()), + ), + ("context".to_string(), json!("matching")), + ]); + let mut credential = get_new_credentials_with_metadata(&claims, "matching-signing-secret") + .expect("temporary credentials should be generated"); + credential.parent_user = "matching-root".to_string(); + iam.set_temp_user(&credential.access_key, &credential, None) + .await + .expect("temporary credentials should be stored in request IAM"); + + let (verified, _) = check_key_valid_with_context(&credential.session_token, &credential.access_key, Some(&matching)) + .await + .expect("the matching request context should verify the token"); + assert_eq!( + verified.claims.as_ref().and_then(|claims| claims.get("context")), + Some(&json!("matching")) + ); + assert!( + check_key_valid_with_context(&credential.session_token, &credential.access_key, Some(&mismatching)) + .await + .is_err(), + "a different server context must not validate the token" + ); + } + #[test] fn test_get_session_token_from_header() { let mut headers = HeaderMap::new(); diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index f0f8b9033..4592ebeb2 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -14,10 +14,7 @@ use super::ObjectOptions; use super::ecfs::FS; -use super::{ - ECStore, PolicySys, ReplicationStatusType, StorageError, get_bucket_metadata, get_bucket_policy_raw, - get_lock_acquire_timeout, get_public_access_block_config, is_err_bucket_not_found, -}; +use super::{ECStore, PolicySys, ReplicationStatusType, StorageError, get_lock_acquire_timeout, is_err_bucket_not_found}; use crate::auth::{ check_key_valid_with_context, get_condition_values_with_client_info, get_condition_values_with_query_and_client_info, get_session_token, @@ -50,8 +47,10 @@ use rustfs_utils::http::{ }; use s3s::access::{S3Access, S3AccessContext}; use s3s::{S3Error, S3ErrorCode, S3Request, S3Result, dto::*, s3_error}; -use std::collections::HashMap; -use std::sync::{Arc, OnceLock}; +use std::collections::{BTreeSet, HashMap}; +use std::sync::Arc; +#[cfg(test)] +use std::sync::OnceLock; use url::{Url, form_urlencoded}; #[derive(Default, Clone, Debug)] @@ -109,6 +108,22 @@ pub(crate) fn recursive_force_delete_is_authorized(headers: &HeaderMap, is_owner #[derive(Clone, Debug)] pub(crate) struct PostObjectRequestMarker; +#[derive(Clone, Debug)] +struct InternalObjectAuthorization; + +#[derive(Clone, Default)] +struct TableDataPlanePublicationGuards { + state: Arc>, +} + +#[derive(Default)] +struct TableDataPlanePublicationState { + keys: BTreeSet<(String, String)>, + guards: Vec>, + resources: HashMap<(String, String), crate::table_catalog::TableDataPlaneResource>, + missing_resources: BTreeSet<(String, String)>, +} + #[derive(Clone, Debug)] pub(crate) struct BucketGenerationGuard { bucket: String, @@ -186,7 +201,7 @@ struct CopySourceBucketGenerationGuard { } #[cfg(test)] -type RestoreAuthorizationTestHook = (String, std::sync::Arc, std::sync::Arc); +type RestoreAuthorizationTestHook = (String, tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>); #[cfg(test)] static RESTORE_AUTHORIZATION_TEST_HOOK: OnceLock>> = OnceLock::new(); @@ -194,8 +209,8 @@ static RESTORE_AUTHORIZATION_TEST_HOOK: OnceLock, - resume: std::sync::Arc, + authorized: tokio::sync::oneshot::Sender<()>, + resume: tokio::sync::oneshot::Receiver<()>, ) { *RESTORE_AUTHORIZATION_TEST_HOOK .get_or_init(|| std::sync::Mutex::new(None)) @@ -217,8 +232,8 @@ async fn wait_for_restore_authorization_test_hook(bucket: &str) { } }; if let Some((_bucket, authorized, resume)) = hook { - authorized.wait().await; - resume.wait().await; + let _ = authorized.send(()); + let _ = resume.await; } } @@ -410,8 +425,12 @@ fn classify_bucket_policy_raw_load_error(err: &StorageError) -> BucketPolicyRawL } /// Load and parse bucket policy once for ExistingObjectTag hint checks. -async fn load_bucket_policy_existing_object_tag_hint(bucket: &str, action: Action) -> BucketPolicyExistingObjectTagHint { - let (policy_str, _) = match get_bucket_policy_raw(bucket).await { +async fn load_bucket_policy_existing_object_tag_hint( + store: &ECStore, + bucket: &str, + action: Action, +) -> BucketPolicyExistingObjectTagHint { + let (policy_str, _) = match store.get_bucket_policy_raw(bucket).await { Ok(v) => v, Err(err) => match classify_bucket_policy_raw_load_error(&err) { BucketPolicyRawLoadErrorKind::PolicyMissing => { @@ -567,11 +586,45 @@ fn authorization_conditions( req.uri.query(), client_info, ); + if req.extensions.get::().is_some() { + retain_internal_object_authorization_conditions(&mut conditions, cred); + return Ok(conditions); + } merge_list_bucket_query_conditions(action, req.uri.query(), &mut conditions); merge_request_object_tag_conditions(action, &req.headers, &mut conditions)?; Ok(conditions) } +fn retain_internal_object_authorization_conditions( + conditions: &mut HashMap>, + cred: &rustfs_credentials::Credentials, +) { + conditions.retain(|key, _| { + matches!( + key.as_str(), + "CurrentTime" + | "EpochTime" + | "SecureTransport" + | "SourceIp" + | "UserAgent" + | "Referer" + | "userid" + | "username" + | "principaltype" + | "signatureversion" + | "signatureAge" + | "authType" + | "LocationConstraint" + | "groups" + | "roles" + ) || cred.claims.as_ref().is_some_and(|claims| { + claims + .keys() + .any(|claim| claim.trim_start_matches("ldap").to_lowercase() == *key) + }) + }); +} + /// Condition values for an authorization decision that is not scoped to a bucket or object. /// /// Lives here rather than at the call site because this module already owns the request @@ -585,9 +638,17 @@ pub(crate) fn resource_free_condition_values( get_condition_values_with_client_info(&req.headers, cred, None, None, remote_addr, req.extensions.get::()) } -fn auth_fs() -> &'static FS { - static AUTH_FS: OnceLock = OnceLock::new(); - AUTH_FS.get_or_init(FS::new) +fn request_object_store(req: &S3Request) -> S3Result> { + match req.extensions.get::>() { + Some(server_ctx) => server_ctx + .installed_object_store() + .ok_or_else(object_store_not_initialized_error), + None => runtime_sources::current_object_store_handle().ok_or_else(object_store_not_initialized_error), + } +} + +fn object_store_not_initialized_error() -> S3Error { + S3Error::with_message(S3ErrorCode::InternalError, "object store is not initialized") } fn request_iam_store(req: &S3Request) -> S3Result>> { @@ -738,6 +799,7 @@ fn versioned_read_action(version_id: Option<&str>) -> Action { async fn get_or_fetch_object_tag_conditions( req: &mut S3Request, + store: &ECStore, bucket: &str, object: &str, version_id: Option<&str>, @@ -750,9 +812,7 @@ async fn get_or_fetch_object_tag_conditions( } counter!("rustfs_object_tag_conditions_fetched_total", "op" => action_tag_metric_label(&action)).increment(1); - let fetched = auth_fs() - .get_object_tag_conditions_for_policy(bucket, object, version_id) - .await?; + let fetched = FS::get_object_tag_conditions_for_policy_from_store(store, bucket, object, version_id).await?; req.extensions .insert(ObjectTagConditions::new(bucket, object, version_id, fetched.clone())); Ok(fetched) @@ -772,7 +832,8 @@ async fn maybe_merge_object_tag_conditions( return Ok(()); } - let tags = get_or_fetch_object_tag_conditions(req, bucket, object, version_id, action).await?; + let store = request_object_store(req)?; + let tags = get_or_fetch_object_tag_conditions(req, store.as_ref(), bucket, object, version_id, action).await?; merge_object_tag_conditions(conditions, &tags); Ok(()) } @@ -847,6 +908,7 @@ impl DenialContext<'_> { /// Authorizes the request based on the action and credentials. pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3Result<()> { + let internal_object_authorization = req.extensions.get::().is_some(); let remote_addr = req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)); let req_info = req_info_ref(req)?; let cred = req_info.cred.clone(); @@ -855,6 +917,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R let object = req_info.object.clone().unwrap_or_default(); let version_id = req_info.version_id.clone(); let quiet_denial = req_info.suppress_denial_log; + let store = request_object_store(req)?; let denial = DenialContext { quiet: quiet_denial, bucket: bucket.as_str(), @@ -887,7 +950,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R let mut needs_tag_from_iam = prepared.needs_existing_object_tag; let bucket_tag_hint = if !bucket.is_empty() && !object.is_empty() { - Some(load_bucket_policy_existing_object_tag_hint(bucket.as_str(), action).await) + Some(load_bucket_policy_existing_object_tag_hint(store.as_ref(), bucket.as_str(), action).await) } else { None }; @@ -965,17 +1028,20 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R let owner_can_bypass_deny = owner_can_bypass_policy_deny(is_owner, &action); if !bucket_name.is_empty() && !owner_can_bypass_deny - && !PolicySys::try_is_allowed(&BucketPolicyArgs { - bucket: bucket_name, - action, - // Early explicit-deny gate for bucket policy: use owner short-circuit path so - // deny statements are enforced before IAM/bucket allow fallback evaluation. - is_owner: true, - account: &cred.access_key, - groups: &cred.groups, - conditions: &conditions, - object: object.as_str(), - }) + && !PolicySys::try_is_allowed_for_store( + store.as_ref(), + &BucketPolicyArgs { + bucket: bucket_name, + action, + // Early explicit-deny gate for bucket policy: use owner short-circuit path so + // deny statements are enforced before IAM/bucket allow fallback evaluation. + is_owner: true, + account: &cred.access_key, + groups: &cred.groups, + conditions: &conditions, + object: object.as_str(), + }, + ) .await .map_err(ApiError::from)? { @@ -996,15 +1062,18 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R }; let delete_version_allowed = iam_store.eval_prepared(&prepared, &delete_version_args).await; if !delete_version_allowed - && !PolicySys::try_is_allowed(&BucketPolicyArgs { - bucket: bucket.as_str(), - action: Action::S3Action(S3Action::DeleteObjectVersionAction), - is_owner, - account: &cred.access_key, - groups: &cred.groups, - conditions: &conditions, - object: object.as_str(), - }) + && !PolicySys::try_is_allowed_for_store( + store.as_ref(), + &BucketPolicyArgs { + bucket: bucket.as_str(), + action: Action::S3Action(S3Action::DeleteObjectVersionAction), + is_owner, + account: &cred.access_key, + groups: &cred.groups, + conditions: &conditions, + object: object.as_str(), + }, + ) .await .map_err(ApiError::from)? { @@ -1028,8 +1097,10 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R }; if iam_allowed { - authorize_table_data_plane_if_needed(action, bucket.as_str(), object.as_str(), cred, is_owner, &conditions, claims) - .await?; + if !internal_object_authorization { + authorize_table_data_plane_if_needed(req, action, bucket.as_str(), object.as_str(), cred, is_owner, &conditions) + .await?; + } return Ok(()); } @@ -1038,21 +1109,26 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R return Err(ApiError::access_denied().into()); } - let policy_allowed_fallback = PolicySys::try_is_allowed(&BucketPolicyArgs { - bucket: bucket.as_str(), - action, - is_owner, - account: &cred.access_key, - groups: &cred.groups, - conditions: &conditions, - object: object.as_str(), - }) + let policy_allowed_fallback = PolicySys::try_is_allowed_for_store( + store.as_ref(), + &BucketPolicyArgs { + bucket: bucket.as_str(), + action, + is_owner, + account: &cred.access_key, + groups: &cred.groups, + conditions: &conditions, + object: object.as_str(), + }, + ) .await .map_err(ApiError::from)?; if policy_allowed_fallback { - authorize_table_data_plane_if_needed(action, bucket.as_str(), object.as_str(), cred, is_owner, &conditions, claims) - .await?; + if !internal_object_authorization { + authorize_table_data_plane_if_needed(req, action, bucket.as_str(), object.as_str(), cred, is_owner, &conditions) + .await?; + } return Ok(()); } @@ -1073,15 +1149,18 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R return Ok(()); } - if PolicySys::try_is_allowed(&BucketPolicyArgs { - bucket: bucket.as_str(), - action: Action::S3Action(S3Action::ListBucketAction), - is_owner, - account: &cred.access_key, - groups: &cred.groups, - conditions: &conditions, - object: object.as_str(), - }) + if PolicySys::try_is_allowed_for_store( + store.as_ref(), + &BucketPolicyArgs { + bucket: bucket.as_str(), + action: Action::S3Action(S3Action::ListBucketAction), + is_owner, + account: &cred.access_key, + groups: &cred.groups, + conditions: &conditions, + object: object.as_str(), + }, + ) .await .map_err(ApiError::from)? { @@ -1103,7 +1182,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R let no_groups: Option> = None; let bucket_tag_hint = if !bucket.is_empty() && !object.is_empty() { - Some(load_bucket_policy_existing_object_tag_hint(bucket.as_str(), action).await) + Some(load_bucket_policy_existing_object_tag_hint(store.as_ref(), bucket.as_str(), action).await) } else { None }; @@ -1157,16 +1236,19 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R let bucket_name = bucket.as_str(); if !bucket_name.is_empty() - && !PolicySys::try_is_allowed(&BucketPolicyArgs { - bucket: bucket_name, - action, - // Early explicit-deny gate for bucket policy in anonymous path. - is_owner: true, - account: "", - groups: &None, - conditions: &conditions, - object: object.as_str(), - }) + && !PolicySys::try_is_allowed_for_store( + store.as_ref(), + &BucketPolicyArgs { + bucket: bucket_name, + action, + // Early explicit-deny gate for bucket policy in anonymous path. + is_owner: true, + account: "", + groups: &None, + conditions: &conditions, + object: object.as_str(), + }, + ) .await .map_err(ApiError::from)? { @@ -1175,15 +1257,18 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R if action != Action::S3Action(S3Action::ListAllMyBucketsAction) { if action == Action::S3Action(S3Action::DeleteObjectAction) && version_id.is_some() { - let delete_version_allowed = PolicySys::try_is_allowed(&BucketPolicyArgs { - bucket: bucket.as_str(), - action: Action::S3Action(S3Action::DeleteObjectVersionAction), - is_owner: false, - account: "", - groups: &None, - conditions: &conditions, - object: object.as_str(), - }) + let delete_version_allowed = PolicySys::try_is_allowed_for_store( + store.as_ref(), + &BucketPolicyArgs { + bucket: bucket.as_str(), + action: Action::S3Action(S3Action::DeleteObjectVersionAction), + is_owner: false, + account: "", + groups: &None, + conditions: &conditions, + object: object.as_str(), + }, + ) .await .map_err(ApiError::from)?; if !delete_version_allowed { @@ -1193,15 +1278,18 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R } } - let policy_allowed = PolicySys::try_is_allowed(&BucketPolicyArgs { - bucket: bucket.as_str(), - action, - is_owner: false, - account: "", - groups: &None, - conditions: &conditions, - object: object.as_str(), - }) + let policy_allowed = PolicySys::try_is_allowed_for_store( + store.as_ref(), + &BucketPolicyArgs { + bucket: bucket.as_str(), + action, + is_owner: false, + account: "", + groups: &None, + conditions: &conditions, + object: object.as_str(), + }, + ) .await .map_err(ApiError::from)?; @@ -1211,27 +1299,27 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R // ListObjectVersions after RestrictPublicBuckets is turned on. let policy_allowed = policy_allowed || (action == Action::S3Action(S3Action::ListBucketVersionsAction) - && PolicySys::try_is_allowed(&BucketPolicyArgs { - bucket: bucket.as_str(), - action: Action::S3Action(S3Action::ListBucketAction), - is_owner: false, - account: "", - groups: &None, - conditions: &conditions, - object: "", - }) + && PolicySys::try_is_allowed_for_store( + store.as_ref(), + &BucketPolicyArgs { + bucket: bucket.as_str(), + action: Action::S3Action(S3Action::ListBucketAction), + is_owner: false, + account: "", + groups: &None, + conditions: &conditions, + object: "", + }, + ) .await .map_err(ApiError::from)?); if policy_allowed { - deny_anonymous_table_data_plane_if_needed(action, bucket.as_str(), object.as_str()).await?; + deny_anonymous_table_data_plane_if_needed(req, action, bucket.as_str(), object.as_str()).await?; // RestrictPublicBuckets: when true, deny public access even if bucket policy allows it. - match get_public_access_block_config(bucket_name).await { - Ok((config, _)) => { - if config.restrict_public_buckets.unwrap_or(false) { - return Err(denial.deny("restrict_public_buckets", action)); - } - } + match store.restricts_public_bucket_access(bucket_name).await { + Ok(true) => return Err(denial.deny("restrict_public_buckets", action)), + Ok(false) => {} Err(StorageError::ConfigNotFound) => {} Err(_) => { return Err(denial.deny("public_access_block_unavailable", action)); @@ -1332,38 +1420,151 @@ fn table_data_plane_admin_action(action: Action) -> Option { } } -fn table_catalog_store_for_data_plane() -> S3Result> { - let store = - runtime_sources::current_object_store_handle().ok_or_else(|| s3_error!(InternalError, "object store not initialized"))?; - let backend = crate::table_catalog::EcStoreTableCatalogObjectBackend::new(store); +fn table_data_plane_content_mutation(action: Action) -> bool { + matches!( + action, + Action::S3Action( + S3Action::PutObjectAction + | S3Action::DeleteObjectAction + | S3Action::DeleteObjectVersionAction + | S3Action::RestoreObjectAction + | S3Action::ReplicateObjectAction + | S3Action::ReplicateDeleteAction + | S3Action::PutObjectFanOutAction + ) + ) +} + +fn table_catalog_backend_for_data_plane( + req: &S3Request, +) -> S3Result> { + let store = request_object_store(req)?; + Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new(store)) +} + +fn table_catalog_store_for_data_plane( + req: &S3Request, +) -> S3Result> { + let backend = table_catalog_backend_for_data_plane(req)?; crate::table_catalog::ConfiguredTableCatalogStore::from_env(backend) .map_err(|err| s3_error!(InternalError, "failed to configure table catalog backing: {}", err)) } -async fn table_data_plane_resource_for_request( - bucket: &str, - object: &str, -) -> S3Result> { - if bucket.is_empty() || object.is_empty() { - return Ok(None); +async fn retain_table_data_plane_publication_guard( + req: &mut S3Request, + table_bucket: &str, + lock_object: &str, +) -> S3Result<()> { + let key = (table_bucket.to_string(), lock_object.to_string()); + let retained = req + .extensions + .get::() + .cloned() + .unwrap_or_default(); + if retained.state.lock().keys.contains(&key) { + return Ok(()); } - match get_bucket_metadata(bucket).await { - Ok(metadata) if metadata.table_bucket_enabled() => {} - Ok(_) | Err(StorageError::ConfigNotFound) => return Ok(None), - Err(err) if is_err_bucket_not_found(&err) => return Ok(None), + let backend = table_catalog_backend_for_data_plane(req)?; + let guard = crate::table_catalog::TableCatalogObjectBackend::acquire_read_lock(&backend, table_bucket, lock_object) + .await + .map_err(|err| s3_error!(InternalError, "failed to acquire table publication guard: {}", err))?; + let mut state = retained.state.lock(); + state.keys.insert(key); + state.guards.push(guard); + drop(state); + req.extensions.insert(retained); + Ok(()) +} + +async fn retain_table_bucket_publication_guard(req: &mut S3Request, table_bucket: &str) -> S3Result<()> { + retain_table_data_plane_publication_guard( + req, + table_bucket, + &crate::table_catalog::default_table_bucket_publication_lock_path(), + ) + .await +} + +async fn retain_table_publication_guard( + req: &mut S3Request, + resource: &crate::table_catalog::TableDataPlaneResource, +) -> S3Result<()> { + let namespace = crate::table_catalog::Namespace::parse(&resource.namespace) + .map_err(|err| s3_error!(InternalError, "persisted table namespace is invalid: {}", err))?; + let table = crate::table_catalog::IdentifierSegment::parse(resource.table.clone()) + .map_err(|err| s3_error!(InternalError, "persisted table name is invalid: {}", err))?; + retain_table_data_plane_publication_guard( + req, + &resource.table_bucket, + &crate::table_catalog::default_table_publication_lock_path(&namespace, &table), + ) + .await?; + let retained = req + .extensions + .get::() + .cloned() + .ok_or_else(|| s3_error!(InternalError, "table publication guard state is missing"))?; + retained.state.lock().resources.insert( + (resource.table_bucket.clone(), resource.warehouse_object_prefix.clone()), + resource.clone(), + ); + Ok(()) +} + +async fn table_bucket_enabled_for_data_plane(req: &S3Request, bucket: &str) -> S3Result { + if bucket.is_empty() { + return Ok(false); + } + + match request_object_store(req)?.get_bucket_metadata(bucket).await { + Ok(metadata) => Ok(metadata.table_bucket_enabled()), + Err(StorageError::ConfigNotFound) => Ok(false), + Err(err) if is_err_bucket_not_found(&err) => Ok(false), Err(err) => { tracing::warn!( bucket = %bucket, error = %err, "failed to load bucket metadata while authorizing table data-plane access" ); - return Err(s3_error!(AccessDenied, "Access Denied")); + Err(s3_error!(AccessDenied, "Access Denied")) + } + } +} + +async fn table_data_plane_resource_for_request( + req: &mut S3Request, + bucket: &str, + object: &str, + table_bucket_enabled: bool, +) -> S3Result> { + if !table_bucket_enabled || bucket.is_empty() || object.is_empty() { + return Ok(None); + } + + let key = (bucket.to_string(), object.to_string()); + let retained = req + .extensions + .get::() + .cloned() + .unwrap_or_default(); + { + let state = retained.state.lock(); + if state.missing_resources.contains(&key) { + return Ok(None); + } + if let Some(resource) = state + .resources + .values() + .find(|resource| resource.table_bucket == bucket && object.starts_with(&resource.warehouse_object_prefix)) + .cloned() + { + return Ok(Some(resource)); } } - let store = table_catalog_store_for_data_plane()?; - crate::table_catalog::table_data_plane_resource_for_object(&store, bucket, object) + let store = table_catalog_store_for_data_plane(req)?; + let resource = crate::table_catalog::table_data_plane_resource_for_object(&store, bucket, object) .await .map_err(|err| { tracing::warn!( @@ -1373,27 +1574,48 @@ async fn table_data_plane_resource_for_request( "failed to resolve table data-plane resource" ); s3_error!(AccessDenied, "Access Denied") - }) + })?; + let bucket_fence_key = (bucket.to_string(), crate::table_catalog::default_table_bucket_publication_lock_path()); + let mut state = retained.state.lock(); + if resource.is_none() && state.keys.contains(&bucket_fence_key) { + state.missing_resources.insert(key); + drop(state); + req.extensions.insert(retained); + } + Ok(resource) } -async fn authorize_table_data_plane_if_needed( +async fn table_data_plane_resource_for_authorization( + req: &mut S3Request, + bucket: &str, + object: &str, + table_bucket_enabled: bool, +) -> S3Result> { + table_data_plane_resource_for_request(req, bucket, object, table_bucket_enabled).await +} + +async fn authorize_table_data_plane_if_needed( + req: &mut S3Request, action: Action, bucket: &str, object: &str, cred: &rustfs_credentials::Credentials, is_owner: bool, conditions: &HashMap>, - claims: &HashMap, ) -> S3Result<()> { let Some(admin_action) = table_data_plane_admin_action(action) else { return Ok(()); }; - let Some(resource) = table_data_plane_resource_for_request(bucket, object).await? else { + if table_data_plane_content_mutation(action) { + retain_table_bucket_publication_guard(req, bucket).await?; + } + let table_bucket_enabled = table_bucket_enabled_for_data_plane(req, bucket).await?; + let Some(resource) = table_data_plane_resource_for_authorization(req, bucket, object, table_bucket_enabled).await? else { return Ok(()); }; - let Ok(iam_store) = runtime_sources::current_ready_iam_handle() else { - return Err(s3_error!(InternalError, "iam not init")); - }; + let iam_store = request_iam_store(req)?; + let default_claims = HashMap::new(); + let claims = cred.claims.as_ref().unwrap_or(&default_claims); let resource_object = resource.catalog_resource_object(); let allowed = iam_store @@ -1410,6 +1632,9 @@ async fn authorize_table_data_plane_if_needed( }) .await; if allowed { + if table_data_plane_content_mutation(action) { + retain_table_publication_guard(req, &resource).await?; + } return Ok(()); } @@ -1426,11 +1651,31 @@ async fn authorize_table_data_plane_if_needed( Err(s3_error!(AccessDenied, "Access Denied")) } -async fn deny_anonymous_table_data_plane_if_needed(action: Action, bucket: &str, object: &str) -> S3Result<()> { +/// Authorizes one exact object key after its enclosing table-catalog operation has been authorized. +pub(crate) async fn authorize_internal_object_request(req: &mut S3Request, action: Action) -> S3Result<()> { + req.extensions.insert(InternalObjectAuthorization); + let result = authorize_request(req, action).await; + req.extensions.remove::(); + result +} + +async fn deny_anonymous_table_data_plane_if_needed( + req: &mut S3Request, + action: Action, + bucket: &str, + object: &str, +) -> S3Result<()> { if table_data_plane_admin_action(action).is_none() { return Ok(()); } - if table_data_plane_resource_for_request(bucket, object).await?.is_some() { + if table_data_plane_content_mutation(action) { + retain_table_bucket_publication_guard(req, bucket).await?; + } + let table_bucket_enabled = table_bucket_enabled_for_data_plane(req, bucket).await?; + if table_data_plane_resource_for_authorization(req, bucket, object, table_bucket_enabled) + .await? + .is_some() + { return Err(s3_error!(AccessDenied, "Access Denied")); } Ok(()) @@ -2709,15 +2954,17 @@ impl S3Access for FS { mod tests { use super::{ AMZ_WRITE_OFFSET_BYTES_HEADER, BucketGenerationGuard, BucketPolicyArgs, BucketPolicyExistingObjectTagHint, - BucketPolicyRawLoadErrorKind, DenialContext, FS, ObjectTagConditions, PostObjectRequestMarker, ReqInfo, S3Access, - StorageError, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, - bucket_policy_needs_existing_object_tag_from_hint, bucket_website_config_authorize_action, - classify_bucket_policy_raw_load_error, complete_multipart_upload_authorize_action, get_bucket_policy_authorize_action, - has_write_offset_bytes_header, install_restore_authorization_test_hook, legal_hold_write_requested, - list_parts_authorize_action, load_bucket_policy_existing_object_tag_hint, maybe_merge_object_tag_conditions, - merge_list_bucket_query_conditions, merge_request_object_tag_conditions, owner_can_bypass_policy_deny, - post_object_authorize_action, put_bucket_policy_authorize_action, request_context_from_req, retention_write_requested, - secondary_tag_hint_action, table_data_plane_admin_action, validate_post_object_success_controls, versioned_read_action, + BucketPolicyRawLoadErrorKind, DenialContext, FS, InternalObjectAuthorization, ObjectTagConditions, + PostObjectRequestMarker, ReqInfo, S3Access, StorageError, TableDataPlanePublicationGuards, apply_bucket_generation_guard, + apply_copy_source_bucket_generation_guard, authorization_conditions, bucket_policy_needs_existing_object_tag_from_hint, + bucket_website_config_authorize_action, classify_bucket_policy_raw_load_error, + complete_multipart_upload_authorize_action, get_bucket_policy_authorize_action, has_write_offset_bytes_header, + install_restore_authorization_test_hook, legal_hold_write_requested, list_parts_authorize_action, + load_bucket_policy_existing_object_tag_hint, maybe_merge_object_tag_conditions, merge_list_bucket_query_conditions, + merge_request_object_tag_conditions, owner_can_bypass_policy_deny, post_object_authorize_action, + put_bucket_policy_authorize_action, request_context_from_req, request_object_store, retention_write_requested, + secondary_tag_hint_action, table_data_plane_admin_action, table_data_plane_content_mutation, + table_data_plane_resource_for_request, validate_post_object_success_controls, versioned_read_action, }; use crate::error::ApiError; use crate::storage::storage_api::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions}; @@ -2856,6 +3103,142 @@ mod tests { assert_eq!(table_data_plane_admin_action(Action::S3Action(S3Action::ListBucketAction)), None); } + #[test] + fn table_data_plane_publication_guard_covers_content_mutations_only() { + for action in [ + S3Action::PutObjectAction, + S3Action::DeleteObjectAction, + S3Action::DeleteObjectVersionAction, + S3Action::RestoreObjectAction, + S3Action::ReplicateObjectAction, + S3Action::ReplicateDeleteAction, + S3Action::PutObjectFanOutAction, + ] { + assert!(table_data_plane_content_mutation(Action::S3Action(action))); + } + for action in [ + S3Action::GetObjectAction, + S3Action::PutObjectTaggingAction, + S3Action::DeleteObjectTaggingAction, + S3Action::ListBucketAction, + ] { + assert!(!table_data_plane_content_mutation(Action::S3Action(action))); + } + } + + #[test] + fn table_data_plane_mutations_fence_before_table_bucket_marker_lookup() { + let source = include_str!("access.rs"); + for (function, end_marker) in [ + ("async fn authorize_table_data_plane_if_needed", "/// Authorizes one exact object key"), + ( + "async fn deny_anonymous_table_data_plane_if_needed", + "fn validate_post_object_success_controls", + ), + ] { + let start = source.find(function).expect("authorization helper should exist"); + let end = source[start..] + .find(end_marker) + .map(|offset| start + offset) + .expect("authorization helper boundary should exist"); + let block = &source[start..end]; + let fence = block + .find("retain_table_bucket_publication_guard(req, bucket).await?;") + .expect("content mutation should retain the bucket publication fence"); + let marker = block + .find("table_bucket_enabled_for_data_plane(req, bucket).await?") + .expect("authorization should load the table bucket marker"); + assert!(fence < marker, "{function} must fence pre-enable writers before reading the marker"); + } + } + + #[tokio::test] + async fn table_data_plane_request_reuses_table_resource_for_distinct_object_while_commit_waits() { + let resource = crate::table_catalog::TableDataPlaneResource { + table_bucket: "warehouse".to_string(), + namespace: "analytics".to_string(), + table: "events".to_string(), + table_id: "table-id".to_string(), + warehouse_object_prefix: "tables/table-id/".to_string(), + }; + let cached_key = ("warehouse".to_string(), "tables/table-id/".to_string()); + let requested_key = ("warehouse".to_string(), "tables/table-id/data/part-00002.parquet".to_string()); + let retained = TableDataPlanePublicationGuards::default(); + retained.state.lock().resources.insert(cached_key, resource.clone()); + + let publication_lock = Arc::new(tokio::sync::Mutex::new(())); + let request_publication_guard = Arc::clone(&publication_lock).lock_owned().await; + retained.state.lock().guards.push(Box::new(request_publication_guard)); + let mut req = build_request((), Method::PUT); + req.extensions.insert(retained.clone()); + + let catalog_lock = Arc::new(tokio::sync::Mutex::new(())); + let commit_started = Arc::new(tokio::sync::Notify::new()); + let commit_catalog_lock = Arc::clone(&catalog_lock); + let commit_publication_lock = Arc::clone(&publication_lock); + let commit_started_signal = Arc::clone(&commit_started); + let commit = tokio::spawn(async move { + let _catalog_guard = commit_catalog_lock.lock().await; + commit_started_signal.notify_one(); + let _publication_guard = commit_publication_lock.lock().await; + }); + commit_started.notified().await; + + let resolved = tokio::time::timeout( + std::time::Duration::from_secs(1), + table_data_plane_resource_for_request(&mut req, &requested_key.0, &requested_key.1, true), + ) + .await + .expect("cached table resolution must not wait for the catalog lock") + .expect("cached table resolution should succeed"); + assert_eq!(resolved, Some(resource)); + + retained.state.lock().guards.clear(); + tokio::time::timeout(std::time::Duration::from_secs(1), commit) + .await + .expect("commit should continue after the request releases its publication guard") + .expect("commit task should join"); + } + + #[tokio::test] + async fn table_data_plane_request_reuses_absence_while_bucket_publication_guard_is_held() { + let key = ("warehouse".to_string(), "tables/new-table/data/file.parquet".to_string()); + let bucket_fence = (key.0.clone(), crate::table_catalog::default_table_bucket_publication_lock_path()); + let retained = TableDataPlanePublicationGuards::default(); + { + let mut state = retained.state.lock(); + state.keys.insert(bucket_fence); + state.missing_resources.insert(key.clone()); + } + + let publication_lock = Arc::new(tokio::sync::Mutex::new(())); + let request_publication_guard = Arc::clone(&publication_lock).lock_owned().await; + retained.state.lock().guards.push(Box::new(request_publication_guard)); + let mut req = build_request((), Method::PUT); + req.extensions.insert(retained.clone()); + + let commit_publication_lock = Arc::clone(&publication_lock); + let commit = tokio::spawn(async move { + let _publication_guard = commit_publication_lock.lock().await; + }); + tokio::task::yield_now().await; + + let resolved = tokio::time::timeout( + std::time::Duration::from_secs(1), + table_data_plane_resource_for_request(&mut req, &key.0, &key.1, true), + ) + .await + .expect("cached absence must not reread the catalog while publication waits") + .expect("cached absence should resolve successfully"); + assert!(resolved.is_none()); + + retained.state.lock().guards.clear(); + tokio::time::timeout(std::time::Duration::from_secs(1), commit) + .await + .expect("publication should continue after the request releases its bucket guard") + .expect("publication task should join"); + } + #[test] fn legal_hold_write_requested_is_true_when_status_present() { assert!(legal_hold_write_requested(Some(&ObjectLockLegalHoldStatus::from_static( @@ -2998,9 +3381,41 @@ mod tests { ); } + #[test] + fn internal_object_authorization_ignores_unapplied_request_options() { + let mut req = build_request((), Method::POST); + req.headers.insert( + "authorization", + HeaderValue::from_static("AWS4-HMAC-SHA256 Credential=test/20260801/us-east-1/s3tables/aws4_request"), + ); + req.headers.insert("user-agent", HeaderValue::from_static("pyiceberg/test")); + req.headers + .insert("x-amz-server-side-encryption", HeaderValue::from_static("AES256")); + req.headers + .insert("x-amz-tagging", HeaderValue::from_static("classification=%ZZ")); + req.extensions.insert(InternalObjectAuthorization); + + let credentials = rustfs_credentials::Credentials::default(); + let conditions = + authorization_conditions(&req, &credentials, None, None, None, None, Action::S3Action(S3Action::PutObjectAction)) + .expect("internal object authorization conditions should build"); + + assert_eq!(conditions.get("authType"), Some(&vec!["REST-HEADER".to_string()])); + assert_eq!(conditions.get("signatureversion"), Some(&vec!["AWS4-HMAC-SHA256".to_string()])); + assert_eq!(conditions.get("UserAgent"), Some(&vec!["pyiceberg/test".to_string()])); + assert!(!conditions.contains_key("authorization")); + assert!(!conditions.contains_key("x-amz-server-side-encryption")); + assert!(!conditions.contains_key("RequestObjectTag/classification")); + assert!(!conditions.contains_key("RequestObjectTagKeys")); + } + #[tokio::test] async fn test_required_existing_object_tags_are_merged_into_authorization_conditions() { + let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await; + let server_ctx = ServerContextSlot::new(); + assert!(server_ctx.install(Arc::new(AppContext::new(store, Arc::new(UnreadyIam), Arc::new(TestKms),)))); let mut request = build_request((), Method::GET); + request.extensions.insert(server_ctx); request.extensions.insert(ObjectTagConditions::new( "bucket", "tagged-object", @@ -3035,7 +3450,9 @@ mod tests { #[ignore = "requires isolated global object layer state"] async fn test_bucket_policy_needs_existing_object_tag_load_failure_is_conservative() { let conditions = HashMap::new(); + let store = crate::app::gating_test_env::shared_gating_ecstore().await; let hint = load_bucket_policy_existing_object_tag_hint( + store.as_ref(), "test-bucket-no-policy-xyz-absent", Action::S3Action(S3Action::GetObjectAction), ) @@ -3349,6 +3766,7 @@ mod tests { .expect("put object input should build"); let mut req = build_request(input, Method::PUT); ensure_req_info(&mut req); + req.extensions.insert(fs.server_ctx().clone()); fs.put_object(&mut req) .await @@ -3366,6 +3784,157 @@ mod tests { ); } + #[tokio::test] + #[serial] + async fn request_slot_keeps_bucket_policy_bound_to_its_store() { + let store_a = crate::app::gating_test_env::shared_gating_ecstore().await; + let (_store_b_temp, _store_b_paths, store_b) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await; + let bucket = format!("request-policy-isolation-{}", uuid::Uuid::new_v4()); + for store in [&store_a, &store_b] { + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create isolated policy test bucket"); + } + + let allow_policy = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"# + ); + let deny_policy = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Deny","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"# + ); + for (store, policy) in [(&store_a, allow_policy), (&store_b, deny_policy)] { + store + .update_bucket_metadata_config( + &bucket, + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG, + policy.into_bytes(), + ) + .await + .expect("publish isolated bucket policy"); + } + + let slot_a = ServerContextSlot::new(); + assert!(slot_a.install(Arc::new(AppContext::new(Arc::clone(&store_a), Arc::new(UnreadyIam), Arc::new(TestKms),)))); + let slot_b = ServerContextSlot::new(); + assert!(slot_b.install(Arc::new(AppContext::new(Arc::clone(&store_b), Arc::new(UnreadyIam), Arc::new(TestKms),)))); + + let mut request_a = build_request((), Method::PUT); + request_a.extensions.insert(ReqInfo { + bucket: Some(bucket.clone()), + object: Some("object".to_string()), + ..Default::default() + }); + request_a.extensions.insert(slot_a); + super::authorize_request(&mut request_a, Action::S3Action(S3Action::PutObjectAction)) + .await + .expect("store A policy should authorize store A request"); + + let mut request_b = build_request((), Method::PUT); + request_b.extensions.insert(ReqInfo { + bucket: Some(bucket.clone()), + object: Some("object".to_string()), + ..Default::default() + }); + request_b.extensions.insert(slot_b); + let err = super::authorize_request(&mut request_b, Action::S3Action(S3Action::PutObjectAction)) + .await + .expect_err("store B policy must deny store B request"); + assert_eq!(err.code(), &S3ErrorCode::AccessDenied); + + let allow_policy = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"# + ); + store_b + .update_bucket_metadata_config( + &bucket, + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG, + allow_policy.into_bytes(), + ) + .await + .expect("replace store B bucket policy"); + let slot_b = ServerContextSlot::new(); + assert!(slot_b.install(Arc::new(AppContext::new(Arc::clone(&store_b), Arc::new(UnreadyIam), Arc::new(TestKms),)))); + let mut allowed_request_b = build_request((), Method::PUT); + allowed_request_b.extensions.insert(ReqInfo { + bucket: Some(bucket.clone()), + object: Some("object".to_string()), + ..Default::default() + }); + allowed_request_b.extensions.insert(slot_b); + super::authorize_request(&mut allowed_request_b, Action::S3Action(S3Action::PutObjectAction)) + .await + .expect("store B policy should allow before its public access block is installed"); + + store_b + .update_bucket_metadata_config( + &bucket, + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, + br#"true"# + .to_vec(), + ) + .await + .expect("publish store B public access block configuration"); + let slot_b = ServerContextSlot::new(); + assert!(slot_b.install(Arc::new(AppContext::new(Arc::clone(&store_b), Arc::new(UnreadyIam), Arc::new(TestKms),)))); + let mut restricted_request_b = build_request((), Method::PUT); + restricted_request_b.extensions.insert(ReqInfo { + bucket: Some(bucket.clone()), + object: Some("object".to_string()), + ..Default::default() + }); + restricted_request_b.extensions.insert(slot_b); + let err = super::authorize_request(&mut restricted_request_b, Action::S3Action(S3Action::PutObjectAction)) + .await + .expect_err("store B public access block must restrict store B request"); + assert_eq!(err.code(), &S3ErrorCode::AccessDenied); + + let tag_bucket = format!("request-tag-isolation-{}", uuid::Uuid::new_v4()); + for (store, tags) in [(&store_a, "classification=ambient"), (&store_b, "classification=request")] { + store + .make_bucket(&tag_bucket, &MakeBucketOptions::default()) + .await + .expect("create isolated tag test bucket"); + let mut reader = crate::storage::PutObjReader::from_vec(b"tagged object".to_vec()); + store + .put_object(&tag_bucket, "object", &mut reader, &crate::storage::ObjectOptions::default()) + .await + .expect("create isolated tagged object"); + store + .put_object_tags(&tag_bucket, "object", tags, &crate::storage::ObjectOptions::default()) + .await + .expect("publish isolated object tags"); + } + let tag_policy = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{tag_bucket}/*"],"Condition":{{"StringEquals":{{"s3:ExistingObjectTag/classification":"request"}}}}}}]}}"# + ); + store_b + .update_bucket_metadata_config( + &tag_bucket, + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG, + tag_policy.into_bytes(), + ) + .await + .expect("publish store B tag-conditioned policy"); + let slot_b = ServerContextSlot::new(); + assert!(slot_b.install(Arc::new(AppContext::new(Arc::clone(&store_b), Arc::new(UnreadyIam), Arc::new(TestKms),)))); + let mut tagged_request_b = build_request((), Method::PUT); + tagged_request_b.extensions.insert(ReqInfo { + bucket: Some(tag_bucket), + object: Some("object".to_string()), + ..Default::default() + }); + tagged_request_b.extensions.insert(slot_b); + super::authorize_request(&mut tagged_request_b, Action::S3Action(S3Action::PutObjectAction)) + .await + .expect("store B existing object tags should authorize store B request"); + + let mut unready_request = build_request((), Method::PUT); + unready_request.extensions.insert(ServerContextSlot::new()); + let err = request_object_store(&unready_request).expect_err("an unready request slot must not use an ambient store"); + assert_eq!(err.code(), &S3ErrorCode::InternalError); + } + #[tokio::test] #[serial] async fn delete_object_access_captures_authorized_bucket_incarnation() { @@ -3412,6 +3981,7 @@ mod tests { .expect("delete object input should build"); let mut req = build_request(input, Method::DELETE); ensure_req_info(&mut req); + req.extensions.insert(fs.server_ctx().clone()); fs.delete_object(&mut req) .await @@ -3510,6 +4080,7 @@ mod tests { .expect("copy input should build"); let mut req = build_request(input, Method::PUT); ensure_req_info(&mut req); + req.extensions.insert(fs.server_ctx().clone()); fs.copy_object(&mut req) .await @@ -3539,6 +4110,7 @@ mod tests { .expect("part copy input should build"); let mut upload_req = build_request(upload_input, Method::PUT); ensure_req_info(&mut upload_req); + upload_req.extensions.insert(fs.server_ctx().clone()); fs.upload_part_copy(&mut upload_req) .await .expect("test policies should authorize the part copy request"); @@ -3627,15 +4199,19 @@ mod tests { .expect("restore object input should build"); let mut req = build_request(input, Method::POST); ensure_req_info(&mut req); - let authorized = Arc::new(tokio::sync::Barrier::new(2)); - let resume = Arc::new(tokio::sync::Barrier::new(2)); - install_restore_authorization_test_hook(bucket.clone(), Arc::clone(&authorized), Arc::clone(&resume)); + req.extensions.insert(fs.server_ctx().clone()); + let (authorized_tx, authorized_rx) = tokio::sync::oneshot::channel(); + let (resume_tx, resume_rx) = tokio::sync::oneshot::channel(); + install_restore_authorization_test_hook(bucket.clone(), authorized_tx, resume_rx); let access = tokio::spawn(async move { let result = fs.restore_object(&mut req).await; (result, req) }); - authorized.wait().await; + tokio::time::timeout(std::time::Duration::from_secs(10), authorized_rx) + .await + .expect("RestoreObject authorization should reach the test hook") + .expect("RestoreObject access must not fail before reaching the test hook"); store .delete_bucket(&bucket, &DeleteBucketOptions::default()) @@ -3650,7 +4226,9 @@ mod tests { .await .expect("read the recreated bucket incarnation"); assert_ne!(authorized_incarnation_id, recreated_incarnation_id); - resume.wait().await; + resume_tx + .send(()) + .expect("RestoreObject access should still be waiting at the test hook"); let (result, req) = access.await.expect("RestoreObject access task should join"); result.expect("the already-authorized request should retain its generation guard"); diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index c946db98e..e27c6dc29 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -34,7 +34,7 @@ use crate::storage::storage_api::ecfs_consumer::contract::{ use crate::storage::storage_api::ecfs_consumer::object_lock::{ parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled, }; -use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; +use crate::storage::storage_api::runtime_sources_consumer::{ECStore, runtime_sources}; use crate::table_catalog; use http::StatusCode; use metrics::{counter, histogram}; @@ -128,6 +128,15 @@ impl FS { let Some(store) = self.server_ctx.object_store() else { return Ok(std::collections::HashMap::new()); }; + Self::get_object_tag_conditions_for_policy_from_store(store.as_ref(), bucket, object, version_id).await + } + + pub(crate) async fn get_object_tag_conditions_for_policy_from_store( + store: &ECStore, + bucket: &str, + object: &str, + version_id: Option<&str>, + ) -> S3Result>> { let opts = ObjectOptions { version_id: version_id.map(String::from), ..Default::default() diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index 84509502e..58b6a0ce9 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -67,15 +67,14 @@ pub(crate) use storage_api::{ ecstore_layout, ecstore_metrics, ecstore_notification, ecstore_rebalance, ecstore_rio, ecstore_rpc, ecstore_set_disk, ecstore_storage, ecstore_tier, encode_tags, find_local_disk_by_ref, get_bucket_accelerate_config, get_bucket_cors_config, get_bucket_logging_config, get_bucket_metadata, get_bucket_notification_config, get_bucket_object_lock_config, - get_bucket_policy_raw, get_bucket_replication_config, get_bucket_request_payment_config, get_bucket_sse_config, - get_bucket_website_config, get_local_server_property, get_lock_acquire_timeout, get_public_access_block_config, - head_prefix_consumer, helper_consumer, init_background_replication, init_bucket_metadata_sys, init_ecstore_config, - init_local_disks_with_instance_ctx, init_lock_clients, is_err_bucket_not_found, is_err_object_not_found, - is_err_version_not_found, is_valid_storage_class, options_consumer, prewarm_local_disk_id_map_with_instance_ctx, read_config, - record_replication_proxy, rpc_consumer, runtime_sources_consumer, s3_api_consumer, serialize, table_catalog_path_hash, - to_s3s_etag, topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata, try_migrate_iam_config, - try_migrate_server_config, update_bucket_metadata_config, update_bucket_metadata_config_if_incarnation, verify_rpc_signature, - wrap_reader, + get_bucket_replication_config, get_bucket_request_payment_config, get_bucket_sse_config, get_bucket_website_config, + get_local_server_property, get_lock_acquire_timeout, head_prefix_consumer, helper_consumer, init_background_replication, + init_bucket_metadata_sys, init_ecstore_config, init_local_disks_with_instance_ctx, init_lock_clients, + is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, is_valid_storage_class, options_consumer, + prewarm_local_disk_id_map_with_instance_ctx, read_config, record_replication_proxy, rpc_consumer, runtime_sources_consumer, + s3_api_consumer, serialize, table_catalog_path_hash, to_s3s_etag, topology_snapshot_from_endpoint_pools_with_capabilities, + try_migrate_bucket_metadata, try_migrate_iam_config, try_migrate_server_config, update_bucket_metadata_config, + update_bucket_metadata_config_if_incarnation, verify_rpc_signature, wrap_reader, }; #[cfg(test)] diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 9387929a5..b765b9069 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -109,9 +109,9 @@ pub(crate) use super::sse::{ pub(crate) mod access_consumer { pub(crate) use super::super::access::{ PostObjectRequestMarker, ReqInfo, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, - authorize_request, bucket_config_mutation_incarnation, has_bypass_governance_header, load_bucket_generation_from_store, - log_list_buckets_iam_implicit_deny, prepare_list_buckets_iam_authorization, recursive_force_delete_is_authorized, - replication_request_authorized, req_info_mut, req_info_ref, + authorize_internal_object_request, authorize_request, bucket_config_mutation_incarnation, has_bypass_governance_header, + load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, prepare_list_buckets_iam_authorization, + recursive_force_delete_is_authorized, replication_request_authorized, req_info_mut, req_info_ref, }; } @@ -1448,10 +1448,6 @@ pub(crate) async fn get_bucket_accelerate_config( ecstore_bucket::metadata_sys::get_accelerate_config(bucket).await } -pub(crate) async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_bucket_policy_raw(bucket).await -} - pub(crate) async fn get_bucket_cors_config(bucket: &str) -> Result<(s3s::dto::CORSConfiguration, time::OffsetDateTime)> { ecstore_bucket::metadata_sys::get_cors_config(bucket).await } @@ -1466,12 +1462,6 @@ pub(crate) async fn get_bucket_object_lock_config( ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await } -pub(crate) async fn get_public_access_block_config( - bucket: &str, -) -> Result<(s3s::dto::PublicAccessBlockConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_public_access_block_config(bucket).await -} - pub(crate) async fn get_bucket_replication_config( bucket: &str, ) -> Result<(s3s::dto::ReplicationConfiguration, time::OffsetDateTime)> { diff --git a/rustfs/src/table_catalog/iceberg/commit.rs b/rustfs/src/table_catalog/iceberg/commit.rs index 89756743e..224654517 100644 --- a/rustfs/src/table_catalog/iceberg/commit.rs +++ b/rustfs/src/table_catalog/iceberg/commit.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::{BTreeMap, BTreeSet}; + use super::super::*; pub(crate) fn commit_log_matches_request(commit_log: &CommitLogEntry, request: &TableCommitRequest, table_id: &str) -> bool { @@ -39,6 +41,49 @@ pub(crate) fn table_matches_staged_base(table: &TableEntry, commit_log: &CommitL && table.version_token == commit_log.expected_version_token } +pub(crate) struct TableCommitHistoryIndex<'a> { + table_id: &'a str, + reachable_states: BTreeSet<(&'a str, &'a str)>, +} + +impl<'a> TableCommitHistoryIndex<'a> { + pub(crate) fn new(table: &'a TableEntry, commits: impl IntoIterator) -> Self { + let mut by_new_state = BTreeMap::<(&str, &str), Option<(&str, &str)>>::new(); + for commit in commits + .into_iter() + .filter(|commit| commit.table_id == table.table_id && !matches!(commit.status, CommitLogStatus::Failed)) + { + let key = (commit.new_metadata_location.as_str(), commit.new_version_token.as_str()); + let previous = (commit.previous_metadata_location.as_str(), commit.expected_version_token.as_str()); + by_new_state + .entry(key) + .and_modify(|candidate| *candidate = None) + .or_insert(Some(previous)); + } + + let mut reachable_states = BTreeSet::new(); + let mut state = (table.metadata_location.as_str(), table.version_token.as_str()); + while reachable_states.insert(state) { + let Some(Some(previous)) = by_new_state.get(&state) else { + break; + }; + state = *previous; + } + Self { + table_id: &table.table_id, + reachable_states, + } + } + + pub(crate) fn proves_committed(&self, target: &CommitLogEntry) -> bool { + self.table_id == target.table_id.as_str() + && !matches!(target.status, CommitLogStatus::Failed) + && self + .reachable_states + .contains(&(target.new_metadata_location.as_str(), target.new_version_token.as_str())) + } +} + pub(crate) fn table_catalog_recovery_summary( metadata_status: &TableMetadataPointerStatus, commit_recovery: &TableCommitRecoveryReport, @@ -76,7 +121,7 @@ pub(crate) fn table_catalog_recovery_summary( (metadata_status.unwrap_or(TableCatalogRecoveryStatus::Healthy), actions) } -fn commit_logs_share_recovery_payload(left: &CommitLogEntry, right: &CommitLogEntry) -> bool { +pub(crate) fn commit_logs_share_recovery_payload(left: &CommitLogEntry, right: &CommitLogEntry) -> bool { left.version == right.version && left.commit_id == right.commit_id && left.idempotency_key == right.idempotency_key @@ -109,6 +154,7 @@ pub(crate) fn table_commit_recovery_entry( table: &TableEntry, commit_log: &CommitLogEntry, idempotency_commit: Option<&CommitLogEntry>, + historically_committed: bool, ) -> TableCommitRecoveryEntry { let idempotency_index_status = commit_idempotency_index_status(commit_log, idempotency_commit); let idempotency_index_present = matches!( @@ -127,6 +173,11 @@ pub(crate) fn table_commit_recovery_entry( TableCommitRecoveryState::ManualReview, "idempotency index points at a different commit payload".to_string(), ) + } else if matches!(commit_log.status, CommitLogStatus::Failed) { + ( + TableCommitRecoveryState::ManualReview, + "failed commit log cannot be finalized automatically".to_string(), + ) } else if table_matches_committed_log(table, commit_log) { if matches!(commit_log.status, CommitLogStatus::Committed) { if idempotency_index_repair_required { @@ -146,6 +197,11 @@ pub(crate) fn table_commit_recovery_entry( "current table pointer already advanced but commit log is not finalized".to_string(), ) } + } else if matches!(commit_log.status, CommitLogStatus::Staged) && historically_committed { + ( + TableCommitRecoveryState::FinalizationRequired, + "a later committed pointer proves this staged commit is part of table history".to_string(), + ) } else if matches!(commit_log.status, CommitLogStatus::Committed) { if idempotency_index_repair_required { ( diff --git a/rustfs/src/table_catalog/iceberg/validation.rs b/rustfs/src/table_catalog/iceberg/validation.rs index ebcc2045f..56953e9ba 100644 --- a/rustfs/src/table_catalog/iceberg/validation.rs +++ b/rustfs/src/table_catalog/iceberg/validation.rs @@ -175,6 +175,46 @@ pub(crate) fn table_metadata_warehouse_location( metadata_warehouse_location(table_bucket, metadata_location, metadata_object, validate_table_warehouse_location) } +pub(crate) fn canonical_json_sha256(metadata: &serde_json::Value) -> TableCatalogStoreResult { + let canonical = serde_json::to_vec(metadata) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to encode metadata digest input: {err}")))?; + Ok(hex_simd::encode_to_string(Sha256::digest(canonical), hex_simd::AsciiCase::Lower)) +} + +pub(crate) fn validate_commit_metadata_digest( + request: &TableCommitRequest, + metadata_object: &TableCatalogObject, +) -> TableCatalogStoreResult<()> { + let mut expected_digest = None; + for requirement in &request.requirements { + if requirement.get("type").and_then(serde_json::Value::as_str) != Some(TABLE_METADATA_DIGEST_REQUIREMENT_TYPE) { + continue; + } + if expected_digest.is_some() { + return Err(TableCatalogStoreError::Invalid( + "commit contains duplicate metadata digest requirements".to_string(), + )); + } + expected_digest = Some( + requirement + .get("sha256") + .and_then(serde_json::Value::as_str) + .filter(|digest| rustfs_utils::crypto::is_sha256_checksum(digest)) + .ok_or_else(|| TableCatalogStoreError::Invalid("commit metadata digest is invalid".to_string()))?, + ); + } + let Some(expected_digest) = expected_digest else { + return Ok(()); + }; + let metadata = decode_table_metadata_json(&request.new_metadata_location, &metadata_object.data)?; + if canonical_json_sha256(&metadata)? != expected_digest { + return Err(TableCatalogStoreError::Conflict( + "new metadata object changed after commit validation".to_string(), + )); + } + Ok(()) +} + pub(crate) fn view_metadata_warehouse_location( table_bucket: &str, metadata_location: &str, @@ -197,6 +237,10 @@ pub(crate) fn warehouse_index_candidate_prefixes(object: &str) -> Vec<&str> { prefixes } +pub(crate) fn warehouse_object_prefixes_overlap(left: &str, right: &str) -> bool { + left.starts_with(right) || right.starts_with(left) +} + pub(crate) fn table_data_plane_resource_from_entry(table: TableEntry, warehouse_object_prefix: String) -> TableDataPlaneResource { TableDataPlaneResource { table_bucket: table.table_bucket, @@ -1352,7 +1396,9 @@ where validate_snapshot_graph_data_file_reference(reference, format_version, manifest_sequence_number)?; let object_key = snapshot_graph_object_key(context, &reference.location, reference.object_kind.clone())?; match reference.entry_status { - Some(0 | 1) if budget.validated_live_objects.insert(object_key.clone()) => live_object_keys.push(object_key), + Some(0 | 1) if budget.validated_live_objects.insert(object_key.clone()) => { + live_object_keys.push(object_key); + } Some(0 | 1) => {} Some(2) => {} Some(_) => { diff --git a/rustfs/src/table_catalog/identifier.rs b/rustfs/src/table_catalog/identifier.rs index 521b89da3..279a246a8 100644 --- a/rustfs/src/table_catalog/identifier.rs +++ b/rustfs/src/table_catalog/identifier.rs @@ -177,6 +177,14 @@ pub(crate) fn default_table_root_prefix(namespace: &Namespace) -> String { format!("{}{}/{}/", default_namespace_root_prefix(), namespace.storage_id(), TABLE_ROOT) } +pub(crate) fn default_table_publication_lock_path(namespace: &Namespace, table: &IdentifierSegment) -> String { + format!("{}{}/publication.lock", default_table_root_prefix(namespace), table.as_str()) +} + +pub(crate) fn default_table_bucket_publication_lock_path() -> String { + rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH.to_string() +} + pub(crate) fn default_table_marker_path(namespace: &Namespace, table: &IdentifierSegment) -> String { format!("{}{}/{}", default_table_root_prefix(namespace), table.as_str(), TABLE_MARKER_FILE) } diff --git a/rustfs/src/table_catalog/mod.rs b/rustfs/src/table_catalog/mod.rs index d7733abbc..51259a855 100644 --- a/rustfs/src/table_catalog/mod.rs +++ b/rustfs/src/table_catalog/mod.rs @@ -72,9 +72,10 @@ pub(crate) use error::{TableCatalogStoreError, TableCatalogStoreResult}; pub(crate) use iceberg::*; pub use identifier::{IdentifierSegment, Namespace, is_reserved_table_object_key}; pub(crate) use identifier::{ - default_table_data_dir_path, default_table_delete_dir_path, default_table_metadata_dir_path, - default_table_metadata_file_path, default_view_metadata_file_path, is_valid_table_metadata_location, - is_valid_view_metadata_location, metadata_location_from_metadata_file_path, validate_bucket_object_mutation, + default_table_bucket_publication_lock_path, default_table_data_dir_path, default_table_delete_dir_path, + default_table_metadata_dir_path, default_table_metadata_file_path, default_table_publication_lock_path, + default_view_metadata_file_path, is_valid_table_metadata_location, is_valid_view_metadata_location, + metadata_location_from_metadata_file_path, validate_bucket_object_mutation, }; pub(crate) use maintenance::*; pub(crate) use model::*; @@ -89,12 +90,16 @@ pub(crate) const TABLE_NAMESPACE_MARKER_VERSION: u16 = 1; pub(crate) const TABLE_RESOURCE_MARKER_VERSION: u16 = 1; pub(crate) const TABLE_METADATA_POINTER_VERSION: u16 = 1; pub(crate) const TABLE_CATALOG_ENTRY_VERSION: u16 = 1; +pub(crate) const TABLE_WAREHOUSE_INDEX_STATE_VERSION: u16 = 2; pub(crate) const TABLE_MAINTENANCE_CONFIG_VERSION: u16 = 1; pub(crate) const TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION: u16 = 1; pub(crate) const TABLE_CATALOG_BACKING_MANIFEST_VERSION: u16 = 1; pub(crate) const ENV_TABLE_CATALOG_BACKING: &str = "RUSTFS_TABLE_CATALOG_BACKING"; +pub(crate) const ENV_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED: &str = + "RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED"; pub(crate) const TABLE_CATALOG_BACKING_OBJECT: &str = "object"; pub(crate) const TABLE_CATALOG_BACKING_DURABLE_STRONG: &str = "durable-strong"; +pub(crate) const TABLE_METADATA_DIGEST_REQUIREMENT_TYPE: &str = "assert-rustfs-metadata-sha256"; pub(crate) const TABLE_METADATA_FILE_NAME_MAX_LEN: usize = 128; pub(crate) const TABLE_METADATA_JSON_MAX_SIZE: usize = 50 * 1024 * 1024; pub(crate) const TABLE_MANIFEST_AVRO_MAX_SIZE: usize = 128 * 1024 * 1024; @@ -104,7 +109,7 @@ const TABLE_MANIFEST_AVRO_MAX_HEADER_ENTRIES: usize = 1_024; const TABLE_COMMIT_MAX_MANIFESTS: usize = 10_000; const TABLE_COMMIT_MAX_AVRO_BYTES: usize = 512 * 1024 * 1024; const TABLE_COMMIT_MAX_FILE_REFERENCES: usize = 1_000_000; -const TABLE_COMMIT_OBJECT_VALIDATION_CONCURRENCY: usize = 16; +pub(crate) const TABLE_COMMIT_OBJECT_VALIDATION_CONCURRENCY: usize = 16; pub const TABLE_RESERVED_PREFIX: &str = BUCKET_TABLE_RESERVED_PREFIX; const WAREHOUSE_ROOT: &str = "warehouses"; const NAMESPACE_ROOT: &str = "namespaces"; diff --git a/rustfs/src/table_catalog/store/migration.rs b/rustfs/src/table_catalog/store/migration.rs index ae949c65d..6b33eebb7 100644 --- a/rustfs/src/table_catalog/store/migration.rs +++ b/rustfs/src/table_catalog/store/migration.rs @@ -390,6 +390,23 @@ where "migration snapshot contains duplicate idempotency lookup keys".to_string(), )); } + let mut commit_logs_by_table = BTreeMap::<&str, Vec<&CommitLogEntry>>::new(); + for record in &snapshot.commits { + commit_logs_by_table + .entry(record.table_id.as_str()) + .or_default() + .push(&record.commit); + } + let history_by_table = tables_by_id + .iter() + .map(|(table_id, table)| { + let commits = commit_logs_by_table + .get(table_id) + .into_iter() + .flat_map(|commits| commits.iter().copied()); + (*table_id, TableCommitHistoryIndex::new(table, commits)) + }) + .collect::>(); for record in &snapshot.commits { let table = tables_by_id.get(record.table_id.as_str()).ok_or_else(|| { TableCatalogStoreError::Invalid(format!("commit {} has no table in migration snapshot", record.commit.commit_id)) @@ -408,7 +425,14 @@ where .idempotency_key .as_deref() .and_then(|idempotency_key| idempotency_by_key.get(&(record.table_id.as_str(), idempotency_key)).copied()); - let recovery = table_commit_recovery_entry(table, &record.commit, indexed); + let recovery = table_commit_recovery_entry( + table, + &record.commit, + indexed, + history_by_table + .get(record.table_id.as_str()) + .is_some_and(|history| history.proves_committed(&record.commit)), + ); if recovery.recovery_state != TableCommitRecoveryState::Committed { return Err(TableCatalogStoreError::Conflict(format!( "commit {} requires catalog recovery before durable strong migration", diff --git a/rustfs/src/table_catalog/store/mod.rs b/rustfs/src/table_catalog/store/mod.rs index 40bd5fb00..fd62c12f6 100644 --- a/rustfs/src/table_catalog/store/mod.rs +++ b/rustfs/src/table_catalog/store/mod.rs @@ -166,6 +166,12 @@ pub(crate) trait TableCatalogStore: Send + Sync { async fn register_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()>; + async fn register_table_with_publication( + &self, + entry: TableEntry, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()>; + async fn list_tables(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult>; async fn list_all_tables(&self, table_bucket: &str) -> TableCatalogStoreResult>; @@ -201,6 +207,12 @@ pub(crate) trait TableCatalogStore: Send + Sync { /// newly introduced or changed snapshots before invoking this persistence boundary. async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult; + async fn commit_table_with_publication( + &self, + request: TableCommitRequest, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult; + async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()>; async fn create_view(&self, entry: ViewEntry) -> TableCatalogStoreResult<()>; @@ -243,6 +255,131 @@ pub(crate) trait TableCatalogStore: Send + Sync { ) -> TableCatalogStoreResult>; } +#[async_trait::async_trait] +pub(crate) trait TableCommitPublication: Send + Sync { + async fn begin_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult<()>; + + async fn prepare(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()>; + + fn holds_table_bucket(&self, table_bucket: &str) -> bool; + + fn holds_table(&self, table_bucket: &str, namespace: &str, table: &str) -> bool; + + fn complete(&self); +} + +pub(crate) struct TableCommitPublicationCompletion<'a> { + publication: &'a (dyn TableCommitPublication + Sync), +} + +impl<'a> TableCommitPublicationCompletion<'a> { + pub(crate) fn new(publication: &'a (dyn TableCommitPublication + Sync)) -> Self { + Self { publication } + } +} + +impl Drop for TableCommitPublicationCompletion<'_> { + fn drop(&mut self) { + self.publication.complete(); + } +} + +struct TableCommitLockPublication<'a, B> { + backend: &'a B, + state: parking_lot::Mutex, +} + +#[derive(Default)] +struct TableCommitLockPublicationState { + table_bucket: Option, + table: Option<(String, String, String)>, + guards: Vec>, +} + +impl<'a, B> TableCommitLockPublication<'a, B> { + fn new(backend: &'a B) -> Self { + Self { + backend, + state: parking_lot::Mutex::new(TableCommitLockPublicationState::default()), + } + } +} + +#[async_trait::async_trait] +impl<'a, B> TableCommitPublication for TableCommitLockPublication<'a, B> +where + B: TableCatalogObjectBackend, +{ + async fn begin_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { + { + let mut state = self.state.lock(); + if state.table_bucket.as_deref() == Some(table_bucket) { + return Ok(()); + } + if state.table_bucket.is_some() || state.table.is_some() { + return Err(TableCatalogStoreError::Internal( + "table-bucket publication lock is already held for another table bucket".to_string(), + )); + } + state.table_bucket = Some(table_bucket.to_string()); + } + let publication_lock = default_table_bucket_publication_lock_path(); + let guard = match self.backend.acquire_write_lock(table_bucket, &publication_lock).await { + Ok(guard) => guard, + Err(err) => { + self.state.lock().table_bucket = None; + return Err(err); + } + }; + self.state.lock().guards.push(guard); + Ok(()) + } + + async fn prepare(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()> { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + let table_key = (table_bucket.to_string(), namespace.public_name(), table.as_str().to_string()); + { + let mut state = self.state.lock(); + if state.table.as_ref() == Some(&table_key) { + return Ok(()); + } + if state.table.is_some() { + return Err(TableCatalogStoreError::Internal( + "table publication lock is already held for another table".to_string(), + )); + } + state.table = Some(table_key); + } + let publication_lock = default_table_publication_lock_path(&namespace, &table); + let guard = match self.backend.acquire_write_lock(table_bucket, &publication_lock).await { + Ok(guard) => guard, + Err(err) => { + self.state.lock().table = None; + return Err(err); + } + }; + self.state.lock().guards.push(guard); + Ok(()) + } + + fn holds_table_bucket(&self, table_bucket: &str) -> bool { + self.state.lock().table_bucket.as_deref() == Some(table_bucket) + } + + fn holds_table(&self, table_bucket: &str, namespace: &str, table: &str) -> bool { + self.state + .lock() + .table + .as_ref() + .is_some_and(|held| held.0 == table_bucket && held.1 == namespace && held.2 == table) + } + + fn complete(&self) { + *self.state.lock() = TableCommitLockPublicationState::default(); + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct TableCatalogObject { pub data: Vec, @@ -317,8 +454,26 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static { })) } + async fn object_metadata_unlocked( + &self, + bucket: &str, + object: &str, + ) -> TableCatalogStoreResult> { + Ok(self + .read_object_unlocked(bucket, object) + .await? + .map(|object| TableCatalogObjectMetadata { + etag: object.etag, + mod_time: object.mod_time, + })) + } + async fn object_exists(&self, bucket: &str, object: &str) -> TableCatalogStoreResult; + async fn object_exists_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { + self.object_exists(bucket, object).await + } + async fn put_object( &self, bucket: &str, @@ -370,6 +525,55 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static { } async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult>; + + async fn begin_table_bucket_commit_publication(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + fn table_bucket_commit_publication_is_held(&self, _table_bucket: &str) -> bool { + false + } + + async fn prepare_table_commit_publication( + &self, + _table_bucket: &str, + _namespace: &str, + _table: &str, + ) -> TableCatalogStoreResult<()> { + Ok(()) + } + + fn table_commit_publication_is_held(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> bool { + false + } + + fn complete_table_commit_publication(&self) {} +} + +#[async_trait::async_trait] +impl TableCommitPublication for B +where + B: TableCatalogObjectBackend, +{ + async fn begin_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { + self.begin_table_bucket_commit_publication(table_bucket).await + } + + async fn prepare(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()> { + self.prepare_table_commit_publication(table_bucket, namespace, table).await + } + + fn holds_table_bucket(&self, table_bucket: &str) -> bool { + self.table_bucket_commit_publication_is_held(table_bucket) + } + + fn holds_table(&self, table_bucket: &str, namespace: &str, table: &str) -> bool { + self.table_commit_publication_is_held(table_bucket, namespace, table) + } + + fn complete(&self) { + self.complete_table_commit_publication(); + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -577,6 +781,10 @@ impl TableCatalogObjectPaths { ) } + pub fn warehouse_index_entries_prefix(&self, table_bucket: &str) -> String { + format!("{}{}/", self.table_bucket_root_prefix(table_bucket), WAREHOUSE_INDEX_ROOT) + } + pub fn warehouse_index_entry_path(&self, table_bucket: &str, warehouse_object_prefix: &str) -> String { format!( "{}{}/{}.json", @@ -779,6 +987,17 @@ where } } + async fn register_table_with_publication( + &self, + entry: TableEntry, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { + match self { + Self::ObjectBacked(store) => store.register_table_with_publication(entry, publication).await, + Self::DurableStrong(store) => store.register_table_with_publication(entry, publication).await, + } + } + async fn list_tables(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { match self { Self::ObjectBacked(store) => store.list_tables(table_bucket, namespace).await, @@ -831,6 +1050,17 @@ where } } + async fn commit_table_with_publication( + &self, + request: TableCommitRequest, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(store) => store.commit_table_with_publication(request, publication).await, + Self::DurableStrong(store) => store.commit_table_with_publication(request, publication).await, + } + } + async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()> { match self { Self::ObjectBacked(store) => store.drop_table(table_bucket, namespace, table).await, @@ -1205,6 +1435,32 @@ where } } + async fn object_metadata_unlocked( + &self, + bucket: &str, + object: &str, + ) -> TableCatalogStoreResult> { + match self + .store + .get_object_info( + bucket, + object, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(info) => Ok(Some(TableCatalogObjectMetadata { + etag: info.etag, + mod_time: info.mod_time, + })), + Err(err) if is_missing_storage_error(&err) => Ok(None), + Err(err) => Err(storage_error_to_catalog("stat catalog object", err)), + } + } + async fn object_exists(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { match self.store.get_object_info(bucket, object, &ObjectOptions::default()).await { Ok(_) => Ok(true), @@ -1213,6 +1469,25 @@ where } } + async fn object_exists_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { + match self + .store + .get_object_info( + bucket, + object, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(_) => Ok(true), + Err(err) if is_missing_storage_error(&err) => Ok(false), + Err(err) => Err(storage_error_to_catalog("check catalog object", err)), + } + } + async fn put_object( &self, bucket: &str, diff --git a/rustfs/src/table_catalog/store/object.rs b/rustfs/src/table_catalog/store/object.rs index 4b7464943..29a41bba7 100644 --- a/rustfs/src/table_catalog/store/object.rs +++ b/rustfs/src/table_catalog/store/object.rs @@ -519,7 +519,7 @@ where async fn write_warehouse_index_state_unlocked(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { let state = TableWarehouseIndexStateEntry { - version: TABLE_CATALOG_ENTRY_VERSION, + version: TABLE_WAREHOUSE_INDEX_STATE_VERSION, table_bucket: table_bucket.to_string(), state: TableCatalogEntryState::Active, }; @@ -542,7 +542,7 @@ where else { return Ok(false); }; - Ok(state.version == TABLE_CATALOG_ENTRY_VERSION + Ok(state.version == TABLE_WAREHOUSE_INDEX_STATE_VERSION && state.table_bucket == table_bucket && state.state == TableCatalogEntryState::Active) } @@ -635,11 +635,51 @@ where Ok(true) } + async fn ensure_table_warehouse_prefix_available(&self, entry: &TableEntry) -> TableCatalogStoreResult<()> { + let candidate = table_warehouse_index_entry(entry)?; + let state_object = self.paths.warehouse_index_state_path(&candidate.table_bucket); + for object in self + .backend + .list_objects(self.catalog_bucket(), &self.paths.warehouse_index_entries_prefix(&candidate.table_bucket)) + .await? + { + if object == state_object { + continue; + } + let Some((existing, _)) = self + .read_entry::(self.catalog_bucket(), &object) + .await? + else { + continue; + }; + if existing.table_id == candidate.table_id || existing.state != TableCatalogEntryState::Active { + continue; + } + if warehouse_object_prefixes_overlap(&existing.warehouse_object_prefix, &candidate.warehouse_object_prefix) + && self.warehouse_index_entry_has_active_owner(&existing).await? + { + return Err(TableCatalogStoreError::Conflict(format!( + "table warehouse location overlaps an active table: {}", + candidate.warehouse_object_prefix + ))); + } + } + Ok(()) + } + async fn reserve_table_warehouse_index(&self, entry: &TableEntry) -> TableCatalogStoreResult { let index = table_warehouse_index_entry(entry)?; let object = self .paths .warehouse_index_entry_path(&index.table_bucket, &index.warehouse_object_prefix); + if let Some((existing, _)) = self + .read_entry::(self.catalog_bucket(), &object) + .await? + && existing == index + { + return Ok(WarehouseIndexReservation::AlreadyReserved); + } + self.ensure_table_warehouse_prefix_available(entry).await?; loop { match self .write_entry(self.catalog_bucket(), &object, &index, TableCatalogPutPrecondition::IfAbsent) @@ -745,7 +785,7 @@ where else { return Ok(false); }; - Ok(state.version == TABLE_CATALOG_ENTRY_VERSION + Ok(state.version == TABLE_WAREHOUSE_INDEX_STATE_VERSION && state.table_bucket == table_bucket && state.state == TableCatalogEntryState::Active) } @@ -868,7 +908,27 @@ where if self.read_warehouse_index_state_unlocked(table_bucket).await? { return Ok(()); } - for table in self.list_all_tables(table_bucket).await? { + let tables = self.list_all_tables(table_bucket).await?; + let mut active_prefixes = tables + .iter() + .filter(|table| table.state == TableCatalogEntryState::Active) + .filter_map(|table| { + table_warehouse_object_prefix(table) + .ok() + .map(|prefix| (prefix, table.table_id.as_str())) + }) + .collect::>(); + active_prefixes.sort_unstable_by(|left, right| left.0.cmp(&right.0)); + if let Some(window) = active_prefixes + .windows(2) + .find(|window| window[0].1 != window[1].1 && warehouse_object_prefixes_overlap(&window[0].0, &window[1].0)) + { + return Err(TableCatalogStoreError::Conflict(format!( + "active table warehouse locations overlap: {} and {}", + window[0].0, window[1].0 + ))); + } + for table in tables { if table.state != TableCatalogEntryState::Active { continue; } @@ -958,12 +1018,30 @@ where &self, entry: TableEntry, precondition: TableCatalogPutPrecondition, + ) -> TableCatalogStoreResult<()> { + let publication = TableCommitLockPublication::new(&self.backend); + self.write_table_entry_with_publication(entry, precondition, &publication) + .await + } + + async fn write_table_entry_with_publication( + &self, + entry: TableEntry, + precondition: TableCatalogPutPrecondition, + publication: &(dyn TableCommitPublication + Sync), ) -> TableCatalogStoreResult<()> { validate_catalog_entry_version("table", entry.version)?; - self.require_table_bucket(&entry.table_bucket).await?; let namespace = parse_namespace_for_store(&entry.namespace)?; let table = parse_table_for_store(&entry.table)?; validate_table_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?; + publication.begin_table_bucket(&entry.table_bucket).await?; + if !publication.holds_table_bucket(&entry.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "table registration requires a table-bucket publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); + self.require_table_bucket(&entry.table_bucket).await?; let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; let namespace_path = self.paths.namespace_entry_path(&entry.table_bucket, &namespace); let _namespace_guard = self @@ -974,6 +1052,15 @@ where .await?; let table_path = self.paths.table_entry_path(&entry.table_bucket, &namespace, &table); let _table_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + // Preserve catalog -> publication -> object lock order across rolling upgrades. + publication + .prepare(&entry.table_bucket, &entry.namespace, &entry.table) + .await?; + if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table) { + return Err(TableCatalogStoreError::Internal( + "table registration requires a table publication fence".to_string(), + )); + } let reservation = self.reserve_table_warehouse_index(&entry).await?; let result = self .write_entry_unlocked(self.catalog_bucket(), &table_path, &entry, precondition) @@ -1056,6 +1143,20 @@ where .map(|entry| entry.map(|(commit, _)| commit)) } + async fn read_table_commit_logs(&self, entry: &TableEntry) -> TableCatalogStoreResult> { + let commit_prefix = self.paths.commit_log_entries_prefix(&entry.table_bucket, &entry.table_id); + let mut commits = Vec::new(); + for object in self.backend.list_objects(self.catalog_bucket(), &commit_prefix).await? { + if !object.ends_with(".json") { + continue; + } + if let Some(commit_log) = self.read_commit_by_path(&object).await? { + commits.push((object, commit_log)); + } + } + Ok(commits) + } + async fn finalize_commit_log( &self, commit_path: &str, @@ -1076,15 +1177,10 @@ where entry: &TableEntry, finalized_count: usize, ) -> TableCatalogStoreResult { - let commit_prefix = self.paths.commit_log_entries_prefix(&entry.table_bucket, &entry.table_id); - let mut commits = Vec::new(); - for object in self.backend.list_objects(self.catalog_bucket(), &commit_prefix).await? { - if !object.ends_with(".json") { - continue; - } - let Some(commit_log) = self.read_commit_by_path(&object).await? else { - continue; - }; + let commit_logs_with_paths = self.read_table_commit_logs(entry).await?; + let history = TableCommitHistoryIndex::new(entry, commit_logs_with_paths.iter().map(|(_, commit_log)| commit_log)); + let mut commits = Vec::with_capacity(commit_logs_with_paths.len()); + for (_, commit_log) in &commit_logs_with_paths { let idempotency_commit = match commit_log.idempotency_key.as_deref() { Some(idempotency_key) => { let idempotency_path = @@ -1094,7 +1190,12 @@ where } None => None, }; - commits.push(table_commit_recovery_entry(entry, &commit_log, idempotency_commit.as_ref())); + commits.push(table_commit_recovery_entry( + entry, + commit_log, + idempotency_commit.as_ref(), + history.proves_committed(commit_log), + )); } commits.sort_by(|left, right| left.commit_id.cmp(&right.commit_id)); @@ -1172,15 +1273,10 @@ where ))); }; - let commit_prefix = self.paths.commit_log_entries_prefix(table_bucket, &entry.table_id); + let commit_logs_with_paths = self.read_table_commit_logs(&entry).await?; + let history = TableCommitHistoryIndex::new(&entry, commit_logs_with_paths.iter().map(|(_, commit_log)| commit_log)); let mut finalized_count = 0; - for commit_path in self.backend.list_objects(self.catalog_bucket(), &commit_prefix).await? { - if !commit_path.ends_with(".json") { - continue; - } - let Some(commit_log) = self.read_commit_by_path(&commit_path).await? else { - continue; - }; + for (commit_path, commit_log) in &commit_logs_with_paths { let idempotency_path = commit_log.idempotency_key.as_deref().map(|idempotency_key| { self.paths .commit_idempotency_entry_path(table_bucket, &entry.table_id, idempotency_key) @@ -1189,14 +1285,19 @@ where Some(idempotency_path) => self.read_commit_by_path(idempotency_path).await?, None => None, }; - let recovery_entry = table_commit_recovery_entry(&entry, &commit_log, idempotency_commit.as_ref()); + let recovery_entry = table_commit_recovery_entry( + &entry, + commit_log, + idempotency_commit.as_ref(), + history.proves_committed(commit_log), + ); if matches!( recovery_entry.recovery_state, TableCommitRecoveryState::FinalizationRequired | TableCommitRecoveryState::IdempotencyIndexRepairRequired ) { - let mut committed = commit_log; + let mut committed = commit_log.clone(); committed.status = CommitLogStatus::Committed; - self.finalize_commit_log(&commit_path, idempotency_path.as_deref(), &committed) + self.finalize_commit_log(commit_path, idempotency_path.as_deref(), &committed) .await?; finalized_count += 1; } @@ -2427,6 +2528,21 @@ where table: &str, config: TableSnapshotExpirationConfig, ) -> TableCatalogStoreResult { + self.plan_table_snapshot_expiration_with_backend(&self.backend, table_bucket, namespace, table, config) + .await + } + + pub(crate) async fn plan_table_snapshot_expiration_with_backend

( + &self, + metadata_backend: &P, + table_bucket: &str, + namespace: &str, + table: &str, + config: TableSnapshotExpirationConfig, + ) -> TableCatalogStoreResult + where + P: TableCatalogObjectBackend, + { validate_table_snapshot_expiration_config(&config)?; let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; @@ -2445,7 +2561,7 @@ where )); } - let Some(current_metadata) = read_table_metadata_value(&self.backend, table_bucket, &entry.metadata_location).await? + let Some(current_metadata) = read_table_metadata_value(metadata_backend, table_bucket, &entry.metadata_location).await? else { return Err(TableCatalogStoreError::NotFound(format!( "current metadata object {}", @@ -2507,6 +2623,23 @@ where table: &str, config: TableCompactionPlanningConfig, ) -> TableCatalogStoreResult { + let publication = TableCommitLockPublication::new(&self.backend); + self.commit_table_compaction_with_publication(&self.backend, &publication, table_bucket, namespace, table, config) + .await + } + + pub(crate) async fn commit_table_compaction_with_publication

( + &self, + object_backend: &P, + publication: &(dyn TableCommitPublication + Sync), + table_bucket: &str, + namespace: &str, + table: &str, + config: TableCompactionPlanningConfig, + ) -> TableCatalogStoreResult + where + P: TableCatalogObjectBackend, + { validate_table_compaction_planning_config(&config)?; let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; @@ -2525,7 +2658,7 @@ where )); } - let Some(current_metadata) = read_table_metadata_value(&self.backend, table_bucket, &entry.metadata_location).await? + let Some(current_metadata) = read_table_metadata_value(object_backend, table_bucket, &entry.metadata_location).await? else { return Err(TableCatalogStoreError::NotFound(format!( "current metadata object {}", @@ -2533,13 +2666,13 @@ where ))); }; let mut report = - table_compaction_planning_report(&self.backend, table_bucket, &namespace, &table, &entry, ¤t_metadata, config) + table_compaction_planning_report(object_backend, table_bucket, &namespace, &table, &entry, ¤t_metadata, config) .await?; if report.status != TableCompactionPlanningStatus::RewriteCandidates { return Err(TableCatalogStoreError::Invalid("compaction has no safe rewrite candidates".to_string())); } let current_data_files = - compaction_current_data_files(&self.backend, table_bucket, &namespace, &table, &entry, ¤t_metadata).await?; + compaction_current_data_files(object_backend, table_bucket, &namespace, &table, &entry, ¤t_metadata).await?; let current_data_files_by_key = current_data_files .iter() .map(|file| (file.object_key.as_str(), file)) @@ -2574,14 +2707,14 @@ where let sort_order_id = compaction_rewrite_group_sort_order(¤t_data_files_by_key, rewrite_group)?; let mut input_files = Vec::with_capacity(rewrite_group.input_file_locations.len()); for input_file in &rewrite_group.input_file_locations { - let Some(input_object) = self.backend.read_object(table_bucket, input_file).await? else { + let Some(input_object) = object_backend.read_object(table_bucket, input_file).await? else { return Err(TableCatalogStoreError::NotFound(format!("compaction input data file {input_file}"))); }; input_files.push((input_file.clone(), input_object.data)); } let compacted_file = compact_parquet_data_files(&input_files)?; let output_bytes = u64::try_from(compacted_file.data.len()).unwrap_or(u64::MAX); - self.backend + object_backend .put_object(table_bucket, &output_file, compacted_file.data, TableCatalogPutPrecondition::IfAbsent) .await?; rewrite_group.output_file_location = Some(output_file_path.clone()); @@ -2608,7 +2741,7 @@ where default_table_metadata_file_path(&namespace, &table, &format!("compaction-{compaction_id}.metadata.json")); let manifest_data = compacted_manifest_avro_bytes(&manifest_data_files)?; let manifest_length = u64::try_from(manifest_data.len()).unwrap_or(u64::MAX); - self.backend + object_backend .put_object(table_bucket, &new_manifest, manifest_data, TableCatalogPutPrecondition::IfAbsent) .await?; let added_files_count = compacted_files.len(); @@ -2631,7 +2764,7 @@ where added_rows_count, existing_rows_count, })?; - self.backend + object_backend .put_object( table_bucket, &new_manifest_list, @@ -2648,24 +2781,27 @@ where &entry.metadata_location, now, )?; - self.backend + object_backend .put_object(table_bucket, &new_metadata, new_metadata_data, TableCatalogPutPrecondition::IfAbsent) .await?; let commit_result = self - .commit_table(TableCommitRequest { - table_bucket: table_bucket.to_string(), - namespace: namespace.public_name(), - table: table.as_str().to_string(), - commit_id: format!("compaction-{compaction_id}"), - idempotency_key: Some(format!("compaction-{compaction_id}")), - operation: "compaction".to_string(), - expected_version_token: entry.version_token, - expected_metadata_location: entry.metadata_location, - new_metadata_location: new_metadata.clone(), - requirements: Vec::new(), - writer: Some("rustfs-maintenance".to_string()), - }) + .commit_table_with_publication( + TableCommitRequest { + table_bucket: table_bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: format!("compaction-{compaction_id}"), + idempotency_key: Some(format!("compaction-{compaction_id}")), + operation: "compaction".to_string(), + expected_version_token: entry.version_token, + expected_metadata_location: entry.metadata_location, + new_metadata_location: new_metadata.clone(), + requirements: Vec::new(), + writer: Some("rustfs-maintenance".to_string()), + }, + publication, + ) .await?; report.status = TableCompactionPlanningStatus::Committed; @@ -3245,6 +3381,8 @@ where let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + let publication_lock = default_table_publication_lock_path(&namespace, &table); + let _publication_guard = self.backend.acquire_write_lock(table_bucket, &publication_lock).await?; let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { return Err(TableCatalogStoreError::NotFound(format!( "table {}/{}/{}", @@ -3635,6 +3773,15 @@ where self.write_table_entry(entry, TableCatalogPutPrecondition::IfAbsent).await } + async fn register_table_with_publication( + &self, + entry: TableEntry, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { + self.write_table_entry_with_publication(entry, TableCatalogPutPrecondition::IfAbsent, publication) + .await + } + async fn list_tables(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { let namespace = parse_namespace_for_store(namespace)?; let mut entries = Vec::new(); @@ -3722,6 +3869,7 @@ where match self.backfill_table_warehouse_index(table_bucket).await { Ok(()) => self.resolve_table_data_plane_resource_from_index(table_bucket, object).await, + Err(err @ TableCatalogStoreError::Conflict(_)) => Err(err), Err(err) => { tracing::warn!( table_bucket = %table_bucket, @@ -3734,6 +3882,16 @@ where } async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { + let publication = TableCommitLockPublication::new(&self.backend); + publication.begin_table_bucket(&request.table_bucket).await?; + self.commit_table_with_publication(request, &publication).await + } + + async fn commit_table_with_publication( + &self, + request: TableCommitRequest, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult { let commit_started = Instant::now(); record_table_commit_attempt(&request.operation); let namespace = parse_namespace_for_store(&request.namespace)?; @@ -3741,6 +3899,16 @@ where let _migration_guard = self.acquire_object_backed_catalog_write_permit(&request.table_bucket).await?; let table_path = self.paths.table_entry_path(&request.table_bucket, &namespace, &table); let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + // Preserve catalog -> publication -> object lock order across rolling upgrades. + publication + .prepare(&request.table_bucket, &request.namespace, &request.table) + .await?; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) { + return Err(TableCatalogStoreError::Internal( + "table commit requires a table publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); let Some((current, current_etag)) = self .read_table_with_etag_unlocked(&request.table_bucket, &namespace, &table) @@ -3773,6 +3941,48 @@ where None => None, }; + if let (Some(existing), Some(indexed)) = (&existing_commit, &existing_idempotency_commit) + && !commit_logs_share_recovery_payload(existing, indexed) + { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Conflict( + "commit record and idempotency index contain different payloads".to_string(), + )), + ); + } + if let Some(existing) = existing_idempotency_commit.as_ref() + && !commit_log_matches_request(existing, &request, ¤t.table_id) + { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Conflict("idempotency key already exists".to_string())), + ); + } + if existing_commit.is_none() && existing_idempotency_commit.is_some() { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Conflict( + "idempotency key exists without a recoverable commit record".to_string(), + )), + ); + } + if let Some(existing) = existing_commit.as_ref() { if !commit_log_matches_request(existing, &request, ¤t.table_id) { return table_commit_result( @@ -3788,7 +3998,48 @@ where ))), ); } - if matches!(existing.status, CommitLogStatus::Committed) || table_matches_committed_log(¤t, existing) { + if matches!(existing.status, CommitLogStatus::Failed) { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Conflict("failed commit record cannot be replayed".to_string())), + ); + } + if matches!(existing.status, CommitLogStatus::Committed) && table_matches_staged_base(¤t, existing) { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Conflict( + "committed record still matches the pre-commit table state".to_string(), + )), + ); + } + let historically_committed = if matches!(existing.status, CommitLogStatus::Staged) + && !table_matches_staged_base(¤t, existing) + && !table_matches_committed_log(¤t, existing) + { + let commit_logs = self + .read_table_commit_logs(¤t) + .await? + .into_iter() + .map(|(_, commit_log)| commit_log) + .collect::>(); + TableCommitHistoryIndex::new(¤t, commit_logs.iter()).proves_committed(existing) + } else { + false + }; + if matches!(existing.status, CommitLogStatus::Committed) + || (matches!(existing.status, CommitLogStatus::Staged) + && (table_matches_committed_log(¤t, existing) || historically_committed)) + { let mut committed = existing.clone(); committed.status = CommitLogStatus::Committed; let _ = self @@ -3821,32 +4072,6 @@ where ); } } - if let Some(existing) = existing_idempotency_commit.as_ref() - && !commit_log_matches_request(existing, &request, ¤t.table_id) - { - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(TableCatalogStoreError::Conflict("idempotency key already exists".to_string())), - ); - } - if existing_commit.is_none() && existing_idempotency_commit.is_some() { - return table_commit_result( - &request.table_bucket, - &request.namespace, - &request.table, - &request.commit_id, - &request.operation, - commit_started, - Err(TableCatalogStoreError::Conflict( - "idempotency key exists without a recoverable commit record".to_string(), - )), - ); - } if current.version_token != request.expected_version_token { return table_commit_result( @@ -3905,6 +4130,7 @@ where ))), ); }; + validate_commit_metadata_digest(&request, &new_metadata_object)?; let table_bucket = request.table_bucket.clone(); let metadata_location = request.new_metadata_location.clone(); let next_warehouse_location = tokio::task::spawn_blocking(move || { @@ -3912,6 +4138,23 @@ where }) .await .map_err(|err| TableCatalogStoreError::Internal(format!("table metadata parser task failed: {err}")))??; + if next_warehouse_location + .as_ref() + .is_some_and(|warehouse_location| warehouse_location != ¤t.warehouse_location) + && !publication.holds_table_bucket(&request.table_bucket) + { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Internal( + "table warehouse relocation requires a table-bucket publication fence".to_string(), + )), + ); + } let has_existing_commit = existing_commit.is_some(); let mut staged_commit_log = existing_commit.unwrap_or_else(|| CommitLogEntry { @@ -4024,6 +4267,8 @@ where } async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()> { + let publication = TableCommitLockPublication::new(&self.backend); + publication.begin_table_bucket(table_bucket).await?; let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; @@ -4034,6 +4279,17 @@ where .await?; let object = self.paths.table_entry_path(table_bucket, &namespace, &table); let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?; + publication + .prepare(table_bucket, &namespace.public_name(), table.as_str()) + .await?; + if !publication.holds_table_bucket(table_bucket) + || !publication.holds_table(table_bucket, &namespace.public_name(), table.as_str()) + { + return Err(TableCatalogStoreError::Internal( + "table drop requires table-bucket and table publication fences".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(&publication); let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { return Err(TableCatalogStoreError::NotFound(format!( "table {}/{}/{}", diff --git a/rustfs/src/table_catalog/store/strong.rs b/rustfs/src/table_catalog/store/strong.rs index a8bf57a4a..f3a885423 100644 --- a/rustfs/src/table_catalog/store/strong.rs +++ b/rustfs/src/table_catalog/store/strong.rs @@ -443,16 +443,20 @@ where continue; }; let table_key = (table_bucket.clone(), namespace.clone(), table.clone()); - if let Some(existing_key) = warehouse_index - .entry(table_bucket.clone()) - .or_default() - .insert(warehouse_object_prefix.clone(), table_key.clone()) + let bucket_index = warehouse_index.entry(table_bucket.clone()).or_default(); + let predecessor = bucket_index.range(..=warehouse_object_prefix.clone()).next_back(); + let successor = bucket_index.range(warehouse_object_prefix.clone()..).next(); + if let Some((existing_prefix, existing_key)) = predecessor + .into_iter() + .chain(successor) + .find(|(existing_prefix, _)| warehouse_object_prefixes_overlap(existing_prefix, &warehouse_object_prefix)) { return Err(TableCatalogStoreError::Invalid(format!( - "duplicate active table warehouse location in strong catalog snapshot: {warehouse_object_prefix} is owned by {}/{}/{} and {}/{}/{}", + "overlapping active table warehouse location in strong catalog snapshot: {warehouse_object_prefix} overlaps {existing_prefix} owned by {}/{}/{} and {}/{}/{}", existing_key.0, existing_key.1, existing_key.2, table_key.0, table_key.1, table_key.2 ))); } + bucket_index.insert(warehouse_object_prefix, table_key); } state.warehouse_index = warehouse_index; Ok(()) @@ -696,9 +700,9 @@ where let Ok(existing_prefix) = table_warehouse_object_prefix(existing) else { continue; }; - if existing_prefix == candidate_prefix { + if warehouse_object_prefixes_overlap(&existing_prefix, &candidate_prefix) { return Err(TableCatalogStoreError::Conflict(format!( - "table warehouse location is already registered: {candidate_prefix}" + "table warehouse location overlaps an active table: {candidate_prefix}" ))); } } @@ -709,6 +713,14 @@ where state: &StrongTableCatalogState, entry: &TableEntry, ) -> TableCommitRecoveryReport { + let history = TableCommitHistoryIndex::new( + entry, + state + .commits + .iter() + .filter(|((table_bucket, table_id, _), _)| table_bucket == &entry.table_bucket && table_id == &entry.table_id) + .map(|(_, commit_log)| commit_log), + ); let mut commits = state .commits .iter() @@ -719,7 +731,7 @@ where .idempotency .get(&Self::idempotency_key(&entry.table_bucket, &entry.table_id, idempotency_key)) }); - table_commit_recovery_entry(entry, commit_log, idempotency_commit) + table_commit_recovery_entry(entry, commit_log, idempotency_commit, history.proves_committed(commit_log)) }) .collect::>(); commits.sort_by(|left, right| left.commit_id.cmp(&right.commit_id)); @@ -783,18 +795,11 @@ where .map(|idempotency_key| Self::idempotency_key(&request.table_bucket, ¤t.table_id, idempotency_key)); let existing_idempotency_commit = idempotency_key.as_ref().and_then(|key| state.idempotency.get(key)); - if let Some(existing) = existing_commit { - if !commit_log_matches_request(existing, request, ¤t.table_id) { - return Err(TableCatalogStoreError::Conflict(format!( - "commit id already exists: {}", - request.commit_id - ))); - } - if matches!(existing.status, CommitLogStatus::Committed) || table_matches_committed_log(¤t, existing) { - return Ok(current); - } + if let (Some(existing), Some(indexed)) = (existing_commit, existing_idempotency_commit) + && !commit_logs_share_recovery_payload(existing, indexed) + { return Err(TableCatalogStoreError::Conflict( - "existing commit record does not match current table state".to_string(), + "commit record and idempotency index contain different payloads".to_string(), )); } if let Some(existing) = existing_idempotency_commit @@ -802,11 +807,55 @@ where { return Err(TableCatalogStoreError::Conflict("idempotency key already exists".to_string())); } - if existing_idempotency_commit.is_some() { + if existing_commit.is_none() && existing_idempotency_commit.is_some() { return Err(TableCatalogStoreError::Conflict( "idempotency key exists without a recoverable commit record".to_string(), )); } + + if let Some(existing) = existing_commit { + if !commit_log_matches_request(existing, request, ¤t.table_id) { + return Err(TableCatalogStoreError::Conflict(format!( + "commit id already exists: {}", + request.commit_id + ))); + } + if matches!(existing.status, CommitLogStatus::Failed) { + return Err(TableCatalogStoreError::Conflict("failed commit record cannot be replayed".to_string())); + } + if matches!(existing.status, CommitLogStatus::Committed) && table_matches_staged_base(¤t, existing) { + return Err(TableCatalogStoreError::Conflict( + "committed record still matches the pre-commit table state".to_string(), + )); + } + let historically_committed = if matches!(existing.status, CommitLogStatus::Staged) + && !table_matches_staged_base(¤t, existing) + && !table_matches_committed_log(¤t, existing) + { + TableCommitHistoryIndex::new( + ¤t, + state + .commits + .iter() + .filter(|((table_bucket, table_id, _), _)| { + table_bucket == &request.table_bucket && table_id == ¤t.table_id + }) + .map(|(_, commit_log)| commit_log), + ) + .proves_committed(existing) + } else { + false + }; + if matches!(existing.status, CommitLogStatus::Committed) + || (matches!(existing.status, CommitLogStatus::Staged) + && (table_matches_committed_log(¤t, existing) || historically_committed)) + { + return Ok(current); + } + return Err(TableCatalogStoreError::Conflict( + "existing commit record does not match current table state".to_string(), + )); + } if current.version_token != request.expected_version_token { return Err(TableCatalogStoreError::Conflict( "current table version token does not match expected token".to_string(), @@ -831,15 +880,38 @@ where current: TableEntry, ) -> Option { let commit_key = Self::commit_key(&request.table_bucket, ¤t.table_id, &request.commit_id); - let existing = state.commits.get(&commit_key)?; - if !commit_log_matches_request(existing, request, ¤t.table_id) { + let existing = state.commits.get(&commit_key)?.clone(); + if !commit_log_matches_request(&existing, request, ¤t.table_id) { return None; } - if !matches!(existing.status, CommitLogStatus::Committed) && !table_matches_committed_log(¤t, existing) { + let historically_committed = if matches!(existing.status, CommitLogStatus::Staged) + && !table_matches_staged_base(¤t, &existing) + && !table_matches_committed_log(¤t, &existing) + { + TableCommitHistoryIndex::new( + ¤t, + state + .commits + .iter() + .filter(|((table_bucket, table_id, _), _)| { + table_bucket == &request.table_bucket && table_id == ¤t.table_id + }) + .map(|(_, commit_log)| commit_log), + ) + .proves_committed(&existing) + } else { + false + }; + if matches!(existing.status, CommitLogStatus::Failed) + || (matches!(existing.status, CommitLogStatus::Committed) && table_matches_staged_base(¤t, &existing)) + || (!matches!(existing.status, CommitLogStatus::Committed) + && !table_matches_committed_log(¤t, &existing) + && !historically_committed) + { return None; } - let mut committed = existing.clone(); + let mut committed = existing; committed.status = CommitLogStatus::Committed; state.commits.insert(commit_key, committed.clone()); if let Some(idempotency_key) = committed.idempotency_key.as_deref() { @@ -1202,12 +1274,36 @@ where } async fn register_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()> { - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; + let publication = TableCommitLockPublication::new(&self.object_backend); + self.register_table_with_publication(entry, &publication).await + } + + async fn register_table_with_publication( + &self, + entry: TableEntry, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { validate_catalog_entry_version("table", entry.version)?; let namespace = parse_namespace_for_store(&entry.namespace)?; let table = parse_table_for_store(&entry.table)?; table_warehouse_object_prefix(&entry)?; + publication.begin_table_bucket(&entry.table_bucket).await?; + if !publication.holds_table_bucket(&entry.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "table registration requires a table-bucket publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); + publication + .prepare(&entry.table_bucket, &entry.namespace, &entry.table) + .await?; + if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table) { + return Err(TableCatalogStoreError::Internal( + "table registration requires a table publication fence".to_string(), + )); + } + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; let key = Self::table_key(&entry.table_bucket, &namespace, &table); let (snapshot, precondition) = { let state = self.state.lock().await; @@ -1328,12 +1424,31 @@ where } async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { - let _write_guard = self.write_lock.lock().await; - self.hydrate_state().await?; + let publication = TableCommitLockPublication::new(&self.object_backend); + publication.begin_table_bucket(&request.table_bucket).await?; + self.commit_table_with_publication(request, &publication).await + } + + async fn commit_table_with_publication( + &self, + request: TableCommitRequest, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult { let commit_started = Instant::now(); record_table_commit_attempt(&request.operation); let namespace = parse_namespace_for_store(&request.namespace)?; let table = parse_table_for_store(&request.table)?; + publication + .prepare(&request.table_bucket, &request.namespace, &request.table) + .await?; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) { + return Err(TableCatalogStoreError::Internal( + "table commit requires a table publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; let key = Self::table_key(&request.table_bucket, &namespace, &table); let committed_existing_result = { @@ -1396,6 +1511,7 @@ where ))), ); }; + validate_commit_metadata_digest(&request, &new_metadata_object)?; let table_bucket = request.table_bucket.clone(); let metadata_location = request.new_metadata_location.clone(); let next_warehouse_location = tokio::task::spawn_blocking(move || { @@ -1403,6 +1519,36 @@ where }) .await .map_err(|err| TableCatalogStoreError::Internal(format!("table metadata parser task failed: {err}")))??; + let current_warehouse_location = { + let state = self.state.lock().await; + state + .tables + .get(&key) + .map(|entry| entry.warehouse_location.clone()) + .ok_or_else(|| { + TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + request.table_bucket, request.namespace, request.table + )) + })? + }; + if next_warehouse_location + .as_ref() + .is_some_and(|warehouse_location| warehouse_location != ¤t_warehouse_location) + && !publication.holds_table_bucket(&request.table_bucket) + { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::Internal( + "table warehouse relocation requires a table-bucket publication fence".to_string(), + )), + ); + } let cas_started = Instant::now(); let prepared_result = { @@ -1433,6 +1579,15 @@ where } async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()> { + let publication = TableCommitLockPublication::new(&self.object_backend); + publication.begin_table_bucket(table_bucket).await?; + publication.prepare(table_bucket, namespace, table).await?; + if !publication.holds_table_bucket(table_bucket) || !publication.holds_table(table_bucket, namespace, table) { + return Err(TableCatalogStoreError::Internal( + "table drop requires table-bucket and table publication fences".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(&publication); let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; let namespace = parse_namespace_for_store(namespace)?; diff --git a/rustfs/src/table_catalog/tests.rs b/rustfs/src/table_catalog/tests.rs index beeb6d7a0..9533ae862 100644 --- a/rustfs/src/table_catalog/tests.rs +++ b/rustfs/src/table_catalog/tests.rs @@ -230,6 +230,29 @@ impl TableCatalogStore for NoopTableCatalogStore { Ok(()) } + async fn register_table_with_publication( + &self, + entry: TableEntry, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { + publication.begin_table_bucket(&entry.table_bucket).await?; + if !publication.holds_table_bucket(&entry.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "table registration requires a table-bucket publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); + publication + .prepare(&entry.table_bucket, &entry.namespace, &entry.table) + .await?; + if !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table) { + return Err(TableCatalogStoreError::Internal( + "table registration requires a table publication fence".to_string(), + )); + } + self.register_table(entry).await + } + async fn list_tables(&self, _table_bucket: &str, _namespace: &str) -> TableCatalogStoreResult> { Ok(Vec::new()) } @@ -286,6 +309,23 @@ impl TableCatalogStore for NoopTableCatalogStore { Ok(TableCommitResult { table, commit_log }) } + async fn commit_table_with_publication( + &self, + request: TableCommitRequest, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult { + publication + .prepare(&request.table_bucket, &request.namespace, &request.table) + .await?; + if !publication.holds_table(&request.table_bucket, &request.namespace, &request.table) { + return Err(TableCatalogStoreError::Internal( + "table commit requires a table publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(publication); + self.commit_table(request).await + } + async fn drop_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> { Ok(()) } @@ -394,6 +434,243 @@ async fn table_catalog_store_trait_has_atomic_commit_shape() { assert_eq!(result.commit_log.status, CommitLogStatus::Committed); } +async fn assert_direct_commit_uses_publication_lock(store: &S, backend: &TestCatalogObjectBackend) +where + S: TableCatalogStore + ?Sized, +{ + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let publication_lock = default_table_publication_lock_path(&namespace, &table); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .expect("table should be created"); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + let acquisitions_before = backend.write_lock_acquisition_count(bucket, &publication_lock).await; + + store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: Some("rustfs-maintenance".to_string()), + }) + .await + .expect("direct commit should succeed"); + + assert_eq!( + backend.write_lock_acquisition_count(bucket, &publication_lock).await, + acquisitions_before + 1 + ); + let guard = tokio::time::timeout( + StdDuration::from_secs(1), + TableCatalogObjectBackend::acquire_write_lock(backend, bucket, &publication_lock), + ) + .await + .expect("publication lock should be released after commit") + .expect("publication lock should be reacquired"); + drop(guard); +} + +#[tokio::test] +async fn catalog_backings_fence_direct_commits_with_publication_lock() { + let object_backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(object_backend.clone()); + assert_direct_commit_uses_publication_lock(&object_store, &object_backend).await; + + let strong_backend = TestCatalogObjectBackend::default(); + let strong_store = StrongTableCatalogStore::new(strong_backend.clone()); + assert_direct_commit_uses_publication_lock(&strong_store, &strong_backend).await; +} + +async fn assert_direct_drop_uses_publication_locks(store: S, backend: &TestCatalogObjectBackend) +where + S: TableCatalogStore + Clone + Send + Sync + 'static, +{ + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata)) + .await + .expect("table should be created"); + + let bucket_lock = default_table_bucket_publication_lock_path(); + let table_lock = default_table_publication_lock_path(&namespace, &table); + let bucket_guard = backend + .acquire_read_lock(bucket, &bucket_lock) + .await + .expect("bucket publication reader should be acquired"); + let table_guard = backend + .acquire_read_lock(bucket, &table_lock) + .await + .expect("table publication reader should be acquired"); + let bucket_attempts = backend.write_lock_acquisition_count(bucket, &bucket_lock).await; + let table_attempts = backend.write_lock_acquisition_count(bucket, &table_lock).await; + + let drop_store = store.clone(); + let drop_task = tokio::spawn(async move { drop_store.drop_table(bucket, "sales", "orders").await }); + tokio::time::timeout(StdDuration::from_secs(1), async { + while backend.write_lock_acquisition_count(bucket, &bucket_lock).await == bucket_attempts { + tokio::task::yield_now().await; + } + }) + .await + .expect("drop should attempt the bucket publication fence"); + assert!(!drop_task.is_finished(), "drop must wait for an in-flight bucket writer"); + + drop(bucket_guard); + tokio::time::timeout(StdDuration::from_secs(1), async { + while backend.write_lock_acquisition_count(bucket, &table_lock).await == table_attempts { + tokio::task::yield_now().await; + } + }) + .await + .expect("drop should attempt the table publication fence"); + assert!(!drop_task.is_finished(), "drop must wait for an in-flight table writer"); + + drop(table_guard); + tokio::time::timeout(StdDuration::from_secs(1), drop_task) + .await + .expect("drop should continue after publication readers finish") + .expect("drop task should join") + .expect("drop should succeed"); + assert!( + store + .load_table(bucket, "sales", "orders") + .await + .expect("table lookup should succeed") + .is_none(), + "table must become invisible before publication fences are released" + ); +} + +#[tokio::test] +async fn catalog_backings_fence_direct_table_drop() { + let object_backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(object_backend.clone()); + assert_direct_drop_uses_publication_locks(object_store, &object_backend).await; + + let strong_backend = TestCatalogObjectBackend::default(); + let strong_store = StrongTableCatalogStore::new(strong_backend.clone()); + assert_direct_drop_uses_publication_locks(strong_store, &strong_backend).await; +} + +#[tokio::test] +async fn strong_catalog_blocked_publication_object_does_not_stall_unrelated_writes() { + let backend = TestCatalogObjectBackend::default(); + let store = Arc::new(StrongTableCatalogStore::new(backend.clone())); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .expect("table should be created"); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + + let blocked_object = "tables/table-id/data/late.parquet"; + let blocker = backend + .acquire_write_lock(bucket, blocked_object) + .await + .expect("blocked object lock should be acquired"); + let publication = Arc::new(BlockingObjectPublication::new(backend.clone(), blocked_object)); + let commit_store = Arc::clone(&store); + let commit_publication = Arc::clone(&publication); + let commit = tokio::spawn(async move { + commit_store + .commit_table_with_publication( + TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "blocked-publication".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: Some("concurrency-test".to_string()), + }, + commit_publication.as_ref(), + ) + .await + }); + publication.wait_started().await; + + tokio::time::timeout(StdDuration::from_secs(5), store.put_table_bucket(test_bucket_entry("independent"))) + .await + .expect("a blocked table publication must not hold the strong catalog write lock") + .expect("the unrelated table bucket write should succeed"); + + drop(blocker); + tokio::time::timeout(StdDuration::from_secs(5), commit) + .await + .expect("the blocked commit should finish after its object lock is released") + .expect("the commit task should join") + .expect("the commit should succeed"); + assert!( + store + .get_table_bucket("independent") + .await + .expect("unrelated table bucket lookup should succeed") + .is_some() + ); + assert_eq!( + store + .load_table(bucket, "sales", "orders") + .await + .expect("committed table lookup should succeed") + .expect("committed table should exist") + .metadata_location, + default_table_metadata_file_path( + &Namespace::parse("sales").expect("namespace should parse"), + &IdentifierSegment::parse("orders").expect("table should parse"), + "00002.metadata.json", + ) + ); +} + #[test] fn catalog_object_entry_paths_use_internal_root_and_hashed_untrusted_ids() { let paths = TableCatalogObjectPaths::default(); @@ -469,6 +746,55 @@ impl TestCatalogObjectPutPause { } } +#[derive(Clone)] +struct BlockingObjectPublication { + backend: TestCatalogObjectBackend, + object: String, + started: Arc, + guard: Arc>>>, +} + +impl BlockingObjectPublication { + fn new(backend: TestCatalogObjectBackend, object: impl Into) -> Self { + Self { + backend, + object: object.into(), + started: Arc::new(tokio::sync::Notify::new()), + guard: Arc::new(parking_lot::Mutex::new(None)), + } + } + + async fn wait_started(&self) { + self.started.notified().await; + } +} + +#[async_trait::async_trait] +impl TableCommitPublication for BlockingObjectPublication { + async fn begin_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn prepare(&self, table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> { + self.started.notify_one(); + let guard = self.backend.acquire_read_lock(table_bucket, &self.object).await?; + *self.guard.lock() = Some(guard); + Ok(()) + } + + fn holds_table_bucket(&self, _table_bucket: &str) -> bool { + true + } + + fn holds_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> bool { + self.guard.lock().is_some() + } + + fn complete(&self) { + drop(self.guard.lock().take()); + } +} + #[derive(Default)] struct TestCatalogObjectState { objects: BTreeMap<(String, String), TestCatalogObjectRecord>, @@ -1342,6 +1668,64 @@ async fn iceberg_snapshot_graph_counts_shared_manifests_once() { .expect("shared manifest objects must consume the commit budget only once"); } +#[tokio::test] +async fn iceberg_snapshot_graph_accepts_more_than_ten_thousand_live_files() { + let backend = TestCatalogObjectBackend::default(); + let namespace = Namespace::parse("analytics").expect("namespace should parse"); + let table = IdentifierSegment::parse("events").expect("table should parse"); + let entry = test_table_entry("warehouse", &namespace, &table, "tables/table-id/metadata/v1.metadata.json".to_string()); + let manifest_list = "s3://warehouse/tables/table-id/metadata/boundary-list.avro"; + let manifest = "s3://warehouse/tables/table-id/metadata/boundary-manifest.avro"; + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/boundary-list.avro", + manifest_list_avro_bytes(&[manifest]), + ) + .await; + + let data_file_count = 10_001; + let data_files = (0..data_file_count) + .map(|index| format!("s3://warehouse/tables/table-id/data/part-{index:05}.parquet")) + .collect::>(); + for data_file in &data_files { + backend + .seed_object( + "warehouse", + data_file + .strip_prefix("s3://warehouse/") + .expect("test data location should be in warehouse"), + vec![1], + ) + .await; + } + let references = data_files.iter().map(|data_file| (data_file.as_str(), 0)).collect::>(); + backend + .seed_object( + "warehouse", + "tables/table-id/metadata/boundary-manifest.avro", + manifest_avro_bytes(&references), + ) + .await; + + let mut metadata = table_metadata_json_for_validation(); + metadata["last-sequence-number"] = serde_json::Value::from(7); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 20, + "sequence-number": 7, + "timestamp-ms": 1, + "manifest-list": manifest_list, + "summary": {"operation": "append"} + }]); + metadata["current-snapshot-id"] = serde_json::Value::from(20); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 20}}); + let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &namespace, &table, &entry); + + validate_table_snapshot_changes(&context, None, &metadata) + .await + .expect("a valid graph with more than ten thousand live files should validate"); +} + #[tokio::test] async fn iceberg_v2_snapshot_graph_accepts_embedded_v2_manifests() { let backend = TestCatalogObjectBackend::default(); @@ -2129,6 +2513,7 @@ async fn table_data_plane_resource_resolves_registered_warehouse_prefix() { let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + backend.reset_call_counts().await; assert_eq!(backend.list_call_count().await, 0); let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") @@ -2155,6 +2540,7 @@ async fn table_data_plane_resource_does_not_match_sibling_prefix() { let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + backend.reset_call_counts().await; let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id-other/data/part-00001.parquet") .await @@ -2165,9 +2551,9 @@ async fn table_data_plane_resource_does_not_match_sibling_prefix() { } #[tokio::test] -async fn table_data_plane_resource_prefers_longest_registered_warehouse_prefix() { +async fn catalog_backings_reject_overlapping_registered_warehouse_prefixes() { let backend = TestCatalogObjectBackend::default(); - let store = ObjectTableCatalogStore::new(backend.clone()); + let store = ObjectTableCatalogStore::new(backend); let bucket = "analytics"; let namespace = Namespace::parse("sales").unwrap(); let parent_table = IdentifierSegment::parse("orders").unwrap(); @@ -2178,18 +2564,78 @@ async fn table_data_plane_resource_prefers_longest_registered_warehouse_prefix() let mut child_entry = test_table_entry(bucket, &namespace, &child_table, current); child_entry.table_id = "table-id-child".to_string(); child_entry.warehouse_location = format!("s3://{bucket}/tables/table-id/child"); - store.create_table(child_entry).await.unwrap(); + assert_matches!(store.create_table(child_entry.clone()).await, Err(TableCatalogStoreError::Conflict(_))); - let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/child/data/part-00001.parquet") + let strong_backend = TestCatalogObjectBackend::default(); + let strong_store = StrongTableCatalogStore::new(strong_backend); + strong_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + strong_store + .create_namespace(test_namespace_entry(bucket, &namespace)) .await - .expect("data-plane resource lookup should succeed") - .expect("object should resolve to the child table"); + .unwrap(); + strong_store + .create_table(test_table_entry( + bucket, + &namespace, + &parent_table, + default_table_metadata_file_path(&namespace, &parent_table, "00001.metadata.json"), + )) + .await + .unwrap(); + assert_matches!(strong_store.create_table(child_entry).await, Err(TableCatalogStoreError::Conflict(_))); +} - assert_eq!(resource.table, "orders_child"); - assert_eq!(resource.table_id, "table-id-child"); - assert_eq!(resource.warehouse_object_prefix, "tables/table-id/child/"); - assert_eq!(resource.catalog_resource_object(), "namespaces/sales/tables/orders_child"); - assert_eq!(backend.list_call_count().await, 0); +#[tokio::test] +async fn object_catalog_revalidates_legacy_warehouse_index_before_resolution() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let parent_table = IdentifierSegment::parse("orders").expect("parent table should parse"); + let child_table = IdentifierSegment::parse("orders_child").expect("child table should parse"); + let current = default_table_metadata_file_path(&namespace, &parent_table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &parent_table, current).await; + let mut child_entry = test_table_entry( + bucket, + &namespace, + &child_table, + default_table_metadata_file_path(&namespace, &child_table, "00001.metadata.json"), + ); + child_entry.table_id = "table-id-child".to_string(); + child_entry.table_uuid = "table-uuid-child".to_string(); + child_entry.warehouse_location = format!("s3://{bucket}/tables/table-id/child"); + store + .write_entry( + store.catalog_bucket(), + &store.paths.table_entry_path(bucket, &namespace, &child_table), + &child_entry, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("legacy overlapping table entry should be seeded"); + store + .write_entry( + store.catalog_bucket(), + &store.paths.warehouse_index_state_path(bucket), + &TableWarehouseIndexStateEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + state: TableCatalogEntryState::Active, + }, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("legacy warehouse index state should be seeded"); + + let error = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/child/data/part-00001.parquet") + .await + .expect_err("legacy overlapping prefixes must fail closed during index rebuild"); + + assert!(matches!( + error, + TableCatalogStoreError::Conflict(message) if message.contains("warehouse locations overlap") + )); } #[tokio::test] @@ -2221,6 +2667,7 @@ async fn table_data_plane_resource_skips_stale_deeper_index_and_matches_parent() ) .await .unwrap(); + backend.reset_call_counts().await; let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/child/data/part-00001.parquet") .await @@ -2295,7 +2742,7 @@ async fn object_table_catalog_store_rejects_duplicate_warehouse_prefix() { assert!(matches!( error, - TableCatalogStoreError::Conflict(message) if message.contains("warehouse location is already registered") + TableCatalogStoreError::Conflict(message) if message.contains("warehouse location overlaps an active table") )); } @@ -2658,8 +3105,8 @@ async fn object_table_catalog_store_restores_table_when_drop_index_delete_fails( let backend = TestCatalogObjectBackend::default(); let store = ObjectTableCatalogStore::new(backend.clone()); let bucket = "analytics"; - let namespace = Namespace::parse("sales").unwrap(); - let table = IdentifierSegment::parse("orders").unwrap(); + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); let index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); @@ -2718,11 +3165,14 @@ async fn object_table_catalog_store_rejects_invalid_table_warehouse_location() { let table = IdentifierSegment::parse("orders").unwrap(); let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); - store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); store .create_namespace(test_namespace_entry(bucket, &namespace)) .await - .unwrap(); + .expect("namespace should be created"); let mut entry = test_table_entry(bucket, &namespace, &table, current); entry.warehouse_location = format!("s3://{bucket}/tables/../table-id"); @@ -2751,7 +3201,7 @@ async fn object_table_catalog_store_rejects_deep_table_warehouse_location() { store .create_namespace(test_namespace_entry(bucket, &namespace)) .await - .unwrap(); + .expect("table should be created"); let mut entry = test_table_entry(bucket, &namespace, &table, current); entry.warehouse_location = format!("s3://{bucket}/{deep_prefix}"); @@ -8047,7 +8497,7 @@ async fn strong_catalog_backing_rejects_duplicate_snapshot_warehouse_index_entri .await .expect_err("duplicate warehouse prefix should fail snapshot hydration"); - assert_matches!(err, TableCatalogStoreError::Invalid(message) if message.contains("duplicate active table warehouse location")); + assert_matches!(err, TableCatalogStoreError::Invalid(message) if message.contains("overlapping active table warehouse location")); } #[tokio::test] @@ -8504,6 +8954,86 @@ async fn object_table_catalog_store_commits_with_token_match_and_writes_log() { ); } +async fn assert_catalog_rejects_metadata_changed_after_validation(store: &S, backend: &TestCatalogObjectBackend, bucket: &str) +where + S: TableCatalogStore + ?Sized, +{ + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .expect("table should be created"); + let validated_metadata = serde_json::json!({"format-version": 2, "properties": {"owner": "validated"}}); + let changed_metadata = serde_json::json!({"format-version": 2, "properties": {"owner": "replaced"}}); + backend + .seed_object( + bucket, + &new_metadata, + serde_json::to_vec(&changed_metadata).expect("changed metadata should encode"), + ) + .await; + + let error = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "digest-commit".to_string(), + idempotency_key: Some("digest-request".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata.clone(), + new_metadata_location: new_metadata, + requirements: vec![serde_json::json!({ + "type": TABLE_METADATA_DIGEST_REQUIREMENT_TYPE, + "sha256": canonical_json_sha256(&validated_metadata).expect("metadata digest should build") + })], + writer: Some("test-client".to_string()), + }) + .await + .expect_err("metadata changed after validation must not publish"); + + assert_matches!(error, TableCatalogStoreError::Conflict(_)); + let unchanged = store + .load_table(bucket, "sales", "orders") + .await + .expect("table lookup should succeed") + .expect("table should remain"); + assert_eq!(unchanged.metadata_location, current_metadata); + assert_eq!(unchanged.version_token, "token-v1"); + assert_eq!(unchanged.generation, 1); +} + +#[tokio::test] +async fn catalog_backings_reject_metadata_changed_after_validation() { + let object_backend = TestCatalogObjectBackend::default(); + assert_catalog_rejects_metadata_changed_after_validation( + &ObjectTableCatalogStore::new(object_backend.clone()), + &object_backend, + "object-catalog", + ) + .await; + + let strong_backend = TestCatalogObjectBackend::default(); + assert_catalog_rejects_metadata_changed_after_validation( + &StrongTableCatalogStore::new(strong_backend.clone()), + &strong_backend, + "strong-catalog", + ) + .await; +} + #[tokio::test] async fn object_table_catalog_store_syncs_warehouse_location_from_committed_metadata() { let backend = TestCatalogObjectBackend::default(); @@ -8555,6 +9085,7 @@ async fn object_table_catalog_store_syncs_warehouse_location_from_committed_meta .unwrap(); assert_eq!(result.table.warehouse_location, "s3://analytics/tables/relocated-table-id"); + backend.reset_call_counts().await; let resource = table_data_plane_resource_for_object(&store, bucket, "tables/relocated-table-id/data/part-00001.parquet") .await .expect("data-plane resource lookup should succeed") @@ -8750,6 +9281,194 @@ async fn object_table_catalog_store_recovers_staged_commit_after_post_cas_finali assert_eq!(committed.status, CommitLogStatus::Committed); } +#[test] +fn failed_commit_does_not_prove_historical_staged_commit() { + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table_name = IdentifierSegment::parse("orders").expect("table should parse"); + let mut table = test_table_entry( + "analytics", + &namespace, + &table_name, + default_table_metadata_file_path(&namespace, &table_name, "00003.metadata.json"), + ); + table.version_token = "token-v3".to_string(); + table.generation = 3; + let target = CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: "commit-1".to_string(), + idempotency_key: None, + table_id: table.table_id.clone(), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + new_version_token: "token-v2".to_string(), + previous_metadata_location: default_table_metadata_file_path(&namespace, &table_name, "00001.metadata.json"), + new_metadata_location: default_table_metadata_file_path(&namespace, &table_name, "00002.metadata.json"), + requirements: Vec::new(), + status: CommitLogStatus::Staged, + writer: None, + created_at: None, + updated_at: None, + }; + let failed = CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: "commit-2".to_string(), + idempotency_key: None, + table_id: table.table_id.clone(), + operation: "append".to_string(), + expected_version_token: "token-v2".to_string(), + new_version_token: "token-v3".to_string(), + previous_metadata_location: target.new_metadata_location.clone(), + new_metadata_location: table.metadata_location.clone(), + requirements: Vec::new(), + status: CommitLogStatus::Failed, + writer: None, + created_at: None, + updated_at: None, + }; + + let commits = [target.clone(), failed.clone()]; + assert!(!TableCommitHistoryIndex::new(&table, commits.iter()).proves_committed(&target)); + let recovery = table_commit_recovery_entry(&table, &failed, None, false); + assert_eq!(recovery.recovery_state, TableCommitRecoveryState::ManualReview); +} + +#[test] +fn commit_history_index_scales_across_a_long_table_history() { + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table_name = IdentifierSegment::parse("orders").expect("table should parse"); + let commit_count = 10_000; + let commits = (0..commit_count) + .map(|index| CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: format!("commit-{index:05}"), + idempotency_key: None, + table_id: "table-id".to_string(), + operation: "append".to_string(), + expected_version_token: format!("token-{index:05}"), + new_version_token: format!("token-{:05}", index + 1), + previous_metadata_location: format!("metadata/v{index:05}.metadata.json"), + new_metadata_location: format!("metadata/v{:05}.metadata.json", index + 1), + requirements: Vec::new(), + status: CommitLogStatus::Staged, + writer: None, + created_at: None, + updated_at: None, + }) + .collect::>(); + let mut table = test_table_entry("analytics", &namespace, &table_name, format!("metadata/v{commit_count:05}.metadata.json")); + table.version_token = format!("token-{commit_count:05}"); + + let history = TableCommitHistoryIndex::new(&table, commits.iter()); + + assert!(commits.iter().all(|commit| history.proves_committed(commit))); +} + +#[tokio::test] +async fn object_table_catalog_store_recovers_historical_staged_commit_after_later_commit() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let first_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let second_metadata = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + let first_commit_path = TableCatalogObjectPaths::default().commit_log_entry_path(bucket, "table-id", "commit-1"); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .expect("table should be created"); + backend.seed_object(bucket, &first_metadata, b"{}".to_vec()).await; + backend.seed_object(bucket, &second_metadata, b"{}".to_vec()).await; + backend.fail_put_attempt(RUSTFS_META_BUCKET, &first_commit_path, 2).await; + + let first_request = TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("request-1".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: first_metadata.clone(), + requirements: Vec::new(), + writer: None, + }; + let first = store + .commit_table(first_request.clone()) + .await + .expect("first commit should publish despite finalization failure"); + assert_eq!( + store + .get_commit_by_id(bucket, "table-id", "commit-1") + .await + .expect("first commit lookup should succeed") + .expect("first commit record should exist") + .status, + CommitLogStatus::Staged + ); + + let second = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-2".to_string(), + idempotency_key: Some("request-2".to_string()), + operation: "append".to_string(), + expected_version_token: first.table.version_token, + expected_metadata_location: first_metadata, + new_metadata_location: second_metadata.clone(), + requirements: Vec::new(), + writer: None, + }) + .await + .expect("second commit should succeed"); + + let report = store + .plan_table_commit_recovery(bucket, "sales", "orders") + .await + .expect("commit recovery plan should succeed"); + assert_eq!(report.finalization_required_count, 1); + assert_eq!(report.manual_review_count, 0); + assert_eq!( + report + .commits + .iter() + .find(|commit| commit.commit_id == "commit-1") + .expect("first commit should be reported") + .recovery_state, + TableCommitRecoveryState::FinalizationRequired + ); + + let replay = store + .commit_table(first_request) + .await + .expect("historical commit retry should finalize"); + assert_eq!(replay.table.metadata_location, second_metadata); + assert_eq!(replay.table.version_token, second.table.version_token); + assert_eq!(replay.commit_log.status, CommitLogStatus::Committed); + assert_eq!( + store + .get_commit_by_id(bucket, "table-id", "commit-1") + .await + .expect("first commit lookup should succeed") + .expect("first commit record should exist") + .status, + CommitLogStatus::Committed + ); +} + #[tokio::test] async fn table_commit_recovery_reports_post_cas_staged_commit() { let backend = TestCatalogObjectBackend::default(); @@ -9023,6 +9742,181 @@ async fn table_commit_recovery_repairs_stale_idempotency_index_after_partial_fin assert_eq!(repaired_index.status, CommitLogStatus::Committed); } +#[tokio::test] +async fn table_commit_recovery_does_not_overwrite_conflicting_idempotency_index() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let paths = TableCatalogObjectPaths::default(); + let commit_path = paths.commit_log_entry_path(bucket, "table-id", "commit-1"); + let idempotency_path = paths.commit_idempotency_entry_path(bucket, "table-id", "client-request-1"); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .expect("table should be created"); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + let request = TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client-request-1".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: None, + }; + let committed = store.commit_table(request.clone()).await.expect("commit should succeed"); + let mut conflicting_index = committed.commit_log; + conflicting_index.new_version_token = "different-token".to_string(); + backend + .seed_object( + RUSTFS_META_BUCKET, + &idempotency_path, + serde_json::to_vec(&conflicting_index).expect("conflicting index should encode"), + ) + .await; + let commit_before = backend + .read_object(RUSTFS_META_BUCKET, &commit_path) + .await + .expect("commit record lookup should succeed") + .expect("commit record should exist") + .data; + let index_before = backend + .read_object(RUSTFS_META_BUCKET, &idempotency_path) + .await + .expect("idempotency index lookup should succeed") + .expect("idempotency index should exist") + .data; + + let retry_error = store + .commit_table(request) + .await + .expect_err("conflicting index must fail closed"); + assert_matches!(retry_error, TableCatalogStoreError::Conflict(_)); + let planned = store + .plan_table_commit_recovery(bucket, "sales", "orders") + .await + .expect("commit recovery plan should succeed"); + assert_eq!(planned.manual_review_count, 1); + assert_eq!(planned.commits[0].recovery_state, TableCommitRecoveryState::ManualReview); + assert_eq!( + planned.commits[0].idempotency_index_status, + TableCommitIdempotencyIndexStatus::Conflicting + ); + + let recovered = store + .recover_table_commits(bucket, "sales", "orders") + .await + .expect("commit recovery should succeed"); + assert_eq!(recovered.manual_review_count, 1); + assert_eq!( + backend + .read_object(RUSTFS_META_BUCKET, &commit_path) + .await + .expect("commit record lookup should succeed") + .expect("commit record should remain") + .data, + commit_before + ); + assert_eq!( + backend + .read_object(RUSTFS_META_BUCKET, &idempotency_path) + .await + .expect("idempotency index lookup should succeed") + .expect("conflicting index should remain") + .data, + index_before + ); +} + +#[tokio::test] +async fn strong_table_commit_retry_rejects_token_only_idempotency_conflict() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .expect("table should be created"); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + let request = TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client-request-1".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: None, + }; + store + .commit_table(request.clone()) + .await + .expect("initial strong catalog commit should succeed"); + + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + let snapshot_object = backend + .read_object(RUSTFS_META_BUCKET, &snapshot_path) + .await + .expect("strong snapshot lookup should succeed") + .expect("strong snapshot should exist"); + let mut snapshot = serde_json::from_slice::(&snapshot_object.data).expect("strong snapshot should decode"); + assert_eq!(snapshot["commits"].as_array().map(Vec::len), Some(1)); + assert_eq!(snapshot["idempotency"].as_array().map(Vec::len), Some(1)); + snapshot["idempotency"][0]["commit"]["new_version_token"] = serde_json::json!("different-token"); + let conflicting_snapshot = serde_json::to_vec(&snapshot).expect("conflicting strong snapshot should encode"); + backend + .seed_object(RUSTFS_META_BUCKET, &snapshot_path, conflicting_snapshot.clone()) + .await; + + let retry_error = store + .commit_table(request) + .await + .expect_err("token-only idempotency mismatch must fail closed in strong mode"); + assert_matches!(retry_error, TableCatalogStoreError::Conflict(_)); + assert_eq!( + backend + .read_object(RUSTFS_META_BUCKET, &snapshot_path) + .await + .expect("strong snapshot lookup should succeed") + .expect("strong snapshot should remain") + .data, + conflicting_snapshot + ); +} + #[tokio::test] async fn object_table_catalog_store_rejects_stale_commit_token() { let backend = TestCatalogObjectBackend::default(); @@ -9222,6 +10116,13 @@ fn table_metadata_file_name_validation_rejects_unsafe_names() { assert!(!is_valid_table_metadata_file_name("00001\nmetadata.json")); } +#[test] +fn warehouse_prefix_overlap_distinguishes_nested_and_sibling_tables() { + assert!(warehouse_object_prefixes_overlap("tables/table-id/", "tables/table-id/child/")); + assert!(warehouse_object_prefixes_overlap("tables/table-id/child/", "tables/table-id/")); + assert!(!warehouse_object_prefixes_overlap("tables/table-id/", "tables/table-id-other/")); +} + #[test] fn table_metadata_location_validation_stays_inside_metadata_dir() { let namespace = Namespace::parse("analytics.daily_events").unwrap();