From c7233d6624541be6e6205588e0f917f6eec8b287 Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Wed, 12 Aug 2026 16:28:56 +0800 Subject: [PATCH] fix(table-catalog): harden strong backing compatibility (#5941) * fix(table-catalog): harden strong backing compatibility * fix(table-catalog): close strong backing recovery gaps * fix(table-catalog): harden strong backing recovery * fix(table-catalog): repair strong backing CI failures * fix(table-catalog): satisfy test clippy lint --------- Co-authored-by: Henry Guo --- docs/architecture/compat-cleanup-register.md | 3 + docs/architecture/s3-tables-support-matrix.md | 60 +- .../src/admin/handlers/table_catalog/mod.rs | 9 +- .../src/admin/handlers/table_catalog/tests.rs | 4 +- rustfs/src/app/context/global.rs | 7 + rustfs/src/storage/access.rs | 11 +- rustfs/src/table_catalog/iceberg/commit.rs | 32 +- .../src/table_catalog/iceberg/validation.rs | 14 +- rustfs/src/table_catalog/mod.rs | 9 +- rustfs/src/table_catalog/model.rs | 12 +- rustfs/src/table_catalog/store/migration.rs | 344 +- rustfs/src/table_catalog/store/mod.rs | 120 +- rustfs/src/table_catalog/store/object.rs | 786 ++- rustfs/src/table_catalog/store/strong.rs | 1502 ++++- rustfs/src/table_catalog/tests.rs | 5654 ++++++++++++++++- 15 files changed, 7873 insertions(+), 694 deletions(-) diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index e985a3441..e1a0b8040 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -13,6 +13,9 @@ for later deletion. ## Open Items - `table-publication-fence-v1` S3 Tables publication fencing: nodes that predate table and table-bucket publication fences can mutate live files while a new node is publishing a catalog pointer. New nodes retain exact object guards until the operator confirms that every serving node uses the new fences. Fleet confirmation also requires non-overlapping active warehouse prefixes and lifecycle workers that exclude table buckets. Remove the exact live-file fallback and the fleet-confirmation gate after the minimum supported RustFS release acquires table fences for registered-table mutations and table-bucket fences for unresolved-prefix mutations. +- `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: mixed-version deployments continue writing version 1 snapshots until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2. +- `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration. +- `table-catalog-backing-manifest-v1-wire-labels` durable strong backing manifest labels: version 1 published "STRONG_KV_WAL" and "CUT_OVER_LINEARIZABLE_READS" before the implementation was narrowed to the ETag-CAS durable snapshot backing. Internal names and operator documentation describe the implemented semantics, while version 1 responses retain those labels for existing clients. Replace the labels only in a new manifest version with an explicit client migration contract. - `cross-pool-fence-v1` authenticated unsupported advertisement: predeployment servers recognize the versioned cross-pool fence capability probe but report support version 0, allowing a later all-peer probe to distinguish predeployment nodes without activating a second lock domain. Replace the unsupported advertisement only when composite lock acquisition, a cluster-wide activation fence, complete fleet proof, commit-time proof revalidation, and fail-closed revocation ship together. - `table-catalog-dotted-namespace` Iceberg REST namespace path compatibility: existing RustFS clients use dotted namespace paths, while the standard multi-level contract uses the URL-encoded unit separator `%1F`. New servers accept both forms so a rolling upgrade does not invalidate existing catalog configuration. Remove the dotted fallback after the minimum supported RustFS release advertises `%1F` and all supported clients have refreshed their catalog configuration. - `rustfs-5509` FileInfo positional MessagePack decoding: beta.11 serialized 28 fields, while beta.12 inserted transition-version fields in the middle and serialized an incompatible 30-field array. New releases write named maps and retain readers for both shipped array layouts so direct and rolling upgrades can read either release. Remove the positional-array readers after every supported direct-upgrade release writes named maps and no retained RPC payload can contain a pre-map FileInfo array. diff --git a/docs/architecture/s3-tables-support-matrix.md b/docs/architecture/s3-tables-support-matrix.md index 1a619c921..5f4f5c0f5 100644 --- a/docs/architecture/s3-tables-support-matrix.md +++ b/docs/architecture/s3-tables-support-matrix.md @@ -116,12 +116,13 @@ catalog extension. | Commit publication fencing | Supported with rolling-upgrade gate | Existing deployments retain exact object guards so older writers cannot mutate referenced files during publication. Set `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` only after every serving node supports table and table-bucket publication fences. In scalable mode, active table warehouse prefixes must not overlap, ordinary lifecycle expiry remains disabled for table buckets, and first enablement, first publication, drop, and warehouse relocation are serialized by the table-bucket fence. | | Post-CAS finalization recovery | Supported | Diagnostics and recovery can repair stale or missing idempotency indexes without changing the current table pointer. | | Catalog export | Supported | Exposes table state, commit recovery state, and backing migration information for operator inspection. | -| Strong backing state transfer | Supported | Object-backed table bucket, namespace, table, view, commit-log, and idempotency state can be materialized into the durable strong snapshot. The transfer is deterministic, ETag-CAS protected, idempotent after an interrupted finalization, and fails closed when a table or view has no owning namespace entry. | -| Durable backing migration preflight | Supported | `GET /iceberg/v1/{warehouse}/catalog/migration` and the `/_iceberg/v1` alias inspect object-backed catalog inventory, recovery blockers, warehouse prefix index readiness, persistent write-fence state, target snapshot agreement, and whether every table bucket is ready for cutover. | -| Durable backing migration execution | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/migration` fences table-bucket registry changes, acquires a persistent per-bucket write fence, drains in-flight catalog mutations, materializes the target snapshot, and reports `ready_to_enable_durable_strong`. `DELETE` safely releases the bucket fence only while its target state has not advanced, and releases the registry fence after the last bucket is cancelled. Both mutations require `admin:MigrateTableCatalog`. | +| Strong backing state transfer | Supported | Object-backed table bucket, namespace, table, view, commit-log, and idempotency state can be materialized into the durable strong snapshot. The transfer is deterministic, ETag-CAS protected, idempotent after an interrupted finalization, validates candidate state through the restart decoder before publication, preserves resource-backed implicit namespaces, and fails closed when an inactive explicit namespace conflicts with active descendants or resources. Snapshot hydration requires a stable non-empty ETag, caps the encoded snapshot at 64 MiB, shares state and reload serialization across requests in one server context, and rejects disappearance or format-version rollback after observation. Configured durable-strong mode rejects a missing snapshot on its first catalog access after startup; only object-backed migration may initialize an empty target. | +| Durable backing migration preflight | Supported | `GET /iceberg/v1/{warehouse}/catalog/migration` and the `/_iceberg/v1` alias inspect object-backed catalog inventory, recovery blockers, warehouse prefix index readiness, active table/view identifier collisions, persistent write-fence state, target snapshot agreement, and whether every table bucket is ready for cutover. | +| Durable backing migration execution | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/migration` fences table-bucket registry changes, acquires a persistent per-bucket write fence, records whether a global strong snapshot existed before publication, drains in-flight catalog mutations, materializes the target snapshot, and reports `ready_to_enable_durable_strong`. Retries and `DELETE` may restore a known-absent initial target after an ambiguous first write, but fail closed if a previously existing or materialized global snapshot disappears. `DELETE` releases the bucket fence only while its target state has not advanced, and releases the registry fence after the last bucket is cancelled. Both mutations require `admin:MigrateTableCatalog`. | +| Strong snapshot rolling compatibility | Supported | Durable strong control-plane reads snapshot versions 1 and 2, writes version 1 by default, and writes version 2 only after both the requested and fleet-confirmed gates are enabled. A running process rejects any lower-format snapshot after observing a higher format. Once version 2 is fleet-confirmed, table data-plane resolution fails closed until the persisted snapshot is version 2; a missing table-bucket entry also fails closed instead of bypassing table-aware authorization. | | Disaster recovery rehearsal | Manual/live harness | `failure_coverage.py --print-disaster-recovery-rehearsal` generates an operator runbook covering catalog export, diagnostics, safe recovery repair, rollback/import, durable backing migration dry-run, post-recovery loadTable, and table data-plane policy probes. | | Scale and fault rehearsal | Manual/live harness | `failure_coverage.py --print-scale-fault-rehearsal` generates an opt-in runbook for concurrent writer stress, maintenance scheduler lease recovery, durable backing cutover preflight, recovery/rollback/import under load, and post-run evidence capture. | -| Strong KV/WAL backing cutover | Preview / controlled | Operators can select durable strong backing with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong` only after every table bucket reports `SNAPSHOT_MATERIALIZED` and `ready_to_enable_durable_strong: true`. Object-only advanced operations fail closed in durable strong mode. | +| Durable strong snapshot backing cutover | Preview / controlled | Operators can select the ETag-CAS snapshot backing with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong` only after every table bucket reports `SNAPSHOT_MATERIALIZED` and `ready_to_enable_durable_strong: true`. This mode does not claim a separate external KV/WAL service, and object-only advanced operations fail closed. Version 1 backing manifests retain the legacy `STRONG_KV_WAL` and `CUT_OVER_LINEARIZABLE_READS` wire labels for client compatibility; those labels do not expand the implementation claim. | | Single active writer region | Supported policy | Diagnostics publish single-active-writer semantics and read-only replica limits. | | Active-active multi-region writes | Not claimed | A table must not accept independent concurrent writers in multiple active regions. | @@ -136,23 +137,60 @@ warehouse: `GetTableCatalogAction` on each table bucket. Treat every `blockers` entry as fail-closed; repair commit recovery state and backfill the warehouse prefix index before continuing. -3. Run `POST /iceberg/v1/{warehouse}/catalog/migration` with - `admin:MigrateTableCatalog`. This persists the source write fence before it - drains in-flight mutations and copies the catalog state. -4. Repeat the preflight and materialization for every table bucket. Do not set +3. Before the migration `POST`, drain every catalog writer that predates the + durable-backing migration fence and restart it on a fence-aware release. An + older writer does not recognize the persisted fence and can otherwise + mutate the object-backed source after the snapshot inventory is captured. + Keep all catalog writers on the fence-aware release until cutover completes. +4. Inventory object-only advanced operations, including maintenance workers, + catalog recovery, export, diagnostics, and external catalog bridge writes. + Quiesce mutating operations before cutover and confirm that each required + operation is supported by durable-strong mode; unsupported operations fail + closed after cutover rather than continuing against object-backed state. +5. Run `POST /iceberg/v1/{warehouse}/catalog/migration` with + `admin:MigrateTableCatalog`. This acquires the exclusive migration fence to + drain in-flight fence-aware mutations, persists the source fence while + exclusivity is held, and then copies the catalog state. +6. Repeat the preflight and materialization for every table bucket. Do not set `RUSTFS_TABLE_CATALOG_BACKING=durable-strong` until the preflight reports `SNAPSHOT_MATERIALIZED`, no blockers, and `ready_to_enable_durable_strong: true`. -5. Restart with durable strong backing enabled, then verify catalog config, +7. Restart with durable strong backing enabled, then verify catalog config, table and view loads, commit idempotency, and table data-plane policy resolution before admitting writers. -6. Before restarting into durable-strong mode, `DELETE` on the migration +8. Before restarting into durable-strong mode, `DELETE` on the migration endpoint can remove a migration-created target bucket snapshot and release the source fence. After the durable-strong state advances, cancellation fails closed; recovery requires an operator-selected restore or reverse migration instead of restarting against the stale object-backed pointer. -7. Preserve the object-backed catalog backup until durable strong backing has +9. Preserve the object-backed catalog backup until durable strong backing has passed the operator's retention window. +10. Keep strong snapshot writes on version 1 during a rolling binary upgrade. + After every catalog writer can read version 2, set both + `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2=true` and + `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED=true`, then restart + the catalog writers. Perform a controlled catalog write or migration + materialization and confirm that the persisted snapshot is version 2 before + serving table data-plane traffic. Setting only one gate does not change the + write format. +11. After any version 2 snapshot is persisted, do not roll catalog writers back + to a binary that only reads version 1. Current binaries preserve version 2 + even when the gates are later disabled. A running process rejects restored + version 1 content after observing version 2, but cannot distinguish an older + snapshot with the same format version from a deliberate restore. The format + high-water mark is process-local: restoring any older snapshot and restarting + every writer is a privileged disaster-recovery rollback that cannot be + inferred from the restored object alone. Recovery must restore a compatible + binary and a snapshot selected through the operator recovery procedure. +12. Migration preflight rejects an active table/view identifier collision before + it writes a migration fence. A pre-existing version 1 strong snapshot with + such a collision is loaded in cleanup-only quarantine. Ambiguous reads fail + closed; each cleanup mutation must reduce the collision set, and unrelated + writes remain blocked until all collisions are removed. Drain catalog + writers that predate cleanup quarantine before starting this repair, and + complete cleanup before the first version 2 write. Restoring any version 1 + snapshot after a writer has observed version 2 fails closed instead of + replacing the in-process catalog state. ## Production Failure Coverage diff --git a/rustfs/src/admin/handlers/table_catalog/mod.rs b/rustfs/src/admin/handlers/table_catalog/mod.rs index b2e90b90c..94ed44b56 100644 --- a/rustfs/src/admin/handlers/table_catalog/mod.rs +++ b/rustfs/src/admin/handlers/table_catalog/mod.rs @@ -2067,9 +2067,12 @@ fn job_id_from_params(params: &Params<'_, '_>) -> S3Result { fn table_catalog_backend_from_extensions( extensions: &http::Extensions, ) -> S3Result> { - let store = runtime_sources::object_store_from_extensions(extensions) - .ok_or_else(|| table_catalog_internal_error("request object store is not initialized"))?; - Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new(store)) + let context = runtime_sources::app_context_from_extensions(extensions) + .ok_or_else(|| table_catalog_internal_error("request application context is not initialized"))?; + Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime( + context.object_store(), + context.table_catalog_strong_runtime(), + )) } type EcStoreObjectTableCatalogStore = diff --git a/rustfs/src/admin/handlers/table_catalog/tests.rs b/rustfs/src/admin/handlers/table_catalog/tests.rs index bfcf4f321..3360f7d70 100644 --- a/rustfs/src/admin/handlers/table_catalog/tests.rs +++ b/rustfs/src/admin/handlers/table_catalog/tests.rs @@ -2512,7 +2512,7 @@ async fn commit_publication_replays_historical_standard_commit_across_backings() crate::table_catalog::TableCatalogBackingMode::DurableStrong, ] { let metadata_backend = TestTableCatalogObjectBackend::default(); - let store = crate::table_catalog::ConfiguredTableCatalogStore::new(metadata_backend.clone(), mode); + let store = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(metadata_backend.clone(), mode); let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); create_standard_events_table(&store, &metadata_backend, &namespace).await; let first_request = serde_json::json!({ @@ -2559,7 +2559,7 @@ async fn commit_publication_replays_historical_standard_commit_across_backings() } } - let store = crate::table_catalog::ConfiguredTableCatalogStore::new(metadata_backend.clone(), mode); + let store = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(metadata_backend.clone(), mode); let second = standard_commit_table_response( &store, &trusted_table_commit_backend(&metadata_backend), diff --git a/rustfs/src/app/context/global.rs b/rustfs/src/app/context/global.rs index 6a0fb763f..50c79498a 100644 --- a/rustfs/src/app/context/global.rs +++ b/rustfs/src/app/context/global.rs @@ -76,6 +76,7 @@ pub struct AppContext { buffer_config: Arc, object_data_cache: Arc, object_traffic_health: Arc, + table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime, } impl AppContext { @@ -125,6 +126,7 @@ impl AppContext { buffer_config: default_buffer_config_interface(), object_data_cache, object_traffic_health: Arc::new(ObjectTrafficHealth::from_env()), + table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime::default(), } } @@ -144,6 +146,10 @@ impl AppContext { Arc::clone(&self.object_traffic_health) } + pub(crate) fn table_catalog_strong_runtime(&self) -> crate::table_catalog::StrongTableCatalogRuntime { + self.table_catalog_strong_runtime.clone() + } + pub fn iam(&self) -> Arc { self.iam.clone() } @@ -350,6 +356,7 @@ impl AppContext { buffer_config: interfaces.buffer_config, object_data_cache: ObjectDataCacheAdapter::disabled_arc(), object_traffic_health: Arc::new(ObjectTrafficHealth::from_env()), + table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime::default(), } } diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index 4592ebeb2..2de6c3dda 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -1438,8 +1438,15 @@ fn table_data_plane_content_mutation(action: Action) -> bool { fn table_catalog_backend_for_data_plane( req: &S3Request, ) -> S3Result> { - let store = request_object_store(req)?; - Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new(store)) + let context = match req.extensions.get::>() { + Some(server_ctx) => server_ctx.installed_app_context(), + None => runtime_sources::current_app_context(), + } + .ok_or_else(object_store_not_initialized_error)?; + Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime( + context.object_store(), + context.table_catalog_strong_runtime(), + )) } fn table_catalog_store_for_data_plane( diff --git a/rustfs/src/table_catalog/iceberg/commit.rs b/rustfs/src/table_catalog/iceberg/commit.rs index 224654517..bfd890a6d 100644 --- a/rustfs/src/table_catalog/iceberg/commit.rs +++ b/rustfs/src/table_catalog/iceberg/commit.rs @@ -44,11 +44,14 @@ pub(crate) fn table_matches_staged_base(table: &TableEntry, commit_log: &CommitL pub(crate) struct TableCommitHistoryIndex<'a> { table_id: &'a str, reachable_states: BTreeSet<(&'a str, &'a str)>, + ambiguous_states: BTreeSet<(&'a str, &'a str)>, + cycle_detected: bool, } impl<'a> TableCommitHistoryIndex<'a> { pub(crate) fn new(table: &'a TableEntry, commits: impl IntoIterator) -> Self { let mut by_new_state = BTreeMap::<(&str, &str), Option<(&str, &str)>>::new(); + let mut ambiguous_states = BTreeSet::new(); for commit in commits .into_iter() .filter(|commit| commit.table_id == table.table_id && !matches!(commit.status, CommitLogStatus::Failed)) @@ -57,27 +60,39 @@ impl<'a> TableCommitHistoryIndex<'a> { let previous = (commit.previous_metadata_location.as_str(), commit.expected_version_token.as_str()); by_new_state .entry(key) - .and_modify(|candidate| *candidate = None) + .and_modify(|candidate| { + *candidate = None; + ambiguous_states.insert(key); + }) .or_insert(Some(previous)); } let mut reachable_states = BTreeSet::new(); let mut state = (table.metadata_location.as_str(), table.version_token.as_str()); - while reachable_states.insert(state) { + let cycle_detected = loop { + if !reachable_states.insert(state) { + break true; + } let Some(Some(previous)) = by_new_state.get(&state) else { - break; + break false; }; state = *previous; - } + }; Self { table_id: &table.table_id, reachable_states, + ambiguous_states, + cycle_detected, } } pub(crate) fn proves_committed(&self, target: &CommitLogEntry) -> bool { - self.table_id == target.table_id.as_str() + !self.cycle_detected + && self.table_id == target.table_id.as_str() && !matches!(target.status, CommitLogStatus::Failed) + && !self + .ambiguous_states + .contains(&(target.new_metadata_location.as_str(), target.new_version_token.as_str())) && self .reachable_states .contains(&(target.new_metadata_location.as_str(), target.new_version_token.as_str())) @@ -202,7 +217,7 @@ pub(crate) fn table_commit_recovery_entry( TableCommitRecoveryState::FinalizationRequired, "a later committed pointer proves this staged commit is part of table history".to_string(), ) - } else if matches!(commit_log.status, CommitLogStatus::Committed) { + } else if matches!(commit_log.status, CommitLogStatus::Committed) && historically_committed { if idempotency_index_repair_required { ( TableCommitRecoveryState::IdempotencyIndexRepairRequired, @@ -214,6 +229,11 @@ pub(crate) fn table_commit_recovery_entry( "commit is finalized and may be older than the current table pointer".to_string(), ) } + } else if matches!(commit_log.status, CommitLogStatus::Committed) { + ( + TableCommitRecoveryState::ManualReview, + "committed log is not reachable from the current table pointer".to_string(), + ) } else if table_matches_staged_base(table, commit_log) { ( TableCommitRecoveryState::StagedBeforeTableUpdate, diff --git a/rustfs/src/table_catalog/iceberg/validation.rs b/rustfs/src/table_catalog/iceberg/validation.rs index 56953e9ba..559f3c62c 100644 --- a/rustfs/src/table_catalog/iceberg/validation.rs +++ b/rustfs/src/table_catalog/iceberg/validation.rs @@ -286,17 +286,15 @@ where if table.state != TableCatalogEntryState::Active { continue; } - let Ok(warehouse_object_prefix) = table_warehouse_object_prefix(&table) else { - continue; - }; + let warehouse_object_prefix = table_warehouse_object_prefix(&table)?; if !object.starts_with(&warehouse_object_prefix) { continue; } - if matched - .as_ref() - .is_some_and(|current| current.warehouse_object_prefix.len() >= warehouse_object_prefix.len()) - { - continue; + if let Some(current) = matched.as_ref() { + return Err(TableCatalogStoreError::Invalid(format!( + "object {object} matches overlapping active table warehouse prefixes {} and {warehouse_object_prefix}", + current.warehouse_object_prefix + ))); } matched = Some(table_data_plane_resource_from_entry(table, warehouse_object_prefix)); } diff --git a/rustfs/src/table_catalog/mod.rs b/rustfs/src/table_catalog/mod.rs index 51259a855..cdcaf7d03 100644 --- a/rustfs/src/table_catalog/mod.rs +++ b/rustfs/src/table_catalog/mod.rs @@ -97,6 +97,9 @@ pub(crate) const TABLE_CATALOG_BACKING_MANIFEST_VERSION: u16 = 1; pub(crate) const ENV_TABLE_CATALOG_BACKING: &str = "RUSTFS_TABLE_CATALOG_BACKING"; pub(crate) const ENV_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED: &str = "RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED"; +pub(crate) const ENV_TABLE_CATALOG_STRONG_SNAPSHOT_V2: &str = "RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2"; +pub(crate) const ENV_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED: &str = + "RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED"; pub(crate) const TABLE_CATALOG_BACKING_OBJECT: &str = "object"; pub(crate) const TABLE_CATALOG_BACKING_DURABLE_STRONG: &str = "durable-strong"; pub(crate) const TABLE_METADATA_DIGEST_REQUIREMENT_TYPE: &str = "assert-rustfs-metadata-sha256"; @@ -159,10 +162,12 @@ const ICEBERG_MAX_REF_AGE_MS_PROPERTY: &str = "history.expire.max-ref-age-ms"; const ICEBERG_REF_MIN_SNAPSHOTS_TO_KEEP_FIELD: &str = "min-snapshots-to-keep"; const ICEBERG_REF_MAX_SNAPSHOT_AGE_MS_FIELD: &str = "max-snapshot-age-ms"; const ICEBERG_REF_MAX_REF_AGE_MS_FIELD: &str = "max-ref-age-ms"; -const STRONG_TABLE_CATALOG_SNAPSHOT_VERSION: u16 = 1; +const STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION: u16 = 1; +const STRONG_TABLE_CATALOG_SNAPSHOT_VERSION: u16 = 2; const STRONG_TABLE_CATALOG_BACKING_ROOT: &str = "strong-backing"; const STRONG_TABLE_CATALOG_SNAPSHOT_FILE: &str = "snapshot.json"; -const TABLE_CATALOG_MIGRATION_VERSION: u16 = 1; +const TABLE_CATALOG_MIGRATION_MIN_READ_VERSION: u16 = 1; +const TABLE_CATALOG_MIGRATION_VERSION: u16 = 2; const TABLE_CATALOG_MIGRATION_ROOT: &str = "backing-migration"; const TABLE_CATALOG_MIGRATION_FENCE_FILE: &str = "durable-strong-fence.json"; const TABLE_CATALOG_MIGRATION_FENCE_LOCK: &str = "durable-strong-fence.lock"; diff --git a/rustfs/src/table_catalog/model.rs b/rustfs/src/table_catalog/model.rs index 242ec71ba..3cef76469 100644 --- a/rustfs/src/table_catalog/model.rs +++ b/rustfs/src/table_catalog/model.rs @@ -1067,7 +1067,9 @@ pub(crate) struct TableCatalogBackingProfile { #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub(crate) enum TableCatalogBackingKind { ObjectBacked, - StrongKvWal, + // RUSTFS_COMPAT_TODO(table-catalog-backing-manifest-v1-wire-labels): Keep the version 1 wire label for existing clients. Remove after a versioned manifest with an explicit client migration contract replaces it. + #[serde(rename = "STRONG_KV_WAL")] + DurableStrongSnapshot, } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] @@ -1203,7 +1205,9 @@ pub(crate) enum TableCatalogBackingMigrationStep { ReplayCommitLog, VerifyCurrentPointer, EnableSingleWriterFencing, - CutOverLinearizableReads, + // RUSTFS_COMPAT_TODO(table-catalog-backing-manifest-v1-wire-labels): Keep the version 1 wire label for existing clients. Remove after a versioned manifest with an explicit client migration contract replaces it. + #[serde(rename = "CUT_OVER_LINEARIZABLE_READS")] + CutOverDurableSnapshotReads, } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] @@ -1213,6 +1217,8 @@ pub(crate) enum TableCatalogBackingMigrationBlocker { CommitManualReviewRequired, WarehouseIndexBackfillRequired, DuplicateWarehousePrefix, + DuplicateTableIdentity, + TableViewIdentifierCollision, DurableStrongSnapshotChanged, } @@ -1222,6 +1228,8 @@ pub(crate) enum TableCatalogBackingMigrationAction { RunCatalogRecovery, BackfillWarehouseIndex, ReviewDuplicateWarehousePrefixes, + ReviewDuplicateTableIdentities, + ReviewTableViewIdentifierCollisions, SnapshotObjectBackedCatalog, EnableDurableStrongBacking, VerifyDurableStrongSnapshot, diff --git a/rustfs/src/table_catalog/store/migration.rs b/rustfs/src/table_catalog/store/migration.rs index 6b33eebb7..d8224e471 100644 --- a/rustfs/src/table_catalog/store/migration.rs +++ b/rustfs/src/table_catalog/store/migration.rs @@ -13,11 +13,13 @@ // limitations under the License. use super::object::{ - ObjectTableCatalogStore, validate_namespace_entry_object, validate_table_entry_object, validate_view_entry_object, + ObjectTableCatalogStore, validate_commit_idempotency_entry_object, validate_commit_log_entry_object, + validate_namespace_entry_object, validate_table_bucket_entry_object, validate_table_entry_object, validate_view_entry_object, }; use super::strong::{ StrongCommitSnapshotRecord, StrongTableCatalogBucketSnapshot, StrongTableCatalogState, TableCatalogBackingMigrationFence, - TableCatalogBackingMigrationFenceStatus, TableCatalogBackingMigrationGlobalFence, table_catalog_bucket_snapshot_fingerprint, + TableCatalogBackingMigrationFenceStatus, TableCatalogBackingMigrationGlobalFence, + TableCatalogBackingMigrationTargetSnapshotState, table_catalog_bucket_snapshot_fingerprint, }; use super::*; @@ -82,14 +84,14 @@ pub(super) fn table_catalog_backing_manifest( }, migration: TableCatalogBackingMigrationPlan { source_kind: TableCatalogBackingKind::ObjectBacked, - target_kind: TableCatalogBackingKind::StrongKvWal, + target_kind: TableCatalogBackingKind::DurableStrongSnapshot, status: migration_status, required_steps: vec![ TableCatalogBackingMigrationStep::SnapshotCatalogExport, TableCatalogBackingMigrationStep::ReplayCommitLog, TableCatalogBackingMigrationStep::VerifyCurrentPointer, TableCatalogBackingMigrationStep::EnableSingleWriterFencing, - TableCatalogBackingMigrationStep::CutOverLinearizableReads, + TableCatalogBackingMigrationStep::CutOverDurableSnapshotReads, ], blockers, }, @@ -118,12 +120,100 @@ impl ObjectTableCatalogStore where B: TableCatalogObjectBackend, { + fn migration_target_snapshot_state( + fence: &TableCatalogBackingMigrationFence, + ) -> TableCatalogBackingMigrationTargetSnapshotState { + // RUSTFS_COMPAT_TODO(table-catalog-migration-fence-v1): Version 1 PREPARING fences have no durable baseline. Remove after all supported upgrade sources write version 2 fences and all version 1 migrations are completed or cancelled. + if fence.version < TABLE_CATALOG_MIGRATION_VERSION { + TableCatalogBackingMigrationTargetSnapshotState::Unknown + } else if fence.target_snapshot_etag.is_some() { + TableCatalogBackingMigrationTargetSnapshotState::Present + } else { + TableCatalogBackingMigrationTargetSnapshotState::Absent + } + } + + fn validate_backing_migration_fence( + table_bucket: &str, + fence: &TableCatalogBackingMigrationFence, + ) -> TableCatalogStoreResult<()> { + if !(TABLE_CATALOG_MIGRATION_MIN_READ_VERSION..=TABLE_CATALOG_MIGRATION_VERSION).contains(&fence.version) + || fence.table_bucket != table_bucket + || fence.migration_id.is_empty() + { + return Err(TableCatalogStoreError::Invalid(format!( + "invalid durable strong migration fence for table bucket {table_bucket}" + ))); + } + let target_snapshot_state = Self::migration_target_snapshot_state(fence); + if fence.target_bucket_existed && target_snapshot_state == TableCatalogBackingMigrationTargetSnapshotState::Absent { + return Err(TableCatalogStoreError::Invalid(format!( + "durable strong migration fence for table bucket {table_bucket} has an inconsistent target snapshot baseline" + ))); + } + match fence.status { + TableCatalogBackingMigrationFenceStatus::Preparing if fence.source_fingerprint.is_some() => { + return Err(TableCatalogStoreError::Invalid(format!( + "preparing durable strong migration fence for table bucket {table_bucket} has materialized state" + ))); + } + TableCatalogBackingMigrationFenceStatus::Materialized + if fence.source_fingerprint.is_none() || fence.target_snapshot_etag.is_none() => + { + return Err(TableCatalogStoreError::Invalid(format!( + "materialized durable strong migration fence for table bucket {table_bucket} is incomplete" + ))); + } + _ => {} + } + Ok(()) + } + + fn validate_global_backing_migration_fence(fence: &TableCatalogBackingMigrationGlobalFence) -> TableCatalogStoreResult<()> { + if !(TABLE_CATALOG_MIGRATION_MIN_READ_VERSION..=TABLE_CATALOG_MIGRATION_VERSION).contains(&fence.version) + || fence.migration_id.is_empty() + { + return Err(TableCatalogStoreError::Invalid( + "invalid durable strong global migration fence".to_string(), + )); + } + Ok(()) + } + + async fn observe_durable_strong_migration_target( + strong_store: &StrongTableCatalogStore, + table_bucket: &str, + fence: Option<&TableCatalogBackingMigrationFence>, + ) -> TableCatalogStoreResult<(Option, Option)> { + let target_snapshot_state = fence.map(Self::migration_target_snapshot_state); + let permits_absent_snapshot = fence.is_some_and(|fence| { + fence.status == TableCatalogBackingMigrationFenceStatus::Preparing + && !fence.target_bucket_existed + && target_snapshot_state == Some(TableCatalogBackingMigrationTargetSnapshotState::Absent) + }); + if permits_absent_snapshot { + strong_store.restore_absent_migration_snapshot_baseline().await?; + } + let observation = strong_store.bucket_snapshot_observation(table_bucket).await?; + if fence.is_some() && observation.1.is_none() && !permits_absent_snapshot { + return Err(TableCatalogStoreError::Conflict( + "durable strong catalog snapshot is missing for an in-progress backing migration".to_string(), + )); + } + Ok(observation) + } + async fn read_backing_migration_fence( &self, table_bucket: &str, ) -> TableCatalogStoreResult)>> { - self.read_entry(self.catalog_bucket(), &self.paths.backing_migration_fence_path(table_bucket)) - .await + let fence = self + .read_entry(self.catalog_bucket(), &self.paths.backing_migration_fence_path(table_bucket)) + .await?; + if let Some((fence, _)) = fence.as_ref() { + Self::validate_backing_migration_fence(table_bucket, fence)?; + } + Ok(fence) } pub(super) async fn acquire_table_bucket_registry_write_permit(&self) -> TableCatalogStoreResult> { @@ -164,11 +254,7 @@ where .read_entry::(self.catalog_bucket(), fence_path) .await? { - if fence.version != TABLE_CATALOG_MIGRATION_VERSION { - return Err(TableCatalogStoreError::Invalid( - "invalid durable strong global migration fence".to_string(), - )); - } + Self::validate_global_backing_migration_fence(&fence)?; return Ok(fence); } let fence = TableCatalogBackingMigrationGlobalFence { @@ -181,6 +267,13 @@ where } async fn clear_global_backing_migration_fence_if_unused(&self, fence_path: &str) -> TableCatalogStoreResult<()> { + let Some((fence, _)) = self + .read_entry::(self.catalog_bucket(), fence_path) + .await? + else { + return Ok(()); + }; + Self::validate_global_backing_migration_fence(&fence)?; let bucket_objects = self .backend .list_objects(self.catalog_bucket(), &self.paths.table_bucket_entries_prefix()) @@ -194,19 +287,6 @@ where self.backend.delete_object(self.catalog_bucket(), fence_path).await } - pub(super) async fn ensure_object_backed_writes_allowed(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { - if self - .backend - .object_exists(self.catalog_bucket(), &self.paths.backing_migration_fence_path(table_bucket)) - .await? - { - return Err(TableCatalogStoreError::Conflict(format!( - "object-backed catalog writes are fenced while table bucket {table_bucket} is prepared for durable strong cutover" - ))); - } - Ok(()) - } - async fn collect_bucket_snapshot_with_locks( &self, table_bucket: &str, @@ -220,11 +300,7 @@ where else { return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); }; - if table_bucket_entry.table_bucket != table_bucket { - return Err(TableCatalogStoreError::Invalid(format!( - "table bucket entry does not match migration target {table_bucket}" - ))); - } + validate_table_bucket_entry_object(&self.paths, &bucket_path, &table_bucket_entry)?; let mut namespaces = Vec::new(); let mut tables = Vec::new(); @@ -286,6 +362,7 @@ where "commit log changed while preparing durable strong snapshot: {commit_object}" ))); }; + validate_commit_log_entry_object(&self.paths, &commit_object, table_bucket, &table_entry.table_id, &commit)?; commits.push(StrongCommitSnapshotRecord { table_bucket: table_bucket.to_string(), table_id: table_entry.table_id.clone(), @@ -313,6 +390,13 @@ where "idempotency index changed while preparing durable strong snapshot: {idempotency_object}" ))); }; + validate_commit_idempotency_entry_object( + &self.paths, + &idempotency_object, + table_bucket, + &table_entry.table_id, + &commit, + )?; let lookup_key = commit.idempotency_key.clone().ok_or_else(|| { TableCatalogStoreError::Invalid(format!("idempotency index {idempotency_object} has no idempotency key")) })?; @@ -360,6 +444,22 @@ where fn validate_bucket_snapshot_for_migration(&self, snapshot: &StrongTableCatalogBucketSnapshot) -> TableCatalogStoreResult<()> { let table_bucket = &snapshot.table_bucket.table_bucket; + let active_table_identifiers = snapshot + .tables + .iter() + .filter(|table| table.state == TableCatalogEntryState::Active) + .map(|table| (&table.namespace, &table.table)) + .collect::>(); + if snapshot + .views + .iter() + .filter(|view| view.state == TableCatalogEntryState::Active) + .any(|view| active_table_identifiers.contains(&(&view.namespace, &view.view))) + { + return Err(TableCatalogStoreError::Conflict(format!( + "table bucket {table_bucket} contains an active table/view identifier collision" + ))); + } let tables_by_id = snapshot .tables .iter() @@ -498,6 +598,15 @@ where if self.get_table_bucket(table_bucket).await?.is_none() { return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); } + if let Some((global_fence, _)) = self + .read_entry::( + self.catalog_bucket(), + &self.paths.backing_migration_global_fence_path(), + ) + .await? + { + Self::validate_global_backing_migration_fence(&global_fence)?; + } let namespace_objects = self .backend @@ -511,6 +620,10 @@ where let mut recovery_required_count: usize = 0; let mut manual_review_count: usize = 0; let mut warehouse_prefix_owners = BTreeMap::::new(); + let mut table_ids = BTreeSet::::new(); + let mut duplicate_table_identity = false; + let mut active_table_identifiers = BTreeSet::<(String, String)>::new(); + let mut active_view_identifiers = BTreeSet::<(String, String)>::new(); for object in namespace_objects { if object.ends_with(NAMESPACE_ENTRY_FILE) { @@ -526,6 +639,9 @@ where continue; }; validate_view_entry_object(&self.paths, &object, &entry)?; + if entry.state == TableCatalogEntryState::Active { + active_view_identifiers.insert((entry.namespace.clone(), entry.view.clone())); + } view_count = view_count.saturating_add(1); continue; } @@ -538,7 +654,11 @@ where }; validate_table_entry_object(&self.paths, &object, &table)?; table_count = table_count.saturating_add(1); + if !table_ids.insert(table.table_id.clone()) { + duplicate_table_identity = true; + } if table.state == TableCatalogEntryState::Active { + active_table_identifiers.insert((table.namespace.clone(), table.table.clone())); let warehouse_prefix = table_warehouse_object_prefix(&table)?; warehouse_prefix_owners .entry(warehouse_prefix) @@ -548,17 +668,31 @@ where let recovery = self.table_commit_recovery_report_for_entry(&table, 0).await?; commit_log_count = commit_log_count.saturating_add(recovery.commits.len()); - idempotency_index_count = idempotency_index_count.saturating_add( - self.backend - .list_objects( - self.catalog_bucket(), - &self.paths.commit_idempotency_entries_prefix(table_bucket, &table.table_id), - ) + for idempotency_object in self + .backend + .list_objects( + self.catalog_bucket(), + &self.paths.commit_idempotency_entries_prefix(table_bucket, &table.table_id), + ) + .await? + .into_iter() + .filter(|object| object.ends_with(".json")) + { + let Some((commit, _)) = self + .read_entry::(self.catalog_bucket(), &idempotency_object) .await? - .into_iter() - .filter(|object| object.ends_with(".json")) - .count(), - ); + else { + continue; + }; + validate_commit_idempotency_entry_object( + &self.paths, + &idempotency_object, + table_bucket, + &table.table_id, + &commit, + )?; + idempotency_index_count = idempotency_index_count.saturating_add(1); + } recovery_required_count = recovery_required_count .saturating_add(recovery.staged_before_table_update_count) .saturating_add(recovery.finalization_required_count) @@ -568,6 +702,13 @@ where let warehouse_index_ready = self.warehouse_index_ready(table_bucket).await?; let duplicate_warehouse_prefix_count = warehouse_prefix_owners.values().filter(|count| **count > 1).count(); + let overlapping_warehouse_prefix = warehouse_prefix_owners + .keys() + .collect::>() + .windows(2) + .any(|window| warehouse_object_prefixes_overlap(window[0], window[1])); + let conflicting_warehouse_prefix = duplicate_warehouse_prefix_count > 0 || overlapping_warehouse_prefix; + let table_view_identifier_collision_count = active_table_identifiers.intersection(&active_view_identifiers).count(); let mut blockers = Vec::new(); let mut recommended_actions = Vec::new(); if recovery_required_count > 0 { @@ -583,12 +724,24 @@ where blockers.push(TableCatalogBackingMigrationBlocker::WarehouseIndexBackfillRequired); recommended_actions.push(TableCatalogBackingMigrationAction::BackfillWarehouseIndex); } - if duplicate_warehouse_prefix_count > 0 { + if conflicting_warehouse_prefix { blockers.push(TableCatalogBackingMigrationBlocker::DuplicateWarehousePrefix); recommended_actions.push(TableCatalogBackingMigrationAction::ReviewDuplicateWarehousePrefixes); } + if duplicate_table_identity { + blockers.push(TableCatalogBackingMigrationBlocker::DuplicateTableIdentity); + recommended_actions.push(TableCatalogBackingMigrationAction::ReviewDuplicateTableIdentities); + } + if table_view_identifier_collision_count > 0 { + blockers.push(TableCatalogBackingMigrationBlocker::TableViewIdentifierCollision); + recommended_actions.push(TableCatalogBackingMigrationAction::ReviewTableViewIdentifierCollisions); + } - let mut status = if manual_review_count > 0 || duplicate_warehouse_prefix_count > 0 { + let mut status = if manual_review_count > 0 + || conflicting_warehouse_prefix + || duplicate_table_identity + || table_view_identifier_collision_count > 0 + { TableCatalogBackingMigrationStatus::ManualReviewRequired } else if recovery_required_count > 0 || !warehouse_index_ready { TableCatalogBackingMigrationStatus::RecoveryRequired @@ -597,6 +750,14 @@ where }; let strong_store = StrongTableCatalogStore::new(self.backend.clone()); + let source_table_buckets = self.object_backed_table_buckets().await?; + let source_table_bucket_names = source_table_buckets.keys().cloned().collect::>(); + let target_table_buckets = strong_store.table_bucket_names().await?; + if !target_table_buckets.is_subset(&source_table_bucket_names) { + status = TableCatalogBackingMigrationStatus::ManualReviewRequired; + blockers.push(TableCatalogBackingMigrationBlocker::DurableStrongSnapshotChanged); + recommended_actions.push(TableCatalogBackingMigrationAction::ReviewDurableStrongSnapshot); + } let migration_fence = self.read_backing_migration_fence(table_bucket).await?.map(|(fence, _)| fence); let object_backed_writes_fenced = migration_fence.is_some(); if status == TableCatalogBackingMigrationStatus::ReadyToSnapshot @@ -633,7 +794,7 @@ where Ok(TableCatalogBackingMigrationDryRunReport { table_bucket: table_bucket.to_string(), source_kind: TableCatalogBackingKind::ObjectBacked, - target_kind: TableCatalogBackingKind::StrongKvWal, + target_kind: TableCatalogBackingKind::DurableStrongSnapshot, status, namespace_count, table_count, @@ -684,11 +845,17 @@ where let existing_fence = self.read_backing_migration_fence(table_bucket).await?; let strong_store = StrongTableCatalogStore::new(self.backend.clone()); if let Some((fence, _)) = existing_fence.as_ref() - && (fence.version != TABLE_CATALOG_MIGRATION_VERSION || fence.table_bucket != table_bucket) + && fence.status == TableCatalogBackingMigrationFenceStatus::Preparing + && !fence.target_bucket_existed + && Self::migration_target_snapshot_state(fence) == TableCatalogBackingMigrationTargetSnapshotState::Absent { - return Err(TableCatalogStoreError::Invalid(format!( - "invalid durable strong migration fence for table bucket {table_bucket}" - ))); + strong_store.restore_absent_migration_snapshot_baseline().await?; + } + let source_table_bucket_names = self.object_backed_table_buckets().await?.into_keys().collect::>(); + if !strong_store.table_bucket_names().await?.is_subset(&source_table_bucket_names) { + return Err(TableCatalogStoreError::Conflict( + "durable strong snapshot contains table buckets outside the object-backed catalog inventory".to_string(), + )); } if !self.warehouse_index_ready(table_bucket).await? { @@ -712,10 +879,16 @@ where } self.ensure_global_backing_migration_fence(&global_fence_path).await?; + let (target_fingerprint, target_snapshot_etag) = Self::observe_durable_strong_migration_target( + &strong_store, + table_bucket, + existing_fence.as_ref().map(|(fence, _)| fence), + ) + .await?; let (migration_id, target_bucket_existed) = if let Some((fence, _)) = existing_fence.as_ref() { (fence.migration_id.clone(), fence.target_bucket_existed) } else { - let target_bucket_existed = strong_store.bucket_snapshot_fingerprint(table_bucket).await?.is_some(); + let target_bucket_existed = target_fingerprint.is_some(); let fence = TableCatalogBackingMigrationFence { version: TABLE_CATALOG_MIGRATION_VERSION, table_bucket: table_bucket.to_string(), @@ -723,7 +896,7 @@ where status: TableCatalogBackingMigrationFenceStatus::Preparing, target_bucket_existed, source_fingerprint: None, - target_snapshot_etag: None, + target_snapshot_etag, }; self.write_entry(self.catalog_bucket(), &fence_path, &fence, TableCatalogPutPrecondition::IfAbsent) .await?; @@ -749,7 +922,7 @@ where Ok(TableCatalogBackingMigrationExecutionReport { table_bucket: table_bucket.to_string(), source_kind: TableCatalogBackingKind::ObjectBacked, - target_kind: TableCatalogBackingKind::StrongKvWal, + target_kind: TableCatalogBackingKind::DurableStrongSnapshot, status: if created { TableCatalogBackingMigrationExecutionStatus::SnapshotMaterialized } else { @@ -793,11 +966,6 @@ where }); }; self.ensure_global_backing_migration_fence(&global_fence_path).await?; - if fence.version != TABLE_CATALOG_MIGRATION_VERSION || fence.table_bucket != table_bucket { - return Err(TableCatalogStoreError::Invalid(format!( - "invalid durable strong migration fence for table bucket {table_bucket}" - ))); - } let mut source_guards = Vec::new(); let source = self @@ -813,12 +981,21 @@ where } let strong_store = StrongTableCatalogStore::new(self.backend.clone()); - if fence.status == TableCatalogBackingMigrationFenceStatus::Materialized - && strong_store.bucket_snapshot_fingerprint(table_bucket).await?.as_deref() != Some(&source_fingerprint) - { - return Err(TableCatalogStoreError::Conflict(format!( - "durable strong catalog state changed after materializing table bucket {table_bucket}" - ))); + let (target_fingerprint, target_snapshot_etag) = + Self::observe_durable_strong_migration_target(&strong_store, table_bucket, Some(&fence)).await?; + if fence.status == TableCatalogBackingMigrationFenceStatus::Materialized { + let target_matches_source = target_fingerprint.as_deref() == Some(&source_fingerprint); + let target_was_already_removed = !fence.target_bucket_existed && target_fingerprint.is_none(); + if !target_matches_source && !target_was_already_removed { + return Err(TableCatalogStoreError::Conflict(format!( + "durable strong catalog state changed after materializing table bucket {table_bucket}" + ))); + } + if target_matches_source && target_snapshot_etag != fence.target_snapshot_etag { + return Err(TableCatalogStoreError::Conflict( + "durable strong catalog snapshot advanced after materialization".to_string(), + )); + } } if !fence.target_bucket_existed { strong_store @@ -836,30 +1013,19 @@ where } async fn all_table_buckets_materialized(&self, strong_store: &StrongTableCatalogStore) -> TableCatalogStoreResult { - if self + let Some((global_fence, _)) = self .read_entry::( self.catalog_bucket(), &self.paths.backing_migration_global_fence_path(), ) .await? - .is_none() - { + else { return Ok(false); - } - let table_bucket_objects = self - .backend - .list_objects(self.catalog_bucket(), &self.paths.table_bucket_entries_prefix()) - .await?; - for table_bucket_object in table_bucket_objects - .iter() - .filter(|object| object.ends_with(TABLE_BUCKET_ENTRY_FILE)) - { - let Some((entry, _)) = self - .read_entry::(self.catalog_bucket(), table_bucket_object) - .await? - else { - return Ok(false); - }; + }; + Self::validate_global_backing_migration_fence(&global_fence)?; + let source_table_buckets = self.object_backed_table_buckets().await?; + let source_table_bucket_names = source_table_buckets.keys().cloned().collect::>(); + for entry in source_table_buckets.values() { let Some((fence, _)) = self.read_backing_migration_fence(&entry.table_bucket).await? else { return Ok(false); }; @@ -878,6 +1044,26 @@ where return Ok(false); } } - Ok(true) + Ok(strong_store.table_bucket_names().await? == source_table_bucket_names) + } + + async fn object_backed_table_buckets(&self) -> TableCatalogStoreResult> { + let mut table_buckets = BTreeMap::new(); + for object in self + .backend + .list_objects(self.catalog_bucket(), &self.paths.table_bucket_entries_prefix()) + .await? + .into_iter() + .filter(|object| object.ends_with(TABLE_BUCKET_ENTRY_FILE)) + { + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? else { + return Err(TableCatalogStoreError::Conflict(format!( + "table bucket changed while reading durable strong migration inventory: {object}" + ))); + }; + validate_table_bucket_entry_object(&self.paths, &object, &entry)?; + table_buckets.insert(entry.table_bucket.clone(), entry); + } + Ok(table_buckets) } } diff --git a/rustfs/src/table_catalog/store/mod.rs b/rustfs/src/table_catalog/store/mod.rs index fd62c12f6..3b7f54d1b 100644 --- a/rustfs/src/table_catalog/store/mod.rs +++ b/rustfs/src/table_catalog/store/mod.rs @@ -21,8 +21,39 @@ mod strong; use migration::table_catalog_backing_manifest; pub(crate) use object::ObjectTableCatalogStore; #[cfg(test)] -pub(super) use strong::StrongTableCatalogSnapshot; -pub(crate) use strong::StrongTableCatalogStore; +pub(super) use strong::{ + STRONG_TABLE_CATALOG_RELOAD_MAX_ATTEMPTS, STRONG_TABLE_CATALOG_SNAPSHOT_MAX_SIZE, StrongCommitSnapshotRecord, + StrongTableCatalogBucketSnapshot, StrongTableCatalogSnapshot, strong_snapshot_write_version, + table_catalog_bucket_snapshot_fingerprint, +}; +pub(crate) use strong::{StrongTableCatalogRuntime, StrongTableCatalogStore}; + +fn validate_table_bucket_entry(entry: &TableBucketEntry) -> TableCatalogStoreResult<()> { + validate_catalog_entry_version("table bucket", entry.version)?; + if entry.table_bucket.is_empty() { + return Err(TableCatalogStoreError::Invalid("table bucket name cannot be empty".to_string())); + } + if entry.catalog_type != TABLE_BUCKET_CATALOG_TYPE { + return Err(TableCatalogStoreError::Invalid("unsupported table bucket catalog type".to_string())); + } + Ok(()) +} + +fn validate_table_entry_version_and_id(entry: &TableEntry) -> TableCatalogStoreResult<()> { + validate_catalog_entry_version("table", entry.version)?; + if entry.table_id.is_empty() { + return Err(TableCatalogStoreError::Invalid("table id cannot be empty".to_string())); + } + Ok(()) +} + +fn validate_view_entry_version_and_id(entry: &ViewEntry) -> TableCatalogStoreResult<()> { + validate_catalog_entry_version("view", entry.version)?; + if entry.view_id.is_empty() { + return Err(TableCatalogStoreError::Invalid("view id cannot be empty".to_string())); + } + Ok(()) +} fn validate_namespace_entry_identity(entry: &NamespaceEntry) -> TableCatalogStoreResult { validate_catalog_entry_version("namespace", entry.version)?; @@ -406,8 +437,31 @@ pub(crate) enum TableCatalogPutPrecondition { IfMatch(String), } +pub(in crate::table_catalog) fn catalog_list_next_continuation( + seen: &mut BTreeSet, + is_truncated: bool, + next: Option, +) -> TableCatalogStoreResult> { + if !is_truncated { + return Ok(None); + } + let next = next.filter(|next| !next.is_empty()).ok_or_else(|| { + TableCatalogStoreError::Internal("truncated catalog object listing has no continuation token".to_string()) + })?; + if !seen.insert(next.clone()) { + return Err(TableCatalogStoreError::Internal( + "catalog object listing continuation token did not advance".to_string(), + )); + } + Ok(Some(next)) +} + #[async_trait::async_trait] pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static { + fn strong_catalog_runtime(&self) -> Option { + None + } + async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult>; async fn read_object_limited( @@ -845,10 +899,16 @@ where B: TableCatalogObjectBackend, { pub(crate) fn from_env(backend: B) -> TableCatalogStoreResult { - Ok(Self::new(backend, TableCatalogBackingMode::from_env()?)) + Ok(match TableCatalogBackingMode::from_env()? { + TableCatalogBackingMode::ObjectBacked => Self::ObjectBacked(ObjectTableCatalogStore::new(backend)), + TableCatalogBackingMode::DurableStrong => { + Self::DurableStrong(StrongTableCatalogStore::new_requiring_snapshot(backend)) + } + }) } - pub(crate) fn new(backend: B, mode: TableCatalogBackingMode) -> Self { + #[cfg(test)] + pub(crate) fn new_for_test(backend: B, mode: TableCatalogBackingMode) -> Self { match mode { TableCatalogBackingMode::ObjectBacked => Self::ObjectBacked(ObjectTableCatalogStore::new(backend)), TableCatalogBackingMode::DurableStrong => Self::DurableStrong(StrongTableCatalogStore::new(backend)), @@ -1352,12 +1412,14 @@ where pub(crate) struct EcStoreTableCatalogObjectBackend { store: Arc, + strong_runtime: StrongTableCatalogRuntime, } impl Clone for EcStoreTableCatalogObjectBackend { fn clone(&self) -> Self { Self { store: self.store.clone(), + strong_runtime: self.strong_runtime.clone(), } } } @@ -1366,8 +1428,8 @@ impl EcStoreTableCatalogObjectBackend where S: TableCatalogStorage, { - pub fn new(store: Arc) -> Self { - Self { store } + pub fn new_with_strong_runtime(store: Arc, strong_runtime: StrongTableCatalogRuntime) -> Self { + Self { store, strong_runtime } } } @@ -1378,6 +1440,10 @@ impl TableCatalogObjectBackend for EcStoreTableCatalogObjectBackend where S: TableCatalogStorage, { + fn strong_catalog_runtime(&self) -> Option { + Some(self.strong_runtime.clone()) + } + async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { self.read_object_with_options(bucket, object, ObjectOptions::default(), None) .await @@ -1537,6 +1603,7 @@ where async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult> { let mut continuation = None; + let mut seen_continuations = BTreeSet::new(); let mut objects = BTreeSet::new(); let max_keys = i32::try_from(TABLE_CATALOG_LIST_MAX_KEYS) .map_err(|_| TableCatalogStoreError::Internal("catalog list limit exceeds storage API range".to_string()))?; @@ -1553,14 +1620,10 @@ where objects.insert(object.name); } - if !result.is_truncated { - break; - } - - let Some(next) = result.next_continuation_token else { - break; + match catalog_list_next_continuation(&mut seen_continuations, result.is_truncated, result.next_continuation_token)? { + Some(next) => continuation = Some(next), + None => break, }; - continuation = Some(next); } Ok(objects.into_iter().collect()) @@ -1627,20 +1690,6 @@ where opts: ObjectOptions, max_size: Option, ) -> TableCatalogStoreResult> { - let info = match self.store.get_object_info(bucket, object, &opts).await { - Ok(info) => info, - Err(err) if is_missing_storage_error(&err) => return Ok(None), - Err(err) => return Err(storage_error_to_catalog("read catalog object info", err)), - }; - if let Some(max_size) = max_size { - let object_size = usize::try_from(info.size) - .map_err(|_| TableCatalogStoreError::Invalid(format!("catalog object {bucket}/{object} has an invalid size")))?; - if object_size > max_size { - return Err(TableCatalogStoreError::Invalid(format!( - "catalog object {bucket}/{object} exceeds the maximum size of {max_size} bytes" - ))); - } - } let mut reader = match self .store .get_object_reader(bucket, object, None, HeaderMap::new(), &opts) @@ -1650,6 +1699,17 @@ where Err(err) if is_missing_storage_error(&err) => return Ok(None), Err(err) => return Err(storage_error_to_catalog("read catalog object", err)), }; + if let Some(max_size) = max_size { + let object_size = usize::try_from(reader.object_info.size) + .map_err(|_| TableCatalogStoreError::Invalid(format!("catalog object {bucket}/{object} has an invalid size")))?; + if object_size > max_size { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog object {bucket}/{object} exceeds the maximum size of {max_size} bytes" + ))); + } + } + let etag = reader.object_info.etag.clone(); + let mod_time = reader.object_info.mod_time; let mut data = Vec::new(); if let Some(max_size) = max_size { let read_limit = u64::try_from(max_size.saturating_add(1)).unwrap_or(u64::MAX); @@ -1666,11 +1726,7 @@ where TableCatalogStoreError::Internal(format!("failed to read catalog object {bucket}/{object}: {err}")) })?; } - Ok(Some(TableCatalogObject { - data, - etag: info.etag, - mod_time: info.mod_time, - })) + Ok(Some(TableCatalogObject { data, etag, mod_time })) } async fn put_object_with_options( diff --git a/rustfs/src/table_catalog/store/object.rs b/rustfs/src/table_catalog/store/object.rs index 29a41bba7..b538ae85f 100644 --- a/rustfs/src/table_catalog/store/object.rs +++ b/rustfs/src/table_catalog/store/object.rs @@ -14,6 +14,20 @@ use super::*; +pub(super) fn validate_table_bucket_entry_object( + paths: &TableCatalogObjectPaths, + object: &str, + entry: &TableBucketEntry, +) -> TableCatalogStoreResult<()> { + validate_table_bucket_entry(entry)?; + if paths.table_bucket_entry_path(&entry.table_bucket) != object { + return Err(TableCatalogStoreError::Invalid( + "catalog table bucket entry identity does not match its object path".to_string(), + )); + } + Ok(()) +} + pub(super) fn validate_namespace_entry_object( paths: &TableCatalogObjectPaths, object: &str, @@ -33,7 +47,7 @@ pub(super) fn validate_table_entry_object( object: &str, entry: &TableEntry, ) -> TableCatalogStoreResult { - validate_catalog_entry_version("table", entry.version)?; + validate_table_entry_version_and_id(entry)?; let namespace = parse_namespace_for_store(&entry.namespace)?; let table = parse_table_for_store(&entry.table)?; if paths.table_entry_path(&entry.table_bucket, &namespace, &table) != object { @@ -49,7 +63,7 @@ pub(super) fn validate_view_entry_object( object: &str, entry: &ViewEntry, ) -> TableCatalogStoreResult { - validate_catalog_entry_version("view", entry.version)?; + validate_view_entry_version_and_id(entry)?; let namespace = parse_namespace_for_store(&entry.namespace)?; let view = parse_table_for_store(&entry.view)?; if paths.view_entry_path(&entry.table_bucket, &namespace, &view) != object { @@ -60,6 +74,107 @@ pub(super) fn validate_view_entry_object( Ok(namespace) } +pub(super) fn validate_commit_log_entry_object( + paths: &TableCatalogObjectPaths, + object: &str, + table_bucket: &str, + table_id: &str, + entry: &CommitLogEntry, +) -> TableCatalogStoreResult<()> { + validate_catalog_entry_version("commit log", entry.version)?; + if entry.table_id != table_id || paths.commit_log_entry_path(table_bucket, table_id, &entry.commit_id) != object { + return Err(TableCatalogStoreError::Invalid( + "catalog commit log identity does not match its object path".to_string(), + )); + } + Ok(()) +} + +pub(super) fn validate_commit_idempotency_entry_object( + paths: &TableCatalogObjectPaths, + object: &str, + table_bucket: &str, + table_id: &str, + entry: &CommitLogEntry, +) -> TableCatalogStoreResult<()> { + validate_catalog_entry_version("commit idempotency index", entry.version)?; + let idempotency_key = entry + .idempotency_key + .as_deref() + .ok_or_else(|| TableCatalogStoreError::Invalid("catalog commit idempotency index has no idempotency key".to_string()))?; + if entry.table_id != table_id || paths.commit_idempotency_entry_path(table_bucket, table_id, idempotency_key) != object { + return Err(TableCatalogStoreError::Invalid( + "catalog commit idempotency identity does not match its object path".to_string(), + )); + } + Ok(()) +} + +fn validate_external_catalog_bridge_entry_object( + paths: &TableCatalogObjectPaths, + object: &str, + entry: &ExternalCatalogBridgeEntry, +) -> TableCatalogStoreResult<()> { + validate_catalog_entry_version("external catalog bridge", entry.version)?; + let namespace = parse_namespace_for_store(&entry.namespace)?; + let table = parse_table_for_store(&entry.table)?; + if paths.external_catalog_bridge_path(&entry.table_bucket, &namespace, &table) != object { + return Err(TableCatalogStoreError::Invalid( + "external catalog bridge identity does not match its object path".to_string(), + )); + } + Ok(()) +} + +fn validate_table_maintenance_report_owner( + report: &TableMetadataMaintenanceReport, + table_bucket: &str, + namespace: &Namespace, + table: &IdentifierSegment, + table_id: &str, +) -> TableCatalogStoreResult<()> { + if report.job.job_id.is_empty() + || report.job.table_bucket != table_bucket + || report.job.namespace != namespace.public_name() + || report.job.table != table.as_str() + || report.job.table_id != table_id + { + return Err(TableCatalogStoreError::Invalid( + "table maintenance report identity does not match its catalog owner".to_string(), + )); + } + Ok(()) +} + +fn validate_table_warehouse_index_entry_object( + paths: &TableCatalogObjectPaths, + object: &str, + index: &TableWarehouseIndexEntry, +) -> TableCatalogStoreResult<()> { + validate_catalog_entry_version("warehouse index", index.version)?; + if paths.warehouse_index_entry_path(&index.table_bucket, &index.warehouse_object_prefix) != object { + return Err(TableCatalogStoreError::Invalid( + "catalog warehouse index identity does not match its object path".to_string(), + )); + } + Ok(()) +} + +fn table_warehouse_index_state_ready(state: &TableWarehouseIndexStateEntry, table_bucket: &str) -> TableCatalogStoreResult { + if state.version == 0 || state.version > TABLE_WAREHOUSE_INDEX_STATE_VERSION { + return Err(TableCatalogStoreError::Invalid(format!( + "unsupported warehouse index state version: {}", + state.version + ))); + } + if state.table_bucket != table_bucket { + return Err(TableCatalogStoreError::Invalid( + "warehouse index state identity does not match its object path".to_string(), + )); + } + Ok(state.version == TABLE_WAREHOUSE_INDEX_STATE_VERSION && state.state == TableCatalogEntryState::Active) +} + struct ActiveNamespaceEvidence { namespace: Namespace, explicit_entry: Option, @@ -236,6 +351,40 @@ where Ok(false) } + async fn has_namespace_resource_entry(&self, table_bucket: &str, namespace: &Namespace) -> TableCatalogStoreResult { + let scan_limit = NonZeroUsize::new(TABLE_CATALOG_LIST_MAX_KEYS) + .ok_or_else(|| TableCatalogStoreError::Internal("catalog object scan limit must be positive".to_string()))?; + for (prefix, entry_file) in [ + (self.paths.table_entries_prefix(table_bucket, namespace), TABLE_ENTRY_FILE), + (self.paths.view_entries_prefix(table_bucket, namespace), VIEW_ENTRY_FILE), + ] { + let mut cursor = None; + loop { + let page = self + .backend + .list_objects_page(self.catalog_bucket(), &prefix, cursor.as_deref(), scan_limit) + .await?; + let last_scanned = page.objects.last().cloned(); + if page.objects.iter().any(|object| object.ends_with(entry_file)) { + return Ok(true); + } + if !page.is_truncated { + break; + } + let next = last_scanned.ok_or_else(|| { + TableCatalogStoreError::Internal("catalog namespace resource scan made no progress".to_string()) + })?; + if cursor.as_deref().is_some_and(|cursor| next.as_str() <= cursor) { + return Err(TableCatalogStoreError::Internal( + "catalog namespace resource scan did not advance".to_string(), + )); + } + cursor = Some(next); + } + } + Ok(false) + } + async fn has_active_namespace_descendant(&self, table_bucket: &str, namespace: &Namespace) -> TableCatalogStoreResult { let parent = namespace.public_name(); let descendant_prefix = format!("{}{}/", self.paths.namespace_entries_prefix(table_bucket), namespace.storage_id()); @@ -542,23 +691,24 @@ where else { return Ok(false); }; - Ok(state.version == TABLE_WAREHOUSE_INDEX_STATE_VERSION - && state.table_bucket == table_bucket - && state.state == TableCatalogEntryState::Active) + table_warehouse_index_state_ready(&state, table_bucket) } async fn warehouse_index_entry_has_active_owner(&self, index: &TableWarehouseIndexEntry) -> TableCatalogStoreResult { if index.state != TableCatalogEntryState::Active { return Ok(false); } - let Some(table) = self.load_table(&index.table_bucket, &index.namespace, &index.table).await? else { + let Some(table) = self + .load_table_entry(&index.table_bucket, &index.namespace, &index.table) + .await? + else { return Ok(false); }; if table.state != TableCatalogEntryState::Active { return Ok(false); } let current_prefix = table_warehouse_object_prefix(&table)?; - Ok(current_prefix == index.warehouse_object_prefix) + Ok(table.table_id == index.table_id && current_prefix == index.warehouse_object_prefix) } async fn delete_warehouse_index_object( @@ -567,6 +717,7 @@ where index: &TableWarehouseIndexEntry, reason: &'static str, ) -> TableCatalogStoreResult { + validate_table_warehouse_index_entry_object(&self.paths, object, index)?; let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), object).await?; let Some((current, _)) = self .read_entry_unlocked::(self.catalog_bucket(), object) @@ -574,6 +725,7 @@ where else { return Ok(false); }; + validate_table_warehouse_index_entry_object(&self.paths, object, ¤t)?; if current != *index { tracing::warn!( table_bucket = %index.table_bucket, @@ -626,6 +778,7 @@ where else { return Ok(false); }; + validate_table_warehouse_index_entry_object(&self.paths, object, ¤t)?; if current != *stale { return Ok(false); } @@ -637,6 +790,27 @@ where async fn ensure_table_warehouse_prefix_available(&self, entry: &TableEntry) -> TableCatalogStoreResult<()> { let candidate = table_warehouse_index_entry(entry)?; + validate_table_entry_version_and_id(entry)?; + for existing in self.list_all_table_entries(&candidate.table_bucket).await? { + if existing.table_id == candidate.table_id { + if existing.namespace != candidate.namespace || existing.table != candidate.table { + return Err(TableCatalogStoreError::Conflict( + "table id is already registered in this table bucket".to_string(), + )); + } + continue; + } + if existing.state != TableCatalogEntryState::Active { + continue; + } + let existing_prefix = table_warehouse_object_prefix(&existing)?; + if warehouse_object_prefixes_overlap(&existing_prefix, &candidate.warehouse_object_prefix) { + return Err(TableCatalogStoreError::Conflict(format!( + "table warehouse location overlaps an active table: {}", + candidate.warehouse_object_prefix + ))); + } + } let state_object = self.paths.warehouse_index_state_path(&candidate.table_bucket); for object in self .backend @@ -652,6 +826,7 @@ where else { continue; }; + validate_table_warehouse_index_entry_object(&self.paths, &object, &existing)?; if existing.table_id == candidate.table_id || existing.state != TableCatalogEntryState::Active { continue; } @@ -675,9 +850,11 @@ where if let Some((existing, _)) = self .read_entry::(self.catalog_bucket(), &object) .await? - && existing == index { - return Ok(WarehouseIndexReservation::AlreadyReserved); + validate_table_warehouse_index_entry_object(&self.paths, &object, &existing)?; + if existing == index { + return Ok(WarehouseIndexReservation::AlreadyReserved); + } } self.ensure_table_warehouse_prefix_available(entry).await?; loop { @@ -693,6 +870,7 @@ where else { continue; }; + validate_table_warehouse_index_entry_object(&self.paths, &object, &existing)?; if existing == index { return Ok(WarehouseIndexReservation::AlreadyReserved); } @@ -746,10 +924,17 @@ where index_object: &str, index: TableWarehouseIndexEntry, ) -> TableCatalogStoreResult> { + validate_table_warehouse_index_entry_object(&self.paths, index_object, &index)?; if index.state != TableCatalogEntryState::Active { - return Ok(None); + return Err(TableCatalogStoreError::Internal(format!( + "warehouse index {index_object} for {}/{}/{} is inactive while the index is authoritative", + index.table_bucket, index.namespace, index.table + ))); } - let Some(table) = self.load_table(&index.table_bucket, &index.namespace, &index.table).await? else { + let Some(table) = self + .load_table_entry(&index.table_bucket, &index.namespace, &index.table) + .await? + else { return self .fail_closed_for_broken_warehouse_index(index_object, &index, "referenced table entry is missing") .await; @@ -785,9 +970,7 @@ where else { return Ok(false); }; - Ok(state.version == TABLE_WAREHOUSE_INDEX_STATE_VERSION - && state.table_bucket == table_bucket - && state.state == TableCatalogEntryState::Active) + table_warehouse_index_state_ready(&state, table_bucket) } async fn delete_created_table_warehouse_index( @@ -829,6 +1012,30 @@ where .map(|_| ()) } + async fn delete_owned_table_warehouse_index_for_drop(&self, entry: &TableEntry) -> TableCatalogStoreResult<()> { + let index = table_warehouse_index_entry(entry)?; + let object = self + .paths + .warehouse_index_entry_path(&index.table_bucket, &index.warehouse_object_prefix); + validate_table_warehouse_index_entry_object(&self.paths, &object, &index)?; + let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?; + let Some((current, _)) = self + .read_entry_unlocked::(self.catalog_bucket(), &object) + .await? + else { + return Ok(()); + }; + validate_table_warehouse_index_entry_object(&self.paths, &object, ¤t)?; + if current != index { + return Err(TableCatalogStoreError::Conflict(format!( + "table warehouse index owner changed before drop: {}", + index.warehouse_object_prefix + ))); + } + self.delete_warehouse_index_object_unlocked(&object, &index, "table warehouse index owner dropped") + .await + } + async fn delete_table_warehouse_index_if_changed(&self, current: &TableEntry, next: &TableEntry) { let Ok(current_index) = table_warehouse_index_entry(current) else { return; @@ -857,6 +1064,7 @@ where table_bucket: &str, object: &str, ) -> TableCatalogStoreResult> { + let mut matched: Option = None; for warehouse_object_prefix in warehouse_index_candidate_prefixes(object) { let index_object = self.paths.warehouse_index_entry_path(table_bucket, warehouse_object_prefix); let Some((index, _)) = self @@ -874,10 +1082,16 @@ where .resolve_table_data_plane_resource_from_index_entry(&index_object, index) .await? { - return Ok(Some(resource)); + if let Some(current) = matched.as_ref() { + return Err(TableCatalogStoreError::Invalid(format!( + "object {object} matches overlapping active table warehouse indexes {} and {}", + current.warehouse_object_prefix, resource.warehouse_object_prefix + ))); + } + matched = Some(resource); } } - Ok(None) + Ok(matched) } pub(in crate::table_catalog) async fn backfill_active_table_warehouse_index( @@ -903,21 +1117,24 @@ where &self, table_bucket: &str, ) -> TableCatalogStoreResult<()> { + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; let state_object = self.paths.warehouse_index_state_path(table_bucket); let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &state_object).await?; if self.read_warehouse_index_state_unlocked(table_bucket).await? { return Ok(()); } - let tables = self.list_all_tables(table_bucket).await?; - let mut active_prefixes = tables - .iter() - .filter(|table| table.state == TableCatalogEntryState::Active) - .filter_map(|table| { - table_warehouse_object_prefix(table) - .ok() - .map(|prefix| (prefix, table.table_id.as_str())) - }) - .collect::>(); + let tables = self.list_all_table_entries(table_bucket).await?; + let mut table_ids = BTreeSet::new(); + if let Some(table) = tables.iter().find(|table| !table_ids.insert(table.table_id.as_str())) { + return Err(TableCatalogStoreError::Conflict(format!( + "table id {} is registered by multiple tables in table bucket {table_bucket}", + table.table_id + ))); + } + let mut active_prefixes = Vec::new(); + for table in tables.iter().filter(|table| table.state == TableCatalogEntryState::Active) { + active_prefixes.push((table_warehouse_object_prefix(table)?, table.table_id.as_str())); + } active_prefixes.sort_unstable_by(|left, right| left.0.cmp(&right.0)); if let Some(window) = active_prefixes .windows(2) @@ -932,32 +1149,21 @@ where if table.state != TableCatalogEntryState::Active { continue; } - if let Err(err) = self - .backfill_active_table_warehouse_index(&table.table_bucket, &table.namespace, &table.table) - .await - { - if matches!(&err, TableCatalogStoreError::Invalid(_)) { - tracing::warn!( - table_bucket = %table.table_bucket, - namespace = %table.namespace, - table = %table.table, - table_id = %table.table_id, - error = %err, - "skipping invalid table warehouse location while backfilling warehouse index" - ); - continue; - } - return Err(err); - } + self.backfill_active_table_warehouse_index(&table.table_bucket, &table.namespace, &table.table) + .await?; } self.write_warehouse_index_state_unlocked(table_bucket).await } async fn require_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { - if self.get_table_bucket(table_bucket).await?.is_none() { - return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); + if self + .get_table_bucket(table_bucket) + .await? + .is_some_and(|entry| entry.state == TableCatalogEntryState::Active) + { + return Ok(()); } - Ok(()) + Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))) } async fn read_table_with_etag( @@ -970,12 +1176,46 @@ where let Some((entry, etag)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { return Ok(None); }; + validate_table_entry_object(&self.paths, &table_path, &entry)?; let Some(etag) = etag else { return Err(TableCatalogStoreError::Internal(format!("catalog table entry has no etag: {table_path}"))); }; Ok(Some((entry, etag))) } + async fn load_table_entry( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + ) -> TableCatalogStoreResult> { + let namespace = parse_namespace_for_store(namespace)?; + let table = parse_table_for_store(table)?; + self.read_table_with_etag(table_bucket, &namespace, &table) + .await + .map(|entry| entry.map(|(table, _)| table)) + } + + async fn list_all_table_entries(&self, table_bucket: &str) -> TableCatalogStoreResult> { + let mut entries = Vec::new(); + for object in self + .backend + .list_objects(self.catalog_bucket(), &self.paths.namespace_entries_prefix(table_bucket)) + .await? + { + if !object.ends_with(TABLE_ENTRY_FILE) { + continue; + } + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? else { + continue; + }; + validate_table_entry_object(&self.paths, &object, &entry)?; + entries.push(entry); + } + entries.sort_by(|left, right| (&left.namespace, &left.table).cmp(&(&right.namespace, &right.table))); + Ok(entries) + } + async fn read_table_with_etag_unlocked( &self, table_bucket: &str, @@ -989,6 +1229,7 @@ where else { return Ok(None); }; + validate_table_entry_object(&self.paths, &table_path, &entry)?; let Some(etag) = etag else { return Err(TableCatalogStoreError::Internal(format!("catalog table entry has no etag: {table_path}"))); }; @@ -1008,6 +1249,7 @@ where else { return Ok(None); }; + validate_view_entry_object(&self.paths, &view_path, &entry)?; let Some(etag) = etag else { return Err(TableCatalogStoreError::Internal(format!("catalog view entry has no etag: {view_path}"))); }; @@ -1030,7 +1272,7 @@ where precondition: TableCatalogPutPrecondition, publication: &(dyn TableCommitPublication + Sync), ) -> TableCatalogStoreResult<()> { - validate_catalog_entry_version("table", entry.version)?; + validate_table_entry_version_and_id(&entry)?; let namespace = parse_namespace_for_store(&entry.namespace)?; let table = parse_table_for_store(&entry.table)?; validate_table_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?; @@ -1052,6 +1294,17 @@ where .await?; let table_path = self.paths.table_entry_path(&entry.table_bucket, &namespace, &table); let _table_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; + let view_path = self.paths.view_entry_path(&entry.table_bucket, &namespace, &table); + if self + .read_entry_unlocked::(self.catalog_bucket(), &view_path) + .await? + .is_some() + { + return Err(TableCatalogStoreError::Conflict(format!( + "catalog object already exists: view {}/{}/{}", + entry.table_bucket, entry.namespace, entry.table + ))); + } // Preserve catalog -> publication -> object lock order across rolling upgrades. publication .prepare(&entry.table_bucket, &entry.namespace, &entry.table) @@ -1061,6 +1314,7 @@ where "table registration requires a table publication fence".to_string(), )); } + self.ensure_table_warehouse_prefix_available(&entry).await?; let reservation = self.reserve_table_warehouse_index(&entry).await?; let result = self .write_entry_unlocked(self.catalog_bucket(), &table_path, &entry, precondition) @@ -1073,7 +1327,7 @@ where } async fn write_view_entry(&self, entry: ViewEntry, precondition: TableCatalogPutPrecondition) -> TableCatalogStoreResult<()> { - validate_catalog_entry_version("view", entry.version)?; + validate_view_entry_version_and_id(&entry)?; self.require_table_bucket(&entry.table_bucket).await?; let namespace = parse_namespace_for_store(&entry.namespace)?; let view = parse_table_for_store(&entry.view)?; @@ -1088,6 +1342,17 @@ where .await?; let view_path = self.paths.view_entry_path(&entry.table_bucket, &namespace, &view); let _view_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &view_path).await?; + let table_path = self.paths.table_entry_path(&entry.table_bucket, &namespace, &view); + if self + .read_entry_unlocked::(self.catalog_bucket(), &table_path) + .await? + .is_some() + { + return Err(TableCatalogStoreError::Conflict(format!( + "catalog object already exists: table {}/{}/{}", + entry.table_bucket, entry.namespace, entry.view + ))); + } self.write_entry_unlocked(self.catalog_bucket(), &view_path, &entry, precondition) .await } @@ -1109,9 +1374,14 @@ where ))); } let bridge_path = self.paths.external_catalog_bridge_path(table_bucket, &namespace, &table); - self.read_entry::(self.catalog_bucket(), &bridge_path) - .await - .map(|entry| entry.map(|(bridge, _)| bridge)) + let Some((entry, _)) = self + .read_entry::(self.catalog_bucket(), &bridge_path) + .await? + else { + return Ok(None); + }; + validate_external_catalog_bridge_entry_object(&self.paths, &bridge_path, &entry)?; + Ok(Some(entry)) } pub(crate) async fn put_external_catalog_bridge( @@ -1120,7 +1390,7 @@ where ) -> TableCatalogStoreResult { validate_catalog_entry_version("external catalog bridge", entry.version)?; self.require_table_bucket(&entry.table_bucket).await?; - self.ensure_object_backed_writes_allowed(&entry.table_bucket).await?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; let namespace = parse_namespace_for_store(&entry.namespace)?; let table = parse_table_for_store(&entry.table)?; if self.get_namespace(&entry.table_bucket, &entry.namespace).await?.is_none() { @@ -1132,6 +1402,7 @@ where let bridge_path = self .paths .external_catalog_bridge_path(&entry.table_bucket, &namespace, &table); + validate_external_catalog_bridge_entry_object(&self.paths, &bridge_path, &entry)?; self.write_entry(self.catalog_bucket(), &bridge_path, &entry, TableCatalogPutPrecondition::Any) .await?; Ok(entry) @@ -1143,6 +1414,36 @@ where .map(|entry| entry.map(|(commit, _)| commit)) } + async fn read_commit_log_entry( + &self, + table_bucket: &str, + table_id: &str, + commit_id: &str, + ) -> TableCatalogStoreResult> { + let object = self.paths.commit_log_entry_path(table_bucket, table_id, commit_id); + let Some(commit) = self.read_commit_by_path(&object).await? else { + return Ok(None); + }; + validate_commit_log_entry_object(&self.paths, &object, table_bucket, table_id, &commit)?; + Ok(Some(commit)) + } + + async fn read_commit_idempotency_entry( + &self, + table_bucket: &str, + table_id: &str, + idempotency_key: &str, + ) -> TableCatalogStoreResult> { + let object = self + .paths + .commit_idempotency_entry_path(table_bucket, table_id, idempotency_key); + let Some(commit) = self.read_commit_by_path(&object).await? else { + return Ok(None); + }; + validate_commit_idempotency_entry_object(&self.paths, &object, table_bucket, table_id, &commit)?; + Ok(Some(commit)) + } + async fn read_table_commit_logs(&self, entry: &TableEntry) -> TableCatalogStoreResult> { let commit_prefix = self.paths.commit_log_entries_prefix(&entry.table_bucket, &entry.table_id); let mut commits = Vec::new(); @@ -1151,6 +1452,7 @@ where continue; } if let Some(commit_log) = self.read_commit_by_path(&object).await? { + validate_commit_log_entry_object(&self.paths, &object, &entry.table_bucket, &entry.table_id, &commit_log)?; commits.push((object, commit_log)); } } @@ -1183,10 +1485,8 @@ where for (_, commit_log) in &commit_logs_with_paths { let idempotency_commit = match commit_log.idempotency_key.as_deref() { Some(idempotency_key) => { - let idempotency_path = - self.paths - .commit_idempotency_entry_path(&entry.table_bucket, &entry.table_id, idempotency_key); - self.read_commit_by_path(&idempotency_path).await? + self.read_commit_idempotency_entry(&entry.table_bucket, &entry.table_id, idempotency_key) + .await? } None => None, }; @@ -1241,8 +1541,7 @@ where ) -> TableCatalogStoreResult { let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + let Some((entry, _)) = self.read_table_with_etag(table_bucket, &namespace, &table).await? else { return Err(TableCatalogStoreError::NotFound(format!( "table {}/{}/{}", table_bucket, @@ -1277,12 +1576,16 @@ where let history = TableCommitHistoryIndex::new(&entry, commit_logs_with_paths.iter().map(|(_, commit_log)| commit_log)); let mut finalized_count = 0; for (commit_path, commit_log) in &commit_logs_with_paths { - let idempotency_path = commit_log.idempotency_key.as_deref().map(|idempotency_key| { + let idempotency_key = commit_log.idempotency_key.as_deref(); + let idempotency_path = idempotency_key.map(|idempotency_key| { self.paths .commit_idempotency_entry_path(table_bucket, &entry.table_id, idempotency_key) }); - let idempotency_commit = match idempotency_path.as_deref() { - Some(idempotency_path) => self.read_commit_by_path(idempotency_path).await?, + let idempotency_commit = match idempotency_key { + Some(idempotency_key) => { + self.read_commit_idempotency_entry(table_bucket, &entry.table_id, idempotency_key) + .await? + } None => None, }; let recovery_entry = table_commit_recovery_entry( @@ -1314,8 +1617,7 @@ where ) -> TableCatalogStoreResult { let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + let Some((entry, _)) = self.read_table_with_etag(table_bucket, &namespace, &table).await? else { return Err(TableCatalogStoreError::NotFound(format!( "table {}/{}/{}", table_bucket, @@ -1343,7 +1645,7 @@ where ) -> TableCatalogStoreResult { validate_table_maintenance_config(&config)?; self.require_table_bucket(table_bucket).await?; - self.ensure_object_backed_writes_allowed(table_bucket).await?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; let config_path = self.paths.table_bucket_maintenance_config_path(table_bucket); self.write_entry(self.catalog_bucket(), &config_path, &config, TableCatalogPutPrecondition::Any) .await?; @@ -1358,8 +1660,7 @@ where ) -> TableCatalogStoreResult { let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + let Some((entry, _)) = self.read_table_with_etag(table_bucket, &namespace, &table).await? else { return Err(TableCatalogStoreError::NotFound(format!( "table {}/{}/{}", table_bucket, @@ -1419,11 +1720,10 @@ where config: TableMaintenanceConfig, ) -> TableCatalogStoreResult { validate_table_maintenance_config(&config)?; - self.ensure_object_backed_writes_allowed(table_bucket).await?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + let Some((entry, _)) = self.read_table_with_etag(table_bucket, &namespace, &table).await? else { return Err(TableCatalogStoreError::NotFound(format!( "table {}/{}/{}", table_bucket, @@ -1443,11 +1743,32 @@ where pub(crate) async fn put_table_metadata_maintenance_report( &self, report: &TableMetadataMaintenanceReport, + ) -> TableCatalogStoreResult<()> { + let _migration_guard = self + .acquire_object_backed_catalog_write_permit(&report.job.table_bucket) + .await?; + self.put_table_metadata_maintenance_report_unfenced(report).await + } + + async fn put_table_metadata_maintenance_report_unfenced( + &self, + report: &TableMetadataMaintenanceReport, ) -> TableCatalogStoreResult<()> { let report = table_maintenance_report_with_recommended_actions(report.clone()); - self.ensure_object_backed_writes_allowed(&report.job.table_bucket).await?; let namespace = parse_namespace_for_store(&report.job.namespace)?; let table = parse_table_for_store(&report.job.table)?; + let Some((entry, _)) = self + .read_table_with_etag(&report.job.table_bucket, &namespace, &table) + .await? + else { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + report.job.table_bucket, + namespace.public_name(), + table.as_str() + ))); + }; + validate_table_maintenance_report_owner(&report, &report.job.table_bucket, &namespace, &table, &entry.table_id)?; let job_path = self.paths.table_maintenance_job_path( &report.job.table_bucket, &namespace, @@ -1478,8 +1799,7 @@ where ) -> TableCatalogStoreResult> { let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + let Some((entry, _)) = self.read_table_with_etag(table_bucket, &namespace, &table).await? else { return Err(TableCatalogStoreError::NotFound(format!( "table {}/{}/{}", table_bucket, @@ -1501,9 +1821,14 @@ where job_id: &str, ) -> TableCatalogStoreResult> { let job_path = self.table_metadata_maintenance_report_path(table_bucket, namespace, table, table_id, job_id); - self.read_entry_unlocked::(self.catalog_bucket(), &job_path) - .await - .map(|entry| entry.map(|(report, _)| table_maintenance_report_with_recommended_actions(report))) + let Some((report, _)) = self + .read_entry_unlocked::(self.catalog_bucket(), &job_path) + .await? + else { + return Ok(None); + }; + validate_table_maintenance_report_owner(&report, table_bucket, namespace, table, table_id)?; + Ok(Some(table_maintenance_report_with_recommended_actions(report))) } fn table_metadata_maintenance_report_path( @@ -1548,8 +1873,7 @@ where ) -> TableCatalogStoreResult { let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + let Some((entry, _)) = self.read_table_with_etag(table_bucket, &namespace, &table).await? else { return Err(TableCatalogStoreError::NotFound(format!( "table {}/{}/{}", table_bucket, @@ -1651,6 +1975,7 @@ where ) -> TableCatalogStoreResult { let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); let namespace_name = namespace.public_name(); let table_name = table.as_str().to_string(); @@ -1772,7 +2097,7 @@ where before_status, before_quarantined_object_count, ); - self.put_table_metadata_maintenance_report(&report).await?; + self.put_table_metadata_maintenance_report_unfenced(&report).await?; report } TableMaintenanceSchedulerPreflight::Complete(report) => *report, @@ -1810,6 +2135,7 @@ where )) })? } else { + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { return Err(TableCatalogStoreError::NotFound(format!( @@ -1884,7 +2210,7 @@ where before_status, before_quarantined_object_count, ); - self.put_table_metadata_maintenance_report(&report).await?; + self.put_table_metadata_maintenance_report_unfenced(&report).await?; report }; @@ -1917,6 +2243,16 @@ where .read_entry::(self.catalog_bucket(), &object) .await? { + validate_table_maintenance_report_owner(&report, table_bucket, namespace, table, table_id)?; + if self + .paths + .table_maintenance_job_path(table_bucket, namespace, table, table_id, &report.job.job_id) + != object + { + return Err(TableCatalogStoreError::Invalid( + "table maintenance report identity does not match its object path".to_string(), + )); + } reports.push(table_maintenance_report_with_recommended_actions(report)); } } @@ -2030,6 +2366,7 @@ where ) -> TableCatalogStoreResult { let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); let namespace_name = namespace.public_name(); let table_name = table.as_str().to_string(); @@ -2158,7 +2495,7 @@ where before_status, before_quarantined_object_count, ); - self.put_table_metadata_maintenance_report(&report).await?; + self.put_table_metadata_maintenance_report_unfenced(&report).await?; let delete = matches!(report.job.operation, TableMetadataMaintenanceOperation::Delete); (report, effective, delete) @@ -2281,6 +2618,9 @@ where ) -> TableCatalogStoreResult { let namespace = parse_namespace_for_store(heartbeat.namespace)?; let table = parse_table_for_store(heartbeat.table)?; + let _migration_guard = self + .acquire_object_backed_catalog_write_permit(heartbeat.table_bucket) + .await?; let table_path = self.paths.table_entry_path(heartbeat.table_bucket, &namespace, &table); let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; let Some((entry, _)) = self @@ -2337,7 +2677,7 @@ where Some(TableMetadataMaintenanceJobStatus::Running), before_quarantined_object_count, ); - self.put_table_metadata_maintenance_report(&report).await?; + self.put_table_metadata_maintenance_report_unfenced(&report).await?; Ok(report) } @@ -2363,7 +2703,7 @@ where before_status, before_quarantined_object_count, ); - self.put_table_metadata_maintenance_report(&report).await?; + self.put_table_metadata_maintenance_report_unfenced(&report).await?; Ok(report) } @@ -2440,7 +2780,7 @@ where None, None, ); - self.put_table_metadata_maintenance_report(&report).await?; + self.put_table_metadata_maintenance_report_unfenced(&report).await?; Ok(report) } @@ -2517,7 +2857,7 @@ where None, None, ); - self.put_table_metadata_maintenance_report(&report).await?; + self.put_table_metadata_maintenance_report_unfenced(&report).await?; Ok(report) } @@ -2546,8 +2886,7 @@ where validate_table_snapshot_expiration_config(&config)?; let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + let Some((entry, _)) = self.read_table_with_etag(table_bucket, &namespace, &table).await? else { return Err(TableCatalogStoreError::NotFound(format!( "table {}/{}/{}", table_bucket, @@ -2590,8 +2929,7 @@ where validate_table_compaction_planning_config(&config)?; let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + let Some((entry, _)) = self.read_table_with_etag(table_bucket, &namespace, &table).await? else { return Err(TableCatalogStoreError::NotFound(format!( "table {}/{}/{}", table_bucket, @@ -2643,8 +2981,7 @@ where validate_table_compaction_planning_config(&config)?; let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + let Some((entry, _)) = self.read_table_with_etag(table_bucket, &namespace, &table).await? else { return Err(TableCatalogStoreError::NotFound(format!( "table {}/{}/{}", table_bucket, @@ -2826,14 +3163,17 @@ where let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; + let table_bucket_path = self.paths.table_bucket_entry_path(table_bucket); let Some((table_bucket_entry, _)) = self - .read_entry::(self.catalog_bucket(), &self.paths.table_bucket_entry_path(table_bucket)) + .read_entry::(self.catalog_bucket(), &table_bucket_path) .await? else { return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); }; + validate_table_bucket_entry_object(&self.paths, &table_bucket_path, &table_bucket_entry)?; + let namespace_path = self.paths.namespace_entry_path(table_bucket, &namespace); let Some((namespace_entry, _)) = self - .read_entry::(self.catalog_bucket(), &self.paths.namespace_entry_path(table_bucket, &namespace)) + .read_entry::(self.catalog_bucket(), &namespace_path) .await? else { return Err(TableCatalogStoreError::NotFound(format!( @@ -2842,10 +3182,8 @@ where namespace.public_name() ))); }; - let Some((table_entry, _)) = self - .read_entry::(self.catalog_bucket(), &self.paths.table_entry_path(table_bucket, &namespace, &table)) - .await? - else { + validate_namespace_entry_object(&self.paths, &namespace_path, &namespace_entry)?; + let Some((table_entry, _)) = self.read_table_with_etag(table_bucket, &namespace, &table).await? else { return Err(TableCatalogStoreError::NotFound(format!( "table {}/{}/{}", table_bucket, @@ -2958,8 +3296,7 @@ where ) -> TableCatalogStoreResult { let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; - let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &table_path).await? else { + let Some((entry, _)) = self.read_table_with_etag(table_bucket, &namespace, &table).await? else { return Err(TableCatalogStoreError::NotFound(format!( "table {}/{}/{}", table_bucket, @@ -3225,6 +3562,7 @@ where worker_id: Option, effective: TableMaintenanceEffectiveConfig, ) -> TableCatalogStoreResult { + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; let mut report = self .plan_table_metadata_maintenance(table_bucket, namespace, table, effective.config.retain_recent_metadata_files) .await?; @@ -3259,7 +3597,7 @@ where Some(TableMetadataMaintenanceJobStatus::Successful), Some(0), ); - self.put_table_metadata_maintenance_report(&report).await?; + self.put_table_metadata_maintenance_report_unfenced(&report).await?; self.finish_table_metadata_maintenance_run(table_bucket, namespace, table, delete, &effective, report) .await @@ -3292,14 +3630,14 @@ where before_status, before_quarantined_object_count, ); - self.put_table_metadata_maintenance_report(&report).await?; + self.put_table_metadata_maintenance_report_unfenced(&report).await?; return Ok(report); } if delete { let running_report = report.clone(); let mut deleted = match self - .delete_table_metadata_maintenance_report(table_bucket, namespace, table, report) + .delete_table_metadata_maintenance_report_unfenced(table_bucket, namespace, table, report) .await { Ok(report) => report, @@ -3323,7 +3661,7 @@ where before_status, before_quarantined_object_count, ); - self.put_table_metadata_maintenance_report(&failed).await?; + self.put_table_metadata_maintenance_report_unfenced(&failed).await?; return Err(err); } }; @@ -3341,7 +3679,7 @@ where before_status, before_quarantined_object_count, ); - self.put_table_metadata_maintenance_report(&deleted).await?; + self.put_table_metadata_maintenance_report_unfenced(&deleted).await?; return Ok(deleted); } @@ -3360,7 +3698,7 @@ where before_status, before_quarantined_object_count, ); - self.put_table_metadata_maintenance_report(&report).await?; + self.put_table_metadata_maintenance_report_unfenced(&report).await?; Ok(report) } @@ -3370,6 +3708,18 @@ where namespace: &str, table: &str, report: TableMetadataMaintenanceReport, + ) -> TableCatalogStoreResult { + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; + self.delete_table_metadata_maintenance_report_unfenced(table_bucket, namespace, table, report) + .await + } + + async fn delete_table_metadata_maintenance_report_unfenced( + &self, + table_bucket: &str, + namespace: &str, + table: &str, + report: TableMetadataMaintenanceReport, ) -> TableCatalogStoreResult { let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; @@ -3555,19 +3905,16 @@ where B: TableCatalogObjectBackend, { async fn get_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult> { - self.read_entry::(self.catalog_bucket(), &self.paths.table_bucket_entry_path(table_bucket)) - .await - .map(|entry| entry.map(|(bucket, _)| bucket)) + let object = self.paths.table_bucket_entry_path(table_bucket); + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? else { + return Ok(None); + }; + validate_table_bucket_entry_object(&self.paths, &object, &entry)?; + Ok(Some(entry)) } async fn put_table_bucket(&self, entry: TableBucketEntry) -> TableCatalogStoreResult<()> { - validate_catalog_entry_version("table bucket", entry.version)?; - if entry.table_bucket.is_empty() { - return Err(TableCatalogStoreError::Invalid("table bucket name cannot be empty".to_string())); - } - if entry.catalog_type != TABLE_BUCKET_CATALOG_TYPE { - return Err(TableCatalogStoreError::Invalid("unsupported table bucket catalog type".to_string())); - } + validate_table_bucket_entry(&entry)?; let _registry_guard = self.acquire_table_bucket_registry_write_permit().await?; let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; let object = self.paths.table_bucket_entry_path(&entry.table_bucket); @@ -3722,7 +4069,7 @@ where namespace.public_name() ))); } - if self.has_active_namespace_object(table_bucket, &namespace).await? { + if self.has_namespace_resource_entry(table_bucket, &namespace).await? { return Err(TableCatalogStoreError::Conflict(format!( "namespace {table_bucket}/{} is not empty", namespace.public_name() @@ -3746,20 +4093,6 @@ where namespace.public_name() ))); } - if !self.list_tables(table_bucket, &namespace.public_name()).await?.is_empty() { - return Err(TableCatalogStoreError::Conflict(format!( - "namespace {}/{} is not empty", - table_bucket, - namespace.public_name() - ))); - } - if !self.list_views(table_bucket, &namespace.public_name()).await?.is_empty() { - return Err(TableCatalogStoreError::Conflict(format!( - "namespace {}/{} is not empty", - table_bucket, - namespace.public_name() - ))); - } self.backend .delete_object_unlocked(self.catalog_bucket(), &namespace_path) .await @@ -3793,7 +4126,11 @@ where if !object.ends_with(TABLE_ENTRY_FILE) { continue; } - if let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? { + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? else { + continue; + }; + validate_table_entry_object(&self.paths, &object, &entry)?; + if entry.state == TableCatalogEntryState::Active { entries.push(entry); } } @@ -3802,23 +4139,12 @@ where } async fn list_all_tables(&self, table_bucket: &str) -> TableCatalogStoreResult> { - let mut entries = Vec::new(); - for object in self - .backend - .list_objects(self.catalog_bucket(), &self.paths.namespace_entries_prefix(table_bucket)) - .await? - { - if !object.ends_with(TABLE_ENTRY_FILE) { - continue; - } - let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? else { - continue; - }; - validate_table_entry_object(&self.paths, &object, &entry)?; - entries.push(entry); - } - entries.sort_by(|left, right| (&left.namespace, &left.table).cmp(&(&right.namespace, &right.table))); - Ok(entries) + self.list_all_table_entries(table_bucket).await.map(|entries| { + entries + .into_iter() + .filter(|entry| entry.state == TableCatalogEntryState::Active) + .collect() + }) } async fn list_tables_page( @@ -3834,18 +4160,16 @@ where TABLE_ENTRY_FILE, cursor, limit, - |_: &TableEntry| true, - |_, _: &TableEntry| Ok(()), + |entry: &TableEntry| entry.state == TableCatalogEntryState::Active, + |object, entry: &TableEntry| validate_table_entry_object(&self.paths, object, entry).map(|_| ()), ) .await } async fn load_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult> { - let namespace = parse_namespace_for_store(namespace)?; - let table = parse_table_for_store(table)?; - self.read_entry::(self.catalog_bucket(), &self.paths.table_entry_path(table_bucket, &namespace, &table)) + self.load_table_entry(table_bucket, namespace, table) .await - .map(|entry| entry.map(|(table, _)| table)) + .map(|entry| entry.filter(|table| table.state == TableCatalogEntryState::Active)) } async fn resolve_table_data_plane_resource( @@ -3857,20 +4181,35 @@ where return Ok(None); } let Some(table_bucket_entry) = self.get_table_bucket(table_bucket).await? else { - return Ok(None); + return Err(TableCatalogStoreError::Internal(format!( + "object-backed catalog has no entry for table-enabled bucket {table_bucket}" + ))); }; if table_bucket_entry.state != TableCatalogEntryState::Active { - return Ok(None); + return Err(TableCatalogStoreError::Internal(format!( + "table-enabled bucket {table_bucket} has an inactive object-backed catalog entry" + ))); } if self.warehouse_index_ready(table_bucket).await? { - return self.resolve_table_data_plane_resource_from_index(table_bucket, object).await; + return match self + .resolve_table_data_plane_resource_from_index(table_bucket, object) + .await? + { + Some(resource) => Ok(Some(resource)), + None => scan_table_data_plane_resource_for_object(self, table_bucket, object).await, + }; } match self.backfill_table_warehouse_index(table_bucket).await { - Ok(()) => self.resolve_table_data_plane_resource_from_index(table_bucket, object).await, - Err(err @ TableCatalogStoreError::Conflict(_)) => Err(err), - Err(err) => { + Ok(()) => match self + .resolve_table_data_plane_resource_from_index(table_bucket, object) + .await? + { + Some(resource) => Ok(Some(resource)), + None => scan_table_data_plane_resource_for_object(self, table_bucket, object).await, + }, + Err(err @ TableCatalogStoreError::Internal(_)) => { tracing::warn!( table_bucket = %table_bucket, error = %err, @@ -3878,6 +4217,7 @@ where ); scan_table_data_plane_resource_for_object(self, table_bucket, object).await } + Err(err) => Err(err), } } @@ -3927,17 +4267,37 @@ where ))), ); }; + if current.state != TableCatalogEntryState::Active { + return table_commit_result( + &request.table_bucket, + &request.namespace, + &request.table, + &request.commit_id, + &request.operation, + commit_started, + Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + request.table_bucket, request.namespace, request.table + ))), + ); + } let commit_path = self .paths .commit_log_entry_path(&request.table_bucket, ¤t.table_id, &request.commit_id); - let existing_commit = self.read_commit_by_path(&commit_path).await?; - let idempotency_path = request.idempotency_key.as_deref().map(|idempotency_key| { + let existing_commit = self + .read_commit_log_entry(&request.table_bucket, ¤t.table_id, &request.commit_id) + .await?; + let idempotency_key = request.idempotency_key.as_deref(); + let idempotency_path = idempotency_key.map(|idempotency_key| { self.paths .commit_idempotency_entry_path(&request.table_bucket, ¤t.table_id, idempotency_key) }); - let existing_idempotency_commit = match idempotency_path.as_deref() { - Some(idempotency_path) => self.read_commit_by_path(idempotency_path).await?, + let existing_idempotency_commit = match idempotency_key { + Some(idempotency_key) => { + self.read_commit_idempotency_entry(&request.table_bucket, ¤t.table_id, idempotency_key) + .await? + } None => None, }; @@ -4182,6 +4542,9 @@ where } next.version_token = staged_commit_log.new_version_token.clone(); next.generation = current.generation.saturating_add(1); + if next.warehouse_location != current.warehouse_location { + self.ensure_table_warehouse_prefix_available(&next).await?; + } let reservation = self.reserve_table_warehouse_index(&next).await?; let staged_write_result = async { @@ -4298,29 +4661,39 @@ where table.as_str() ))); }; - self.backend.delete_object_unlocked(self.catalog_bucket(), &object).await?; - if let Err(err) = self.delete_table_warehouse_index(&entry).await { - if let Err(restore_err) = self - .write_entry_unlocked(self.catalog_bucket(), &object, &entry, TableCatalogPutPrecondition::IfAbsent) - .await - { - tracing::warn!( - table_bucket = %entry.table_bucket, - namespace = %entry.namespace, - table = %entry.table, - table_id = %entry.table_id, - error = %restore_err, - "failed to restore table entry after warehouse index delete failure" - ); + self.delete_owned_table_warehouse_index_for_drop(&entry).await?; + if let Err(err) = self.backend.delete_object_unlocked(self.catalog_bucket(), &object).await { + match self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await { + Ok(None) => return Ok(()), + Ok(Some((current, _))) if current == entry => { + if let Err(restore_err) = self.reserve_table_warehouse_index(&entry).await { + tracing::warn!( + table_bucket = %entry.table_bucket, + namespace = %entry.namespace, + table = %entry.table, + table_id = %entry.table_id, + error = %restore_err, + "failed to restore table warehouse index after table entry delete failure" + ); + } + } + Ok(Some(_)) => { + return Err(TableCatalogStoreError::Internal(format!( + "table {table_bucket}/{}/{} changed identity while its drop result was ambiguous", + namespace.public_name(), + table.as_str() + ))); + } + Err(read_err) => { + tracing::warn!( + table_bucket, + namespace = %namespace.public_name(), + table = %table.as_str(), + error = %read_err, + "failed to verify table state after an ambiguous table entry delete" + ); + } } - tracing::warn!( - table_bucket = %entry.table_bucket, - namespace = %entry.namespace, - table = %entry.table, - table_id = %entry.table_id, - error = %err, - "failed to delete table warehouse index after dropping table" - ); return Err(err); } Ok(()) @@ -4341,7 +4714,11 @@ where if !object.ends_with(VIEW_ENTRY_FILE) { continue; } - if let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? { + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? else { + continue; + }; + validate_view_entry_object(&self.paths, &object, &entry)?; + if entry.state == TableCatalogEntryState::Active { entries.push(entry); } } @@ -4362,8 +4739,8 @@ where VIEW_ENTRY_FILE, cursor, limit, - |_: &ViewEntry| true, - |_, _: &ViewEntry| Ok(()), + |entry: &ViewEntry| entry.state == TableCatalogEntryState::Active, + |object, entry: &ViewEntry| validate_view_entry_object(&self.paths, object, entry).map(|_| ()), ) .await } @@ -4371,9 +4748,12 @@ where async fn load_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult> { let namespace = parse_namespace_for_store(namespace)?; let view = parse_table_for_store(view)?; - self.read_entry::(self.catalog_bucket(), &self.paths.view_entry_path(table_bucket, &namespace, &view)) - .await - .map(|entry| entry.map(|(view, _)| view)) + let object = self.paths.view_entry_path(table_bucket, &namespace, &view); + let Some((entry, _)) = self.read_entry::(self.catalog_bucket(), &object).await? else { + return Ok(None); + }; + validate_view_entry_object(&self.paths, &object, &entry)?; + Ok((entry.state == TableCatalogEntryState::Active).then_some(entry)) } async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { @@ -4396,6 +4776,12 @@ where request.table_bucket, request.namespace, request.view ))); }; + if current.state != TableCatalogEntryState::Active { + return Err(TableCatalogStoreError::NotFound(format!( + "view {}/{}/{}", + request.table_bucket, request.namespace, request.view + ))); + } if current.version_token != request.expected_version_token { return Err(TableCatalogStoreError::Conflict( "current view version token does not match expected token".to_string(), @@ -4457,18 +4843,15 @@ where .await?; let object = self.paths.view_entry_path(table_bucket, &namespace, &view); let _view_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?; - if self - .read_entry_unlocked::(self.catalog_bucket(), &object) - .await? - .is_none() - { + let Some((entry, _)) = self.read_entry_unlocked::(self.catalog_bucket(), &object).await? else { return Err(TableCatalogStoreError::NotFound(format!( "view {}/{}/{}", table_bucket, namespace.public_name(), view.as_str() ))); - } + }; + validate_view_entry_object(&self.paths, &object, &entry)?; self.backend.delete_object_unlocked(self.catalog_bucket(), &object).await } @@ -4478,8 +4861,7 @@ where table_id: &str, commit_id: &str, ) -> TableCatalogStoreResult> { - let object = self.paths.commit_log_entry_path(table_bucket, table_id, commit_id); - self.read_commit_by_path(&object).await + self.read_commit_log_entry(table_bucket, table_id, commit_id).await } async fn get_commit_by_idempotency_key( @@ -4488,9 +4870,7 @@ where table_id: &str, idempotency_key: &str, ) -> TableCatalogStoreResult> { - let object = self - .paths - .commit_idempotency_entry_path(table_bucket, table_id, idempotency_key); - self.read_commit_by_path(&object).await + self.read_commit_idempotency_entry(table_bucket, table_id, idempotency_key) + .await } } diff --git a/rustfs/src/table_catalog/store/strong.rs b/rustfs/src/table_catalog/store/strong.rs index f3a885423..e2d525764 100644 --- a/rustfs/src/table_catalog/store/strong.rs +++ b/rustfs/src/table_catalog/store/strong.rs @@ -19,12 +19,40 @@ type StrongResourceKey = (String, String, String); type StrongCommitKey = (String, String, String); type StrongNamespaceChildKey = (String, String, String); type StrongWarehouseIndex = BTreeMap>; +#[derive(Clone, PartialEq, Eq)] +struct StrongSnapshotObservation { + hydrated: bool, + snapshot_required: bool, + etag: Option, + version: Option, +} +pub(in crate::table_catalog) const STRONG_TABLE_CATALOG_RELOAD_MAX_ATTEMPTS: usize = 3; +pub(in crate::table_catalog) const STRONG_TABLE_CATALOG_SNAPSHOT_MAX_SIZE: usize = 64 * 1024 * 1024; + +pub(in crate::table_catalog) fn strong_snapshot_write_version(version_two_requested: bool, fleet_confirmed: bool) -> u16 { + if version_two_requested && fleet_confirmed { + STRONG_TABLE_CATALOG_SNAPSHOT_VERSION + } else { + STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION + } +} + +#[derive(Clone, Default)] +pub(crate) struct StrongTableCatalogRuntime { + state: Arc>, + write_lock: Arc>, + reload_lock: Arc>, + #[cfg(test)] + reload_lock_attempts: Arc, +} #[derive(Clone, Default)] pub(in crate::table_catalog) struct StrongTableCatalogState { pub(super) hydrated: bool, - pub(super) snapshot_etag: Option, - pub(super) table_buckets: BTreeMap, + snapshot_required: bool, + pub(in crate::table_catalog) snapshot_etag: Option, + snapshot_version: Option, + pub(in crate::table_catalog) table_buckets: BTreeMap, pub(in crate::table_catalog) namespaces: BTreeMap, namespace_children: BTreeMap, namespace_objects: BTreeSet, @@ -33,17 +61,37 @@ pub(in crate::table_catalog) struct StrongTableCatalogState { pub(super) commits: BTreeMap, pub(super) idempotency: BTreeMap, pub(super) warehouse_index: StrongWarehouseIndex, + identifier_collisions: BTreeSet, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub(in crate::table_catalog) struct StrongCommitSnapshotRecord { pub(super) table_bucket: String, pub(super) table_id: String, pub(super) lookup_key: String, - pub(super) commit: CommitLogEntry, + pub(in crate::table_catalog) commit: CommitLogEntry, +} + +#[cfg(test)] +impl StrongCommitSnapshotRecord { + pub(in crate::table_catalog) fn new_for_test( + table_bucket: String, + table_id: String, + lookup_key: String, + commit: CommitLogEntry, + ) -> Self { + Self { + table_bucket, + table_id, + lookup_key, + commit, + } + } } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub(in crate::table_catalog) struct StrongTableCatalogSnapshot { pub(in crate::table_catalog) version: u16, pub(in crate::table_catalog) table_buckets: Vec, @@ -55,7 +103,7 @@ pub(in crate::table_catalog) struct StrongTableCatalogSnapshot { } #[derive(Debug, Clone, PartialEq, Serialize)] -pub(super) struct StrongTableCatalogBucketSnapshot { +pub(in crate::table_catalog) struct StrongTableCatalogBucketSnapshot { pub(super) table_bucket: TableBucketEntry, pub(super) namespaces: Vec, pub(super) tables: Vec, @@ -64,6 +112,129 @@ pub(super) struct StrongTableCatalogBucketSnapshot { pub(super) idempotency: Vec, } +#[derive(Clone)] +enum StrongSnapshotWritePostcondition { + TableBucketPresent(TableBucketEntry), + TableBucketAbsent(String), + NamespacePresent(NamespaceEntry), + NamespaceAbsent(StrongNamespaceKey), + TablePresent(TableEntry), + TableAbsent { + key: StrongResourceKey, + table_id: String, + }, + ViewPresent(ViewEntry), + ViewAbsent { + key: StrongResourceKey, + view_id: String, + }, + CommitPresent { + table_bucket: String, + table_id: String, + commit: CommitLogEntry, + }, + BucketSnapshotPresent(StrongTableCatalogBucketSnapshot), +} + +impl StrongSnapshotWritePostcondition { + fn is_satisfied_by(&self, state: &StrongTableCatalogState) -> bool + where + B: TableCatalogObjectBackend, + { + match self { + Self::TableBucketPresent(expected) => state.table_buckets.get(&expected.table_bucket) == Some(expected), + Self::TableBucketAbsent(table_bucket) => !state.table_buckets.contains_key(table_bucket), + Self::NamespacePresent(expected) => { + let Ok(namespace) = parse_namespace_for_store(&expected.namespace) else { + return false; + }; + state + .namespaces + .get(&StrongTableCatalogStore::::namespace_key(&expected.table_bucket, &namespace)) + == Some(expected) + } + Self::NamespaceAbsent(key) => !state.namespaces.contains_key(key), + Self::TablePresent(expected) => { + let (Ok(namespace), Ok(table)) = + (parse_namespace_for_store(&expected.namespace), parse_table_for_store(&expected.table)) + else { + return false; + }; + state + .tables + .get(&StrongTableCatalogStore::::table_key(&expected.table_bucket, &namespace, &table)) + == Some(expected) + } + Self::TableAbsent { key, table_id } => state.tables.get(key).is_none_or(|current| current.table_id != *table_id), + Self::ViewPresent(expected) => { + let (Ok(namespace), Ok(view)) = + (parse_namespace_for_store(&expected.namespace), parse_table_for_store(&expected.view)) + else { + return false; + }; + state + .views + .get(&StrongTableCatalogStore::::table_key(&expected.table_bucket, &namespace, &view)) + == Some(expected) + } + Self::ViewAbsent { key, view_id } => state.views.get(key).is_none_or(|current| current.view_id != *view_id), + Self::CommitPresent { + table_bucket, + table_id, + commit, + } => { + let commit_key = StrongTableCatalogStore::::commit_key(table_bucket, table_id, &commit.commit_id); + if state.commits.get(&commit_key) != Some(commit) { + return false; + } + if !commit.idempotency_key.as_deref().is_none_or(|idempotency_key| { + state + .idempotency + .get(&StrongTableCatalogStore::::idempotency_key(table_bucket, table_id, idempotency_key)) + == Some(commit) + }) { + return false; + } + let Some(table) = state.tables.values().find(|table| { + table.table_bucket == *table_bucket + && table.table_id == *table_id + && table.state == TableCatalogEntryState::Active + }) else { + return false; + }; + TableCommitHistoryIndex::new( + table, + StrongTableCatalogStore::::table_commits_locked(state, table_bucket, table_id), + ) + .proves_committed(commit) + } + Self::BucketSnapshotPresent(expected) => { + StrongTableCatalogStore::::bucket_snapshot_from_state_locked(state, &expected.table_bucket.table_bucket) + .as_ref() + == Some(expected) + } + } + } +} + +#[cfg(test)] +impl StrongTableCatalogBucketSnapshot { + pub(in crate::table_catalog) fn new_for_test(table_bucket: TableBucketEntry) -> Self { + Self { + table_bucket, + namespaces: Vec::new(), + tables: Vec::new(), + views: Vec::new(), + commits: Vec::new(), + idempotency: Vec::new(), + } + } + + pub(in crate::table_catalog) fn push_commit_for_test(&mut self, record: StrongCommitSnapshotRecord) { + self.commits.push(record); + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub(super) enum TableCatalogBackingMigrationFenceStatus { @@ -71,6 +242,13 @@ pub(super) enum TableCatalogBackingMigrationFenceStatus { Materialized, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum TableCatalogBackingMigrationTargetSnapshotState { + Unknown, + Absent, + Present, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct TableCatalogBackingMigrationGlobalFence { @@ -90,7 +268,7 @@ pub(super) struct TableCatalogBackingMigrationFence { pub(super) target_snapshot_etag: Option, } -pub(super) fn table_catalog_bucket_snapshot_fingerprint( +pub(in crate::table_catalog) fn table_catalog_bucket_snapshot_fingerprint( snapshot: &StrongTableCatalogBucketSnapshot, ) -> TableCatalogStoreResult { let data = serde_json::to_vec(snapshot) @@ -101,6 +279,8 @@ pub(super) fn table_catalog_bucket_snapshot_fingerprint( #[derive(Clone)] pub(crate) struct StrongTableCatalogStore { object_backend: B, + snapshot_write_version: u16, + snapshot_required_on_start: bool, // Single mutex protecting all catalog state (table_buckets, namespaces, tables, views, commits, idempotency). // This is intentional: many operations require atomic read-modify-write across multiple fields. // Splitting into per-field locks would introduce deadlock risk and complexity. @@ -111,6 +291,10 @@ pub(crate) struct StrongTableCatalogStore { pub(in crate::table_catalog) state: Arc>, // Serializes local snapshot mutations; object ETags fence independent store instances. write_lock: Arc>, + // Coalesces reloads for clones of one store so only one task reads and decodes a changed snapshot. + reload_lock: Arc>, + #[cfg(test)] + reload_lock_attempts: Arc, } impl StrongTableCatalogStore @@ -118,13 +302,63 @@ where B: TableCatalogObjectBackend, { pub fn new(object_backend: B) -> Self { + Self::new_with_snapshot_requirement(object_backend, false) + } + + pub(in crate::table_catalog) fn new_requiring_snapshot(object_backend: B) -> Self { + Self::new_with_snapshot_requirement(object_backend, true) + } + + #[cfg(test)] + pub(in crate::table_catalog) async fn is_hydrated_for_test(&self) -> bool { + self.state.lock().await.hydrated + } + + #[cfg(test)] + pub(in crate::table_catalog) fn reload_lock_attempts_for_test(&self) -> usize { + self.reload_lock_attempts.load(std::sync::atomic::Ordering::Relaxed) + } + + fn new_with_snapshot_requirement(object_backend: B, snapshot_required_on_start: bool) -> Self { + let snapshot_write_version = strong_snapshot_write_version( + rustfs_utils::get_env_bool(ENV_TABLE_CATALOG_STRONG_SNAPSHOT_V2, false), + rustfs_utils::get_env_bool(ENV_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED, false), + ); + // RUSTFS_COMPAT_TODO(table-catalog-strong-snapshot-v1): Keep version 1 writes during mixed-version rollout. Remove after all supported releases read version 2 and every retained snapshot is upgraded. + // Remove after the minimum supported release reads version 2 and operators no longer need collision cleanup. + let runtime = object_backend.strong_catalog_runtime().unwrap_or_default(); + Self::new_with_runtime_and_snapshot_write_version( + object_backend, + runtime, + snapshot_write_version, + snapshot_required_on_start, + ) + } + + fn new_with_runtime_and_snapshot_write_version( + object_backend: B, + runtime: StrongTableCatalogRuntime, + snapshot_write_version: u16, + snapshot_required_on_start: bool, + ) -> Self { Self { object_backend, - state: Arc::new(tokio::sync::Mutex::new(StrongTableCatalogState::default())), - write_lock: Arc::new(tokio::sync::Mutex::new(())), + snapshot_write_version, + snapshot_required_on_start, + state: runtime.state, + write_lock: runtime.write_lock, + reload_lock: runtime.reload_lock, + #[cfg(test)] + reload_lock_attempts: runtime.reload_lock_attempts, } } + #[cfg(test)] + pub(in crate::table_catalog) fn new_with_snapshot_write_version(object_backend: B, snapshot_write_version: u16) -> Self { + let runtime = object_backend.strong_catalog_runtime().unwrap_or_default(); + Self::new_with_runtime_and_snapshot_write_version(object_backend, runtime, snapshot_write_version, false) + } + pub(in crate::table_catalog) fn namespace_key(table_bucket: &str, namespace: &Namespace) -> StrongNamespaceKey { (table_bucket.to_string(), namespace.public_name()) } @@ -140,11 +374,10 @@ where fn namespace_exists_locked(state: &StrongTableCatalogState, table_bucket: &str, namespace: &Namespace) -> bool { let key = Self::namespace_key(table_bucket, namespace); - state - .namespaces - .get(&key) - .is_some_and(|entry| entry.state == TableCatalogEntryState::Active) - || state.namespace_objects.contains(&key) + if let Some(entry) = state.namespaces.get(&key) { + return entry.state == TableCatalogEntryState::Active; + } + state.namespace_objects.contains(&key) || Self::has_active_namespace_descendant_locked(state, table_bucket, &namespace.public_name()) } @@ -220,6 +453,52 @@ where (table_bucket.to_string(), namespace.public_name(), table.as_str().to_string()) } + fn ensure_identifier_is_unambiguous_locked( + state: &StrongTableCatalogState, + key: &StrongResourceKey, + ) -> TableCatalogStoreResult<()> { + if state.identifier_collisions.contains(key) { + return Err(TableCatalogStoreError::Internal(format!( + "legacy table/view identifier collision requires operator cleanup: {}/{}/{}", + key.0, key.1, key.2 + ))); + } + Ok(()) + } + + fn ensure_namespace_identifiers_are_unambiguous_locked( + state: &StrongTableCatalogState, + table_bucket: &str, + namespace: &str, + ) -> TableCatalogStoreResult<()> { + if state + .identifier_collisions + .iter() + .any(|(bucket, entry_namespace, _)| bucket == table_bucket && entry_namespace == namespace) + { + return Err(TableCatalogStoreError::Internal(format!( + "legacy table/view identifier collision requires operator cleanup in {table_bucket}/{namespace}" + ))); + } + Ok(()) + } + + fn ensure_table_bucket_identifiers_are_unambiguous_locked( + state: &StrongTableCatalogState, + table_bucket: &str, + ) -> TableCatalogStoreResult<()> { + if state + .identifier_collisions + .iter() + .any(|(bucket, _, _)| bucket == table_bucket) + { + return Err(TableCatalogStoreError::Internal(format!( + "legacy table/view identifier collision requires operator cleanup in {table_bucket}" + ))); + } + Ok(()) + } + fn commit_key(table_bucket: &str, table_id: &str, commit_id: &str) -> StrongCommitKey { (table_bucket.to_string(), table_id.to_string(), commit_id.to_string()) } @@ -228,13 +507,49 @@ where (table_bucket.to_string(), table_id.to_string(), idempotency_key.to_string()) } + fn table_commits_locked<'a>( + state: &'a StrongTableCatalogState, + table_bucket: &str, + table_id: &str, + ) -> impl Iterator + 'a { + let owner = (table_bucket.to_string(), table_id.to_string()); + let start = (owner.0.clone(), owner.1.clone(), String::new()); + state + .commits + .range(start..) + .take_while(move |((entry_bucket, entry_table_id, _), _)| entry_bucket == &owner.0 && entry_table_id == &owner.1) + .map(|(_, commit)| commit) + } + + fn commit_write_postcondition(table_bucket: &str, commit: &CommitLogEntry) -> StrongSnapshotWritePostcondition { + StrongSnapshotWritePostcondition::CommitPresent { + table_bucket: table_bucket.to_string(), + table_id: commit.table_id.clone(), + commit: commit.clone(), + } + } + pub(in crate::table_catalog) fn snapshot_object_path() -> String { format!("{INTERNAL_CATALOG_ROOT}/{STRONG_TABLE_CATALOG_BACKING_ROOT}/{STRONG_TABLE_CATALOG_SNAPSHOT_FILE}") } - fn snapshot_from_state_locked(state: &StrongTableCatalogState) -> StrongTableCatalogSnapshot { + // Ordinary mutations hold the global migration read lock before the local write lock; migration takes the + // write side before invoking its dedicated snapshot mutation methods. + async fn acquire_snapshot_write_permit(&self) -> TableCatalogStoreResult> { + let lock_path = TableCatalogObjectPaths::default().backing_migration_global_fence_lock_path(); + self.object_backend.acquire_read_lock(RUSTFS_META_BUCKET, &lock_path).await + } + + fn effective_snapshot_write_version(state: &StrongTableCatalogState, configured_write_version: u16) -> u16 { + state + .snapshot_version + .unwrap_or(STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION) + .max(configured_write_version) + } + + fn snapshot_from_state_locked(state: &StrongTableCatalogState, snapshot_version: u16) -> StrongTableCatalogSnapshot { StrongTableCatalogSnapshot { - version: STRONG_TABLE_CATALOG_SNAPSHOT_VERSION, + version: snapshot_version, table_buckets: state.table_buckets.values().cloned().collect(), namespaces: state.namespaces.values().cloned().collect(), tables: state.tables.values().cloned().collect(), @@ -328,43 +643,73 @@ where .idempotency .retain(|(entry_bucket, _, _), _| entry_bucket != table_bucket); state.warehouse_index.remove(table_bucket); + state + .identifier_collisions + .retain(|(entry_bucket, _, _)| entry_bucket != table_bucket); } pub(super) fn insert_bucket_snapshot_locked( state: &mut StrongTableCatalogState, snapshot: StrongTableCatalogBucketSnapshot, ) -> TableCatalogStoreResult<()> { + let expected_counts = ( + 1, + snapshot.namespaces.len(), + snapshot.tables.len(), + snapshot.views.len(), + snapshot.commits.len(), + snapshot.idempotency.len(), + ); let table_bucket = snapshot.table_bucket.table_bucket.clone(); + let unexpected_owner = snapshot + .namespaces + .iter() + .map(|entry| entry.table_bucket.as_str()) + .chain(snapshot.tables.iter().map(|entry| entry.table_bucket.as_str())) + .chain(snapshot.views.iter().map(|entry| entry.table_bucket.as_str())) + .chain(snapshot.commits.iter().map(|record| record.table_bucket.as_str())) + .chain(snapshot.idempotency.iter().map(|record| record.table_bucket.as_str())) + .find(|owner| *owner != table_bucket); + if let Some(owner) = unexpected_owner { + return Err(TableCatalogStoreError::Invalid(format!( + "durable strong bucket snapshot for {table_bucket} contains state owned by {owner}" + ))); + } + let validated = Self::state_from_snapshot( + StrongTableCatalogSnapshot { + version: STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION, + table_buckets: vec![snapshot.table_bucket], + namespaces: snapshot.namespaces, + tables: snapshot.tables, + views: snapshot.views, + commits: snapshot.commits, + idempotency: snapshot.idempotency, + }, + None, + )?; + let validated_counts = ( + validated.table_buckets.len(), + validated.namespaces.len(), + validated.tables.len(), + validated.views.len(), + validated.commits.len(), + validated.idempotency.len(), + ); + if validated_counts != expected_counts { + return Err(TableCatalogStoreError::Invalid( + "durable strong bucket snapshot contains state without a valid owner".to_string(), + )); + } Self::remove_bucket_from_state_locked(state, &table_bucket); - state.table_buckets.insert(table_bucket.clone(), snapshot.table_bucket); - for entry in snapshot.namespaces { - let namespace = validate_namespace_entry_identity(&entry)?; - state.namespaces.insert(Self::namespace_key(&table_bucket, &namespace), entry); - } - for entry in snapshot.tables { - let namespace = parse_namespace_for_store(&entry.namespace)?; - let table = parse_table_for_store(&entry.table)?; - state.tables.insert(Self::table_key(&table_bucket, &namespace, &table), entry); - } - for entry in snapshot.views { - let namespace = parse_namespace_for_store(&entry.namespace)?; - let view = parse_table_for_store(&entry.view)?; - state.views.insert(Self::table_key(&table_bucket, &namespace, &view), entry); - } - for record in snapshot.commits { - state.commits.insert( - Self::commit_key(&record.table_bucket, &record.table_id, &record.lookup_key), - record.commit, - ); - } - for record in snapshot.idempotency { - state.idempotency.insert( - Self::idempotency_key(&record.table_bucket, &record.table_id, &record.lookup_key), - record.commit, - ); - } + state.table_buckets.extend(validated.table_buckets); + state.namespaces.extend(validated.namespaces); + state.tables.extend(validated.tables); + state.views.extend(validated.views); + state.commits.extend(validated.commits); + state.idempotency.extend(validated.idempotency); Self::rebuild_namespace_indexes_locked(state)?; - Self::rebuild_warehouse_index_locked(state) + Self::rebuild_warehouse_index_locked(state)?; + Ok(()) } fn index_namespace_children( @@ -390,58 +735,106 @@ where fn rebuild_namespace_indexes_locked(state: &mut StrongTableCatalogState) -> TableCatalogStoreResult<()> { let mut children = BTreeMap::new(); - for entry in state - .namespaces - .values() - .filter(|entry| entry.state == TableCatalogEntryState::Active) - { + for ((table_bucket, namespace_name), entry) in &state.namespaces { let namespace = validate_namespace_entry_identity(entry)?; + validate_namespace_properties(&entry.properties)?; + if entry.table_bucket != *table_bucket || entry.namespace != *namespace_name { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog namespace entry identity does not match its snapshot key: {table_bucket}/{namespace_name}" + ))); + } + let Some(bucket_entry) = state.table_buckets.get(table_bucket) else { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog namespace {table_bucket}/{namespace_name} has no table bucket" + ))); + }; + if entry.state != TableCatalogEntryState::Active { + continue; + } + if bucket_entry.state != TableCatalogEntryState::Active { + return Err(TableCatalogStoreError::Invalid(format!( + "active namespace {table_bucket}/{namespace_name} has no active table bucket" + ))); + } Self::index_namespace_children(&mut children, &entry.table_bucket, &namespace); } let mut objects = BTreeSet::new(); - for (table_bucket, namespace_name) in state + for (table_bucket, namespace_name, active) in state .tables .values() - .filter(|entry| entry.state == TableCatalogEntryState::Active) - .map(|entry| (&entry.table_bucket, &entry.namespace)) + .map(|entry| (&entry.table_bucket, &entry.namespace, entry.state == TableCatalogEntryState::Active)) .chain( state .views .values() - .filter(|entry| entry.state == TableCatalogEntryState::Active) - .map(|entry| (&entry.table_bucket, &entry.namespace)), + .map(|entry| (&entry.table_bucket, &entry.namespace, entry.state == TableCatalogEntryState::Active)), ) { let namespace = parse_namespace_for_store(namespace_name)?; + if !active { + continue; + } objects.insert(Self::namespace_key(table_bucket, &namespace)); Self::index_namespace_children(&mut children, table_bucket, &namespace); } state.namespace_children = children; state.namespace_objects = objects; + for ((table_bucket, namespace), entry) in &state.namespaces { + if entry.state == TableCatalogEntryState::Active { + continue; + } + let key = (table_bucket.clone(), namespace.clone()); + if state.namespace_objects.contains(&key) + || Self::has_active_namespace_descendant_locked(state, table_bucket, namespace) + { + return Err(TableCatalogStoreError::Invalid(format!( + "inactive namespace {table_bucket}/{namespace} has active resources or descendants" + ))); + } + } Ok(()) } fn rebuild_warehouse_index_locked(state: &mut StrongTableCatalogState) -> TableCatalogStoreResult<()> { let mut warehouse_index: StrongWarehouseIndex = BTreeMap::new(); for ((table_bucket, namespace, table), entry) in &state.tables { + if entry.table_bucket != *table_bucket || entry.namespace != *namespace || entry.table != *table { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog table entry identity does not match its snapshot key: {table_bucket}/{namespace}/{table}" + ))); + } + if !state.table_buckets.contains_key(table_bucket) { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog table {table_bucket}/{namespace}/{table} has no table bucket" + ))); + } if entry.state != TableCatalogEntryState::Active { continue; } + let namespace_identity = parse_namespace_for_store(namespace)?; + let table_identity = parse_table_for_store(table)?; + validate_table_warehouse_location(table_bucket, &entry.warehouse_location)?; + if !is_valid_table_metadata_location(&namespace_identity, &table_identity, &entry.metadata_location) { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog table {table_bucket}/{namespace}/{table} has an invalid metadata location" + ))); + } + if !Self::namespace_exists_locked(state, table_bucket, &namespace_identity) { + return Err(TableCatalogStoreError::Invalid(format!( + "table {table_bucket}/{namespace}/{table} has no active namespace" + ))); + } if !state .table_buckets .get(table_bucket) .is_some_and(|entry| entry.state == TableCatalogEntryState::Active) { - continue; + return Err(TableCatalogStoreError::Invalid(format!( + "active table {table_bucket}/{namespace}/{table} has no active table bucket" + ))); } - let namespace_identity = parse_namespace_for_store(namespace)?; - if !Self::namespace_exists_locked(state, table_bucket, &namespace_identity) { - continue; - } - let Ok(warehouse_object_prefix) = table_warehouse_object_prefix(entry) else { - continue; - }; + let warehouse_object_prefix = table_warehouse_object_prefix(entry)?; let table_key = (table_bucket.clone(), namespace.clone(), table.clone()); let bucket_index = warehouse_index.entry(table_bucket.clone()).or_default(); let predecessor = bucket_index.range(..=warehouse_object_prefix.clone()).next_back(); @@ -458,71 +851,320 @@ where } bucket_index.insert(warehouse_object_prefix, table_key); } + + let mut identifier_collisions = BTreeSet::new(); + for ((table_bucket, namespace, view), entry) in &state.views { + if entry.table_bucket != *table_bucket || entry.namespace != *namespace || entry.view != *view { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog view entry identity does not match its snapshot key: {table_bucket}/{namespace}/{view}" + ))); + } + if !state.table_buckets.contains_key(table_bucket) { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog view {table_bucket}/{namespace}/{view} has no table bucket" + ))); + } + if entry.state != TableCatalogEntryState::Active { + continue; + } + let namespace_identity = parse_namespace_for_store(namespace)?; + let view_identity = parse_table_for_store(view)?; + validate_view_warehouse_location(table_bucket, &entry.warehouse_location)?; + if !is_valid_view_metadata_location(&namespace_identity, &view_identity, &entry.metadata_location) { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog view {table_bucket}/{namespace}/{view} has an invalid metadata location" + ))); + } + if !Self::namespace_exists_locked(state, table_bucket, &namespace_identity) { + return Err(TableCatalogStoreError::Invalid(format!( + "view {table_bucket}/{namespace}/{view} has no active namespace" + ))); + } + if !state + .table_buckets + .get(table_bucket) + .is_some_and(|entry| entry.state == TableCatalogEntryState::Active) + { + return Err(TableCatalogStoreError::Invalid(format!( + "active view {table_bucket}/{namespace}/{view} has no active table bucket" + ))); + } + let view_key = (table_bucket.clone(), namespace.clone(), view.clone()); + if state + .tables + .get(&view_key) + .is_some_and(|entry| entry.state == TableCatalogEntryState::Active) + { + identifier_collisions.insert(view_key); + } + } state.warehouse_index = warehouse_index; + state.identifier_collisions = identifier_collisions; + Ok(()) + } + + fn validate_inactive_resource_locations_locked(state: &StrongTableCatalogState) -> TableCatalogStoreResult<()> { + for ((table_bucket, namespace, table), entry) in &state.tables { + if entry.state == TableCatalogEntryState::Active { + continue; + } + let namespace_identity = parse_namespace_for_store(namespace)?; + let table_identity = parse_table_for_store(table)?; + validate_table_warehouse_location(table_bucket, &entry.warehouse_location)?; + if !is_valid_table_metadata_location(&namespace_identity, &table_identity, &entry.metadata_location) { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog table {table_bucket}/{namespace}/{table} has an invalid metadata location" + ))); + } + if !Self::namespace_exists_locked(state, table_bucket, &namespace_identity) { + return Err(TableCatalogStoreError::Invalid(format!( + "table {table_bucket}/{namespace}/{table} has no active namespace" + ))); + } + } + for ((table_bucket, namespace, view), entry) in &state.views { + if entry.state == TableCatalogEntryState::Active { + continue; + } + let namespace_identity = parse_namespace_for_store(namespace)?; + let view_identity = parse_table_for_store(view)?; + validate_view_warehouse_location(table_bucket, &entry.warehouse_location)?; + if !is_valid_view_metadata_location(&namespace_identity, &view_identity, &entry.metadata_location) { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog view {table_bucket}/{namespace}/{view} has an invalid metadata location" + ))); + } + if !Self::namespace_exists_locked(state, table_bucket, &namespace_identity) { + return Err(TableCatalogStoreError::Invalid(format!( + "view {table_bucket}/{namespace}/{view} has no active namespace" + ))); + } + } Ok(()) } fn snapshot_from_mutated_state_locked( state: &mut StrongTableCatalogState, + configured_write_version: u16, ) -> TableCatalogStoreResult { + let previous_identifier_collisions = state.identifier_collisions.clone(); Self::rebuild_namespace_indexes_locked(state)?; Self::rebuild_warehouse_index_locked(state)?; - Ok(Self::snapshot_from_state_locked(state)) + if !state.identifier_collisions.is_empty() + && state + .snapshot_version + .is_some_and(|version| version >= STRONG_TABLE_CATALOG_SNAPSHOT_VERSION) + { + return Err(TableCatalogStoreError::Invalid( + "version 1 collision quarantine cannot be persisted after snapshot version 2 was observed".to_string(), + )); + } + let collision_cleanup_progress = !previous_identifier_collisions.is_empty() + && state.identifier_collisions.is_subset(&previous_identifier_collisions) + && state.identifier_collisions.len() < previous_identifier_collisions.len(); + // RUSTFS_COMPAT_TODO(table-catalog-strong-snapshot-v1): Version 1 collision quarantine permits cleanup-only writes. Remove after all supported releases read version 2 and every retained snapshot is collision-free. + // Remove with version 1 snapshot write support after every supported deployment writes collision-free version 2 snapshots. + if !state.identifier_collisions.is_empty() && !collision_cleanup_progress { + return Err(TableCatalogStoreError::Conflict( + "legacy table/view identifier collision permits only collision cleanup".to_string(), + )); + } + let snapshot_version = if state.identifier_collisions.is_empty() { + Self::effective_snapshot_write_version(state, configured_write_version) + } else { + STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION + }; + if snapshot_version >= STRONG_TABLE_CATALOG_SNAPSHOT_VERSION { + Self::validate_inactive_resource_locations_locked(state)?; + } + let snapshot = Self::snapshot_from_state_locked(state, snapshot_version); + Self::state_from_snapshot(snapshot.clone(), None)?; + state.snapshot_version = Some(state.snapshot_version.unwrap_or(snapshot.version).max(snapshot.version)); + Ok(snapshot) } fn state_from_snapshot( snapshot: StrongTableCatalogSnapshot, snapshot_etag: Option, ) -> TableCatalogStoreResult { - if snapshot.version != STRONG_TABLE_CATALOG_SNAPSHOT_VERSION { + if !(STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION..=STRONG_TABLE_CATALOG_SNAPSHOT_VERSION).contains(&snapshot.version) { return Err(TableCatalogStoreError::Invalid(format!( "unsupported strong catalog snapshot version: {}", snapshot.version ))); } + let snapshot_version = snapshot.version; let mut state = StrongTableCatalogState { hydrated: true, + snapshot_required: true, snapshot_etag, + snapshot_version: Some(snapshot_version), ..StrongTableCatalogState::default() }; for entry in snapshot.table_buckets { - state.table_buckets.insert(entry.table_bucket.clone(), entry); + validate_table_bucket_entry(&entry)?; + let table_bucket = entry.table_bucket.clone(); + if state.table_buckets.insert(table_bucket.clone(), entry).is_some() { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog snapshot contains duplicate table bucket {table_bucket}" + ))); + } } for entry in snapshot.namespaces { let namespace = validate_namespace_entry_identity(&entry)?; - state - .namespaces - .insert(Self::namespace_key(&entry.table_bucket, &namespace), entry); + validate_namespace_properties(&entry.properties)?; + let Some(table_bucket_entry) = state.table_buckets.get(&entry.table_bucket) else { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog namespace {}/{} has no table bucket", + entry.table_bucket, entry.namespace + ))); + }; + if entry.state == TableCatalogEntryState::Active && table_bucket_entry.state != TableCatalogEntryState::Active { + return Err(TableCatalogStoreError::Invalid(format!( + "active namespace {}/{} has no active table bucket", + entry.table_bucket, entry.namespace + ))); + } + let key = Self::namespace_key(&entry.table_bucket, &namespace); + if state.namespaces.insert(key, entry).is_some() { + return Err(TableCatalogStoreError::Invalid( + "strong catalog snapshot contains duplicate namespace identifiers".to_string(), + )); + } } + let mut table_ids = BTreeSet::new(); for entry in snapshot.tables { + validate_table_entry_version_and_id(&entry)?; let namespace = parse_namespace_for_store(&entry.namespace)?; let table = parse_table_for_store(&entry.table)?; - state - .tables - .insert(Self::table_key(&entry.table_bucket, &namespace, &table), entry); + if !state.table_buckets.contains_key(&entry.table_bucket) { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog table {}/{}/{} has no table bucket", + entry.table_bucket, entry.namespace, entry.table + ))); + } + if !table_ids.insert((entry.table_bucket.clone(), entry.table_id.clone())) { + return Err(TableCatalogStoreError::Invalid( + "strong catalog snapshot contains duplicate table ids".to_string(), + )); + } + let key = Self::table_key(&entry.table_bucket, &namespace, &table); + if state.tables.insert(key, entry).is_some() { + return Err(TableCatalogStoreError::Invalid( + "strong catalog snapshot contains duplicate table identifiers".to_string(), + )); + } } for entry in snapshot.views { + validate_view_entry_version_and_id(&entry)?; let namespace = parse_namespace_for_store(&entry.namespace)?; let view = parse_table_for_store(&entry.view)?; - state - .views - .insert(Self::table_key(&entry.table_bucket, &namespace, &view), entry); + if !state.table_buckets.contains_key(&entry.table_bucket) { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog view {}/{}/{} has no table bucket", + entry.table_bucket, entry.namespace, entry.view + ))); + } + let key = Self::table_key(&entry.table_bucket, &namespace, &view); + if state.views.insert(key, entry).is_some() { + return Err(TableCatalogStoreError::Invalid( + "strong catalog snapshot contains duplicate view identifiers".to_string(), + )); + } } for record in snapshot.commits { - state.commits.insert( - Self::commit_key(&record.table_bucket, &record.table_id, &record.lookup_key), - record.commit, - ); + validate_catalog_entry_version("commit log", record.commit.version)?; + if record.commit.table_id != record.table_id || record.commit.commit_id != record.lookup_key { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog commit {} does not match its snapshot owner", + record.lookup_key + ))); + } + if !table_ids.contains(&(record.table_bucket.clone(), record.table_id.clone())) { + if snapshot_version >= STRONG_TABLE_CATALOG_SNAPSHOT_VERSION { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog commit {} has no owning table", + record.lookup_key + ))); + } + continue; + } + let key = Self::commit_key(&record.table_bucket, &record.table_id, &record.lookup_key); + if state.commits.insert(key, record.commit).is_some() { + return Err(TableCatalogStoreError::Invalid( + "strong catalog snapshot contains duplicate commit lookup keys".to_string(), + )); + } } for record in snapshot.idempotency { - state.idempotency.insert( - Self::idempotency_key(&record.table_bucket, &record.table_id, &record.lookup_key), - record.commit, - ); + validate_catalog_entry_version("commit idempotency", record.commit.version)?; + if record.commit.table_id != record.table_id + || record.commit.idempotency_key.as_deref() != Some(record.lookup_key.as_str()) + { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog idempotency index {} does not match its snapshot owner", + record.lookup_key + ))); + } + if !table_ids.contains(&(record.table_bucket.clone(), record.table_id.clone())) { + if snapshot_version >= STRONG_TABLE_CATALOG_SNAPSHOT_VERSION { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog idempotency index {} has no owning table", + record.lookup_key + ))); + } + continue; + } + let commit_key = Self::commit_key(&record.table_bucket, &record.table_id, &record.commit.commit_id); + if state.commits.get(&commit_key) != Some(&record.commit) { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog idempotency index {} has no matching commit", + record.lookup_key + ))); + } + let key = Self::idempotency_key(&record.table_bucket, &record.table_id, &record.lookup_key); + if state.idempotency.insert(key, record.commit).is_some() { + return Err(TableCatalogStoreError::Invalid( + "strong catalog snapshot contains duplicate idempotency lookup keys".to_string(), + )); + } + } + for ((table_bucket, table_id, _), commit) in &state.commits { + let Some(idempotency_key) = commit.idempotency_key.as_deref() else { + continue; + }; + let index_key = Self::idempotency_key(table_bucket, table_id, idempotency_key); + if state.idempotency.get(&index_key) != Some(commit) { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog commit {} has no matching idempotency index", + commit.commit_id + ))); + } + } + for table in state.tables.values() { + let table_commits = Self::table_commits_locked(&state, &table.table_bucket, &table.table_id).collect::>(); + let history = TableCommitHistoryIndex::new(table, table_commits.iter().copied()); + if let Some(commit) = table_commits + .into_iter() + .find(|commit| matches!(commit.status, CommitLogStatus::Failed) || !history.proves_committed(commit)) + { + return Err(TableCatalogStoreError::Invalid(format!( + "strong catalog commit {} is not recoverable in the current table history", + commit.commit_id + ))); + } } Self::rebuild_namespace_indexes_locked(&mut state)?; Self::rebuild_warehouse_index_locked(&mut state)?; + if snapshot_version >= STRONG_TABLE_CATALOG_SNAPSHOT_VERSION { + Self::validate_inactive_resource_locations_locked(&state)?; + } + if snapshot_version >= STRONG_TABLE_CATALOG_SNAPSHOT_VERSION && !state.identifier_collisions.is_empty() { + return Err(TableCatalogStoreError::Invalid( + "strong catalog snapshot contains a table/view identifier collision".to_string(), + )); + } Ok(state) } @@ -539,11 +1181,41 @@ where (Self::snapshot_write_precondition_locked(state), state.clone()) } + fn snapshot_observation_locked(state: &StrongTableCatalogState) -> StrongSnapshotObservation { + StrongSnapshotObservation { + hydrated: state.hydrated, + snapshot_required: state.snapshot_required, + etag: state.snapshot_etag.clone(), + version: state.snapshot_version, + } + } + + fn snapshot_etag(etag: Option) -> TableCatalogStoreResult { + etag.filter(|etag| !etag.is_empty()) + .ok_or_else(|| TableCatalogStoreError::Internal("durable strong catalog snapshot has no etag".to_string())) + } + + fn state_from_snapshot_object(snapshot_object: TableCatalogObject) -> TableCatalogStoreResult { + let snapshot_etag = Self::snapshot_etag(snapshot_object.etag)?; + let snapshot = serde_json::from_slice::(&snapshot_object.data) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to decode strong catalog snapshot: {err}")))?; + Self::state_from_snapshot(snapshot, Some(snapshot_etag)) + } + + async fn mark_snapshot_reload_failed(&self, err: &TableCatalogStoreError, phase: &'static str) { + self.state.lock().await.hydrated = false; + tracing::warn!( + error = %err, + phase, + "durable strong catalog state reload failed after a snapshot write outcome" + ); + } + async fn hydrate_state(&self) -> TableCatalogStoreResult<()> { - let Some(current_snapshot_etag) = ({ + let Some((current_snapshot_etag, current_snapshot_required)) = ({ let state = self.state.lock().await; if state.hydrated { - Some(state.snapshot_etag.clone()) + Some((state.snapshot_etag.clone(), state.snapshot_required)) } else { None } @@ -556,65 +1228,151 @@ where .object_metadata(RUSTFS_META_BUCKET, &Self::snapshot_object_path()) .await?; match (snapshot_metadata, current_snapshot_etag.as_deref()) { - (None, None) => Ok(()), + (None, None) if !self.snapshot_required_on_start && !current_snapshot_required => Ok(()), (Some(metadata), Some(current_etag)) if metadata.etag.as_deref() == Some(current_etag) => Ok(()), _ => self.reload_state_from_durable().await, } } - async fn reload_state_from_durable(&self) -> TableCatalogStoreResult<()> { - let snapshot_object = self - .object_backend - .read_object(RUSTFS_META_BUCKET, &Self::snapshot_object_path()) - .await?; - let mut state = self.state.lock().await; - if let Some(snapshot_object) = snapshot_object { - let snapshot = serde_json::from_slice::(&snapshot_object.data) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to decode strong catalog snapshot: {err}")))?; - *state = Self::state_from_snapshot(snapshot, snapshot_object.etag)?; - } else { - *state = StrongTableCatalogState { - hydrated: true, - ..StrongTableCatalogState::default() - }; + pub(in crate::table_catalog) async fn reload_state_from_durable(&self) -> TableCatalogStoreResult<()> { + let initial_observation = { + let state = self.state.lock().await; + Self::snapshot_observation_locked(&state) + }; + #[cfg(test)] + self.reload_lock_attempts.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let _reload_guard = self.reload_lock.lock().await; + { + let state = self.state.lock().await; + if state.hydrated + && (!self.snapshot_required_on_start || state.snapshot_required) + && Self::snapshot_observation_locked(&state) != initial_observation + { + return Ok(()); + } } - Ok(()) - } - async fn persist_snapshot( - &self, - snapshot: StrongTableCatalogSnapshot, - precondition: TableCatalogPutPrecondition, - ) -> TableCatalogStoreResult<()> { - let data = serde_json::to_vec(&snapshot) - .map_err(|err| TableCatalogStoreError::Internal(format!("failed to encode strong catalog snapshot: {err}")))?; - self.object_backend - .put_object(RUSTFS_META_BUCKET, &Self::snapshot_object_path(), data, precondition) - .await + for _ in 0..STRONG_TABLE_CATALOG_RELOAD_MAX_ATTEMPTS { + let observed_state = { + let state = self.state.lock().await; + Self::snapshot_observation_locked(&state) + }; + let snapshot_object = self + .object_backend + .read_object_limited(RUSTFS_META_BUCKET, &Self::snapshot_object_path(), STRONG_TABLE_CATALOG_SNAPSHOT_MAX_SIZE) + .await?; + let snapshot_metadata = self + .object_backend + .object_metadata(RUSTFS_META_BUCKET, &Self::snapshot_object_path()) + .await?; + let loaded_state = match (snapshot_object, snapshot_metadata) { + (Some(snapshot_object), Some(snapshot_metadata)) => { + let object_etag = Self::snapshot_etag(snapshot_object.etag.clone())?; + let metadata_etag = Self::snapshot_etag(snapshot_metadata.etag)?; + if object_etag != metadata_etag { + continue; + } + Some(Self::state_from_snapshot_object(snapshot_object)?) + } + (Some(_), None) => { + return Err(TableCatalogStoreError::Internal( + "durable strong catalog snapshot disappeared while it was being loaded".to_string(), + )); + } + (None, Some(_)) => continue, + (None, None) => None, + }; + let mut state = self.state.lock().await; + if Self::snapshot_observation_locked(&state) != observed_state { + continue; + } + let loaded_state = match loaded_state { + Some(loaded_state) => loaded_state, + None if observed_state.snapshot_required || self.snapshot_required_on_start => { + return Err(TableCatalogStoreError::Internal( + "durable strong catalog snapshot disappeared after it was observed".to_string(), + )); + } + None => StrongTableCatalogState { + hydrated: true, + ..StrongTableCatalogState::default() + }, + }; + if let (Some(observed), Some(loaded)) = (observed_state.version, loaded_state.snapshot_version) + && loaded < observed + { + return Err(TableCatalogStoreError::Invalid(format!( + "durable strong catalog snapshot version {loaded} cannot replace process high-water version {observed}" + ))); + } + *state = loaded_state; + return Ok(()); + } + Err(TableCatalogStoreError::Internal( + "durable strong catalog state changed repeatedly while reloading".to_string(), + )) } async fn finalize_snapshot_write( &self, snapshot: StrongTableCatalogSnapshot, precondition: TableCatalogPutPrecondition, + postcondition: StrongSnapshotWritePostcondition, ) -> TableCatalogStoreResult<()> { - match self.persist_snapshot(snapshot, precondition).await { - Ok(()) => self.reload_state_from_durable().await, - Err(err) => { - let _ = self.reload_state_from_durable().await; + let data = serde_json::to_vec(&snapshot) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to encode strong catalog snapshot: {err}")))?; + if data.len() > STRONG_TABLE_CATALOG_SNAPSHOT_MAX_SIZE { + return Err(TableCatalogStoreError::Invalid(format!( + "durable strong catalog snapshot exceeds the maximum encoded size of {STRONG_TABLE_CATALOG_SNAPSHOT_MAX_SIZE} bytes" + ))); + } + match self + .object_backend + .put_object(RUSTFS_META_BUCKET, &Self::snapshot_object_path(), data, precondition) + .await + { + Ok(()) => { + self.state.lock().await.snapshot_required = true; + if let Err(err) = self.reload_state_from_durable().await { + self.mark_snapshot_reload_failed(&err, "confirmed-write").await; + } + Ok(()) + } + Err(err @ TableCatalogStoreError::Conflict(_)) => { + self.state.lock().await.snapshot_required = true; + if let Err(reload_err) = self.reload_state_from_durable().await { + self.mark_snapshot_reload_failed(&reload_err, "write-conflict").await; + } Err(err) } + Err(err) => { + self.state.lock().await.snapshot_required = true; + match self.reload_state_from_durable().await { + Ok(()) => { + let state = self.state.lock().await; + if postcondition.is_satisfied_by::(&state) { + Ok(()) + } else { + Err(err) + } + } + Err(reload_err) => { + self.mark_snapshot_reload_failed(&reload_err, "ambiguous-write").await; + Err(err) + } + } + } } } - pub(super) async fn materialize_bucket_snapshot( + pub(in crate::table_catalog) async fn materialize_bucket_snapshot( &self, source: StrongTableCatalogBucketSnapshot, ) -> TableCatalogStoreResult<(String, bool)> { let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; let table_bucket = source.table_bucket.table_bucket.clone(); - let (snapshot, precondition) = { + let (snapshot, precondition, postcondition) = { let state = self.state.lock().await; if let Some(current) = Self::bucket_snapshot_from_state_locked(&state, &table_bucket) { if current != source { @@ -629,10 +1387,16 @@ where return Ok((snapshot_etag, false)); } let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); + let postcondition = StrongSnapshotWritePostcondition::BucketSnapshotPresent(source.clone()); Self::insert_bucket_snapshot_locked(&mut draft_state, source)?; - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + ( + Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version)?, + precondition, + postcondition, + ) }; - self.finalize_snapshot_write(snapshot, precondition).await?; + self.finalize_snapshot_write(snapshot, precondition, postcondition).await?; + self.hydrate_state().await?; let state = self.state.lock().await; let snapshot_etag = state .snapshot_etag @@ -641,14 +1405,14 @@ where Ok((snapshot_etag, true)) } - pub(super) async fn remove_bucket_snapshot_if_unchanged( + pub(in crate::table_catalog) async fn remove_bucket_snapshot_if_unchanged( &self, table_bucket: &str, expected_fingerprint: &str, ) -> TableCatalogStoreResult<()> { let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; - let (snapshot, precondition) = { + let (snapshot, precondition, postcondition) = { let state = self.state.lock().await; let Some(current) = Self::bucket_snapshot_from_state_locked(&state, table_bucket) else { return Ok(()); @@ -660,22 +1424,91 @@ where } let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); Self::remove_bucket_from_state_locked(&mut draft_state, table_bucket); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + ( + Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version)?, + precondition, + StrongSnapshotWritePostcondition::TableBucketAbsent(table_bucket.to_string()), + ) }; - self.finalize_snapshot_write(snapshot, precondition).await + self.finalize_snapshot_write(snapshot, precondition, postcondition).await + } + + pub(super) async fn restore_absent_migration_snapshot_baseline(&self) -> TableCatalogStoreResult<()> { + let _write_guard = self.write_lock.lock().await; + let _reload_guard = self.reload_lock.lock().await; + let snapshot_object = self + .object_backend + .read_object_limited(RUSTFS_META_BUCKET, &Self::snapshot_object_path(), STRONG_TABLE_CATALOG_SNAPSHOT_MAX_SIZE) + .await?; + let snapshot_metadata = self + .object_backend + .object_metadata(RUSTFS_META_BUCKET, &Self::snapshot_object_path()) + .await?; + match (snapshot_object, snapshot_metadata) { + (None, None) => {} + (Some(_), Some(_)) => return Ok(()), + _ => { + return Err(TableCatalogStoreError::Internal( + "durable strong catalog snapshot changed while restoring an absent migration baseline".to_string(), + )); + } + } + + let mut state = self.state.lock().await; + if state.snapshot_etag.is_some() + || state.snapshot_version.is_some() + || !state.table_buckets.is_empty() + || !state.namespaces.is_empty() + || !state.namespace_children.is_empty() + || !state.namespace_objects.is_empty() + || !state.tables.is_empty() + || !state.views.is_empty() + || !state.commits.is_empty() + || !state.idempotency.is_empty() + || !state.warehouse_index.is_empty() + || !state.identifier_collisions.is_empty() + { + return Err(TableCatalogStoreError::Invalid( + "cannot restore an absent migration baseline after durable strong catalog state was observed".to_string(), + )); + } + *state = StrongTableCatalogState { + hydrated: true, + ..StrongTableCatalogState::default() + }; + Ok(()) + } + + pub(super) async fn bucket_snapshot_observation( + &self, + table_bucket: &str, + ) -> TableCatalogStoreResult<(Option, Option)> { + self.hydrate_state().await?; + let state = self.state.lock().await; + let fingerprint = Self::bucket_snapshot_from_state_locked(&state, table_bucket) + .as_ref() + .map(table_catalog_bucket_snapshot_fingerprint) + .transpose()?; + Ok((fingerprint, state.snapshot_etag.clone())) } pub(super) async fn bucket_snapshot_fingerprint(&self, table_bucket: &str) -> TableCatalogStoreResult> { + self.bucket_snapshot_observation(table_bucket) + .await + .map(|(fingerprint, _)| fingerprint) + } + + pub(super) async fn table_bucket_names(&self) -> TableCatalogStoreResult> { self.hydrate_state().await?; - let state = self.state.lock().await; - Self::bucket_snapshot_from_state_locked(&state, table_bucket) - .as_ref() - .map(table_catalog_bucket_snapshot_fingerprint) - .transpose() + Ok(self.state.lock().await.table_buckets.keys().cloned().collect()) } fn require_table_bucket_in_state(state: &StrongTableCatalogState, table_bucket: &str) -> TableCatalogStoreResult<()> { - if !state.table_buckets.contains_key(table_bucket) { + if !state + .table_buckets + .get(table_bucket) + .is_some_and(|entry| entry.state == TableCatalogEntryState::Active) + { return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); } Ok(()) @@ -690,21 +1523,26 @@ where return Ok(()); } let candidate_prefix = table_warehouse_object_prefix(candidate)?; - for (existing_key, existing) in &state.tables { - if existing_key == candidate_key - || existing.table_bucket != candidate.table_bucket - || existing.state != TableCatalogEntryState::Active - { - continue; - } - let Ok(existing_prefix) = table_warehouse_object_prefix(existing) else { - continue; - }; - if warehouse_object_prefixes_overlap(&existing_prefix, &candidate_prefix) { - return Err(TableCatalogStoreError::Conflict(format!( - "table warehouse location overlaps an active table: {candidate_prefix}" - ))); - } + let Some(bucket_index) = state.warehouse_index.get(&candidate.table_bucket) else { + return Ok(()); + }; + let predecessor = bucket_index + .range(..=candidate_prefix.clone()) + .rev() + .find(|(_, existing_key)| *existing_key != candidate_key) + .map(|(prefix, existing_key)| (prefix.as_str(), existing_key)); + let successor = bucket_index + .range(candidate_prefix.clone()..) + .find(|(_, existing_key)| *existing_key != candidate_key) + .map(|(prefix, existing_key)| (prefix.as_str(), existing_key)); + if predecessor + .into_iter() + .chain(successor) + .any(|(existing_prefix, _)| warehouse_object_prefixes_overlap(existing_prefix, &candidate_prefix)) + { + return Err(TableCatalogStoreError::Conflict(format!( + "table warehouse location overlaps an active table: {candidate_prefix}" + ))); } Ok(()) } @@ -713,19 +1551,10 @@ where state: &StrongTableCatalogState, entry: &TableEntry, ) -> TableCommitRecoveryReport { - let history = TableCommitHistoryIndex::new( - entry, - state - .commits - .iter() - .filter(|((table_bucket, table_id, _), _)| table_bucket == &entry.table_bucket && table_id == &entry.table_id) - .map(|(_, commit_log)| commit_log), - ); - let mut commits = state - .commits - .iter() - .filter(|((table_bucket, table_id, _), _)| table_bucket == &entry.table_bucket && table_id == &entry.table_id) - .map(|((_, _, _), commit_log)| { + let history = + TableCommitHistoryIndex::new(entry, Self::table_commits_locked(state, &entry.table_bucket, &entry.table_id)); + let mut commits = Self::table_commits_locked(state, &entry.table_bucket, &entry.table_id) + .map(|commit_log| { let idempotency_commit = commit_log.idempotency_key.as_deref().and_then(|idempotency_key| { state .idempotency @@ -781,12 +1610,19 @@ where namespace: &Namespace, table: &IdentifierSegment, ) -> TableCatalogStoreResult { + Self::ensure_identifier_is_unambiguous_locked(state, key)?; let Some(current) = state.tables.get(key).cloned() else { return Err(TableCatalogStoreError::NotFound(format!( "table {}/{}/{}", request.table_bucket, request.namespace, request.table ))); }; + if current.state != TableCatalogEntryState::Active { + return Err(TableCatalogStoreError::NotFound(format!( + "table {}/{}/{}", + request.table_bucket, request.namespace, request.table + ))); + } let commit_key = Self::commit_key(&request.table_bucket, ¤t.table_id, &request.commit_id); let existing_commit = state.commits.get(&commit_key); let idempotency_key = request @@ -834,13 +1670,7 @@ where { TableCommitHistoryIndex::new( ¤t, - state - .commits - .iter() - .filter(|((table_bucket, table_id, _), _)| { - table_bucket == &request.table_bucket && table_id == ¤t.table_id - }) - .map(|(_, commit_log)| commit_log), + Self::table_commits_locked(state, &request.table_bucket, ¤t.table_id), ) .proves_committed(existing) } else { @@ -878,7 +1708,7 @@ where state: &mut StrongTableCatalogState, request: &TableCommitRequest, current: TableEntry, - ) -> Option { + ) -> Option<(TableCommitResult, bool)> { let commit_key = Self::commit_key(&request.table_bucket, ¤t.table_id, &request.commit_id); let existing = state.commits.get(&commit_key)?.clone(); if !commit_log_matches_request(&existing, request, ¤t.table_id) { @@ -888,17 +1718,8 @@ where && !table_matches_staged_base(¤t, &existing) && !table_matches_committed_log(¤t, &existing) { - TableCommitHistoryIndex::new( - ¤t, - state - .commits - .iter() - .filter(|((table_bucket, table_id, _), _)| { - table_bucket == &request.table_bucket && table_id == ¤t.table_id - }) - .map(|(_, commit_log)| commit_log), - ) - .proves_committed(&existing) + TableCommitHistoryIndex::new(¤t, Self::table_commits_locked(state, &request.table_bucket, ¤t.table_id)) + .proves_committed(&existing) } else { false }; @@ -913,17 +1734,25 @@ where let mut committed = existing; committed.status = CommitLogStatus::Committed; - state.commits.insert(commit_key, committed.clone()); - if let Some(idempotency_key) = committed.idempotency_key.as_deref() { - state.idempotency.insert( - Self::idempotency_key(&request.table_bucket, ¤t.table_id, idempotency_key), - committed.clone(), - ); + let commit_changed = state.commits.get(&commit_key) != Some(&committed); + if commit_changed { + state.commits.insert(commit_key, committed.clone()); } - Some(TableCommitResult { - table: current, - commit_log: committed, - }) + let mut idempotency_changed = false; + if let Some(idempotency_key) = committed.idempotency_key.as_deref() { + let idempotency_key = Self::idempotency_key(&request.table_bucket, ¤t.table_id, idempotency_key); + idempotency_changed = state.idempotency.get(&idempotency_key) != Some(&committed); + if idempotency_changed { + state.idempotency.insert(idempotency_key, committed.clone()); + } + } + Some(( + TableCommitResult { + table: current, + commit_log: committed, + }, + commit_changed || idempotency_changed, + )) } fn apply_commit_locked( @@ -935,7 +1764,7 @@ where ) -> TableCatalogStoreResult { let key = Self::table_key(&request.table_bucket, namespace, table); let current = Self::validate_new_table_commit_locked(state, &key, request, namespace, table)?; - if let Some(result) = Self::committed_existing_result_locked(state, request, current.clone()) { + if let Some((result, _)) = Self::committed_existing_result_locked(state, request, current.clone()) { return Ok(result); } @@ -1013,32 +1842,32 @@ where } async fn put_table_bucket(&self, entry: TableBucketEntry) -> TableCatalogStoreResult<()> { + let _migration_guard = self.acquire_snapshot_write_permit().await?; let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; - validate_catalog_entry_version("table bucket", entry.version)?; - if entry.table_bucket.is_empty() { - return Err(TableCatalogStoreError::Invalid("table bucket name cannot be empty".to_string())); - } - if entry.catalog_type != TABLE_BUCKET_CATALOG_TYPE { - return Err(TableCatalogStoreError::Invalid("unsupported table bucket catalog type".to_string())); - } + validate_table_bucket_entry(&entry)?; - let (snapshot, precondition) = { + let (snapshot, precondition, postcondition) = { let state = self.state.lock().await; let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.table_buckets.insert(entry.table_bucket.clone(), entry); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + draft_state.table_buckets.insert(entry.table_bucket.clone(), entry.clone()); + ( + Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version)?, + precondition, + StrongSnapshotWritePostcondition::TableBucketPresent(entry), + ) }; - self.finalize_snapshot_write(snapshot, precondition).await + self.finalize_snapshot_write(snapshot, precondition, postcondition).await } async fn create_namespace(&self, entry: NamespaceEntry) -> TableCatalogStoreResult<()> { + let _migration_guard = self.acquire_snapshot_write_permit().await?; let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; let namespace = validate_namespace_entry_identity(&entry)?; validate_namespace_properties(&entry.properties)?; let key = Self::namespace_key(&entry.table_bucket, &namespace); - let (snapshot, precondition) = { + let (snapshot, precondition, postcondition) = { let state = self.state.lock().await; Self::require_table_bucket_in_state(&state, &entry.table_bucket)?; let existing = state.namespaces.get(&key); @@ -1051,10 +1880,14 @@ where ))); } let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.namespaces.insert(key, entry); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + draft_state.namespaces.insert(key, entry.clone()); + ( + Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version)?, + precondition, + StrongSnapshotWritePostcondition::NamespacePresent(entry), + ) }; - self.finalize_snapshot_write(snapshot, precondition).await + self.finalize_snapshot_write(snapshot, precondition, postcondition).await } async fn list_namespaces(&self, table_bucket: &str) -> TableCatalogStoreResult> { @@ -1181,11 +2014,12 @@ where namespace: &str, update: NamespacePropertiesUpdate, ) -> TableCatalogStoreResult { + let _migration_guard = self.acquire_snapshot_write_permit().await?; let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; let namespace = parse_namespace_for_store(namespace)?; let key = Self::namespace_key(table_bucket, &namespace); - let (snapshot, precondition, result) = { + let (snapshot, precondition, result, postcondition) = { let state = self.state.lock().await; let current = state .namespaces @@ -1211,19 +2045,25 @@ where return Ok(result); } let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.namespaces.insert(key, next); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition, result) + draft_state.namespaces.insert(key, next.clone()); + ( + Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version)?, + precondition, + result, + StrongSnapshotWritePostcondition::NamespacePresent(next), + ) }; - self.finalize_snapshot_write(snapshot, precondition).await?; + self.finalize_snapshot_write(snapshot, precondition, postcondition).await?; Ok(result) } async fn drop_namespace(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult<()> { + let _migration_guard = self.acquire_snapshot_write_permit().await?; let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; let namespace = parse_namespace_for_store(namespace)?; let key = Self::namespace_key(table_bucket, &namespace); - let (snapshot, precondition) = { + let (snapshot, precondition, postcondition) = { let state = self.state.lock().await; let parent = namespace.public_name(); if Self::has_active_namespace_descendant_locked(&state, table_bucket, &parent) { @@ -1264,9 +2104,13 @@ where } let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); draft_state.namespaces.remove(&key); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + ( + Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version)?, + precondition, + StrongSnapshotWritePostcondition::NamespaceAbsent(key), + ) }; - self.finalize_snapshot_write(snapshot, precondition).await + self.finalize_snapshot_write(snapshot, precondition, postcondition).await } async fn create_table(&self, entry: TableEntry) -> TableCatalogStoreResult<()> { @@ -1283,10 +2127,15 @@ where entry: TableEntry, publication: &(dyn TableCommitPublication + Sync), ) -> TableCatalogStoreResult<()> { - validate_catalog_entry_version("table", entry.version)?; + validate_table_entry_version_and_id(&entry)?; let namespace = parse_namespace_for_store(&entry.namespace)?; let table = parse_table_for_store(&entry.table)?; table_warehouse_object_prefix(&entry)?; + if !is_valid_table_metadata_location(&namespace, &table, &entry.metadata_location) { + return Err(TableCatalogStoreError::Invalid( + "table metadata location must be inside the table metadata directory".to_string(), + )); + } publication.begin_table_bucket(&entry.table_bucket).await?; if !publication.holds_table_bucket(&entry.table_bucket) { return Err(TableCatalogStoreError::Internal( @@ -1294,6 +2143,7 @@ where )); } let _publication_completion = TableCommitPublicationCompletion::new(publication); + let _migration_guard = self.acquire_snapshot_write_permit().await?; publication .prepare(&entry.table_bucket, &entry.namespace, &entry.table) .await?; @@ -1305,32 +2155,49 @@ where let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; let key = Self::table_key(&entry.table_bucket, &namespace, &table); - let (snapshot, precondition) = { + let (snapshot, precondition, postcondition) = { let state = self.state.lock().await; Self::require_table_bucket_in_state(&state, &entry.table_bucket)?; Self::require_active_namespace_locked(&state, &entry.table_bucket, &namespace)?; - if state.tables.contains_key(&key) { + if state.tables.contains_key(&key) || state.views.contains_key(&key) { return Err(TableCatalogStoreError::Conflict(format!( "catalog object already exists: table {}/{}/{}", entry.table_bucket, entry.namespace, entry.table ))); } + if state + .tables + .values() + .any(|existing| existing.table_bucket == entry.table_bucket && existing.table_id == entry.table_id) + { + return Err(TableCatalogStoreError::Conflict( + "table id is already registered in this table bucket".to_string(), + )); + } Self::ensure_table_warehouse_prefix_available_locked(&state, &entry, &key)?; let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.tables.insert(key, entry); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + draft_state.tables.insert(key, entry.clone()); + ( + Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version)?, + precondition, + StrongSnapshotWritePostcondition::TablePresent(entry), + ) }; - self.finalize_snapshot_write(snapshot, precondition).await + self.finalize_snapshot_write(snapshot, precondition, postcondition).await } async fn list_tables(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { self.hydrate_state().await?; let namespace = parse_namespace_for_store(namespace)?; let state = self.state.lock().await; + let namespace_name = namespace.public_name(); + Self::ensure_namespace_identifiers_are_unambiguous_locked(&state, table_bucket, &namespace_name)?; let mut entries = state .tables .iter() - .filter(|((bucket, namespace_name, _), _)| bucket == table_bucket && namespace_name == &namespace.public_name()) + .filter(|((bucket, entry_namespace, _), entry)| { + bucket == table_bucket && entry_namespace == &namespace_name && entry.state == TableCatalogEntryState::Active + }) .map(|(_, entry)| entry.clone()) .collect::>(); entries.sort_by(|left, right| left.table.cmp(&right.table)); @@ -1340,10 +2207,12 @@ where async fn list_all_tables(&self, table_bucket: &str) -> TableCatalogStoreResult> { self.hydrate_state().await?; let state = self.state.lock().await; + Self::ensure_table_bucket_identifiers_are_unambiguous_locked(&state, table_bucket)?; Ok(state .tables .range((table_bucket.to_string(), String::new(), String::new())..) .take_while(|((bucket, _, _), _)| bucket == table_bucket) + .filter(|(_, entry)| entry.state == TableCatalogEntryState::Active) .map(|(_, entry)| entry.clone()) .collect()) } @@ -1367,10 +2236,12 @@ where None => Bound::Included((table_bucket.to_string(), namespace.clone(), String::new())), }; let state = self.state.lock().await; + Self::ensure_namespace_identifiers_are_unambiguous_locked(&state, table_bucket, &namespace)?; let entries = state .tables .range((start, Bound::Unbounded)) .take_while(|((bucket, entry_namespace, _), _)| bucket == table_bucket && entry_namespace == &namespace) + .filter(|(_, entry)| entry.state == TableCatalogEntryState::Active) .take(limit.get().saturating_add(1)) .map(|(_, entry)| entry.clone()) .collect(); @@ -1384,7 +2255,13 @@ where let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; let state = self.state.lock().await; - Ok(state.tables.get(&Self::table_key(table_bucket, &namespace, &table)).cloned()) + let key = Self::table_key(table_bucket, &namespace, &table); + Self::ensure_identifier_is_unambiguous_locked(&state, &key)?; + Ok(state + .tables + .get(&key) + .filter(|entry| entry.state == TableCatalogEntryState::Active) + .cloned()) } async fn resolve_table_data_plane_resource( @@ -1399,10 +2276,23 @@ where self.hydrate_state().await?; let state = self.state.lock().await; let Some(bucket_entry) = state.table_buckets.get(table_bucket) else { - return Ok(None); + return Err(TableCatalogStoreError::Internal(format!( + "durable strong catalog has no entry for table-enabled bucket {table_bucket}" + ))); }; if bucket_entry.state != TableCatalogEntryState::Active { - return Ok(None); + return Err(TableCatalogStoreError::Internal(format!( + "table-enabled bucket {table_bucket} has an inactive durable strong catalog entry" + ))); + } + if self.snapshot_write_version >= STRONG_TABLE_CATALOG_SNAPSHOT_VERSION + && state + .snapshot_version + .is_none_or(|version| version < STRONG_TABLE_CATALOG_SNAPSHOT_VERSION) + { + return Err(TableCatalogStoreError::Internal( + "durable strong catalog data-plane access requires a version 2 snapshot after fleet confirmation".to_string(), + )); } let Some(bucket_index) = state.warehouse_index.get(table_bucket) else { @@ -1411,6 +2301,7 @@ where for warehouse_object_prefix in warehouse_index_candidate_prefixes(object) { if let Some(table_key) = bucket_index.get(warehouse_object_prefix) { + Self::ensure_identifier_is_unambiguous_locked(&state, table_key)?; let Some(table) = state.tables.get(table_key) else { continue; }; @@ -1434,6 +2325,7 @@ where request: TableCommitRequest, publication: &(dyn TableCommitPublication + Sync), ) -> TableCatalogStoreResult { + let _migration_guard = self.acquire_snapshot_write_permit().await?; let commit_started = Instant::now(); record_table_commit_attempt(&request.operation); let namespace = parse_namespace_for_store(&request.namespace)?; @@ -1447,7 +2339,7 @@ where )); } let _publication_completion = TableCommitPublicationCompletion::new(publication); - let _write_guard = self.write_lock.lock().await; + let write_guard = self.write_lock.lock().await; self.hydrate_state().await?; let key = Self::table_key(&request.table_bucket, &namespace, &table); @@ -1457,9 +2349,13 @@ where match current { Ok(current) => { let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - Self::committed_existing_result_locked(&mut draft_state, &request, current).map(|result| { - Self::snapshot_from_mutated_state_locked(&mut draft_state) - .map(|snapshot| (result, snapshot, precondition)) + Self::committed_existing_result_locked(&mut draft_state, &request, current).map(|(result, state_changed)| { + if state_changed { + Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version) + .map(|snapshot| (result, Some((snapshot, precondition)))) + } else { + Ok((result, None)) + } }) } Err(error) => { @@ -1477,9 +2373,13 @@ where }; if let Some(prepared_result) = committed_existing_result { let result = match prepared_result { - Ok((result, snapshot, precondition)) => { - self.finalize_snapshot_write(snapshot, precondition).await.map(|_| result) + Ok((result, Some((snapshot, precondition)))) => { + let postcondition = Self::commit_write_postcondition(&request.table_bucket, &result.commit_log); + self.finalize_snapshot_write(snapshot, precondition, postcondition) + .await + .map(|_| result) } + Ok((result, None)) => Ok(result), Err(err) => Err(err), }; return table_commit_result( @@ -1492,6 +2392,7 @@ where result, ); } + drop(write_guard); let Some(new_metadata_object) = self .object_backend @@ -1519,6 +2420,8 @@ where }) .await .map_err(|err| TableCatalogStoreError::Internal(format!("table metadata parser task failed: {err}")))??; + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; let current_warehouse_location = { let state = self.state.lock().await; state @@ -1555,14 +2458,31 @@ where let state = self.state.lock().await; let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); match Self::apply_commit_locked(&mut draft_state, &request, &namespace, &table, next_warehouse_location) { - Ok(result) => { - Self::snapshot_from_mutated_state_locked(&mut draft_state).map(|snapshot| (result, snapshot, precondition)) - } + Ok(result) => Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version) + .map(|snapshot| (result, snapshot, precondition)), Err(err) => Err(err), } }; let result = match prepared_result { - Ok((result, snapshot, precondition)) => self.finalize_snapshot_write(snapshot, precondition).await.map(|_| result), + Ok((result, snapshot, precondition)) => { + let postcondition = Self::commit_write_postcondition(&request.table_bucket, &result.commit_log); + match self.finalize_snapshot_write(snapshot, precondition, postcondition).await { + Ok(()) => Ok(result), + Err(err) => { + let replay = { + let state = self.state.lock().await; + let mut replay_state = state.clone(); + state + .tables + .get(&key) + .cloned() + .and_then(|current| Self::committed_existing_result_locked(&mut replay_state, &request, current)) + .and_then(|(result, state_changed)| (!state_changed).then_some(result)) + }; + replay.ok_or(err) + } + } + } Err(err) => Err(err), }; let cas_result = result.as_ref().map(|_| ()).map_err(Clone::clone); @@ -1581,6 +2501,7 @@ where async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()> { let publication = TableCommitLockPublication::new(&self.object_backend); publication.begin_table_bucket(table_bucket).await?; + let _migration_guard = self.acquire_snapshot_write_permit().await?; publication.prepare(table_bucket, namespace, table).await?; if !publication.holds_table_bucket(table_bucket) || !publication.holds_table(table_bucket, namespace, table) { return Err(TableCatalogStoreError::Internal( @@ -1593,7 +2514,7 @@ where let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; let key = Self::table_key(table_bucket, &namespace, &table); - let (snapshot, precondition) = { + let (snapshot, precondition, postcondition) = { let state = self.state.lock().await; if !state.tables.contains_key(&key) { return Err(TableCatalogStoreError::NotFound(format!( @@ -1604,45 +2525,74 @@ where ))); } let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.tables.remove(&key); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + let removed = draft_state.tables.remove(&key).ok_or_else(|| { + TableCatalogStoreError::Internal("table disappeared while preparing the strong catalog snapshot".to_string()) + })?; + draft_state + .commits + .retain(|(entry_bucket, table_id, _), _| entry_bucket != table_bucket || table_id != &removed.table_id); + draft_state + .idempotency + .retain(|(entry_bucket, table_id, _), _| entry_bucket != table_bucket || table_id != &removed.table_id); + ( + Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version)?, + precondition, + StrongSnapshotWritePostcondition::TableAbsent { + key, + table_id: removed.table_id, + }, + ) }; - self.finalize_snapshot_write(snapshot, precondition).await + self.finalize_snapshot_write(snapshot, precondition, postcondition).await } async fn create_view(&self, entry: ViewEntry) -> TableCatalogStoreResult<()> { + let _migration_guard = self.acquire_snapshot_write_permit().await?; let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; - validate_catalog_entry_version("view", entry.version)?; + validate_view_entry_version_and_id(&entry)?; validate_view_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?; let namespace = parse_namespace_for_store(&entry.namespace)?; let view = parse_table_for_store(&entry.view)?; + if !is_valid_view_metadata_location(&namespace, &view, &entry.metadata_location) { + return Err(TableCatalogStoreError::Invalid( + "view metadata location must be inside the view metadata directory".to_string(), + )); + } let key = Self::table_key(&entry.table_bucket, &namespace, &view); - let (snapshot, precondition) = { + let (snapshot, precondition, postcondition) = { let state = self.state.lock().await; Self::require_table_bucket_in_state(&state, &entry.table_bucket)?; Self::require_active_namespace_locked(&state, &entry.table_bucket, &namespace)?; - if state.views.contains_key(&key) { + if state.views.contains_key(&key) || state.tables.contains_key(&key) { return Err(TableCatalogStoreError::Conflict(format!( "catalog object already exists: view {}/{}/{}", entry.table_bucket, entry.namespace, entry.view ))); } let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.views.insert(key, entry); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + draft_state.views.insert(key, entry.clone()); + ( + Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version)?, + precondition, + StrongSnapshotWritePostcondition::ViewPresent(entry), + ) }; - self.finalize_snapshot_write(snapshot, precondition).await + self.finalize_snapshot_write(snapshot, precondition, postcondition).await } async fn list_views(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { self.hydrate_state().await?; let namespace = parse_namespace_for_store(namespace)?; let state = self.state.lock().await; + let namespace_name = namespace.public_name(); + Self::ensure_namespace_identifiers_are_unambiguous_locked(&state, table_bucket, &namespace_name)?; let mut entries = state .views .iter() - .filter(|((bucket, namespace_name, _), _)| bucket == table_bucket && namespace_name == &namespace.public_name()) + .filter(|((bucket, entry_namespace, _), entry)| { + bucket == table_bucket && entry_namespace == &namespace_name && entry.state == TableCatalogEntryState::Active + }) .map(|(_, entry)| entry.clone()) .collect::>(); entries.sort_by(|left, right| left.view.cmp(&right.view)); @@ -1668,10 +2618,12 @@ where None => Bound::Included((table_bucket.to_string(), namespace.clone(), String::new())), }; let state = self.state.lock().await; + Self::ensure_namespace_identifiers_are_unambiguous_locked(&state, table_bucket, &namespace)?; let entries = state .views .range((start, Bound::Unbounded)) .take_while(|((bucket, entry_namespace, _), _)| bucket == table_bucket && entry_namespace == &namespace) + .filter(|(_, entry)| entry.state == TableCatalogEntryState::Active) .take(limit.get().saturating_add(1)) .map(|(_, entry)| entry.clone()) .collect(); @@ -1685,14 +2637,38 @@ where let namespace = parse_namespace_for_store(namespace)?; let view = parse_table_for_store(view)?; let state = self.state.lock().await; - Ok(state.views.get(&Self::table_key(table_bucket, &namespace, &view)).cloned()) + let key = Self::table_key(table_bucket, &namespace, &view); + Self::ensure_identifier_is_unambiguous_locked(&state, &key)?; + Ok(state + .views + .get(&key) + .filter(|entry| entry.state == TableCatalogEntryState::Active) + .cloned()) } async fn replace_view(&self, request: ViewCommitRequest) -> TableCatalogStoreResult { - let _write_guard = self.write_lock.lock().await; + let _migration_guard = self.acquire_snapshot_write_permit().await?; + let write_guard = self.write_lock.lock().await; self.hydrate_state().await?; let namespace = parse_namespace_for_store(&request.namespace)?; let view = parse_table_for_store(&request.view)?; + let key = Self::table_key(&request.table_bucket, &namespace, &view); + let expected_view_id = { + let state = self.state.lock().await; + Self::ensure_identifier_is_unambiguous_locked(&state, &key)?; + let Some(current) = state + .views + .get(&key) + .filter(|entry| entry.state == TableCatalogEntryState::Active) + else { + return Err(TableCatalogStoreError::NotFound(format!( + "view {}/{}/{}", + request.table_bucket, request.namespace, request.view + ))); + }; + current.view_id.clone() + }; + drop(write_guard); if !is_valid_view_metadata_location(&namespace, &view, &request.new_metadata_location) { return Err(TableCatalogStoreError::Invalid( "new metadata location must be inside the view metadata directory".to_string(), @@ -1716,15 +2692,28 @@ where .await .map_err(|err| TableCatalogStoreError::Internal(format!("view metadata parser task failed: {err}")))??; - let key = Self::table_key(&request.table_bucket, &namespace, &view); - let (snapshot, precondition, next) = { + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; + let (snapshot, precondition, next, postcondition) = { let state = self.state.lock().await; + Self::ensure_identifier_is_unambiguous_locked(&state, &key)?; let Some(current) = state.views.get(&key).cloned() else { return Err(TableCatalogStoreError::NotFound(format!( "view {}/{}/{}", request.table_bucket, request.namespace, request.view ))); }; + if current.state != TableCatalogEntryState::Active { + return Err(TableCatalogStoreError::NotFound(format!( + "view {}/{}/{}", + request.table_bucket, request.namespace, request.view + ))); + } + if current.view_id != expected_view_id { + return Err(TableCatalogStoreError::Conflict( + "current view identity changed while metadata was being validated".to_string(), + )); + } if current.version_token != request.expected_version_token { return Err(TableCatalogStoreError::Conflict( "current view version token does not match expected token".to_string(), @@ -1745,19 +2734,25 @@ where next.generation = next.generation.saturating_add(1); let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); draft_state.views.insert(key, next.clone()); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition, next) + ( + Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version)?, + precondition, + next.clone(), + StrongSnapshotWritePostcondition::ViewPresent(next), + ) }; - self.finalize_snapshot_write(snapshot, precondition).await?; + self.finalize_snapshot_write(snapshot, precondition, postcondition).await?; Ok(ViewCommitResult { view: next }) } async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()> { + let _migration_guard = self.acquire_snapshot_write_permit().await?; let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; let namespace = parse_namespace_for_store(namespace)?; let view = parse_table_for_store(view)?; let key = Self::table_key(table_bucket, &namespace, &view); - let (snapshot, precondition) = { + let (snapshot, precondition, postcondition) = { let state = self.state.lock().await; if !state.views.contains_key(&key) { return Err(TableCatalogStoreError::NotFound(format!( @@ -1768,10 +2763,19 @@ where ))); } let (precondition, mut draft_state) = Self::snapshot_draft_context_locked(&state); - draft_state.views.remove(&key); - (Self::snapshot_from_mutated_state_locked(&mut draft_state)?, precondition) + let removed = draft_state.views.remove(&key).ok_or_else(|| { + TableCatalogStoreError::Internal("view disappeared while preparing the strong catalog snapshot".to_string()) + })?; + ( + Self::snapshot_from_mutated_state_locked(&mut draft_state, self.snapshot_write_version)?, + precondition, + StrongSnapshotWritePostcondition::ViewAbsent { + key, + view_id: removed.view_id, + }, + ) }; - self.finalize_snapshot_write(snapshot, precondition).await + self.finalize_snapshot_write(snapshot, precondition, postcondition).await } async fn get_commit_by_id( diff --git a/rustfs/src/table_catalog/tests.rs b/rustfs/src/table_catalog/tests.rs index 997fa49ab..1074add1a 100644 --- a/rustfs/src/table_catalog/tests.rs +++ b/rustfs/src/table_catalog/tests.rs @@ -15,6 +15,8 @@ use datafusion::{ use std::assert_matches; use std::sync::Arc; +const TABLE_CATALOG_TEST_TIMEOUT: StdDuration = StdDuration::from_secs(30); + #[test] fn reserved_table_object_key_matches_exact_prefix_and_children_only() { assert!(is_reserved_table_object_key(".rustfs-table")); @@ -194,6 +196,31 @@ fn table_catalog_backing_mode_rejects_unknown_value() { }); } +#[test] +fn catalog_object_listing_rejects_missing_or_stalled_continuation_tokens() { + let mut seen = BTreeSet::new(); + assert_eq!( + catalog_list_next_continuation(&mut seen, false, None).expect("complete listings need no token"), + None + ); + assert_eq!( + catalog_list_next_continuation(&mut seen, true, Some("next".to_string())).expect("truncated listing should advance"), + Some("next".to_string()) + ); + assert_matches!( + catalog_list_next_continuation(&mut seen, true, None), + Err(TableCatalogStoreError::Internal(message)) if message.contains("no continuation token") + ); + assert_matches!( + catalog_list_next_continuation(&mut seen, true, Some(String::new())), + Err(TableCatalogStoreError::Internal(message)) if message.contains("no continuation token") + ); + assert_matches!( + catalog_list_next_continuation(&mut seen, true, Some("next".to_string())), + Err(TableCatalogStoreError::Internal(message)) if message.contains("did not advance") + ); +} + struct NoopTableCatalogStore; #[async_trait::async_trait] @@ -482,7 +509,7 @@ where acquisitions_before + 1 ); let guard = tokio::time::timeout( - StdDuration::from_secs(1), + TABLE_CATALOG_TEST_TIMEOUT, TableCatalogObjectBackend::acquire_write_lock(backend, bucket, &publication_lock), ) .await @@ -502,6 +529,108 @@ async fn catalog_backings_fence_direct_commits_with_publication_lock() { assert_direct_commit_uses_publication_lock(&strong_store, &strong_backend).await; } +#[tokio::test] +async fn strong_table_registration_and_drop_acquire_publication_before_migration_read_lock() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + + let publication_lock = default_table_bucket_publication_lock_path(); + let migration_lock = TableCatalogObjectPaths::default().backing_migration_global_fence_lock_path(); + let publication_guard = backend + .acquire_write_lock(bucket, &publication_lock) + .await + .expect("publication lock should be acquired"); + let publication_attempts = backend.write_lock_acquisition_count(bucket, &publication_lock).await; + let migration_reads = backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock).await; + let register_store = store.clone(); + let register = tokio::spawn(async move { + register_store + .register_table(test_table_entry(bucket, &namespace, &table, metadata_location)) + .await + }); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { + while backend.write_lock_acquisition_count(bucket, &publication_lock).await == publication_attempts { + tokio::task::yield_now().await; + } + }) + .await + .expect("registration should wait on the table-bucket publication lock"); + assert_eq!( + backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock).await, + migration_reads, + "registration must not retain a migration read lock while waiting for publication" + ); + let migration_guard = tokio::time::timeout( + TABLE_CATALOG_TEST_TIMEOUT, + backend.acquire_write_lock(RUSTFS_META_BUCKET, &migration_lock), + ) + .await + .expect("migration writer must not be blocked by registration waiting on publication") + .expect("migration write lock should be acquired"); + drop(publication_guard); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { + while backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock).await == migration_reads { + tokio::task::yield_now().await; + } + }) + .await + .expect("registration should request the migration read lock after publication"); + assert!(!register.is_finished()); + drop(migration_guard); + register + .await + .expect("registration task should join") + .expect("registration should succeed"); + + let publication_guard = backend + .acquire_read_lock(bucket, &publication_lock) + .await + .expect("publication reader should be acquired"); + let publication_attempts = backend.write_lock_acquisition_count(bucket, &publication_lock).await; + let migration_reads = backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock).await; + let drop_store = store.clone(); + let drop_task = tokio::spawn(async move { drop_store.drop_table(bucket, "sales", "orders").await }); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { + while backend.write_lock_acquisition_count(bucket, &publication_lock).await == publication_attempts { + tokio::task::yield_now().await; + } + }) + .await + .expect("drop should wait on the table-bucket publication lock"); + assert_eq!( + backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock).await, + migration_reads, + "drop must not retain a migration read lock while waiting for publication" + ); + let migration_guard = tokio::time::timeout( + TABLE_CATALOG_TEST_TIMEOUT, + backend.acquire_write_lock(RUSTFS_META_BUCKET, &migration_lock), + ) + .await + .expect("migration writer must not be blocked by drop waiting on publication") + .expect("migration write lock should be acquired"); + drop(publication_guard); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { + while backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock).await == migration_reads { + tokio::task::yield_now().await; + } + }) + .await + .expect("drop should request the migration read lock after publication"); + assert!(!drop_task.is_finished()); + drop(migration_guard); + drop_task.await.expect("drop task should join").expect("drop should succeed"); +} + async fn assert_direct_drop_uses_publication_locks(store: S, backend: &TestCatalogObjectBackend) where S: TableCatalogStore + Clone + Send + Sync + 'static, @@ -538,7 +667,7 @@ where let drop_store = store.clone(); let drop_task = tokio::spawn(async move { drop_store.drop_table(bucket, "sales", "orders").await }); - tokio::time::timeout(StdDuration::from_secs(1), async { + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { while backend.write_lock_acquisition_count(bucket, &bucket_lock).await == bucket_attempts { tokio::task::yield_now().await; } @@ -548,7 +677,7 @@ where assert!(!drop_task.is_finished(), "drop must wait for an in-flight bucket writer"); drop(bucket_guard); - tokio::time::timeout(StdDuration::from_secs(1), async { + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { while backend.write_lock_acquisition_count(bucket, &table_lock).await == table_attempts { tokio::task::yield_now().await; } @@ -558,7 +687,7 @@ where assert!(!drop_task.is_finished(), "drop must wait for an in-flight table writer"); drop(table_guard); - tokio::time::timeout(StdDuration::from_secs(1), drop_task) + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, drop_task) .await .expect("drop should continue after publication readers finish") .expect("drop task should join") @@ -638,13 +767,13 @@ async fn strong_catalog_blocked_publication_object_does_not_stall_unrelated_writ }); publication.wait_started().await; - tokio::time::timeout(StdDuration::from_secs(5), store.put_table_bucket(test_bucket_entry("independent"))) + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, store.put_table_bucket(test_bucket_entry("independent"))) .await .expect("a blocked table publication must not hold the strong catalog write lock") .expect("the unrelated table bucket write should succeed"); drop(blocker); - tokio::time::timeout(StdDuration::from_secs(5), commit) + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, commit) .await .expect("the blocked commit should finish after its object lock is released") .expect("the commit task should join") @@ -671,6 +800,234 @@ async fn strong_catalog_blocked_publication_object_does_not_stall_unrelated_writ ); } +#[tokio::test] +async fn strong_catalog_metadata_read_does_not_stall_unrelated_writes() { + let backend = TestCatalogObjectBackend::default(); + let store = Arc::new(StrongTableCatalogStore::new(backend.clone())); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .expect("table should be created"); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + let metadata_read = backend.pause_next_read(bucket, &new_metadata).await; + + let commit_store = Arc::clone(&store); + let commit = tokio::spawn(async move { + commit_store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "slow-metadata-read".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: Some("concurrency-test".to_string()), + }) + .await + }); + metadata_read.wait_started().await; + + let independent = Namespace::parse("independent").expect("namespace should parse"); + tokio::time::timeout( + TABLE_CATALOG_TEST_TIMEOUT, + store.create_namespace(test_namespace_entry(bucket, &independent)), + ) + .await + .expect("a slow metadata read must not hold the strong catalog write lock") + .expect("the unrelated namespace write should succeed"); + + metadata_read.release(); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, commit) + .await + .expect("the commit should finish after the metadata read is released") + .expect("the commit task should join") + .expect("the commit should succeed"); + assert!( + store + .get_namespace(bucket, &independent.public_name()) + .await + .expect("unrelated namespace lookup should succeed") + .is_some() + ); +} + +#[tokio::test] +async fn strong_catalog_view_metadata_read_does_not_stall_unrelated_writes() { + let backend = TestCatalogObjectBackend::default(); + let store = Arc::new(StrongTableCatalogStore::new(backend.clone())); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); + let current_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + let new_metadata = default_view_metadata_file_path(&namespace, &view, "00002.metadata.json"); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_view(test_view_entry(bucket, &namespace, &view, current_metadata.clone())) + .await + .expect("view should be created"); + backend + .seed_object( + bucket, + &new_metadata, + serde_json::to_vec(&serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": "s3://analytics/views/view-id" + })) + .expect("view metadata should encode"), + ) + .await; + let metadata_read = backend.pause_next_read(bucket, &new_metadata).await; + + let replace_store = Arc::clone(&store); + let replace = tokio::spawn(async move { + replace_store + .replace_view(ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + }) + .await + }); + metadata_read.wait_started().await; + + let independent = Namespace::parse("independent").expect("namespace should parse"); + tokio::time::timeout( + TABLE_CATALOG_TEST_TIMEOUT, + store.create_namespace(test_namespace_entry(bucket, &independent)), + ) + .await + .expect("a slow view metadata read must not hold the strong catalog write lock") + .expect("the unrelated namespace write should succeed"); + + metadata_read.release(); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, replace) + .await + .expect("the view replacement should finish after the metadata read is released") + .expect("the view replacement task should join") + .expect("the view replacement should succeed"); + assert!( + store + .get_namespace(bucket, &independent.public_name()) + .await + .expect("unrelated namespace lookup should succeed") + .is_some() + ); +} + +#[tokio::test] +async fn strong_catalog_view_replace_rejects_identity_recreation_during_metadata_read() { + let backend = TestCatalogObjectBackend::default(); + let store = Arc::new(StrongTableCatalogStore::new(backend.clone())); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); + let current_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + let new_metadata = default_view_metadata_file_path(&namespace, &view, "00002.metadata.json"); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_view(test_view_entry(bucket, &namespace, &view, current_metadata.clone())) + .await + .expect("view should be created"); + backend + .seed_object( + bucket, + &new_metadata, + serde_json::to_vec(&serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": "s3://analytics/views/view-id" + })) + .expect("view metadata should encode"), + ) + .await; + let metadata_read = backend.pause_next_read(bucket, &new_metadata).await; + + let replace_store = Arc::clone(&store); + let replace_namespace = namespace.clone(); + let replace_view = view.clone(); + let replace_current_metadata = current_metadata.clone(); + let replace = tokio::spawn(async move { + replace_store + .replace_view(ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: replace_namespace.public_name(), + view: replace_view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: replace_current_metadata, + new_metadata_location: new_metadata, + }) + .await + }); + metadata_read.wait_started().await; + + store + .drop_view(bucket, &namespace.public_name(), view.as_str()) + .await + .expect("original view should be dropped"); + let mut recreated = test_view_entry(bucket, &namespace, &view, current_metadata); + recreated.view_id = "replacement-view-id".to_string(); + recreated.view_uuid = "replacement-view-uuid".to_string(); + recreated.warehouse_location = format!("s3://{bucket}/views/replacement-view-id"); + store + .create_view(recreated.clone()) + .await + .expect("replacement view should be created"); + + metadata_read.release(); + assert_matches!( + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, replace) + .await + .expect("the view replacement should finish after the metadata read is released") + .expect("the view replacement task should join"), + Err(TableCatalogStoreError::Conflict(message)) if message.contains("identity changed") + ); + assert_eq!( + store + .load_view(bucket, &namespace.public_name(), view.as_str()) + .await + .expect("replacement view lookup should succeed"), + Some(recreated) + ); +} + #[test] fn catalog_object_entry_paths_use_internal_root_and_hashed_untrusted_ids() { let paths = TableCatalogObjectPaths::default(); @@ -724,19 +1081,20 @@ fn catalog_object_entry_paths_use_internal_root_and_hashed_untrusted_ids() { struct TestCatalogObjectBackend { state: Arc>, locks: TestCatalogObjectLocks, + strong_runtime: Option, } type TestCatalogObjectLockKey = (String, String); -type TestCatalogObjectLock = Arc>; +type TestCatalogObjectLock = Arc>; type TestCatalogObjectLocks = Arc>>; #[derive(Clone, Default)] -struct TestCatalogObjectPutPause { +struct TestCatalogObjectPause { started: Arc, release: Arc, } -impl TestCatalogObjectPutPause { +impl TestCatalogObjectPause { async fn wait_started(&self) { self.started.notified().await; } @@ -769,6 +1127,30 @@ impl BlockingObjectPublication { } } +#[derive(Default)] +struct UnserializedTestPublication; + +#[async_trait::async_trait] +impl TableCommitPublication for UnserializedTestPublication { + async fn begin_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn prepare(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + fn holds_table_bucket(&self, _table_bucket: &str) -> bool { + true + } + + fn holds_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> bool { + true + } + + fn complete(&self) {} +} + #[async_trait::async_trait] impl TableCommitPublication for BlockingObjectPublication { async fn begin_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> { @@ -798,14 +1180,21 @@ impl TableCommitPublication for BlockingObjectPublication { #[derive(Default)] struct TestCatalogObjectState { objects: BTreeMap<(String, String), TestCatalogObjectRecord>, + etagless_objects: BTreeSet<(String, String)>, fail_read_attempts: BTreeMap<(String, String), BTreeSet>, + pause_before_read_attempts: BTreeMap<(String, String), BTreeMap>, + pause_read_attempts: BTreeMap<(String, String), BTreeMap>, read_attempts: BTreeMap<(String, String), usize>, + read_limits: Vec<((String, String), usize)>, fail_put_attempts: BTreeMap<(String, String), BTreeSet>, - pause_put_attempts: BTreeMap<(String, String), BTreeMap>, + fail_after_put_attempts: BTreeMap<(String, String), BTreeSet>, + pause_put_attempts: BTreeMap<(String, String), BTreeMap>, fail_delete_attempts: BTreeMap<(String, String), BTreeSet>, + fail_after_delete_attempts: BTreeMap<(String, String), BTreeSet>, put_attempts: BTreeMap<(String, String), usize>, delete_attempts: BTreeMap<(String, String), usize>, write_lock_acquisitions: BTreeMap<(String, String), usize>, + read_lock_acquisitions: BTreeMap<(String, String), usize>, read_calls: usize, metadata_calls: usize, list_calls: usize, @@ -880,6 +1269,16 @@ impl TestCatalogObjectBackend { .unwrap_or_default() } + async fn read_lock_acquisition_count(&self, bucket: &str, object: &str) -> usize { + self.state + .lock() + .await + .read_lock_acquisitions + .get(&(bucket.to_string(), object.to_string())) + .copied() + .unwrap_or_default() + } + async fn fail_next_read(&self, bucket: &str, object: &str) { let mut state = self.state.lock().await; let key = (bucket.to_string(), object.to_string()); @@ -887,6 +1286,51 @@ impl TestCatalogObjectBackend { state.fail_read_attempts.entry(key).or_default().insert(next_attempt); } + async fn pause_next_read(&self, bucket: &str, object: &str) -> TestCatalogObjectPause { + let mut state = self.state.lock().await; + let key = (bucket.to_string(), object.to_string()); + let next_attempt = state.read_attempts.get(&key).copied().unwrap_or_default() + 1; + let pause = TestCatalogObjectPause::default(); + state + .pause_read_attempts + .entry(key) + .or_default() + .insert(next_attempt, pause.clone()); + pause + } + + async fn pause_before_next_read(&self, bucket: &str, object: &str) -> TestCatalogObjectPause { + let mut state = self.state.lock().await; + let key = (bucket.to_string(), object.to_string()); + let next_attempt = state.read_attempts.get(&key).copied().unwrap_or_default() + 1; + let pause = TestCatalogObjectPause::default(); + state + .pause_before_read_attempts + .entry(key) + .or_default() + .insert(next_attempt, pause.clone()); + pause + } + + async fn omit_etag_for_object(&self, bucket: &str, object: &str) { + self.state + .lock() + .await + .etagless_objects + .insert((bucket.to_string(), object.to_string())); + } + + async fn last_read_limit(&self, bucket: &str, object: &str) -> Option { + let key = (bucket.to_string(), object.to_string()); + self.state + .lock() + .await + .read_limits + .iter() + .rev() + .find_map(|(read_key, limit)| (read_key == &key).then_some(*limit)) + } + async fn fail_next_put(&self, bucket: &str, object: &str) { let mut state = self.state.lock().await; let key = (bucket.to_string(), object.to_string()); @@ -894,11 +1338,25 @@ impl TestCatalogObjectBackend { state.fail_put_attempts.entry(key).or_default().insert(next_attempt); } - async fn pause_next_put(&self, bucket: &str, object: &str) -> TestCatalogObjectPutPause { + async fn fail_after_next_put(&self, bucket: &str, object: &str) { let mut state = self.state.lock().await; let key = (bucket.to_string(), object.to_string()); let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1; - let pause = TestCatalogObjectPutPause::default(); + state.fail_after_put_attempts.entry(key).or_default().insert(next_attempt); + } + + async fn fail_after_next_delete(&self, bucket: &str, object: &str) { + let mut state = self.state.lock().await; + let key = (bucket.to_string(), object.to_string()); + let next_attempt = state.delete_attempts.get(&key).copied().unwrap_or_default() + 1; + state.fail_after_delete_attempts.entry(key).or_default().insert(next_attempt); + } + + async fn pause_next_put(&self, bucket: &str, object: &str) -> TestCatalogObjectPause { + let mut state = self.state.lock().await; + let key = (bucket.to_string(), object.to_string()); + let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1; + let pause = TestCatalogObjectPause::default(); state .pause_put_attempts .entry(key) @@ -906,6 +1364,16 @@ impl TestCatalogObjectBackend { .insert(next_attempt, pause.clone()); pause } + + async fn put_attempt_count(&self, bucket: &str, object: &str) -> usize { + self.state + .lock() + .await + .put_attempts + .get(&(bucket.to_string(), object.to_string())) + .copied() + .unwrap_or_default() + } } impl TestCatalogObjectState { @@ -2084,41 +2552,89 @@ fn parquet_i32_values(data: Vec) -> Vec { #[async_trait::async_trait] impl TableCatalogObjectBackend for TestCatalogObjectBackend { + fn strong_catalog_runtime(&self) -> Option { + self.strong_runtime.clone() + } + async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { - let mut state = self.state.lock().await; - state.read_calls += 1; let key = (bucket.to_string(), object.to_string()); - let attempt = { - let attempts = state.read_attempts.entry(key.clone()).or_default(); - *attempts += 1; - *attempts + let (attempt, pause_before) = { + let mut state = self.state.lock().await; + state.read_calls += 1; + let attempt = { + let attempts = state.read_attempts.entry(key.clone()).or_default(); + *attempts += 1; + *attempts + }; + if state + .fail_read_attempts + .get(&key) + .is_some_and(|attempts| attempts.contains(&attempt)) + { + return Err(TableCatalogStoreError::Internal(format!( + "injected read failure for {object} attempt {attempt}" + ))); + } + let pause = state + .pause_before_read_attempts + .get_mut(&key) + .and_then(|attempts| attempts.remove(&attempt)); + (attempt, pause) }; - if state - .fail_read_attempts - .get(&key) - .is_some_and(|attempts| attempts.contains(&attempt)) - { - return Err(TableCatalogStoreError::Internal(format!( - "injected read failure for {object} attempt {attempt}" + if let Some(pause) = pause_before { + pause.started.notify_one(); + pause.release.notified().await; + } + let (result, pause) = { + let mut state = self.state.lock().await; + let etagless = state.etagless_objects.contains(&key); + let result = state.objects.get(&key).map(|record| TableCatalogObject { + data: record.data.clone(), + etag: (!etagless).then(|| record.etag.clone()), + mod_time: record.mod_time, + }); + let pause = state + .pause_read_attempts + .get_mut(&key) + .and_then(|attempts| attempts.remove(&attempt)); + (result, pause) + }; + if let Some(pause) = pause { + pause.started.notify_one(); + pause.release.notified().await; + } + Ok(result) + } + + async fn read_object_limited( + &self, + bucket: &str, + object: &str, + max_size: usize, + ) -> TableCatalogStoreResult> { + self.state + .lock() + .await + .read_limits + .push(((bucket.to_string(), object.to_string()), max_size)); + let result = self.read_object(bucket, object).await?; + if result.as_ref().is_some_and(|object| object.data.len() > max_size) { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog object {bucket}/{object} exceeds the maximum size of {max_size} bytes" ))); } - Ok(state.objects.get(&key).map(|record| TableCatalogObject { - data: record.data.clone(), - etag: Some(record.etag.clone()), - mod_time: record.mod_time, - })) + Ok(result) } async fn object_metadata(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { let mut state = self.state.lock().await; state.metadata_calls += 1; - Ok(state - .objects - .get(&(bucket.to_string(), object.to_string())) - .map(|record| TableCatalogObjectMetadata { - etag: Some(record.etag.clone()), - mod_time: record.mod_time, - })) + let key = (bucket.to_string(), object.to_string()); + let etagless = state.etagless_objects.contains(&key); + Ok(state.objects.get(&key).map(|record| TableCatalogObjectMetadata { + etag: (!etagless).then(|| record.etag.clone()), + mod_time: record.mod_time, + })) } async fn object_exists(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { @@ -2134,7 +2650,7 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { precondition: TableCatalogPutPrecondition, ) -> TableCatalogStoreResult<()> { let key = (bucket.to_string(), object.to_string()); - let pause = { + let (attempt, pause) = { let mut state = self.state.lock().await; let attempt = { let attempts = state.put_attempts.entry(key.clone()).or_default(); @@ -2150,10 +2666,11 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { "injected put failure for {object} attempt {attempt}" ))); } - state + let pause = state .pause_put_attempts .get_mut(&key) - .and_then(|attempts| attempts.remove(&attempt)) + .and_then(|attempts| attempts.remove(&attempt)); + (attempt, pause) }; if let Some(pause) = pause { pause.started.notify_one(); @@ -2178,13 +2695,22 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { let etag = state.next_etag(); state.objects.insert( - key, + key.clone(), TestCatalogObjectRecord { data, etag, mod_time: Some(OffsetDateTime::now_utc()), }, ); + if state + .fail_after_put_attempts + .get(&key) + .is_some_and(|attempts| attempts.contains(&attempt)) + { + return Err(TableCatalogStoreError::Internal(format!( + "injected post-commit put failure for {object} attempt {attempt}" + ))); + } Ok(()) } @@ -2206,6 +2732,15 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { ))); } state.objects.remove(&key); + if state + .fail_after_delete_attempts + .get(&key) + .is_some_and(|attempts| attempts.contains(&attempt)) + { + return Err(TableCatalogStoreError::Internal(format!( + "injected post-commit delete failure for {object} attempt {attempt}" + ))); + } Ok(()) } @@ -2232,10 +2767,28 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { let mut locks = self.locks.lock().await; locks .entry((bucket.to_string(), object.to_string())) - .or_insert_with(|| std::sync::Arc::new(tokio::sync::Mutex::new(()))) + .or_insert_with(|| std::sync::Arc::new(tokio::sync::RwLock::new(()))) .clone() }; - Ok(Box::new(lock.lock_owned().await)) + Ok(Box::new(lock.write_owned().await)) + } + + async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { + { + let mut state = self.state.lock().await; + *state + .read_lock_acquisitions + .entry((bucket.to_string(), object.to_string())) + .or_default() += 1; + } + let lock = { + let mut locks = self.locks.lock().await; + locks + .entry((bucket.to_string(), object.to_string())) + .or_insert_with(|| std::sync::Arc::new(tokio::sync::RwLock::new(()))) + .clone() + }; + Ok(Box::new(lock.read_owned().await)) } } @@ -2307,6 +2860,59 @@ fn test_view_entry(bucket: &str, namespace: &Namespace, view: &IdentifierSegment } } +fn test_strong_snapshot( + bucket: &str, + namespace: &Namespace, + tables: Vec, + views: Vec, +) -> StrongTableCatalogSnapshot { + StrongTableCatalogSnapshot { + version: STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION, + table_buckets: vec![test_bucket_entry(bucket)], + namespaces: vec![test_namespace_entry(bucket, namespace)], + tables, + views, + commits: Vec::new(), + idempotency: Vec::new(), + } +} + +async fn seed_strong_snapshot(backend: &TestCatalogObjectBackend, snapshot: &StrongTableCatalogSnapshot) { + backend + .seed_object( + RUSTFS_META_BUCKET, + &StrongTableCatalogStore::::snapshot_object_path(), + serde_json::to_vec(snapshot).expect("strong snapshot should encode"), + ) + .await; +} + +async fn read_strong_snapshot(backend: &TestCatalogObjectBackend) -> StrongTableCatalogSnapshot { + let object = backend + .read_object( + RUSTFS_META_BUCKET, + &StrongTableCatalogStore::::snapshot_object_path(), + ) + .await + .expect("strong snapshot should load") + .expect("strong snapshot should exist"); + serde_json::from_slice(&object.data).expect("strong snapshot should decode") +} + +async fn strong_snapshot_hydration_error(snapshot: StrongTableCatalogSnapshot) -> TableCatalogStoreError { + let backend = TestCatalogObjectBackend::default(); + let bucket = snapshot + .table_buckets + .first() + .map(|entry| entry.table_bucket.clone()) + .unwrap_or_else(|| "missing".to_string()); + seed_strong_snapshot(&backend, &snapshot).await; + StrongTableCatalogStore::new(backend) + .get_table_bucket(&bucket) + .await + .expect_err("strong snapshot hydration should fail") +} + async fn seed_catalog_list_entries(store: &S, bucket: &str, namespace: &Namespace) where S: TableCatalogStore + ?Sized, @@ -2325,15 +2931,17 @@ where .expect("namespace should be created"); } for name in ["alpha", "beta"] { - let identifier = IdentifierSegment::parse(name).expect("table and view name should parse"); - let metadata_location = default_table_metadata_file_path(namespace, &identifier, "00001.metadata.json"); - let mut table = test_table_entry(bucket, namespace, &identifier, metadata_location.clone()); + let table_identifier = IdentifierSegment::parse(name).expect("table name should parse"); + let metadata_location = default_table_metadata_file_path(namespace, &table_identifier, "00001.metadata.json"); + let mut table = test_table_entry(bucket, namespace, &table_identifier, metadata_location); table.table_id = format!("table-{name}"); table.table_uuid = format!("table-uuid-{name}"); table.warehouse_location = format!("s3://{bucket}/tables/table-{name}"); store.create_table(table).await.expect("table should be created"); - let mut view = test_view_entry(bucket, namespace, &identifier, metadata_location); + let view_identifier = IdentifierSegment::parse(format!("view_{name}")).expect("view name should parse"); + let view_metadata_location = default_view_metadata_file_path(namespace, &view_identifier, "00001.metadata.json"); + let mut view = test_view_entry(bucket, namespace, &view_identifier, view_metadata_location); view.view_id = format!("view-{name}"); view.view_uuid = format!("view-uuid-{name}"); view.warehouse_location = format!("s3://{bucket}/views/view-{name}"); @@ -2432,7 +3040,7 @@ async fn object_table_catalog_store_writes_catalog_entries_to_internal_meta_buck #[tokio::test] async fn configured_table_catalog_store_uses_durable_strong_snapshot() { let backend = TestCatalogObjectBackend::default(); - let store = ConfiguredTableCatalogStore::new(backend.clone(), TableCatalogBackingMode::DurableStrong); + let store = ConfiguredTableCatalogStore::new_for_test(backend.clone(), TableCatalogBackingMode::DurableStrong); let bucket = "analytics"; assert_eq!(store.backing_mode(), TableCatalogBackingMode::DurableStrong); @@ -2441,7 +3049,7 @@ async fn configured_table_catalog_store_uses_durable_strong_snapshot() { let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); assert!(backend.object_exists(RUSTFS_META_BUCKET, &snapshot_path).await.unwrap()); - let reloaded = ConfiguredTableCatalogStore::new(backend.clone(), TableCatalogBackingMode::DurableStrong); + let reloaded = ConfiguredTableCatalogStore::new_for_test(backend.clone(), TableCatalogBackingMode::DurableStrong); assert!(reloaded.get_table_bucket(bucket).await.unwrap().is_some()); } @@ -2587,7 +3195,7 @@ async fn table_data_plane_resource_does_not_match_sibling_prefix() { .expect("data-plane resource lookup should succeed"); assert!(resource.is_none()); - assert_eq!(backend.list_call_count().await, 0); + assert_eq!(backend.list_call_count().await, 1); } #[tokio::test] @@ -2599,9 +3207,10 @@ async fn catalog_backings_reject_overlapping_registered_warehouse_prefixes() { let parent_table = IdentifierSegment::parse("orders").unwrap(); let child_table = IdentifierSegment::parse("orders_child").unwrap(); let current = default_table_metadata_file_path(&namespace, &parent_table, "00001.metadata.json"); + let child_metadata = default_table_metadata_file_path(&namespace, &child_table, "00001.metadata.json"); - seed_table_for_metadata_maintenance(&store, bucket, &namespace, &parent_table, current.clone()).await; - let mut child_entry = test_table_entry(bucket, &namespace, &child_table, current); + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &parent_table, current).await; + let mut child_entry = test_table_entry(bucket, &namespace, &child_table, child_metadata); child_entry.table_id = "table-id-child".to_string(); child_entry.warehouse_location = format!("s3://{bucket}/tables/table-id/child"); assert_matches!(store.create_table(child_entry.clone()).await, Err(TableCatalogStoreError::Conflict(_))); @@ -2625,6 +3234,132 @@ async fn catalog_backings_reject_overlapping_registered_warehouse_prefixes() { assert_matches!(strong_store.create_table(child_entry).await, Err(TableCatalogStoreError::Conflict(_))); } +async fn assert_resource_id_registration_contract(store: &S) +where + S: TableCatalogStore, +{ + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let orders = IdentifierSegment::parse("orders").expect("table should parse"); + let returns = IdentifierSegment::parse("returns").expect("table should parse"); + let empty = IdentifierSegment::parse("empty_id").expect("table should parse"); + let empty_view = IdentifierSegment::parse("empty_view_id").expect("view should parse"); + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_table(test_table_entry( + bucket, + &namespace, + &orders, + default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"), + )) + .await + .expect("first table should be created"); + + let mut duplicate = test_table_entry( + bucket, + &namespace, + &returns, + default_table_metadata_file_path(&namespace, &returns, "00001.metadata.json"), + ); + duplicate.warehouse_location = format!("s3://{bucket}/tables/returns"); + assert_matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_))); + + let mut empty_id = test_table_entry( + bucket, + &namespace, + &empty, + default_table_metadata_file_path(&namespace, &empty, "00001.metadata.json"), + ); + empty_id.table_id.clear(); + empty_id.warehouse_location = format!("s3://{bucket}/tables/empty-id"); + assert_matches!(store.create_table(empty_id).await, Err(TableCatalogStoreError::Invalid(_))); + + let mut empty_view_id = test_view_entry( + bucket, + &namespace, + &empty_view, + default_view_metadata_file_path(&namespace, &empty_view, "00001.view-metadata.json"), + ); + empty_view_id.view_id.clear(); + assert_matches!(store.create_view(empty_view_id).await, Err(TableCatalogStoreError::Invalid(_))); +} + +#[tokio::test] +async fn catalog_backings_reject_empty_resource_ids_and_duplicate_table_ids() { + assert_resource_id_registration_contract(&ObjectTableCatalogStore::new(TestCatalogObjectBackend::default())).await; + assert_resource_id_registration_contract(&StrongTableCatalogStore::new(TestCatalogObjectBackend::default())).await; +} + +#[tokio::test] +async fn object_catalog_rejects_overlapping_registration_when_a_ready_index_entry_is_missing() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let parent_table = IdentifierSegment::parse("orders").expect("parent table should parse"); + let child_table = IdentifierSegment::parse("orders_child").expect("child table should parse"); + let current = default_table_metadata_file_path(&namespace, &parent_table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &parent_table, current.clone()).await; + backend + .delete_object(RUSTFS_META_BUCKET, &store.paths.warehouse_index_entry_path(bucket, "tables/table-id/")) + .await + .expect("warehouse index entry should be removed"); + assert!( + store + .warehouse_index_ready(bucket) + .await + .expect("warehouse index state should remain ready") + ); + + let mut child_entry = test_table_entry(bucket, &namespace, &child_table, current); + child_entry.table_id = "table-id-child".to_string(); + child_entry.warehouse_location = format!("s3://{bucket}/tables/table-id/child"); + + assert_matches!(store.create_table(child_entry).await, Err(TableCatalogStoreError::Conflict(_))); +} + +#[tokio::test] +async fn object_catalog_rechecks_overlap_when_candidate_has_a_stale_reservation() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let parent_table = IdentifierSegment::parse("orders").expect("parent table should parse"); + let child_table = IdentifierSegment::parse("orders_child").expect("child table should parse"); + let current = default_table_metadata_file_path(&namespace, &parent_table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &parent_table, current).await; + let mut child_entry = test_table_entry( + bucket, + &namespace, + &child_table, + default_table_metadata_file_path(&namespace, &child_table, "00001.metadata.json"), + ); + child_entry.table_id = "table-id-child".to_string(); + child_entry.warehouse_location = format!("s3://{bucket}/tables/table-id/child"); + let child_index = table_warehouse_index_entry(&child_entry).expect("child index should be valid"); + let child_index_path = store + .paths + .warehouse_index_entry_path(bucket, &child_index.warehouse_object_prefix); + backend + .seed_object( + RUSTFS_META_BUCKET, + &child_index_path, + serde_json::to_vec(&child_index).expect("child index should serialize"), + ) + .await; + + assert_matches!(store.create_table(child_entry).await, Err(TableCatalogStoreError::Conflict(_))); +} + #[tokio::test] async fn object_catalog_revalidates_legacy_warehouse_index_before_resolution() { let backend = TestCatalogObjectBackend::default(); @@ -2678,6 +3413,151 @@ async fn object_catalog_revalidates_legacy_warehouse_index_before_resolution() { )); } +#[tokio::test] +async fn object_catalog_ready_index_rejects_overlapping_active_owners() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let parent_table = IdentifierSegment::parse("orders").expect("parent table should parse"); + let child_table = IdentifierSegment::parse("orders_child").expect("child table should parse"); + let current = default_table_metadata_file_path(&namespace, &parent_table, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &parent_table, current).await; + let mut child_entry = test_table_entry( + bucket, + &namespace, + &child_table, + default_table_metadata_file_path(&namespace, &child_table, "00001.metadata.json"), + ); + child_entry.table_id = "table-id-child".to_string(); + child_entry.table_uuid = "table-uuid-child".to_string(); + child_entry.warehouse_location = format!("s3://{bucket}/tables/table-id/child"); + store + .write_entry( + store.catalog_bucket(), + &store.paths.table_entry_path(bucket, &namespace, &child_table), + &child_entry, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("legacy overlapping table entry should be seeded"); + let child_index = table_warehouse_index_entry(&child_entry).expect("child warehouse index should be valid"); + store + .write_entry( + store.catalog_bucket(), + &store + .paths + .warehouse_index_entry_path(bucket, &child_index.warehouse_object_prefix), + &child_index, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("legacy overlapping warehouse index should be seeded"); + assert!( + store + .warehouse_index_ready(bucket) + .await + .expect("warehouse index should remain ready") + ); + + let error = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/child/data/part-00001.parquet") + .await + .expect_err("overlapping authoritative indexes must fail closed"); + + assert_matches!( + error, + TableCatalogStoreError::Invalid(message) if message.contains("overlapping active table warehouse indexes") + ); +} + +#[tokio::test] +async fn object_catalog_index_backfill_rejects_legacy_duplicate_table_ids() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let orders = IdentifierSegment::parse("orders").expect("orders table should parse"); + let returns = IdentifierSegment::parse("returns").expect("returns table should parse"); + let current = default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &orders, current).await; + let mut duplicate = test_table_entry( + bucket, + &namespace, + &returns, + default_table_metadata_file_path(&namespace, &returns, "00001.metadata.json"), + ); + duplicate.warehouse_location = format!("s3://{bucket}/tables/returns"); + store + .write_entry( + store.catalog_bucket(), + &store.paths.table_entry_path(bucket, &namespace, &returns), + &duplicate, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("legacy duplicate table id should be seeded"); + backend + .delete_object(RUSTFS_META_BUCKET, &store.paths.warehouse_index_state_path(bucket)) + .await + .expect("warehouse index readiness marker should be removed"); + + let error = store + .backfill_table_warehouse_index(bucket) + .await + .expect_err("duplicate table ids must block warehouse index readiness"); + + assert_matches!( + error, + TableCatalogStoreError::Conflict(message) if message.contains("registered by multiple tables") + ); + assert!( + !store + .warehouse_index_ready(bucket) + .await + .expect("warehouse index should remain unready") + ); +} + +#[tokio::test] +async fn object_catalog_data_plane_rejects_invalid_active_warehouse_location() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let mut entry = test_table_entry( + bucket, + &namespace, + &table, + default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), + ); + entry.warehouse_location = "s3://other-bucket/tables/table-id".to_string(); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .write_entry( + store.catalog_bucket(), + &store.paths.table_entry_path(bucket, &namespace, &table), + &entry, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("invalid persisted table should be seeded"); + + let error = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") + .await + .expect_err("invalid active warehouse locations must fail closed"); + + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("must be inside the table bucket")); + assert!(!store.warehouse_index_ready(bucket).await.unwrap()); +} + #[tokio::test] async fn table_data_plane_resource_skips_stale_deeper_index_and_matches_parent() { let backend = TestCatalogObjectBackend::default(); @@ -2763,6 +3643,109 @@ async fn object_catalog_rebuilds_warehouse_index_for_table_backed_namespace() { ); } +#[tokio::test] +async fn table_data_plane_resource_scans_when_a_ready_index_entry_is_missing() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let object = "tables/table-id/data/part-00001.parquet"; + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + backend + .delete_object(RUSTFS_META_BUCKET, &store.paths.warehouse_index_entry_path(bucket, "tables/table-id/")) + .await + .expect("warehouse index entry should be removed"); + backend.reset_call_counts().await; + + let resource = table_data_plane_resource_for_object(&store, bucket, object) + .await + .expect("missing ready index entries must use the authoritative table scan") + .expect("the table scan must retain table-aware protection"); + + assert_eq!(resource.table, "orders"); + assert!(backend.list_call_count().await > 0); +} + +#[tokio::test] +async fn table_data_plane_resource_scans_when_an_index_disappears_during_backfill() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let object = "tables/table-id/data/part-00001.parquet"; + let index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); + let state_path = store.paths.warehouse_index_state_path(bucket); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + backend + .delete_object(RUSTFS_META_BUCKET, &index_path) + .await + .expect("warehouse index entry should be removed before backfill"); + backend + .delete_object(RUSTFS_META_BUCKET, &state_path) + .await + .expect("warehouse index readiness should be cleared before backfill"); + let paused_state_write = backend.pause_next_put(RUSTFS_META_BUCKET, &state_path).await; + let resolve_store = store.clone(); + let resolution = tokio::spawn(async move { table_data_plane_resource_for_object(&resolve_store, bucket, object).await }); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, paused_state_write.wait_started()) + .await + .expect("warehouse index state publication should start"); + backend + .delete_object(RUSTFS_META_BUCKET, &index_path) + .await + .expect("warehouse index entry should be removed after backfill publication"); + paused_state_write.release(); + + let resource = resolution + .await + .expect("data-plane resolution task should join") + .expect("authoritative catalog scan should preserve table-aware resolution") + .expect("table-aware resource must not be lost after an index race"); + assert_eq!(resource.table, "orders"); +} + +#[tokio::test] +async fn table_data_plane_resource_fails_closed_for_an_inactive_ready_index() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let prefix = "tables/table-id/"; + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + let inactive = TableWarehouseIndexEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + table_id: "table-id".to_string(), + warehouse_object_prefix: prefix.to_string(), + state: TableCatalogEntryState::Deleted, + }; + store + .write_entry( + store.catalog_bucket(), + &store.paths.warehouse_index_entry_path(bucket, prefix), + &inactive, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("inactive warehouse index should be seeded"); + + assert_matches!( + table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet").await, + Err(TableCatalogStoreError::Internal(message)) if message.contains("inactive while the index is authoritative") + ); +} + #[tokio::test] async fn object_table_catalog_store_rejects_duplicate_warehouse_prefix() { let backend = TestCatalogObjectBackend::default(); @@ -2825,6 +3808,280 @@ async fn table_data_plane_resource_fails_closed_for_missing_indexed_table() { )); } +#[tokio::test] +async fn table_data_plane_resource_fails_closed_for_mismatched_indexed_table_identity() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + + let mut mismatched = store + .load_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("table lookup should succeed") + .expect("table should exist"); + mismatched.namespace = "finance".to_string(); + store + .write_entry( + store.catalog_bucket(), + &store.paths.table_entry_path(bucket, &namespace, &table), + &mismatched, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("mismatched table identity should be seeded"); + let index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); + + let error = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") + .await + .expect_err("mismatched persisted table identity must fail closed"); + let recovery_error = store + .plan_table_commit_recovery(bucket, &namespace.public_name(), table.as_str()) + .await + .expect_err("catalog recovery must reject a mismatched persisted table identity"); + + for error in [error, recovery_error] { + assert_matches!( + error, + TableCatalogStoreError::Invalid(message) + if message.contains("catalog table entry identity does not match its object path") + ); + } + assert!( + backend + .object_exists(RUSTFS_META_BUCKET, &index_path) + .await + .expect("warehouse index lookup should succeed") + ); +} + +#[tokio::test] +async fn table_data_plane_resource_rejects_unknown_warehouse_index_version() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + + let prefix = "tables/table-id/"; + let unsupported = TableWarehouseIndexEntry { + version: TABLE_CATALOG_ENTRY_VERSION + 1, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + table_id: "table-id".to_string(), + warehouse_object_prefix: prefix.to_string(), + state: TableCatalogEntryState::Active, + }; + store + .write_entry( + store.catalog_bucket(), + &store.paths.warehouse_index_entry_path(bucket, prefix), + &unsupported, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("unsupported warehouse index should be seeded"); + store + .backend + .delete_object(store.catalog_bucket(), &store.paths.warehouse_index_state_path(bucket)) + .await + .expect("warehouse index state should be removed"); + + let error = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") + .await + .expect_err("unknown warehouse index versions must fail closed"); + + assert!(matches!( + error, + TableCatalogStoreError::Invalid(message) if message.contains("unsupported warehouse index entry version") + )); + assert!( + !store + .warehouse_index_ready(bucket) + .await + .expect("warehouse index state lookup should succeed") + ); +} + +#[tokio::test] +async fn object_catalog_does_not_overwrite_unknown_warehouse_index_state_version() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let state_path = store.paths.warehouse_index_state_path(bucket); + let future_state = TableWarehouseIndexStateEntry { + version: TABLE_WAREHOUSE_INDEX_STATE_VERSION + 1, + table_bucket: bucket.to_string(), + state: TableCatalogEntryState::Active, + }; + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .write_entry(store.catalog_bucket(), &state_path, &future_state, TableCatalogPutPrecondition::Any) + .await + .expect("future warehouse index state should be seeded"); + + let error = store + .backfill_table_warehouse_index(bucket) + .await + .expect_err("unknown warehouse index state versions must fail closed"); + assert_matches!( + error, + TableCatalogStoreError::Invalid(message) if message.contains("unsupported warehouse index state version") + ); + let persisted = store + .read_entry::(store.catalog_bucket(), &state_path) + .await + .expect("warehouse index state should load") + .expect("warehouse index state should remain") + .0; + assert_eq!(persisted, future_state); +} + +#[tokio::test] +async fn object_catalog_rejects_mismatched_table_bucket_identity() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let requested_bucket = "analytics"; + let mut mismatched = test_bucket_entry("finance"); + mismatched.state = TableCatalogEntryState::Active; + store + .write_entry( + store.catalog_bucket(), + &store.paths.table_bucket_entry_path(requested_bucket), + &mismatched, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("mismatched table bucket identity should be seeded"); + + let error = store + .get_table_bucket(requested_bucket) + .await + .expect_err("mismatched persisted table bucket identity must fail closed"); + + assert!(matches!( + error, + TableCatalogStoreError::Invalid(message) + if message.contains("catalog table bucket entry identity does not match its object path") + )); +} + +#[tokio::test] +async fn object_catalog_view_reads_reject_mismatched_persisted_identity() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let view = IdentifierSegment::parse("orders_view").expect("view should parse"); + let metadata_location = default_view_metadata_file_path(&namespace, &view, "00001.view-metadata.json"); + let mut mismatched = test_view_entry(bucket, &namespace, &view, metadata_location); + mismatched.namespace = "finance".to_string(); + store + .write_entry( + store.catalog_bucket(), + &store.paths.view_entry_path(bucket, &namespace, &view), + &mismatched, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("mismatched view identity should be seeded"); + + let load_error = store + .load_view(bucket, &namespace.public_name(), view.as_str()) + .await + .expect_err("mismatched persisted view identity must fail closed on load"); + let list_error = store + .list_views(bucket, &namespace.public_name()) + .await + .expect_err("mismatched persisted view identity must fail closed on list"); + let drop_error = store + .drop_view(bucket, &namespace.public_name(), view.as_str()) + .await + .expect_err("mismatched persisted view identity must fail closed before deletion"); + + for error in [load_error, list_error, drop_error] { + assert!(matches!( + error, + TableCatalogStoreError::Invalid(message) + if message.contains("catalog view entry identity does not match its object path") + )); + } +} + +#[tokio::test] +async fn object_catalog_view_reads_reject_empty_persisted_id() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let view = IdentifierSegment::parse("orders_view").expect("view should parse"); + let metadata_location = default_view_metadata_file_path(&namespace, &view, "00001.view-metadata.json"); + let mut invalid = test_view_entry(bucket, &namespace, &view, metadata_location); + invalid.view_id.clear(); + store + .write_entry( + store.catalog_bucket(), + &store.paths.view_entry_path(bucket, &namespace, &view), + &invalid, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("invalid view entry should be seeded"); + + let error = store + .load_view(bucket, &namespace.public_name(), view.as_str()) + .await + .expect_err("an empty persisted view id must fail closed"); + + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("view id cannot be empty")); +} + +#[tokio::test] +async fn object_catalog_data_plane_removes_index_for_inactive_table() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + let mut inactive = store + .load_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("table lookup should succeed") + .expect("table should exist"); + inactive.state = TableCatalogEntryState::Deleted; + store + .write_entry( + store.catalog_bucket(), + &store.paths.table_entry_path(bucket, &namespace, &table), + &inactive, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("inactive table state should be seeded"); + let index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); + + let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") + .await + .expect("inactive table lookup should safely clean its stale index"); + + assert!(resource.is_none()); + assert!( + !backend + .object_exists(RUSTFS_META_BUCKET, &index_path) + .await + .expect("warehouse index lookup should succeed") + ); +} + #[tokio::test] async fn object_table_catalog_store_replaces_stale_warehouse_index_on_create() { let backend = TestCatalogObjectBackend::default(); @@ -2871,6 +4128,128 @@ async fn object_table_catalog_store_replaces_stale_warehouse_index_on_create() { assert_eq!(index.table_id, "table-id"); } +#[tokio::test] +async fn object_table_catalog_store_does_not_replace_unknown_warehouse_index_version() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let prefix = "tables/shared-table/"; + let index_path = store.paths.warehouse_index_entry_path(bucket, prefix); + let unsupported = TableWarehouseIndexEntry { + version: TABLE_CATALOG_ENTRY_VERSION + 1, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: "missing_orders".to_string(), + table_id: "missing-table-id".to_string(), + warehouse_object_prefix: prefix.to_string(), + state: TableCatalogEntryState::Active, + }; + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .write_entry(store.catalog_bucket(), &index_path, &unsupported, TableCatalogPutPrecondition::Any) + .await + .expect("unsupported warehouse index should be seeded"); + + let mut entry = test_table_entry(bucket, &namespace, &table, current); + entry.warehouse_location = format!("s3://{bucket}/tables/shared-table"); + let error = store + .create_table(entry) + .await + .expect_err("unknown warehouse index versions must not be replaced as stale"); + + assert!(matches!( + error, + TableCatalogStoreError::Invalid(message) if message.contains("unsupported warehouse index entry version") + )); +} + +#[tokio::test] +async fn object_table_catalog_store_does_not_ignore_unknown_unrelated_warehouse_index_version() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let prefix = "tables/future-table/"; + let index_path = store.paths.warehouse_index_entry_path(bucket, prefix); + let unsupported = TableWarehouseIndexEntry { + version: TABLE_CATALOG_ENTRY_VERSION + 1, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: "future_orders".to_string(), + table_id: "future-table-id".to_string(), + warehouse_object_prefix: prefix.to_string(), + state: TableCatalogEntryState::Active, + }; + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .write_entry(store.catalog_bucket(), &index_path, &unsupported, TableCatalogPutPrecondition::Any) + .await + .expect("future warehouse index should be seeded"); + + let error = store + .create_table(test_table_entry(bucket, &namespace, &table, current)) + .await + .expect_err("unknown index versions must not be skipped during warehouse uniqueness checks"); + assert_matches!( + error, + TableCatalogStoreError::Invalid(message) if message.contains("unsupported warehouse index entry version") + ); +} + +#[tokio::test] +async fn object_table_catalog_store_rejects_misplaced_warehouse_index_before_create() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let misplaced = TableWarehouseIndexEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: "missing_orders".to_string(), + table_id: "missing-table-id".to_string(), + warehouse_object_prefix: "tables/payload-prefix/".to_string(), + state: TableCatalogEntryState::Active, + }; + let misplaced_path = store.paths.warehouse_index_entry_path(bucket, "tables/object-prefix/"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .write_entry(store.catalog_bucket(), &misplaced_path, &misplaced, TableCatalogPutPrecondition::Any) + .await + .expect("misplaced warehouse index should be seeded"); + + let error = store + .create_table(test_table_entry(bucket, &namespace, &table, current)) + .await + .expect_err("misplaced warehouse indexes must fail closed during uniqueness checks"); + assert_matches!( + error, + TableCatalogStoreError::Invalid(message) if message.contains("warehouse index identity does not match its object path") + ); +} + #[tokio::test] async fn table_data_plane_resource_falls_back_to_scan_without_index_state() { let backend = TestCatalogObjectBackend::default(); @@ -2928,7 +4307,7 @@ async fn table_data_plane_resource_falls_back_to_scan_without_index_state() { assert_eq!(indexed_resource.table, "orders"); assert_eq!(backend.list_call_count().await, 0); - assert!(backend.read_call_count().await <= 5); + assert!(backend.read_call_count().await <= 6); } #[tokio::test] @@ -3041,12 +4420,12 @@ async fn object_catalog_pagination_bounds_reads_and_covers_rest_resources() { .list_views_page(bucket, &namespace_name, None, one) .await .expect("first view page should load"); - assert_eq!(view_page.entries[0].view, "alpha"); + assert_eq!(view_page.entries[0].view, "view_alpha"); let view_page = store .list_views_page(bucket, &namespace_name, view_page.next_cursor.as_deref(), one) .await .expect("second view page should load"); - assert_eq!(view_page.entries[0].view, "beta"); + assert_eq!(view_page.entries[0].view, "view_beta"); assert!(view_page.next_cursor.is_none()); let exact_page = store @@ -3107,6 +4486,53 @@ async fn object_catalog_pagination_bounds_sparse_namespace_scans() { assert_eq!(backend.list_call_count().await, 2); } +#[tokio::test] +async fn object_catalog_drop_namespace_scans_all_pages_for_inactive_resources() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let prefix = store.paths.table_entries_prefix(bucket, &namespace); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + for index in 0..TABLE_CATALOG_LIST_MAX_KEYS { + backend + .seed_object(RUSTFS_META_BUCKET, &format!("{prefix}0000-spacer/{index:04}.json"), Vec::new()) + .await; + } + let mut inactive = test_table_entry( + bucket, + &namespace, + &table, + default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), + ); + inactive.state = TableCatalogEntryState::Deleted; + store + .write_entry( + store.catalog_bucket(), + &store.paths.table_entry_path(bucket, &namespace, &table), + &inactive, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("inactive table entry should be seeded after the sparse first page"); + backend.reset_call_counts().await; + + assert_matches!( + store.drop_namespace(bucket, &namespace.public_name()).await, + Err(TableCatalogStoreError::Conflict(message)) if message.contains("not empty") + ); + assert_eq!(backend.list_call_count().await, 4); +} + #[tokio::test] async fn object_table_catalog_store_rolls_back_warehouse_index_when_table_entry_write_fails() { let backend = TestCatalogObjectBackend::default(); @@ -3141,17 +4567,19 @@ async fn object_table_catalog_store_rolls_back_warehouse_index_when_table_entry_ } #[tokio::test] -async fn object_table_catalog_store_restores_table_when_drop_index_delete_fails() { +async fn object_table_catalog_store_keeps_table_when_drop_index_delete_fails() { let backend = TestCatalogObjectBackend::default(); let store = ObjectTableCatalogStore::new(backend.clone()); let bucket = "analytics"; let namespace = Namespace::parse("sales").expect("namespace should parse"); let table = IdentifierSegment::parse("orders").expect("table should parse"); let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let table_path = store.paths.table_entry_path(bucket, &namespace, &table); let index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; backend.fail_delete_attempt(RUSTFS_META_BUCKET, &index_path, 1).await; + backend.fail_next_put(RUSTFS_META_BUCKET, &table_path).await; let error = store .drop_table(bucket, &namespace.public_name(), table.as_str()) @@ -3159,19 +4587,212 @@ async fn object_table_catalog_store_restores_table_when_drop_index_delete_fails( .unwrap_err(); assert!(matches!(error, TableCatalogStoreError::Internal(_))); - let restored = store + let retained = store .load_table(bucket, &namespace.public_name(), table.as_str()) .await - .expect("restored table lookup should succeed") - .expect("table entry should be restored"); - assert_eq!(restored.table_id, "table-id"); + .expect("retained table lookup should succeed") + .expect("table entry should remain present"); + assert_eq!(retained.table_id, "table-id"); let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") .await - .expect("restored table data-plane lookup should succeed") - .expect("restored table should keep data-plane protection"); + .expect("retained table data-plane lookup should succeed") + .expect("retained table should keep data-plane protection"); assert_eq!(resource.table, "orders"); } +#[tokio::test] +async fn object_table_catalog_store_rejects_drop_when_warehouse_index_owner_changed() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + let conflicting_index = TableWarehouseIndexEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.to_string(), + namespace: "finance".to_string(), + table: "returns".to_string(), + table_id: "other-table-id".to_string(), + warehouse_object_prefix: "tables/table-id/".to_string(), + state: TableCatalogEntryState::Active, + }; + store + .write_entry(store.catalog_bucket(), &index_path, &conflicting_index, TableCatalogPutPrecondition::Any) + .await + .expect("conflicting warehouse index should be seeded"); + + assert_matches!( + store.drop_table(bucket, &namespace.public_name(), table.as_str()).await, + Err(TableCatalogStoreError::Conflict(message)) if message.contains("owner changed") + ); + let retained = store + .load_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("retained table lookup should succeed") + .expect("table entry should remain present"); + assert_eq!(retained.table_id, "table-id"); + let (retained_index, _) = store + .read_entry::(RUSTFS_META_BUCKET, &index_path) + .await + .expect("conflicting warehouse index lookup should succeed") + .expect("conflicting warehouse index should remain present"); + assert_eq!(retained_index, conflicting_index); +} + +#[tokio::test] +async fn object_table_catalog_store_drops_table_when_warehouse_index_is_missing() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + let entry = store + .load_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("table lookup should succeed") + .expect("table entry should exist"); + store + .delete_table_warehouse_index(&entry) + .await + .expect("warehouse index should be removed"); + assert!( + store + .read_entry::(RUSTFS_META_BUCKET, &index_path) + .await + .expect("warehouse index lookup should succeed") + .is_none() + ); + + store + .drop_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("a missing warehouse index should not block table drop"); + assert!( + store + .load_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("dropped table lookup should succeed") + .is_none() + ); +} + +#[tokio::test] +async fn object_table_catalog_store_restores_index_when_table_entry_delete_fails() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let table_path = store.paths.table_entry_path(bucket, &namespace, &table); + let index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + backend.fail_delete_attempt(RUSTFS_META_BUCKET, &table_path, 1).await; + + assert_matches!( + store.drop_table(bucket, &namespace.public_name(), table.as_str()).await, + Err(TableCatalogStoreError::Internal(_)) + ); + let retained = store + .load_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("retained table lookup should succeed") + .expect("table entry should remain present"); + assert_eq!(retained.table_id, "table-id"); + let (restored_index, _) = store + .read_entry::(RUSTFS_META_BUCKET, &index_path) + .await + .expect("restored warehouse index lookup should succeed") + .expect("warehouse index should be restored"); + assert_eq!(restored_index.table_id, "table-id"); + let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") + .await + .expect("restored index lookup should succeed") + .expect("restored index should keep data-plane protection"); + assert_eq!(resource.table, "orders"); +} + +#[tokio::test] +async fn object_table_catalog_store_falls_back_to_scan_when_drop_index_restore_fails() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let table_path = store.paths.table_entry_path(bucket, &namespace, &table); + let index_path = store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + backend.fail_delete_attempt(RUSTFS_META_BUCKET, &table_path, 1).await; + backend.fail_next_put(RUSTFS_META_BUCKET, &index_path).await; + + assert_matches!( + store.drop_table(bucket, &namespace.public_name(), table.as_str()).await, + Err(TableCatalogStoreError::Internal(_)) + ); + let retained = store + .load_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("retained table lookup should succeed") + .expect("table entry should remain present"); + assert_eq!(retained.table_id, "table-id"); + assert!( + store + .read_entry::(RUSTFS_META_BUCKET, &index_path) + .await + .expect("warehouse index lookup should succeed") + .is_none(), + "the injected index restore failure must leave the scan fallback under test" + ); + let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") + .await + .expect("catalog scan fallback should succeed") + .expect("catalog scan fallback should keep data-plane protection"); + assert_eq!(resource.table, "orders"); +} + +#[tokio::test] +async fn object_table_catalog_store_accepts_ambiguous_delete_when_table_is_absent() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let table_path = store.paths.table_entry_path(bucket, &namespace, &table); + + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await; + backend.fail_after_next_delete(RUSTFS_META_BUCKET, &table_path).await; + + store + .drop_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("a confirmed absent table should satisfy an ambiguous delete"); + assert!( + store + .load_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("dropped table lookup should succeed") + .is_none() + ); + assert!( + table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") + .await + .expect("dropped table data-plane lookup should succeed") + .is_none() + ); +} + #[tokio::test] async fn table_data_plane_resource_bounds_deep_object_index_reads() { let backend = TestCatalogObjectBackend::default(); @@ -3338,14 +4959,15 @@ async fn object_table_catalog_store_allows_deep_view_warehouse_location() { } #[tokio::test] -async fn table_data_plane_resource_skips_invalid_warehouse_locations() { +async fn table_data_plane_resource_fails_closed_when_any_active_warehouse_location_is_invalid() { let backend = TestCatalogObjectBackend::default(); let store = ObjectTableCatalogStore::new(backend); let bucket = "analytics"; let namespace = Namespace::parse("sales").unwrap(); let invalid_table = IdentifierSegment::parse("bad_orders").unwrap(); let valid_table = IdentifierSegment::parse("orders").unwrap(); - let current = default_table_metadata_file_path(&namespace, &valid_table, "00001.metadata.json"); + let invalid_metadata = default_table_metadata_file_path(&namespace, &invalid_table, "00001.metadata.json"); + let valid_metadata = default_table_metadata_file_path(&namespace, &valid_table, "00001.metadata.json"); store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); store @@ -3353,7 +4975,7 @@ async fn table_data_plane_resource_skips_invalid_warehouse_locations() { .await .unwrap(); - let mut invalid_entry = test_table_entry(bucket, &namespace, &invalid_table, current.clone()); + let mut invalid_entry = test_table_entry(bucket, &namespace, &invalid_table, invalid_metadata); invalid_entry.table_id = "bad-table-id".to_string(); invalid_entry.warehouse_location = format!("s3://{bucket}/"); let invalid_path = store.paths.table_entry_path(bucket, &namespace, &invalid_table); @@ -3366,23 +4988,23 @@ async fn table_data_plane_resource_skips_invalid_warehouse_locations() { ) .await .unwrap(); + let valid_entry = test_table_entry(bucket, &namespace, &valid_table, valid_metadata); store - .create_table(test_table_entry(bucket, &namespace, &valid_table, current)) + .write_entry( + store.catalog_bucket(), + &store.paths.table_entry_path(bucket, &namespace, &valid_table), + &valid_entry, + TableCatalogPutPrecondition::IfAbsent, + ) .await .unwrap(); - let unrelated = table_data_plane_resource_for_object(&store, bucket, "ordinary/object.parquet") - .await - .expect("invalid table warehouse location should not deny unrelated object lookup"); - assert!(unrelated.is_none()); - - let resource = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part-00001.parquet") - .await - .expect("invalid table warehouse location should not block a later valid match") - .expect("valid table warehouse object should resolve to the table"); - assert_eq!(resource.table, "orders"); - assert_eq!(resource.table_id, "table-id"); - assert_eq!(resource.warehouse_object_prefix, "tables/table-id/"); + for object in ["ordinary/object.parquet", "tables/table-id/data/part-00001.parquet"] { + let error = table_data_plane_resource_for_object(&store, bucket, object) + .await + .expect_err("an invalid active warehouse location must fail closed for the whole table bucket"); + assert_matches!(error, TableCatalogStoreError::Invalid(_)); + } } #[tokio::test] @@ -7234,7 +8856,10 @@ async fn export_catalog_entry_includes_backing_migration_manifest() { TableCatalogConsistencyMode::ConditionalObjectCas ); assert_eq!(export.backing_manifest.current.wal.finalization_required_count, 0); - assert_eq!(export.backing_manifest.migration.target_kind, TableCatalogBackingKind::StrongKvWal); + assert_eq!( + export.backing_manifest.migration.target_kind, + TableCatalogBackingKind::DurableStrongSnapshot + ); assert_eq!( export.backing_manifest.migration.status, TableCatalogBackingMigrationStatus::ReadyToSnapshot @@ -7251,6 +8876,19 @@ async fn export_catalog_entry_includes_backing_migration_manifest() { TableCatalogHaWriterModel::SingleActiveWriterRegion ); assert!(!export.backing_manifest.ha.active_active_supported); + let wire_manifest = serde_json::to_value(&export.backing_manifest).expect("backing manifest should serialize"); + assert_eq!( + wire_manifest + .pointer("/migration/target_kind") + .and_then(serde_json::Value::as_str), + Some("STRONG_KV_WAL") + ); + assert!( + wire_manifest + .pointer("/migration/required_steps") + .and_then(serde_json::Value::as_array) + .is_some_and(|steps| { steps.iter().any(|step| step.as_str() == Some("CUT_OVER_LINEARIZABLE_READS")) }) + ); } #[tokio::test] @@ -7275,14 +8913,25 @@ async fn object_table_catalog_store_serializes_namespace_drop_with_table_creatio let create = tokio::spawn(async move { create_store.create_table(table_entry).await }); pause.wait_started().await; + let namespace_path = store.paths.namespace_entry_path(bucket, &namespace); + let namespace_lock_attempts = backend + .write_lock_acquisition_count(RUSTFS_META_BUCKET, &namespace_path) + .await; let drop_store = store.clone(); let namespace_name = namespace.public_name(); - let mut drop_namespace = tokio::spawn(async move { drop_store.drop_namespace(bucket, &namespace_name).await }); - assert!( - tokio::time::timeout(std::time::Duration::from_millis(50), &mut drop_namespace) + let drop_namespace = tokio::spawn(async move { drop_store.drop_namespace(bucket, &namespace_name).await }); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { + while backend + .write_lock_acquisition_count(RUSTFS_META_BUCKET, &namespace_path) .await - .is_err() - ); + == namespace_lock_attempts + { + tokio::task::yield_now().await; + } + }) + .await + .expect("namespace drop should attempt the namespace catalog lock"); + assert!(!drop_namespace.is_finished(), "namespace drop must wait for table creation"); pause.release(); create.await.unwrap().unwrap(); @@ -7338,7 +8987,7 @@ async fn durable_strong_migration_dry_run_reports_ready_catalog_inventory() { assert_eq!(report.table_bucket, bucket); assert_eq!(report.source_kind, TableCatalogBackingKind::ObjectBacked); - assert_eq!(report.target_kind, TableCatalogBackingKind::StrongKvWal); + assert_eq!(report.target_kind, TableCatalogBackingKind::DurableStrongSnapshot); assert_eq!(report.status, TableCatalogBackingMigrationStatus::ReadyToSnapshot); assert_eq!(report.namespace_count, 1); assert_eq!(report.table_count, 1); @@ -7721,6 +9370,144 @@ async fn durable_strong_migration_cancel_restores_object_backed_writes_before_ta assert_eq!(result.table.generation, 2); } +#[tokio::test] +async fn durable_strong_migration_cancel_rejects_unrelated_strong_snapshot_changes() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata).await; + object_store + .materialize_durable_strong_backing_migration(bucket) + .await + .expect("migration should materialize the source bucket"); + + StrongTableCatalogStore::new(backend.clone()) + .put_table_bucket(test_bucket_entry("research")) + .await + .expect("unrelated strong state should advance the global snapshot"); + + let error = object_store + .cancel_durable_strong_backing_migration(bucket) + .await + .expect_err("cancellation must not reopen object writes after any strong snapshot change"); + assert_matches!( + error, + TableCatalogStoreError::Conflict(message) if message.contains("snapshot advanced after materialization") + ); + assert_matches!( + object_store + .create_namespace(test_namespace_entry(bucket, &Namespace::parse("blocked").unwrap())) + .await, + Err(TableCatalogStoreError::Conflict(message)) if message.contains("writes are fenced") + ); +} + +#[tokio::test] +async fn durable_strong_migration_cancel_serializes_with_strong_snapshot_writes() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata).await; + object_store + .materialize_durable_strong_backing_migration(bucket) + .await + .expect("migration should materialize the source bucket"); + + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + let paused_put = backend.pause_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let strong_store = StrongTableCatalogStore::new(backend.clone()); + let strong_write = tokio::spawn(async move { + strong_store + .update_namespace_properties( + bucket, + &namespace.public_name(), + NamespacePropertiesUpdate::try_new(Vec::new(), BTreeMap::from([("owner".to_string(), "platform".to_string())])) + .expect("namespace update should validate"), + ) + .await + }); + paused_put.wait_started().await; + + let global_lock = object_store.paths.backing_migration_global_fence_lock_path(); + let lock_attempts = backend.write_lock_acquisition_count(RUSTFS_META_BUCKET, &global_lock).await; + let cancel_store = object_store.clone(); + let cancel = tokio::spawn(async move { cancel_store.cancel_durable_strong_backing_migration(bucket).await }); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { + while backend.write_lock_acquisition_count(RUSTFS_META_BUCKET, &global_lock).await == lock_attempts { + tokio::task::yield_now().await; + } + }) + .await + .expect("cancellation should attempt the global migration write lock"); + assert!(!cancel.is_finished(), "cancellation must wait for the in-flight strong snapshot write"); + + paused_put.release(); + strong_write + .await + .expect("strong writer task should join") + .expect("strong snapshot write should complete"); + let error = cancel + .await + .expect("cancellation task should join") + .expect_err("cancellation must reject the advanced strong snapshot"); + assert_matches!( + error, + TableCatalogStoreError::Conflict(message) if message.contains("changed after materializing") + ); +} + +#[tokio::test] +async fn durable_strong_migration_cancel_retries_after_fence_delete_failure() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + seed_table_for_metadata_maintenance(&object_store, bucket, &namespace, &table, current_metadata).await; + object_store + .materialize_durable_strong_backing_migration(bucket) + .await + .unwrap(); + + let fence_path = object_store.paths.backing_migration_fence_path(bucket); + backend.fail_delete_attempt(RUSTFS_META_BUCKET, &fence_path, 1).await; + let first = object_store + .cancel_durable_strong_backing_migration(bucket) + .await + .expect_err("fence deletion failure should leave object-backed writes fenced"); + assert_matches!(first, TableCatalogStoreError::Internal(message) if message.contains("injected delete failure")); + assert!( + StrongTableCatalogStore::new(backend.clone()) + .get_table_bucket(bucket) + .await + .expect("strong snapshot should remain readable") + .is_none() + ); + assert_matches!( + object_store + .create_namespace(test_namespace_entry(bucket, &Namespace::parse("blocked").unwrap())) + .await, + Err(TableCatalogStoreError::Conflict(message)) if message.contains("writes are fenced") + ); + + let retry = object_store + .cancel_durable_strong_backing_migration(bucket) + .await + .expect("cancellation should resume after the target snapshot was already removed"); + assert_eq!(retry.status, TableCatalogBackingMigrationCancelStatus::FenceReleased); + object_store + .create_namespace(test_namespace_entry(bucket, &Namespace::parse("unblocked").unwrap())) + .await + .expect("object-backed writes should resume after the retry removes the fence"); +} + #[tokio::test] async fn durable_strong_migration_waits_for_in_flight_catalog_writes() { let backend = TestCatalogObjectBackend::default(); @@ -7740,12 +9527,26 @@ async fn durable_strong_migration_waits_for_in_flight_catalog_writes() { }); pause.wait_started().await; + let migration_lock = object_store.paths.backing_migration_global_fence_lock_path(); + let migration_attempts = backend + .write_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock) + .await; let migration_store = object_store.clone(); - let mut migration = tokio::spawn(async move { migration_store.materialize_durable_strong_backing_migration(bucket).await }); - assert!( - tokio::time::timeout(std::time::Duration::from_millis(50), &mut migration) + let migration = tokio::spawn(async move { migration_store.materialize_durable_strong_backing_migration(bucket).await }); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { + while backend + .write_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock) .await - .is_err() + == migration_attempts + { + tokio::task::yield_now().await; + } + }) + .await + .expect("migration should attempt to acquire its global write fence"); + assert!( + !migration.is_finished(), + "migration must wait for the catalog write's migration read permit" ); pause.release(); @@ -7761,6 +9562,230 @@ async fn durable_strong_migration_waits_for_in_flight_catalog_writes() { ); } +#[tokio::test] +async fn durable_strong_migration_waits_for_in_flight_maintenance_heartbeat() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + + let mut report = store + .plan_table_metadata_maintenance(bucket, &namespace.public_name(), table.as_str(), 0) + .await + .expect("maintenance report should be planned"); + report.job.status = TableMetadataMaintenanceJobStatus::Running; + report.job.worker_id = Some("worker-a".to_string()); + report.job.lease_id = "lease-a".to_string(); + store + .put_table_metadata_maintenance_report(&report) + .await + .expect("running maintenance report should persist"); + + let job_path = store + .paths + .table_maintenance_job_path(bucket, &namespace, &table, &report.job.table_id, &report.job.job_id); + let pause = backend.pause_next_put(RUSTFS_META_BUCKET, &job_path).await; + let heartbeat_store = store.clone(); + let heartbeat_job_id = report.job.job_id.clone(); + let heartbeat = tokio::spawn(async move { + heartbeat_store + .heartbeat_table_metadata_maintenance_job(bucket, "sales", "orders", &heartbeat_job_id, "lease-a", "worker-a") + .await + }); + pause.wait_started().await; + + let migration_lock = store.paths.backing_migration_global_fence_lock_path(); + let migration_writes = backend + .write_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock) + .await; + let migration_store = store.clone(); + let migration = tokio::spawn(async move { migration_store.materialize_durable_strong_backing_migration(bucket).await }); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { + while backend + .write_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock) + .await + == migration_writes + { + tokio::task::yield_now().await; + } + }) + .await + .expect("migration should attempt to acquire its global write fence"); + assert!(!migration.is_finished(), "migration must wait for the heartbeat's migration read permit"); + + pause.release(); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, heartbeat) + .await + .expect("heartbeat should finish after its report write resumes") + .expect("heartbeat task should join") + .expect("heartbeat should succeed"); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, migration) + .await + .expect("migration should finish after the heartbeat releases its permit") + .expect("migration task should join") + .expect("migration should succeed"); +} + +#[tokio::test] +async fn object_only_catalog_mutations_hold_the_migration_read_permit() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let (namespace, table, report) = + seed_quarantined_table_maintenance(&store, &backend, bucket, OffsetDateTime::UNIX_EPOCH, None).await; + let lock_path = store.paths.backing_migration_fence_lock_path(bucket); + let mut expected = backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &lock_path).await; + + store + .put_external_catalog_bridge(ExternalCatalogBridgeEntry { + version: TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + catalog: "glue".to_string(), + external_catalog_id: None, + external_namespace: "sales".to_string(), + external_table: "orders".to_string(), + external_table_uuid: None, + metadata_location: None, + external_version_token: None, + policy_mode: "rustfs-authoritative".to_string(), + credential_mode: "rustfs-vended".to_string(), + sync_mode: "manual".to_string(), + rollback_strategy: "retain-current".to_string(), + last_sync_status: None, + last_synced_metadata_location: None, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }) + .await + .expect("external catalog bridge should persist"); + expected += 1; + assert_eq!(backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &lock_path).await, expected); + + store + .put_table_bucket_maintenance_config(bucket, TableMaintenanceConfig::default()) + .await + .expect("table-bucket maintenance config should persist"); + expected += 1; + assert_eq!(backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &lock_path).await, expected); + + store + .put_table_maintenance_config(bucket, &namespace.public_name(), table.as_str(), TableMaintenanceConfig::default()) + .await + .expect("table maintenance config should persist"); + expected += 1; + assert_eq!(backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &lock_path).await, expected); + + store + .put_table_metadata_maintenance_report(&report) + .await + .expect("maintenance report should persist"); + expected += 1; + assert_eq!(backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &lock_path).await, expected); + + store + .apply_table_maintenance_quarantine_operation( + bucket, + &namespace.public_name(), + table.as_str(), + &report.job.job_id, + TableMaintenanceQuarantineOperationRequest { + action: TableMaintenanceQuarantineAction::Retry, + reason: Some("migration permit coverage".to_string()), + }, + ) + .await + .expect("quarantine retry should persist under one migration permit"); + expected += 1; + assert_eq!(backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &lock_path).await, expected); + + store + .backfill_table_warehouse_index(bucket) + .await + .expect("warehouse index backfill should remain idempotent"); + expected += 1; + assert_eq!(backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &lock_path).await, expected); +} + +#[tokio::test] +async fn object_catalog_rejects_misplaced_bridge_and_maintenance_state() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await; + backend + .seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec()) + .await; + + let bridge_path = store.paths.external_catalog_bridge_path(bucket, &namespace, &table); + let misplaced_bridge = ExternalCatalogBridgeEntry { + version: TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION, + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: "invoices".to_string(), + catalog: "glue".to_string(), + external_catalog_id: None, + external_namespace: "sales".to_string(), + external_table: "orders".to_string(), + external_table_uuid: None, + metadata_location: None, + external_version_token: None, + policy_mode: "rustfs-authoritative".to_string(), + credential_mode: "rustfs-vended".to_string(), + sync_mode: "manual".to_string(), + rollback_strategy: "retain-current".to_string(), + last_sync_status: None, + last_synced_metadata_location: None, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }; + backend + .seed_object( + RUSTFS_META_BUCKET, + &bridge_path, + serde_json::to_vec(&misplaced_bridge).expect("bridge should encode"), + ) + .await; + assert_matches!( + store.get_external_catalog_bridge(bucket, "sales", "orders").await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("bridge identity") + ); + + let mut misplaced_report = store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .expect("maintenance report should be planned"); + misplaced_report.job.table_id = "different-table-id".to_string(); + let current_job_path = store + .paths + .table_maintenance_current_job_path(bucket, &namespace, &table, "table-id"); + backend + .seed_object( + RUSTFS_META_BUCKET, + ¤t_job_path, + serde_json::to_vec(&misplaced_report).expect("maintenance report should encode"), + ) + .await; + assert_matches!( + store + .get_table_metadata_maintenance_report(bucket, "sales", "orders", MAINTENANCE_JOB_ALIAS_CURRENT) + .await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("maintenance report identity") + ); +} + #[tokio::test] async fn durable_strong_migration_retry_recovers_after_fence_finalization_failure() { let backend = TestCatalogObjectBackend::default(); @@ -7794,6 +9819,324 @@ async fn durable_strong_migration_retry_recovers_after_fence_finalization_failur assert!(retry.ready_to_enable_durable_strong); } +#[tokio::test] +async fn durable_strong_migration_retry_recovers_after_initial_snapshot_write_failure() { + let backend = TestCatalogObjectBackend { + strong_runtime: Some(StrongTableCatalogRuntime::default()), + ..TestCatalogObjectBackend::default() + }; + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + object_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + object_store.backfill_table_warehouse_index(bucket).await.unwrap(); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + backend.fail_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + + assert_matches!( + object_store.materialize_durable_strong_backing_migration(bucket).await, + Err(TableCatalogStoreError::Internal(message)) if message.contains("injected put failure") + ); + + let retry = object_store + .materialize_durable_strong_backing_migration(bucket) + .await + .expect("migration retry should restore the known-absent target baseline"); + assert_eq!(retry.status, TableCatalogBackingMigrationExecutionStatus::SnapshotMaterialized); + assert!(retry.ready_to_enable_durable_strong); +} + +#[tokio::test] +async fn durable_strong_migration_cancel_recovers_after_initial_snapshot_write_failure() { + let backend = TestCatalogObjectBackend { + strong_runtime: Some(StrongTableCatalogRuntime::default()), + ..TestCatalogObjectBackend::default() + }; + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + object_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + object_store.backfill_table_warehouse_index(bucket).await.unwrap(); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + backend.fail_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + + assert_matches!( + object_store.materialize_durable_strong_backing_migration(bucket).await, + Err(TableCatalogStoreError::Internal(message)) if message.contains("injected put failure") + ); + assert_matches!( + object_store + .create_namespace(test_namespace_entry(bucket, &Namespace::parse("blocked").unwrap())) + .await, + Err(TableCatalogStoreError::Conflict(message)) if message.contains("writes are fenced") + ); + + let cancelled = object_store + .cancel_durable_strong_backing_migration(bucket) + .await + .expect("migration cancellation should restore the known-absent target baseline"); + assert_eq!(cancelled.status, TableCatalogBackingMigrationCancelStatus::FenceReleased); + object_store + .create_namespace(test_namespace_entry(bucket, &Namespace::parse("unblocked").unwrap())) + .await + .expect("object-backed writes should resume after cancellation"); +} + +#[tokio::test] +async fn durable_strong_migration_cancel_rejects_a_missing_materialized_snapshot() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + object_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + object_store.backfill_table_warehouse_index(bucket).await.unwrap(); + object_store + .materialize_durable_strong_backing_migration(bucket) + .await + .expect("migration should materialize"); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + backend + .delete_object(RUSTFS_META_BUCKET, &snapshot_path) + .await + .expect("materialized snapshot should be removed for the fault injection"); + + assert_matches!( + object_store.cancel_durable_strong_backing_migration(bucket).await, + Err(TableCatalogStoreError::Conflict(message)) if message.contains("snapshot is missing") + ); + assert_matches!( + object_store + .create_namespace(test_namespace_entry(bucket, &Namespace::parse("blocked").unwrap())) + .await, + Err(TableCatalogStoreError::Conflict(message)) if message.contains("writes are fenced") + ); +} + +#[tokio::test] +async fn durable_strong_migration_cancel_fails_closed_for_a_v1_preparing_fence_without_a_snapshot() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + object_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + object_store.backfill_table_warehouse_index(bucket).await.unwrap(); + let fence_path = object_store.paths.backing_migration_fence_path(bucket); + backend + .seed_object( + RUSTFS_META_BUCKET, + &fence_path, + serde_json::to_vec(&serde_json::json!({ + "version": TABLE_CATALOG_MIGRATION_MIN_READ_VERSION, + "table_bucket": bucket, + "migration_id": "legacy-migration", + "status": "PREPARING", + "target_bucket_existed": false, + "source_fingerprint": null, + "target_snapshot_etag": null + })) + .expect("legacy migration fence should encode"), + ) + .await; + + assert_matches!( + object_store.cancel_durable_strong_backing_migration(bucket).await, + Err(TableCatalogStoreError::Conflict(message)) if message.contains("snapshot is missing") + ); + assert!( + backend + .object_exists(RUSTFS_META_BUCKET, &fence_path) + .await + .expect("legacy migration fence lookup should succeed") + ); +} + +#[tokio::test] +async fn durable_strong_migration_rejects_unknown_bucket_and_global_fence_versions() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + object_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + object_store.backfill_table_warehouse_index(bucket).await.unwrap(); + let fence_path = object_store.paths.backing_migration_fence_path(bucket); + backend + .seed_object( + RUSTFS_META_BUCKET, + &fence_path, + serde_json::to_vec(&serde_json::json!({ + "version": TABLE_CATALOG_MIGRATION_VERSION + 1, + "table_bucket": bucket, + "migration_id": "future-migration", + "status": "PREPARING", + "target_bucket_existed": false, + "source_fingerprint": null, + "target_snapshot_etag": null + })) + .expect("future migration fence should encode"), + ) + .await; + assert_matches!( + object_store.plan_durable_strong_backing_migration(bucket).await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("invalid durable strong migration fence") + ); + + backend + .delete_object(RUSTFS_META_BUCKET, &fence_path) + .await + .expect("future bucket fence should be removed"); + let global_fence_path = object_store.paths.backing_migration_global_fence_path(); + backend + .seed_object( + RUSTFS_META_BUCKET, + &global_fence_path, + serde_json::to_vec(&serde_json::json!({ + "version": TABLE_CATALOG_MIGRATION_VERSION + 1, + "migration_id": "future-global-migration" + })) + .expect("future global migration fence should encode"), + ) + .await; + assert_matches!( + object_store.plan_durable_strong_backing_migration(bucket).await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("invalid durable strong global migration fence") + ); + assert_matches!( + object_store.materialize_durable_strong_backing_migration(bucket).await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("invalid durable strong global migration fence") + ); + assert_matches!( + object_store.cancel_durable_strong_backing_migration(bucket).await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("invalid durable strong global migration fence") + ); + assert!( + backend + .object_exists(RUSTFS_META_BUCKET, &global_fence_path) + .await + .expect("future global migration fence lookup should succeed") + ); + assert!( + !backend + .object_exists(RUSTFS_META_BUCKET, &fence_path) + .await + .expect("bucket migration fence lookup should succeed") + ); +} + +#[tokio::test] +async fn durable_strong_migration_rejects_commit_log_identity_outside_its_object_path() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let next_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current_metadata.clone()).await; + backend.seed_object(bucket, &next_metadata, b"{}".to_vec()).await; + store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: next_metadata, + requirements: Vec::new(), + writer: None, + }) + .await + .expect("commit should succeed"); + + let correct_path = store.paths.commit_log_entry_path(bucket, "table-id", "commit-1"); + let misplaced_path = store.paths.commit_log_entry_path(bucket, "table-id", "different-commit"); + let commit = backend + .read_object(RUSTFS_META_BUCKET, &correct_path) + .await + .expect("commit lookup should succeed") + .expect("commit should exist") + .data; + backend + .delete_object(RUSTFS_META_BUCKET, &correct_path) + .await + .expect("correct commit path should be removed"); + backend.seed_object(RUSTFS_META_BUCKET, &misplaced_path, commit).await; + + assert_matches!( + store.get_commit_by_id(bucket, "table-id", "different-commit").await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("commit log identity") + ); + assert_matches!( + store.plan_durable_strong_backing_migration(bucket).await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("commit log identity") + ); + assert!( + !backend + .object_exists(RUSTFS_META_BUCKET, &store.paths.backing_migration_fence_path(bucket)) + .await + .expect("migration fence lookup should succeed") + ); +} + +#[tokio::test] +async fn durable_strong_migration_rejects_idempotency_identity_outside_its_object_path() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let next_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current_metadata.clone()).await; + backend.seed_object(bucket, &next_metadata, b"{}".to_vec()).await; + store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("request-1".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: next_metadata, + requirements: Vec::new(), + writer: None, + }) + .await + .expect("commit should succeed"); + + let correct_path = store.paths.commit_idempotency_entry_path(bucket, "table-id", "request-1"); + let misplaced_path = store + .paths + .commit_idempotency_entry_path(bucket, "table-id", "different-request"); + let index = backend + .read_object(RUSTFS_META_BUCKET, &correct_path) + .await + .expect("idempotency lookup should succeed") + .expect("idempotency index should exist") + .data; + backend + .delete_object(RUSTFS_META_BUCKET, &correct_path) + .await + .expect("correct idempotency path should be removed"); + backend.seed_object(RUSTFS_META_BUCKET, &misplaced_path, index).await; + + assert_matches!( + store + .get_commit_by_idempotency_key(bucket, "table-id", "different-request") + .await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("commit idempotency identity") + ); + assert_matches!( + store.plan_durable_strong_backing_migration(bucket).await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("commit idempotency identity") + ); + assert!( + !backend + .object_exists(RUSTFS_META_BUCKET, &store.paths.backing_migration_fence_path(bucket)) + .await + .expect("migration fence lookup should succeed") + ); +} + #[tokio::test] async fn durable_strong_migration_cancel_preserves_preexisting_target_state() { let backend = TestCatalogObjectBackend::default(); @@ -7874,6 +10217,52 @@ async fn durable_strong_migration_requires_every_table_bucket_before_cutover() { ); } +#[tokio::test] +async fn durable_strong_migration_rejects_target_buckets_outside_the_source_inventory() { + let backend = TestCatalogObjectBackend::default(); + let object_store = ObjectTableCatalogStore::new(backend.clone()); + object_store + .put_table_bucket(test_bucket_entry("analytics")) + .await + .expect("source table bucket should be created"); + object_store + .backfill_table_warehouse_index("analytics") + .await + .expect("source warehouse index should be ready"); + StrongTableCatalogStore::new(backend.clone()) + .put_table_bucket(test_bucket_entry("stale-target")) + .await + .expect("stale target table bucket should be seeded"); + + let report = object_store + .plan_durable_strong_backing_migration("analytics") + .await + .expect("migration preflight should report the stale target state"); + assert_eq!(report.status, TableCatalogBackingMigrationStatus::ManualReviewRequired); + assert!( + report + .blockers + .contains(&TableCatalogBackingMigrationBlocker::DurableStrongSnapshotChanged) + ); + assert!( + report + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::ReviewDurableStrongSnapshot) + ); + assert_matches!( + object_store + .materialize_durable_strong_backing_migration("analytics") + .await, + Err(TableCatalogStoreError::Conflict(message)) if message.contains("not ready") + ); + assert!( + !backend + .object_exists(RUSTFS_META_BUCKET, &object_store.paths.backing_migration_fence_path("analytics"),) + .await + .expect("migration fence lookup should succeed") + ); +} + #[tokio::test] async fn durable_strong_migration_dry_run_reports_recovery_blockers() { let backend = TestCatalogObjectBackend::default(); @@ -7989,6 +10378,273 @@ async fn durable_strong_migration_dry_run_requires_ready_warehouse_index() { ); } +#[tokio::test] +async fn durable_strong_migration_rejects_overlapping_warehouse_prefixes_before_fencing() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let parent = IdentifierSegment::parse("orders").expect("table should parse"); + let child = IdentifierSegment::parse("order_items").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &parent, "00001.metadata.json"); + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &parent, current_metadata).await; + + let mut child_entry = test_table_entry( + bucket, + &namespace, + &child, + default_table_metadata_file_path(&namespace, &child, "00001.metadata.json"), + ); + child_entry.table_id = "table-id-child".to_string(); + child_entry.table_uuid = "table-uuid-child".to_string(); + child_entry.warehouse_location = format!("s3://{bucket}/tables/table-id/child"); + store + .write_entry( + store.catalog_bucket(), + &store.paths.table_entry_path(bucket, &namespace, &child), + &child_entry, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("legacy overlapping table should be seeded"); + + assert_matches!( + scan_table_data_plane_resource_for_object( + &store, + bucket, + "tables/table-id/child/data/part-00001.parquet", + ) + .await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("overlapping active table warehouse prefixes") + ); + + let report = store + .plan_durable_strong_backing_migration(bucket) + .await + .expect("migration dry run should report the overlap"); + + assert_eq!(report.status, TableCatalogBackingMigrationStatus::ManualReviewRequired); + assert!(report.warehouse_index_ready); + assert_eq!(report.warehouse_prefix_count, 2); + assert!( + report + .blockers + .contains(&TableCatalogBackingMigrationBlocker::DuplicateWarehousePrefix) + ); + assert!( + report + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::ReviewDuplicateWarehousePrefixes) + ); + assert_matches!( + store.materialize_durable_strong_backing_migration(bucket).await, + Err(TableCatalogStoreError::Conflict(_)) + ); + assert!( + !backend + .object_exists(RUSTFS_META_BUCKET, &store.paths.backing_migration_fence_path(bucket)) + .await + .expect("migration fence lookup should succeed") + ); +} + +#[tokio::test] +async fn durable_strong_migration_rejects_committed_log_outside_current_history() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current_metadata.clone()).await; + let commit = CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: "commit-1".to_string(), + idempotency_key: None, + table_id: "table-id".to_string(), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + new_version_token: "token-v2".to_string(), + previous_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + status: CommitLogStatus::Committed, + writer: None, + created_at: None, + updated_at: None, + }; + store + .write_entry( + store.catalog_bucket(), + &store.paths.commit_log_entry_path(bucket, "table-id", "commit-1"), + &commit, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("disconnected committed log should be seeded"); + + let report = store + .plan_durable_strong_backing_migration(bucket) + .await + .expect("migration dry run should report disconnected history"); + + assert_eq!(report.status, TableCatalogBackingMigrationStatus::ManualReviewRequired); + assert!( + report + .blockers + .contains(&TableCatalogBackingMigrationBlocker::CommitManualReviewRequired) + ); + assert_matches!( + store.materialize_durable_strong_backing_migration(bucket).await, + Err(TableCatalogStoreError::Conflict(message)) if message.contains("not ready") + ); + assert!( + !backend + .object_exists(RUSTFS_META_BUCKET, &store.paths.backing_migration_fence_path(bucket)) + .await + .expect("migration fence lookup should succeed") + ); +} + +#[tokio::test] +async fn durable_strong_migration_rejects_duplicate_table_ids_before_fencing() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let orders = IdentifierSegment::parse("orders").expect("table should parse"); + let invoices = IdentifierSegment::parse("invoices").expect("table should parse"); + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry( + bucket, + &namespace, + &orders, + default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"), + )) + .await + .unwrap(); + let mut duplicate = test_table_entry( + bucket, + &namespace, + &invoices, + default_table_metadata_file_path(&namespace, &invoices, "00001.metadata.json"), + ); + duplicate.table_uuid = "duplicate-table-uuid".to_string(); + duplicate.warehouse_location = format!("s3://{bucket}/tables/duplicate-table-location"); + store + .write_entry( + store.catalog_bucket(), + &store.paths.table_entry_path(bucket, &namespace, &invoices), + &duplicate, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("legacy duplicate table id should be seeded"); + + let report = store.plan_durable_strong_backing_migration(bucket).await.unwrap(); + + assert_eq!(report.status, TableCatalogBackingMigrationStatus::ManualReviewRequired); + assert!( + report + .blockers + .contains(&TableCatalogBackingMigrationBlocker::DuplicateTableIdentity) + ); + assert!( + report + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::ReviewDuplicateTableIdentities) + ); + assert_matches!( + store.materialize_durable_strong_backing_migration(bucket).await, + Err(TableCatalogStoreError::Conflict(message)) if message.contains("not ready") + ); + for fence_path in [ + store.paths.backing_migration_global_fence_path(), + store.paths.backing_migration_fence_path(bucket), + ] { + assert!( + !backend + .object_exists(RUSTFS_META_BUCKET, &fence_path) + .await + .expect("migration fence lookup should succeed") + ); + } +} + +#[tokio::test] +async fn durable_strong_migration_rejects_table_view_collision_before_fencing() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let identifier = IdentifierSegment::parse("orders").expect("identifier should parse"); + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry( + bucket, + &namespace, + &identifier, + default_table_metadata_file_path(&namespace, &identifier, "00001.metadata.json"), + )) + .await + .unwrap(); + let view = test_view_entry( + bucket, + &namespace, + &identifier, + default_view_metadata_file_path(&namespace, &identifier, "00001.metadata.json"), + ); + store + .write_entry( + store.catalog_bucket(), + &store.paths.view_entry_path(bucket, &namespace, &identifier), + &view, + TableCatalogPutPrecondition::Any, + ) + .await + .expect("legacy colliding view should be seeded"); + store.backfill_table_warehouse_index(bucket).await.unwrap(); + + let report = store.plan_durable_strong_backing_migration(bucket).await.unwrap(); + + assert_eq!(report.status, TableCatalogBackingMigrationStatus::ManualReviewRequired); + assert!( + report + .blockers + .contains(&TableCatalogBackingMigrationBlocker::TableViewIdentifierCollision) + ); + assert!( + report + .recommended_actions + .contains(&TableCatalogBackingMigrationAction::ReviewTableViewIdentifierCollisions) + ); + assert_matches!( + store.materialize_durable_strong_backing_migration(bucket).await, + Err(TableCatalogStoreError::Conflict(message)) if message.contains("not ready") + ); + for fence_path in [ + store.paths.backing_migration_global_fence_path(), + store.paths.backing_migration_fence_path(bucket), + ] { + assert!( + !backend + .object_exists(RUSTFS_META_BUCKET, &fence_path) + .await + .expect("migration fence lookup should succeed") + ); + } +} + #[tokio::test] async fn strong_catalog_backing_commit_is_atomic_with_wal_and_idempotency() { let backend = TestCatalogObjectBackend::default(); @@ -8100,12 +10756,12 @@ async fn strong_catalog_pagination_uses_ordered_state_and_rejects_object_cursors .list_views_page(bucket, &namespace_name, None, one) .await .expect("first strong view page should load"); - assert_eq!(first.entries[0].view, "alpha"); + assert_eq!(first.entries[0].view, "view_alpha"); let second = store .list_views_page(bucket, &namespace_name, first.next_cursor.as_deref(), one) .await .expect("second strong view page should load"); - assert_eq!(second.entries[0].view, "beta"); + assert_eq!(second.entries[0].view, "view_beta"); assert!(second.next_cursor.is_none()); let exact = NonZeroUsize::new(2).expect("exact page size should be non-zero"); @@ -8220,6 +10876,121 @@ async fn strong_catalog_backing_replays_durable_commit_state_after_restart() { assert_eq!(recovery.manual_review_count, 0); } +#[tokio::test] +async fn strong_catalog_exact_commit_replay_does_not_rewrite_snapshot() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .expect("table should be created"); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + let request = TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client-request".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }; + let committed = store.commit_table(request.clone()).await.expect("commit should succeed"); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + let put_count = backend.put_attempt_count(RUSTFS_META_BUCKET, &snapshot_path).await; + + let replay = store.commit_table(request).await.expect("exact replay should succeed"); + + assert_eq!(replay, committed); + assert_eq!(backend.put_attempt_count(RUSTFS_META_BUCKET, &snapshot_path).await, put_count); +} + +#[tokio::test] +async fn strong_catalog_drop_removes_commit_indexes_before_restart() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .expect("table should be created"); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client-request".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }) + .await + .expect("commit should succeed"); + + store + .drop_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("table should be dropped"); + + let restarted = StrongTableCatalogStore::new(backend); + assert!( + restarted + .load_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("snapshot should hydrate after table drop") + .is_none() + ); + assert!( + restarted + .get_commit_by_id(bucket, "table-id", "commit-1") + .await + .expect("commit lookup should succeed") + .is_none() + ); + assert!( + restarted + .get_commit_by_idempotency_key(bucket, "table-id", "client-request") + .await + .expect("idempotency lookup should succeed") + .is_none() + ); +} + #[tokio::test] async fn strong_catalog_backing_rejects_stale_snapshot_cas_after_concurrent_restart() { let backend = TestCatalogObjectBackend::default(); @@ -8311,6 +11082,139 @@ async fn strong_catalog_backing_rejects_stale_snapshot_cas_after_concurrent_rest ); } +#[tokio::test] +async fn strong_catalog_replays_identical_commit_after_snapshot_cas_loss() { + let backend = TestCatalogObjectBackend::default(); + let first_store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + + first_store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + first_store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + first_store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone())) + .await + .expect("table should be created"); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + + let request = TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("request-1".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }; + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + let pause = backend.pause_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let first_request = request.clone(); + let commit_store = first_store.clone(); + let first_writer = tokio::spawn(async move { + commit_store + .commit_table_with_publication(first_request, &UnserializedTestPublication) + .await + }); + pause.wait_started().await; + let winner = StrongTableCatalogStore::new(backend.clone()) + .commit_table_with_publication(request.clone(), &UnserializedTestPublication) + .await; + let winner = winner.expect("independent competing writer should commit the shared request"); + pause.release(); + let replay = first_writer + .await + .expect("first writer task should join") + .expect("CAS loser should replay the durable winner"); + + assert_eq!(replay.table.metadata_location, winner.table.metadata_location); + assert_eq!(replay.table.version_token, winner.table.version_token); + assert_eq!(replay.table.generation, winner.table.generation); + assert_eq!(replay.commit_log, winner.commit_log); + + let second_metadata = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json"); + backend.seed_object(bucket, &second_metadata, b"{}".to_vec()).await; + let second_request = TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-2".to_string(), + idempotency_key: Some("request-2".to_string()), + operation: "append".to_string(), + expected_version_token: replay.table.version_token.clone(), + expected_metadata_location: replay.table.metadata_location.clone(), + new_metadata_location: second_metadata, + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }; + let pause = backend.pause_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let commit_store = first_store.clone(); + let staged_request = second_request.clone(); + let staged_writer = tokio::spawn(async move { + commit_store + .commit_table_with_publication(staged_request, &UnserializedTestPublication) + .await + }); + pause.wait_started().await; + let committed_winner = StrongTableCatalogStore::new(backend.clone()) + .commit_table_with_publication(second_request.clone(), &UnserializedTestPublication) + .await + .expect("independent competing writer should commit the second request"); + assert_eq!(committed_winner.commit_log.status, CommitLogStatus::Committed); + let mut staged_snapshot = read_strong_snapshot(&backend).await; + for record in staged_snapshot + .commits + .iter_mut() + .chain(staged_snapshot.idempotency.iter_mut()) + .filter(|record| record.commit.commit_id == second_request.commit_id) + { + record.commit.status = CommitLogStatus::Staged; + } + seed_strong_snapshot(&backend, &staged_snapshot).await; + pause.release(); + + assert_matches!( + staged_writer.await.expect("staged writer task should join"), + Err(TableCatalogStoreError::Conflict(_)) + ); + assert_eq!( + first_store + .get_commit_by_id(bucket, "table-id", "commit-2") + .await + .expect("staged commit lookup should succeed") + .expect("staged durable winner should remain visible") + .status, + CommitLogStatus::Staged + ); + + let finalized = first_store + .commit_table(second_request) + .await + .expect("retry should durably finalize the staged winner"); + assert_eq!(finalized.commit_log.status, CommitLogStatus::Committed); + assert_eq!( + first_store + .get_commit_by_id(bucket, "table-id", "commit-2") + .await + .expect("finalized commit lookup should succeed") + .expect("finalized commit should exist") + .status, + CommitLogStatus::Committed + ); +} + #[tokio::test] async fn strong_catalog_backing_refreshes_hydrated_reads_after_independent_commit() { let backend = TestCatalogObjectBackend::default(); @@ -8369,6 +11273,367 @@ async fn strong_catalog_backing_refreshes_hydrated_reads_after_independent_commi assert_eq!(loaded_after_commit.version_token, result.table.version_token); } +#[tokio::test] +async fn strong_catalog_does_not_install_stale_concurrent_reload() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + + let pause = backend.pause_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; + let stale_store = store.clone(); + let stale_reload = tokio::spawn(async move { stale_store.reload_state_from_durable().await }); + pause.wait_started().await; + + let writer = store.clone(); + let namespace_name = namespace.public_name(); + let write = tokio::spawn(async move { writer.create_namespace(test_namespace_entry(bucket, &namespace)).await }); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { + loop { + let durable = read_strong_snapshot(&backend).await; + if durable.namespaces.iter().any(|entry| entry.namespace == namespace_name) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("namespace snapshot should become durable while stale reload is paused"); + pause.release(); + write + .await + .expect("namespace writer task should join") + .expect("namespace should be durably created while stale reload is paused"); + stale_reload + .await + .expect("stale reload task should join") + .expect("stale reload must not replace the current snapshot"); + + assert!( + store + .get_namespace(bucket, "sales") + .await + .expect("namespace lookup should succeed") + .is_some() + ); +} + +#[tokio::test] +async fn strong_catalog_bounds_reload_under_repeated_local_state_changes() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + + store + .put_table_bucket(test_bucket_entry("analytics")) + .await + .expect("table bucket should be created"); + + let mut pause = backend.pause_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; + let reload_store = store.clone(); + let reload = tokio::spawn(async move { reload_store.reload_state_from_durable().await }); + for attempt in 0..STRONG_TABLE_CATALOG_RELOAD_MAX_ATTEMPTS { + pause.wait_started().await; + let next_pause = if attempt + 1 < STRONG_TABLE_CATALOG_RELOAD_MAX_ATTEMPTS { + Some(backend.pause_next_read(RUSTFS_META_BUCKET, &snapshot_path).await) + } else { + None + }; + store.state.lock().await.snapshot_etag = Some(format!("concurrent-etag-{attempt}")); + pause.release(); + if let Some(next_pause) = next_pause { + pause = next_pause; + } + } + + let error = reload + .await + .expect("reload task should join") + .expect_err("repeated local state changes must stop after the retry bound"); + assert_matches!(error, TableCatalogStoreError::Internal(message) if message.contains("changed repeatedly")); +} + +#[tokio::test] +async fn strong_catalog_fails_closed_when_observed_snapshot_disappears() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + let snapshot = backend + .read_object(RUSTFS_META_BUCKET, &snapshot_path) + .await + .expect("snapshot read should succeed") + .expect("snapshot should exist"); + backend + .delete_object(RUSTFS_META_BUCKET, &snapshot_path) + .await + .expect("snapshot deletion should be injected"); + + assert_matches!( + store.get_table_bucket(bucket).await, + Err(TableCatalogStoreError::Internal(message)) if message.contains("snapshot disappeared") + ); + assert_matches!( + store.put_table_bucket(test_bucket_entry("replacement")).await, + Err(TableCatalogStoreError::Internal(message)) if message.contains("snapshot disappeared") + ); + assert!(store.state.lock().await.table_buckets.contains_key(bucket)); + assert!( + backend + .read_object(RUSTFS_META_BUCKET, &snapshot_path) + .await + .expect("snapshot lookup should succeed") + .is_none() + ); + backend.seed_object(RUSTFS_META_BUCKET, &snapshot_path, snapshot.data).await; + assert_eq!( + store + .get_table_bucket(bucket) + .await + .expect("restored snapshot should reload") + .expect("table bucket should be restored") + .table_bucket, + bucket + ); +} + +#[tokio::test] +async fn strong_catalog_configured_mode_requires_snapshot_after_restart() { + let backend = TestCatalogObjectBackend::default(); + let required = StrongTableCatalogStore::new_requiring_snapshot(backend.clone()); + assert_matches!( + required.get_table_bucket("analytics").await, + Err(TableCatalogStoreError::Internal(message)) if message.contains("snapshot disappeared") + ); + + let bootstrap = StrongTableCatalogStore::new(backend.clone()); + bootstrap + .put_table_bucket(test_bucket_entry("analytics")) + .await + .expect("migration path should materialize the first snapshot"); + let restarted = StrongTableCatalogStore::new_requiring_snapshot(backend.clone()); + assert!( + restarted + .get_table_bucket("analytics") + .await + .expect("configured mode should load a materialized snapshot") + .is_some() + ); + + backend + .delete_object( + RUSTFS_META_BUCKET, + &StrongTableCatalogStore::::snapshot_object_path(), + ) + .await + .expect("snapshot deletion should be injected"); + let restarted_after_loss = StrongTableCatalogStore::new_requiring_snapshot(backend); + assert_matches!( + restarted_after_loss.get_table_bucket("analytics").await, + Err(TableCatalogStoreError::Internal(message)) if message.contains("snapshot disappeared") + ); +} + +#[tokio::test] +async fn strong_catalog_required_reload_does_not_reuse_empty_migration_state() { + let runtime = StrongTableCatalogRuntime::default(); + let backend = TestCatalogObjectBackend { + strong_runtime: Some(runtime), + ..TestCatalogObjectBackend::default() + }; + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + let pause = backend.pause_before_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; + let migration = StrongTableCatalogStore::new(backend.clone()); + let migration_reload = tokio::spawn(async move { migration.get_table_bucket("analytics").await }); + pause.wait_started().await; + + let configured = StrongTableCatalogStore::new_requiring_snapshot(backend); + let reload_attempts = configured.reload_lock_attempts_for_test(); + let configured_task = configured.clone(); + let configured_reload = tokio::spawn(async move { configured_task.get_table_bucket("analytics").await }); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { + while configured.reload_lock_attempts_for_test() == reload_attempts { + tokio::task::yield_now().await; + } + }) + .await + .expect("configured reload should attempt the shared reload lock"); + assert!( + !configured_reload.is_finished(), + "configured reload should wait for the in-flight migration reload" + ); + + pause.release(); + assert!( + migration_reload + .await + .expect("migration reload task should join") + .expect("migration may hydrate an empty target") + .is_none() + ); + assert_matches!( + configured_reload.await.expect("configured reload task should join"), + Err(TableCatalogStoreError::Internal(message)) if message.contains("snapshot disappeared") + ); +} + +#[tokio::test] +async fn strong_catalog_first_confirmed_write_requires_snapshot_after_reload_failure() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + assert!( + store + .get_table_bucket(bucket) + .await + .expect("migration target should hydrate without a snapshot") + .is_none() + ); + backend.fail_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("confirmed snapshot write should survive a local reload failure"); + assert!(!store.is_hydrated_for_test().await); + backend + .delete_object(RUSTFS_META_BUCKET, &snapshot_path) + .await + .expect("snapshot deletion should be injected"); + + assert_matches!( + store.get_table_bucket(bucket).await, + Err(TableCatalogStoreError::Internal(message)) if message.contains("snapshot disappeared") + ); +} + +#[tokio::test] +async fn strong_catalog_first_ambiguous_write_requires_snapshot_after_recovery_failure() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + assert!( + store + .get_table_bucket(bucket) + .await + .expect("migration target should hydrate without a snapshot") + .is_none() + ); + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + backend.fail_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect_err("ambiguous first write should report the original transport failure"); + assert!(!store.is_hydrated_for_test().await); + backend + .delete_object(RUSTFS_META_BUCKET, &snapshot_path) + .await + .expect("snapshot deletion should be injected"); + + assert_matches!( + store.get_table_bucket(bucket).await, + Err(TableCatalogStoreError::Internal(message)) if message.contains("snapshot disappeared") + ); +} + +#[tokio::test] +async fn strong_catalog_does_not_expose_empty_state_during_ambiguous_first_write_recovery() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + store + .get_table_bucket(bucket) + .await + .expect("migration target may start without a snapshot"); + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let recovery_read = backend.pause_before_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; + + let writer = store.clone(); + let write = tokio::spawn(async move { writer.put_table_bucket(test_bucket_entry(bucket)).await }); + recovery_read.wait_started().await; + backend + .delete_object(RUSTFS_META_BUCKET, &snapshot_path) + .await + .expect("snapshot loss should be injected while recovery is in flight"); + + let reload_attempts = store.reload_lock_attempts_for_test(); + let reader = store.clone(); + let load = tokio::spawn(async move { reader.get_table_bucket(bucket).await }); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { + while store.reload_lock_attempts_for_test() == reload_attempts { + tokio::task::yield_now().await; + } + }) + .await + .expect("reader should attempt the shared reload lock"); + assert!( + !load.is_finished(), + "a reader must wait for ambiguous first-write recovery instead of exposing the old empty state" + ); + + recovery_read.release(); + assert_matches!( + write.await.expect("writer task should join"), + Err(TableCatalogStoreError::Internal(message)) if message.contains("post-commit put failure") + ); + assert_matches!( + load.await.expect("reader task should join"), + Err(TableCatalogStoreError::Internal(message)) if message.contains("snapshot disappeared") + ); +} + +#[tokio::test] +async fn strong_catalog_first_write_conflict_requires_snapshot_after_recovery_failure() { + let backend = TestCatalogObjectBackend::default(); + let stale = StrongTableCatalogStore::new(backend.clone()); + let winner = StrongTableCatalogStore::new(backend.clone()); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + stale + .get_table_bucket("missing") + .await + .expect("stale writer should hydrate the empty state"); + + let paused_put = backend.pause_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let stale_writer = stale.clone(); + let stale_write = tokio::spawn(async move { stale_writer.put_table_bucket(test_bucket_entry("stale")).await }); + paused_put.wait_started().await; + winner + .put_table_bucket(test_bucket_entry("winner")) + .await + .expect("winning writer should publish the first snapshot"); + backend.fail_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; + paused_put.release(); + + assert_matches!( + stale_write.await.expect("stale writer task should join"), + Err(TableCatalogStoreError::Conflict(_)) + ); + assert!(!stale.is_hydrated_for_test().await); + backend + .delete_object(RUSTFS_META_BUCKET, &snapshot_path) + .await + .expect("snapshot deletion should be injected"); + assert_matches!( + stale.get_table_bucket("winner").await, + Err(TableCatalogStoreError::Internal(message)) if message.contains("snapshot disappeared") + ); +} + #[tokio::test] async fn strong_catalog_backing_skips_snapshot_body_when_etag_is_unchanged() { let backend = TestCatalogObjectBackend::default(); @@ -8393,6 +11658,84 @@ async fn strong_catalog_backing_skips_snapshot_body_when_etag_is_unchanged() { assert_eq!(backend.list_call_count().await, 0); } +#[tokio::test] +async fn strong_catalog_reload_requires_etag_and_bounds_snapshot_reads() { + let backend = TestCatalogObjectBackend::default(); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let snapshot = test_strong_snapshot(bucket, &namespace, Vec::new(), Vec::new()); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + seed_strong_snapshot(&backend, &snapshot).await; + backend.omit_etag_for_object(RUSTFS_META_BUCKET, &snapshot_path).await; + + let error = StrongTableCatalogStore::new(backend.clone()) + .get_table_bucket(bucket) + .await + .expect_err("a durable snapshot without an etag must fail closed"); + + assert_matches!(error, TableCatalogStoreError::Internal(message) if message.contains("snapshot has no etag")); + assert_eq!( + backend.last_read_limit(RUSTFS_META_BUCKET, &snapshot_path).await, + Some(STRONG_TABLE_CATALOG_SNAPSHOT_MAX_SIZE) + ); +} + +#[tokio::test] +async fn strong_catalog_rejects_oversized_snapshot_before_publication() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + let mut entry = test_bucket_entry("analytics"); + entry + .properties + .insert("oversized".to_string(), "x".repeat(STRONG_TABLE_CATALOG_SNAPSHOT_MAX_SIZE)); + + let error = store + .put_table_bucket(entry) + .await + .expect_err("an oversized durable snapshot must be rejected before publication"); + + assert_matches!( + error, + TableCatalogStoreError::Invalid(message) if message.contains("exceeds the maximum encoded size") + ); + assert!( + backend + .read_object(RUSTFS_META_BUCKET, &snapshot_path) + .await + .expect("snapshot lookup should succeed") + .is_none() + ); +} + +#[tokio::test] +async fn strong_catalog_coalesces_concurrent_snapshot_reloads() { + let backend = TestCatalogObjectBackend::default(); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let snapshot = test_strong_snapshot(bucket, &namespace, Vec::new(), Vec::new()); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + seed_strong_snapshot(&backend, &snapshot).await; + backend.reset_call_counts().await; + let store = StrongTableCatalogStore::new(backend.clone()); + let first = store.clone(); + let second = store.clone(); + let pause = backend.pause_before_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; + let release = pause.clone(); + let release_task = tokio::spawn(async move { + release.wait_started().await; + release.release(); + }); + + let (first_result, second_result) = tokio::join!(first.reload_state_from_durable(), second.reload_state_from_durable()); + release_task.await.expect("reload release task should join"); + + first_result.expect("first reload should succeed"); + second_result.expect("coalesced reload should succeed"); + assert_eq!(backend.read_call_count().await, 1); + assert_eq!(backend.metadata_call_count().await, 1); +} + #[tokio::test] async fn strong_catalog_backing_resolves_data_plane_resource_without_catalog_scan() { let backend = TestCatalogObjectBackend::default(); @@ -8431,6 +11774,95 @@ async fn strong_catalog_backing_resolves_data_plane_resource_without_catalog_sca assert_eq!(backend.list_call_count().await, 0); } +#[tokio::test] +async fn strong_catalog_data_plane_fails_closed_for_missing_bucket_snapshot() { + let store = StrongTableCatalogStore::new(TestCatalogObjectBackend::default()); + + let error = table_data_plane_resource_for_object(&store, "analytics", "tables/table-id/data/file.parquet") + .await + .expect_err("a table-enabled bucket missing from the strong snapshot must fail closed"); + + assert_matches!(error, TableCatalogStoreError::Internal(message) if message.contains("no entry for table-enabled bucket")); +} + +#[tokio::test] +async fn object_catalog_data_plane_fails_closed_for_missing_bucket_entry() { + let store = ObjectTableCatalogStore::new(TestCatalogObjectBackend::default()); + + let error = table_data_plane_resource_for_object(&store, "analytics", "tables/table-id/data/file.parquet") + .await + .expect_err("a table-enabled bucket missing from the object catalog must fail closed"); + + assert_matches!(error, TableCatalogStoreError::Internal(message) if message.contains("no entry for table-enabled bucket")); +} + +#[tokio::test] +async fn catalog_backings_fail_closed_for_inactive_table_bucket_data_plane() { + let bucket = "analytics"; + let object = "tables/table-id/data/file.parquet"; + let mut inactive = test_bucket_entry(bucket); + inactive.state = TableCatalogEntryState::Deleted; + + let object_store = ObjectTableCatalogStore::new(TestCatalogObjectBackend::default()); + object_store + .put_table_bucket(inactive.clone()) + .await + .expect("inactive object-backed bucket should be seeded"); + let object_error = table_data_plane_resource_for_object(&object_store, bucket, object) + .await + .expect_err("inactive object-backed table buckets must fail closed"); + assert_matches!( + object_error, + TableCatalogStoreError::Internal(message) if message.contains("inactive object-backed catalog entry") + ); + + let strong_store = StrongTableCatalogStore::new(TestCatalogObjectBackend::default()); + strong_store + .put_table_bucket(inactive) + .await + .expect("inactive strong bucket should be seeded"); + let strong_error = table_data_plane_resource_for_object(&strong_store, bucket, object) + .await + .expect_err("inactive durable strong table buckets must fail closed"); + assert_matches!( + strong_error, + TableCatalogStoreError::Internal(message) if message.contains("inactive durable strong catalog entry") + ); +} + +#[tokio::test] +async fn strong_catalog_data_plane_requires_v2_snapshot_after_fleet_confirmation() { + let backend = TestCatalogObjectBackend::default(); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let table_entry = test_table_entry( + bucket, + &namespace, + &table, + default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), + ); + seed_strong_snapshot(&backend, &test_strong_snapshot(bucket, &namespace, vec![table_entry], Vec::new())).await; + let store = StrongTableCatalogStore::new_with_snapshot_write_version(backend.clone(), STRONG_TABLE_CATALOG_SNAPSHOT_VERSION); + + let error = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/file.parquet") + .await + .expect_err("fleet-confirmed readers must reject a version 1 snapshot on the data plane"); + assert_matches!(error, TableCatalogStoreError::Internal(message) if message.contains("requires a version 2 snapshot")); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("a catalog write should upgrade the snapshot"); + assert_eq!(read_strong_snapshot(&backend).await.version, STRONG_TABLE_CATALOG_SNAPSHOT_VERSION); + assert!( + table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/file.parquet") + .await + .expect("version 2 data-plane lookup should succeed") + .is_some() + ); +} + #[tokio::test] async fn strong_catalog_backing_does_not_publish_warehouse_index_before_snapshot_persist() { let backend = TestCatalogObjectBackend::default(); @@ -8540,6 +11972,279 @@ async fn strong_catalog_backing_rejects_duplicate_snapshot_warehouse_index_entri assert_matches!(err, TableCatalogStoreError::Invalid(message) if message.contains("overlapping active table warehouse location")); } +#[test] +fn strong_catalog_snapshot_v2_requires_fleet_confirmation() { + for (requested, fleet_confirmed, expected) in [ + (false, false, STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION), + (true, false, STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION), + (false, true, STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION), + (true, true, STRONG_TABLE_CATALOG_SNAPSHOT_VERSION), + ] { + assert_eq!(strong_snapshot_write_version(requested, fleet_confirmed), expected); + } +} + +#[tokio::test] +async fn strong_catalog_reads_literal_v1_snapshot_fixture() { + let backend = TestCatalogObjectBackend::default(); + backend + .seed_object( + RUSTFS_META_BUCKET, + &StrongTableCatalogStore::::snapshot_object_path(), + br#"{ + "version": 1, + "table_buckets": [{ + "version": 1, + "table_bucket": "analytics", + "catalog_type": "iceberg-rest", + "warehouse_root": "s3://analytics/", + "state": "ACTIVE", + "properties": {}, + "created_at": null, + "updated_at": null + }], + "namespaces": [], + "tables": [], + "views": [], + "commits": [], + "idempotency": [] + }"# + .to_vec(), + ) + .await; + + let store = StrongTableCatalogStore::new(backend); + let entry = store + .get_table_bucket("analytics") + .await + .expect("literal version 1 snapshot should load") + .expect("table bucket should exist"); + + assert_eq!(entry, test_bucket_entry("analytics")); +} + +#[tokio::test] +async fn strong_catalog_snapshot_upgrade_preserves_v2_and_rejects_stale_restore() { + let backend = TestCatalogObjectBackend::default(); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let snapshot = test_strong_snapshot(bucket, &namespace, Vec::new(), Vec::new()); + seed_strong_snapshot(&backend, &snapshot).await; + + let legacy = + StrongTableCatalogStore::new_with_snapshot_write_version(backend.clone(), STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION); + let mut legacy_bucket = test_bucket_entry(bucket); + legacy_bucket.properties.insert("writer".to_string(), "legacy".to_string()); + legacy + .put_table_bucket(legacy_bucket) + .await + .expect("legacy-compatible write should succeed"); + assert_eq!( + read_strong_snapshot(&backend).await.version, + STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION + ); + + let fleet = StrongTableCatalogStore::new_with_snapshot_write_version(backend.clone(), STRONG_TABLE_CATALOG_SNAPSHOT_VERSION); + let mut upgraded_bucket = test_bucket_entry(bucket); + upgraded_bucket.properties.insert("writer".to_string(), "fleet".to_string()); + fleet + .put_table_bucket(upgraded_bucket) + .await + .expect("fleet-confirmed write should upgrade the snapshot"); + assert_eq!(read_strong_snapshot(&backend).await.version, STRONG_TABLE_CATALOG_SNAPSHOT_VERSION); + + let compatibility_writer = + StrongTableCatalogStore::new_with_snapshot_write_version(backend.clone(), STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION); + let mut post_upgrade_bucket = test_bucket_entry(bucket); + post_upgrade_bucket + .properties + .insert("writer".to_string(), "compatibility".to_string()); + compatibility_writer + .put_table_bucket(post_upgrade_bucket) + .await + .expect("current binary should preserve a previously upgraded snapshot"); + assert_eq!(read_strong_snapshot(&backend).await.version, STRONG_TABLE_CATALOG_SNAPSHOT_VERSION); + + let restored_snapshot = snapshot; + seed_strong_snapshot(&backend, &restored_snapshot).await; + let mut after_restore_bucket = test_bucket_entry(bucket); + after_restore_bucket + .properties + .insert("writer".to_string(), "post-restore".to_string()); + let error = compatibility_writer + .put_table_bucket(after_restore_bucket) + .await + .expect_err("a running writer that observed version 2 must reject stale version 1 content"); + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("high-water version 2")); + assert_eq!(read_strong_snapshot(&backend).await, restored_snapshot); + assert_matches!( + compatibility_writer.get_table_bucket(bucket).await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("high-water version 2") + ); +} + +#[tokio::test] +async fn shared_strong_runtime_preserves_snapshot_high_water_across_store_instances() { + let runtime = StrongTableCatalogRuntime::default(); + let backend = TestCatalogObjectBackend { + strong_runtime: Some(runtime), + ..TestCatalogObjectBackend::default() + }; + let bucket = "analytics"; + let writer = StrongTableCatalogStore::new_with_snapshot_write_version(backend.clone(), STRONG_TABLE_CATALOG_SNAPSHOT_VERSION); + writer + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("fleet-confirmed writer should publish version 2"); + assert_eq!(read_strong_snapshot(&backend).await.version, STRONG_TABLE_CATALOG_SNAPSHOT_VERSION); + + let next_request = + StrongTableCatalogStore::new_with_snapshot_write_version(backend.clone(), STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION); + next_request + .get_table_bucket(bucket) + .await + .expect("a new request should reuse the hydrated runtime") + .expect("table bucket should exist"); + let mut restored = read_strong_snapshot(&backend).await; + restored.version = STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION; + seed_strong_snapshot(&backend, &restored).await; + + let after_restore = + StrongTableCatalogStore::new_with_snapshot_write_version(backend.clone(), STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION); + assert_matches!( + after_restore.get_table_bucket(bucket).await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("high-water version 2") + ); + + backend + .delete_object( + RUSTFS_META_BUCKET, + &StrongTableCatalogStore::::snapshot_object_path(), + ) + .await + .expect("snapshot deletion should be injected"); + let after_deletion = + StrongTableCatalogStore::new_with_snapshot_write_version(backend, STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION); + assert_matches!( + after_deletion.get_table_bucket(bucket).await, + Err(TableCatalogStoreError::Internal(message)) if message.contains("snapshot disappeared") + ); +} + +#[tokio::test] +async fn strong_catalog_snapshot_rejects_unknown_fields_and_versions() { + let backend = TestCatalogObjectBackend::default(); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let snapshot = test_strong_snapshot(bucket, &namespace, Vec::new(), Vec::new()); + let mut value = serde_json::to_value(snapshot).expect("snapshot should encode"); + value + .as_object_mut() + .expect("snapshot should be an object") + .insert("future-field".to_string(), serde_json::Value::Bool(true)); + backend + .seed_object( + RUSTFS_META_BUCKET, + &StrongTableCatalogStore::::snapshot_object_path(), + serde_json::to_vec(&value).expect("snapshot should encode"), + ) + .await; + + let unknown_field = StrongTableCatalogStore::new(backend.clone()) + .get_table_bucket(bucket) + .await + .expect_err("unknown persisted fields must fail closed"); + assert_matches!(unknown_field, TableCatalogStoreError::Internal(message) if message.contains("unknown field")); + + let nested_unknown = serde_json::json!({ + "version": STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION, + "table_buckets": [], + "namespaces": [], + "tables": [], + "views": [], + "commits": [{ + "table_bucket": bucket, + "table_id": "table-id", + "lookup_key": "commit-1", + "commit": { + "version": TABLE_CATALOG_ENTRY_VERSION, + "commit_id": "commit-1", + "idempotency_key": null, + "table_id": "table-id", + "operation": "append", + "expected_version_token": "token-v1", + "new_version_token": "token-v2", + "previous_metadata_location": "metadata/00001.metadata.json", + "new_metadata_location": "metadata/00002.metadata.json", + "requirements": [], + "status": "COMMITTED", + "writer": null, + "created_at": null, + "updated_at": null + }, + "future-field": true + }], + "idempotency": [] + }); + backend + .seed_object( + RUSTFS_META_BUCKET, + &StrongTableCatalogStore::::snapshot_object_path(), + serde_json::to_vec(&nested_unknown).expect("snapshot should encode"), + ) + .await; + let nested_unknown_field = StrongTableCatalogStore::new(backend.clone()) + .get_table_bucket(bucket) + .await + .expect_err("unknown commit record fields must fail closed"); + assert_matches!(nested_unknown_field, TableCatalogStoreError::Internal(message) if message.contains("unknown field")); + + value + .as_object_mut() + .expect("snapshot should be an object") + .remove("future-field"); + value["version"] = serde_json::Value::from(STRONG_TABLE_CATALOG_SNAPSHOT_VERSION.saturating_add(1)); + backend + .seed_object( + RUSTFS_META_BUCKET, + &StrongTableCatalogStore::::snapshot_object_path(), + serde_json::to_vec(&value).expect("snapshot should encode"), + ) + .await; + let unsupported = StrongTableCatalogStore::new(backend) + .get_table_bucket(bucket) + .await + .expect_err("unsupported snapshot versions must fail closed"); + assert_matches!(unsupported, TableCatalogStoreError::Invalid(message) if message.contains("unsupported")); +} + +#[tokio::test] +async fn strong_catalog_rejects_overlapping_warehouse_prefixes_on_hydration() { + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let parent = IdentifierSegment::parse("orders").expect("table should parse"); + let child = IdentifierSegment::parse("orders_child").expect("table should parse"); + let parent_entry = test_table_entry( + bucket, + &namespace, + &parent, + default_table_metadata_file_path(&namespace, &parent, "00001.metadata.json"), + ); + let mut child_entry = test_table_entry( + bucket, + &namespace, + &child, + default_table_metadata_file_path(&namespace, &child, "00001.metadata.json"), + ); + child_entry.table_id = "child-table-id".to_string(); + child_entry.table_uuid = "child-table-uuid".to_string(); + child_entry.warehouse_location = format!("s3://{bucket}/tables/table-id/child"); + let snapshot = test_strong_snapshot(bucket, &namespace, vec![parent_entry, child_entry], Vec::new()); + let error = strong_snapshot_hydration_error(snapshot).await; + + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("overlapping active table warehouse")); +} + #[tokio::test] async fn strong_catalog_backing_does_not_publish_draft_when_persist_and_reload_fail() { let backend = TestCatalogObjectBackend::default(); @@ -8578,6 +12283,651 @@ async fn strong_catalog_backing_does_not_publish_draft_when_persist_and_reload_f ); } +#[tokio::test] +async fn strong_catalog_recovers_resource_mutations_after_committed_put_response_loss() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); + let table_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let view_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + let next_view_metadata = default_view_metadata_file_path(&namespace, &view, "00002.metadata.json"); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("committed table bucket write should survive a lost PUT response"); + + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("committed namespace write should survive a lost PUT response"); + + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + store + .update_namespace_properties( + bucket, + &namespace.public_name(), + NamespacePropertiesUpdate::try_new(Vec::new(), BTreeMap::from([("owner".to_string(), "platform".to_string())])) + .expect("namespace update should validate"), + ) + .await + .expect("committed namespace update should survive a lost PUT response"); + + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + store + .create_table(test_table_entry(bucket, &namespace, &table, table_metadata)) + .await + .expect("committed table write should survive a lost PUT response"); + + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + store + .create_view(test_view_entry(bucket, &namespace, &view, view_metadata.clone())) + .await + .expect("committed view write should survive a lost PUT response"); + + backend + .seed_object( + bucket, + &next_view_metadata, + serde_json::to_vec(&serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": format!("s3://{bucket}/views/view-id") + })) + .expect("view metadata should encode"), + ) + .await; + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let replaced = store + .replace_view(ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: view_metadata, + new_metadata_location: next_view_metadata.clone(), + }) + .await + .expect("committed view replacement should survive a lost PUT response"); + assert_eq!(replaced.view.metadata_location, next_view_metadata); + assert_eq!(replaced.view.generation, 2); + + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + store + .drop_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("committed table drop should survive a lost PUT response"); + + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + store + .drop_view(bucket, &namespace.public_name(), view.as_str()) + .await + .expect("committed view drop should survive a lost PUT response"); + + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + store + .drop_namespace(bucket, &namespace.public_name()) + .await + .expect("committed namespace drop should survive a lost PUT response"); + + let restarted = StrongTableCatalogStore::new(backend); + assert!( + restarted + .get_table_bucket(bucket) + .await + .expect("table bucket lookup should succeed after restart") + .is_some() + ); + assert!( + restarted + .get_namespace(bucket, &namespace.public_name()) + .await + .expect("namespace lookup should succeed after restart") + .is_none() + ); + assert!( + restarted + .list_tables(bucket, &namespace.public_name()) + .await + .expect("table listing should succeed after restart") + .is_empty() + ); + assert!( + restarted + .list_views(bucket, &namespace.public_name()) + .await + .expect("view listing should succeed after restart") + .is_empty() + ); +} + +#[tokio::test] +async fn strong_catalog_does_not_guess_view_history_from_a_concurrent_replacement() { + let backend = TestCatalogObjectBackend::default(); + let bootstrap = StrongTableCatalogStore::new(backend.clone()); + let first = StrongTableCatalogStore::new(backend.clone()); + let second = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); + let initial_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + let first_metadata = default_view_metadata_file_path(&namespace, &view, "00002.metadata.json"); + let second_metadata = default_view_metadata_file_path(&namespace, &view, "00003.metadata.json"); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + + bootstrap + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + bootstrap + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + bootstrap + .create_view(test_view_entry(bucket, &namespace, &view, initial_metadata.clone())) + .await + .expect("view should be created"); + first + .get_table_bucket(bucket) + .await + .expect("first store should hydrate") + .expect("table bucket should exist"); + second + .get_table_bucket(bucket) + .await + .expect("second store should hydrate") + .expect("table bucket should exist"); + for metadata in [&first_metadata, &second_metadata] { + backend + .seed_object( + bucket, + metadata, + serde_json::to_vec(&serde_json::json!({ + "format-version": 1, + "view-uuid": "view-uuid", + "location": format!("s3://{bucket}/views/view-id") + })) + .expect("view metadata should encode"), + ) + .await; + } + + backend.fail_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let recovery_read = backend.pause_before_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; + let first_replace_store = first.clone(); + let first_namespace_name = namespace.public_name(); + let first_view_name = view.as_str().to_string(); + let first_expected_metadata = initial_metadata.clone(); + let first_replace = tokio::spawn(async move { + first_replace_store + .replace_view(ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: first_namespace_name, + view: first_view_name, + expected_version_token: "token-v1".to_string(), + expected_metadata_location: first_expected_metadata, + new_metadata_location: first_metadata, + }) + .await + }); + recovery_read.wait_started().await; + second + .replace_view(ViewCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + view: view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: initial_metadata, + new_metadata_location: second_metadata.clone(), + }) + .await + .expect("second writer should publish a different replacement"); + recovery_read.release(); + let error = first_replace + .await + .expect("first replacement task should join") + .expect_err("a different generation-two view must not prove the first replacement succeeded"); + assert_matches!(error, TableCatalogStoreError::Internal(message) if message.contains("injected put failure")); + + let loaded = first + .load_view(bucket, &namespace.public_name(), view.as_str()) + .await + .expect("view lookup should succeed") + .expect("view should remain present"); + assert_eq!(loaded.metadata_location, second_metadata); +} + +#[tokio::test] +async fn strong_catalog_recovers_view_drop_when_a_different_identity_is_recreated() { + let backend = TestCatalogObjectBackend::default(); + let bootstrap = StrongTableCatalogStore::new(backend.clone()); + let first = StrongTableCatalogStore::new(backend.clone()); + let second = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); + let initial_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + let replacement_metadata = default_view_metadata_file_path(&namespace, &view, "00002.metadata.json"); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + + bootstrap + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + bootstrap + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + bootstrap + .create_view(test_view_entry(bucket, &namespace, &view, initial_metadata)) + .await + .expect("view should be created"); + first + .get_table_bucket(bucket) + .await + .expect("first store should hydrate") + .expect("table bucket should exist"); + second + .get_table_bucket(bucket) + .await + .expect("second store should hydrate") + .expect("table bucket should exist"); + + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let drop_recovery_read = backend.pause_before_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; + let first_drop = first.clone(); + let namespace_name = namespace.public_name(); + let view_name = view.as_str().to_string(); + let drop = tokio::spawn(async move { first_drop.drop_view(bucket, &namespace_name, &view_name).await }); + drop_recovery_read.wait_started().await; + let mut replacement = test_view_entry(bucket, &namespace, &view, replacement_metadata); + replacement.view_id = "replacement-view-id".to_string(); + replacement.view_uuid = "replacement-view-uuid".to_string(); + second + .create_view(replacement.clone()) + .await + .expect("second writer should recreate the dropped view"); + drop_recovery_read.release(); + drop.await + .expect("drop task should join") + .expect("lost drop response should recover when a different view identity was recreated"); + + let loaded = first + .load_view(bucket, &namespace.public_name(), view.as_str()) + .await + .expect("view lookup should succeed") + .expect("replacement view should exist"); + assert_eq!(loaded.view_id, replacement.view_id); + assert_eq!(loaded.view_uuid, replacement.view_uuid); +} + +#[tokio::test] +async fn strong_catalog_recovers_migration_snapshot_mutations_after_committed_put_response_loss() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + let source = StrongTableCatalogBucketSnapshot::new_for_test(test_bucket_entry(bucket)); + let source_fingerprint = table_catalog_bucket_snapshot_fingerprint(&source).expect("source snapshot should hash"); + + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let (snapshot_etag, created) = store + .materialize_bucket_snapshot(source) + .await + .expect("committed migration materialization should survive a lost PUT response"); + assert!(created); + assert!(!snapshot_etag.is_empty()); + + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + store + .remove_bucket_snapshot_if_unchanged(bucket, &source_fingerprint) + .await + .expect("committed migration rollback should survive a lost PUT response"); + + let restarted = StrongTableCatalogStore::new(backend); + assert!( + restarted + .get_table_bucket(bucket) + .await + .expect("table bucket lookup should succeed after restart") + .is_none() + ); +} + +#[tokio::test] +async fn strong_catalog_rejects_snapshot_with_unreachable_commit_log() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + let base_table = test_table_entry(bucket, &namespace, &table, current_metadata.clone()); + store.create_table(base_table.clone()).await.expect("table should be created"); + backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; + + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let recovery_read = backend.pause_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; + let request = TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("client-request".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }; + let commit_store = store.clone(); + let commit = tokio::spawn(async move { commit_store.commit_table(request).await }); + recovery_read.wait_started().await; + + let mut inconsistent = read_strong_snapshot(&backend).await; + inconsistent.tables = vec![base_table]; + seed_strong_snapshot(&backend, &inconsistent).await; + recovery_read.release(); + + let error = commit + .await + .expect("commit task should join") + .expect_err("an unreachable commit log must not prove that the table pointer advanced"); + assert_matches!(error, TableCatalogStoreError::Internal(message) if message.contains("post-commit put failure")); + assert_matches!( + store.load_table(bucket, "sales", "orders").await, + Err(TableCatalogStoreError::Invalid(message)) + if message.contains("not recoverable in the current table history") + ); +} + +#[tokio::test] +async fn strong_catalog_post_commit_recovery_does_not_install_stale_descendant() { + let backend = TestCatalogObjectBackend::default(); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + let bucket = "analytics"; + let bootstrap = StrongTableCatalogStore::new(backend.clone()); + bootstrap + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + + let first = StrongTableCatalogStore::new(backend.clone()); + let second = StrongTableCatalogStore::new(backend.clone()); + first + .get_table_bucket(bucket) + .await + .expect("first store should hydrate") + .expect("table bucket should exist"); + second + .get_table_bucket(bucket) + .await + .expect("second store should hydrate") + .expect("table bucket should exist"); + + backend.fail_after_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let recovery_read = backend.pause_before_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; + let first_namespace = Namespace::parse("first").expect("namespace should parse"); + let first_store = first.clone(); + let first_write = tokio::spawn(async move { + first_store + .create_namespace(test_namespace_entry(bucket, &first_namespace)) + .await + }); + recovery_read.wait_started().await; + + let second_namespace = Namespace::parse("second").expect("namespace should parse"); + second + .create_namespace(test_namespace_entry(bucket, &second_namespace)) + .await + .expect("second writer should publish a descendant snapshot"); + recovery_read.release(); + first_write + .await + .expect("first writer task should finish") + .expect("lost response should recover as committed"); + + let state = first.state.lock().await; + assert!(state.namespaces.contains_key(&(bucket.to_string(), "first".to_string()))); + assert!(state.namespaces.contains_key(&(bucket.to_string(), "second".to_string()))); +} + +#[tokio::test] +async fn strong_catalog_materialization_recovers_etag_after_transient_reload_failure() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + assert!( + store + .get_table_bucket(bucket) + .await + .expect("empty durable snapshot should hydrate") + .is_none() + ); + backend.fail_next_read(RUSTFS_META_BUCKET, &snapshot_path).await; + + let (snapshot_etag, created) = store + .materialize_bucket_snapshot(StrongTableCatalogBucketSnapshot::new_for_test(test_bucket_entry(bucket))) + .await + .expect("materialization should recover the committed snapshot etag"); + + assert!(created); + let durable = backend + .read_object(RUSTFS_META_BUCKET, &snapshot_path) + .await + .expect("durable snapshot read should succeed") + .expect("durable snapshot should exist"); + assert_eq!(snapshot_etag, durable.etag.expect("durable snapshot should have an etag")); + assert!( + store + .get_table_bucket(bucket) + .await + .expect("materialized bucket should load") + .is_some() + ); +} + +#[tokio::test] +async fn strong_catalog_rejects_cross_bucket_state_before_materialization() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let mut source = StrongTableCatalogBucketSnapshot::new_for_test(test_bucket_entry("analytics")); + source.push_commit_for_test(StrongCommitSnapshotRecord::new_for_test( + "victim".to_string(), + "table-id".to_string(), + "commit-1".to_string(), + CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: "commit-1".to_string(), + idempotency_key: None, + table_id: "table-id".to_string(), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + new_version_token: "token-v2".to_string(), + previous_metadata_location: "metadata/00001.metadata.json".to_string(), + new_metadata_location: "metadata/00002.metadata.json".to_string(), + requirements: Vec::new(), + status: CommitLogStatus::Committed, + writer: None, + created_at: None, + updated_at: None, + }, + )); + + assert_matches!( + store.materialize_bucket_snapshot(source).await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("owned by victim") + ); + assert!( + backend + .read_object( + RUSTFS_META_BUCKET, + &StrongTableCatalogStore::::snapshot_object_path(), + ) + .await + .expect("snapshot lookup should succeed") + .is_none() + ); +} + +#[tokio::test] +async fn strong_catalog_rejects_invalid_bucket_before_materialization() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let mut bucket = test_bucket_entry("analytics"); + bucket.catalog_type = "unsupported".to_string(); + + assert_matches!( + store + .materialize_bucket_snapshot(StrongTableCatalogBucketSnapshot::new_for_test(bucket)) + .await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("catalog type") + ); + assert!( + backend + .read_object( + RUSTFS_META_BUCKET, + &StrongTableCatalogStore::::snapshot_object_path(), + ) + .await + .expect("snapshot lookup should succeed") + .is_none() + ); +} + +#[tokio::test] +async fn strong_catalog_rejects_unreadable_commit_before_materialization() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let mut source = StrongTableCatalogBucketSnapshot::new_for_test(test_bucket_entry(bucket)); + source.push_commit_for_test(StrongCommitSnapshotRecord::new_for_test( + bucket.to_string(), + "table-id".to_string(), + "commit-1".to_string(), + CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION.saturating_add(1), + commit_id: "commit-1".to_string(), + idempotency_key: None, + table_id: "table-id".to_string(), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + new_version_token: "token-v2".to_string(), + previous_metadata_location: "metadata/00001.metadata.json".to_string(), + new_metadata_location: "metadata/00002.metadata.json".to_string(), + requirements: Vec::new(), + status: CommitLogStatus::Committed, + writer: None, + created_at: None, + updated_at: None, + }, + )); + + assert_matches!( + store.materialize_bucket_snapshot(source).await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("unsupported commit log entry version") + ); + assert!( + backend + .read_object( + RUSTFS_META_BUCKET, + &StrongTableCatalogStore::::snapshot_object_path(), + ) + .await + .expect("snapshot lookup should succeed") + .is_none() + ); +} + +#[tokio::test] +async fn strong_catalog_rejects_inactive_bucket_with_active_namespace_before_persisting() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + let before = read_strong_snapshot(&backend).await; + let mut inactive = test_bucket_entry(bucket); + inactive.state = TableCatalogEntryState::Deleted; + + assert_matches!( + store.put_table_bucket(inactive).await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("active namespace") + ); + assert_eq!(read_strong_snapshot(&backend).await, before); + assert_eq!( + StrongTableCatalogStore::new(backend) + .get_table_bucket(bucket) + .await + .expect("durable bucket should reload") + .expect("table bucket should remain") + .state, + TableCatalogEntryState::Active + ); +} + +#[tokio::test] +async fn strong_catalog_validates_candidate_snapshot_before_publication() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + let put_attempts = backend.put_attempt_count(RUSTFS_META_BUCKET, &snapshot_path).await; + let mut entry = test_table_entry( + bucket, + &namespace, + &table, + default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), + ); + entry.table_id.clear(); + + assert_matches!( + store.register_table(entry).await, + Err(TableCatalogStoreError::Invalid(message)) if message.contains("table id cannot be empty") + ); + assert_eq!( + backend.put_attempt_count(RUSTFS_META_BUCKET, &snapshot_path).await, + put_attempts, + "an unreadable candidate must be rejected before snapshot publication" + ); +} + #[tokio::test] async fn strong_catalog_backing_commit_conflict_keeps_pointer_and_wal_unchanged() { let backend = TestCatalogObjectBackend::default(); @@ -8758,6 +13108,988 @@ async fn strong_catalog_backing_rejects_duplicate_table_warehouse_location_on_co ); } +#[tokio::test] +async fn strong_catalog_relocation_checks_past_the_current_warehouse_prefix() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let orders = IdentifierSegment::parse("orders").expect("table should parse"); + let customers = IdentifierSegment::parse("customers").expect("table should parse"); + let orders_metadata = default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"); + let relocated_metadata = default_table_metadata_file_path(&namespace, &orders, "00002.metadata.json"); + let customers_metadata = default_table_metadata_file_path(&namespace, &customers, "00001.metadata.json"); + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + let mut orders_entry = test_table_entry(bucket, &namespace, &orders, orders_metadata.clone()); + orders_entry.warehouse_location = format!("s3://{bucket}/tables/group/orders"); + store.create_table(orders_entry).await.unwrap(); + let mut customers_entry = test_table_entry(bucket, &namespace, &customers, customers_metadata); + customers_entry.table_id = "table-id-2".to_string(); + customers_entry.table_uuid = "table-uuid-2".to_string(); + customers_entry.warehouse_location = format!("s3://{bucket}/tables/group/customers"); + store.create_table(customers_entry).await.unwrap(); + backend + .seed_object( + bucket, + &relocated_metadata, + serde_json::to_vec(&serde_json::json!({ + "location": "s3://analytics/tables/group", + "table-uuid": "table-uuid" + })) + .unwrap(), + ) + .await; + + let error = store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: namespace.public_name(), + table: orders.as_str().to_string(), + commit_id: "commit-parent-relocation".to_string(), + idempotency_key: None, + operation: "set-location".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: orders_metadata, + new_metadata_location: relocated_metadata, + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }) + .await + .expect_err("parent relocation must detect the sibling after skipping the table's current prefix"); + + assert_matches!(error, TableCatalogStoreError::Conflict(message) if message.contains("overlaps an active table")); +} + +#[tokio::test] +async fn catalog_backings_reject_table_view_identifier_collisions() { + for mode in [TableCatalogBackingMode::ObjectBacked, TableCatalogBackingMode::DurableStrong] { + let store = ConfiguredTableCatalogStore::new_for_test(TestCatalogObjectBackend::default(), mode); + let bucket = format!("collision-{mode:?}").to_ascii_lowercase(); + let table_first = Namespace::parse("table_first").expect("namespace should parse"); + let view_first = Namespace::parse("view_first").expect("namespace should parse"); + let identifier = IdentifierSegment::parse("orders").expect("identifier should parse"); + store + .put_table_bucket(test_bucket_entry(&bucket)) + .await + .expect("table bucket should be created"); + for namespace in [&table_first, &view_first] { + store + .create_namespace(test_namespace_entry(&bucket, namespace)) + .await + .expect("namespace should be created"); + } + + store + .create_table(test_table_entry( + &bucket, + &table_first, + &identifier, + default_table_metadata_file_path(&table_first, &identifier, "00001.metadata.json"), + )) + .await + .expect("table should be created"); + let view_error = store + .create_view(test_view_entry( + &bucket, + &table_first, + &identifier, + default_view_metadata_file_path(&table_first, &identifier, "00001.metadata.json"), + )) + .await + .expect_err("view must not reuse a table identifier"); + assert_matches!(view_error, TableCatalogStoreError::Conflict(_)); + + store + .create_view(test_view_entry( + &bucket, + &view_first, + &identifier, + default_view_metadata_file_path(&view_first, &identifier, "00001.metadata.json"), + )) + .await + .expect("view should be created"); + let mut table_entry = test_table_entry( + &bucket, + &view_first, + &identifier, + default_table_metadata_file_path(&view_first, &identifier, "00001.metadata.json"), + ); + table_entry.table_id = "table-id-2".to_string(); + table_entry.table_uuid = "table-uuid-2".to_string(); + table_entry.warehouse_location = format!("s3://{bucket}/tables/table-id-2"); + let table_error = store + .create_table(table_entry) + .await + .expect_err("table must not reuse a view identifier"); + assert_matches!(table_error, TableCatalogStoreError::Conflict(_)); + } +} + +#[tokio::test] +async fn catalog_backings_hide_and_reject_mutation_of_inactive_resources() { + for mode in [TableCatalogBackingMode::ObjectBacked, TableCatalogBackingMode::DurableStrong] { + let backend = TestCatalogObjectBackend::default(); + let store = ConfiguredTableCatalogStore::new_for_test(backend.clone(), mode); + let bucket = format!("inactive-{mode:?}").to_ascii_lowercase(); + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let view = IdentifierSegment::parse("summary").expect("view should parse"); + let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let current_view_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"); + let new_view_metadata = default_view_metadata_file_path(&namespace, &view, "00002.metadata.json"); + store + .put_table_bucket(test_bucket_entry(&bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(&bucket, &namespace)) + .await + .expect("namespace should be created"); + let mut table_entry = test_table_entry(&bucket, &namespace, &table, current_metadata.clone()); + table_entry.state = TableCatalogEntryState::Deleted; + store + .create_table(table_entry) + .await + .expect("inactive table should be retained"); + let mut view_entry = test_view_entry(&bucket, &namespace, &view, current_view_metadata.clone()); + view_entry.state = TableCatalogEntryState::Deleted; + store.create_view(view_entry).await.expect("inactive view should be retained"); + + assert!( + store + .load_table(&bucket, &namespace.public_name(), table.as_str()) + .await + .expect("table lookup should succeed") + .is_none() + ); + assert!( + store + .load_view(&bucket, &namespace.public_name(), view.as_str()) + .await + .expect("view lookup should succeed") + .is_none() + ); + assert!( + store + .list_tables(&bucket, &namespace.public_name()) + .await + .expect("tables should list") + .is_empty() + ); + assert!( + store + .list_all_tables(&bucket) + .await + .expect("all tables should list") + .is_empty() + ); + assert!( + store + .list_views(&bucket, &namespace.public_name()) + .await + .expect("views should list") + .is_empty() + ); + assert_matches!( + store.drop_namespace(&bucket, &namespace.public_name()).await, + Err(TableCatalogStoreError::Conflict(_)) + ); + + backend.seed_object(&bucket, &new_metadata, b"{}".to_vec()).await; + let commit_error = store + .commit_table(TableCommitRequest { + table_bucket: bucket.clone(), + namespace: namespace.public_name(), + table: table.as_str().to_string(), + commit_id: "commit-1".to_string(), + idempotency_key: Some("request-1".to_string()), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_metadata, + new_metadata_location: new_metadata, + requirements: Vec::new(), + writer: Some("pyiceberg/test".to_string()), + }) + .await + .expect_err("inactive table must reject commits"); + assert_matches!(commit_error, TableCatalogStoreError::NotFound(_)); + + let view_error = store + .replace_view(ViewCommitRequest { + table_bucket: bucket.clone(), + namespace: namespace.public_name(), + view: view.as_str().to_string(), + expected_version_token: "token-v1".to_string(), + expected_metadata_location: current_view_metadata, + new_metadata_location: new_view_metadata, + }) + .await + .expect_err("inactive view must reject replacement"); + assert_matches!(view_error, TableCatalogStoreError::NotFound(_)); + } +} + +#[tokio::test] +async fn strong_catalog_rejects_invalid_inactive_resources_before_snapshot_write() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let view = IdentifierSegment::parse("summary").expect("view should parse"); + + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + + let mut table_entry = test_table_entry( + bucket, + &namespace, + &table, + default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), + ); + table_entry.state = TableCatalogEntryState::Deleted; + table_entry.metadata_location = "data/inactive.parquet".to_string(); + assert_matches!(store.create_table(table_entry).await, Err(TableCatalogStoreError::Invalid(_))); + + let mut view_entry = test_view_entry( + bucket, + &namespace, + &view, + default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"), + ); + view_entry.state = TableCatalogEntryState::Deleted; + view_entry.metadata_location = "data/inactive-view.parquet".to_string(); + assert_matches!(store.create_view(view_entry).await, Err(TableCatalogStoreError::Invalid(_))); + + let durable = read_strong_snapshot(&backend).await; + assert!(durable.tables.is_empty()); + assert!(durable.views.is_empty()); + let restarted = StrongTableCatalogStore::new(backend); + assert!( + restarted + .list_all_tables(bucket) + .await + .expect("tables should list") + .is_empty() + ); + assert!( + restarted + .list_views(bucket, &namespace.public_name()) + .await + .expect("views should list") + .is_empty() + ); +} + +#[tokio::test] +async fn object_catalog_concurrent_table_view_creation_has_one_winner() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let identifier = IdentifierSegment::parse("orders").expect("identifier should parse"); + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + let table_entry = test_table_entry( + bucket, + &namespace, + &identifier, + default_table_metadata_file_path(&namespace, &identifier, "00001.metadata.json"), + ); + let view_entry = test_view_entry( + bucket, + &namespace, + &identifier, + default_view_metadata_file_path(&namespace, &identifier, "00001.metadata.json"), + ); + + let (table_result, view_result) = tokio::join!(store.create_table(table_entry), store.create_view(view_entry)); + + assert_eq!(usize::from(table_result.is_ok()) + usize::from(view_result.is_ok()), 1); + assert!(table_result.is_ok() || matches!(table_result, Err(TableCatalogStoreError::Conflict(_)))); + assert!(view_result.is_ok() || matches!(view_result, Err(TableCatalogStoreError::Conflict(_)))); + let table_exists = store + .load_table(bucket, &namespace.public_name(), identifier.as_str()) + .await + .expect("table lookup should succeed") + .is_some(); + let view_exists = store + .load_view(bucket, &namespace.public_name(), identifier.as_str()) + .await + .expect("view lookup should succeed") + .is_some(); + assert_ne!(table_exists, view_exists); +} + +#[tokio::test] +async fn strong_catalog_independent_table_view_creation_has_one_winner() { + let backend = TestCatalogObjectBackend::default(); + let bootstrap = StrongTableCatalogStore::new(backend.clone()); + let table_store = StrongTableCatalogStore::new(backend.clone()); + let view_store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let identifier = IdentifierSegment::parse("orders").expect("identifier should parse"); + bootstrap + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + bootstrap + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + table_store + .get_namespace(bucket, &namespace.public_name()) + .await + .expect("table writer should hydrate") + .expect("namespace should exist"); + view_store + .get_namespace(bucket, &namespace.public_name()) + .await + .expect("view writer should hydrate") + .expect("namespace should exist"); + + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + let paused_put = backend.pause_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let table_entry = test_table_entry( + bucket, + &namespace, + &identifier, + default_table_metadata_file_path(&namespace, &identifier, "00001.metadata.json"), + ); + let table_write = tokio::spawn(async move { + table_store + .register_table_with_publication(table_entry, &UnserializedTestPublication) + .await + }); + paused_put.wait_started().await; + + view_store + .create_view(test_view_entry( + bucket, + &namespace, + &identifier, + default_view_metadata_file_path(&namespace, &identifier, "00001.metadata.json"), + )) + .await + .expect("view writer should win the snapshot CAS"); + paused_put.release(); + assert_matches!( + table_write.await.expect("table writer task should join"), + Err(TableCatalogStoreError::Conflict(_)) + ); + + let reader = StrongTableCatalogStore::new(backend); + assert!( + reader + .load_table(bucket, &namespace.public_name(), identifier.as_str()) + .await + .expect("table lookup should succeed") + .is_none() + ); + assert!( + reader + .load_view(bucket, &namespace.public_name(), identifier.as_str()) + .await + .expect("view lookup should succeed") + .is_some() + ); +} + +#[tokio::test] +async fn strong_catalog_quarantines_and_repairs_legacy_identifier_collisions() { + let backend = TestCatalogObjectBackend::default(); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let identifier = IdentifierSegment::parse("orders").expect("identifier should parse"); + let table = test_table_entry( + bucket, + &namespace, + &identifier, + default_table_metadata_file_path(&namespace, &identifier, "00001.metadata.json"), + ); + let view = test_view_entry( + bucket, + &namespace, + &identifier, + default_view_metadata_file_path(&namespace, &identifier, "00001.metadata.json"), + ); + let second_identifier = IdentifierSegment::parse("returns").expect("identifier should parse"); + let mut second_table = test_table_entry( + bucket, + &namespace, + &second_identifier, + default_table_metadata_file_path(&namespace, &second_identifier, "00001.metadata.json"), + ); + second_table.table_id = "table-id-2".to_string(); + second_table.table_uuid = "table-uuid-2".to_string(); + second_table.warehouse_location = format!("s3://{bucket}/tables/table-id-2"); + let mut second_view = test_view_entry( + bucket, + &namespace, + &second_identifier, + default_view_metadata_file_path(&namespace, &second_identifier, "00001.metadata.json"), + ); + second_view.view_id = "view-id-2".to_string(); + second_view.view_uuid = "view-uuid-2".to_string(); + second_view.warehouse_location = format!("s3://{bucket}/views/view-id-2"); + let snapshot = test_strong_snapshot(bucket, &namespace, vec![table.clone(), second_table], vec![view, second_view]); + seed_strong_snapshot(&backend, &snapshot).await; + let store = + StrongTableCatalogStore::new_with_snapshot_write_version(backend.clone(), STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION); + + let error = store + .load_table(bucket, &namespace.public_name(), identifier.as_str()) + .await + .expect_err("ambiguous legacy identifiers must fail closed"); + assert_matches!(error, TableCatalogStoreError::Internal(message) if message.contains("operator cleanup")); + assert_matches!( + store.list_views(bucket, &namespace.public_name()).await, + Err(TableCatalogStoreError::Internal(_)) + ); + assert_matches!(store.list_all_tables(bucket).await, Err(TableCatalogStoreError::Internal(_))); + + let unrelated = Namespace::parse("unrelated").expect("namespace should parse"); + assert_matches!( + store.create_namespace(test_namespace_entry(bucket, &unrelated)).await, + Err(TableCatalogStoreError::Conflict(message)) if message.contains("collision cleanup") + ); + let v2_writer = + StrongTableCatalogStore::new_with_snapshot_write_version(backend.clone(), STRONG_TABLE_CATALOG_SNAPSHOT_VERSION); + let blocked = Namespace::parse("blocked").expect("namespace should parse"); + assert_matches!( + v2_writer.create_namespace(test_namespace_entry(bucket, &blocked)).await, + Err(TableCatalogStoreError::Conflict(_)) + ); + + store + .drop_view(bucket, &namespace.public_name(), identifier.as_str()) + .await + .expect("dropping one ambiguous resource must make cleanup progress"); + assert_eq!( + read_strong_snapshot(&backend).await.version, + STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION + ); + assert_matches!( + v2_writer.create_namespace(test_namespace_entry(bucket, &unrelated)).await, + Err(TableCatalogStoreError::Conflict(_)) + ); + store + .drop_view(bucket, &namespace.public_name(), second_identifier.as_str()) + .await + .expect("dropping the final ambiguous resource must finish cleanup"); + assert_eq!( + read_strong_snapshot(&backend).await.version, + STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION + ); + v2_writer + .create_namespace(test_namespace_entry(bucket, &unrelated)) + .await + .expect("ordinary writes should resume after collision cleanup"); + assert_eq!(read_strong_snapshot(&backend).await.version, STRONG_TABLE_CATALOG_SNAPSHOT_VERSION); + assert_eq!( + store + .load_table(bucket, &namespace.public_name(), identifier.as_str()) + .await + .expect("repaired table should load"), + Some(table.clone()) + ); + + let restarted = + StrongTableCatalogStore::new_with_snapshot_write_version(backend, STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION); + assert_eq!( + restarted + .load_table(bucket, &namespace.public_name(), identifier.as_str()) + .await + .expect("repaired snapshot should survive restart"), + Some(table) + ); + assert!( + restarted + .get_namespace(bucket, &unrelated.public_name()) + .await + .expect("unrelated namespace lookup should succeed") + .is_some() + ); +} + +#[tokio::test] +async fn strong_catalog_rejects_restored_v1_collision_after_observing_v2() { + let backend = TestCatalogObjectBackend::default(); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let identifier = IdentifierSegment::parse("orders").expect("identifier should parse"); + let mut upgraded = test_strong_snapshot(bucket, &namespace, Vec::new(), Vec::new()); + upgraded.version = STRONG_TABLE_CATALOG_SNAPSHOT_VERSION; + seed_strong_snapshot(&backend, &upgraded).await; + let store = StrongTableCatalogStore::new(backend.clone()); + store + .get_table_bucket(bucket) + .await + .expect("version 2 snapshot should load") + .expect("table bucket should exist"); + + let table = test_table_entry( + bucket, + &namespace, + &identifier, + default_table_metadata_file_path(&namespace, &identifier, "00001.metadata.json"), + ); + let view = test_view_entry( + bucket, + &namespace, + &identifier, + default_view_metadata_file_path(&namespace, &identifier, "00001.metadata.json"), + ); + seed_strong_snapshot(&backend, &test_strong_snapshot(bucket, &namespace, vec![table], vec![view])).await; + + let error = store + .get_table_bucket(bucket) + .await + .expect_err("a restored v1 collision must not replace observed v2 state"); + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("version 2")); +} + +#[tokio::test] +async fn strong_catalog_v2_snapshot_rejects_table_view_identifier_collision() { + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let identifier = IdentifierSegment::parse("orders").expect("identifier should parse"); + let table = test_table_entry( + bucket, + &namespace, + &identifier, + default_table_metadata_file_path(&namespace, &identifier, "00001.metadata.json"), + ); + let view = test_view_entry( + bucket, + &namespace, + &identifier, + default_view_metadata_file_path(&namespace, &identifier, "00001.metadata.json"), + ); + let mut snapshot = test_strong_snapshot(bucket, &namespace, vec![table], vec![view]); + snapshot.version = STRONG_TABLE_CATALOG_SNAPSHOT_VERSION; + + let error = strong_snapshot_hydration_error(snapshot).await; + + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("table/view")); +} + +#[tokio::test] +async fn strong_catalog_snapshot_rejects_corrupt_resource_ownership() { + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let orders = IdentifierSegment::parse("orders").expect("table should parse"); + let returns = IdentifierSegment::parse("returns").expect("table should parse"); + let mut invalid_metadata = test_table_entry( + bucket, + &namespace, + &orders, + default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"), + ); + invalid_metadata.metadata_location = "data/part.parquet".to_string(); + let error = + strong_snapshot_hydration_error(test_strong_snapshot(bucket, &namespace, vec![invalid_metadata], Vec::new())).await; + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("invalid metadata location")); + + let mut inactive_table = test_table_entry( + bucket, + &namespace, + &orders, + default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"), + ); + inactive_table.state = TableCatalogEntryState::Deleted; + inactive_table.metadata_location = "data/inactive.parquet".to_string(); + let mut invalid_inactive_table_snapshot = test_strong_snapshot(bucket, &namespace, vec![inactive_table], Vec::new()); + invalid_inactive_table_snapshot.version = STRONG_TABLE_CATALOG_SNAPSHOT_VERSION; + let error = strong_snapshot_hydration_error(invalid_inactive_table_snapshot).await; + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("invalid metadata location")); + + let view = IdentifierSegment::parse("summary").expect("view should parse"); + let mut empty_view_id = test_view_entry( + bucket, + &namespace, + &view, + default_view_metadata_file_path(&namespace, &view, "00001.view-metadata.json"), + ); + empty_view_id.view_id.clear(); + let error = strong_snapshot_hydration_error(test_strong_snapshot(bucket, &namespace, Vec::new(), vec![empty_view_id])).await; + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("view id cannot be empty")); + + let mut inactive_view = test_view_entry( + bucket, + &namespace, + &view, + default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"), + ); + inactive_view.state = TableCatalogEntryState::Deleted; + inactive_view.metadata_location = "data/inactive-view.parquet".to_string(); + let mut invalid_inactive_view_snapshot = test_strong_snapshot(bucket, &namespace, Vec::new(), vec![inactive_view]); + invalid_inactive_view_snapshot.version = STRONG_TABLE_CATALOG_SNAPSHOT_VERSION; + let error = strong_snapshot_hydration_error(invalid_inactive_view_snapshot).await; + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("invalid metadata location")); + + let resource_backed_namespace = test_table_entry( + bucket, + &namespace, + &orders, + default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"), + ); + let resource_backed_snapshot = StrongTableCatalogSnapshot { + version: STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION, + table_buckets: vec![test_bucket_entry(bucket)], + namespaces: Vec::new(), + tables: vec![resource_backed_namespace], + views: Vec::new(), + commits: Vec::new(), + idempotency: Vec::new(), + }; + let backend = TestCatalogObjectBackend::default(); + seed_strong_snapshot(&backend, &resource_backed_snapshot).await; + let store = StrongTableCatalogStore::new(backend); + assert!( + store + .get_namespace(bucket, &namespace.public_name()) + .await + .expect("resource-backed namespace should load") + .is_some() + ); + + let mut inactive_resource = test_table_entry( + bucket, + &namespace, + &orders, + default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"), + ); + inactive_resource.state = TableCatalogEntryState::Deleted; + let inactive_resource_snapshot = StrongTableCatalogSnapshot { + version: STRONG_TABLE_CATALOG_SNAPSHOT_VERSION, + table_buckets: vec![test_bucket_entry(bucket)], + namespaces: Vec::new(), + tables: vec![inactive_resource], + views: Vec::new(), + commits: Vec::new(), + idempotency: Vec::new(), + }; + let error = strong_snapshot_hydration_error(inactive_resource_snapshot).await; + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("no active namespace")); + + let child = Namespace::parse("sales.daily").expect("child namespace should parse"); + let mut inactive_parent = test_namespace_entry(bucket, &namespace); + inactive_parent.state = TableCatalogEntryState::Deleted; + let inactive_parent_snapshot = StrongTableCatalogSnapshot { + version: STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION, + table_buckets: vec![test_bucket_entry(bucket)], + namespaces: vec![inactive_parent, test_namespace_entry(bucket, &child)], + tables: Vec::new(), + views: Vec::new(), + commits: Vec::new(), + idempotency: Vec::new(), + }; + let error = strong_snapshot_hydration_error(inactive_parent_snapshot).await; + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("inactive namespace")); + + let mut inactive_bucket = test_bucket_entry(bucket); + inactive_bucket.state = TableCatalogEntryState::Deleted; + let inactive_bucket_snapshot = StrongTableCatalogSnapshot { + version: STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION, + table_buckets: vec![inactive_bucket], + namespaces: vec![test_namespace_entry(bucket, &namespace)], + tables: Vec::new(), + views: Vec::new(), + commits: Vec::new(), + idempotency: Vec::new(), + }; + let error = strong_snapshot_hydration_error(inactive_bucket_snapshot).await; + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("no active table bucket")); + + let orders_entry = test_table_entry( + bucket, + &namespace, + &orders, + default_table_metadata_file_path(&namespace, &orders, "00001.metadata.json"), + ); + let mut duplicate_table_id = test_table_entry( + bucket, + &namespace, + &returns, + default_table_metadata_file_path(&namespace, &returns, "00001.metadata.json"), + ); + duplicate_table_id.warehouse_location = format!("s3://{bucket}/tables/returns-id"); + let error = strong_snapshot_hydration_error(test_strong_snapshot( + bucket, + &namespace, + vec![orders_entry, duplicate_table_id], + Vec::new(), + )) + .await; + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("table ids")); +} + +#[tokio::test] +async fn strong_catalog_v1_inactive_resources_are_hidden_and_cleanup_only() { + let backend = TestCatalogObjectBackend::default(); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let view = IdentifierSegment::parse("summary").expect("view should parse"); + let mut table_entry = test_table_entry( + bucket, + &namespace, + &table, + default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), + ); + table_entry.state = TableCatalogEntryState::Deleted; + table_entry.warehouse_location = "legacy-invalid-location".to_string(); + table_entry.metadata_location = "legacy-invalid-metadata".to_string(); + let mut view_entry = test_view_entry( + bucket, + &namespace, + &view, + default_view_metadata_file_path(&namespace, &view, "00001.metadata.json"), + ); + view_entry.state = TableCatalogEntryState::Deleted; + view_entry.warehouse_location = "legacy-invalid-view-location".to_string(); + view_entry.metadata_location = "legacy-invalid-view-metadata".to_string(); + let snapshot = StrongTableCatalogSnapshot { + version: STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION, + table_buckets: vec![test_bucket_entry(bucket)], + namespaces: Vec::new(), + tables: vec![table_entry], + views: vec![view_entry], + commits: Vec::new(), + idempotency: Vec::new(), + }; + seed_strong_snapshot(&backend, &snapshot).await; + let store = + StrongTableCatalogStore::new_with_snapshot_write_version(backend.clone(), STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION); + + assert!(store.list_all_tables(bucket).await.expect("tables should list").is_empty()); + assert!( + store + .list_views(bucket, &namespace.public_name()) + .await + .expect("views should list") + .is_empty() + ); + store + .drop_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("legacy inactive table should be removable"); + store + .drop_view(bucket, &namespace.public_name(), view.as_str()) + .await + .expect("legacy inactive view should be removable"); + + let cleaned = read_strong_snapshot(&backend).await; + assert!(cleaned.tables.is_empty()); + assert!(cleaned.views.is_empty()); + assert_eq!(cleaned.version, STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION); +} + +#[tokio::test] +async fn strong_catalog_snapshot_rejects_mismatched_commit_indexes() { + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let table_entry = test_table_entry( + bucket, + &namespace, + &table, + default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), + ); + let commit = CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: "commit-1".to_string(), + idempotency_key: Some("request-1".to_string()), + table_id: table_entry.table_id.clone(), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + new_version_token: "token-v2".to_string(), + previous_metadata_location: table_entry.metadata_location.clone(), + new_metadata_location: default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"), + requirements: Vec::new(), + status: CommitLogStatus::Committed, + writer: Some("pyiceberg/test".to_string()), + created_at: None, + updated_at: None, + }; + let record = |lookup_key: &str, commit: CommitLogEntry| { + StrongCommitSnapshotRecord::new_for_test(bucket.to_string(), table_entry.table_id.clone(), lookup_key.to_string(), commit) + }; + let snapshot = |commits, idempotency| StrongTableCatalogSnapshot { + version: STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION, + table_buckets: vec![test_bucket_entry(bucket)], + namespaces: vec![test_namespace_entry(bucket, &namespace)], + tables: vec![table_entry.clone()], + views: Vec::new(), + commits, + idempotency, + }; + + let error = strong_snapshot_hydration_error(snapshot(vec![record("wrong", commit.clone())], Vec::new())).await; + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("snapshot owner")); + + let error = strong_snapshot_hydration_error(snapshot(vec![record("commit-1", commit.clone())], Vec::new())).await; + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("idempotency index")); + + let mut mismatched = commit.clone(); + mismatched.new_version_token = "token-v3".to_string(); + let error = strong_snapshot_hydration_error(snapshot( + vec![record("commit-1", commit.clone())], + vec![record("request-1", mismatched)], + )) + .await; + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("matching commit")); + + let error = strong_snapshot_hydration_error(snapshot( + vec![record("commit-1", commit.clone())], + vec![record("request-1", commit.clone()), record("request-1", commit)], + )) + .await; + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("duplicate idempotency")); +} + +#[tokio::test] +async fn strong_catalog_snapshot_rejects_commit_outside_current_history() { + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let table_entry = test_table_entry( + bucket, + &namespace, + &table, + default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), + ); + let commit = CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: "commit-1".to_string(), + idempotency_key: None, + table_id: table_entry.table_id.clone(), + operation: "append".to_string(), + expected_version_token: table_entry.version_token.clone(), + new_version_token: "token-v2".to_string(), + previous_metadata_location: table_entry.metadata_location.clone(), + new_metadata_location: default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"), + requirements: Vec::new(), + status: CommitLogStatus::Committed, + writer: None, + created_at: None, + updated_at: None, + }; + let snapshot = StrongTableCatalogSnapshot { + version: STRONG_TABLE_CATALOG_SNAPSHOT_VERSION, + table_buckets: vec![test_bucket_entry(bucket)], + namespaces: vec![test_namespace_entry(bucket, &namespace)], + tables: vec![table_entry.clone()], + views: Vec::new(), + commits: vec![StrongCommitSnapshotRecord::new_for_test( + bucket.to_string(), + table_entry.table_id, + commit.commit_id.clone(), + commit, + )], + idempotency: Vec::new(), + }; + + let error = strong_snapshot_hydration_error(snapshot).await; + assert_matches!( + error, + TableCatalogStoreError::Invalid(message) if message.contains("not recoverable in the current table history") + ); +} + +#[tokio::test] +async fn strong_catalog_snapshot_discards_dropped_table_commit_indexes() { + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let commit = CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: "commit-1".to_string(), + idempotency_key: Some("request-1".to_string()), + table_id: "dropped-table-id".to_string(), + operation: "append".to_string(), + expected_version_token: "token-v1".to_string(), + new_version_token: "token-v2".to_string(), + previous_metadata_location: "tables/dropped-table-id/metadata/00001.metadata.json".to_string(), + new_metadata_location: "tables/dropped-table-id/metadata/00002.metadata.json".to_string(), + requirements: Vec::new(), + status: CommitLogStatus::Committed, + writer: Some("pyiceberg/test".to_string()), + created_at: None, + updated_at: None, + }; + let record = |lookup_key: &str, commit: CommitLogEntry| { + StrongCommitSnapshotRecord::new_for_test( + bucket.to_string(), + "dropped-table-id".to_string(), + lookup_key.to_string(), + commit, + ) + }; + let snapshot = StrongTableCatalogSnapshot { + version: STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION, + table_buckets: vec![test_bucket_entry(bucket)], + namespaces: vec![test_namespace_entry(bucket, &namespace)], + tables: Vec::new(), + views: Vec::new(), + commits: vec![record("commit-1", commit.clone())], + idempotency: vec![record("request-1", commit)], + }; + let mut strict_snapshot = snapshot.clone(); + strict_snapshot.version = STRONG_TABLE_CATALOG_SNAPSHOT_VERSION; + let error = strong_snapshot_hydration_error(strict_snapshot).await; + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("no owning table")); + + let mut unsupported_idempotency = snapshot.clone(); + unsupported_idempotency.commits.clear(); + unsupported_idempotency.idempotency[0].commit.version = u16::MAX; + let error = strong_snapshot_hydration_error(unsupported_idempotency).await; + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("commit idempotency entry version")); + + let backend = TestCatalogObjectBackend::default(); + seed_strong_snapshot(&backend, &snapshot).await; + + let store = StrongTableCatalogStore::new(backend); + store + .get_table_bucket(bucket) + .await + .expect("legacy dropped-table indexes should not break hydration") + .expect("table bucket should exist"); + assert!( + store + .get_commit_by_id(bucket, "dropped-table-id", "commit-1") + .await + .expect("commit lookup should succeed") + .is_none() + ); + assert!( + store + .get_commit_by_idempotency_key(bucket, "dropped-table-id", "request-1") + .await + .expect("idempotency lookup should succeed") + .is_none() + ); +} + #[tokio::test] async fn diagnostics_backing_manifest_requires_recovery_before_migration() { let backend = TestCatalogObjectBackend::default(); @@ -9136,7 +14468,7 @@ async fn object_table_catalog_store_syncs_warehouse_location_from_committed_meta .await .expect("old table warehouse lookup should succeed"); assert!(old_resource.is_none()); - assert_eq!(backend.list_call_count().await, 0); + assert_eq!(backend.list_call_count().await, 1); } #[tokio::test] @@ -9372,6 +14704,40 @@ fn failed_commit_does_not_prove_historical_staged_commit() { assert_eq!(recovery.recovery_state, TableCommitRecoveryState::ManualReview); } +#[test] +fn committed_commit_outside_current_history_requires_manual_review() { + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table_name = IdentifierSegment::parse("orders").expect("table should parse"); + let table = test_table_entry( + "analytics", + &namespace, + &table_name, + default_table_metadata_file_path(&namespace, &table_name, "00001.metadata.json"), + ); + let commit = CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: "commit-1".to_string(), + idempotency_key: None, + table_id: table.table_id.clone(), + operation: "append".to_string(), + expected_version_token: table.version_token.clone(), + new_version_token: "token-v2".to_string(), + previous_metadata_location: table.metadata_location.clone(), + new_metadata_location: default_table_metadata_file_path(&namespace, &table_name, "00002.metadata.json"), + requirements: Vec::new(), + status: CommitLogStatus::Committed, + writer: None, + created_at: None, + updated_at: None, + }; + + let history = TableCommitHistoryIndex::new(&table, [&commit]); + assert!(!history.proves_committed(&commit)); + let recovery = table_commit_recovery_entry(&table, &commit, None, false); + assert_eq!(recovery.recovery_state, TableCommitRecoveryState::ManualReview); + assert!(recovery.reason.contains("not reachable")); +} + #[test] fn commit_history_index_scales_across_a_long_table_history() { let namespace = Namespace::parse("sales").expect("namespace should parse"); @@ -9403,6 +14769,45 @@ fn commit_history_index_scales_across_a_long_table_history() { assert!(commits.iter().all(|commit| history.proves_committed(commit))); } +#[test] +fn commit_history_index_rejects_ambiguous_states_and_cycles() { + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table_name = IdentifierSegment::parse("orders").expect("table should parse"); + let metadata = |version| default_table_metadata_file_path(&namespace, &table_name, &format!("{version:05}.metadata.json")); + let commit = |commit_id: &str, previous: u8, next: u8| CommitLogEntry { + version: TABLE_CATALOG_ENTRY_VERSION, + commit_id: commit_id.to_string(), + idempotency_key: None, + table_id: "table-id".to_string(), + operation: "append".to_string(), + expected_version_token: format!("token-v{previous}"), + new_version_token: format!("token-v{next}"), + previous_metadata_location: metadata(previous), + new_metadata_location: metadata(next), + requirements: Vec::new(), + status: CommitLogStatus::Committed, + writer: None, + created_at: None, + updated_at: None, + }; + + let mut table = test_table_entry("analytics", &namespace, &table_name, metadata(2)); + table.version_token = "token-v2".to_string(); + let first = commit("commit-1", 1, 2); + let duplicate = commit("commit-2", 3, 2); + let ambiguous = TableCommitHistoryIndex::new(&table, [&first, &duplicate]); + assert!(!ambiguous.proves_committed(&first)); + assert!(!ambiguous.proves_committed(&duplicate)); + + table.metadata_location = metadata(3); + table.version_token = "token-v3".to_string(); + let forward = commit("commit-3", 2, 3); + let backward = commit("commit-4", 3, 2); + let cyclic = TableCommitHistoryIndex::new(&table, [&forward, &backward]); + assert!(!cyclic.proves_committed(&forward)); + assert!(!cyclic.proves_committed(&backward)); +} + #[tokio::test] async fn object_table_catalog_store_recovers_historical_staged_commit_after_later_commit() { let backend = TestCatalogObjectBackend::default(); @@ -9886,7 +15291,7 @@ async fn table_commit_recovery_does_not_overwrite_conflicting_idempotency_index( } #[tokio::test] -async fn strong_table_commit_retry_rejects_token_only_idempotency_conflict() { +async fn strong_table_commit_retry_rejects_token_only_idempotency_corruption() { let backend = TestCatalogObjectBackend::default(); let store = StrongTableCatalogStore::new(backend.clone()); let bucket = "analytics"; @@ -9945,7 +15350,10 @@ async fn strong_table_commit_retry_rejects_token_only_idempotency_conflict() { .commit_table(request) .await .expect_err("token-only idempotency mismatch must fail closed in strong mode"); - assert_matches!(retry_error, TableCatalogStoreError::Conflict(_)); + assert_matches!( + retry_error, + TableCatalogStoreError::Invalid(message) if message.contains("has no matching commit") + ); assert_eq!( backend .read_object(RUSTFS_META_BUCKET, &snapshot_path) @@ -10438,7 +15846,7 @@ fn namespace_property_update_and_limits_reject_ambiguous_or_oversized_state() { #[tokio::test] async fn configured_object_catalog_rejects_namespace_property_update_without_mutation() { let backend = TestCatalogObjectBackend::default(); - let store = ConfiguredTableCatalogStore::new(backend, TableCatalogBackingMode::ObjectBacked); + let store = ConfiguredTableCatalogStore::new_for_test(backend, TableCatalogBackingMode::ObjectBacked); let bucket = "analytics"; let namespace = Namespace::parse("sales").expect("namespace should parse"); let mut entry = test_namespace_entry(bucket, &namespace); @@ -11121,6 +16529,48 @@ where ); } +async fn assert_inactive_table_bucket_rejects_catalog_creation(store: &S, bucket: &str) +where + S: TableCatalogStore + ?Sized, +{ + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); + let mut inactive = test_bucket_entry(bucket); + inactive.state = TableCatalogEntryState::Deleted; + store + .put_table_bucket(inactive) + .await + .expect("inactive table bucket marker should be seeded"); + + assert_matches!( + store.create_namespace(test_namespace_entry(bucket, &namespace)).await, + Err(TableCatalogStoreError::NotFound(_)) + ); + assert_matches!( + store + .create_table(test_table_entry( + bucket, + &namespace, + &table, + default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), + )) + .await, + Err(TableCatalogStoreError::NotFound(_)) + ); + assert_matches!( + store + .create_view(test_view_entry( + bucket, + &namespace, + &view, + default_view_metadata_file_path(&namespace, &view, "00001.view.json"), + )) + .await, + Err(TableCatalogStoreError::NotFound(_)) + ); +} + #[tokio::test] async fn catalog_backings_keep_implicit_parents_implicit_during_resource_creation() { let bucket = "analytics"; @@ -11177,6 +16627,20 @@ async fn catalog_backings_reject_resource_creation_in_inactive_namespace() { .await; } +#[tokio::test] +async fn catalog_backings_reject_creation_in_inactive_table_bucket() { + assert_inactive_table_bucket_rejects_catalog_creation( + &ObjectTableCatalogStore::new(TestCatalogObjectBackend::default()), + "object-catalog", + ) + .await; + assert_inactive_table_bucket_rejects_catalog_creation( + &StrongTableCatalogStore::new(TestCatalogObjectBackend::default()), + "strong-catalog", + ) + .await; +} + #[test] fn resolver_builds_paths_under_reserved_table_boundary() { let table = TableIdentifier::new(