feat(s3-tables): support object-backed table rename (#6899)

This commit is contained in:
GatewayJ
2026-08-31 00:30:49 +08:00
committed by GitHub
parent c2d8488728
commit 8fd364a99c
12 changed files with 1403 additions and 74 deletions
@@ -168,6 +168,7 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[
"GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials",
"POST /v1/{prefix}/namespaces/{namespace}/tables/{table}",
"DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}",
"POST /v1/{prefix}/tables/rename",
"GET /v1/{prefix}/namespaces/{namespace}/views",
"POST /v1/{prefix}/namespaces/{namespace}/views",
"GET /v1/{prefix}/namespaces/{namespace}/views/{view}",
@@ -175,10 +176,7 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[
"POST /v1/{prefix}/namespaces/{namespace}/views/{view}",
"DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}",
];
const TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS: &[&str] = &[
"POST /v1/{prefix}/namespaces/{namespace}/properties",
"POST /v1/{prefix}/tables/rename",
];
const TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS: &[&str] = &["POST /v1/{prefix}/namespaces/{namespace}/properties"];
static GET_CONFIG_HANDLER: GetCatalogConfigHandler = GetCatalogConfigHandler {};
static ENABLE_TABLE_BUCKET_HANDLER: EnableTableBucketHandler = EnableTableBucketHandler {};
@@ -2298,6 +2296,7 @@ fn table_bucket_entry_from_metadata_marker(bucket: &str) -> crate::table_catalog
warehouse_root: format!("s3://{bucket}/"),
state: crate::table_catalog::TableCatalogEntryState::Active,
properties: BTreeMap::new(),
active_rename_id: None,
created_at: None,
updated_at: None,
}
@@ -438,7 +438,7 @@ fn catalog_config_response_lists_standard_rest_endpoints() {
.endpoints
.contains(&"POST /v1/{prefix}/namespaces/{namespace}/properties")
);
assert!(!response.endpoints.contains(&"POST /v1/{prefix}/tables/rename"));
assert!(response.endpoints.contains(&"POST /v1/{prefix}/tables/rename"));
assert_eq!(response.admin_discovery.runtime_capabilities, "/rustfs/admin/v4/runtime/capabilities");
assert_eq!(response.admin_discovery.cluster_snapshot, "/rustfs/admin/v4/cluster/snapshot");
assert_eq!(response.admin_discovery.extensions_catalog, "/rustfs/admin/v4/extensions/catalog");
@@ -11120,6 +11120,7 @@ async fn seed_object_table_for_metadata_maintenance(
warehouse_root: format!("s3://{bucket}/"),
state: crate::table_catalog::TableCatalogEntryState::Active,
properties: BTreeMap::new(),
active_rename_id: None,
created_at: None,
updated_at: None,
})
+8
View File
@@ -86,6 +86,14 @@ impl ApiError {
}
}
pub fn service_unavailable() -> Self {
ApiError {
code: S3ErrorCode::ServiceUnavailable,
message: Self::error_code_to_message(&S3ErrorCode::ServiceUnavailable),
source: None,
}
}
pub fn invalid_request(message: impl std::fmt::Display) -> Self {
ApiError {
code: S3ErrorCode::InvalidRequest,
+5 -1
View File
@@ -1598,7 +1598,11 @@ async fn table_data_plane_resource_for_request<T>(
error = %err,
"failed to resolve table data-plane resource"
);
s3_error!(AccessDenied, "Access Denied")
if matches!(err, crate::table_catalog::TableCatalogStoreError::Unavailable(_)) {
S3Error::from(ApiError::service_unavailable())
} else {
s3_error!(AccessDenied, "Access Denied")
}
})?;
let bucket_fence_key = (bucket.to_string(), crate::table_catalog::default_table_bucket_publication_lock_path());
let mut state = retained.state.lock();
+2
View File
@@ -101,6 +101,7 @@ pub(crate) const TABLE_RESOURCE_MARKER_VERSION: u16 = 1;
)]
pub(crate) const TABLE_METADATA_POINTER_VERSION: u16 = 1;
pub(crate) const TABLE_CATALOG_ENTRY_VERSION: u16 = 1;
pub(crate) const TABLE_RENAME_INTENT_VERSION: u16 = 1;
pub(crate) const TABLE_WAREHOUSE_INDEX_STATE_VERSION: u16 = 2;
pub(crate) const TABLE_MAINTENANCE_CONFIG_VERSION: u16 = 1;
pub(crate) const TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION: u16 = 1;
@@ -166,6 +167,7 @@ const COMMIT_LOG_ROOT: &str = "commits";
const COMMIT_IDEMPOTENCY_ROOT: &str = "commit-idempotency";
const WAREHOUSE_INDEX_ROOT: &str = "warehouse-index";
const WAREHOUSE_INDEX_STATE_FILE: &str = "state.json";
const TABLE_RENAME_ROOT: &str = "renames";
const WAREHOUSE_INDEX_MAX_PREFIX_DEPTH: usize = 64;
const EXTERNAL_CATALOG_ROOT: &str = "external-catalog";
const EXTERNAL_CATALOG_BRIDGE_FILE: &str = "bridge.json";
+33
View File
@@ -39,6 +39,8 @@ pub(crate) fn table_bucket_marker_json() -> Result<Vec<u8>, serde_json::Error> {
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogEntryState {
Active,
/// Persisted only behind a rename intent so older readers reject the unknown state and fail closed.
Renaming,
Deleting,
Deleted,
}
@@ -53,10 +55,40 @@ pub(crate) struct TableBucketEntry {
pub state: TableCatalogEntryState,
#[serde(default)]
pub properties: BTreeMap<String, String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub active_rename_id: Option<String>,
pub created_at: Option<String>,
pub updated_at: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableRenameIntentState {
Prepared,
SourceFenced,
DestinationWritten,
SourceTombstoned,
IndexPublished,
DestinationPublished,
Completed,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct TableRenameIntent {
pub version: u16,
pub rename_id: String,
pub table_bucket: String,
pub source: TableEntry,
pub destination: TableEntry,
pub source_etag: String,
pub destination_etag: Option<String>,
pub warehouse_index_etag: String,
pub state: TableRenameIntentState,
pub created_at: String,
pub updated_at: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct NamespaceEntry {
@@ -1225,6 +1257,7 @@ pub(crate) enum TableCatalogBackingMigrationStep {
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogBackingMigrationBlocker {
TableRenameRecoveryRequired,
CommitRecoveryRequired,
CommitManualReviewRequired,
WarehouseIndexBackfillRequired,
+33 -15
View File
@@ -301,6 +301,11 @@ where
return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}")));
};
validate_table_bucket_entry_object(&self.paths, &bucket_path, &table_bucket_entry)?;
if table_bucket_entry.active_rename_id.is_some() {
return Err(TableCatalogStoreError::Conflict(format!(
"table bucket {table_bucket} has a table rename requiring recovery"
)));
}
let mut namespaces = Vec::new();
let mut tables = Vec::new();
@@ -343,6 +348,9 @@ where
)));
};
validate_table_entry_object(&self.paths, table_object, &table_entry)?;
if table_entry.state != TableCatalogEntryState::Active {
continue;
}
for commit_object in self
.backend
@@ -595,9 +603,10 @@ where
&self,
table_bucket: &str,
) -> TableCatalogStoreResult<TableCatalogBackingMigrationDryRunReport> {
if self.get_table_bucket(table_bucket).await?.is_none() {
let Some(table_bucket_entry) = self.get_table_bucket(table_bucket).await? else {
return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}")));
}
};
let rename_recovery_required = table_bucket_entry.active_rename_id.is_some();
if let Some((global_fence, _)) = self
.read_entry::<TableCatalogBackingMigrationGlobalFence>(
self.catalog_bucket(),
@@ -653,18 +662,19 @@ where
continue;
};
validate_table_entry_object(&self.paths, &object, &table)?;
if table.state != TableCatalogEntryState::Active {
continue;
}
table_count = table_count.saturating_add(1);
if !table_ids.insert(table.table_id.clone()) {
duplicate_table_identity = true;
}
if table.state == TableCatalogEntryState::Active {
active_table_identifiers.insert((table.namespace.clone(), table.table.clone()));
let warehouse_prefix = table_warehouse_object_prefix(&table)?;
warehouse_prefix_owners
.entry(warehouse_prefix)
.and_modify(|count| *count = count.saturating_add(1))
.or_insert(1);
}
active_table_identifiers.insert((table.namespace.clone(), table.table.clone()));
let warehouse_prefix = table_warehouse_object_prefix(&table)?;
warehouse_prefix_owners
.entry(warehouse_prefix)
.and_modify(|count| *count = count.saturating_add(1))
.or_insert(1);
let recovery = self.table_commit_recovery_report_for_entry(&table, 0).await?;
commit_log_count = commit_log_count.saturating_add(recovery.commits.len());
@@ -711,33 +721,41 @@ where
let table_view_identifier_collision_count = active_table_identifiers.intersection(&active_view_identifiers).count();
let mut blockers = Vec::new();
let mut recommended_actions = Vec::new();
if rename_recovery_required {
blockers.push(TableCatalogBackingMigrationBlocker::TableRenameRecoveryRequired);
recommended_actions.push(TableCatalogBackingMigrationAction::RunCatalogRecovery);
}
if recovery_required_count > 0 {
blockers.push(TableCatalogBackingMigrationBlocker::CommitRecoveryRequired);
}
if manual_review_count > 0 {
blockers.push(TableCatalogBackingMigrationBlocker::CommitManualReviewRequired);
}
if recovery_required_count > 0 || manual_review_count > 0 {
if (recovery_required_count > 0 || manual_review_count > 0)
&& !recommended_actions.contains(&TableCatalogBackingMigrationAction::RunCatalogRecovery)
{
recommended_actions.push(TableCatalogBackingMigrationAction::RunCatalogRecovery);
}
if !warehouse_index_ready {
blockers.push(TableCatalogBackingMigrationBlocker::WarehouseIndexBackfillRequired);
recommended_actions.push(TableCatalogBackingMigrationAction::BackfillWarehouseIndex);
}
if conflicting_warehouse_prefix {
if conflicting_warehouse_prefix && !rename_recovery_required {
blockers.push(TableCatalogBackingMigrationBlocker::DuplicateWarehousePrefix);
recommended_actions.push(TableCatalogBackingMigrationAction::ReviewDuplicateWarehousePrefixes);
}
if duplicate_table_identity {
if duplicate_table_identity && !rename_recovery_required {
blockers.push(TableCatalogBackingMigrationBlocker::DuplicateTableIdentity);
recommended_actions.push(TableCatalogBackingMigrationAction::ReviewDuplicateTableIdentities);
}
if table_view_identifier_collision_count > 0 {
if table_view_identifier_collision_count > 0 && !rename_recovery_required {
blockers.push(TableCatalogBackingMigrationBlocker::TableViewIdentifierCollision);
recommended_actions.push(TableCatalogBackingMigrationAction::ReviewTableViewIdentifierCollisions);
}
let mut status = if manual_review_count > 0
let mut status = if rename_recovery_required {
TableCatalogBackingMigrationStatus::RecoveryRequired
} else if manual_review_count > 0
|| conflicting_warehouse_prefix
|| duplicate_table_identity
|| table_view_identifier_collision_count > 0
+17 -3
View File
@@ -57,6 +57,9 @@ fn validate_table_bucket_entry(entry: &TableBucketEntry) -> TableCatalogStoreRes
if entry.catalog_type != TABLE_BUCKET_CATALOG_TYPE {
return Err(TableCatalogStoreError::Invalid("unsupported table bucket catalog type".to_string()));
}
if entry.active_rename_id.as_ref().is_some_and(String::is_empty) {
return Err(TableCatalogStoreError::Invalid("active table rename id cannot be empty".to_string()));
}
Ok(())
}
@@ -984,6 +987,15 @@ impl TableCatalogObjectPaths {
)
}
pub fn table_rename_intent_path(&self, table_bucket: &str, rename_id: &str) -> String {
format!(
"{}{}/{}.json",
self.table_bucket_root_prefix(table_bucket),
TABLE_RENAME_ROOT,
table_catalog_path_hash(rename_id)
)
}
pub fn backing_migration_fence_path(&self, table_bucket: &str) -> String {
format!(
"{}{}/{}",
@@ -1241,9 +1253,11 @@ where
destination_table: &str,
) -> TableCatalogStoreResult<()> {
match self {
Self::ObjectBacked(_) => Err(TableCatalogStoreError::Unsupported(
"table rename requires durable-strong catalog backing".to_string(),
)),
Self::ObjectBacked(store) => {
store
.rename_table(table_bucket, source_namespace, source_table, destination_namespace, destination_table)
.await
}
Self::DurableStrong(store) => {
store
.rename_table(table_bucket, source_namespace, source_table, destination_namespace, destination_table)
File diff suppressed because it is too large Load Diff
+15 -1
View File
@@ -642,12 +642,26 @@ impl TestCatalogObjectBackend {
let mut state = self.state.lock().await;
let key = (bucket.to_string(), object.to_string());
let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1;
Self::pause_put_attempt_unlocked(&mut state, key, next_attempt)
}
pub(crate) async fn pause_put_attempt(&self, bucket: &str, object: &str, attempt: usize) -> TestCatalogObjectPause {
let mut state = self.state.lock().await;
let key = (bucket.to_string(), object.to_string());
Self::pause_put_attempt_unlocked(&mut state, key, attempt)
}
fn pause_put_attempt_unlocked(
state: &mut TestCatalogObjectState,
key: (String, String),
attempt: usize,
) -> TestCatalogObjectPause {
let pause = TestCatalogObjectPause::default();
state
.pause_put_attempts
.entry(key)
.or_default()
.insert(next_attempt, pause.clone());
.insert(attempt, pause.clone());
pause
}
+460 -3
View File
@@ -144,6 +144,7 @@ fn catalog_entry_structures_serialize_stable_fields() {
warehouse_root: "s3://analytics/".to_string(),
state: TableCatalogEntryState::Active,
properties: BTreeMap::from([("owner".to_string(), "platform".to_string())]),
active_rename_id: None,
created_at: Some("2026-05-23T00:00:00Z".to_string()),
updated_at: Some("2026-05-23T00:00:00Z".to_string()),
};
@@ -3441,6 +3442,7 @@ fn test_bucket_entry(bucket: &str) -> TableBucketEntry {
warehouse_root: format!("s3://{bucket}/"),
state: TableCatalogEntryState::Active,
properties: BTreeMap::new(),
active_rename_id: None,
created_at: None,
updated_at: None,
}
@@ -5107,7 +5109,8 @@ async fn object_catalog_pagination_bounds_reads_and_covers_rest_resources() {
.await
.expect("first table page should load");
assert_eq!(table_page.entries[0].table, "alpha");
assert_eq!(backend.read_call_count().await, 1);
// One read snapshots the bucket rename fence and one loads the page entry.
assert_eq!(backend.read_call_count().await, 2);
let table_page = store
.list_tables_page(bucket, &namespace_name, table_page.next_cursor.as_deref(), one)
.await
@@ -17752,7 +17755,461 @@ async fn strong_catalog_table_rename_returns_success_after_committed_snapshot_re
}
#[tokio::test]
async fn configured_object_catalog_rejects_table_rename() {
async fn object_catalog_table_rename_preserves_identity_index_and_reuses_source_tombstone() {
let backend = TestCatalogObjectBackend {
content_addressed_etags: true,
..Default::default()
};
let store = ObjectTableCatalogStore::new(backend);
let bucket = "analytics";
let source_namespace = Namespace::parse("sales").expect("source namespace should parse");
let destination_namespace = Namespace::parse("curated").expect("destination namespace should parse");
let source_table = IdentifierSegment::parse("orders").expect("source table should parse");
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
store
.create_namespace(test_namespace_entry(bucket, &source_namespace))
.await
.unwrap();
store
.create_namespace(test_namespace_entry(bucket, &destination_namespace))
.await
.unwrap();
let source = test_table_entry(
bucket,
&source_namespace,
&source_table,
default_table_metadata_file_path(&source_namespace, &source_table, "00001.metadata.json"),
);
store.create_table(source.clone()).await.unwrap();
let bucket_object = store.paths.table_bucket_entry_path(bucket);
let bucket_etag_before = store
.read_entry::<TableBucketEntry>(RUSTFS_META_BUCKET, &bucket_object)
.await
.unwrap()
.unwrap()
.1
.expect("table bucket should have an etag");
store
.rename_table(bucket, "sales", "orders", "curated", "orders_v2")
.await
.expect("object-backed table rename should complete");
let bucket_etag_after = store
.read_entry::<TableBucketEntry>(RUSTFS_META_BUCKET, &bucket_object)
.await
.unwrap()
.unwrap()
.1
.expect("table bucket should have an etag");
assert_ne!(bucket_etag_after, bucket_etag_before);
assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_none());
let destination = store
.load_table(bucket, "curated", "orders_v2")
.await
.unwrap()
.expect("destination table should exist");
let mut expected_destination = source.clone();
expected_destination.namespace = "curated".to_string();
expected_destination.table = "orders_v2".to_string();
assert_ne!(destination.updated_at, source.updated_at);
expected_destination.updated_at.clone_from(&destination.updated_at);
assert_eq!(destination, expected_destination);
let source_object = store.paths.table_entry_path(bucket, &source_namespace, &source_table);
let source_tombstone = store
.read_entry::<TableEntry>(RUSTFS_META_BUCKET, &source_object)
.await
.unwrap()
.expect("source tombstone should remain")
.0;
assert_eq!(source_tombstone.state, TableCatalogEntryState::Deleted);
assert_eq!(source_tombstone.table_id, destination.table_id);
assert!(
store
.get_table_bucket(bucket)
.await
.unwrap()
.expect("table bucket should exist")
.active_rename_id
.is_none()
);
let destination_index = table_warehouse_index_entry(&destination).unwrap();
let index_object = store
.paths
.warehouse_index_entry_path(bucket, &destination_index.warehouse_object_prefix);
let persisted_index = store
.read_entry::<TableWarehouseIndexEntry>(RUSTFS_META_BUCKET, &index_object)
.await
.unwrap()
.expect("warehouse index should exist")
.0;
assert_eq!(persisted_index, destination_index);
let resource = store
.resolve_table_data_plane_resource(bucket, "tables/table-id/data/part.parquet")
.await
.unwrap()
.expect("renamed table should resolve its stable warehouse prefix");
assert_eq!(resource.namespace, "curated");
assert_eq!(resource.table, "orders_v2");
store
.backfill_table_warehouse_index(bucket)
.await
.expect("retained source tombstone should not make the warehouse index ambiguous");
let migration = store.plan_durable_strong_backing_migration(bucket).await.unwrap();
assert_eq!(migration.table_count, 1);
assert!(
!migration
.blockers
.contains(&TableCatalogBackingMigrationBlocker::DuplicateTableIdentity)
);
store
.rename_table(bucket, "curated", "orders_v2", "sales", "orders")
.await
.expect("rename should conditionally replace the retained source tombstone");
store
.rename_table(bucket, "sales", "orders", "curated", "orders_v2")
.await
.expect("rename should conditionally replace a destination tombstone");
let mut replacement = test_table_entry(
bucket,
&source_namespace,
&source_table,
default_table_metadata_file_path(&source_namespace, &source_table, "00002.metadata.json"),
);
replacement.table_id = "replacement-table-id".to_string();
replacement.table_uuid = "replacement-table-uuid".to_string();
replacement.warehouse_location = "s3://analytics/tables/replacement-table-id".to_string();
store
.create_table(replacement.clone())
.await
.expect("create should conditionally replace the source tombstone");
assert_eq!(
store
.load_table(bucket, "sales", "orders")
.await
.unwrap()
.expect("source identifier should be reusable"),
replacement
);
}
#[tokio::test]
async fn object_catalog_table_rename_rejects_missing_and_conflicting_destinations() {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend);
let bucket = "analytics";
let source_namespace = Namespace::parse("sales").unwrap();
let destination_namespace = Namespace::parse("curated").unwrap();
let source_table = IdentifierSegment::parse("orders").unwrap();
let destination_table = IdentifierSegment::parse("orders_v2").unwrap();
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
store
.create_namespace(test_namespace_entry(bucket, &source_namespace))
.await
.unwrap();
store
.create_namespace(test_namespace_entry(bucket, &destination_namespace))
.await
.unwrap();
store
.create_table(test_table_entry(
bucket,
&source_namespace,
&source_table,
default_table_metadata_file_path(&source_namespace, &source_table, "00001.metadata.json"),
))
.await
.unwrap();
assert_matches!(
store.rename_table(bucket, "sales", "orders", "missing", "orders_v2").await,
Err(TableCatalogStoreError::NamespaceNotFound(_))
);
assert_matches!(
store.rename_table(bucket, "sales", "missing", "curated", "orders_v2").await,
Err(TableCatalogStoreError::TableNotFound(_))
);
let mut existing = test_table_entry(
bucket,
&destination_namespace,
&destination_table,
default_table_metadata_file_path(&destination_namespace, &destination_table, "00001.metadata.json"),
);
existing.table_id = "destination-table-id".to_string();
existing.table_uuid = "destination-table-uuid".to_string();
existing.warehouse_location = "s3://analytics/tables/destination-table-id".to_string();
store.create_table(existing).await.unwrap();
assert_matches!(
store.rename_table(bucket, "sales", "orders", "curated", "orders_v2").await,
Err(TableCatalogStoreError::AlreadyExists(_))
);
let destination_view = IdentifierSegment::parse("orders_view").unwrap();
store
.create_view(test_view_entry(
bucket,
&destination_namespace,
&destination_view,
default_view_metadata_file_path(&destination_namespace, &destination_view, "00001.metadata.json"),
))
.await
.unwrap();
assert_matches!(
store.rename_table(bucket, "sales", "orders", "curated", "orders_view").await,
Err(TableCatalogStoreError::AlreadyExists(_))
);
assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_some());
}
#[tokio::test]
async fn object_catalog_table_rename_fails_closed_around_durable_fence_creation() {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let bucket = "analytics";
let source_namespace = Namespace::parse("sales").unwrap();
let destination_namespace = Namespace::parse("curated").unwrap();
let source_table = IdentifierSegment::parse("orders").unwrap();
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
store
.create_namespace(test_namespace_entry(bucket, &source_namespace))
.await
.unwrap();
store
.create_namespace(test_namespace_entry(bucket, &destination_namespace))
.await
.unwrap();
store
.create_table(test_table_entry(
bucket,
&source_namespace,
&source_table,
default_table_metadata_file_path(&source_namespace, &source_table, "00001.metadata.json"),
))
.await
.unwrap();
let bucket_object = store.paths.table_bucket_entry_path(bucket);
backend.fail_next_put(RUSTFS_META_BUCKET, &bucket_object).await;
assert_matches!(
store.rename_table(bucket, "sales", "orders", "curated", "orders_v2").await,
Err(TableCatalogStoreError::Internal(_))
);
assert!(
store
.get_table_bucket(bucket)
.await
.unwrap()
.expect("table bucket should remain")
.active_rename_id
.is_none()
);
assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_some());
assert!(store.load_table(bucket, "curated", "orders_v2").await.unwrap().is_none());
let mut fenced_bucket = store.get_table_bucket(bucket).await.unwrap().unwrap();
fenced_bucket.active_rename_id = Some("missing-intent".to_string());
backend
.seed_object(
RUSTFS_META_BUCKET,
&bucket_object,
serde_json::to_vec(&fenced_bucket).expect("fenced bucket should serialize"),
)
.await;
assert_matches!(
store.rename_table(bucket, "sales", "orders", "curated", "orders_v2").await,
Err(TableCatalogStoreError::Unavailable(_))
);
assert_eq!(
store
.get_table_bucket(bucket)
.await
.unwrap()
.expect("table bucket should remain fail-closed")
.active_rename_id
.as_deref(),
Some("missing-intent")
);
assert_matches!(
store
.resolve_table_data_plane_resource(bucket, "tables/table-id/data/part.parquet")
.await,
Err(TableCatalogStoreError::Unavailable(_))
);
}
#[tokio::test]
async fn object_catalog_table_rename_recovers_after_destination_publish_and_fences_concurrent_mutations() {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let bucket = "analytics";
let source_namespace = Namespace::parse("sales").expect("source namespace should parse");
let destination_namespace = Namespace::parse("curated").expect("destination namespace should parse");
let source_table = IdentifierSegment::parse("orders").expect("source table should parse");
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
store
.create_namespace(test_namespace_entry(bucket, &source_namespace))
.await
.unwrap();
store
.create_namespace(test_namespace_entry(bucket, &destination_namespace))
.await
.unwrap();
let source = test_table_entry(
bucket,
&source_namespace,
&source_table,
default_table_metadata_file_path(&source_namespace, &source_table, "00001.metadata.json"),
);
store.create_table(source.clone()).await.unwrap();
let source_object = store.paths.table_entry_path(bucket, &source_namespace, &source_table);
let source_tombstone_attempt = backend.put_attempt_count(RUSTFS_META_BUCKET, &source_object).await + 2;
let source_tombstone_pause = backend
.pause_put_attempt(RUSTFS_META_BUCKET, &source_object, source_tombstone_attempt)
.await;
let rename_store = store.clone();
let rename = tokio::spawn(async move {
rename_store
.rename_table(bucket, "sales", "orders", "curated", "orders_v2")
.await
});
source_tombstone_pause.wait_started().await;
let active_rename_id = store
.get_table_bucket(bucket)
.await
.unwrap()
.expect("table bucket should exist")
.active_rename_id
.expect("rename fence should be durable before destination publication");
assert_matches!(
store.load_table(bucket, "sales", "orders").await,
Err(TableCatalogStoreError::Unavailable(_))
);
assert_matches!(store.list_tables(bucket, "sales").await, Err(TableCatalogStoreError::Unavailable(_)));
assert_matches!(
store
.resolve_table_data_plane_resource(bucket, "tables/table-id/data/part.parquet")
.await,
Err(TableCatalogStoreError::Unavailable(_))
);
let migration = store.plan_durable_strong_backing_migration(bucket).await.unwrap();
assert_eq!(migration.status, TableCatalogBackingMigrationStatus::RecoveryRequired);
assert!(
migration
.blockers
.contains(&TableCatalogBackingMigrationBlocker::TableRenameRecoveryRequired)
);
let publication_lock = default_table_bucket_publication_lock_path();
let publication_attempts = backend.write_lock_acquisition_count(bucket, &publication_lock).await;
let commit_store = store.clone();
let commit = tokio::spawn(async move {
commit_store
.commit_table(TableCommitRequest {
table_bucket: bucket.to_string(),
namespace: "sales".to_string(),
table: "orders".to_string(),
commit_id: "concurrent-commit".to_string(),
idempotency_key: None,
operation: "append".to_string(),
expected_version_token: source.version_token,
expected_metadata_location: source.metadata_location,
new_metadata_location: "unused.metadata.json".to_string(),
requirements: Vec::new(),
writer: Some("rename-test".to_string()),
})
.await
});
let drop_store = store.clone();
let drop_table = tokio::spawn(async move { drop_store.drop_table(bucket, "sales", "orders").await });
let create_store = store.clone();
let mut replacement = test_table_entry(
bucket,
&source_namespace,
&source_table,
default_table_metadata_file_path(&source_namespace, &source_table, "00002.metadata.json"),
);
replacement.table_id = "replacement-table-id".to_string();
replacement.table_uuid = "replacement-table-uuid".to_string();
replacement.warehouse_location = "s3://analytics/tables/replacement-table-id".to_string();
let create = tokio::spawn(async move { create_store.create_table(replacement).await });
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async {
while backend.write_lock_acquisition_count(bucket, &publication_lock).await < publication_attempts + 3 {
tokio::task::yield_now().await;
}
})
.await
.expect("concurrent mutations should reach the table-bucket publication fence");
assert!(!commit.is_finished());
assert!(!drop_table.is_finished());
assert!(!create.is_finished());
commit.abort();
drop_table.abort();
create.abort();
rename.abort();
source_tombstone_pause.release();
let _ = rename.await;
let destination_object = store.paths.table_entry_path(
bucket,
&destination_namespace,
&IdentifierSegment::parse("orders_v2").expect("destination table should parse"),
);
let source_fence = store
.read_entry::<TableEntry>(RUSTFS_META_BUCKET, &source_object)
.await
.unwrap()
.expect("source rename fence should be durable")
.0;
let destination_fence = store
.read_entry::<TableEntry>(RUSTFS_META_BUCKET, &destination_object)
.await
.unwrap()
.expect("destination rename fence should be durable")
.0;
assert_eq!(source_fence.state, TableCatalogEntryState::Renaming);
assert_eq!(destination_fence.state, TableCatalogEntryState::Renaming);
assert_matches!(
store.load_table(bucket, "curated", "orders_v2").await,
Err(TableCatalogStoreError::Unavailable(_))
);
assert_matches!(
store.rename_table(bucket, "sales", "orders", "curated", "orders_v2").await,
Err(TableCatalogStoreError::TableNotFound(_))
);
assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_none());
let destination = store
.load_table(bucket, "curated", "orders_v2")
.await
.unwrap()
.expect("recovery should finish the destination publication");
assert_eq!(destination.table_id, "table-id");
assert!(
store
.get_table_bucket(bucket)
.await
.unwrap()
.expect("table bucket should exist")
.active_rename_id
.is_none()
);
let intent_object = store.paths.table_rename_intent_path(bucket, &active_rename_id);
let intent = store
.read_entry::<TableRenameIntent>(RUSTFS_META_BUCKET, &intent_object)
.await
.unwrap()
.expect("completed rename intent should be retained as a recovery record")
.0;
assert_eq!(intent.state, TableRenameIntentState::Completed);
}
#[tokio::test]
async fn configured_object_catalog_dispatches_table_rename() {
let store =
ConfiguredTableCatalogStore::new_for_test(TestCatalogObjectBackend::default(), TableCatalogBackingMode::ObjectBacked);
@@ -17760,6 +18217,6 @@ async fn configured_object_catalog_rejects_table_rename() {
store
.rename_table("analytics", "sales", "orders", "curated", "orders_v2")
.await,
Err(TableCatalogStoreError::Unsupported(_))
Err(TableCatalogStoreError::NotFound(_))
);
}