mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-01 17:58:22 +00:00
feat(s3-tables): support object-backed table rename (#6899)
This commit is contained in:
@@ -168,6 +168,7 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[
|
||||
"GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials",
|
||||
"POST /v1/{prefix}/namespaces/{namespace}/tables/{table}",
|
||||
"DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}",
|
||||
"POST /v1/{prefix}/tables/rename",
|
||||
"GET /v1/{prefix}/namespaces/{namespace}/views",
|
||||
"POST /v1/{prefix}/namespaces/{namespace}/views",
|
||||
"GET /v1/{prefix}/namespaces/{namespace}/views/{view}",
|
||||
@@ -175,10 +176,7 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[
|
||||
"POST /v1/{prefix}/namespaces/{namespace}/views/{view}",
|
||||
"DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}",
|
||||
];
|
||||
const TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS: &[&str] = &[
|
||||
"POST /v1/{prefix}/namespaces/{namespace}/properties",
|
||||
"POST /v1/{prefix}/tables/rename",
|
||||
];
|
||||
const TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS: &[&str] = &["POST /v1/{prefix}/namespaces/{namespace}/properties"];
|
||||
|
||||
static GET_CONFIG_HANDLER: GetCatalogConfigHandler = GetCatalogConfigHandler {};
|
||||
static ENABLE_TABLE_BUCKET_HANDLER: EnableTableBucketHandler = EnableTableBucketHandler {};
|
||||
@@ -2298,6 +2296,7 @@ fn table_bucket_entry_from_metadata_marker(bucket: &str) -> crate::table_catalog
|
||||
warehouse_root: format!("s3://{bucket}/"),
|
||||
state: crate::table_catalog::TableCatalogEntryState::Active,
|
||||
properties: BTreeMap::new(),
|
||||
active_rename_id: None,
|
||||
created_at: None,
|
||||
updated_at: None,
|
||||
}
|
||||
|
||||
@@ -438,7 +438,7 @@ fn catalog_config_response_lists_standard_rest_endpoints() {
|
||||
.endpoints
|
||||
.contains(&"POST /v1/{prefix}/namespaces/{namespace}/properties")
|
||||
);
|
||||
assert!(!response.endpoints.contains(&"POST /v1/{prefix}/tables/rename"));
|
||||
assert!(response.endpoints.contains(&"POST /v1/{prefix}/tables/rename"));
|
||||
assert_eq!(response.admin_discovery.runtime_capabilities, "/rustfs/admin/v4/runtime/capabilities");
|
||||
assert_eq!(response.admin_discovery.cluster_snapshot, "/rustfs/admin/v4/cluster/snapshot");
|
||||
assert_eq!(response.admin_discovery.extensions_catalog, "/rustfs/admin/v4/extensions/catalog");
|
||||
@@ -11120,6 +11120,7 @@ async fn seed_object_table_for_metadata_maintenance(
|
||||
warehouse_root: format!("s3://{bucket}/"),
|
||||
state: crate::table_catalog::TableCatalogEntryState::Active,
|
||||
properties: BTreeMap::new(),
|
||||
active_rename_id: None,
|
||||
created_at: None,
|
||||
updated_at: None,
|
||||
})
|
||||
|
||||
@@ -86,6 +86,14 @@ impl ApiError {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn service_unavailable() -> Self {
|
||||
ApiError {
|
||||
code: S3ErrorCode::ServiceUnavailable,
|
||||
message: Self::error_code_to_message(&S3ErrorCode::ServiceUnavailable),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn invalid_request(message: impl std::fmt::Display) -> Self {
|
||||
ApiError {
|
||||
code: S3ErrorCode::InvalidRequest,
|
||||
|
||||
@@ -1598,7 +1598,11 @@ async fn table_data_plane_resource_for_request<T>(
|
||||
error = %err,
|
||||
"failed to resolve table data-plane resource"
|
||||
);
|
||||
s3_error!(AccessDenied, "Access Denied")
|
||||
if matches!(err, crate::table_catalog::TableCatalogStoreError::Unavailable(_)) {
|
||||
S3Error::from(ApiError::service_unavailable())
|
||||
} else {
|
||||
s3_error!(AccessDenied, "Access Denied")
|
||||
}
|
||||
})?;
|
||||
let bucket_fence_key = (bucket.to_string(), crate::table_catalog::default_table_bucket_publication_lock_path());
|
||||
let mut state = retained.state.lock();
|
||||
|
||||
@@ -101,6 +101,7 @@ pub(crate) const TABLE_RESOURCE_MARKER_VERSION: u16 = 1;
|
||||
)]
|
||||
pub(crate) const TABLE_METADATA_POINTER_VERSION: u16 = 1;
|
||||
pub(crate) const TABLE_CATALOG_ENTRY_VERSION: u16 = 1;
|
||||
pub(crate) const TABLE_RENAME_INTENT_VERSION: u16 = 1;
|
||||
pub(crate) const TABLE_WAREHOUSE_INDEX_STATE_VERSION: u16 = 2;
|
||||
pub(crate) const TABLE_MAINTENANCE_CONFIG_VERSION: u16 = 1;
|
||||
pub(crate) const TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION: u16 = 1;
|
||||
@@ -166,6 +167,7 @@ const COMMIT_LOG_ROOT: &str = "commits";
|
||||
const COMMIT_IDEMPOTENCY_ROOT: &str = "commit-idempotency";
|
||||
const WAREHOUSE_INDEX_ROOT: &str = "warehouse-index";
|
||||
const WAREHOUSE_INDEX_STATE_FILE: &str = "state.json";
|
||||
const TABLE_RENAME_ROOT: &str = "renames";
|
||||
const WAREHOUSE_INDEX_MAX_PREFIX_DEPTH: usize = 64;
|
||||
const EXTERNAL_CATALOG_ROOT: &str = "external-catalog";
|
||||
const EXTERNAL_CATALOG_BRIDGE_FILE: &str = "bridge.json";
|
||||
|
||||
@@ -39,6 +39,8 @@ pub(crate) fn table_bucket_marker_json() -> Result<Vec<u8>, serde_json::Error> {
|
||||
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||
pub(crate) enum TableCatalogEntryState {
|
||||
Active,
|
||||
/// Persisted only behind a rename intent so older readers reject the unknown state and fail closed.
|
||||
Renaming,
|
||||
Deleting,
|
||||
Deleted,
|
||||
}
|
||||
@@ -53,10 +55,40 @@ pub(crate) struct TableBucketEntry {
|
||||
pub state: TableCatalogEntryState,
|
||||
#[serde(default)]
|
||||
pub properties: BTreeMap<String, String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub active_rename_id: Option<String>,
|
||||
pub created_at: Option<String>,
|
||||
pub updated_at: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||
pub(crate) enum TableRenameIntentState {
|
||||
Prepared,
|
||||
SourceFenced,
|
||||
DestinationWritten,
|
||||
SourceTombstoned,
|
||||
IndexPublished,
|
||||
DestinationPublished,
|
||||
Completed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct TableRenameIntent {
|
||||
pub version: u16,
|
||||
pub rename_id: String,
|
||||
pub table_bucket: String,
|
||||
pub source: TableEntry,
|
||||
pub destination: TableEntry,
|
||||
pub source_etag: String,
|
||||
pub destination_etag: Option<String>,
|
||||
pub warehouse_index_etag: String,
|
||||
pub state: TableRenameIntentState,
|
||||
pub created_at: String,
|
||||
pub updated_at: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct NamespaceEntry {
|
||||
@@ -1225,6 +1257,7 @@ pub(crate) enum TableCatalogBackingMigrationStep {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||
pub(crate) enum TableCatalogBackingMigrationBlocker {
|
||||
TableRenameRecoveryRequired,
|
||||
CommitRecoveryRequired,
|
||||
CommitManualReviewRequired,
|
||||
WarehouseIndexBackfillRequired,
|
||||
|
||||
@@ -301,6 +301,11 @@ where
|
||||
return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}")));
|
||||
};
|
||||
validate_table_bucket_entry_object(&self.paths, &bucket_path, &table_bucket_entry)?;
|
||||
if table_bucket_entry.active_rename_id.is_some() {
|
||||
return Err(TableCatalogStoreError::Conflict(format!(
|
||||
"table bucket {table_bucket} has a table rename requiring recovery"
|
||||
)));
|
||||
}
|
||||
|
||||
let mut namespaces = Vec::new();
|
||||
let mut tables = Vec::new();
|
||||
@@ -343,6 +348,9 @@ where
|
||||
)));
|
||||
};
|
||||
validate_table_entry_object(&self.paths, table_object, &table_entry)?;
|
||||
if table_entry.state != TableCatalogEntryState::Active {
|
||||
continue;
|
||||
}
|
||||
|
||||
for commit_object in self
|
||||
.backend
|
||||
@@ -595,9 +603,10 @@ where
|
||||
&self,
|
||||
table_bucket: &str,
|
||||
) -> TableCatalogStoreResult<TableCatalogBackingMigrationDryRunReport> {
|
||||
if self.get_table_bucket(table_bucket).await?.is_none() {
|
||||
let Some(table_bucket_entry) = self.get_table_bucket(table_bucket).await? else {
|
||||
return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}")));
|
||||
}
|
||||
};
|
||||
let rename_recovery_required = table_bucket_entry.active_rename_id.is_some();
|
||||
if let Some((global_fence, _)) = self
|
||||
.read_entry::<TableCatalogBackingMigrationGlobalFence>(
|
||||
self.catalog_bucket(),
|
||||
@@ -653,18 +662,19 @@ where
|
||||
continue;
|
||||
};
|
||||
validate_table_entry_object(&self.paths, &object, &table)?;
|
||||
if table.state != TableCatalogEntryState::Active {
|
||||
continue;
|
||||
}
|
||||
table_count = table_count.saturating_add(1);
|
||||
if !table_ids.insert(table.table_id.clone()) {
|
||||
duplicate_table_identity = true;
|
||||
}
|
||||
if table.state == TableCatalogEntryState::Active {
|
||||
active_table_identifiers.insert((table.namespace.clone(), table.table.clone()));
|
||||
let warehouse_prefix = table_warehouse_object_prefix(&table)?;
|
||||
warehouse_prefix_owners
|
||||
.entry(warehouse_prefix)
|
||||
.and_modify(|count| *count = count.saturating_add(1))
|
||||
.or_insert(1);
|
||||
}
|
||||
active_table_identifiers.insert((table.namespace.clone(), table.table.clone()));
|
||||
let warehouse_prefix = table_warehouse_object_prefix(&table)?;
|
||||
warehouse_prefix_owners
|
||||
.entry(warehouse_prefix)
|
||||
.and_modify(|count| *count = count.saturating_add(1))
|
||||
.or_insert(1);
|
||||
|
||||
let recovery = self.table_commit_recovery_report_for_entry(&table, 0).await?;
|
||||
commit_log_count = commit_log_count.saturating_add(recovery.commits.len());
|
||||
@@ -711,33 +721,41 @@ where
|
||||
let table_view_identifier_collision_count = active_table_identifiers.intersection(&active_view_identifiers).count();
|
||||
let mut blockers = Vec::new();
|
||||
let mut recommended_actions = Vec::new();
|
||||
if rename_recovery_required {
|
||||
blockers.push(TableCatalogBackingMigrationBlocker::TableRenameRecoveryRequired);
|
||||
recommended_actions.push(TableCatalogBackingMigrationAction::RunCatalogRecovery);
|
||||
}
|
||||
if recovery_required_count > 0 {
|
||||
blockers.push(TableCatalogBackingMigrationBlocker::CommitRecoveryRequired);
|
||||
}
|
||||
if manual_review_count > 0 {
|
||||
blockers.push(TableCatalogBackingMigrationBlocker::CommitManualReviewRequired);
|
||||
}
|
||||
if recovery_required_count > 0 || manual_review_count > 0 {
|
||||
if (recovery_required_count > 0 || manual_review_count > 0)
|
||||
&& !recommended_actions.contains(&TableCatalogBackingMigrationAction::RunCatalogRecovery)
|
||||
{
|
||||
recommended_actions.push(TableCatalogBackingMigrationAction::RunCatalogRecovery);
|
||||
}
|
||||
if !warehouse_index_ready {
|
||||
blockers.push(TableCatalogBackingMigrationBlocker::WarehouseIndexBackfillRequired);
|
||||
recommended_actions.push(TableCatalogBackingMigrationAction::BackfillWarehouseIndex);
|
||||
}
|
||||
if conflicting_warehouse_prefix {
|
||||
if conflicting_warehouse_prefix && !rename_recovery_required {
|
||||
blockers.push(TableCatalogBackingMigrationBlocker::DuplicateWarehousePrefix);
|
||||
recommended_actions.push(TableCatalogBackingMigrationAction::ReviewDuplicateWarehousePrefixes);
|
||||
}
|
||||
if duplicate_table_identity {
|
||||
if duplicate_table_identity && !rename_recovery_required {
|
||||
blockers.push(TableCatalogBackingMigrationBlocker::DuplicateTableIdentity);
|
||||
recommended_actions.push(TableCatalogBackingMigrationAction::ReviewDuplicateTableIdentities);
|
||||
}
|
||||
if table_view_identifier_collision_count > 0 {
|
||||
if table_view_identifier_collision_count > 0 && !rename_recovery_required {
|
||||
blockers.push(TableCatalogBackingMigrationBlocker::TableViewIdentifierCollision);
|
||||
recommended_actions.push(TableCatalogBackingMigrationAction::ReviewTableViewIdentifierCollisions);
|
||||
}
|
||||
|
||||
let mut status = if manual_review_count > 0
|
||||
let mut status = if rename_recovery_required {
|
||||
TableCatalogBackingMigrationStatus::RecoveryRequired
|
||||
} else if manual_review_count > 0
|
||||
|| conflicting_warehouse_prefix
|
||||
|| duplicate_table_identity
|
||||
|| table_view_identifier_collision_count > 0
|
||||
|
||||
@@ -57,6 +57,9 @@ fn validate_table_bucket_entry(entry: &TableBucketEntry) -> TableCatalogStoreRes
|
||||
if entry.catalog_type != TABLE_BUCKET_CATALOG_TYPE {
|
||||
return Err(TableCatalogStoreError::Invalid("unsupported table bucket catalog type".to_string()));
|
||||
}
|
||||
if entry.active_rename_id.as_ref().is_some_and(String::is_empty) {
|
||||
return Err(TableCatalogStoreError::Invalid("active table rename id cannot be empty".to_string()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -984,6 +987,15 @@ impl TableCatalogObjectPaths {
|
||||
)
|
||||
}
|
||||
|
||||
pub fn table_rename_intent_path(&self, table_bucket: &str, rename_id: &str) -> String {
|
||||
format!(
|
||||
"{}{}/{}.json",
|
||||
self.table_bucket_root_prefix(table_bucket),
|
||||
TABLE_RENAME_ROOT,
|
||||
table_catalog_path_hash(rename_id)
|
||||
)
|
||||
}
|
||||
|
||||
pub fn backing_migration_fence_path(&self, table_bucket: &str) -> String {
|
||||
format!(
|
||||
"{}{}/{}",
|
||||
@@ -1241,9 +1253,11 @@ where
|
||||
destination_table: &str,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
match self {
|
||||
Self::ObjectBacked(_) => Err(TableCatalogStoreError::Unsupported(
|
||||
"table rename requires durable-strong catalog backing".to_string(),
|
||||
)),
|
||||
Self::ObjectBacked(store) => {
|
||||
store
|
||||
.rename_table(table_bucket, source_namespace, source_table, destination_namespace, destination_table)
|
||||
.await
|
||||
}
|
||||
Self::DurableStrong(store) => {
|
||||
store
|
||||
.rename_table(table_bucket, source_namespace, source_table, destination_namespace, destination_table)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -642,12 +642,26 @@ impl TestCatalogObjectBackend {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1;
|
||||
Self::pause_put_attempt_unlocked(&mut state, key, next_attempt)
|
||||
}
|
||||
|
||||
pub(crate) async fn pause_put_attempt(&self, bucket: &str, object: &str, attempt: usize) -> TestCatalogObjectPause {
|
||||
let mut state = self.state.lock().await;
|
||||
let key = (bucket.to_string(), object.to_string());
|
||||
Self::pause_put_attempt_unlocked(&mut state, key, attempt)
|
||||
}
|
||||
|
||||
fn pause_put_attempt_unlocked(
|
||||
state: &mut TestCatalogObjectState,
|
||||
key: (String, String),
|
||||
attempt: usize,
|
||||
) -> TestCatalogObjectPause {
|
||||
let pause = TestCatalogObjectPause::default();
|
||||
state
|
||||
.pause_put_attempts
|
||||
.entry(key)
|
||||
.or_default()
|
||||
.insert(next_attempt, pause.clone());
|
||||
.insert(attempt, pause.clone());
|
||||
pause
|
||||
}
|
||||
|
||||
|
||||
@@ -144,6 +144,7 @@ fn catalog_entry_structures_serialize_stable_fields() {
|
||||
warehouse_root: "s3://analytics/".to_string(),
|
||||
state: TableCatalogEntryState::Active,
|
||||
properties: BTreeMap::from([("owner".to_string(), "platform".to_string())]),
|
||||
active_rename_id: None,
|
||||
created_at: Some("2026-05-23T00:00:00Z".to_string()),
|
||||
updated_at: Some("2026-05-23T00:00:00Z".to_string()),
|
||||
};
|
||||
@@ -3441,6 +3442,7 @@ fn test_bucket_entry(bucket: &str) -> TableBucketEntry {
|
||||
warehouse_root: format!("s3://{bucket}/"),
|
||||
state: TableCatalogEntryState::Active,
|
||||
properties: BTreeMap::new(),
|
||||
active_rename_id: None,
|
||||
created_at: None,
|
||||
updated_at: None,
|
||||
}
|
||||
@@ -5107,7 +5109,8 @@ async fn object_catalog_pagination_bounds_reads_and_covers_rest_resources() {
|
||||
.await
|
||||
.expect("first table page should load");
|
||||
assert_eq!(table_page.entries[0].table, "alpha");
|
||||
assert_eq!(backend.read_call_count().await, 1);
|
||||
// One read snapshots the bucket rename fence and one loads the page entry.
|
||||
assert_eq!(backend.read_call_count().await, 2);
|
||||
let table_page = store
|
||||
.list_tables_page(bucket, &namespace_name, table_page.next_cursor.as_deref(), one)
|
||||
.await
|
||||
@@ -17752,7 +17755,461 @@ async fn strong_catalog_table_rename_returns_success_after_committed_snapshot_re
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn configured_object_catalog_rejects_table_rename() {
|
||||
async fn object_catalog_table_rename_preserves_identity_index_and_reuses_source_tombstone() {
|
||||
let backend = TestCatalogObjectBackend {
|
||||
content_addressed_etags: true,
|
||||
..Default::default()
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend);
|
||||
let bucket = "analytics";
|
||||
let source_namespace = Namespace::parse("sales").expect("source namespace should parse");
|
||||
let destination_namespace = Namespace::parse("curated").expect("destination namespace should parse");
|
||||
let source_table = IdentifierSegment::parse("orders").expect("source table should parse");
|
||||
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &source_namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &destination_namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
let source = test_table_entry(
|
||||
bucket,
|
||||
&source_namespace,
|
||||
&source_table,
|
||||
default_table_metadata_file_path(&source_namespace, &source_table, "00001.metadata.json"),
|
||||
);
|
||||
store.create_table(source.clone()).await.unwrap();
|
||||
let bucket_object = store.paths.table_bucket_entry_path(bucket);
|
||||
let bucket_etag_before = store
|
||||
.read_entry::<TableBucketEntry>(RUSTFS_META_BUCKET, &bucket_object)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.1
|
||||
.expect("table bucket should have an etag");
|
||||
|
||||
store
|
||||
.rename_table(bucket, "sales", "orders", "curated", "orders_v2")
|
||||
.await
|
||||
.expect("object-backed table rename should complete");
|
||||
let bucket_etag_after = store
|
||||
.read_entry::<TableBucketEntry>(RUSTFS_META_BUCKET, &bucket_object)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.1
|
||||
.expect("table bucket should have an etag");
|
||||
assert_ne!(bucket_etag_after, bucket_etag_before);
|
||||
|
||||
assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_none());
|
||||
let destination = store
|
||||
.load_table(bucket, "curated", "orders_v2")
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("destination table should exist");
|
||||
let mut expected_destination = source.clone();
|
||||
expected_destination.namespace = "curated".to_string();
|
||||
expected_destination.table = "orders_v2".to_string();
|
||||
assert_ne!(destination.updated_at, source.updated_at);
|
||||
expected_destination.updated_at.clone_from(&destination.updated_at);
|
||||
assert_eq!(destination, expected_destination);
|
||||
|
||||
let source_object = store.paths.table_entry_path(bucket, &source_namespace, &source_table);
|
||||
let source_tombstone = store
|
||||
.read_entry::<TableEntry>(RUSTFS_META_BUCKET, &source_object)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("source tombstone should remain")
|
||||
.0;
|
||||
assert_eq!(source_tombstone.state, TableCatalogEntryState::Deleted);
|
||||
assert_eq!(source_tombstone.table_id, destination.table_id);
|
||||
assert!(
|
||||
store
|
||||
.get_table_bucket(bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("table bucket should exist")
|
||||
.active_rename_id
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let destination_index = table_warehouse_index_entry(&destination).unwrap();
|
||||
let index_object = store
|
||||
.paths
|
||||
.warehouse_index_entry_path(bucket, &destination_index.warehouse_object_prefix);
|
||||
let persisted_index = store
|
||||
.read_entry::<TableWarehouseIndexEntry>(RUSTFS_META_BUCKET, &index_object)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("warehouse index should exist")
|
||||
.0;
|
||||
assert_eq!(persisted_index, destination_index);
|
||||
let resource = store
|
||||
.resolve_table_data_plane_resource(bucket, "tables/table-id/data/part.parquet")
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("renamed table should resolve its stable warehouse prefix");
|
||||
assert_eq!(resource.namespace, "curated");
|
||||
assert_eq!(resource.table, "orders_v2");
|
||||
store
|
||||
.backfill_table_warehouse_index(bucket)
|
||||
.await
|
||||
.expect("retained source tombstone should not make the warehouse index ambiguous");
|
||||
let migration = store.plan_durable_strong_backing_migration(bucket).await.unwrap();
|
||||
assert_eq!(migration.table_count, 1);
|
||||
assert!(
|
||||
!migration
|
||||
.blockers
|
||||
.contains(&TableCatalogBackingMigrationBlocker::DuplicateTableIdentity)
|
||||
);
|
||||
|
||||
store
|
||||
.rename_table(bucket, "curated", "orders_v2", "sales", "orders")
|
||||
.await
|
||||
.expect("rename should conditionally replace the retained source tombstone");
|
||||
store
|
||||
.rename_table(bucket, "sales", "orders", "curated", "orders_v2")
|
||||
.await
|
||||
.expect("rename should conditionally replace a destination tombstone");
|
||||
|
||||
let mut replacement = test_table_entry(
|
||||
bucket,
|
||||
&source_namespace,
|
||||
&source_table,
|
||||
default_table_metadata_file_path(&source_namespace, &source_table, "00002.metadata.json"),
|
||||
);
|
||||
replacement.table_id = "replacement-table-id".to_string();
|
||||
replacement.table_uuid = "replacement-table-uuid".to_string();
|
||||
replacement.warehouse_location = "s3://analytics/tables/replacement-table-id".to_string();
|
||||
store
|
||||
.create_table(replacement.clone())
|
||||
.await
|
||||
.expect("create should conditionally replace the source tombstone");
|
||||
assert_eq!(
|
||||
store
|
||||
.load_table(bucket, "sales", "orders")
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("source identifier should be reusable"),
|
||||
replacement
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_table_rename_rejects_missing_and_conflicting_destinations() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = ObjectTableCatalogStore::new(backend);
|
||||
let bucket = "analytics";
|
||||
let source_namespace = Namespace::parse("sales").unwrap();
|
||||
let destination_namespace = Namespace::parse("curated").unwrap();
|
||||
let source_table = IdentifierSegment::parse("orders").unwrap();
|
||||
let destination_table = IdentifierSegment::parse("orders_v2").unwrap();
|
||||
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &source_namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &destination_namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.create_table(test_table_entry(
|
||||
bucket,
|
||||
&source_namespace,
|
||||
&source_table,
|
||||
default_table_metadata_file_path(&source_namespace, &source_table, "00001.metadata.json"),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_matches!(
|
||||
store.rename_table(bucket, "sales", "orders", "missing", "orders_v2").await,
|
||||
Err(TableCatalogStoreError::NamespaceNotFound(_))
|
||||
);
|
||||
assert_matches!(
|
||||
store.rename_table(bucket, "sales", "missing", "curated", "orders_v2").await,
|
||||
Err(TableCatalogStoreError::TableNotFound(_))
|
||||
);
|
||||
let mut existing = test_table_entry(
|
||||
bucket,
|
||||
&destination_namespace,
|
||||
&destination_table,
|
||||
default_table_metadata_file_path(&destination_namespace, &destination_table, "00001.metadata.json"),
|
||||
);
|
||||
existing.table_id = "destination-table-id".to_string();
|
||||
existing.table_uuid = "destination-table-uuid".to_string();
|
||||
existing.warehouse_location = "s3://analytics/tables/destination-table-id".to_string();
|
||||
store.create_table(existing).await.unwrap();
|
||||
assert_matches!(
|
||||
store.rename_table(bucket, "sales", "orders", "curated", "orders_v2").await,
|
||||
Err(TableCatalogStoreError::AlreadyExists(_))
|
||||
);
|
||||
let destination_view = IdentifierSegment::parse("orders_view").unwrap();
|
||||
store
|
||||
.create_view(test_view_entry(
|
||||
bucket,
|
||||
&destination_namespace,
|
||||
&destination_view,
|
||||
default_view_metadata_file_path(&destination_namespace, &destination_view, "00001.metadata.json"),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_matches!(
|
||||
store.rename_table(bucket, "sales", "orders", "curated", "orders_view").await,
|
||||
Err(TableCatalogStoreError::AlreadyExists(_))
|
||||
);
|
||||
assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_table_rename_fails_closed_around_durable_fence_creation() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let bucket = "analytics";
|
||||
let source_namespace = Namespace::parse("sales").unwrap();
|
||||
let destination_namespace = Namespace::parse("curated").unwrap();
|
||||
let source_table = IdentifierSegment::parse("orders").unwrap();
|
||||
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &source_namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &destination_namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.create_table(test_table_entry(
|
||||
bucket,
|
||||
&source_namespace,
|
||||
&source_table,
|
||||
default_table_metadata_file_path(&source_namespace, &source_table, "00001.metadata.json"),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let bucket_object = store.paths.table_bucket_entry_path(bucket);
|
||||
backend.fail_next_put(RUSTFS_META_BUCKET, &bucket_object).await;
|
||||
assert_matches!(
|
||||
store.rename_table(bucket, "sales", "orders", "curated", "orders_v2").await,
|
||||
Err(TableCatalogStoreError::Internal(_))
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_table_bucket(bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("table bucket should remain")
|
||||
.active_rename_id
|
||||
.is_none()
|
||||
);
|
||||
assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_some());
|
||||
assert!(store.load_table(bucket, "curated", "orders_v2").await.unwrap().is_none());
|
||||
|
||||
let mut fenced_bucket = store.get_table_bucket(bucket).await.unwrap().unwrap();
|
||||
fenced_bucket.active_rename_id = Some("missing-intent".to_string());
|
||||
backend
|
||||
.seed_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
&bucket_object,
|
||||
serde_json::to_vec(&fenced_bucket).expect("fenced bucket should serialize"),
|
||||
)
|
||||
.await;
|
||||
assert_matches!(
|
||||
store.rename_table(bucket, "sales", "orders", "curated", "orders_v2").await,
|
||||
Err(TableCatalogStoreError::Unavailable(_))
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.get_table_bucket(bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("table bucket should remain fail-closed")
|
||||
.active_rename_id
|
||||
.as_deref(),
|
||||
Some("missing-intent")
|
||||
);
|
||||
assert_matches!(
|
||||
store
|
||||
.resolve_table_data_plane_resource(bucket, "tables/table-id/data/part.parquet")
|
||||
.await,
|
||||
Err(TableCatalogStoreError::Unavailable(_))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_table_rename_recovers_after_destination_publish_and_fences_concurrent_mutations() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let bucket = "analytics";
|
||||
let source_namespace = Namespace::parse("sales").expect("source namespace should parse");
|
||||
let destination_namespace = Namespace::parse("curated").expect("destination namespace should parse");
|
||||
let source_table = IdentifierSegment::parse("orders").expect("source table should parse");
|
||||
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &source_namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &destination_namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
let source = test_table_entry(
|
||||
bucket,
|
||||
&source_namespace,
|
||||
&source_table,
|
||||
default_table_metadata_file_path(&source_namespace, &source_table, "00001.metadata.json"),
|
||||
);
|
||||
store.create_table(source.clone()).await.unwrap();
|
||||
|
||||
let source_object = store.paths.table_entry_path(bucket, &source_namespace, &source_table);
|
||||
let source_tombstone_attempt = backend.put_attempt_count(RUSTFS_META_BUCKET, &source_object).await + 2;
|
||||
let source_tombstone_pause = backend
|
||||
.pause_put_attempt(RUSTFS_META_BUCKET, &source_object, source_tombstone_attempt)
|
||||
.await;
|
||||
let rename_store = store.clone();
|
||||
let rename = tokio::spawn(async move {
|
||||
rename_store
|
||||
.rename_table(bucket, "sales", "orders", "curated", "orders_v2")
|
||||
.await
|
||||
});
|
||||
source_tombstone_pause.wait_started().await;
|
||||
|
||||
let active_rename_id = store
|
||||
.get_table_bucket(bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("table bucket should exist")
|
||||
.active_rename_id
|
||||
.expect("rename fence should be durable before destination publication");
|
||||
assert_matches!(
|
||||
store.load_table(bucket, "sales", "orders").await,
|
||||
Err(TableCatalogStoreError::Unavailable(_))
|
||||
);
|
||||
assert_matches!(store.list_tables(bucket, "sales").await, Err(TableCatalogStoreError::Unavailable(_)));
|
||||
assert_matches!(
|
||||
store
|
||||
.resolve_table_data_plane_resource(bucket, "tables/table-id/data/part.parquet")
|
||||
.await,
|
||||
Err(TableCatalogStoreError::Unavailable(_))
|
||||
);
|
||||
let migration = store.plan_durable_strong_backing_migration(bucket).await.unwrap();
|
||||
assert_eq!(migration.status, TableCatalogBackingMigrationStatus::RecoveryRequired);
|
||||
assert!(
|
||||
migration
|
||||
.blockers
|
||||
.contains(&TableCatalogBackingMigrationBlocker::TableRenameRecoveryRequired)
|
||||
);
|
||||
|
||||
let publication_lock = default_table_bucket_publication_lock_path();
|
||||
let publication_attempts = backend.write_lock_acquisition_count(bucket, &publication_lock).await;
|
||||
let commit_store = store.clone();
|
||||
let commit = tokio::spawn(async move {
|
||||
commit_store
|
||||
.commit_table(TableCommitRequest {
|
||||
table_bucket: bucket.to_string(),
|
||||
namespace: "sales".to_string(),
|
||||
table: "orders".to_string(),
|
||||
commit_id: "concurrent-commit".to_string(),
|
||||
idempotency_key: None,
|
||||
operation: "append".to_string(),
|
||||
expected_version_token: source.version_token,
|
||||
expected_metadata_location: source.metadata_location,
|
||||
new_metadata_location: "unused.metadata.json".to_string(),
|
||||
requirements: Vec::new(),
|
||||
writer: Some("rename-test".to_string()),
|
||||
})
|
||||
.await
|
||||
});
|
||||
let drop_store = store.clone();
|
||||
let drop_table = tokio::spawn(async move { drop_store.drop_table(bucket, "sales", "orders").await });
|
||||
let create_store = store.clone();
|
||||
let mut replacement = test_table_entry(
|
||||
bucket,
|
||||
&source_namespace,
|
||||
&source_table,
|
||||
default_table_metadata_file_path(&source_namespace, &source_table, "00002.metadata.json"),
|
||||
);
|
||||
replacement.table_id = "replacement-table-id".to_string();
|
||||
replacement.table_uuid = "replacement-table-uuid".to_string();
|
||||
replacement.warehouse_location = "s3://analytics/tables/replacement-table-id".to_string();
|
||||
let create = tokio::spawn(async move { create_store.create_table(replacement).await });
|
||||
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async {
|
||||
while backend.write_lock_acquisition_count(bucket, &publication_lock).await < publication_attempts + 3 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("concurrent mutations should reach the table-bucket publication fence");
|
||||
assert!(!commit.is_finished());
|
||||
assert!(!drop_table.is_finished());
|
||||
assert!(!create.is_finished());
|
||||
commit.abort();
|
||||
drop_table.abort();
|
||||
create.abort();
|
||||
|
||||
rename.abort();
|
||||
source_tombstone_pause.release();
|
||||
let _ = rename.await;
|
||||
let destination_object = store.paths.table_entry_path(
|
||||
bucket,
|
||||
&destination_namespace,
|
||||
&IdentifierSegment::parse("orders_v2").expect("destination table should parse"),
|
||||
);
|
||||
let source_fence = store
|
||||
.read_entry::<TableEntry>(RUSTFS_META_BUCKET, &source_object)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("source rename fence should be durable")
|
||||
.0;
|
||||
let destination_fence = store
|
||||
.read_entry::<TableEntry>(RUSTFS_META_BUCKET, &destination_object)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("destination rename fence should be durable")
|
||||
.0;
|
||||
assert_eq!(source_fence.state, TableCatalogEntryState::Renaming);
|
||||
assert_eq!(destination_fence.state, TableCatalogEntryState::Renaming);
|
||||
assert_matches!(
|
||||
store.load_table(bucket, "curated", "orders_v2").await,
|
||||
Err(TableCatalogStoreError::Unavailable(_))
|
||||
);
|
||||
assert_matches!(
|
||||
store.rename_table(bucket, "sales", "orders", "curated", "orders_v2").await,
|
||||
Err(TableCatalogStoreError::TableNotFound(_))
|
||||
);
|
||||
|
||||
assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_none());
|
||||
let destination = store
|
||||
.load_table(bucket, "curated", "orders_v2")
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("recovery should finish the destination publication");
|
||||
assert_eq!(destination.table_id, "table-id");
|
||||
assert!(
|
||||
store
|
||||
.get_table_bucket(bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("table bucket should exist")
|
||||
.active_rename_id
|
||||
.is_none()
|
||||
);
|
||||
let intent_object = store.paths.table_rename_intent_path(bucket, &active_rename_id);
|
||||
let intent = store
|
||||
.read_entry::<TableRenameIntent>(RUSTFS_META_BUCKET, &intent_object)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("completed rename intent should be retained as a recovery record")
|
||||
.0;
|
||||
assert_eq!(intent.state, TableRenameIntentState::Completed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn configured_object_catalog_dispatches_table_rename() {
|
||||
let store =
|
||||
ConfiguredTableCatalogStore::new_for_test(TestCatalogObjectBackend::default(), TableCatalogBackingMode::ObjectBacked);
|
||||
|
||||
@@ -17760,6 +18217,6 @@ async fn configured_object_catalog_rejects_table_rename() {
|
||||
store
|
||||
.rename_table("analytics", "sales", "orders", "curated", "orders_v2")
|
||||
.await,
|
||||
Err(TableCatalogStoreError::Unsupported(_))
|
||||
Err(TableCatalogStoreError::NotFound(_))
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user