feat(table-catalog): add durable backing state transfer (#4952)

* feat(table-catalog): add durable backing state transfer

* fix(table-catalog): reject orphaned migration entries

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
Henry Guo
2026-07-18 10:45:32 +08:00
committed by GitHub
parent 230e5fc31a
commit 906805568b
10 changed files with 1730 additions and 46 deletions
+6
View File
@@ -508,6 +508,8 @@ pub enum AdminAction {
ExportBucketMetadataAction,
#[strum(serialize = "admin:GetTableCatalog")]
GetTableCatalogAction,
#[strum(serialize = "admin:MigrateTableCatalog")]
MigrateTableCatalogAction,
#[strum(serialize = "admin:GetTableBucket")]
GetTableBucketAction,
#[strum(serialize = "admin:SetTableBucket")]
@@ -658,6 +660,7 @@ impl AdminAction {
| AdminAction::ImportBucketMetadataAction
| AdminAction::ExportBucketMetadataAction
| AdminAction::GetTableCatalogAction
| AdminAction::MigrateTableCatalogAction
| AdminAction::GetTableBucketAction
| AdminAction::SetTableBucketAction
| AdminAction::GetTableNamespaceAction
@@ -832,9 +835,12 @@ mod tests {
#[test]
fn test_table_catalog_admin_action_is_valid() {
let get_action = AdminAction::try_from("admin:GetTableCatalog").expect("Should parse GetTableCatalog action");
let migrate_action = AdminAction::try_from("admin:MigrateTableCatalog").expect("Should parse MigrateTableCatalog action");
assert_eq!(get_action, AdminAction::GetTableCatalogAction);
assert_eq!(migrate_action, AdminAction::MigrateTableCatalogAction);
assert!(get_action.is_valid());
assert!(migrate_action.is_valid());
}
#[test]
+27 -17
View File
@@ -114,11 +114,12 @@ catalog extension.
| Idempotent retry | Supported | Repeated commit IDs can return the already finalized result or surface recoverable finalization gaps. |
| Post-CAS finalization recovery | Supported | Diagnostics and recovery can repair stale or missing idempotency indexes without changing the current table pointer. |
| Catalog export | Supported | Exposes table state, commit recovery state, and backing migration information for operator inspection. |
| Strong backing migration contract | Supported as a contract | Diagnostics publish object-backed manifest state, recoverable commit-log WAL state, target backing type, replay requirements, and blockers. |
| Durable backing migration dry-run | Supported | `GET /iceberg/v1/{warehouse}/catalog/migration` and the `/_iceberg/v1` alias inspect object-backed catalog inventory, commit recovery blockers, idempotency indexes, warehouse prefix index readiness, recommended actions, and rollback configuration without mutating catalog state. |
| Strong backing state transfer | Supported | Object-backed table bucket, namespace, table, view, commit-log, and idempotency state can be materialized into the durable strong snapshot. The transfer is deterministic, ETag-CAS protected, idempotent after an interrupted finalization, and fails closed when a table or view has no owning namespace entry. |
| Durable backing migration preflight | Supported | `GET /iceberg/v1/{warehouse}/catalog/migration` and the `/_iceberg/v1` alias inspect object-backed catalog inventory, recovery blockers, warehouse prefix index readiness, persistent write-fence state, target snapshot agreement, and whether every table bucket is ready for cutover. |
| Durable backing migration execution | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/migration` fences table-bucket registry changes, acquires a persistent per-bucket write fence, drains in-flight catalog mutations, materializes the target snapshot, and reports `ready_to_enable_durable_strong`. `DELETE` safely releases the bucket fence only while its target state has not advanced, and releases the registry fence after the last bucket is cancelled. Both mutations require `admin:MigrateTableCatalog`. |
| Disaster recovery rehearsal | Manual/live harness | `failure_coverage.py --print-disaster-recovery-rehearsal` generates an operator runbook covering catalog export, diagnostics, safe recovery repair, rollback/import, durable backing migration dry-run, post-recovery loadTable, and table data-plane policy probes. |
| Scale and fault rehearsal | Manual/live harness | `failure_coverage.py --print-scale-fault-rehearsal` generates an opt-in runbook for concurrent writer stress, maintenance scheduler lease recovery, durable backing cutover preflight, recovery/rollback/import under load, and post-run evidence capture. |
| Strong KV/WAL backing cutover | Preview / controlled | Operators can explicitly select durable strong backing with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong`. Run the migration dry-run first and keep the object-backed catalog available for rollback. Object-only advanced operations fail closed in durable strong mode. |
| Strong KV/WAL backing cutover | Preview / controlled | Operators can select durable strong backing with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong` only after every table bucket reports `SNAPSHOT_MATERIALIZED` and `ready_to_enable_durable_strong: true`. Object-only advanced operations fail closed in durable strong mode. |
| Single active writer region | Supported policy | Diagnostics publish single-active-writer semantics and read-only replica limits. |
| Active-active multi-region writes | Not claimed | A table must not accept independent concurrent writers in multiple active regions. |
@@ -127,20 +128,29 @@ catalog extension.
Use the migration dry-run before changing the table catalog backing for a
warehouse:
1. Run `GET /iceberg/v1/{warehouse}/catalog/migration` with a principal that has
`GetTableCatalogAction` on the table bucket.
2. Treat any `blockers` entry as fail-closed. Run catalog recovery first when
commit recovery or idempotency repair is required, and backfill the warehouse
prefix index before switching backing modes.
3. Take an object-backed catalog snapshot or backup before setting
`RUSTFS_TABLE_CATALOG_BACKING=durable-strong`.
4. Restart with durable strong backing enabled, then verify catalog config,
table load, commit recovery diagnostics, and table data-plane policy
resolution for representative tables.
5. To roll back, restore `RUSTFS_TABLE_CATALOG_BACKING=object-backed` and restart
while preserving the object-backed catalog objects. Do not delete object-backed
catalog state until durable strong backing has passed the operator's retention
window.
1. Take an object-backed catalog backup and record the current metadata pointer
and version token for representative tables.
2. Run `GET /iceberg/v1/{warehouse}/catalog/migration` with a principal that has
`GetTableCatalogAction` on each table bucket. Treat every `blockers` entry as
fail-closed; repair commit recovery state and backfill the warehouse prefix
index before continuing.
3. Run `POST /iceberg/v1/{warehouse}/catalog/migration` with
`admin:MigrateTableCatalog`. This persists the source write fence before it
drains in-flight mutations and copies the catalog state.
4. Repeat the preflight and materialization for every table bucket. Do not set
`RUSTFS_TABLE_CATALOG_BACKING=durable-strong` until the preflight reports
`SNAPSHOT_MATERIALIZED`, no blockers, and
`ready_to_enable_durable_strong: true`.
5. Restart with durable strong backing enabled, then verify catalog config,
table and view loads, commit idempotency, and table data-plane policy
resolution before admitting writers.
6. Before restarting into durable-strong mode, `DELETE` on the migration
endpoint can remove a migration-created target bucket snapshot and release
the source fence. After the durable-strong state advances, cancellation
fails closed; recovery requires an operator-selected restore or reverse
migration instead of restarting against the stale object-backed pointer.
7. Preserve the object-backed catalog backup until durable strong backing has
passed the operator's retention window.
## Production Failure Coverage
@@ -118,6 +118,8 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[
"PUT /buckets/{warehouse}",
"GET /buckets/{warehouse}",
"GET /{warehouse}/catalog/migration",
"POST /{warehouse}/catalog/migration",
"DELETE /{warehouse}/catalog/migration",
"GET /{warehouse}/namespaces",
"POST /{warehouse}/namespaces",
"GET /{warehouse}/namespaces/{namespace}",
@@ -165,6 +167,9 @@ static GET_CONFIG_HANDLER: GetCatalogConfigHandler = GetCatalogConfigHandler {};
static ENABLE_TABLE_BUCKET_HANDLER: EnableTableBucketHandler = EnableTableBucketHandler {};
static GET_TABLE_BUCKET_HANDLER: GetTableBucketHandler = GetTableBucketHandler {};
static GET_TABLE_CATALOG_MIGRATION_HANDLER: GetTableCatalogMigrationHandler = GetTableCatalogMigrationHandler {};
static MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER: MaterializeTableCatalogMigrationHandler =
MaterializeTableCatalogMigrationHandler {};
static CANCEL_TABLE_CATALOG_MIGRATION_HANDLER: CancelTableCatalogMigrationHandler = CancelTableCatalogMigrationHandler {};
static LIST_NAMESPACES_HANDLER: RestListNamespacesHandler = RestListNamespacesHandler {};
static CREATE_NAMESPACE_HANDLER: RestCreateNamespaceHandler = RestCreateNamespaceHandler {};
static GET_NAMESPACE_HANDLER: RestGetNamespaceHandler = RestGetNamespaceHandler {};
@@ -832,6 +837,16 @@ fn register_table_catalog_prefix_routes(r: &mut S3Router<AdminOperation>, prefix
format!("{prefix}/{{warehouse}}/catalog/migration").as_str(),
AdminOperation(&GET_TABLE_CATALOG_MIGRATION_HANDLER),
)?;
r.insert(
Method::POST,
format!("{prefix}/{{warehouse}}/catalog/migration").as_str(),
AdminOperation(&MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER),
)?;
r.insert(
Method::DELETE,
format!("{prefix}/{{warehouse}}/catalog/migration").as_str(),
AdminOperation(&CANCEL_TABLE_CATALOG_MIGRATION_HANDLER),
)?;
r.insert(
Method::GET,
format!("{prefix}/{{warehouse}}/namespaces").as_str(),
@@ -5012,6 +5027,46 @@ impl Operation for GetTableCatalogMigrationHandler {
}
}
pub struct MaterializeTableCatalogMigrationHandler {}
#[async_trait::async_trait]
impl Operation for MaterializeTableCatalogMigrationHandler {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let warehouse = warehouse_from_params(&params)?;
authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
ensure_table_bucket_enabled(&warehouse).await?;
let store = table_catalog_object_store()?;
let started = Instant::now();
let result = store
.materialize_durable_strong_backing_migration(&warehouse)
.await
.map_err(catalog_store_error);
record_table_catalog_admin_operation_result("migration-materialize", &warehouse, "", "", started, &result);
let response = result?;
build_json_response(StatusCode::OK, &response)
}
}
pub struct CancelTableCatalogMigrationHandler {}
#[async_trait::async_trait]
impl Operation for CancelTableCatalogMigrationHandler {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let warehouse = warehouse_from_params(&params)?;
authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
ensure_table_bucket_enabled(&warehouse).await?;
let store = table_catalog_object_store()?;
let started = Instant::now();
let result = store
.cancel_durable_strong_backing_migration(&warehouse)
.await
.map_err(catalog_store_error);
record_table_catalog_admin_operation_result("migration-cancel", &warehouse, "", "", started, &result);
let response = result?;
build_json_response(StatusCode::OK, &response)
}
}
pub struct RestListNamespacesHandler {}
#[async_trait::async_trait]
@@ -5871,6 +5926,8 @@ mod tests {
assert_eq!(response.admin_discovery.extensions_catalog, "/rustfs/admin/v4/extensions/catalog");
assert!(response.endpoints.contains(&"GET /v1/{prefix}/namespaces"));
assert!(response.endpoints.contains(&"GET /{warehouse}/catalog/migration"));
assert!(response.endpoints.contains(&"POST /{warehouse}/catalog/migration"));
assert!(response.endpoints.contains(&"DELETE /{warehouse}/catalog/migration"));
assert!(response.endpoints.contains(&"HEAD /v1/{prefix}/namespaces/{namespace}"));
assert!(
response
@@ -6122,6 +6179,20 @@ mod tests {
migration_block.contains("TableCatalogResource::warehouse(&warehouse)"),
"catalog migration dry-run should authorize against the warehouse resource"
);
for handler in [
"MaterializeTableCatalogMigrationHandler",
"CancelTableCatalogMigrationHandler",
] {
let block = operation_block(src, handler);
assert!(
block.contains("authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;"),
"{handler} should require the global catalog migration action"
);
assert!(
!block.contains("authorize_table_catalog_resource_request("),
"{handler} must not imply that a global backing cutover is warehouse-scoped"
);
}
for (handler, action) in [
("RestLoadTableHandler", "AdminAction::GetTableMetadataAction"),
@@ -6166,6 +6237,8 @@ mod tests {
for handler in [
"GetTableCatalogMigrationHandler",
"MaterializeTableCatalogMigrationHandler",
"CancelTableCatalogMigrationHandler",
"RestListNamespacesHandler",
"RestCreateNamespaceHandler",
"RestGetNamespaceHandler",
@@ -6280,6 +6353,8 @@ mod tests {
let _: &EnableTableBucketHandler = &ENABLE_TABLE_BUCKET_HANDLER;
let _: &GetTableBucketHandler = &GET_TABLE_BUCKET_HANDLER;
let _: &GetTableCatalogMigrationHandler = &GET_TABLE_CATALOG_MIGRATION_HANDLER;
let _: &MaterializeTableCatalogMigrationHandler = &MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER;
let _: &CancelTableCatalogMigrationHandler = &CANCEL_TABLE_CATALOG_MIGRATION_HANDLER;
let _: &RestListNamespacesHandler = &LIST_NAMESPACES_HANDLER;
let _: &RestCreateNamespaceHandler = &CREATE_NAMESPACE_HANDLER;
let _: &RestGetNamespaceHandler = &GET_NAMESPACE_HANDLER;
@@ -6320,6 +6395,8 @@ mod tests {
assert_operation::<EnableTableBucketHandler>();
assert_operation::<GetTableBucketHandler>();
assert_operation::<GetTableCatalogMigrationHandler>();
assert_operation::<MaterializeTableCatalogMigrationHandler>();
assert_operation::<CancelTableCatalogMigrationHandler>();
assert_operation::<RestListNamespacesHandler>();
assert_operation::<RestCreateNamespaceHandler>();
assert_operation::<RestGetNamespaceHandler>();
+30 -1
View File
@@ -65,6 +65,7 @@ const LIST_TEMPORARY_ACCOUNTS: AdminActionRef = AdminActionRef::new("ListTempora
const LIST_TIER: AdminActionRef = AdminActionRef::new("ListTierAction");
const LIST_USER_POLICIES: AdminActionRef = AdminActionRef::new("ListUserPoliciesAdminAction");
const LIST_USERS: AdminActionRef = AdminActionRef::new("ListUsersAdminAction");
const MIGRATE_TABLE_CATALOG: AdminActionRef = AdminActionRef::new("MigrateTableCatalogAction");
const PROFILING: AdminActionRef = AdminActionRef::new("ProfilingAdminAction");
const REBALANCE: AdminActionRef = AdminActionRef::new("RebalanceAdminAction");
const REGISTER_TABLE: AdminActionRef = AdminActionRef::new("RegisterTableAction");
@@ -789,6 +790,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
GET_TABLE_CATALOG,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/catalog/migration",
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Delete,
"/iceberg/v1/{warehouse}/catalog/migration",
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/iceberg/v1/{warehouse}/namespaces",
@@ -1054,6 +1067,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
GET_TABLE_CATALOG,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/_iceberg/v1/{warehouse}/catalog/migration",
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Delete,
"/_iceberg/v1/{warehouse}/catalog/migration",
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/_iceberg/v1/{warehouse}/namespaces",
@@ -1531,7 +1556,7 @@ mod tests {
let table_specs = ADMIN_ROUTE_POLICY_SPECS
.iter()
.filter(|spec| spec.path().starts_with("/iceberg/v1") || spec.path().starts_with("/_iceberg/v1"));
assert_eq!(table_specs.count(), 90);
assert_eq!(table_specs.count(), 94);
assert_action(HttpMethod::Put, "/iceberg/v1/buckets/{warehouse}", SET_TABLE_BUCKET);
assert_action(HttpMethod::Get, "/_iceberg/v1/buckets/{warehouse}", GET_TABLE_BUCKET);
assert_action(HttpMethod::Get, "/iceberg/v1/{warehouse}/namespaces", GET_TABLE_NAMESPACE);
@@ -1698,6 +1723,10 @@ mod tests {
);
assert_action(HttpMethod::Get, "/iceberg/v1/{warehouse}/catalog/migration", GET_TABLE_CATALOG);
assert_action(HttpMethod::Get, "/_iceberg/v1/{warehouse}/catalog/migration", GET_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Delete, "/iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Delete, "/_iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
assert_action(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import",
@@ -342,6 +342,8 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
table_route_sample(Method::PUT, "/buckets/{warehouse}", "/buckets/analytics"),
table_route_sample(Method::GET, "/buckets/{warehouse}", "/buckets/analytics"),
table_route_sample(Method::GET, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
table_route_sample(Method::POST, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
table_route_sample(Method::DELETE, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
table_route_sample(Method::GET, "/{warehouse}/namespaces", "/analytics/namespaces"),
table_route_sample(Method::POST, "/{warehouse}/namespaces", "/analytics/namespaces"),
table_route_sample(Method::GET, "/{warehouse}/namespaces/{namespace}", "/analytics/namespaces/sales"),
@@ -531,6 +533,8 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
compat_table_route_sample(Method::PUT, "/buckets/{warehouse}", "/buckets/analytics"),
compat_table_route_sample(Method::GET, "/buckets/{warehouse}", "/buckets/analytics"),
compat_table_route_sample(Method::GET, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
compat_table_route_sample(Method::DELETE, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
compat_table_route_sample(Method::GET, "/{warehouse}/namespaces", "/analytics/namespaces"),
compat_table_route_sample(Method::POST, "/{warehouse}/namespaces", "/analytics/namespaces"),
compat_table_route_sample(Method::GET, "/{warehouse}/namespaces/{namespace}", "/analytics/namespaces/sales"),
File diff suppressed because it is too large Load Diff
+15 -1
View File
@@ -335,7 +335,14 @@ def disaster_recovery_rehearsal_plan(
"GET",
warehouse_path(warehouse, "/catalog/migration", rest_path),
"200",
"migration blockers must be empty before cutover",
"migration blockers must be empty and status must be READY_TO_SNAPSHOT before state transfer",
),
probe_step(
"durable-backing-migration-materialize",
"POST",
warehouse_path(warehouse, "/catalog/migration", rest_path),
"200",
"source writes are fenced and the durable strong snapshot is materialized before cutover",
),
],
),
@@ -504,6 +511,13 @@ def scale_fault_rehearsal_plan(
"200",
"migration dry-run reports inventory, replay blockers, idempotency blockers, and recommended actions",
),
probe_step(
"materialize-durable-backing-before-cutover",
"POST",
warehouse_path(warehouse, "/catalog/migration", rest_path),
"200",
"materialization drains in-flight writes and reports SNAPSHOT_MATERIALIZED before backing restart",
),
],
),
rehearsal_phase(
+2 -2
View File
@@ -239,8 +239,8 @@ UNSUPPORTED_INVENTORY: list[dict[str, str]] = [
PRODUCTION_READINESS_INVENTORY: list[dict[str, str]] = [
{
"capability": "strong-catalog-backing",
"status": "migration-contract-supported",
"validation": "catalog export and diagnostics expose an object-backed manifest, recoverable commit-log WAL state, strong-kv-wal migration target, required migration steps, and recovery blockers before cutover",
"status": "state-transfer-supported",
"validation": "preflight exposes recovery blockers and target agreement; migration execution fences object-backed writes, drains in-flight catalog mutations, and idempotently materializes table bucket, namespace, table, view, commit-log, and idempotency state before cutover",
},
{
"capability": "single-active-writer-ha",
@@ -132,7 +132,11 @@ class FailureCoverageTest(unittest.TestCase):
migration_steps = {step["name"]: step for step in phases["migration-preflight"]["steps"]}
self.assertEqual(migration_steps["durable-backing-migration-dry-run"]["path"], "/iceberg/v1/lake/catalog/migration")
self.assertEqual(migration_steps["durable-backing-migration-dry-run"]["expected_behavior"], "migration blockers must be empty before cutover")
self.assertEqual(migration_steps["durable-backing-migration-materialize"]["method"], "POST")
self.assertIn(
"READY_TO_SNAPSHOT",
migration_steps["durable-backing-migration-dry-run"]["expected_behavior"],
)
validation_steps = {step["name"]: step for step in phases["post-recovery-validation"]["steps"]}
self.assertEqual(validation_steps["load-table-after-recovery"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders")
@@ -202,6 +206,7 @@ class FailureCoverageTest(unittest.TestCase):
cutover_steps = {step["name"]: step for step in phases["durable-backing-cutover-under-load"]["steps"]}
self.assertEqual(cutover_steps["migration-dry-run-before-cutover"]["path"], "/iceberg/v1/lake/catalog/migration")
self.assertEqual(cutover_steps["materialize-durable-backing-before-cutover"]["method"], "POST")
self.assertIn("catalog-backup", cutover_steps["capture-cutover-backup"]["expected_behavior"])
recovery_steps = {step["name"]: step for step in phases["recovery-rollback-import-under-load"]["steps"]}
@@ -864,7 +864,7 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
self.assertIn("single-active-writer-ha", capabilities)
self.assertIn("scale-validation-matrix", capabilities)
strong_backing = next(entry for entry in inventory if entry["capability"] == "strong-catalog-backing")
self.assertEqual(strong_backing["status"], "migration-contract-supported")
self.assertEqual(strong_backing["status"], "state-transfer-supported")
for entry in inventory:
self.assertIn("status", entry)
self.assertIn("validation", entry)