mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
feat(table-catalog): add durable backing state transfer (#4952)
* feat(table-catalog): add durable backing state transfer * fix(table-catalog): reject orphaned migration entries --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com> Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
@@ -508,6 +508,8 @@ pub enum AdminAction {
|
||||
ExportBucketMetadataAction,
|
||||
#[strum(serialize = "admin:GetTableCatalog")]
|
||||
GetTableCatalogAction,
|
||||
#[strum(serialize = "admin:MigrateTableCatalog")]
|
||||
MigrateTableCatalogAction,
|
||||
#[strum(serialize = "admin:GetTableBucket")]
|
||||
GetTableBucketAction,
|
||||
#[strum(serialize = "admin:SetTableBucket")]
|
||||
@@ -658,6 +660,7 @@ impl AdminAction {
|
||||
| AdminAction::ImportBucketMetadataAction
|
||||
| AdminAction::ExportBucketMetadataAction
|
||||
| AdminAction::GetTableCatalogAction
|
||||
| AdminAction::MigrateTableCatalogAction
|
||||
| AdminAction::GetTableBucketAction
|
||||
| AdminAction::SetTableBucketAction
|
||||
| AdminAction::GetTableNamespaceAction
|
||||
@@ -832,9 +835,12 @@ mod tests {
|
||||
#[test]
|
||||
fn test_table_catalog_admin_action_is_valid() {
|
||||
let get_action = AdminAction::try_from("admin:GetTableCatalog").expect("Should parse GetTableCatalog action");
|
||||
let migrate_action = AdminAction::try_from("admin:MigrateTableCatalog").expect("Should parse MigrateTableCatalog action");
|
||||
|
||||
assert_eq!(get_action, AdminAction::GetTableCatalogAction);
|
||||
assert_eq!(migrate_action, AdminAction::MigrateTableCatalogAction);
|
||||
assert!(get_action.is_valid());
|
||||
assert!(migrate_action.is_valid());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -114,11 +114,12 @@ catalog extension.
|
||||
| Idempotent retry | Supported | Repeated commit IDs can return the already finalized result or surface recoverable finalization gaps. |
|
||||
| Post-CAS finalization recovery | Supported | Diagnostics and recovery can repair stale or missing idempotency indexes without changing the current table pointer. |
|
||||
| Catalog export | Supported | Exposes table state, commit recovery state, and backing migration information for operator inspection. |
|
||||
| Strong backing migration contract | Supported as a contract | Diagnostics publish object-backed manifest state, recoverable commit-log WAL state, target backing type, replay requirements, and blockers. |
|
||||
| Durable backing migration dry-run | Supported | `GET /iceberg/v1/{warehouse}/catalog/migration` and the `/_iceberg/v1` alias inspect object-backed catalog inventory, commit recovery blockers, idempotency indexes, warehouse prefix index readiness, recommended actions, and rollback configuration without mutating catalog state. |
|
||||
| Strong backing state transfer | Supported | Object-backed table bucket, namespace, table, view, commit-log, and idempotency state can be materialized into the durable strong snapshot. The transfer is deterministic, ETag-CAS protected, idempotent after an interrupted finalization, and fails closed when a table or view has no owning namespace entry. |
|
||||
| Durable backing migration preflight | Supported | `GET /iceberg/v1/{warehouse}/catalog/migration` and the `/_iceberg/v1` alias inspect object-backed catalog inventory, recovery blockers, warehouse prefix index readiness, persistent write-fence state, target snapshot agreement, and whether every table bucket is ready for cutover. |
|
||||
| Durable backing migration execution | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/migration` fences table-bucket registry changes, acquires a persistent per-bucket write fence, drains in-flight catalog mutations, materializes the target snapshot, and reports `ready_to_enable_durable_strong`. `DELETE` safely releases the bucket fence only while its target state has not advanced, and releases the registry fence after the last bucket is cancelled. Both mutations require `admin:MigrateTableCatalog`. |
|
||||
| Disaster recovery rehearsal | Manual/live harness | `failure_coverage.py --print-disaster-recovery-rehearsal` generates an operator runbook covering catalog export, diagnostics, safe recovery repair, rollback/import, durable backing migration dry-run, post-recovery loadTable, and table data-plane policy probes. |
|
||||
| Scale and fault rehearsal | Manual/live harness | `failure_coverage.py --print-scale-fault-rehearsal` generates an opt-in runbook for concurrent writer stress, maintenance scheduler lease recovery, durable backing cutover preflight, recovery/rollback/import under load, and post-run evidence capture. |
|
||||
| Strong KV/WAL backing cutover | Preview / controlled | Operators can explicitly select durable strong backing with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong`. Run the migration dry-run first and keep the object-backed catalog available for rollback. Object-only advanced operations fail closed in durable strong mode. |
|
||||
| Strong KV/WAL backing cutover | Preview / controlled | Operators can select durable strong backing with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong` only after every table bucket reports `SNAPSHOT_MATERIALIZED` and `ready_to_enable_durable_strong: true`. Object-only advanced operations fail closed in durable strong mode. |
|
||||
| Single active writer region | Supported policy | Diagnostics publish single-active-writer semantics and read-only replica limits. |
|
||||
| Active-active multi-region writes | Not claimed | A table must not accept independent concurrent writers in multiple active regions. |
|
||||
|
||||
@@ -127,20 +128,29 @@ catalog extension.
|
||||
Use the migration dry-run before changing the table catalog backing for a
|
||||
warehouse:
|
||||
|
||||
1. Run `GET /iceberg/v1/{warehouse}/catalog/migration` with a principal that has
|
||||
`GetTableCatalogAction` on the table bucket.
|
||||
2. Treat any `blockers` entry as fail-closed. Run catalog recovery first when
|
||||
commit recovery or idempotency repair is required, and backfill the warehouse
|
||||
prefix index before switching backing modes.
|
||||
3. Take an object-backed catalog snapshot or backup before setting
|
||||
`RUSTFS_TABLE_CATALOG_BACKING=durable-strong`.
|
||||
4. Restart with durable strong backing enabled, then verify catalog config,
|
||||
table load, commit recovery diagnostics, and table data-plane policy
|
||||
resolution for representative tables.
|
||||
5. To roll back, restore `RUSTFS_TABLE_CATALOG_BACKING=object-backed` and restart
|
||||
while preserving the object-backed catalog objects. Do not delete object-backed
|
||||
catalog state until durable strong backing has passed the operator's retention
|
||||
window.
|
||||
1. Take an object-backed catalog backup and record the current metadata pointer
|
||||
and version token for representative tables.
|
||||
2. Run `GET /iceberg/v1/{warehouse}/catalog/migration` with a principal that has
|
||||
`GetTableCatalogAction` on each table bucket. Treat every `blockers` entry as
|
||||
fail-closed; repair commit recovery state and backfill the warehouse prefix
|
||||
index before continuing.
|
||||
3. Run `POST /iceberg/v1/{warehouse}/catalog/migration` with
|
||||
`admin:MigrateTableCatalog`. This persists the source write fence before it
|
||||
drains in-flight mutations and copies the catalog state.
|
||||
4. Repeat the preflight and materialization for every table bucket. Do not set
|
||||
`RUSTFS_TABLE_CATALOG_BACKING=durable-strong` until the preflight reports
|
||||
`SNAPSHOT_MATERIALIZED`, no blockers, and
|
||||
`ready_to_enable_durable_strong: true`.
|
||||
5. Restart with durable strong backing enabled, then verify catalog config,
|
||||
table and view loads, commit idempotency, and table data-plane policy
|
||||
resolution before admitting writers.
|
||||
6. Before restarting into durable-strong mode, `DELETE` on the migration
|
||||
endpoint can remove a migration-created target bucket snapshot and release
|
||||
the source fence. After the durable-strong state advances, cancellation
|
||||
fails closed; recovery requires an operator-selected restore or reverse
|
||||
migration instead of restarting against the stale object-backed pointer.
|
||||
7. Preserve the object-backed catalog backup until durable strong backing has
|
||||
passed the operator's retention window.
|
||||
|
||||
## Production Failure Coverage
|
||||
|
||||
|
||||
@@ -118,6 +118,8 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[
|
||||
"PUT /buckets/{warehouse}",
|
||||
"GET /buckets/{warehouse}",
|
||||
"GET /{warehouse}/catalog/migration",
|
||||
"POST /{warehouse}/catalog/migration",
|
||||
"DELETE /{warehouse}/catalog/migration",
|
||||
"GET /{warehouse}/namespaces",
|
||||
"POST /{warehouse}/namespaces",
|
||||
"GET /{warehouse}/namespaces/{namespace}",
|
||||
@@ -165,6 +167,9 @@ static GET_CONFIG_HANDLER: GetCatalogConfigHandler = GetCatalogConfigHandler {};
|
||||
static ENABLE_TABLE_BUCKET_HANDLER: EnableTableBucketHandler = EnableTableBucketHandler {};
|
||||
static GET_TABLE_BUCKET_HANDLER: GetTableBucketHandler = GetTableBucketHandler {};
|
||||
static GET_TABLE_CATALOG_MIGRATION_HANDLER: GetTableCatalogMigrationHandler = GetTableCatalogMigrationHandler {};
|
||||
static MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER: MaterializeTableCatalogMigrationHandler =
|
||||
MaterializeTableCatalogMigrationHandler {};
|
||||
static CANCEL_TABLE_CATALOG_MIGRATION_HANDLER: CancelTableCatalogMigrationHandler = CancelTableCatalogMigrationHandler {};
|
||||
static LIST_NAMESPACES_HANDLER: RestListNamespacesHandler = RestListNamespacesHandler {};
|
||||
static CREATE_NAMESPACE_HANDLER: RestCreateNamespaceHandler = RestCreateNamespaceHandler {};
|
||||
static GET_NAMESPACE_HANDLER: RestGetNamespaceHandler = RestGetNamespaceHandler {};
|
||||
@@ -832,6 +837,16 @@ fn register_table_catalog_prefix_routes(r: &mut S3Router<AdminOperation>, prefix
|
||||
format!("{prefix}/{{warehouse}}/catalog/migration").as_str(),
|
||||
AdminOperation(&GET_TABLE_CATALOG_MIGRATION_HANDLER),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::POST,
|
||||
format!("{prefix}/{{warehouse}}/catalog/migration").as_str(),
|
||||
AdminOperation(&MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::DELETE,
|
||||
format!("{prefix}/{{warehouse}}/catalog/migration").as_str(),
|
||||
AdminOperation(&CANCEL_TABLE_CATALOG_MIGRATION_HANDLER),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::GET,
|
||||
format!("{prefix}/{{warehouse}}/namespaces").as_str(),
|
||||
@@ -5012,6 +5027,46 @@ impl Operation for GetTableCatalogMigrationHandler {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MaterializeTableCatalogMigrationHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for MaterializeTableCatalogMigrationHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let warehouse = warehouse_from_params(¶ms)?;
|
||||
authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
|
||||
ensure_table_bucket_enabled(&warehouse).await?;
|
||||
let store = table_catalog_object_store()?;
|
||||
let started = Instant::now();
|
||||
let result = store
|
||||
.materialize_durable_strong_backing_migration(&warehouse)
|
||||
.await
|
||||
.map_err(catalog_store_error);
|
||||
record_table_catalog_admin_operation_result("migration-materialize", &warehouse, "", "", started, &result);
|
||||
let response = result?;
|
||||
build_json_response(StatusCode::OK, &response)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CancelTableCatalogMigrationHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for CancelTableCatalogMigrationHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let warehouse = warehouse_from_params(¶ms)?;
|
||||
authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
|
||||
ensure_table_bucket_enabled(&warehouse).await?;
|
||||
let store = table_catalog_object_store()?;
|
||||
let started = Instant::now();
|
||||
let result = store
|
||||
.cancel_durable_strong_backing_migration(&warehouse)
|
||||
.await
|
||||
.map_err(catalog_store_error);
|
||||
record_table_catalog_admin_operation_result("migration-cancel", &warehouse, "", "", started, &result);
|
||||
let response = result?;
|
||||
build_json_response(StatusCode::OK, &response)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RestListNamespacesHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -5871,6 +5926,8 @@ mod tests {
|
||||
assert_eq!(response.admin_discovery.extensions_catalog, "/rustfs/admin/v4/extensions/catalog");
|
||||
assert!(response.endpoints.contains(&"GET /v1/{prefix}/namespaces"));
|
||||
assert!(response.endpoints.contains(&"GET /{warehouse}/catalog/migration"));
|
||||
assert!(response.endpoints.contains(&"POST /{warehouse}/catalog/migration"));
|
||||
assert!(response.endpoints.contains(&"DELETE /{warehouse}/catalog/migration"));
|
||||
assert!(response.endpoints.contains(&"HEAD /v1/{prefix}/namespaces/{namespace}"));
|
||||
assert!(
|
||||
response
|
||||
@@ -6122,6 +6179,20 @@ mod tests {
|
||||
migration_block.contains("TableCatalogResource::warehouse(&warehouse)"),
|
||||
"catalog migration dry-run should authorize against the warehouse resource"
|
||||
);
|
||||
for handler in [
|
||||
"MaterializeTableCatalogMigrationHandler",
|
||||
"CancelTableCatalogMigrationHandler",
|
||||
] {
|
||||
let block = operation_block(src, handler);
|
||||
assert!(
|
||||
block.contains("authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;"),
|
||||
"{handler} should require the global catalog migration action"
|
||||
);
|
||||
assert!(
|
||||
!block.contains("authorize_table_catalog_resource_request("),
|
||||
"{handler} must not imply that a global backing cutover is warehouse-scoped"
|
||||
);
|
||||
}
|
||||
|
||||
for (handler, action) in [
|
||||
("RestLoadTableHandler", "AdminAction::GetTableMetadataAction"),
|
||||
@@ -6166,6 +6237,8 @@ mod tests {
|
||||
|
||||
for handler in [
|
||||
"GetTableCatalogMigrationHandler",
|
||||
"MaterializeTableCatalogMigrationHandler",
|
||||
"CancelTableCatalogMigrationHandler",
|
||||
"RestListNamespacesHandler",
|
||||
"RestCreateNamespaceHandler",
|
||||
"RestGetNamespaceHandler",
|
||||
@@ -6280,6 +6353,8 @@ mod tests {
|
||||
let _: &EnableTableBucketHandler = &ENABLE_TABLE_BUCKET_HANDLER;
|
||||
let _: &GetTableBucketHandler = &GET_TABLE_BUCKET_HANDLER;
|
||||
let _: &GetTableCatalogMigrationHandler = &GET_TABLE_CATALOG_MIGRATION_HANDLER;
|
||||
let _: &MaterializeTableCatalogMigrationHandler = &MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER;
|
||||
let _: &CancelTableCatalogMigrationHandler = &CANCEL_TABLE_CATALOG_MIGRATION_HANDLER;
|
||||
let _: &RestListNamespacesHandler = &LIST_NAMESPACES_HANDLER;
|
||||
let _: &RestCreateNamespaceHandler = &CREATE_NAMESPACE_HANDLER;
|
||||
let _: &RestGetNamespaceHandler = &GET_NAMESPACE_HANDLER;
|
||||
@@ -6320,6 +6395,8 @@ mod tests {
|
||||
assert_operation::<EnableTableBucketHandler>();
|
||||
assert_operation::<GetTableBucketHandler>();
|
||||
assert_operation::<GetTableCatalogMigrationHandler>();
|
||||
assert_operation::<MaterializeTableCatalogMigrationHandler>();
|
||||
assert_operation::<CancelTableCatalogMigrationHandler>();
|
||||
assert_operation::<RestListNamespacesHandler>();
|
||||
assert_operation::<RestCreateNamespaceHandler>();
|
||||
assert_operation::<RestGetNamespaceHandler>();
|
||||
|
||||
@@ -65,6 +65,7 @@ const LIST_TEMPORARY_ACCOUNTS: AdminActionRef = AdminActionRef::new("ListTempora
|
||||
const LIST_TIER: AdminActionRef = AdminActionRef::new("ListTierAction");
|
||||
const LIST_USER_POLICIES: AdminActionRef = AdminActionRef::new("ListUserPoliciesAdminAction");
|
||||
const LIST_USERS: AdminActionRef = AdminActionRef::new("ListUsersAdminAction");
|
||||
const MIGRATE_TABLE_CATALOG: AdminActionRef = AdminActionRef::new("MigrateTableCatalogAction");
|
||||
const PROFILING: AdminActionRef = AdminActionRef::new("ProfilingAdminAction");
|
||||
const REBALANCE: AdminActionRef = AdminActionRef::new("RebalanceAdminAction");
|
||||
const REGISTER_TABLE: AdminActionRef = AdminActionRef::new("RegisterTableAction");
|
||||
@@ -789,6 +790,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
GET_TABLE_CATALOG,
|
||||
RouteRiskLevel::Sensitive,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/iceberg/v1/{warehouse}/catalog/migration",
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Delete,
|
||||
"/iceberg/v1/{warehouse}/catalog/migration",
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/iceberg/v1/{warehouse}/namespaces",
|
||||
@@ -1054,6 +1067,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
GET_TABLE_CATALOG,
|
||||
RouteRiskLevel::Sensitive,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/_iceberg/v1/{warehouse}/catalog/migration",
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Delete,
|
||||
"/_iceberg/v1/{warehouse}/catalog/migration",
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/_iceberg/v1/{warehouse}/namespaces",
|
||||
@@ -1531,7 +1556,7 @@ mod tests {
|
||||
let table_specs = ADMIN_ROUTE_POLICY_SPECS
|
||||
.iter()
|
||||
.filter(|spec| spec.path().starts_with("/iceberg/v1") || spec.path().starts_with("/_iceberg/v1"));
|
||||
assert_eq!(table_specs.count(), 90);
|
||||
assert_eq!(table_specs.count(), 94);
|
||||
assert_action(HttpMethod::Put, "/iceberg/v1/buckets/{warehouse}", SET_TABLE_BUCKET);
|
||||
assert_action(HttpMethod::Get, "/_iceberg/v1/buckets/{warehouse}", GET_TABLE_BUCKET);
|
||||
assert_action(HttpMethod::Get, "/iceberg/v1/{warehouse}/namespaces", GET_TABLE_NAMESPACE);
|
||||
@@ -1698,6 +1723,10 @@ mod tests {
|
||||
);
|
||||
assert_action(HttpMethod::Get, "/iceberg/v1/{warehouse}/catalog/migration", GET_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Get, "/_iceberg/v1/{warehouse}/catalog/migration", GET_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Delete, "/iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Delete, "/_iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(
|
||||
HttpMethod::Post,
|
||||
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import",
|
||||
|
||||
@@ -342,6 +342,8 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
||||
table_route_sample(Method::PUT, "/buckets/{warehouse}", "/buckets/analytics"),
|
||||
table_route_sample(Method::GET, "/buckets/{warehouse}", "/buckets/analytics"),
|
||||
table_route_sample(Method::GET, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
|
||||
table_route_sample(Method::POST, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
|
||||
table_route_sample(Method::DELETE, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
|
||||
table_route_sample(Method::GET, "/{warehouse}/namespaces", "/analytics/namespaces"),
|
||||
table_route_sample(Method::POST, "/{warehouse}/namespaces", "/analytics/namespaces"),
|
||||
table_route_sample(Method::GET, "/{warehouse}/namespaces/{namespace}", "/analytics/namespaces/sales"),
|
||||
@@ -531,6 +533,8 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
||||
compat_table_route_sample(Method::PUT, "/buckets/{warehouse}", "/buckets/analytics"),
|
||||
compat_table_route_sample(Method::GET, "/buckets/{warehouse}", "/buckets/analytics"),
|
||||
compat_table_route_sample(Method::GET, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
|
||||
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
|
||||
compat_table_route_sample(Method::DELETE, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
|
||||
compat_table_route_sample(Method::GET, "/{warehouse}/namespaces", "/analytics/namespaces"),
|
||||
compat_table_route_sample(Method::POST, "/{warehouse}/namespaces", "/analytics/namespaces"),
|
||||
compat_table_route_sample(Method::GET, "/{warehouse}/namespaces/{namespace}", "/analytics/namespaces/sales"),
|
||||
|
||||
+1562
-23
File diff suppressed because it is too large
Load Diff
@@ -335,7 +335,14 @@ def disaster_recovery_rehearsal_plan(
|
||||
"GET",
|
||||
warehouse_path(warehouse, "/catalog/migration", rest_path),
|
||||
"200",
|
||||
"migration blockers must be empty before cutover",
|
||||
"migration blockers must be empty and status must be READY_TO_SNAPSHOT before state transfer",
|
||||
),
|
||||
probe_step(
|
||||
"durable-backing-migration-materialize",
|
||||
"POST",
|
||||
warehouse_path(warehouse, "/catalog/migration", rest_path),
|
||||
"200",
|
||||
"source writes are fenced and the durable strong snapshot is materialized before cutover",
|
||||
),
|
||||
],
|
||||
),
|
||||
@@ -504,6 +511,13 @@ def scale_fault_rehearsal_plan(
|
||||
"200",
|
||||
"migration dry-run reports inventory, replay blockers, idempotency blockers, and recommended actions",
|
||||
),
|
||||
probe_step(
|
||||
"materialize-durable-backing-before-cutover",
|
||||
"POST",
|
||||
warehouse_path(warehouse, "/catalog/migration", rest_path),
|
||||
"200",
|
||||
"materialization drains in-flight writes and reports SNAPSHOT_MATERIALIZED before backing restart",
|
||||
),
|
||||
],
|
||||
),
|
||||
rehearsal_phase(
|
||||
|
||||
@@ -239,8 +239,8 @@ UNSUPPORTED_INVENTORY: list[dict[str, str]] = [
|
||||
PRODUCTION_READINESS_INVENTORY: list[dict[str, str]] = [
|
||||
{
|
||||
"capability": "strong-catalog-backing",
|
||||
"status": "migration-contract-supported",
|
||||
"validation": "catalog export and diagnostics expose an object-backed manifest, recoverable commit-log WAL state, strong-kv-wal migration target, required migration steps, and recovery blockers before cutover",
|
||||
"status": "state-transfer-supported",
|
||||
"validation": "preflight exposes recovery blockers and target agreement; migration execution fences object-backed writes, drains in-flight catalog mutations, and idempotently materializes table bucket, namespace, table, view, commit-log, and idempotency state before cutover",
|
||||
},
|
||||
{
|
||||
"capability": "single-active-writer-ha",
|
||||
|
||||
@@ -132,7 +132,11 @@ class FailureCoverageTest(unittest.TestCase):
|
||||
|
||||
migration_steps = {step["name"]: step for step in phases["migration-preflight"]["steps"]}
|
||||
self.assertEqual(migration_steps["durable-backing-migration-dry-run"]["path"], "/iceberg/v1/lake/catalog/migration")
|
||||
self.assertEqual(migration_steps["durable-backing-migration-dry-run"]["expected_behavior"], "migration blockers must be empty before cutover")
|
||||
self.assertEqual(migration_steps["durable-backing-migration-materialize"]["method"], "POST")
|
||||
self.assertIn(
|
||||
"READY_TO_SNAPSHOT",
|
||||
migration_steps["durable-backing-migration-dry-run"]["expected_behavior"],
|
||||
)
|
||||
|
||||
validation_steps = {step["name"]: step for step in phases["post-recovery-validation"]["steps"]}
|
||||
self.assertEqual(validation_steps["load-table-after-recovery"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders")
|
||||
@@ -202,6 +206,7 @@ class FailureCoverageTest(unittest.TestCase):
|
||||
|
||||
cutover_steps = {step["name"]: step for step in phases["durable-backing-cutover-under-load"]["steps"]}
|
||||
self.assertEqual(cutover_steps["migration-dry-run-before-cutover"]["path"], "/iceberg/v1/lake/catalog/migration")
|
||||
self.assertEqual(cutover_steps["materialize-durable-backing-before-cutover"]["method"], "POST")
|
||||
self.assertIn("catalog-backup", cutover_steps["capture-cutover-backup"]["expected_behavior"])
|
||||
|
||||
recovery_steps = {step["name"]: step for step in phases["recovery-rollback-import-under-load"]["steps"]}
|
||||
|
||||
@@ -864,7 +864,7 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
|
||||
self.assertIn("single-active-writer-ha", capabilities)
|
||||
self.assertIn("scale-validation-matrix", capabilities)
|
||||
strong_backing = next(entry for entry in inventory if entry["capability"] == "strong-catalog-backing")
|
||||
self.assertEqual(strong_backing["status"], "migration-contract-supported")
|
||||
self.assertEqual(strong_backing["status"], "state-transfer-supported")
|
||||
for entry in inventory:
|
||||
self.assertIn("status", entry)
|
||||
self.assertIn("validation", entry)
|
||||
|
||||
Reference in New Issue
Block a user