mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-03 02:38:12 +00:00
feat(table-catalog): expose backing migration contract (#3574)
* feat(table-catalog): expose backing migration contract * fix(table-catalog): require register auth for external sync --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
@@ -5193,9 +5193,17 @@ impl Operation for SyncExternalCatalogBridgeHandler {
|
|||||||
let table = table_name_from_params(¶ms)?;
|
let table = table_name_from_params(¶ms)?;
|
||||||
let resource = TableCatalogResource::table(&warehouse, &namespace, &table);
|
let resource = TableCatalogResource::table(&warehouse, &namespace, &table);
|
||||||
authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableMetadataLocationAction).await?;
|
authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableMetadataLocationAction).await?;
|
||||||
let request = read_json_body::<ExternalCatalogBridgeSyncRequest>(req.input).await?;
|
|
||||||
let metadata_backend = table_catalog_backend()?;
|
let metadata_backend = table_catalog_backend()?;
|
||||||
let store = crate::table_catalog::ObjectTableCatalogStore::new(metadata_backend.clone());
|
let store = crate::table_catalog::ObjectTableCatalogStore::new(metadata_backend.clone());
|
||||||
|
if store
|
||||||
|
.load_table(&warehouse, &namespace.public_name(), &table)
|
||||||
|
.await
|
||||||
|
.map_err(catalog_store_error)?
|
||||||
|
.is_none()
|
||||||
|
{
|
||||||
|
authorize_table_catalog_resource_request(&req, &resource, AdminAction::RegisterTableAction).await?;
|
||||||
|
}
|
||||||
|
let request = read_json_body::<ExternalCatalogBridgeSyncRequest>(req.input).await?;
|
||||||
let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?;
|
let table_bucket_enabled = table_bucket_enabled_from_metadata(&warehouse).await?;
|
||||||
let response = sync_external_catalog_bridge_response(
|
let response = sync_external_catalog_bridge_response(
|
||||||
&store,
|
&store,
|
||||||
@@ -5474,6 +5482,16 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let sync_bridge_block = operation_block(src, "SyncExternalCatalogBridgeHandler");
|
||||||
|
assert!(
|
||||||
|
sync_bridge_block.contains("AdminAction::RegisterTableAction"),
|
||||||
|
"external catalog sync should require register authorization before creating a missing table"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
sync_bridge_block.contains(".load_table(&warehouse, &namespace.public_name(), &table)"),
|
||||||
|
"external catalog sync should branch authorization on current table existence"
|
||||||
|
);
|
||||||
|
|
||||||
for (handler, action) in [
|
for (handler, action) in [
|
||||||
("RestLoadTableHandler", "AdminAction::GetTableMetadataAction"),
|
("RestLoadTableHandler", "AdminAction::GetTableMetadataAction"),
|
||||||
("RestTableExistsHandler", "AdminAction::GetTableAction"),
|
("RestTableExistsHandler", "AdminAction::GetTableAction"),
|
||||||
|
|||||||
@@ -72,6 +72,7 @@ pub(crate) const TABLE_METADATA_POINTER_VERSION: u16 = 1;
|
|||||||
pub(crate) const TABLE_CATALOG_ENTRY_VERSION: u16 = 1;
|
pub(crate) const TABLE_CATALOG_ENTRY_VERSION: u16 = 1;
|
||||||
pub(crate) const TABLE_MAINTENANCE_CONFIG_VERSION: u16 = 1;
|
pub(crate) const TABLE_MAINTENANCE_CONFIG_VERSION: u16 = 1;
|
||||||
pub(crate) const TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION: u16 = 1;
|
pub(crate) const TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION: u16 = 1;
|
||||||
|
pub(crate) const TABLE_CATALOG_BACKING_MANIFEST_VERSION: u16 = 1;
|
||||||
pub(crate) const TABLE_METADATA_FILE_NAME_MAX_LEN: usize = 128;
|
pub(crate) const TABLE_METADATA_FILE_NAME_MAX_LEN: usize = 128;
|
||||||
pub const TABLE_RESERVED_PREFIX: &str = BUCKET_TABLE_RESERVED_PREFIX;
|
pub const TABLE_RESERVED_PREFIX: &str = BUCKET_TABLE_RESERVED_PREFIX;
|
||||||
const WAREHOUSE_ROOT: &str = "warehouses";
|
const WAREHOUSE_ROOT: &str = "warehouses";
|
||||||
@@ -843,6 +844,168 @@ pub(crate) struct TableCatalogExport {
|
|||||||
pub table_bucket: TableBucketEntry,
|
pub table_bucket: TableBucketEntry,
|
||||||
pub namespace: NamespaceEntry,
|
pub namespace: NamespaceEntry,
|
||||||
pub table: TableEntry,
|
pub table: TableEntry,
|
||||||
|
pub backing_manifest: TableCatalogBackingManifest,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct TableCatalogBackingManifest {
|
||||||
|
pub version: u16,
|
||||||
|
pub current: TableCatalogBackingProfile,
|
||||||
|
pub migration: TableCatalogBackingMigrationPlan,
|
||||||
|
pub ha: TableCatalogHaPolicy,
|
||||||
|
pub scale_validation: TableCatalogScaleValidation,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct TableCatalogBackingProfile {
|
||||||
|
pub kind: TableCatalogBackingKind,
|
||||||
|
pub authority: TableCatalogAuthority,
|
||||||
|
pub consistency: TableCatalogConsistencyMode,
|
||||||
|
pub durability: TableCatalogDurabilityMode,
|
||||||
|
pub current_pointer_path: String,
|
||||||
|
pub wal: TableCatalogWalState,
|
||||||
|
pub snapshot: TableCatalogSnapshotState,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||||
|
pub(crate) enum TableCatalogBackingKind {
|
||||||
|
ObjectBacked,
|
||||||
|
StrongKvWal,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||||
|
pub(crate) enum TableCatalogAuthority {
|
||||||
|
RustfsSysObject,
|
||||||
|
LinearizableMetadataKv,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||||
|
pub(crate) enum TableCatalogConsistencyMode {
|
||||||
|
ConditionalObjectCas,
|
||||||
|
LinearizableCas,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||||
|
pub(crate) enum TableCatalogDurabilityMode {
|
||||||
|
StagedCommitLogBeforePointerUpdate,
|
||||||
|
WalBeforeStateMachineApply,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct TableCatalogWalState {
|
||||||
|
pub status: TableCatalogWalStatus,
|
||||||
|
pub commit_log_prefix: String,
|
||||||
|
pub idempotency_index_prefix: String,
|
||||||
|
pub committed_generation: u64,
|
||||||
|
pub staged_before_table_update_count: usize,
|
||||||
|
pub finalization_required_count: usize,
|
||||||
|
pub idempotency_repair_required_count: usize,
|
||||||
|
pub manual_review_count: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||||
|
pub(crate) enum TableCatalogWalStatus {
|
||||||
|
Recoverable,
|
||||||
|
RecoveryRequired,
|
||||||
|
ManualReviewRequired,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct TableCatalogSnapshotState {
|
||||||
|
pub export_api: String,
|
||||||
|
pub includes_table_bucket: bool,
|
||||||
|
pub includes_namespace: bool,
|
||||||
|
pub includes_table_pointer: bool,
|
||||||
|
pub includes_backing_manifest: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct TableCatalogBackingMigrationPlan {
|
||||||
|
pub source_kind: TableCatalogBackingKind,
|
||||||
|
pub target_kind: TableCatalogBackingKind,
|
||||||
|
pub status: TableCatalogBackingMigrationStatus,
|
||||||
|
pub required_steps: Vec<TableCatalogBackingMigrationStep>,
|
||||||
|
pub blockers: Vec<TableCatalogBackingMigrationBlocker>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||||
|
pub(crate) enum TableCatalogBackingMigrationStatus {
|
||||||
|
ReadyToSnapshot,
|
||||||
|
RecoveryRequired,
|
||||||
|
ManualReviewRequired,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||||
|
pub(crate) enum TableCatalogBackingMigrationStep {
|
||||||
|
SnapshotCatalogExport,
|
||||||
|
ReplayCommitLog,
|
||||||
|
VerifyCurrentPointer,
|
||||||
|
EnableSingleWriterFencing,
|
||||||
|
CutOverLinearizableReads,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||||
|
pub(crate) enum TableCatalogBackingMigrationBlocker {
|
||||||
|
CommitRecoveryRequired,
|
||||||
|
CommitManualReviewRequired,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct TableCatalogHaPolicy {
|
||||||
|
pub writer_region_model: TableCatalogHaWriterModel,
|
||||||
|
pub read_replica_strategy: TableCatalogReadReplicaStrategy,
|
||||||
|
pub commit_read_requirement: TableCatalogCommitReadRequirement,
|
||||||
|
pub active_active_supported: bool,
|
||||||
|
pub failover_requires_operator_promotion: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||||
|
pub(crate) enum TableCatalogHaWriterModel {
|
||||||
|
SingleActiveWriterRegion,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||||
|
pub(crate) enum TableCatalogReadReplicaStrategy {
|
||||||
|
ReadOnlyReplicasForListAndLoad,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||||
|
pub(crate) enum TableCatalogCommitReadRequirement {
|
||||||
|
LinearizableLeaderRead,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
pub(crate) struct TableCatalogScaleValidation {
|
||||||
|
pub status: TableCatalogScaleValidationStatus,
|
||||||
|
pub benchmark_required: bool,
|
||||||
|
pub required_scenarios: Vec<TableCatalogScaleValidationScenario>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||||
|
pub(crate) enum TableCatalogScaleValidationStatus {
|
||||||
|
MatrixPublished,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
|
||||||
|
pub(crate) enum TableCatalogScaleValidationScenario {
|
||||||
|
ConcurrentCommitCas,
|
||||||
|
CommitLogRecoveryReplay,
|
||||||
|
MigrationSnapshotReplay,
|
||||||
|
ReadReplicaStaleReadGuard,
|
||||||
|
ClientConformanceMatrix,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
@@ -899,6 +1062,7 @@ pub(crate) struct TableCatalogDiagnosticsReport {
|
|||||||
pub recovery_status: TableCatalogRecoveryStatus,
|
pub recovery_status: TableCatalogRecoveryStatus,
|
||||||
pub recommended_actions: Vec<TableCatalogRecoveryAction>,
|
pub recommended_actions: Vec<TableCatalogRecoveryAction>,
|
||||||
pub commit_recovery: TableCommitRecoveryReport,
|
pub commit_recovery: TableCommitRecoveryReport,
|
||||||
|
pub backing_manifest: TableCatalogBackingManifest,
|
||||||
pub orphan_metadata_candidate_locations: Vec<String>,
|
pub orphan_metadata_candidate_locations: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1366,6 +1530,15 @@ impl TableCatalogObjectPaths {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn commit_idempotency_entries_prefix(&self, table_bucket: &str, table_id: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"{}{}/{}/",
|
||||||
|
self.table_bucket_root_prefix(table_bucket),
|
||||||
|
COMMIT_IDEMPOTENCY_ROOT,
|
||||||
|
table_catalog_path_hash(table_id)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn table_bucket_root_prefix(&self, table_bucket: &str) -> String {
|
fn table_bucket_root_prefix(&self, table_bucket: &str) -> String {
|
||||||
format!("{}/{}/{}/", self.catalog_root, TABLE_BUCKET_ROOT, table_catalog_path_hash(table_bucket))
|
format!("{}/{}/{}/", self.catalog_root, TABLE_BUCKET_ROOT, table_catalog_path_hash(table_bucket))
|
||||||
}
|
}
|
||||||
@@ -1377,6 +1550,99 @@ pub(crate) struct ObjectTableCatalogStore<B> {
|
|||||||
paths: TableCatalogObjectPaths,
|
paths: TableCatalogObjectPaths,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn table_catalog_backing_manifest(
|
||||||
|
paths: &TableCatalogObjectPaths,
|
||||||
|
namespace: &Namespace,
|
||||||
|
table: &IdentifierSegment,
|
||||||
|
entry: &TableEntry,
|
||||||
|
commit_recovery: &TableCommitRecoveryReport,
|
||||||
|
) -> TableCatalogBackingManifest {
|
||||||
|
let recovery_required = commit_recovery.staged_before_table_update_count > 0
|
||||||
|
|| commit_recovery.finalization_required_count > 0
|
||||||
|
|| commit_recovery.idempotency_repair_required_count > 0;
|
||||||
|
let manual_review_required = commit_recovery.manual_review_count > 0;
|
||||||
|
let wal_status = if manual_review_required {
|
||||||
|
TableCatalogWalStatus::ManualReviewRequired
|
||||||
|
} else if recovery_required {
|
||||||
|
TableCatalogWalStatus::RecoveryRequired
|
||||||
|
} else {
|
||||||
|
TableCatalogWalStatus::Recoverable
|
||||||
|
};
|
||||||
|
let migration_status = if manual_review_required {
|
||||||
|
TableCatalogBackingMigrationStatus::ManualReviewRequired
|
||||||
|
} else if recovery_required {
|
||||||
|
TableCatalogBackingMigrationStatus::RecoveryRequired
|
||||||
|
} else {
|
||||||
|
TableCatalogBackingMigrationStatus::ReadyToSnapshot
|
||||||
|
};
|
||||||
|
let mut blockers = Vec::new();
|
||||||
|
if recovery_required {
|
||||||
|
blockers.push(TableCatalogBackingMigrationBlocker::CommitRecoveryRequired);
|
||||||
|
}
|
||||||
|
if manual_review_required {
|
||||||
|
blockers.push(TableCatalogBackingMigrationBlocker::CommitManualReviewRequired);
|
||||||
|
}
|
||||||
|
|
||||||
|
TableCatalogBackingManifest {
|
||||||
|
version: TABLE_CATALOG_BACKING_MANIFEST_VERSION,
|
||||||
|
current: TableCatalogBackingProfile {
|
||||||
|
kind: TableCatalogBackingKind::ObjectBacked,
|
||||||
|
authority: TableCatalogAuthority::RustfsSysObject,
|
||||||
|
consistency: TableCatalogConsistencyMode::ConditionalObjectCas,
|
||||||
|
durability: TableCatalogDurabilityMode::StagedCommitLogBeforePointerUpdate,
|
||||||
|
current_pointer_path: paths.table_entry_path(&entry.table_bucket, namespace, table),
|
||||||
|
wal: TableCatalogWalState {
|
||||||
|
status: wal_status,
|
||||||
|
commit_log_prefix: paths.commit_log_entries_prefix(&entry.table_bucket, &entry.table_id),
|
||||||
|
idempotency_index_prefix: paths.commit_idempotency_entries_prefix(&entry.table_bucket, &entry.table_id),
|
||||||
|
committed_generation: entry.generation,
|
||||||
|
staged_before_table_update_count: commit_recovery.staged_before_table_update_count,
|
||||||
|
finalization_required_count: commit_recovery.finalization_required_count,
|
||||||
|
idempotency_repair_required_count: commit_recovery.idempotency_repair_required_count,
|
||||||
|
manual_review_count: commit_recovery.manual_review_count,
|
||||||
|
},
|
||||||
|
snapshot: TableCatalogSnapshotState {
|
||||||
|
export_api: "GET /iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export".to_string(),
|
||||||
|
includes_table_bucket: true,
|
||||||
|
includes_namespace: true,
|
||||||
|
includes_table_pointer: true,
|
||||||
|
includes_backing_manifest: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
migration: TableCatalogBackingMigrationPlan {
|
||||||
|
source_kind: TableCatalogBackingKind::ObjectBacked,
|
||||||
|
target_kind: TableCatalogBackingKind::StrongKvWal,
|
||||||
|
status: migration_status,
|
||||||
|
required_steps: vec![
|
||||||
|
TableCatalogBackingMigrationStep::SnapshotCatalogExport,
|
||||||
|
TableCatalogBackingMigrationStep::ReplayCommitLog,
|
||||||
|
TableCatalogBackingMigrationStep::VerifyCurrentPointer,
|
||||||
|
TableCatalogBackingMigrationStep::EnableSingleWriterFencing,
|
||||||
|
TableCatalogBackingMigrationStep::CutOverLinearizableReads,
|
||||||
|
],
|
||||||
|
blockers,
|
||||||
|
},
|
||||||
|
ha: TableCatalogHaPolicy {
|
||||||
|
writer_region_model: TableCatalogHaWriterModel::SingleActiveWriterRegion,
|
||||||
|
read_replica_strategy: TableCatalogReadReplicaStrategy::ReadOnlyReplicasForListAndLoad,
|
||||||
|
commit_read_requirement: TableCatalogCommitReadRequirement::LinearizableLeaderRead,
|
||||||
|
active_active_supported: false,
|
||||||
|
failover_requires_operator_promotion: true,
|
||||||
|
},
|
||||||
|
scale_validation: TableCatalogScaleValidation {
|
||||||
|
status: TableCatalogScaleValidationStatus::MatrixPublished,
|
||||||
|
benchmark_required: true,
|
||||||
|
required_scenarios: vec![
|
||||||
|
TableCatalogScaleValidationScenario::ConcurrentCommitCas,
|
||||||
|
TableCatalogScaleValidationScenario::CommitLogRecoveryReplay,
|
||||||
|
TableCatalogScaleValidationScenario::MigrationSnapshotReplay,
|
||||||
|
TableCatalogScaleValidationScenario::ReadReplicaStaleReadGuard,
|
||||||
|
TableCatalogScaleValidationScenario::ClientConformanceMatrix,
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl<B> ObjectTableCatalogStore<B>
|
impl<B> ObjectTableCatalogStore<B>
|
||||||
where
|
where
|
||||||
B: TableCatalogObjectBackend,
|
B: TableCatalogObjectBackend,
|
||||||
@@ -2473,11 +2739,14 @@ where
|
|||||||
table.as_str()
|
table.as_str()
|
||||||
)));
|
)));
|
||||||
};
|
};
|
||||||
|
let commit_recovery = self.table_commit_recovery_report_for_entry(&table_entry, 0).await?;
|
||||||
|
let backing_manifest = table_catalog_backing_manifest(&self.paths, &namespace, &table, &table_entry, &commit_recovery);
|
||||||
|
|
||||||
Ok(TableCatalogExport {
|
Ok(TableCatalogExport {
|
||||||
table_bucket: table_bucket_entry,
|
table_bucket: table_bucket_entry,
|
||||||
namespace: namespace_entry,
|
namespace: namespace_entry,
|
||||||
table: table_entry,
|
table: table_entry,
|
||||||
|
backing_manifest,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2550,6 +2819,8 @@ where
|
|||||||
|
|
||||||
let commit_recovery = self.plan_table_commit_recovery(table_bucket, namespace, table).await?;
|
let commit_recovery = self.plan_table_commit_recovery(table_bucket, namespace, table).await?;
|
||||||
let (recovery_status, recommended_actions) = table_catalog_recovery_summary(¤t_metadata_status, &commit_recovery);
|
let (recovery_status, recommended_actions) = table_catalog_recovery_summary(¤t_metadata_status, &commit_recovery);
|
||||||
|
let backing_manifest =
|
||||||
|
table_catalog_backing_manifest(&self.paths, &parsed_namespace, &parsed_table, &catalog.table, &commit_recovery);
|
||||||
|
|
||||||
Ok(TableCatalogDiagnosticsReport {
|
Ok(TableCatalogDiagnosticsReport {
|
||||||
catalog,
|
catalog,
|
||||||
@@ -2557,6 +2828,7 @@ where
|
|||||||
recovery_status,
|
recovery_status,
|
||||||
recommended_actions,
|
recommended_actions,
|
||||||
commit_recovery,
|
commit_recovery,
|
||||||
|
backing_manifest,
|
||||||
orphan_metadata_candidate_locations,
|
orphan_metadata_candidate_locations,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -10148,6 +10420,104 @@ mod tests {
|
|||||||
assert_eq!(export.table.generation, 1);
|
assert_eq!(export.table.generation, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn export_catalog_entry_includes_backing_migration_manifest() {
|
||||||
|
let backend = TestCatalogObjectBackend::default();
|
||||||
|
let store = ObjectTableCatalogStore::new(backend);
|
||||||
|
let bucket = "analytics";
|
||||||
|
let namespace = Namespace::parse("sales").unwrap();
|
||||||
|
let table = IdentifierSegment::parse("orders").unwrap();
|
||||||
|
let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json");
|
||||||
|
|
||||||
|
seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await;
|
||||||
|
|
||||||
|
let export = store.export_table_catalog_entry(bucket, "sales", "orders").await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(export.backing_manifest.version, TABLE_CATALOG_BACKING_MANIFEST_VERSION);
|
||||||
|
assert_eq!(export.backing_manifest.current.kind, TableCatalogBackingKind::ObjectBacked);
|
||||||
|
assert_eq!(export.backing_manifest.current.authority, TableCatalogAuthority::RustfsSysObject);
|
||||||
|
assert_eq!(
|
||||||
|
export.backing_manifest.current.consistency,
|
||||||
|
TableCatalogConsistencyMode::ConditionalObjectCas
|
||||||
|
);
|
||||||
|
assert_eq!(export.backing_manifest.current.wal.finalization_required_count, 0);
|
||||||
|
assert_eq!(export.backing_manifest.migration.target_kind, TableCatalogBackingKind::StrongKvWal);
|
||||||
|
assert_eq!(
|
||||||
|
export.backing_manifest.migration.status,
|
||||||
|
TableCatalogBackingMigrationStatus::ReadyToSnapshot
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
export
|
||||||
|
.backing_manifest
|
||||||
|
.migration
|
||||||
|
.required_steps
|
||||||
|
.contains(&TableCatalogBackingMigrationStep::ReplayCommitLog)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
export.backing_manifest.ha.writer_region_model,
|
||||||
|
TableCatalogHaWriterModel::SingleActiveWriterRegion
|
||||||
|
);
|
||||||
|
assert!(!export.backing_manifest.ha.active_active_supported);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn diagnostics_backing_manifest_requires_recovery_before_migration() {
|
||||||
|
let backend = TestCatalogObjectBackend::default();
|
||||||
|
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||||
|
let bucket = "analytics";
|
||||||
|
let namespace = Namespace::parse("sales").unwrap();
|
||||||
|
let table = IdentifierSegment::parse("orders").unwrap();
|
||||||
|
let current_metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json");
|
||||||
|
let new_metadata = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json");
|
||||||
|
let commit_path = TableCatalogObjectPaths::default().commit_log_entry_path(bucket, "table-id", "commit-1");
|
||||||
|
|
||||||
|
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
|
||||||
|
store
|
||||||
|
.create_namespace(test_namespace_entry(bucket, &namespace))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
store
|
||||||
|
.create_table(test_table_entry(bucket, &namespace, &table, current_metadata.clone()))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||||
|
backend
|
||||||
|
.fail_put_attempt(rustfs_ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
store
|
||||||
|
.commit_table(TableCommitRequest {
|
||||||
|
table_bucket: bucket.to_string(),
|
||||||
|
namespace: namespace.public_name(),
|
||||||
|
table: table.as_str().to_string(),
|
||||||
|
commit_id: "commit-1".to_string(),
|
||||||
|
idempotency_key: None,
|
||||||
|
operation: "append".to_string(),
|
||||||
|
expected_version_token: "token-v1".to_string(),
|
||||||
|
expected_metadata_location: current_metadata,
|
||||||
|
new_metadata_location: new_metadata,
|
||||||
|
requirements: Vec::new(),
|
||||||
|
writer: None,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let diagnostics = store.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(diagnostics.backing_manifest.current.wal.finalization_required_count, 1);
|
||||||
|
assert_eq!(
|
||||||
|
diagnostics.backing_manifest.migration.status,
|
||||||
|
TableCatalogBackingMigrationStatus::RecoveryRequired
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
diagnostics
|
||||||
|
.backing_manifest
|
||||||
|
.migration
|
||||||
|
.blockers
|
||||||
|
.contains(&TableCatalogBackingMigrationBlocker::CommitRecoveryRequired)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn consistency_check_reports_missing_metadata_object() {
|
async fn consistency_check_reports_missing_metadata_object() {
|
||||||
let backend = TestCatalogObjectBackend::default();
|
let backend = TestCatalogObjectBackend::default();
|
||||||
|
|||||||
@@ -125,6 +125,7 @@ PyIceberg, PyArrow, or boto3:
|
|||||||
python3 scripts/table-catalog/pyiceberg_smoke.py --print-client-matrix
|
python3 scripts/table-catalog/pyiceberg_smoke.py --print-client-matrix
|
||||||
python3 scripts/table-catalog/pyiceberg_smoke.py --print-vendor-profiles
|
python3 scripts/table-catalog/pyiceberg_smoke.py --print-vendor-profiles
|
||||||
python3 scripts/table-catalog/pyiceberg_smoke.py --print-unsupported-inventory
|
python3 scripts/table-catalog/pyiceberg_smoke.py --print-unsupported-inventory
|
||||||
|
python3 scripts/table-catalog/pyiceberg_smoke.py --print-production-readiness
|
||||||
```
|
```
|
||||||
|
|
||||||
Use these outputs when updating release notes, PR descriptions, or follow-up
|
Use these outputs when updating release notes, PR descriptions, or follow-up
|
||||||
@@ -143,6 +144,9 @@ The smoke test also probes catalog-backed advanced Iceberg surfaces:
|
|||||||
execution checks
|
execution checks
|
||||||
- catalog diagnostics exposes the table recovery and consistency state used by
|
- catalog diagnostics exposes the table recovery and consistency state used by
|
||||||
operators
|
operators
|
||||||
|
- catalog export and diagnostics expose the current catalog backing manifest,
|
||||||
|
recoverable commit-log WAL state, strong backing migration target, single
|
||||||
|
active writer HA policy, and scale validation matrix
|
||||||
|
|
||||||
## Client Matrix
|
## Client Matrix
|
||||||
|
|
||||||
|
|||||||
@@ -178,6 +178,24 @@ UNSUPPORTED_INVENTORY: list[dict[str, str]] = [
|
|||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
PRODUCTION_READINESS_INVENTORY: list[dict[str, str]] = [
|
||||||
|
{
|
||||||
|
"capability": "strong-catalog-backing",
|
||||||
|
"status": "migration-contract-supported",
|
||||||
|
"validation": "catalog export and diagnostics expose an object-backed manifest, recoverable commit-log WAL state, strong-kv-wal migration target, required migration steps, and recovery blockers before cutover",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"capability": "single-active-writer-ha",
|
||||||
|
"status": "policy-published",
|
||||||
|
"validation": "diagnostics publish single-active-writer-region semantics, linearizable leader-read requirements for commits, read-only replica limits, and explicit no active-active support",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"capability": "scale-validation-matrix",
|
||||||
|
"status": "matrix-published",
|
||||||
|
"validation": "required validation scenarios are machine-readable: concurrent commit CAS, commit-log recovery replay, migration snapshot replay, stale replica read guards, and long-running client conformance",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class RuntimeDeps:
|
class RuntimeDeps:
|
||||||
@@ -225,6 +243,10 @@ def unsupported_inventory() -> list[dict[str, str]]:
|
|||||||
return [entry.copy() for entry in UNSUPPORTED_INVENTORY]
|
return [entry.copy() for entry in UNSUPPORTED_INVENTORY]
|
||||||
|
|
||||||
|
|
||||||
|
def production_readiness_inventory() -> list[dict[str, str]]:
|
||||||
|
return [entry.copy() for entry in PRODUCTION_READINESS_INVENTORY]
|
||||||
|
|
||||||
|
|
||||||
def normalized_rest_path(rest_path: str) -> str:
|
def normalized_rest_path(rest_path: str) -> str:
|
||||||
stripped = rest_path.strip()
|
stripped = rest_path.strip()
|
||||||
if not stripped:
|
if not stripped:
|
||||||
@@ -278,6 +300,11 @@ def parse_args() -> argparse.Namespace:
|
|||||||
parser.add_argument("--print-client-matrix", action="store_true", help="Print the current client conformance matrix as JSON and exit.")
|
parser.add_argument("--print-client-matrix", action="store_true", help="Print the current client conformance matrix as JSON and exit.")
|
||||||
parser.add_argument("--print-vendor-profiles", action="store_true", help="Print vendor connection profile references as JSON and exit.")
|
parser.add_argument("--print-vendor-profiles", action="store_true", help="Print vendor connection profile references as JSON and exit.")
|
||||||
parser.add_argument("--print-unsupported-inventory", action="store_true", help="Print unsupported capability inventory as JSON and exit.")
|
parser.add_argument("--print-unsupported-inventory", action="store_true", help="Print unsupported capability inventory as JSON and exit.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--print-production-readiness",
|
||||||
|
action="store_true",
|
||||||
|
help="Print catalog backing, HA, and scale-readiness inventory as JSON and exit.",
|
||||||
|
)
|
||||||
return apply_profile_defaults(parser.parse_args())
|
return apply_profile_defaults(parser.parse_args())
|
||||||
|
|
||||||
|
|
||||||
@@ -700,6 +727,9 @@ def printed_metadata(args: argparse.Namespace) -> bool:
|
|||||||
if args.print_unsupported_inventory:
|
if args.print_unsupported_inventory:
|
||||||
print_json_document({"unsupported_inventory": unsupported_inventory()})
|
print_json_document({"unsupported_inventory": unsupported_inventory()})
|
||||||
printed = True
|
printed = True
|
||||||
|
if args.print_production_readiness:
|
||||||
|
print_json_document({"production_readiness": production_readiness_inventory()})
|
||||||
|
printed = True
|
||||||
return printed
|
return printed
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -753,11 +753,25 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
|
|||||||
self.assertIn("status", entry)
|
self.assertIn("status", entry)
|
||||||
self.assertIn("roadmap_area", entry)
|
self.assertIn("roadmap_area", entry)
|
||||||
|
|
||||||
|
def test_production_readiness_inventory_tracks_catalog_backing(self) -> None:
|
||||||
|
inventory = pyiceberg_smoke.production_readiness_inventory()
|
||||||
|
capabilities = {entry["capability"] for entry in inventory}
|
||||||
|
|
||||||
|
self.assertIn("strong-catalog-backing", capabilities)
|
||||||
|
self.assertIn("single-active-writer-ha", capabilities)
|
||||||
|
self.assertIn("scale-validation-matrix", capabilities)
|
||||||
|
strong_backing = next(entry for entry in inventory if entry["capability"] == "strong-catalog-backing")
|
||||||
|
self.assertEqual(strong_backing["status"], "migration-contract-supported")
|
||||||
|
for entry in inventory:
|
||||||
|
self.assertIn("status", entry)
|
||||||
|
self.assertIn("validation", entry)
|
||||||
|
|
||||||
def test_published_table_catalog_docs_do_not_use_internal_roadmap_labels(self) -> None:
|
def test_published_table_catalog_docs_do_not_use_internal_roadmap_labels(self) -> None:
|
||||||
readme = (SCRIPT_DIR / "README.md").read_text(encoding="utf-8")
|
readme = (SCRIPT_DIR / "README.md").read_text(encoding="utf-8")
|
||||||
|
|
||||||
self.assertIsNone(INTERNAL_ROADMAP_LABEL_RE.search(readme))
|
self.assertIsNone(INTERNAL_ROADMAP_LABEL_RE.search(readme))
|
||||||
self.assertIsNone(INTERNAL_ROADMAP_LABEL_RE.search(str(pyiceberg_smoke.unsupported_inventory())))
|
self.assertIsNone(INTERNAL_ROADMAP_LABEL_RE.search(str(pyiceberg_smoke.unsupported_inventory())))
|
||||||
|
self.assertIsNone(INTERNAL_ROADMAP_LABEL_RE.search(str(pyiceberg_smoke.production_readiness_inventory())))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
Reference in New Issue
Block a user