gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(table-catalog): add product API surface (#3320)

Browse Source 
* feat(table-catalog): add product API surface

* fix(table-catalog): harden product API commits

* fix(table-catalog): close product API review gaps

* fix(table-catalog): validate rollback metadata before commit

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-06-10 15:58:22 +08:00
committed by GitHub
parent 66fd55a8e0
commit 64c0ede026
4 changed files with 1700 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/handlers/table_catalog.rs
+1071 -2
View File
File diff suppressed because it is too large Load Diff
rustfs/src/admin/route_policy.rs
+166 -1
View File
@@ -39,8 +39,11 @@ const GET_GROUP: AdminActionRef = AdminActionRef::new("GetGroupAdminAction");
const GET_POLICY: AdminActionRef = AdminActionRef::new("GetPolicyAdminAction");
const GET_REPLICATION_METRICS: AdminActionRef = AdminActionRef::new("GetReplicationMetricsAction");
const GET_TABLE: AdminActionRef = AdminActionRef::new("GetTableAction");
const GET_TABLE_BUCKET: AdminActionRef = AdminActionRef::new("GetTableBucketAction");
const GET_TABLE_CATALOG: AdminActionRef = AdminActionRef::new("GetTableCatalogAction");
const GET_TABLE_LIFECYCLE: AdminActionRef = AdminActionRef::new("GetTableLifecycleAction");
const GET_TABLE_METADATA: AdminActionRef = AdminActionRef::new("GetTableMetadataAction");
const GET_TABLE_METADATA_LOCATION: AdminActionRef = AdminActionRef::new("GetTableMetadataLocationAction");
const GET_TABLE_NAMESPACE: AdminActionRef = AdminActionRef::new("GetTableNamespaceAction");
const HEAL: AdminActionRef = AdminActionRef::new("HealAdminAction");
const IMPORT_BUCKET_METADATA: AdminActionRef = AdminActionRef::new("ImportBucketMetadataAction");
@@ -66,6 +69,9 @@ const SERVER_INFO: AdminActionRef = AdminActionRef::new("ServerInfoAdminAction")
const SET_BUCKET_QUOTA: AdminActionRef = AdminActionRef::new("SetBucketQuotaAdminAction");
const SET_BUCKET_TARGET: AdminActionRef = AdminActionRef::new("SetBucketTargetAction");
const SET_TABLE: AdminActionRef = AdminActionRef::new("SetTableAction");
const SET_TABLE_BUCKET: AdminActionRef = AdminActionRef::new("SetTableBucketAction");
const SET_TABLE_LIFECYCLE: AdminActionRef = AdminActionRef::new("SetTableLifecycleAction");
const SET_TABLE_METADATA_LOCATION: AdminActionRef = AdminActionRef::new("SetTableMetadataLocationAction");
const SET_TABLE_NAMESPACE: AdminActionRef = AdminActionRef::new("SetTableNamespaceAction");
const SET_TIER: AdminActionRef = AdminActionRef::new("SetTierAction");
const SITE_REPLICATION_ADD: AdminActionRef = AdminActionRef::new("SiteReplicationAddAction");
@@ -590,6 +596,13 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
),
admin(HttpMethod::Post, "/rustfs/admin/v3/oidc/validate", SERVER_INFO, RouteRiskLevel::High),
admin(HttpMethod::Get, "/iceberg/v1/config", GET_TABLE_CATALOG, RouteRiskLevel::Sensitive),
admin(HttpMethod::Put, "/iceberg/v1/buckets/{warehouse}", SET_TABLE_BUCKET, RouteRiskLevel::High),
admin(
HttpMethod::Get,
"/iceberg/v1/buckets/{warehouse}",
GET_TABLE_BUCKET,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Get,
"/iceberg/v1/{warehouse}/namespaces",
@@ -650,13 +663,79 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
DELETE_TABLE,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
GET_TABLE_METADATA_LOCATION,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Put,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
SET_TABLE_METADATA_LOCATION,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
RUN_TABLE_MAINTENANCE,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
GET_TABLE_LIFECYCLE,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Put,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
SET_TABLE_LIFECYCLE,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}",
GET_TABLE_LIFECYCLE,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Get,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export",
GET_TABLE_METADATA,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import",
REGISTER_TABLE,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/diagnostics",
GET_TABLE_METADATA,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/rollback",
COMMIT_TABLE,
RouteRiskLevel::High,
),
admin(HttpMethod::Get, "/_iceberg/v1/config", GET_TABLE_CATALOG, RouteRiskLevel::Sensitive),
admin(
HttpMethod::Put,
"/_iceberg/v1/buckets/{warehouse}",
SET_TABLE_BUCKET,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/_iceberg/v1/buckets/{warehouse}",
GET_TABLE_BUCKET,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Get,
"/_iceberg/v1/{warehouse}/namespaces",
@@ -717,12 +796,66 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
DELETE_TABLE,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
GET_TABLE_METADATA_LOCATION,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Put,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
SET_TABLE_METADATA_LOCATION,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
RUN_TABLE_MAINTENANCE,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
GET_TABLE_LIFECYCLE,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Put,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
SET_TABLE_LIFECYCLE,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}",
GET_TABLE_LIFECYCLE,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Get,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export",
GET_TABLE_METADATA,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import",
REGISTER_TABLE,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/diagnostics",
GET_TABLE_METADATA,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/rollback",
COMMIT_TABLE,
RouteRiskLevel::High,
),
];
pub const DEFERRED_ADMIN_ROUTE_POLICIES: &[DeferredAdminRoutePolicy] = &[
@@ -896,7 +1029,9 @@ mod tests {
let table_specs = ADMIN_ROUTE_POLICY_SPECS
.iter()
.filter(|spec| spec.path().starts_with("/iceberg/v1") || spec.path().starts_with("/_iceberg/v1"));
assert_eq!(table_specs.count(), 24);
assert_eq!(table_specs.count(), 46);
assert_action(HttpMethod::Put, "/iceberg/v1/buckets/{warehouse}", SET_TABLE_BUCKET);
assert_action(HttpMethod::Get, "/_iceberg/v1/buckets/{warehouse}", GET_TABLE_BUCKET);
assert_action(HttpMethod::Get, "/iceberg/v1/{warehouse}/namespaces", GET_TABLE_NAMESPACE);
assert_action(HttpMethod::Get, "/_iceberg/v1/{warehouse}/namespaces", GET_TABLE_NAMESPACE);
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/namespaces/{namespace}/tables", CREATE_TABLE);
@@ -921,6 +1056,16 @@ mod tests {
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}",
COMMIT_TABLE,
);
assert_action(
HttpMethod::Get,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
GET_TABLE_METADATA_LOCATION,
);
assert_action(
HttpMethod::Put,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
SET_TABLE_METADATA_LOCATION,
);
assert_action(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
@@ -931,6 +1076,26 @@ mod tests {
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
RUN_TABLE_MAINTENANCE,
);
assert_action(
HttpMethod::Put,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
SET_TABLE_LIFECYCLE,
);
assert_action(
HttpMethod::Get,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}",
GET_TABLE_LIFECYCLE,
);
assert_action(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import",
REGISTER_TABLE,
);
assert_action(
HttpMethod::Post,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/rollback",
COMMIT_TABLE,
);
}
#[test]
rustfs/src/admin/route_registration_test.rs
+188
View File
@@ -281,6 +281,8 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
admin_route_sample(Method::DELETE, "/v3/oidc/config/{provider_id}", "/v3/oidc/config/default"),
admin_route(Method::POST, "/v3/oidc/validate"),
table_route(Method::GET, "/config"),
table_route_sample(Method::PUT, "/buckets/{warehouse}", "/buckets/analytics"),
table_route_sample(Method::GET, "/buckets/{warehouse}", "/buckets/analytics"),
table_route_sample(Method::GET, "/{warehouse}/namespaces", "/analytics/namespaces"),
table_route_sample(Method::POST, "/{warehouse}/namespaces", "/analytics/namespaces"),
table_route_sample(Method::GET, "/{warehouse}/namespaces/{namespace}", "/analytics/namespaces/sales"),
@@ -320,7 +322,54 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
"/analytics/namespaces/sales/tables/orders/maintenance/metadata",
),
table_route_sample(
Method::GET,
"/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
"/analytics/namespaces/sales/tables/orders/metadata-location",
),
table_route_sample(
Method::PUT,
"/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
"/analytics/namespaces/sales/tables/orders/metadata-location",
),
table_route_sample(
Method::GET,
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
"/analytics/namespaces/sales/tables/orders/maintenance/config",
),
table_route_sample(
Method::PUT,
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
"/analytics/namespaces/sales/tables/orders/maintenance/config",
),
table_route_sample(
Method::GET,
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}",
"/analytics/namespaces/sales/tables/orders/maintenance/jobs/job-1",
),
table_route_sample(
Method::GET,
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export",
"/analytics/namespaces/sales/tables/orders/catalog/export",
),
table_route_sample(
Method::POST,
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import",
"/analytics/namespaces/sales/tables/orders/catalog/import",
),
table_route_sample(
Method::GET,
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/diagnostics",
"/analytics/namespaces/sales/tables/orders/catalog/diagnostics",
),
table_route_sample(
Method::POST,
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/rollback",
"/analytics/namespaces/sales/tables/orders/catalog/rollback",
),
compat_table_route(Method::GET, "/config"),
compat_table_route_sample(Method::PUT, "/buckets/{warehouse}", "/buckets/analytics"),
compat_table_route_sample(Method::GET, "/buckets/{warehouse}", "/buckets/analytics"),
compat_table_route_sample(Method::GET, "/{warehouse}/namespaces", "/analytics/namespaces"),
compat_table_route_sample(Method::POST, "/{warehouse}/namespaces", "/analytics/namespaces"),
compat_table_route_sample(Method::GET, "/{warehouse}/namespaces/{namespace}", "/analytics/namespaces/sales"),
@@ -360,6 +409,51 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
"/analytics/namespaces/sales/tables/orders/maintenance/metadata",
),
compat_table_route_sample(
Method::GET,
"/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
"/analytics/namespaces/sales/tables/orders/metadata-location",
),
compat_table_route_sample(
Method::PUT,
"/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
"/analytics/namespaces/sales/tables/orders/metadata-location",
),
compat_table_route_sample(
Method::GET,
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
"/analytics/namespaces/sales/tables/orders/maintenance/config",
),
compat_table_route_sample(
Method::PUT,
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
"/analytics/namespaces/sales/tables/orders/maintenance/config",
),
compat_table_route_sample(
Method::GET,
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}",
"/analytics/namespaces/sales/tables/orders/maintenance/jobs/job-1",
),
compat_table_route_sample(
Method::GET,
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export",
"/analytics/namespaces/sales/tables/orders/catalog/export",
),
compat_table_route_sample(
Method::POST,
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import",
"/analytics/namespaces/sales/tables/orders/catalog/import",
),
compat_table_route_sample(
Method::GET,
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/diagnostics",
"/analytics/namespaces/sales/tables/orders/catalog/diagnostics",
),
compat_table_route_sample(
Method::POST,
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/rollback",
"/analytics/namespaces/sales/tables/orders/catalog/rollback",
),
]
}
@@ -467,6 +561,8 @@ fn test_register_routes_cover_representative_admin_paths() {
assert_route(&router, Method::GET, &admin_path("/v3/scanner/status"));
assert_route(&router, Method::GET, &table_catalog_path("/config"));
assert_route(&router, Method::PUT, &table_catalog_path("/buckets/analytics"));
assert_route(&router, Method::GET, &table_catalog_path("/buckets/analytics"));
assert_route(&router, Method::GET, &table_catalog_path("/analytics/namespaces"));
assert_route(&router, Method::POST, &table_catalog_path("/analytics/namespaces"));
assert_route(&router, Method::GET, &table_catalog_path("/analytics/namespaces/sales"));
@@ -482,7 +578,54 @@ fn test_register_routes_cover_representative_admin_paths() {
Method::POST,
&table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/metadata"),
);
assert_route(
&router,
Method::GET,
&table_catalog_path("/analytics/namespaces/sales/tables/orders/metadata-location"),
);
assert_route(
&router,
Method::PUT,
&table_catalog_path("/analytics/namespaces/sales/tables/orders/metadata-location"),
);
assert_route(
&router,
Method::PUT,
&table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/config"),
);
assert_route(
&router,
Method::GET,
&table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/config"),
);
assert_route(
&router,
Method::GET,
&table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/jobs/job-1"),
);
assert_route(
&router,
Method::GET,
&table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/export"),
);
assert_route(
&router,
Method::POST,
&table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/import"),
);
assert_route(
&router,
Method::GET,
&table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/diagnostics"),
);
assert_route(
&router,
Method::POST,
&table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/rollback"),
);
assert_route(&router, Method::GET, &compat_table_catalog_path("/config"));
assert_route(&router, Method::PUT, &compat_table_catalog_path("/buckets/analytics"));
assert_route(&router, Method::GET, &compat_table_catalog_path("/buckets/analytics"));
assert_route(&router, Method::GET, &compat_table_catalog_path("/analytics/namespaces"));
assert_route(&router, Method::POST, &compat_table_catalog_path("/analytics/namespaces"));
assert_route(&router, Method::GET, &compat_table_catalog_path("/analytics/namespaces/sales"));
@@ -510,6 +653,51 @@ fn test_register_routes_cover_representative_admin_paths() {
Method::POST,
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/metadata"),
);
assert_route(
&router,
Method::GET,
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/metadata-location"),
);
assert_route(
&router,
Method::PUT,
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/metadata-location"),
);
assert_route(
&router,
Method::PUT,
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/config"),
);
assert_route(
&router,
Method::GET,
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/config"),
);
assert_route(
&router,
Method::GET,
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/jobs/job-1"),
);
assert_route(
&router,
Method::GET,
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/export"),
);
assert_route(
&router,
Method::POST,
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/import"),
);
assert_route(
&router,
Method::GET,
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/diagnostics"),
);
assert_route(
&router,
Method::POST,
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/rollback"),
);
assert_route(&router, Method::POST, &admin_path("/v3/service"));
assert_route(&router, Method::GET, &admin_path("/v3/info"));
rustfs/src/table_catalog.rs
+275 -3
View File
@@ -54,6 +54,7 @@ pub(crate) const TABLE_NAMESPACE_MARKER_VERSION: u16 = 1;
pub(crate) const TABLE_RESOURCE_MARKER_VERSION: u16 = 1;
pub(crate) const TABLE_METADATA_POINTER_VERSION: u16 = 1;
pub(crate) const TABLE_CATALOG_ENTRY_VERSION: u16 = 1;
pub(crate) const TABLE_MAINTENANCE_CONFIG_VERSION: u16 = 1;
pub(crate) const TABLE_METADATA_FILE_NAME_MAX_LEN: usize = 128;
pub const TABLE_RESERVED_PREFIX: &str = BUCKET_TABLE_RESERVED_PREFIX;
const WAREHOUSE_ROOT: &str = "warehouses";
@@ -71,6 +72,9 @@ const INTERNAL_CATALOG_ROOT: &str = BUCKET_TABLE_CATALOG_META_PREFIX;
const TABLE_BUCKET_ROOT: &str = BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX;
const COMMIT_LOG_ROOT: &str = "commits";
const COMMIT_IDEMPOTENCY_ROOT: &str = "commit-idempotency";
const MAINTENANCE_ROOT: &str = "maintenance";
const MAINTENANCE_CONFIG_FILE: &str = "config.json";
const MAINTENANCE_JOB_ROOT: &str = "jobs";
const TABLE_CATALOG_LIST_MAX_KEYS: i32 = 1000;
const TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS: i64 = 15 * 60;
@@ -245,12 +249,36 @@ pub(crate) struct TableCommitResult {
pub commit_log: CommitLogEntry,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct TableMaintenanceConfig {
pub version: u16,
#[serde(rename = "retain-recent-metadata-files")]
pub retain_recent_metadata_files: usize,
#[serde(rename = "delete-enabled")]
pub delete_enabled: bool,
#[serde(rename = "background-enabled")]
pub background_enabled: bool,
}
impl Default for TableMaintenanceConfig {
fn default() -> Self {
Self {
version: TABLE_MAINTENANCE_CONFIG_VERSION,
retain_recent_metadata_files: 0,
delete_enabled: false,
background_enabled: false,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct TableMetadataMaintenanceJob {
pub job_id: String,
pub table_bucket: String,
pub namespace: String,
pub table: String,
pub table_id: String,
pub current_metadata_location: String,
pub current_generation: u64,
pub retain_recent_metadata_files: usize,
@@ -258,7 +286,7 @@ pub(crate) struct TableMetadataMaintenanceJob {
pub cleanup_watermark_unix_seconds: i64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct TableMetadataMaintenanceReport {
pub job: TableMetadataMaintenanceJob,
pub current_metadata_location: String,
@@ -438,6 +466,38 @@ impl TableCatalogObjectPaths {
)
}
pub fn table_maintenance_config_path(
&self,
table_bucket: &str,
namespace: &Namespace,
table: &IdentifierSegment,
table_id: &str,
) -> String {
format!(
"{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_CONFIG_FILE}",
self.table_entries_prefix(table_bucket, namespace),
table.as_str(),
table_catalog_path_hash(table_id)
)
}
pub fn table_maintenance_job_path(
&self,
table_bucket: &str,
namespace: &Namespace,
table: &IdentifierSegment,
table_id: &str,
job_id: &str,
) -> String {
format!(
"{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_JOB_ROOT}/{}.json",
self.table_entries_prefix(table_bucket, namespace),
table.as_str(),
table_catalog_path_hash(table_id),
table_catalog_path_hash(job_id)
)
}
pub fn commit_log_entry_path(&self, table_bucket: &str, table_id: &str, commit_id: &str) -> String {
format!(
"{}{}/{}/{}.json",
@@ -576,6 +636,108 @@ where
Ok(())
}
pub(crate) async fn get_table_maintenance_config(
&self,
table_bucket: &str,
namespace: &str,
table: &str,
) -> TableCatalogStoreResult<TableMaintenanceConfig> {
let namespace = parse_namespace_for_store(namespace)?;
let table = parse_table_for_store(table)?;
let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table);
let Some((entry, _)) = self.read_entry::<TableEntry>(self.catalog_bucket(), &table_path).await? else {
return Err(TableCatalogStoreError::NotFound(format!(
"table {}/{}/{}",
table_bucket,
namespace.public_name(),
table.as_str()
)));
};
let config_path = self
.paths
.table_maintenance_config_path(table_bucket, &namespace, &table, &entry.table_id);
self.read_entry::<TableMaintenanceConfig>(self.catalog_bucket(), &config_path)
.await
.map(|entry| entry.map(|(config, _)| config).unwrap_or_default())
}
pub(crate) async fn put_table_maintenance_config(
&self,
table_bucket: &str,
namespace: &str,
table: &str,
config: TableMaintenanceConfig,
) -> TableCatalogStoreResult<TableMaintenanceConfig> {
validate_table_maintenance_config_version(config.version)?;
if config.background_enabled {
return Err(TableCatalogStoreError::Invalid(
"background table maintenance is not supported".to_string(),
));
}
let namespace = parse_namespace_for_store(namespace)?;
let table = parse_table_for_store(table)?;
let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table);
let Some((entry, _)) = self.read_entry::<TableEntry>(self.catalog_bucket(), &table_path).await? else {
return Err(TableCatalogStoreError::NotFound(format!(
"table {}/{}/{}",
table_bucket,
namespace.public_name(),
table.as_str()
)));
};
let config_path = self
.paths
.table_maintenance_config_path(table_bucket, &namespace, &table, &entry.table_id);
self.write_entry(self.catalog_bucket(), &config_path, &config, TableCatalogPutPrecondition::Any)
.await?;
Ok(config)
}
pub(crate) async fn put_table_metadata_maintenance_report(
&self,
report: &TableMetadataMaintenanceReport,
) -> TableCatalogStoreResult<()> {
let namespace = parse_namespace_for_store(&report.job.namespace)?;
let table = parse_table_for_store(&report.job.table)?;
let job_path = self.paths.table_maintenance_job_path(
&report.job.table_bucket,
&namespace,
&table,
&report.job.table_id,
&report.job.job_id,
);
self.write_entry(self.catalog_bucket(), &job_path, report, TableCatalogPutPrecondition::Any)
.await
}
pub(crate) async fn get_table_metadata_maintenance_report(
&self,
table_bucket: &str,
namespace: &str,
table: &str,
job_id: &str,
) -> TableCatalogStoreResult<Option<TableMetadataMaintenanceReport>> {
let namespace = parse_namespace_for_store(namespace)?;
let table = parse_table_for_store(table)?;
let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table);
let Some((entry, _)) = self.read_entry::<TableEntry>(self.catalog_bucket(), &table_path).await? else {
return Err(TableCatalogStoreError::NotFound(format!(
"table {}/{}/{}",
table_bucket,
namespace.public_name(),
table.as_str()
)));
};
let job_path = self
.paths
.table_maintenance_job_path(table_bucket, &namespace, &table, &entry.table_id, job_id);
self.read_entry::<TableMetadataMaintenanceReport>(self.catalog_bucket(), &job_path)
.await
.map(|entry| entry.map(|(report, _)| report))
}
pub(crate) async fn export_table_catalog_entry(
&self,
table_bucket: &str,
@@ -759,6 +921,7 @@ where
table_bucket: table_bucket.to_string(),
namespace: namespace.public_name(),
table: table.as_str().to_string(),
table_id: entry.table_id,
current_metadata_location: current_metadata_location.clone(),
current_generation: entry.generation,
retain_recent_metadata_files,
@@ -1365,6 +1528,15 @@ fn validate_catalog_entry_version(kind: &str, version: u16) -> TableCatalogStore
Ok(())
}
fn validate_table_maintenance_config_version(version: u16) -> TableCatalogStoreResult<()> {
if version != TABLE_MAINTENANCE_CONFIG_VERSION {
return Err(TableCatalogStoreError::Invalid(
"unsupported table maintenance config entry version".to_string(),
));
}
Ok(())
}
fn commit_log_matches_request(commit_log: &CommitLogEntry, request: &TableCommitRequest, table_id: &str) -> bool {
commit_log.version == TABLE_CATALOG_ENTRY_VERSION
&& commit_log.commit_id == request.commit_id
@@ -2108,8 +2280,11 @@ mod tests {
let commit_path = paths.commit_log_entry_path("table/../bucket", "table/../id", "commit/%2f\nid");
let idempotency_path = paths.commit_idempotency_entry_path("table/../bucket", "table/../id", "client/%2f\nrequest");
let maintenance_config_path = paths.table_maintenance_config_path("table/../bucket", &namespace, &table, "table/../id");
let maintenance_job_path =
paths.table_maintenance_job_path("table/../bucket", &namespace, &table, "table/../id", "job/%2f\nid");
for path in [commit_path, idempotency_path] {
for path in [commit_path, idempotency_path, maintenance_config_path, maintenance_job_path] {
assert!(path.starts_with("s3tables/catalog/table-buckets/"));
assert!(path.ends_with(".json"));
assert!(!path.contains(".."));
@@ -2400,6 +2575,7 @@ mod tests {
assert_eq!(report.job.table_bucket, bucket);
assert_eq!(report.job.namespace, "sales");
assert_eq!(report.job.table, "orders");
assert_eq!(report.job.table_id, "table-id");
assert_eq!(report.job.current_generation, 1);
assert_eq!(report.job.safety_window_seconds, TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS);
assert!(!report.job.job_id.is_empty());
@@ -2408,6 +2584,102 @@ mod tests {
assert_eq!(report.deletable_metadata_locations, vec![old]);
}
#[tokio::test]
async fn maintenance_state_is_scoped_to_current_table_identity() {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let bucket = "analytics";
let namespace = Namespace::parse("sales").unwrap();
let table = IdentifierSegment::parse("orders").unwrap();
let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json");
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
store
.create_namespace(test_namespace_entry(bucket, &namespace))
.await
.unwrap();
let mut first_table = test_table_entry(bucket, &namespace, &table, current.clone());
first_table.table_id = "table-id-1".to_string();
store.create_table(first_table).await.unwrap();
store
.put_table_maintenance_config(
bucket,
"sales",
"orders",
TableMaintenanceConfig {
version: TABLE_MAINTENANCE_CONFIG_VERSION,
retain_recent_metadata_files: 7,
delete_enabled: true,
background_enabled: false,
},
)
.await
.unwrap();
backend
.seed_object(bucket, &current, br#"{"metadata-log":[]}"#.to_vec())
.await;
let report = store
.plan_table_metadata_maintenance(bucket, "sales", "orders", 0)
.await
.unwrap();
store.put_table_metadata_maintenance_report(&report).await.unwrap();
assert!(
store
.get_table_metadata_maintenance_report(bucket, "sales", "orders", &report.job.job_id)
.await
.unwrap()
.is_some()
);
store.drop_table(bucket, "sales", "orders").await.unwrap();
let mut second_table = test_table_entry(bucket, &namespace, &table, current);
second_table.table_id = "table-id-2".to_string();
store.create_table(second_table).await.unwrap();
assert_eq!(
store.get_table_maintenance_config(bucket, "sales", "orders").await.unwrap(),
TableMaintenanceConfig::default()
);
assert!(
store
.get_table_metadata_maintenance_report(bucket, "sales", "orders", &report.job.job_id)
.await
.unwrap()
.is_none()
);
}
#[tokio::test]
async fn maintenance_config_rejects_unsupported_config_version() {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend);
let bucket = "analytics";
let namespace = Namespace::parse("sales").unwrap();
let table = IdentifierSegment::parse("orders").unwrap();
let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json");
seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await;
let err = store
.put_table_maintenance_config(
bucket,
"sales",
"orders",
TableMaintenanceConfig {
version: TABLE_MAINTENANCE_CONFIG_VERSION.saturating_add(1),
retain_recent_metadata_files: 1,
delete_enabled: false,
background_enabled: false,
},
)
.await
.unwrap_err();
assert_matches!(err, TableCatalogStoreError::Invalid(_));
}
#[tokio::test]
async fn maintenance_dry_run_keeps_metadata_log_references() {
let backend = TestCatalogObjectBackend::default();