mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-25 13:36:50 +00:00
feat(table-catalog): add product API surface (#3320)
* feat(table-catalog): add product API surface * fix(table-catalog): harden product API commits * fix(table-catalog): close product API review gaps * fix(table-catalog): validate rollback metadata before commit --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -39,8 +39,11 @@ const GET_GROUP: AdminActionRef = AdminActionRef::new("GetGroupAdminAction");
|
|||||||
const GET_POLICY: AdminActionRef = AdminActionRef::new("GetPolicyAdminAction");
|
const GET_POLICY: AdminActionRef = AdminActionRef::new("GetPolicyAdminAction");
|
||||||
const GET_REPLICATION_METRICS: AdminActionRef = AdminActionRef::new("GetReplicationMetricsAction");
|
const GET_REPLICATION_METRICS: AdminActionRef = AdminActionRef::new("GetReplicationMetricsAction");
|
||||||
const GET_TABLE: AdminActionRef = AdminActionRef::new("GetTableAction");
|
const GET_TABLE: AdminActionRef = AdminActionRef::new("GetTableAction");
|
||||||
|
const GET_TABLE_BUCKET: AdminActionRef = AdminActionRef::new("GetTableBucketAction");
|
||||||
const GET_TABLE_CATALOG: AdminActionRef = AdminActionRef::new("GetTableCatalogAction");
|
const GET_TABLE_CATALOG: AdminActionRef = AdminActionRef::new("GetTableCatalogAction");
|
||||||
|
const GET_TABLE_LIFECYCLE: AdminActionRef = AdminActionRef::new("GetTableLifecycleAction");
|
||||||
const GET_TABLE_METADATA: AdminActionRef = AdminActionRef::new("GetTableMetadataAction");
|
const GET_TABLE_METADATA: AdminActionRef = AdminActionRef::new("GetTableMetadataAction");
|
||||||
|
const GET_TABLE_METADATA_LOCATION: AdminActionRef = AdminActionRef::new("GetTableMetadataLocationAction");
|
||||||
const GET_TABLE_NAMESPACE: AdminActionRef = AdminActionRef::new("GetTableNamespaceAction");
|
const GET_TABLE_NAMESPACE: AdminActionRef = AdminActionRef::new("GetTableNamespaceAction");
|
||||||
const HEAL: AdminActionRef = AdminActionRef::new("HealAdminAction");
|
const HEAL: AdminActionRef = AdminActionRef::new("HealAdminAction");
|
||||||
const IMPORT_BUCKET_METADATA: AdminActionRef = AdminActionRef::new("ImportBucketMetadataAction");
|
const IMPORT_BUCKET_METADATA: AdminActionRef = AdminActionRef::new("ImportBucketMetadataAction");
|
||||||
@@ -66,6 +69,9 @@ const SERVER_INFO: AdminActionRef = AdminActionRef::new("ServerInfoAdminAction")
|
|||||||
const SET_BUCKET_QUOTA: AdminActionRef = AdminActionRef::new("SetBucketQuotaAdminAction");
|
const SET_BUCKET_QUOTA: AdminActionRef = AdminActionRef::new("SetBucketQuotaAdminAction");
|
||||||
const SET_BUCKET_TARGET: AdminActionRef = AdminActionRef::new("SetBucketTargetAction");
|
const SET_BUCKET_TARGET: AdminActionRef = AdminActionRef::new("SetBucketTargetAction");
|
||||||
const SET_TABLE: AdminActionRef = AdminActionRef::new("SetTableAction");
|
const SET_TABLE: AdminActionRef = AdminActionRef::new("SetTableAction");
|
||||||
|
const SET_TABLE_BUCKET: AdminActionRef = AdminActionRef::new("SetTableBucketAction");
|
||||||
|
const SET_TABLE_LIFECYCLE: AdminActionRef = AdminActionRef::new("SetTableLifecycleAction");
|
||||||
|
const SET_TABLE_METADATA_LOCATION: AdminActionRef = AdminActionRef::new("SetTableMetadataLocationAction");
|
||||||
const SET_TABLE_NAMESPACE: AdminActionRef = AdminActionRef::new("SetTableNamespaceAction");
|
const SET_TABLE_NAMESPACE: AdminActionRef = AdminActionRef::new("SetTableNamespaceAction");
|
||||||
const SET_TIER: AdminActionRef = AdminActionRef::new("SetTierAction");
|
const SET_TIER: AdminActionRef = AdminActionRef::new("SetTierAction");
|
||||||
const SITE_REPLICATION_ADD: AdminActionRef = AdminActionRef::new("SiteReplicationAddAction");
|
const SITE_REPLICATION_ADD: AdminActionRef = AdminActionRef::new("SiteReplicationAddAction");
|
||||||
@@ -590,6 +596,13 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
|||||||
),
|
),
|
||||||
admin(HttpMethod::Post, "/rustfs/admin/v3/oidc/validate", SERVER_INFO, RouteRiskLevel::High),
|
admin(HttpMethod::Post, "/rustfs/admin/v3/oidc/validate", SERVER_INFO, RouteRiskLevel::High),
|
||||||
admin(HttpMethod::Get, "/iceberg/v1/config", GET_TABLE_CATALOG, RouteRiskLevel::Sensitive),
|
admin(HttpMethod::Get, "/iceberg/v1/config", GET_TABLE_CATALOG, RouteRiskLevel::Sensitive),
|
||||||
|
admin(HttpMethod::Put, "/iceberg/v1/buckets/{warehouse}", SET_TABLE_BUCKET, RouteRiskLevel::High),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/iceberg/v1/buckets/{warehouse}",
|
||||||
|
GET_TABLE_BUCKET,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
admin(
|
admin(
|
||||||
HttpMethod::Get,
|
HttpMethod::Get,
|
||||||
"/iceberg/v1/{warehouse}/namespaces",
|
"/iceberg/v1/{warehouse}/namespaces",
|
||||||
@@ -650,13 +663,79 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
|||||||
DELETE_TABLE,
|
DELETE_TABLE,
|
||||||
RouteRiskLevel::High,
|
RouteRiskLevel::High,
|
||||||
),
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
|
||||||
|
GET_TABLE_METADATA_LOCATION,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Put,
|
||||||
|
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
|
||||||
|
SET_TABLE_METADATA_LOCATION,
|
||||||
|
RouteRiskLevel::High,
|
||||||
|
),
|
||||||
admin(
|
admin(
|
||||||
HttpMethod::Post,
|
HttpMethod::Post,
|
||||||
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
|
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
|
||||||
RUN_TABLE_MAINTENANCE,
|
RUN_TABLE_MAINTENANCE,
|
||||||
RouteRiskLevel::High,
|
RouteRiskLevel::High,
|
||||||
),
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
|
||||||
|
GET_TABLE_LIFECYCLE,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Put,
|
||||||
|
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
|
||||||
|
SET_TABLE_LIFECYCLE,
|
||||||
|
RouteRiskLevel::High,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}",
|
||||||
|
GET_TABLE_LIFECYCLE,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export",
|
||||||
|
GET_TABLE_METADATA,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Post,
|
||||||
|
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import",
|
||||||
|
REGISTER_TABLE,
|
||||||
|
RouteRiskLevel::High,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/diagnostics",
|
||||||
|
GET_TABLE_METADATA,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Post,
|
||||||
|
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/rollback",
|
||||||
|
COMMIT_TABLE,
|
||||||
|
RouteRiskLevel::High,
|
||||||
|
),
|
||||||
admin(HttpMethod::Get, "/_iceberg/v1/config", GET_TABLE_CATALOG, RouteRiskLevel::Sensitive),
|
admin(HttpMethod::Get, "/_iceberg/v1/config", GET_TABLE_CATALOG, RouteRiskLevel::Sensitive),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Put,
|
||||||
|
"/_iceberg/v1/buckets/{warehouse}",
|
||||||
|
SET_TABLE_BUCKET,
|
||||||
|
RouteRiskLevel::High,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/_iceberg/v1/buckets/{warehouse}",
|
||||||
|
GET_TABLE_BUCKET,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
admin(
|
admin(
|
||||||
HttpMethod::Get,
|
HttpMethod::Get,
|
||||||
"/_iceberg/v1/{warehouse}/namespaces",
|
"/_iceberg/v1/{warehouse}/namespaces",
|
||||||
@@ -717,12 +796,66 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
|||||||
DELETE_TABLE,
|
DELETE_TABLE,
|
||||||
RouteRiskLevel::High,
|
RouteRiskLevel::High,
|
||||||
),
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
|
||||||
|
GET_TABLE_METADATA_LOCATION,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Put,
|
||||||
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
|
||||||
|
SET_TABLE_METADATA_LOCATION,
|
||||||
|
RouteRiskLevel::High,
|
||||||
|
),
|
||||||
admin(
|
admin(
|
||||||
HttpMethod::Post,
|
HttpMethod::Post,
|
||||||
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
|
||||||
RUN_TABLE_MAINTENANCE,
|
RUN_TABLE_MAINTENANCE,
|
||||||
RouteRiskLevel::High,
|
RouteRiskLevel::High,
|
||||||
),
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
|
||||||
|
GET_TABLE_LIFECYCLE,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Put,
|
||||||
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
|
||||||
|
SET_TABLE_LIFECYCLE,
|
||||||
|
RouteRiskLevel::High,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}",
|
||||||
|
GET_TABLE_LIFECYCLE,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export",
|
||||||
|
GET_TABLE_METADATA,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Post,
|
||||||
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import",
|
||||||
|
REGISTER_TABLE,
|
||||||
|
RouteRiskLevel::High,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/diagnostics",
|
||||||
|
GET_TABLE_METADATA,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Post,
|
||||||
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/rollback",
|
||||||
|
COMMIT_TABLE,
|
||||||
|
RouteRiskLevel::High,
|
||||||
|
),
|
||||||
];
|
];
|
||||||
|
|
||||||
pub const DEFERRED_ADMIN_ROUTE_POLICIES: &[DeferredAdminRoutePolicy] = &[
|
pub const DEFERRED_ADMIN_ROUTE_POLICIES: &[DeferredAdminRoutePolicy] = &[
|
||||||
@@ -896,7 +1029,9 @@ mod tests {
|
|||||||
let table_specs = ADMIN_ROUTE_POLICY_SPECS
|
let table_specs = ADMIN_ROUTE_POLICY_SPECS
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|spec| spec.path().starts_with("/iceberg/v1") || spec.path().starts_with("/_iceberg/v1"));
|
.filter(|spec| spec.path().starts_with("/iceberg/v1") || spec.path().starts_with("/_iceberg/v1"));
|
||||||
assert_eq!(table_specs.count(), 24);
|
assert_eq!(table_specs.count(), 46);
|
||||||
|
assert_action(HttpMethod::Put, "/iceberg/v1/buckets/{warehouse}", SET_TABLE_BUCKET);
|
||||||
|
assert_action(HttpMethod::Get, "/_iceberg/v1/buckets/{warehouse}", GET_TABLE_BUCKET);
|
||||||
assert_action(HttpMethod::Get, "/iceberg/v1/{warehouse}/namespaces", GET_TABLE_NAMESPACE);
|
assert_action(HttpMethod::Get, "/iceberg/v1/{warehouse}/namespaces", GET_TABLE_NAMESPACE);
|
||||||
assert_action(HttpMethod::Get, "/_iceberg/v1/{warehouse}/namespaces", GET_TABLE_NAMESPACE);
|
assert_action(HttpMethod::Get, "/_iceberg/v1/{warehouse}/namespaces", GET_TABLE_NAMESPACE);
|
||||||
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/namespaces/{namespace}/tables", CREATE_TABLE);
|
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/namespaces/{namespace}/tables", CREATE_TABLE);
|
||||||
@@ -921,6 +1056,16 @@ mod tests {
|
|||||||
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}",
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}",
|
||||||
COMMIT_TABLE,
|
COMMIT_TABLE,
|
||||||
);
|
);
|
||||||
|
assert_action(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
|
||||||
|
GET_TABLE_METADATA_LOCATION,
|
||||||
|
);
|
||||||
|
assert_action(
|
||||||
|
HttpMethod::Put,
|
||||||
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
|
||||||
|
SET_TABLE_METADATA_LOCATION,
|
||||||
|
);
|
||||||
assert_action(
|
assert_action(
|
||||||
HttpMethod::Post,
|
HttpMethod::Post,
|
||||||
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
|
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
|
||||||
@@ -931,6 +1076,26 @@ mod tests {
|
|||||||
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
|
||||||
RUN_TABLE_MAINTENANCE,
|
RUN_TABLE_MAINTENANCE,
|
||||||
);
|
);
|
||||||
|
assert_action(
|
||||||
|
HttpMethod::Put,
|
||||||
|
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
|
||||||
|
SET_TABLE_LIFECYCLE,
|
||||||
|
);
|
||||||
|
assert_action(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}",
|
||||||
|
GET_TABLE_LIFECYCLE,
|
||||||
|
);
|
||||||
|
assert_action(
|
||||||
|
HttpMethod::Post,
|
||||||
|
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import",
|
||||||
|
REGISTER_TABLE,
|
||||||
|
);
|
||||||
|
assert_action(
|
||||||
|
HttpMethod::Post,
|
||||||
|
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/rollback",
|
||||||
|
COMMIT_TABLE,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -281,6 +281,8 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
|||||||
admin_route_sample(Method::DELETE, "/v3/oidc/config/{provider_id}", "/v3/oidc/config/default"),
|
admin_route_sample(Method::DELETE, "/v3/oidc/config/{provider_id}", "/v3/oidc/config/default"),
|
||||||
admin_route(Method::POST, "/v3/oidc/validate"),
|
admin_route(Method::POST, "/v3/oidc/validate"),
|
||||||
table_route(Method::GET, "/config"),
|
table_route(Method::GET, "/config"),
|
||||||
|
table_route_sample(Method::PUT, "/buckets/{warehouse}", "/buckets/analytics"),
|
||||||
|
table_route_sample(Method::GET, "/buckets/{warehouse}", "/buckets/analytics"),
|
||||||
table_route_sample(Method::GET, "/{warehouse}/namespaces", "/analytics/namespaces"),
|
table_route_sample(Method::GET, "/{warehouse}/namespaces", "/analytics/namespaces"),
|
||||||
table_route_sample(Method::POST, "/{warehouse}/namespaces", "/analytics/namespaces"),
|
table_route_sample(Method::POST, "/{warehouse}/namespaces", "/analytics/namespaces"),
|
||||||
table_route_sample(Method::GET, "/{warehouse}/namespaces/{namespace}", "/analytics/namespaces/sales"),
|
table_route_sample(Method::GET, "/{warehouse}/namespaces/{namespace}", "/analytics/namespaces/sales"),
|
||||||
@@ -320,7 +322,54 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
|||||||
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
|
||||||
"/analytics/namespaces/sales/tables/orders/maintenance/metadata",
|
"/analytics/namespaces/sales/tables/orders/maintenance/metadata",
|
||||||
),
|
),
|
||||||
|
table_route_sample(
|
||||||
|
Method::GET,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/metadata-location",
|
||||||
|
),
|
||||||
|
table_route_sample(
|
||||||
|
Method::PUT,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/metadata-location",
|
||||||
|
),
|
||||||
|
table_route_sample(
|
||||||
|
Method::GET,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/maintenance/config",
|
||||||
|
),
|
||||||
|
table_route_sample(
|
||||||
|
Method::PUT,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/maintenance/config",
|
||||||
|
),
|
||||||
|
table_route_sample(
|
||||||
|
Method::GET,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/maintenance/jobs/job-1",
|
||||||
|
),
|
||||||
|
table_route_sample(
|
||||||
|
Method::GET,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/catalog/export",
|
||||||
|
),
|
||||||
|
table_route_sample(
|
||||||
|
Method::POST,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/catalog/import",
|
||||||
|
),
|
||||||
|
table_route_sample(
|
||||||
|
Method::GET,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/diagnostics",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/catalog/diagnostics",
|
||||||
|
),
|
||||||
|
table_route_sample(
|
||||||
|
Method::POST,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/rollback",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/catalog/rollback",
|
||||||
|
),
|
||||||
compat_table_route(Method::GET, "/config"),
|
compat_table_route(Method::GET, "/config"),
|
||||||
|
compat_table_route_sample(Method::PUT, "/buckets/{warehouse}", "/buckets/analytics"),
|
||||||
|
compat_table_route_sample(Method::GET, "/buckets/{warehouse}", "/buckets/analytics"),
|
||||||
compat_table_route_sample(Method::GET, "/{warehouse}/namespaces", "/analytics/namespaces"),
|
compat_table_route_sample(Method::GET, "/{warehouse}/namespaces", "/analytics/namespaces"),
|
||||||
compat_table_route_sample(Method::POST, "/{warehouse}/namespaces", "/analytics/namespaces"),
|
compat_table_route_sample(Method::POST, "/{warehouse}/namespaces", "/analytics/namespaces"),
|
||||||
compat_table_route_sample(Method::GET, "/{warehouse}/namespaces/{namespace}", "/analytics/namespaces/sales"),
|
compat_table_route_sample(Method::GET, "/{warehouse}/namespaces/{namespace}", "/analytics/namespaces/sales"),
|
||||||
@@ -360,6 +409,51 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
|||||||
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata",
|
||||||
"/analytics/namespaces/sales/tables/orders/maintenance/metadata",
|
"/analytics/namespaces/sales/tables/orders/maintenance/metadata",
|
||||||
),
|
),
|
||||||
|
compat_table_route_sample(
|
||||||
|
Method::GET,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/metadata-location",
|
||||||
|
),
|
||||||
|
compat_table_route_sample(
|
||||||
|
Method::PUT,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/metadata-location",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/metadata-location",
|
||||||
|
),
|
||||||
|
compat_table_route_sample(
|
||||||
|
Method::GET,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/maintenance/config",
|
||||||
|
),
|
||||||
|
compat_table_route_sample(
|
||||||
|
Method::PUT,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/maintenance/config",
|
||||||
|
),
|
||||||
|
compat_table_route_sample(
|
||||||
|
Method::GET,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/maintenance/jobs/job-1",
|
||||||
|
),
|
||||||
|
compat_table_route_sample(
|
||||||
|
Method::GET,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/catalog/export",
|
||||||
|
),
|
||||||
|
compat_table_route_sample(
|
||||||
|
Method::POST,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/import",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/catalog/import",
|
||||||
|
),
|
||||||
|
compat_table_route_sample(
|
||||||
|
Method::GET,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/diagnostics",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/catalog/diagnostics",
|
||||||
|
),
|
||||||
|
compat_table_route_sample(
|
||||||
|
Method::POST,
|
||||||
|
"/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/rollback",
|
||||||
|
"/analytics/namespaces/sales/tables/orders/catalog/rollback",
|
||||||
|
),
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -467,6 +561,8 @@ fn test_register_routes_cover_representative_admin_paths() {
|
|||||||
assert_route(&router, Method::GET, &admin_path("/v3/scanner/status"));
|
assert_route(&router, Method::GET, &admin_path("/v3/scanner/status"));
|
||||||
|
|
||||||
assert_route(&router, Method::GET, &table_catalog_path("/config"));
|
assert_route(&router, Method::GET, &table_catalog_path("/config"));
|
||||||
|
assert_route(&router, Method::PUT, &table_catalog_path("/buckets/analytics"));
|
||||||
|
assert_route(&router, Method::GET, &table_catalog_path("/buckets/analytics"));
|
||||||
assert_route(&router, Method::GET, &table_catalog_path("/analytics/namespaces"));
|
assert_route(&router, Method::GET, &table_catalog_path("/analytics/namespaces"));
|
||||||
assert_route(&router, Method::POST, &table_catalog_path("/analytics/namespaces"));
|
assert_route(&router, Method::POST, &table_catalog_path("/analytics/namespaces"));
|
||||||
assert_route(&router, Method::GET, &table_catalog_path("/analytics/namespaces/sales"));
|
assert_route(&router, Method::GET, &table_catalog_path("/analytics/namespaces/sales"));
|
||||||
@@ -482,7 +578,54 @@ fn test_register_routes_cover_representative_admin_paths() {
|
|||||||
Method::POST,
|
Method::POST,
|
||||||
&table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/metadata"),
|
&table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/metadata"),
|
||||||
);
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::GET,
|
||||||
|
&table_catalog_path("/analytics/namespaces/sales/tables/orders/metadata-location"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::PUT,
|
||||||
|
&table_catalog_path("/analytics/namespaces/sales/tables/orders/metadata-location"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::PUT,
|
||||||
|
&table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/config"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::GET,
|
||||||
|
&table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/config"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::GET,
|
||||||
|
&table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/jobs/job-1"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::GET,
|
||||||
|
&table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/export"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::POST,
|
||||||
|
&table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/import"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::GET,
|
||||||
|
&table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/diagnostics"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::POST,
|
||||||
|
&table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/rollback"),
|
||||||
|
);
|
||||||
assert_route(&router, Method::GET, &compat_table_catalog_path("/config"));
|
assert_route(&router, Method::GET, &compat_table_catalog_path("/config"));
|
||||||
|
assert_route(&router, Method::PUT, &compat_table_catalog_path("/buckets/analytics"));
|
||||||
|
assert_route(&router, Method::GET, &compat_table_catalog_path("/buckets/analytics"));
|
||||||
assert_route(&router, Method::GET, &compat_table_catalog_path("/analytics/namespaces"));
|
assert_route(&router, Method::GET, &compat_table_catalog_path("/analytics/namespaces"));
|
||||||
assert_route(&router, Method::POST, &compat_table_catalog_path("/analytics/namespaces"));
|
assert_route(&router, Method::POST, &compat_table_catalog_path("/analytics/namespaces"));
|
||||||
assert_route(&router, Method::GET, &compat_table_catalog_path("/analytics/namespaces/sales"));
|
assert_route(&router, Method::GET, &compat_table_catalog_path("/analytics/namespaces/sales"));
|
||||||
@@ -510,6 +653,51 @@ fn test_register_routes_cover_representative_admin_paths() {
|
|||||||
Method::POST,
|
Method::POST,
|
||||||
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/metadata"),
|
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/metadata"),
|
||||||
);
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::GET,
|
||||||
|
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/metadata-location"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::PUT,
|
||||||
|
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/metadata-location"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::PUT,
|
||||||
|
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/config"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::GET,
|
||||||
|
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/config"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::GET,
|
||||||
|
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/jobs/job-1"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::GET,
|
||||||
|
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/export"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::POST,
|
||||||
|
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/import"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::GET,
|
||||||
|
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/diagnostics"),
|
||||||
|
);
|
||||||
|
assert_route(
|
||||||
|
&router,
|
||||||
|
Method::POST,
|
||||||
|
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/catalog/rollback"),
|
||||||
|
);
|
||||||
|
|
||||||
assert_route(&router, Method::POST, &admin_path("/v3/service"));
|
assert_route(&router, Method::POST, &admin_path("/v3/service"));
|
||||||
assert_route(&router, Method::GET, &admin_path("/v3/info"));
|
assert_route(&router, Method::GET, &admin_path("/v3/info"));
|
||||||
|
|||||||
+275
-3
@@ -54,6 +54,7 @@ pub(crate) const TABLE_NAMESPACE_MARKER_VERSION: u16 = 1;
|
|||||||
pub(crate) const TABLE_RESOURCE_MARKER_VERSION: u16 = 1;
|
pub(crate) const TABLE_RESOURCE_MARKER_VERSION: u16 = 1;
|
||||||
pub(crate) const TABLE_METADATA_POINTER_VERSION: u16 = 1;
|
pub(crate) const TABLE_METADATA_POINTER_VERSION: u16 = 1;
|
||||||
pub(crate) const TABLE_CATALOG_ENTRY_VERSION: u16 = 1;
|
pub(crate) const TABLE_CATALOG_ENTRY_VERSION: u16 = 1;
|
||||||
|
pub(crate) const TABLE_MAINTENANCE_CONFIG_VERSION: u16 = 1;
|
||||||
pub(crate) const TABLE_METADATA_FILE_NAME_MAX_LEN: usize = 128;
|
pub(crate) const TABLE_METADATA_FILE_NAME_MAX_LEN: usize = 128;
|
||||||
pub const TABLE_RESERVED_PREFIX: &str = BUCKET_TABLE_RESERVED_PREFIX;
|
pub const TABLE_RESERVED_PREFIX: &str = BUCKET_TABLE_RESERVED_PREFIX;
|
||||||
const WAREHOUSE_ROOT: &str = "warehouses";
|
const WAREHOUSE_ROOT: &str = "warehouses";
|
||||||
@@ -71,6 +72,9 @@ const INTERNAL_CATALOG_ROOT: &str = BUCKET_TABLE_CATALOG_META_PREFIX;
|
|||||||
const TABLE_BUCKET_ROOT: &str = BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX;
|
const TABLE_BUCKET_ROOT: &str = BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX;
|
||||||
const COMMIT_LOG_ROOT: &str = "commits";
|
const COMMIT_LOG_ROOT: &str = "commits";
|
||||||
const COMMIT_IDEMPOTENCY_ROOT: &str = "commit-idempotency";
|
const COMMIT_IDEMPOTENCY_ROOT: &str = "commit-idempotency";
|
||||||
|
const MAINTENANCE_ROOT: &str = "maintenance";
|
||||||
|
const MAINTENANCE_CONFIG_FILE: &str = "config.json";
|
||||||
|
const MAINTENANCE_JOB_ROOT: &str = "jobs";
|
||||||
const TABLE_CATALOG_LIST_MAX_KEYS: i32 = 1000;
|
const TABLE_CATALOG_LIST_MAX_KEYS: i32 = 1000;
|
||||||
const TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS: i64 = 15 * 60;
|
const TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS: i64 = 15 * 60;
|
||||||
|
|
||||||
@@ -245,12 +249,36 @@ pub(crate) struct TableCommitResult {
|
|||||||
pub commit_log: CommitLogEntry,
|
pub commit_log: CommitLogEntry,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub(crate) struct TableMaintenanceConfig {
|
||||||
|
pub version: u16,
|
||||||
|
#[serde(rename = "retain-recent-metadata-files")]
|
||||||
|
pub retain_recent_metadata_files: usize,
|
||||||
|
#[serde(rename = "delete-enabled")]
|
||||||
|
pub delete_enabled: bool,
|
||||||
|
#[serde(rename = "background-enabled")]
|
||||||
|
pub background_enabled: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for TableMaintenanceConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
version: TABLE_MAINTENANCE_CONFIG_VERSION,
|
||||||
|
retain_recent_metadata_files: 0,
|
||||||
|
delete_enabled: false,
|
||||||
|
background_enabled: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub(crate) struct TableMetadataMaintenanceJob {
|
pub(crate) struct TableMetadataMaintenanceJob {
|
||||||
pub job_id: String,
|
pub job_id: String,
|
||||||
pub table_bucket: String,
|
pub table_bucket: String,
|
||||||
pub namespace: String,
|
pub namespace: String,
|
||||||
pub table: String,
|
pub table: String,
|
||||||
|
pub table_id: String,
|
||||||
pub current_metadata_location: String,
|
pub current_metadata_location: String,
|
||||||
pub current_generation: u64,
|
pub current_generation: u64,
|
||||||
pub retain_recent_metadata_files: usize,
|
pub retain_recent_metadata_files: usize,
|
||||||
@@ -258,7 +286,7 @@ pub(crate) struct TableMetadataMaintenanceJob {
|
|||||||
pub cleanup_watermark_unix_seconds: i64,
|
pub cleanup_watermark_unix_seconds: i64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub(crate) struct TableMetadataMaintenanceReport {
|
pub(crate) struct TableMetadataMaintenanceReport {
|
||||||
pub job: TableMetadataMaintenanceJob,
|
pub job: TableMetadataMaintenanceJob,
|
||||||
pub current_metadata_location: String,
|
pub current_metadata_location: String,
|
||||||
@@ -438,6 +466,38 @@ impl TableCatalogObjectPaths {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn table_maintenance_config_path(
|
||||||
|
&self,
|
||||||
|
table_bucket: &str,
|
||||||
|
namespace: &Namespace,
|
||||||
|
table: &IdentifierSegment,
|
||||||
|
table_id: &str,
|
||||||
|
) -> String {
|
||||||
|
format!(
|
||||||
|
"{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_CONFIG_FILE}",
|
||||||
|
self.table_entries_prefix(table_bucket, namespace),
|
||||||
|
table.as_str(),
|
||||||
|
table_catalog_path_hash(table_id)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn table_maintenance_job_path(
|
||||||
|
&self,
|
||||||
|
table_bucket: &str,
|
||||||
|
namespace: &Namespace,
|
||||||
|
table: &IdentifierSegment,
|
||||||
|
table_id: &str,
|
||||||
|
job_id: &str,
|
||||||
|
) -> String {
|
||||||
|
format!(
|
||||||
|
"{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_JOB_ROOT}/{}.json",
|
||||||
|
self.table_entries_prefix(table_bucket, namespace),
|
||||||
|
table.as_str(),
|
||||||
|
table_catalog_path_hash(table_id),
|
||||||
|
table_catalog_path_hash(job_id)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn commit_log_entry_path(&self, table_bucket: &str, table_id: &str, commit_id: &str) -> String {
|
pub fn commit_log_entry_path(&self, table_bucket: &str, table_id: &str, commit_id: &str) -> String {
|
||||||
format!(
|
format!(
|
||||||
"{}{}/{}/{}.json",
|
"{}{}/{}/{}.json",
|
||||||
@@ -576,6 +636,108 @@ where
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn get_table_maintenance_config(
|
||||||
|
&self,
|
||||||
|
table_bucket: &str,
|
||||||
|
namespace: &str,
|
||||||
|
table: &str,
|
||||||
|
) -> TableCatalogStoreResult<TableMaintenanceConfig> {
|
||||||
|
let namespace = parse_namespace_for_store(namespace)?;
|
||||||
|
let table = parse_table_for_store(table)?;
|
||||||
|
let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table);
|
||||||
|
let Some((entry, _)) = self.read_entry::<TableEntry>(self.catalog_bucket(), &table_path).await? else {
|
||||||
|
return Err(TableCatalogStoreError::NotFound(format!(
|
||||||
|
"table {}/{}/{}",
|
||||||
|
table_bucket,
|
||||||
|
namespace.public_name(),
|
||||||
|
table.as_str()
|
||||||
|
)));
|
||||||
|
};
|
||||||
|
|
||||||
|
let config_path = self
|
||||||
|
.paths
|
||||||
|
.table_maintenance_config_path(table_bucket, &namespace, &table, &entry.table_id);
|
||||||
|
self.read_entry::<TableMaintenanceConfig>(self.catalog_bucket(), &config_path)
|
||||||
|
.await
|
||||||
|
.map(|entry| entry.map(|(config, _)| config).unwrap_or_default())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn put_table_maintenance_config(
|
||||||
|
&self,
|
||||||
|
table_bucket: &str,
|
||||||
|
namespace: &str,
|
||||||
|
table: &str,
|
||||||
|
config: TableMaintenanceConfig,
|
||||||
|
) -> TableCatalogStoreResult<TableMaintenanceConfig> {
|
||||||
|
validate_table_maintenance_config_version(config.version)?;
|
||||||
|
if config.background_enabled {
|
||||||
|
return Err(TableCatalogStoreError::Invalid(
|
||||||
|
"background table maintenance is not supported".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let namespace = parse_namespace_for_store(namespace)?;
|
||||||
|
let table = parse_table_for_store(table)?;
|
||||||
|
let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table);
|
||||||
|
let Some((entry, _)) = self.read_entry::<TableEntry>(self.catalog_bucket(), &table_path).await? else {
|
||||||
|
return Err(TableCatalogStoreError::NotFound(format!(
|
||||||
|
"table {}/{}/{}",
|
||||||
|
table_bucket,
|
||||||
|
namespace.public_name(),
|
||||||
|
table.as_str()
|
||||||
|
)));
|
||||||
|
};
|
||||||
|
|
||||||
|
let config_path = self
|
||||||
|
.paths
|
||||||
|
.table_maintenance_config_path(table_bucket, &namespace, &table, &entry.table_id);
|
||||||
|
self.write_entry(self.catalog_bucket(), &config_path, &config, TableCatalogPutPrecondition::Any)
|
||||||
|
.await?;
|
||||||
|
Ok(config)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn put_table_metadata_maintenance_report(
|
||||||
|
&self,
|
||||||
|
report: &TableMetadataMaintenanceReport,
|
||||||
|
) -> TableCatalogStoreResult<()> {
|
||||||
|
let namespace = parse_namespace_for_store(&report.job.namespace)?;
|
||||||
|
let table = parse_table_for_store(&report.job.table)?;
|
||||||
|
let job_path = self.paths.table_maintenance_job_path(
|
||||||
|
&report.job.table_bucket,
|
||||||
|
&namespace,
|
||||||
|
&table,
|
||||||
|
&report.job.table_id,
|
||||||
|
&report.job.job_id,
|
||||||
|
);
|
||||||
|
self.write_entry(self.catalog_bucket(), &job_path, report, TableCatalogPutPrecondition::Any)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn get_table_metadata_maintenance_report(
|
||||||
|
&self,
|
||||||
|
table_bucket: &str,
|
||||||
|
namespace: &str,
|
||||||
|
table: &str,
|
||||||
|
job_id: &str,
|
||||||
|
) -> TableCatalogStoreResult<Option<TableMetadataMaintenanceReport>> {
|
||||||
|
let namespace = parse_namespace_for_store(namespace)?;
|
||||||
|
let table = parse_table_for_store(table)?;
|
||||||
|
let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table);
|
||||||
|
let Some((entry, _)) = self.read_entry::<TableEntry>(self.catalog_bucket(), &table_path).await? else {
|
||||||
|
return Err(TableCatalogStoreError::NotFound(format!(
|
||||||
|
"table {}/{}/{}",
|
||||||
|
table_bucket,
|
||||||
|
namespace.public_name(),
|
||||||
|
table.as_str()
|
||||||
|
)));
|
||||||
|
};
|
||||||
|
let job_path = self
|
||||||
|
.paths
|
||||||
|
.table_maintenance_job_path(table_bucket, &namespace, &table, &entry.table_id, job_id);
|
||||||
|
self.read_entry::<TableMetadataMaintenanceReport>(self.catalog_bucket(), &job_path)
|
||||||
|
.await
|
||||||
|
.map(|entry| entry.map(|(report, _)| report))
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn export_table_catalog_entry(
|
pub(crate) async fn export_table_catalog_entry(
|
||||||
&self,
|
&self,
|
||||||
table_bucket: &str,
|
table_bucket: &str,
|
||||||
@@ -759,6 +921,7 @@ where
|
|||||||
table_bucket: table_bucket.to_string(),
|
table_bucket: table_bucket.to_string(),
|
||||||
namespace: namespace.public_name(),
|
namespace: namespace.public_name(),
|
||||||
table: table.as_str().to_string(),
|
table: table.as_str().to_string(),
|
||||||
|
table_id: entry.table_id,
|
||||||
current_metadata_location: current_metadata_location.clone(),
|
current_metadata_location: current_metadata_location.clone(),
|
||||||
current_generation: entry.generation,
|
current_generation: entry.generation,
|
||||||
retain_recent_metadata_files,
|
retain_recent_metadata_files,
|
||||||
@@ -1365,6 +1528,15 @@ fn validate_catalog_entry_version(kind: &str, version: u16) -> TableCatalogStore
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn validate_table_maintenance_config_version(version: u16) -> TableCatalogStoreResult<()> {
|
||||||
|
if version != TABLE_MAINTENANCE_CONFIG_VERSION {
|
||||||
|
return Err(TableCatalogStoreError::Invalid(
|
||||||
|
"unsupported table maintenance config entry version".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn commit_log_matches_request(commit_log: &CommitLogEntry, request: &TableCommitRequest, table_id: &str) -> bool {
|
fn commit_log_matches_request(commit_log: &CommitLogEntry, request: &TableCommitRequest, table_id: &str) -> bool {
|
||||||
commit_log.version == TABLE_CATALOG_ENTRY_VERSION
|
commit_log.version == TABLE_CATALOG_ENTRY_VERSION
|
||||||
&& commit_log.commit_id == request.commit_id
|
&& commit_log.commit_id == request.commit_id
|
||||||
@@ -2108,8 +2280,11 @@ mod tests {
|
|||||||
|
|
||||||
let commit_path = paths.commit_log_entry_path("table/../bucket", "table/../id", "commit/%2f\nid");
|
let commit_path = paths.commit_log_entry_path("table/../bucket", "table/../id", "commit/%2f\nid");
|
||||||
let idempotency_path = paths.commit_idempotency_entry_path("table/../bucket", "table/../id", "client/%2f\nrequest");
|
let idempotency_path = paths.commit_idempotency_entry_path("table/../bucket", "table/../id", "client/%2f\nrequest");
|
||||||
|
let maintenance_config_path = paths.table_maintenance_config_path("table/../bucket", &namespace, &table, "table/../id");
|
||||||
|
let maintenance_job_path =
|
||||||
|
paths.table_maintenance_job_path("table/../bucket", &namespace, &table, "table/../id", "job/%2f\nid");
|
||||||
|
|
||||||
for path in [commit_path, idempotency_path] {
|
for path in [commit_path, idempotency_path, maintenance_config_path, maintenance_job_path] {
|
||||||
assert!(path.starts_with("s3tables/catalog/table-buckets/"));
|
assert!(path.starts_with("s3tables/catalog/table-buckets/"));
|
||||||
assert!(path.ends_with(".json"));
|
assert!(path.ends_with(".json"));
|
||||||
assert!(!path.contains(".."));
|
assert!(!path.contains(".."));
|
||||||
@@ -2400,6 +2575,7 @@ mod tests {
|
|||||||
assert_eq!(report.job.table_bucket, bucket);
|
assert_eq!(report.job.table_bucket, bucket);
|
||||||
assert_eq!(report.job.namespace, "sales");
|
assert_eq!(report.job.namespace, "sales");
|
||||||
assert_eq!(report.job.table, "orders");
|
assert_eq!(report.job.table, "orders");
|
||||||
|
assert_eq!(report.job.table_id, "table-id");
|
||||||
assert_eq!(report.job.current_generation, 1);
|
assert_eq!(report.job.current_generation, 1);
|
||||||
assert_eq!(report.job.safety_window_seconds, TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS);
|
assert_eq!(report.job.safety_window_seconds, TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS);
|
||||||
assert!(!report.job.job_id.is_empty());
|
assert!(!report.job.job_id.is_empty());
|
||||||
@@ -2408,6 +2584,102 @@ mod tests {
|
|||||||
assert_eq!(report.deletable_metadata_locations, vec![old]);
|
assert_eq!(report.deletable_metadata_locations, vec![old]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn maintenance_state_is_scoped_to_current_table_identity() {
|
||||||
|
let backend = TestCatalogObjectBackend::default();
|
||||||
|
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||||
|
let bucket = "analytics";
|
||||||
|
let namespace = Namespace::parse("sales").unwrap();
|
||||||
|
let table = IdentifierSegment::parse("orders").unwrap();
|
||||||
|
let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json");
|
||||||
|
|
||||||
|
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
|
||||||
|
store
|
||||||
|
.create_namespace(test_namespace_entry(bucket, &namespace))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let mut first_table = test_table_entry(bucket, &namespace, &table, current.clone());
|
||||||
|
first_table.table_id = "table-id-1".to_string();
|
||||||
|
store.create_table(first_table).await.unwrap();
|
||||||
|
store
|
||||||
|
.put_table_maintenance_config(
|
||||||
|
bucket,
|
||||||
|
"sales",
|
||||||
|
"orders",
|
||||||
|
TableMaintenanceConfig {
|
||||||
|
version: TABLE_MAINTENANCE_CONFIG_VERSION,
|
||||||
|
retain_recent_metadata_files: 7,
|
||||||
|
delete_enabled: true,
|
||||||
|
background_enabled: false,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
backend
|
||||||
|
.seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec())
|
||||||
|
.await;
|
||||||
|
let report = store
|
||||||
|
.plan_table_metadata_maintenance(bucket, "sales", "orders", 0)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
store.put_table_metadata_maintenance_report(&report).await.unwrap();
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.get_table_metadata_maintenance_report(bucket, "sales", "orders", &report.job.job_id)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
|
||||||
|
store.drop_table(bucket, "sales", "orders").await.unwrap();
|
||||||
|
|
||||||
|
let mut second_table = test_table_entry(bucket, &namespace, &table, current);
|
||||||
|
second_table.table_id = "table-id-2".to_string();
|
||||||
|
store.create_table(second_table).await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
store.get_table_maintenance_config(bucket, "sales", "orders").await.unwrap(),
|
||||||
|
TableMaintenanceConfig::default()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.get_table_metadata_maintenance_report(bucket, "sales", "orders", &report.job.job_id)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn maintenance_config_rejects_unsupported_config_version() {
|
||||||
|
let backend = TestCatalogObjectBackend::default();
|
||||||
|
let store = ObjectTableCatalogStore::new(backend);
|
||||||
|
let bucket = "analytics";
|
||||||
|
let namespace = Namespace::parse("sales").unwrap();
|
||||||
|
let table = IdentifierSegment::parse("orders").unwrap();
|
||||||
|
let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json");
|
||||||
|
|
||||||
|
seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current).await;
|
||||||
|
|
||||||
|
let err = store
|
||||||
|
.put_table_maintenance_config(
|
||||||
|
bucket,
|
||||||
|
"sales",
|
||||||
|
"orders",
|
||||||
|
TableMaintenanceConfig {
|
||||||
|
version: TABLE_MAINTENANCE_CONFIG_VERSION.saturating_add(1),
|
||||||
|
retain_recent_metadata_files: 1,
|
||||||
|
delete_enabled: false,
|
||||||
|
background_enabled: false,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
|
||||||
|
assert_matches!(err, TableCatalogStoreError::Invalid(_));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn maintenance_dry_run_keeps_metadata_log_references() {
|
async fn maintenance_dry_run_keeps_metadata_log_references() {
|
||||||
let backend = TestCatalogObjectBackend::default();
|
let backend = TestCatalogObjectBackend::default();
|
||||||
|
|||||||
Reference in New Issue
Block a user