gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-09 22:59:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(table-catalog): add distributed maintenance scheduling (#4257)

Browse Source 
* feat(table-catalog): add distributed maintenance scheduling

* fix(table-catalog): address scheduler review feedback

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
Henry Guo
2026-07-04 23:44:08 +08:00
committed by GitHub
parent f3147c5f8c
commit be45b35472
7 changed files with 973 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/handlers/table_catalog.rs
+79
View File
@@ -74,6 +74,7 @@ const TABLE_CATALOG_NAMESPACE_RESOURCE_ROOT: &str = "namespaces";
const TABLE_CATALOG_TABLE_RESOURCE_ROOT: &str = "tables";
const TABLE_CATALOG_VIEW_RESOURCE_ROOT: &str = "views";
const TABLE_CATALOG_ADMIN_OPERATION_SLOW_LOG_THRESHOLD: StdDuration = StdDuration::from_secs(2);
const DEFAULT_TABLE_MAINTENANCE_SCHEDULER_ID: &str = "rustfs-maintenance-scheduler";
const DEFAULT_TABLE_MAINTENANCE_WORKER_ID: &str = "rustfs-maintenance-worker";
const EXTERNAL_CATALOG_BRIDGE_STATUS_UNCONFIGURED: &str = "bridge-unconfigured";
const EXTERNAL_CATALOG_BRIDGE_STATUS_CONFIGURED: &str = "bridge-configured";
@@ -136,6 +137,7 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[
"PUT /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/config",
"GET /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}",
"GET /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler",
"POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler/run",
"POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/worker/run",
"POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}/heartbeat",
"POST /{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/jobs/{job}/quarantine",
@@ -182,6 +184,7 @@ static GET_TABLE_MAINTENANCE_CONFIG_HANDLER: GetTableMaintenanceConfigHandler =
static PUT_TABLE_MAINTENANCE_CONFIG_HANDLER: PutTableMaintenanceConfigHandler = PutTableMaintenanceConfigHandler {};
static GET_TABLE_MAINTENANCE_JOB_HANDLER: GetTableMaintenanceJobHandler = GetTableMaintenanceJobHandler {};
static GET_TABLE_MAINTENANCE_SCHEDULER_HANDLER: GetTableMaintenanceSchedulerHandler = GetTableMaintenanceSchedulerHandler {};
static RUN_TABLE_MAINTENANCE_SCHEDULER_HANDLER: RunTableMaintenanceSchedulerHandler = RunTableMaintenanceSchedulerHandler {};
static RUN_TABLE_MAINTENANCE_WORKER_HANDLER: RunTableMaintenanceWorkerHandler = RunTableMaintenanceWorkerHandler {};
static HEARTBEAT_TABLE_MAINTENANCE_JOB_HANDLER: HeartbeatTableMaintenanceJobHandler = HeartbeatTableMaintenanceJobHandler {};
static TABLE_MAINTENANCE_QUARANTINE_HANDLER: TableMaintenanceQuarantineHandler = TableMaintenanceQuarantineHandler {};
@@ -357,6 +360,19 @@ struct TableMetadataMaintenanceRequest {
commit_compaction: bool,
}
#[derive(Debug, Default, Deserialize)]
#[serde(deny_unknown_fields)]
struct TableMaintenanceSchedulerRunRequest {
#[serde(default, rename = "scheduler-id")]
scheduler_id: Option<String>,
}
impl TableMaintenanceSchedulerRunRequest {
fn scheduler_id(&self) -> &str {
self.scheduler_id.as_deref().unwrap_or(DEFAULT_TABLE_MAINTENANCE_SCHEDULER_ID)
}
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct TableMaintenanceWorkerRunRequest {
@@ -951,6 +967,11 @@ fn register_table_catalog_prefix_routes(r: &mut S3Router<AdminOperation>, prefix
format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/scheduler").as_str(),
AdminOperation(&GET_TABLE_MAINTENANCE_SCHEDULER_HANDLER),
)?;
r.insert(
Method::POST,
format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/scheduler/run").as_str(),
AdminOperation(&RUN_TABLE_MAINTENANCE_SCHEDULER_HANDLER),
)?;
r.insert(
Method::POST,
format!("{prefix}/{{warehouse}}/namespaces/{{namespace}}/tables/{{table}}/maintenance/worker/run").as_str(),
@@ -5237,6 +5258,32 @@ impl Operation for GetTableMaintenanceSchedulerHandler {
pub struct RunTableMaintenanceWorkerHandler {}
#[async_trait::async_trait]
impl Operation for RunTableMaintenanceSchedulerHandler {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let warehouse = warehouse_from_params(&params)?;
let namespace = namespace_from_params(&params)?;
let table = table_name_from_params(&params)?;
let resource = TableCatalogResource::table(&warehouse, &namespace, &table);
authorize_table_catalog_resource_request(&req, &resource, AdminAction::RunTableMaintenanceAction).await?;
ensure_table_bucket_enabled(&warehouse).await?;
let request = read_json_body_or_default::<TableMaintenanceSchedulerRunRequest>(req.input).await?;
let store = table_catalog_store()?;
let response = store
.run_table_maintenance_scheduler_once(
&warehouse,
&namespace.public_name(),
&table,
request.scheduler_id().to_string(),
)
.await
.map_err(catalog_store_error)?;
build_json_response(StatusCode::OK, &response)
}
}
pub struct RunTableMaintenanceSchedulerHandler {}
#[async_trait::async_trait]
impl Operation for RunTableMaintenanceWorkerHandler {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
@@ -5698,6 +5745,7 @@ mod tests {
("PutTableMaintenanceConfigHandler", "AdminAction::SetTableLifecycleAction"),
("GetTableMaintenanceJobHandler", "AdminAction::GetTableLifecycleAction"),
("GetTableMaintenanceSchedulerHandler", "AdminAction::GetTableLifecycleAction"),
("RunTableMaintenanceSchedulerHandler", "AdminAction::RunTableMaintenanceAction"),
("TableMaintenanceQuarantineHandler", "AdminAction::RunTableMaintenanceAction"),
("ExportTableCatalogHandler", "AdminAction::GetTableMetadataAction"),
("ImportTableCatalogHandler", "AdminAction::RegisterTableAction"),
@@ -5753,6 +5801,7 @@ mod tests {
("PutTableMaintenanceConfigHandler", "AdminAction::SetTableLifecycleAction"),
("GetTableMaintenanceJobHandler", "AdminAction::GetTableLifecycleAction"),
("GetTableMaintenanceSchedulerHandler", "AdminAction::GetTableLifecycleAction"),
("RunTableMaintenanceSchedulerHandler", "AdminAction::RunTableMaintenanceAction"),
("TableMaintenanceQuarantineHandler", "AdminAction::RunTableMaintenanceAction"),
("ExportTableCatalogHandler", "AdminAction::GetTableMetadataAction"),
("ImportTableCatalogHandler", "AdminAction::RegisterTableAction"),
@@ -5810,6 +5859,7 @@ mod tests {
"PutTableMaintenanceConfigHandler",
"GetTableMaintenanceJobHandler",
"GetTableMaintenanceSchedulerHandler",
"RunTableMaintenanceSchedulerHandler",
"RunTableMaintenanceWorkerHandler",
"HeartbeatTableMaintenanceJobHandler",
"TableMaintenanceQuarantineHandler",
@@ -5920,6 +5970,7 @@ mod tests {
let _: &PutTableMaintenanceConfigHandler = &PUT_TABLE_MAINTENANCE_CONFIG_HANDLER;
let _: &GetTableMaintenanceJobHandler = &GET_TABLE_MAINTENANCE_JOB_HANDLER;
let _: &GetTableMaintenanceSchedulerHandler = &GET_TABLE_MAINTENANCE_SCHEDULER_HANDLER;
let _: &RunTableMaintenanceSchedulerHandler = &RUN_TABLE_MAINTENANCE_SCHEDULER_HANDLER;
let _: &TableMaintenanceQuarantineHandler = &TABLE_MAINTENANCE_QUARANTINE_HANDLER;
let _: &ExportTableCatalogHandler = &EXPORT_TABLE_CATALOG_HANDLER;
let _: &ImportTableCatalogHandler = &IMPORT_TABLE_CATALOG_HANDLER;
@@ -5959,6 +6010,7 @@ mod tests {
assert_operation::<PutTableMaintenanceConfigHandler>();
assert_operation::<GetTableMaintenanceJobHandler>();
assert_operation::<GetTableMaintenanceSchedulerHandler>();
assert_operation::<RunTableMaintenanceSchedulerHandler>();
assert_operation::<RunTableMaintenanceWorkerHandler>();
assert_operation::<HeartbeatTableMaintenanceJobHandler>();
assert_operation::<TableMaintenanceQuarantineHandler>();
@@ -6033,6 +6085,33 @@ mod tests {
assert_eq!(compaction.max_rewrite_bytes_per_job, 10_737_418_240);
}
#[test]
fn table_maintenance_scheduler_run_request_uses_stable_default_scheduler_id() {
let request: TableMaintenanceSchedulerRunRequest =
serde_json::from_value(serde_json::json!({})).expect("scheduler run request should parse");
assert_eq!(request.scheduler_id(), "rustfs-maintenance-scheduler");
}
#[tokio::test]
async fn table_maintenance_scheduler_run_request_empty_body_uses_default_scheduler_id() {
let request: TableMaintenanceSchedulerRunRequest = read_json_body_or_default(Body::empty())
.await
.expect("bodyless scheduler run should use the default scheduler id");
assert_eq!(request.scheduler_id(), "rustfs-maintenance-scheduler");
}
#[test]
fn table_maintenance_scheduler_run_request_accepts_scheduler_id() {
let request: TableMaintenanceSchedulerRunRequest = serde_json::from_value(serde_json::json!({
"scheduler-id": "scheduler-a"
}))
.expect("scheduler run request should parse scheduler id");
assert_eq!(request.scheduler_id(), "scheduler-a");
}
#[test]
fn table_maintenance_worker_run_request_uses_stable_default_worker_id() {
let request: TableMaintenanceWorkerRunRequest =
rustfs/src/admin/route_policy.rs
+23 -1
View File
@@ -835,6 +835,12 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
GET_TABLE_LIFECYCLE,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler/run",
RUN_TABLE_MAINTENANCE,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/worker/run",
@@ -1094,6 +1100,12 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
GET_TABLE_LIFECYCLE,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler/run",
RUN_TABLE_MAINTENANCE,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/worker/run",
@@ -1342,7 +1354,7 @@ mod tests {
let table_specs = ADMIN_ROUTE_POLICY_SPECS
.iter()
.filter(|spec| spec.path().starts_with("/iceberg/v1") || spec.path().starts_with("/_iceberg/v1"));
assert_eq!(table_specs.count(), 88);
assert_eq!(table_specs.count(), 90);
assert_action(HttpMethod::Put, "/iceberg/v1/buckets/{warehouse}", SET_TABLE_BUCKET);
assert_action(HttpMethod::Get, "/_iceberg/v1/buckets/{warehouse}", GET_TABLE_BUCKET);
assert_action(HttpMethod::Get, "/iceberg/v1/{warehouse}/namespaces", GET_TABLE_NAMESPACE);
@@ -1477,6 +1489,16 @@ mod tests {
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler",
GET_TABLE_LIFECYCLE,
);
assert_action(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler/run",
RUN_TABLE_MAINTENANCE,
);
assert_action(
HttpMethod::Post,
"/_iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler/run",
RUN_TABLE_MAINTENANCE,
);
assert_action(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/worker/run",
rustfs/src/admin/route_registration_test.rs
+20
View File
@@ -424,6 +424,11 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler",
"/analytics/namespaces/sales/tables/orders/maintenance/scheduler",
),
table_route_sample(
Method::POST,
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler/run",
"/analytics/namespaces/sales/tables/orders/maintenance/scheduler/run",
),
table_route_sample(
Method::POST,
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/worker/run",
@@ -608,6 +613,11 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler",
"/analytics/namespaces/sales/tables/orders/maintenance/scheduler",
),
compat_table_route_sample(
Method::POST,
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler/run",
"/analytics/namespaces/sales/tables/orders/maintenance/scheduler/run",
),
compat_table_route_sample(
Method::POST,
"/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/worker/run",
@@ -864,6 +874,11 @@ fn test_register_routes_cover_representative_admin_paths() {
Method::GET,
&table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/scheduler"),
);
assert_route(
&router,
Method::POST,
&table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/scheduler/run"),
);
assert_route(
&router,
Method::POST,
@@ -1021,6 +1036,11 @@ fn test_register_routes_cover_representative_admin_paths() {
Method::GET,
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/scheduler"),
);
assert_route(
&router,
Method::POST,
&compat_table_catalog_path("/analytics/namespaces/sales/tables/orders/maintenance/scheduler/run"),
);
assert_route(
&router,
Method::POST,
rustfs/src/table_catalog.rs
+823 -31
View File
File diff suppressed because it is too large Load Diff