gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(table-catalog): add maintenance scheduler guardrails (#4123)

Browse Source
This commit is contained in:
Henry Guo
2026-07-01 22:31:19 +08:00
committed by GitHub
parent f7769884ff
commit a305160087
8 changed files with 536 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/table-catalog/README.md
+5 -4
View File
@@ -229,8 +229,9 @@ The smoke test also probes catalog-backed advanced Iceberg surfaces:
`main` cannot be deleted
- Iceberg views support basic create, list, load, replace, existence check, and
drop routes with persisted view metadata and view-scoped authorization
- metadata maintenance supports safe dry-run planning and controlled worker
execution checks
- metadata maintenance supports safe dry-run planning, controlled worker
execution checks, and a scheduler status report with disabled, paused,
backpressure, retry, quarantine, and audit-timeline state
- catalog diagnostics exposes the table recovery and consistency state used by
operators
- catalog export and diagnostics expose the current catalog backing manifest,
@@ -333,10 +334,10 @@ Unsupported behavior is documented instead of hidden behind internal errors. The
current unsupported inventory is:
- credential vending: automated after table bootstrap with exact-prefix validation and a data-plane scope probe; full no-long-term-data-credential bootstrap is not claimed
- background maintenance worker: controlled run-once and heartbeat endpoints are registered; continuous in-process scheduling is not claimed
- background maintenance worker: controlled run-once, heartbeat, and scheduler status endpoints are registered; disabled/paused/backpressure/retry/quarantine/audit-timeline state is machine-readable; continuous in-process scheduling is not claimed
- manifest/data reachability cleanup: metadata maintenance reads manifest-list and manifest Avro references, reports manifest/data/delete reachability, and deletes only unreferenced table objects that pass the safety window
- snapshot expiration dry-run planning and manual catalog commit: supported through metadata maintenance reports
- automatic maintenance scheduling: external scheduler hook supported through the worker run endpoint; built-in periodic scheduling is not claimed
- automatic maintenance scheduling: external scheduler hook supported through the worker run endpoint and scheduler status report; built-in periodic scheduling is not claimed
- compaction rewrite: controlled run-once support for partition-local Parquet binpack through metadata maintenance; built-in periodic scheduling, sort compaction, delete-file rewrite, and row-level compaction are not claimed
- row-level delete/update/merge commits: standard catalog commit validates append, overwrite, delete, and replace snapshot manifests for table-warehouse scope, referenced object existence, current-live-file deletes, and stale add/delete conflicts; end-to-end SQL DML client coverage remains a compatibility validation item
- external catalog bridges: metadata import/register and operator-supplied metadata pointer sync are supported for Polaris/Glue/DLF/Hive identity boundaries; online vendor SDK polling and policy mirroring are not claimed
scripts/table-catalog/pyiceberg_smoke.py
+9 -2
View File
@@ -182,8 +182,8 @@ UNSUPPORTED_INVENTORY: list[dict[str, str]] = [
"capability": "background-maintenance-worker",
"status": "controlled-run-once-supported",
"roadmap_area": "maintenance-worker",
"catalog_endpoint": "POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/maintenance/worker/run",
"expected_behavior": "background-enabled maintenance can be driven by the worker run endpoint with current-job backpressure, retry deferral, lease expiry recovery, and heartbeat updates; built-in periodic scheduling is not claimed",
"catalog_endpoint": "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/maintenance/scheduler",
"expected_behavior": "background-enabled maintenance can be driven by the worker run endpoint and inspected through the scheduler status endpoint with disabled/paused state, current-job backpressure, retry deferral, quarantine boundary, audit timeline, lease expiry recovery, and heartbeat updates; built-in periodic scheduling is not claimed",
},
{
"capability": "manifest-data-reachability-cleanup",
@@ -1049,6 +1049,7 @@ def run_view_probe(args: argparse.Namespace, deps: RuntimeDeps) -> None:
def run_maintenance_probe(args: argparse.Namespace, deps: RuntimeDeps) -> None:
config_path = table_endpoint_path(args, "/maintenance/config")
scheduler_path = table_endpoint_path(args, "/maintenance/scheduler")
signed_rest_request(args, deps, "PUT", config_path, default_maintenance_config())
config = signed_rest_request(args, deps, "GET", config_path)
if config.get("version") != TABLE_MAINTENANCE_CONFIG_VERSION:
@@ -1065,6 +1066,12 @@ def run_maintenance_probe(args: argparse.Namespace, deps: RuntimeDeps) -> None:
if not isinstance(job, dict) or not job.get("job-id"):
raise RuntimeError("maintenance metadata endpoint did not return a job id")
signed_rest_request(args, deps, "GET", table_endpoint_path(args, f"/maintenance/jobs/{job['job-id']}"))
scheduler = signed_rest_request(args, deps, "GET", scheduler_path)
expected_scheduler_statuses = {"READY", "DISABLED", "PAUSED", "BACKPRESSURED", "RETRY_DEFERRED", "QUARANTINED"}
if scheduler.get("status") not in expected_scheduler_statuses:
raise RuntimeError("maintenance scheduler endpoint did not return a stable scheduler status")
if "audit_timeline" not in scheduler:
raise RuntimeError("maintenance scheduler endpoint did not return an audit timeline")
worker = signed_rest_request(args, deps, "POST", table_endpoint_path(args, "/maintenance/worker/run"), {})
worker_job = worker.get("job")
scripts/table-catalog/test_pyiceberg_smoke.py
+7
View File
@@ -294,6 +294,7 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
config_path = pyiceberg_smoke.table_endpoint_path(args, "/maintenance/config")
maintenance_path = pyiceberg_smoke.table_endpoint_path(args, "/maintenance/metadata")
job_path = pyiceberg_smoke.table_endpoint_path(args, "/maintenance/jobs/job-1")
scheduler_path = pyiceberg_smoke.table_endpoint_path(args, "/maintenance/scheduler")
worker_path = pyiceberg_smoke.table_endpoint_path(args, "/maintenance/worker/run")
def fake_signed_request(
@@ -311,6 +312,8 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
return {"job": {"job-id": "job-1"}}
if (method, path) == ("GET", job_path):
return {"job": {"job-id": "job-1", "status": "SUCCESSFUL"}}
if (method, path) == ("GET", scheduler_path):
return {"status": "DISABLED", "audit_timeline": [{"job_id": "job-1"}]}
if (method, path) == ("POST", worker_path):
return {"job": {"status": "UNKNOWN"}}
raise AssertionError(f"unexpected REST request: {method} {path}")
@@ -330,6 +333,7 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
view_path = pyiceberg_smoke.view_endpoint_path(args)
config_path = pyiceberg_smoke.table_endpoint_path(args, "/maintenance/config")
maintenance_path = pyiceberg_smoke.table_endpoint_path(args, "/maintenance/metadata")
scheduler_path = pyiceberg_smoke.table_endpoint_path(args, "/maintenance/scheduler")
worker_path = pyiceberg_smoke.table_endpoint_path(args, "/maintenance/worker/run")
diagnostics_path = pyiceberg_smoke.table_endpoint_path(args, "/catalog/diagnostics")
@@ -369,6 +373,8 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
return {"job": {"job-id": "job-1"}}
if (method, path) == ("GET", pyiceberg_smoke.table_endpoint_path(args, "/maintenance/jobs/job-1")):
return {"job": {"job-id": "job-1", "status": "SUCCESSFUL"}}
if (method, path) == ("GET", scheduler_path):
return {"status": "DISABLED", "audit_timeline": [{"job_id": "job-1"}]}
if (method, path) == ("POST", worker_path):
return {"job": {"status": "PAUSED"}}
if (method, path) == ("GET", diagnostics_path):
@@ -398,6 +404,7 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
)
self.assertIn(("GET", metadata_location_path, None), calls)
self.assertIn(("GET", diagnostics_path, None), calls)
self.assertIn(("GET", scheduler_path, None), calls)
self.assertIn(("POST", worker_path, {}), calls)
def test_table_ref_probe_force_deletes_smoke_ref_after_validation_failure(self) -> None: