gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(table-catalog): add maintenance audit timeline (#4207)

Browse Source 
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-07-03 00:36:17 +08:00
committed by GitHub
parent 9dfeffc4c1
commit 782e73ca92
5 changed files with 376 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/table-catalog/README.md
+5 -3
View File
@@ -232,8 +232,10 @@ The smoke test also probes catalog-backed advanced Iceberg surfaces:
drop routes with persisted view metadata and view-scoped authorization
- metadata maintenance supports safe dry-run planning, controlled worker
execution checks, and a scheduler status report with disabled, paused,
backpressure, retry, quarantine, and audit-timeline state; quarantined jobs
can be inspected, released, retried, or abandoned through an operator endpoint
backpressure, retry, quarantine, and audit-timeline state; job reports expose
structured audit events for planning, worker transitions, heartbeat updates,
lease expiry, and mutating quarantine operations; quarantined jobs can be
inspected, released, retried, or abandoned through an operator endpoint
- catalog diagnostics exposes the table recovery and consistency state used by
operators
- catalog export and diagnostics expose the current catalog backing manifest,
@@ -336,7 +338,7 @@ Unsupported behavior is documented instead of hidden behind internal errors. The
current unsupported inventory is:
- credential vending: automated after table bootstrap with exact-prefix validation and a data-plane scope probe; full no-long-term-data-credential bootstrap is not claimed
- background maintenance worker: controlled run-once, heartbeat, quarantine operation, and scheduler status endpoints are registered; disabled/paused/backpressure/retry/quarantine/audit-timeline state is machine-readable; continuous in-process scheduling is not claimed
- background maintenance worker: controlled run-once, heartbeat, quarantine operation, and scheduler status endpoints are registered; disabled/paused/backpressure/retry/quarantine/audit-timeline state and per-job audit events are machine-readable; continuous in-process scheduling is not claimed
- manifest/data reachability cleanup: metadata maintenance reads manifest-list and manifest Avro references, reports manifest/data/delete reachability, and deletes only unreferenced table objects that pass the safety window
- snapshot expiration dry-run planning and manual catalog commit: supported through metadata maintenance reports
- automatic maintenance scheduling: external scheduler hook supported through the worker run endpoint and scheduler status report; built-in periodic scheduling is not claimed
scripts/table-catalog/pyiceberg_smoke.py
+15 -3
View File
@@ -183,7 +183,7 @@ UNSUPPORTED_INVENTORY: list[dict[str, str]] = [
"status": "controlled-run-once-supported",
"roadmap_area": "maintenance-worker",
"catalog_endpoint": "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/maintenance/scheduler",
"expected_behavior": "background-enabled maintenance can be driven by the worker run endpoint and inspected through the scheduler status endpoint with disabled/paused state, current-job backpressure, retry deferral, quarantine boundary, audit timeline, lease expiry recovery, heartbeat updates, and operator quarantine inspect/release/retry/abandon actions; built-in periodic scheduling is not claimed",
"expected_behavior": "background-enabled maintenance can be driven by the worker run endpoint and inspected through the scheduler status endpoint with disabled/paused state, current-job backpressure, retry deferral, quarantine boundary, audit timeline, per-job audit events, lease expiry recovery, heartbeat updates, and operator quarantine inspect/release/retry/abandon actions; built-in periodic scheduling is not claimed",
},
{
"capability": "manifest-data-reachability-cleanup",
@@ -1065,7 +1065,14 @@ def run_maintenance_probe(args: argparse.Namespace, deps: RuntimeDeps) -> None:
job = report.get("job")
if not isinstance(job, dict) or not job.get("job-id"):
raise RuntimeError("maintenance metadata endpoint did not return a job id")
signed_rest_request(args, deps, "GET", table_endpoint_path(args, f"/maintenance/jobs/{job['job-id']}"))
audit_events = report.get("audit-events")
if not isinstance(audit_events, list) or not audit_events:
raise RuntimeError("maintenance metadata endpoint did not return audit events")
if not any(isinstance(event, dict) and event.get("action") == "PLANNED" for event in audit_events):
raise RuntimeError("maintenance metadata endpoint did not report a planning audit event")
job_report = signed_rest_request(args, deps, "GET", table_endpoint_path(args, f"/maintenance/jobs/{job['job-id']}"))
if not isinstance(job_report.get("audit-events"), list):
raise RuntimeError("maintenance job endpoint did not return audit events")
quarantine = signed_rest_request(
args,
deps,
@@ -1079,14 +1086,19 @@ def run_maintenance_probe(args: argparse.Namespace, deps: RuntimeDeps) -> None:
expected_scheduler_statuses = {"READY", "DISABLED", "PAUSED", "BACKPRESSURED", "RETRY_DEFERRED", "QUARANTINED"}
if scheduler.get("status") not in expected_scheduler_statuses:
raise RuntimeError("maintenance scheduler endpoint did not return a stable scheduler status")
if "audit_timeline" not in scheduler:
scheduler_timeline = scheduler.get("audit_timeline")
if not isinstance(scheduler_timeline, list):
raise RuntimeError("maintenance scheduler endpoint did not return an audit timeline")
if scheduler_timeline and not isinstance(scheduler_timeline[0].get("audit-events"), list):
raise RuntimeError("maintenance scheduler audit timeline did not include job audit events")
worker = signed_rest_request(args, deps, "POST", table_endpoint_path(args, "/maintenance/worker/run"), {})
worker_job = worker.get("job")
expected_statuses = {"DISABLED", "PAUSED", "FAILED", "SUCCESSFUL", "RUNNING"}
if not isinstance(worker_job, dict) or worker_job.get("status") not in expected_statuses:
raise RuntimeError("maintenance worker endpoint did not return a stable job status")
if not isinstance(worker.get("audit-events"), list):
raise RuntimeError("maintenance worker endpoint did not return audit events")
def run_catalog_api_probes(args: argparse.Namespace, deps: RuntimeDeps) -> None:
scripts/table-catalog/test_pyiceberg_smoke.py
+8 -8
View File
@@ -310,15 +310,15 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
if (method, path) == ("GET", config_path):
return {"version": 1}
if (method, path) == ("POST", maintenance_path):
return {"job": {"job-id": "job-1"}}
return {"job": {"job-id": "job-1"}, "audit-events": [{"action": "PLANNED"}]}
if (method, path) == ("GET", job_path):
return {"job": {"job-id": "job-1", "status": "SUCCESSFUL"}}
return {"job": {"job-id": "job-1", "status": "SUCCESSFUL"}, "audit-events": [{"action": "PLANNED"}]}
if (method, path) == ("POST", quarantine_path):
return {"action": "INSPECT", "report": {"job": {"job-id": "job-1"}}}
if (method, path) == ("GET", scheduler_path):
return {"status": "DISABLED", "audit_timeline": [{"job_id": "job-1"}]}
return {"status": "DISABLED", "audit_timeline": [{"job_id": "job-1", "audit-events": [{"action": "PLANNED"}]}]}
if (method, path) == ("POST", worker_path):
return {"job": {"status": "UNKNOWN"}}
return {"job": {"status": "UNKNOWN"}, "audit-events": [{"action": "WORKER_CONTROL"}]}
raise AssertionError(f"unexpected REST request: {method} {path}")
with mock.patch.object(pyiceberg_smoke, "signed_rest_request", side_effect=fake_signed_request):
@@ -374,15 +374,15 @@ class PyIcebergSmokeConfigTest(unittest.TestCase):
if (method, path) == ("GET", config_path):
return {"version": 1}
if (method, path) == ("POST", maintenance_path):
return {"job": {"job-id": "job-1"}}
return {"job": {"job-id": "job-1"}, "audit-events": [{"action": "PLANNED"}]}
if (method, path) == ("GET", pyiceberg_smoke.table_endpoint_path(args, "/maintenance/jobs/job-1")):
return {"job": {"job-id": "job-1", "status": "SUCCESSFUL"}}
return {"job": {"job-id": "job-1", "status": "SUCCESSFUL"}, "audit-events": [{"action": "PLANNED"}]}
if (method, path) == ("POST", quarantine_path):
return {"action": "INSPECT", "report": {"job": {"job-id": "job-1"}}}
if (method, path) == ("GET", scheduler_path):
return {"status": "DISABLED", "audit_timeline": [{"job_id": "job-1"}]}
return {"status": "DISABLED", "audit_timeline": [{"job_id": "job-1", "audit-events": [{"action": "PLANNED"}]}]}
if (method, path) == ("POST", worker_path):
return {"job": {"status": "PAUSED"}}
return {"job": {"status": "PAUSED"}, "audit-events": [{"action": "WORKER_CONTROL"}]}
if (method, path) == ("GET", diagnostics_path):
return {"status": "ok"}
raise AssertionError(f"unexpected REST request: {method} {path}")