diff --git a/docs/architecture/s3-tables-support-matrix.md b/docs/architecture/s3-tables-support-matrix.md index 68796f37a..5be48bbea 100644 --- a/docs/architecture/s3-tables-support-matrix.md +++ b/docs/architecture/s3-tables-support-matrix.md @@ -117,6 +117,7 @@ catalog extension. | Strong backing migration contract | Supported as a contract | Diagnostics publish object-backed manifest state, recoverable commit-log WAL state, target backing type, replay requirements, and blockers. | | Durable backing migration dry-run | Supported | `GET /iceberg/v1/{warehouse}/catalog/migration` and the `/_iceberg/v1` alias inspect object-backed catalog inventory, commit recovery blockers, idempotency indexes, warehouse prefix index readiness, recommended actions, and rollback configuration without mutating catalog state. | | Disaster recovery rehearsal | Manual/live harness | `failure_coverage.py --print-disaster-recovery-rehearsal` generates an operator runbook covering catalog export, diagnostics, safe recovery repair, rollback/import, durable backing migration dry-run, post-recovery loadTable, and table data-plane policy probes. | +| Scale and fault rehearsal | Manual/live harness | `failure_coverage.py --print-scale-fault-rehearsal` generates an opt-in runbook for concurrent writer stress, maintenance scheduler lease recovery, durable backing cutover preflight, recovery/rollback/import under load, and post-run evidence capture. | | Strong KV/WAL backing cutover | Preview / controlled | Operators can explicitly select durable strong backing with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong`. Run the migration dry-run first and keep the object-backed catalog available for rollback. Object-only advanced operations fail closed in durable strong mode. | | Single active writer region | Supported policy | Diagnostics publish single-active-writer semantics and read-only replica limits. | | Active-active multi-region writes | Not claimed | A table must not accept independent concurrent writers in multiple active regions. | @@ -225,6 +226,15 @@ python3 scripts/table-catalog/failure_coverage.py \ --table events \ --table-warehouse-location s3://rustfs-s3table-smoke/tables/table-id \ --print-disaster-recovery-rehearsal +python3 scripts/table-catalog/failure_coverage.py \ + --warehouse rustfs-s3table-smoke \ + --namespace smoke \ + --table events \ + --table-warehouse-location s3://rustfs-s3table-smoke/tables/table-id \ + --writer-count 8 \ + --maintenance-worker-count 2 \ + --iteration-count 50 \ + --print-scale-fault-rehearsal ``` ## Release Claim Guidance @@ -237,8 +247,8 @@ Acceptable wording: > with PyIceberg smoke coverage, table-aware S3 data-plane policy checks, > controlled maintenance, catalog recovery diagnostics, manual conformance > input for Spark, Trino, DuckDB, Databend, and Snowflake, production-failure -> probe harnesses, disaster-recovery rehearsal probes, and a machine-readable -> production operations evidence guide. +> probe harnesses, disaster-recovery and scale/fault rehearsal probes, and a +> machine-readable production operations evidence guide. Do not claim: diff --git a/scripts/table-catalog/README.md b/scripts/table-catalog/README.md index 16e679689..425090a13 100644 --- a/scripts/table-catalog/README.md +++ b/scripts/table-catalog/README.md @@ -234,6 +234,15 @@ python3 scripts/table-catalog/failure_coverage.py \ --table events \ --table-warehouse-location s3://rustfs-s3table-smoke/tables/table-id \ --print-disaster-recovery-rehearsal +python3 scripts/table-catalog/failure_coverage.py \ + --warehouse rustfs-s3table-smoke \ + --namespace smoke \ + --table events \ + --table-warehouse-location s3://rustfs-s3table-smoke/tables/table-id \ + --writer-count 8 \ + --maintenance-worker-count 2 \ + --iteration-count 50 \ + --print-scale-fault-rehearsal ``` The generated probe plan covers stale-token commit conflicts, missing metadata @@ -343,6 +352,48 @@ manual-review diagnostics, stale rollback/import conflicts, and data-plane policy failures as fail-closed results that require operator investigation before cutover or release claims. +## Scale And Fault Rehearsal + +The scale and fault rehearsal plan is a machine-readable opt-in runbook for +production-style stress and failure evidence. It does not run the stress test by +itself and does not promote compatibility or scale claims without recorded live +results. + +Generate the rehearsal plan: + +```bash +python3 scripts/table-catalog/failure_coverage.py \ + --warehouse rustfs-s3table-smoke \ + --namespace smoke \ + --table events \ + --table-warehouse-location s3://rustfs-s3table-smoke/tables/table-id \ + --writer-count 8 \ + --maintenance-worker-count 2 \ + --iteration-count 50 \ + --catalog-backing durable-strong \ + --print-scale-fault-rehearsal +``` + +The plan is gated for CI by: + +```text +RUSTFS_TABLE_CATALOG_SCALE_FAULT_REHEARSAL=1 +``` + +The generated phases cover: + +- concurrent writer cohorts that must produce a single successful commit per + conflict group and retryable conflicts for stale writers +- maintenance scheduler queueing, worker claim, lease expiry recovery, and stale + plan fail-closed checks +- durable backing migration dry-run and catalog backup evidence before cutover +- safe recovery repair, rollback, and import conflict behavior after stress +- final `loadTable`, table data-plane policy, and operator artifact capture + +Record the RustFS build, catalog backing mode, writer count, worker count, +iteration count, final metadata location, conflict counts, recovered leases, and +failed-closed operations before using the run as release evidence. + ## Production Operations Guide The engine helper can print a machine-readable operations guide that ties client diff --git a/scripts/table-catalog/failure_coverage.py b/scripts/table-catalog/failure_coverage.py index f7c905a49..bed9915b9 100644 --- a/scripts/table-catalog/failure_coverage.py +++ b/scripts/table-catalog/failure_coverage.py @@ -209,6 +209,16 @@ def rehearsal_phase(name: str, objective: str, steps: list[dict[str, Any]]) -> d } +def positive_int(value: str) -> int: + try: + parsed = int(value) + except ValueError as err: + raise argparse.ArgumentTypeError("value must be a positive integer") from err + if parsed <= 0: + raise argparse.ArgumentTypeError("value must be a positive integer") + return parsed + + def disaster_recovery_rehearsal_plan( *, warehouse: str, @@ -360,6 +370,213 @@ def disaster_recovery_rehearsal_plan( } +def scale_fault_rehearsal_plan( + *, + warehouse: str, + namespace: str, + table: str, + rest_path: str = "/iceberg", + table_warehouse_location: str | None = None, + writer_count: int = 8, + maintenance_worker_count: int = 2, + iteration_count: int = 50, + catalog_backing: str = "object-backed-or-durable-strong", +) -> dict[str, Any]: + table_endpoint = table_path(warehouse, namespace, table, rest_path=rest_path) + table_warehouse_location = table_warehouse_location or f"s3://{warehouse}/tables/table-id" + return { + "mode": "manual-or-ci-optional", + "ci_gate": "RUSTFS_TABLE_CATALOG_SCALE_FAULT_REHEARSAL=1", + "parameters": { + "writer_count": writer_count, + "maintenance_worker_count": maintenance_worker_count, + "iteration_count": iteration_count, + "catalog_backing": catalog_backing, + }, + "preconditions": [ + "run against a disposable table or a table restored from backup", + "record the RustFS build, catalog backing mode, node count, and object backend before starting", + "capture baseline loadTable metadata location, version token, generation, and warehouse prefix", + "enable only opt-in clients and workers for this rehearsal run", + ], + "expected_invariants": [ + "current metadata generation is monotonic", + "each conflicting writer cohort has one winner and retryable conflicts for stale writers", + "maintenance jobs never delete reachable table objects", + "scheduler and worker leases recover without duplicate active jobs for the same table", + "durable backing migration stays blocked while recovery or replay blockers exist", + "rollback/import under load either commits a validated metadata location or conflicts without pointer movement", + ], + "phases": [ + rehearsal_phase( + "concurrent-commit-stress", + "Stress single-table CAS behavior while preserving pointer and generation invariants.", + [ + probe_step( + "spawn-concurrent-writers", + "CLIENT-STRESS", + table_endpoint, + "single-winner-per-conflict-cohort", + "run concurrent append/commit attempts and record winner commit id, conflicts, final metadata location, token, and generation", + { + "writer-count": writer_count, + "iteration-count": iteration_count, + }, + ), + probe_step( + "load-table-after-writer-cohort", + "GET", + table_endpoint, + "200", + "loadTable returns one current metadata location and a generation that advanced monotonically", + ), + probe_step( + "diagnostics-after-writer-cohort", + "GET", + table_path(warehouse, namespace, table, "/catalog/diagnostics", rest_path), + "200", + "diagnostics report no unexpected commit recovery blockers after the writer cohort", + ), + ], + ), + rehearsal_phase( + "maintenance-scheduler-failover", + "Exercise queued maintenance, worker claim, expired lease recovery, and stale plan rejection.", + [ + probe_step( + "queue-maintenance-job", + "POST", + table_path(warehouse, namespace, table, "/maintenance/scheduler/run", rest_path), + "200", + "scheduler queues at most one active maintenance job for the table", + { + "scheduler-id": "scale-fault-rehearsal-scheduler", + }, + ), + probe_step( + "claim-worker-job", + "POST", + table_path(warehouse, namespace, table, "/maintenance/worker/run", rest_path), + "200-or-409", + "one worker claims or executes the queued job while peer workers observe backpressure or no work", + { + "worker-count": maintenance_worker_count, + }, + ), + probe_step( + "recover-expired-lease", + "POST", + table_path(warehouse, namespace, table, "/maintenance/scheduler/run", rest_path), + "200-or-409", + "expired scheduler or worker leases are recovered without creating duplicate active jobs", + { + "scheduler-id": "scale-fault-rehearsal-recovery", + }, + ), + probe_step( + "stale-maintenance-plan-check", + "POST", + table_path(warehouse, namespace, table, "/maintenance/metadata", rest_path), + "409-or-manual-review", + "stale maintenance plans fail closed before deleting objects or committing metadata", + { + "dry-run": False, + "expected-metadata-location": "stale-metadata-location-from-before-writer-cohort", + }, + ), + ], + ), + rehearsal_phase( + "durable-backing-cutover-under-load", + "Verify durable backing cutover preflight remains explainable under recent write and maintenance churn.", + [ + probe_step( + "capture-cutover-backup", + "OPERATOR", + table_path(warehouse, namespace, table, "/catalog/export", rest_path), + "catalog-backup-recorded", + "record catalog-backup artifacts, current metadata location, idempotency indexes, and rollback mode before cutover", + ), + probe_step( + "migration-dry-run-before-cutover", + "GET", + warehouse_path(warehouse, "/catalog/migration", rest_path), + "200", + "migration dry-run reports inventory, replay blockers, idempotency blockers, and recommended actions", + ), + ], + ), + rehearsal_phase( + "recovery-rollback-import-under-load", + "Exercise recovery and operator-selected rollback/import paths after stress activity.", + [ + probe_step( + "safe-recovery-repair-under-load", + "POST", + table_path(warehouse, namespace, table, "/catalog/recovery", rest_path), + "200", + "safe repair handles recoverable idempotency gaps without moving the table pointer", + { + "mode": "safe-repair", + }, + ), + probe_step( + "rollback-conflict-check", + "POST", + table_path(warehouse, namespace, table, "/catalog/rollback", rest_path), + "200-or-409", + "rollback either commits the operator-selected metadata location or conflicts without pointer movement", + { + "metadata-location": "metadata-location-from-backup", + "version-token": "current-version-token-from-load-table", + }, + ), + probe_step( + "import-conflict-check", + "POST", + table_path(warehouse, namespace, table, "/catalog/import", rest_path), + "200-or-409", + "import validates metadata identity and conflicts without pointer/token/generation movement when stale", + { + "metadata-location": "metadata-location-from-backup", + "properties": { + "rehearsal-source": "scale-fault-backup", + }, + }, + ), + ], + ), + rehearsal_phase( + "post-run-evidence", + "Record the evidence needed before any scale or fault claim can be promoted.", + [ + probe_step( + "load-table-after-scale-fault-run", + "GET", + table_endpoint, + "200", + "loadTable returns the final intended metadata location, version token, and generation", + ), + probe_step( + "table-data-plane-policy-probe", + "S3-PROBE", + table_warehouse_location, + "inside-allowed-outside-denied", + "ordinary S3 object access still maps table warehouse objects to the table policy boundary", + ), + probe_step( + "capture-run-artifacts", + "EVIDENCE", + "operator-recorded-artifacts", + "recorded", + "record RustFS build, catalog backing, writer count, worker count, iteration count, final metadata location, observed conflicts, recovered leases, and failed-closed operations", + ), + ], + ), + ], + } + + def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Print RustFS S3 Tables production failure coverage helpers.") parser.add_argument("--warehouse", default="rustfs-s3table-smoke") @@ -367,9 +584,14 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--table", default="events") parser.add_argument("--rest-path", default="/iceberg") parser.add_argument("--table-warehouse-location") + parser.add_argument("--writer-count", type=positive_int, default=8) + parser.add_argument("--maintenance-worker-count", type=positive_int, default=2) + parser.add_argument("--iteration-count", type=positive_int, default=50) + parser.add_argument("--catalog-backing", default="object-backed-or-durable-strong") parser.add_argument("--print-failure-matrix", action="store_true") parser.add_argument("--print-failure-probes", action="store_true") parser.add_argument("--print-disaster-recovery-rehearsal", action="store_true") + parser.add_argument("--print-scale-fault-rehearsal", action="store_true") return parser.parse_args(argv) @@ -419,6 +641,24 @@ def run(args: argparse.Namespace, output: StringIO | None = None) -> None: output, ) printed = True + if args.print_scale_fault_rehearsal: + print_json( + { + "scale_fault_rehearsal": scale_fault_rehearsal_plan( + warehouse=args.warehouse, + namespace=args.namespace, + table=args.table, + rest_path=args.rest_path, + table_warehouse_location=args.table_warehouse_location, + writer_count=args.writer_count, + maintenance_worker_count=args.maintenance_worker_count, + iteration_count=args.iteration_count, + catalog_backing=args.catalog_backing, + ) + }, + output, + ) + printed = True if not printed: print_json({"production_failure_coverage": production_failure_matrix()}, output) diff --git a/scripts/table-catalog/test_failure_coverage.py b/scripts/table-catalog/test_failure_coverage.py index 27dbcf2a2..8e015939c 100644 --- a/scripts/table-catalog/test_failure_coverage.py +++ b/scripts/table-catalog/test_failure_coverage.py @@ -5,6 +5,8 @@ from __future__ import annotations import json import unittest +from contextlib import redirect_stderr +from io import StringIO import failure_coverage @@ -159,6 +161,103 @@ class FailureCoverageTest(unittest.TestCase): first_step = document["disaster_recovery_rehearsal"]["phases"][0]["steps"][0] self.assertEqual(first_step["path"], "/_iceberg/v1/lake/namespaces/sales/tables/orders/catalog/export") + def test_scale_fault_rehearsal_plan_covers_load_and_fault_paths(self) -> None: + rehearsal = failure_coverage.scale_fault_rehearsal_plan( + warehouse="lake", + namespace="sales", + table="orders", + rest_path="/iceberg", + table_warehouse_location="s3://lake/tables/orders", + writer_count=8, + maintenance_worker_count=3, + iteration_count=50, + catalog_backing="durable-strong", + ) + + self.assertEqual(rehearsal["mode"], "manual-or-ci-optional") + self.assertEqual(rehearsal["ci_gate"], "RUSTFS_TABLE_CATALOG_SCALE_FAULT_REHEARSAL=1") + self.assertEqual(rehearsal["parameters"]["writer_count"], 8) + self.assertEqual(rehearsal["parameters"]["maintenance_worker_count"], 3) + self.assertEqual(rehearsal["parameters"]["iteration_count"], 50) + self.assertEqual(rehearsal["parameters"]["catalog_backing"], "durable-strong") + self.assertIn("current metadata generation is monotonic", rehearsal["expected_invariants"]) + self.assertIn("maintenance jobs never delete reachable table objects", rehearsal["expected_invariants"]) + + phases = {phase["name"]: phase for phase in rehearsal["phases"]} + self.assertIn("concurrent-commit-stress", phases) + self.assertIn("maintenance-scheduler-failover", phases) + self.assertIn("durable-backing-cutover-under-load", phases) + self.assertIn("recovery-rollback-import-under-load", phases) + self.assertIn("post-run-evidence", phases) + + commit_steps = {step["name"]: step for step in phases["concurrent-commit-stress"]["steps"]} + self.assertEqual(commit_steps["spawn-concurrent-writers"]["method"], "CLIENT-STRESS") + self.assertEqual(commit_steps["spawn-concurrent-writers"]["expected_status"], "single-winner-per-conflict-cohort") + self.assertEqual(commit_steps["load-table-after-writer-cohort"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders") + + scheduler_steps = {step["name"]: step for step in phases["maintenance-scheduler-failover"]["steps"]} + self.assertEqual(scheduler_steps["queue-maintenance-job"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders/maintenance/scheduler/run") + self.assertEqual(scheduler_steps["claim-worker-job"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders/maintenance/worker/run") + self.assertEqual(scheduler_steps["recover-expired-lease"]["expected_status"], "200-or-409") + + cutover_steps = {step["name"]: step for step in phases["durable-backing-cutover-under-load"]["steps"]} + self.assertEqual(cutover_steps["migration-dry-run-before-cutover"]["path"], "/iceberg/v1/lake/catalog/migration") + self.assertIn("catalog-backup", cutover_steps["capture-cutover-backup"]["expected_behavior"]) + + recovery_steps = {step["name"]: step for step in phases["recovery-rollback-import-under-load"]["steps"]} + self.assertEqual(recovery_steps["safe-recovery-repair-under-load"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders/catalog/recovery") + self.assertEqual(recovery_steps["rollback-conflict-check"]["expected_status"], "200-or-409") + + evidence_steps = {step["name"]: step for step in phases["post-run-evidence"]["steps"]} + self.assertEqual(evidence_steps["table-data-plane-policy-probe"]["path"], "s3://lake/tables/orders") + self.assertEqual(evidence_steps["capture-run-artifacts"]["method"], "EVIDENCE") + + def test_cli_prints_scale_fault_rehearsal_plan(self) -> None: + payload = failure_coverage.cli_json( + [ + "--warehouse", + "lake", + "--namespace", + "sales", + "--table", + "orders", + "--rest-path", + "/_iceberg", + "--table-warehouse-location", + "s3://lake/tables/orders", + "--writer-count", + "6", + "--maintenance-worker-count", + "2", + "--iteration-count", + "25", + "--catalog-backing", + "durable-strong", + "--print-scale-fault-rehearsal", + ] + ) + document = json.loads(payload) + + rehearsal = document["scale_fault_rehearsal"] + self.assertEqual(rehearsal["parameters"]["writer_count"], 6) + self.assertEqual(rehearsal["parameters"]["maintenance_worker_count"], 2) + self.assertEqual(rehearsal["parameters"]["iteration_count"], 25) + first_step = rehearsal["phases"][0]["steps"][0] + self.assertEqual(first_step["path"], "/_iceberg/v1/lake/namespaces/sales/tables/orders") + + def test_cli_rejects_non_positive_scale_fault_counts(self) -> None: + invalid_count_flags = [ + ("--writer-count", "0"), + ("--maintenance-worker-count", "-1"), + ("--iteration-count", "0"), + ] + + for flag, value in invalid_count_flags: + with self.subTest(flag=flag): + with redirect_stderr(StringIO()): + with self.assertRaises(SystemExit): + failure_coverage.cli_json([flag, value, "--print-scale-fault-rehearsal"]) + if __name__ == "__main__": unittest.main()