mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
test(table-catalog): add scale fault rehearsal (#4359)
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
Henry Guo
@@ -234,6 +234,15 @@ python3 scripts/table-catalog/failure_coverage.py \
|
||||
--table events \
|
||||
--table-warehouse-location s3://rustfs-s3table-smoke/tables/table-id \
|
||||
--print-disaster-recovery-rehearsal
|
||||
python3 scripts/table-catalog/failure_coverage.py \
|
||||
--warehouse rustfs-s3table-smoke \
|
||||
--namespace smoke \
|
||||
--table events \
|
||||
--table-warehouse-location s3://rustfs-s3table-smoke/tables/table-id \
|
||||
--writer-count 8 \
|
||||
--maintenance-worker-count 2 \
|
||||
--iteration-count 50 \
|
||||
--print-scale-fault-rehearsal
|
||||
```
|
||||
|
||||
The generated probe plan covers stale-token commit conflicts, missing metadata
|
||||
@@ -343,6 +352,48 @@ manual-review diagnostics, stale rollback/import conflicts, and data-plane
|
||||
policy failures as fail-closed results that require operator investigation
|
||||
before cutover or release claims.
|
||||
|
||||
## Scale And Fault Rehearsal
|
||||
|
||||
The scale and fault rehearsal plan is a machine-readable opt-in runbook for
|
||||
production-style stress and failure evidence. It does not run the stress test by
|
||||
itself and does not promote compatibility or scale claims without recorded live
|
||||
results.
|
||||
|
||||
Generate the rehearsal plan:
|
||||
|
||||
```bash
|
||||
python3 scripts/table-catalog/failure_coverage.py \
|
||||
--warehouse rustfs-s3table-smoke \
|
||||
--namespace smoke \
|
||||
--table events \
|
||||
--table-warehouse-location s3://rustfs-s3table-smoke/tables/table-id \
|
||||
--writer-count 8 \
|
||||
--maintenance-worker-count 2 \
|
||||
--iteration-count 50 \
|
||||
--catalog-backing durable-strong \
|
||||
--print-scale-fault-rehearsal
|
||||
```
|
||||
|
||||
The plan is gated for CI by:
|
||||
|
||||
```text
|
||||
RUSTFS_TABLE_CATALOG_SCALE_FAULT_REHEARSAL=1
|
||||
```
|
||||
|
||||
The generated phases cover:
|
||||
|
||||
- concurrent writer cohorts that must produce a single successful commit per
|
||||
conflict group and retryable conflicts for stale writers
|
||||
- maintenance scheduler queueing, worker claim, lease expiry recovery, and stale
|
||||
plan fail-closed checks
|
||||
- durable backing migration dry-run and catalog backup evidence before cutover
|
||||
- safe recovery repair, rollback, and import conflict behavior after stress
|
||||
- final `loadTable`, table data-plane policy, and operator artifact capture
|
||||
|
||||
Record the RustFS build, catalog backing mode, writer count, worker count,
|
||||
iteration count, final metadata location, conflict counts, recovered leases, and
|
||||
failed-closed operations before using the run as release evidence.
|
||||
|
||||
## Production Operations Guide
|
||||
|
||||
The engine helper can print a machine-readable operations guide that ties client
|
||||
|
||||
@@ -209,6 +209,16 @@ def rehearsal_phase(name: str, objective: str, steps: list[dict[str, Any]]) -> d
|
||||
}
|
||||
|
||||
|
||||
def positive_int(value: str) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except ValueError as err:
|
||||
raise argparse.ArgumentTypeError("value must be a positive integer") from err
|
||||
if parsed <= 0:
|
||||
raise argparse.ArgumentTypeError("value must be a positive integer")
|
||||
return parsed
|
||||
|
||||
|
||||
def disaster_recovery_rehearsal_plan(
|
||||
*,
|
||||
warehouse: str,
|
||||
@@ -360,6 +370,213 @@ def disaster_recovery_rehearsal_plan(
|
||||
}
|
||||
|
||||
|
||||
def scale_fault_rehearsal_plan(
|
||||
*,
|
||||
warehouse: str,
|
||||
namespace: str,
|
||||
table: str,
|
||||
rest_path: str = "/iceberg",
|
||||
table_warehouse_location: str | None = None,
|
||||
writer_count: int = 8,
|
||||
maintenance_worker_count: int = 2,
|
||||
iteration_count: int = 50,
|
||||
catalog_backing: str = "object-backed-or-durable-strong",
|
||||
) -> dict[str, Any]:
|
||||
table_endpoint = table_path(warehouse, namespace, table, rest_path=rest_path)
|
||||
table_warehouse_location = table_warehouse_location or f"s3://{warehouse}/tables/table-id"
|
||||
return {
|
||||
"mode": "manual-or-ci-optional",
|
||||
"ci_gate": "RUSTFS_TABLE_CATALOG_SCALE_FAULT_REHEARSAL=1",
|
||||
"parameters": {
|
||||
"writer_count": writer_count,
|
||||
"maintenance_worker_count": maintenance_worker_count,
|
||||
"iteration_count": iteration_count,
|
||||
"catalog_backing": catalog_backing,
|
||||
},
|
||||
"preconditions": [
|
||||
"run against a disposable table or a table restored from backup",
|
||||
"record the RustFS build, catalog backing mode, node count, and object backend before starting",
|
||||
"capture baseline loadTable metadata location, version token, generation, and warehouse prefix",
|
||||
"enable only opt-in clients and workers for this rehearsal run",
|
||||
],
|
||||
"expected_invariants": [
|
||||
"current metadata generation is monotonic",
|
||||
"each conflicting writer cohort has one winner and retryable conflicts for stale writers",
|
||||
"maintenance jobs never delete reachable table objects",
|
||||
"scheduler and worker leases recover without duplicate active jobs for the same table",
|
||||
"durable backing migration stays blocked while recovery or replay blockers exist",
|
||||
"rollback/import under load either commits a validated metadata location or conflicts without pointer movement",
|
||||
],
|
||||
"phases": [
|
||||
rehearsal_phase(
|
||||
"concurrent-commit-stress",
|
||||
"Stress single-table CAS behavior while preserving pointer and generation invariants.",
|
||||
[
|
||||
probe_step(
|
||||
"spawn-concurrent-writers",
|
||||
"CLIENT-STRESS",
|
||||
table_endpoint,
|
||||
"single-winner-per-conflict-cohort",
|
||||
"run concurrent append/commit attempts and record winner commit id, conflicts, final metadata location, token, and generation",
|
||||
{
|
||||
"writer-count": writer_count,
|
||||
"iteration-count": iteration_count,
|
||||
},
|
||||
),
|
||||
probe_step(
|
||||
"load-table-after-writer-cohort",
|
||||
"GET",
|
||||
table_endpoint,
|
||||
"200",
|
||||
"loadTable returns one current metadata location and a generation that advanced monotonically",
|
||||
),
|
||||
probe_step(
|
||||
"diagnostics-after-writer-cohort",
|
||||
"GET",
|
||||
table_path(warehouse, namespace, table, "/catalog/diagnostics", rest_path),
|
||||
"200",
|
||||
"diagnostics report no unexpected commit recovery blockers after the writer cohort",
|
||||
),
|
||||
],
|
||||
),
|
||||
rehearsal_phase(
|
||||
"maintenance-scheduler-failover",
|
||||
"Exercise queued maintenance, worker claim, expired lease recovery, and stale plan rejection.",
|
||||
[
|
||||
probe_step(
|
||||
"queue-maintenance-job",
|
||||
"POST",
|
||||
table_path(warehouse, namespace, table, "/maintenance/scheduler/run", rest_path),
|
||||
"200",
|
||||
"scheduler queues at most one active maintenance job for the table",
|
||||
{
|
||||
"scheduler-id": "scale-fault-rehearsal-scheduler",
|
||||
},
|
||||
),
|
||||
probe_step(
|
||||
"claim-worker-job",
|
||||
"POST",
|
||||
table_path(warehouse, namespace, table, "/maintenance/worker/run", rest_path),
|
||||
"200-or-409",
|
||||
"one worker claims or executes the queued job while peer workers observe backpressure or no work",
|
||||
{
|
||||
"worker-count": maintenance_worker_count,
|
||||
},
|
||||
),
|
||||
probe_step(
|
||||
"recover-expired-lease",
|
||||
"POST",
|
||||
table_path(warehouse, namespace, table, "/maintenance/scheduler/run", rest_path),
|
||||
"200-or-409",
|
||||
"expired scheduler or worker leases are recovered without creating duplicate active jobs",
|
||||
{
|
||||
"scheduler-id": "scale-fault-rehearsal-recovery",
|
||||
},
|
||||
),
|
||||
probe_step(
|
||||
"stale-maintenance-plan-check",
|
||||
"POST",
|
||||
table_path(warehouse, namespace, table, "/maintenance/metadata", rest_path),
|
||||
"409-or-manual-review",
|
||||
"stale maintenance plans fail closed before deleting objects or committing metadata",
|
||||
{
|
||||
"dry-run": False,
|
||||
"expected-metadata-location": "stale-metadata-location-from-before-writer-cohort",
|
||||
},
|
||||
),
|
||||
],
|
||||
),
|
||||
rehearsal_phase(
|
||||
"durable-backing-cutover-under-load",
|
||||
"Verify durable backing cutover preflight remains explainable under recent write and maintenance churn.",
|
||||
[
|
||||
probe_step(
|
||||
"capture-cutover-backup",
|
||||
"OPERATOR",
|
||||
table_path(warehouse, namespace, table, "/catalog/export", rest_path),
|
||||
"catalog-backup-recorded",
|
||||
"record catalog-backup artifacts, current metadata location, idempotency indexes, and rollback mode before cutover",
|
||||
),
|
||||
probe_step(
|
||||
"migration-dry-run-before-cutover",
|
||||
"GET",
|
||||
warehouse_path(warehouse, "/catalog/migration", rest_path),
|
||||
"200",
|
||||
"migration dry-run reports inventory, replay blockers, idempotency blockers, and recommended actions",
|
||||
),
|
||||
],
|
||||
),
|
||||
rehearsal_phase(
|
||||
"recovery-rollback-import-under-load",
|
||||
"Exercise recovery and operator-selected rollback/import paths after stress activity.",
|
||||
[
|
||||
probe_step(
|
||||
"safe-recovery-repair-under-load",
|
||||
"POST",
|
||||
table_path(warehouse, namespace, table, "/catalog/recovery", rest_path),
|
||||
"200",
|
||||
"safe repair handles recoverable idempotency gaps without moving the table pointer",
|
||||
{
|
||||
"mode": "safe-repair",
|
||||
},
|
||||
),
|
||||
probe_step(
|
||||
"rollback-conflict-check",
|
||||
"POST",
|
||||
table_path(warehouse, namespace, table, "/catalog/rollback", rest_path),
|
||||
"200-or-409",
|
||||
"rollback either commits the operator-selected metadata location or conflicts without pointer movement",
|
||||
{
|
||||
"metadata-location": "metadata-location-from-backup",
|
||||
"version-token": "current-version-token-from-load-table",
|
||||
},
|
||||
),
|
||||
probe_step(
|
||||
"import-conflict-check",
|
||||
"POST",
|
||||
table_path(warehouse, namespace, table, "/catalog/import", rest_path),
|
||||
"200-or-409",
|
||||
"import validates metadata identity and conflicts without pointer/token/generation movement when stale",
|
||||
{
|
||||
"metadata-location": "metadata-location-from-backup",
|
||||
"properties": {
|
||||
"rehearsal-source": "scale-fault-backup",
|
||||
},
|
||||
},
|
||||
),
|
||||
],
|
||||
),
|
||||
rehearsal_phase(
|
||||
"post-run-evidence",
|
||||
"Record the evidence needed before any scale or fault claim can be promoted.",
|
||||
[
|
||||
probe_step(
|
||||
"load-table-after-scale-fault-run",
|
||||
"GET",
|
||||
table_endpoint,
|
||||
"200",
|
||||
"loadTable returns the final intended metadata location, version token, and generation",
|
||||
),
|
||||
probe_step(
|
||||
"table-data-plane-policy-probe",
|
||||
"S3-PROBE",
|
||||
table_warehouse_location,
|
||||
"inside-allowed-outside-denied",
|
||||
"ordinary S3 object access still maps table warehouse objects to the table policy boundary",
|
||||
),
|
||||
probe_step(
|
||||
"capture-run-artifacts",
|
||||
"EVIDENCE",
|
||||
"operator-recorded-artifacts",
|
||||
"recorded",
|
||||
"record RustFS build, catalog backing, writer count, worker count, iteration count, final metadata location, observed conflicts, recovered leases, and failed-closed operations",
|
||||
),
|
||||
],
|
||||
),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Print RustFS S3 Tables production failure coverage helpers.")
|
||||
parser.add_argument("--warehouse", default="rustfs-s3table-smoke")
|
||||
@@ -367,9 +584,14 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser.add_argument("--table", default="events")
|
||||
parser.add_argument("--rest-path", default="/iceberg")
|
||||
parser.add_argument("--table-warehouse-location")
|
||||
parser.add_argument("--writer-count", type=positive_int, default=8)
|
||||
parser.add_argument("--maintenance-worker-count", type=positive_int, default=2)
|
||||
parser.add_argument("--iteration-count", type=positive_int, default=50)
|
||||
parser.add_argument("--catalog-backing", default="object-backed-or-durable-strong")
|
||||
parser.add_argument("--print-failure-matrix", action="store_true")
|
||||
parser.add_argument("--print-failure-probes", action="store_true")
|
||||
parser.add_argument("--print-disaster-recovery-rehearsal", action="store_true")
|
||||
parser.add_argument("--print-scale-fault-rehearsal", action="store_true")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
@@ -419,6 +641,24 @@ def run(args: argparse.Namespace, output: StringIO | None = None) -> None:
|
||||
output,
|
||||
)
|
||||
printed = True
|
||||
if args.print_scale_fault_rehearsal:
|
||||
print_json(
|
||||
{
|
||||
"scale_fault_rehearsal": scale_fault_rehearsal_plan(
|
||||
warehouse=args.warehouse,
|
||||
namespace=args.namespace,
|
||||
table=args.table,
|
||||
rest_path=args.rest_path,
|
||||
table_warehouse_location=args.table_warehouse_location,
|
||||
writer_count=args.writer_count,
|
||||
maintenance_worker_count=args.maintenance_worker_count,
|
||||
iteration_count=args.iteration_count,
|
||||
catalog_backing=args.catalog_backing,
|
||||
)
|
||||
},
|
||||
output,
|
||||
)
|
||||
printed = True
|
||||
if not printed:
|
||||
print_json({"production_failure_coverage": production_failure_matrix()}, output)
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from contextlib import redirect_stderr
|
||||
from io import StringIO
|
||||
|
||||
import failure_coverage
|
||||
|
||||
@@ -159,6 +161,103 @@ class FailureCoverageTest(unittest.TestCase):
|
||||
first_step = document["disaster_recovery_rehearsal"]["phases"][0]["steps"][0]
|
||||
self.assertEqual(first_step["path"], "/_iceberg/v1/lake/namespaces/sales/tables/orders/catalog/export")
|
||||
|
||||
def test_scale_fault_rehearsal_plan_covers_load_and_fault_paths(self) -> None:
|
||||
rehearsal = failure_coverage.scale_fault_rehearsal_plan(
|
||||
warehouse="lake",
|
||||
namespace="sales",
|
||||
table="orders",
|
||||
rest_path="/iceberg",
|
||||
table_warehouse_location="s3://lake/tables/orders",
|
||||
writer_count=8,
|
||||
maintenance_worker_count=3,
|
||||
iteration_count=50,
|
||||
catalog_backing="durable-strong",
|
||||
)
|
||||
|
||||
self.assertEqual(rehearsal["mode"], "manual-or-ci-optional")
|
||||
self.assertEqual(rehearsal["ci_gate"], "RUSTFS_TABLE_CATALOG_SCALE_FAULT_REHEARSAL=1")
|
||||
self.assertEqual(rehearsal["parameters"]["writer_count"], 8)
|
||||
self.assertEqual(rehearsal["parameters"]["maintenance_worker_count"], 3)
|
||||
self.assertEqual(rehearsal["parameters"]["iteration_count"], 50)
|
||||
self.assertEqual(rehearsal["parameters"]["catalog_backing"], "durable-strong")
|
||||
self.assertIn("current metadata generation is monotonic", rehearsal["expected_invariants"])
|
||||
self.assertIn("maintenance jobs never delete reachable table objects", rehearsal["expected_invariants"])
|
||||
|
||||
phases = {phase["name"]: phase for phase in rehearsal["phases"]}
|
||||
self.assertIn("concurrent-commit-stress", phases)
|
||||
self.assertIn("maintenance-scheduler-failover", phases)
|
||||
self.assertIn("durable-backing-cutover-under-load", phases)
|
||||
self.assertIn("recovery-rollback-import-under-load", phases)
|
||||
self.assertIn("post-run-evidence", phases)
|
||||
|
||||
commit_steps = {step["name"]: step for step in phases["concurrent-commit-stress"]["steps"]}
|
||||
self.assertEqual(commit_steps["spawn-concurrent-writers"]["method"], "CLIENT-STRESS")
|
||||
self.assertEqual(commit_steps["spawn-concurrent-writers"]["expected_status"], "single-winner-per-conflict-cohort")
|
||||
self.assertEqual(commit_steps["load-table-after-writer-cohort"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders")
|
||||
|
||||
scheduler_steps = {step["name"]: step for step in phases["maintenance-scheduler-failover"]["steps"]}
|
||||
self.assertEqual(scheduler_steps["queue-maintenance-job"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders/maintenance/scheduler/run")
|
||||
self.assertEqual(scheduler_steps["claim-worker-job"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders/maintenance/worker/run")
|
||||
self.assertEqual(scheduler_steps["recover-expired-lease"]["expected_status"], "200-or-409")
|
||||
|
||||
cutover_steps = {step["name"]: step for step in phases["durable-backing-cutover-under-load"]["steps"]}
|
||||
self.assertEqual(cutover_steps["migration-dry-run-before-cutover"]["path"], "/iceberg/v1/lake/catalog/migration")
|
||||
self.assertIn("catalog-backup", cutover_steps["capture-cutover-backup"]["expected_behavior"])
|
||||
|
||||
recovery_steps = {step["name"]: step for step in phases["recovery-rollback-import-under-load"]["steps"]}
|
||||
self.assertEqual(recovery_steps["safe-recovery-repair-under-load"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders/catalog/recovery")
|
||||
self.assertEqual(recovery_steps["rollback-conflict-check"]["expected_status"], "200-or-409")
|
||||
|
||||
evidence_steps = {step["name"]: step for step in phases["post-run-evidence"]["steps"]}
|
||||
self.assertEqual(evidence_steps["table-data-plane-policy-probe"]["path"], "s3://lake/tables/orders")
|
||||
self.assertEqual(evidence_steps["capture-run-artifacts"]["method"], "EVIDENCE")
|
||||
|
||||
def test_cli_prints_scale_fault_rehearsal_plan(self) -> None:
|
||||
payload = failure_coverage.cli_json(
|
||||
[
|
||||
"--warehouse",
|
||||
"lake",
|
||||
"--namespace",
|
||||
"sales",
|
||||
"--table",
|
||||
"orders",
|
||||
"--rest-path",
|
||||
"/_iceberg",
|
||||
"--table-warehouse-location",
|
||||
"s3://lake/tables/orders",
|
||||
"--writer-count",
|
||||
"6",
|
||||
"--maintenance-worker-count",
|
||||
"2",
|
||||
"--iteration-count",
|
||||
"25",
|
||||
"--catalog-backing",
|
||||
"durable-strong",
|
||||
"--print-scale-fault-rehearsal",
|
||||
]
|
||||
)
|
||||
document = json.loads(payload)
|
||||
|
||||
rehearsal = document["scale_fault_rehearsal"]
|
||||
self.assertEqual(rehearsal["parameters"]["writer_count"], 6)
|
||||
self.assertEqual(rehearsal["parameters"]["maintenance_worker_count"], 2)
|
||||
self.assertEqual(rehearsal["parameters"]["iteration_count"], 25)
|
||||
first_step = rehearsal["phases"][0]["steps"][0]
|
||||
self.assertEqual(first_step["path"], "/_iceberg/v1/lake/namespaces/sales/tables/orders")
|
||||
|
||||
def test_cli_rejects_non_positive_scale_fault_counts(self) -> None:
|
||||
invalid_count_flags = [
|
||||
("--writer-count", "0"),
|
||||
("--maintenance-worker-count", "-1"),
|
||||
("--iteration-count", "0"),
|
||||
]
|
||||
|
||||
for flag, value in invalid_count_flags:
|
||||
with self.subTest(flag=flag):
|
||||
with redirect_stderr(StringIO()):
|
||||
with self.assertRaises(SystemExit):
|
||||
failure_coverage.cli_json([flag, value, "--print-scale-fault-rehearsal"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user