mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
Henry Guo
0e61ba7c63
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>
264 lines
14 KiB
Python
264 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Unit tests for RustFS table catalog production failure coverage helpers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import unittest
|
|
from contextlib import redirect_stderr
|
|
from io import StringIO
|
|
|
|
import failure_coverage
|
|
|
|
|
|
class FailureCoverageTest(unittest.TestCase):
|
|
def test_matrix_tracks_production_failure_boundaries_without_overclaiming(self) -> None:
|
|
matrix = failure_coverage.production_failure_matrix()
|
|
by_case = {entry["case"]: entry for entry in matrix}
|
|
|
|
self.assertIn("commit-cas-conflict", by_case)
|
|
self.assertIn("post-cas-finalization-gap", by_case)
|
|
self.assertIn("missing-referenced-object", by_case)
|
|
self.assertIn("maintenance-stale-plan", by_case)
|
|
self.assertIn("external-bridge-conflict", by_case)
|
|
self.assertIn("backing-migration-blocked", by_case)
|
|
self.assertIn("permission-negative", by_case)
|
|
|
|
self.assertEqual(by_case["commit-cas-conflict"]["expected_behavior"], "conflict-without-pointer-advance")
|
|
self.assertEqual(by_case["post-cas-finalization-gap"]["expected_behavior"], "recoverable-diagnostics-and-idempotency-repair")
|
|
for entry in matrix:
|
|
self.assertIn("coverage_status", entry)
|
|
self.assertIn("evidence", entry)
|
|
self.assertNotEqual(entry["coverage_status"], "claimed-automated-live")
|
|
|
|
def test_failure_probe_plan_generates_rest_negative_steps(self) -> None:
|
|
plan = failure_coverage.failure_probe_plan(
|
|
warehouse="lake",
|
|
namespace="sales",
|
|
table="orders",
|
|
)
|
|
by_name = {step["name"]: step for step in plan}
|
|
|
|
self.assertEqual(
|
|
by_name["stale-token-commit-conflict"]["path"],
|
|
"/iceberg/v1/lake/namespaces/sales/tables/orders",
|
|
)
|
|
self.assertEqual(by_name["stale-token-commit-conflict"]["method"], "POST")
|
|
self.assertEqual(by_name["stale-token-commit-conflict"]["expected_status"], "409")
|
|
self.assertIn("expected-version-token", by_name["stale-token-commit-conflict"]["body"])
|
|
self.assertIn("expected-metadata-location", by_name["stale-token-commit-conflict"]["body"])
|
|
self.assertIn("new-metadata-location", by_name["stale-token-commit-conflict"]["body"])
|
|
self.assertNotIn("base", by_name["stale-token-commit-conflict"]["body"])
|
|
self.assertEqual(
|
|
by_name["diagnostics-after-finalization-gap"]["path"],
|
|
"/iceberg/v1/lake/namespaces/sales/tables/orders/catalog/diagnostics",
|
|
)
|
|
self.assertEqual(by_name["diagnostics-after-finalization-gap"]["method"], "GET")
|
|
self.assertEqual(by_name["recovery-repairs-idempotency-index"]["method"], "POST")
|
|
self.assertIn("does_not_exist.metadata.json", json.dumps(by_name["missing-metadata-object-rejected"]))
|
|
self.assertNotIn("base", by_name["missing-metadata-object-rejected"]["body"])
|
|
|
|
def test_cli_prints_failure_matrix_and_probe_plan(self) -> None:
|
|
matrix_payload = failure_coverage.cli_json(["--print-failure-matrix"])
|
|
matrix_document = json.loads(matrix_payload)
|
|
|
|
self.assertIn("production_failure_coverage", matrix_document)
|
|
self.assertTrue(matrix_document["production_failure_coverage"])
|
|
|
|
probe_payload = failure_coverage.cli_json(
|
|
[
|
|
"--warehouse",
|
|
"lake",
|
|
"--namespace",
|
|
"sales",
|
|
"--table",
|
|
"orders",
|
|
"--rest-path",
|
|
"/_iceberg",
|
|
"--print-failure-probes",
|
|
]
|
|
)
|
|
probe_document = json.loads(probe_payload)
|
|
|
|
self.assertIn("failure_probe_plan", probe_document)
|
|
self.assertTrue(probe_document["failure_probe_plan"])
|
|
self.assertEqual(
|
|
probe_document["failure_probe_plan"][0]["path"],
|
|
"/_iceberg/v1/lake/namespaces/sales/tables/orders",
|
|
)
|
|
|
|
def test_disaster_recovery_rehearsal_plan_covers_operator_recovery_path(self) -> None:
|
|
rehearsal = failure_coverage.disaster_recovery_rehearsal_plan(
|
|
warehouse="lake",
|
|
namespace="sales",
|
|
table="orders",
|
|
rest_path="/iceberg",
|
|
table_warehouse_location="s3://lake/tables/orders",
|
|
)
|
|
|
|
self.assertEqual(rehearsal["mode"], "manual-or-ci-optional")
|
|
self.assertEqual(rehearsal["ci_gate"], "RUSTFS_TABLE_CATALOG_DR_REHEARSAL=1")
|
|
self.assertIn("record the RustFS build and catalog backing mode before starting", rehearsal["preconditions"])
|
|
self.assertIn("current metadata pointer remains recoverable or deliberately rolled back", rehearsal["expected_invariants"])
|
|
|
|
phases = {phase["name"]: phase for phase in rehearsal["phases"]}
|
|
self.assertIn("capture-baseline", phases)
|
|
self.assertIn("diagnose-and-repair", phases)
|
|
self.assertIn("rollback-or-import", phases)
|
|
self.assertIn("migration-preflight", phases)
|
|
self.assertIn("post-recovery-validation", phases)
|
|
|
|
baseline_steps = {step["name"]: step for step in phases["capture-baseline"]["steps"]}
|
|
self.assertEqual(
|
|
baseline_steps["export-catalog-state"]["path"],
|
|
"/iceberg/v1/lake/namespaces/sales/tables/orders/catalog/export",
|
|
)
|
|
self.assertEqual(baseline_steps["load-table-before-recovery"]["method"], "GET")
|
|
|
|
repair_steps = {step["name"]: step for step in phases["diagnose-and-repair"]["steps"]}
|
|
self.assertEqual(repair_steps["read-recovery-diagnostics"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders/catalog/diagnostics")
|
|
self.assertEqual(repair_steps["safe-recovery-repair"]["method"], "POST")
|
|
self.assertEqual(repair_steps["safe-recovery-repair"]["body"], {"mode": "safe-repair"})
|
|
|
|
rollback_steps = {step["name"]: step for step in phases["rollback-or-import"]["steps"]}
|
|
self.assertEqual(rollback_steps["rollback-to-known-metadata"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders/catalog/rollback")
|
|
self.assertIn("metadata-location", rollback_steps["rollback-to-known-metadata"]["body"])
|
|
self.assertIn("version-token", rollback_steps["rollback-to-known-metadata"]["body"])
|
|
self.assertNotIn("expected-version-token", rollback_steps["rollback-to-known-metadata"]["body"])
|
|
self.assertEqual(rollback_steps["import-known-metadata"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders/catalog/import")
|
|
self.assertIn("metadata-location", rollback_steps["import-known-metadata"]["body"])
|
|
self.assertIn("properties", rollback_steps["import-known-metadata"]["body"])
|
|
self.assertNotIn("external-version-token", rollback_steps["import-known-metadata"]["body"])
|
|
|
|
migration_steps = {step["name"]: step for step in phases["migration-preflight"]["steps"]}
|
|
self.assertEqual(migration_steps["durable-backing-migration-dry-run"]["path"], "/iceberg/v1/lake/catalog/migration")
|
|
self.assertEqual(migration_steps["durable-backing-migration-dry-run"]["expected_behavior"], "migration blockers must be empty before cutover")
|
|
|
|
validation_steps = {step["name"]: step for step in phases["post-recovery-validation"]["steps"]}
|
|
self.assertEqual(validation_steps["load-table-after-recovery"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders")
|
|
self.assertEqual(validation_steps["table-data-plane-policy-probe"]["path"], "s3://lake/tables/orders")
|
|
self.assertEqual(validation_steps["table-data-plane-policy-probe"]["method"], "S3-PROBE")
|
|
|
|
def test_cli_prints_disaster_recovery_rehearsal_plan(self) -> None:
|
|
payload = failure_coverage.cli_json(
|
|
[
|
|
"--warehouse",
|
|
"lake",
|
|
"--namespace",
|
|
"sales",
|
|
"--table",
|
|
"orders",
|
|
"--rest-path",
|
|
"/_iceberg",
|
|
"--table-warehouse-location",
|
|
"s3://lake/tables/orders",
|
|
"--print-disaster-recovery-rehearsal",
|
|
]
|
|
)
|
|
document = json.loads(payload)
|
|
|
|
self.assertEqual(document["disaster_recovery_rehearsal"]["phases"][0]["name"], "capture-baseline")
|
|
first_step = document["disaster_recovery_rehearsal"]["phases"][0]["steps"][0]
|
|
self.assertEqual(first_step["path"], "/_iceberg/v1/lake/namespaces/sales/tables/orders/catalog/export")
|
|
|
|
def test_scale_fault_rehearsal_plan_covers_load_and_fault_paths(self) -> None:
|
|
rehearsal = failure_coverage.scale_fault_rehearsal_plan(
|
|
warehouse="lake",
|
|
namespace="sales",
|
|
table="orders",
|
|
rest_path="/iceberg",
|
|
table_warehouse_location="s3://lake/tables/orders",
|
|
writer_count=8,
|
|
maintenance_worker_count=3,
|
|
iteration_count=50,
|
|
catalog_backing="durable-strong",
|
|
)
|
|
|
|
self.assertEqual(rehearsal["mode"], "manual-or-ci-optional")
|
|
self.assertEqual(rehearsal["ci_gate"], "RUSTFS_TABLE_CATALOG_SCALE_FAULT_REHEARSAL=1")
|
|
self.assertEqual(rehearsal["parameters"]["writer_count"], 8)
|
|
self.assertEqual(rehearsal["parameters"]["maintenance_worker_count"], 3)
|
|
self.assertEqual(rehearsal["parameters"]["iteration_count"], 50)
|
|
self.assertEqual(rehearsal["parameters"]["catalog_backing"], "durable-strong")
|
|
self.assertIn("current metadata generation is monotonic", rehearsal["expected_invariants"])
|
|
self.assertIn("maintenance jobs never delete reachable table objects", rehearsal["expected_invariants"])
|
|
|
|
phases = {phase["name"]: phase for phase in rehearsal["phases"]}
|
|
self.assertIn("concurrent-commit-stress", phases)
|
|
self.assertIn("maintenance-scheduler-failover", phases)
|
|
self.assertIn("durable-backing-cutover-under-load", phases)
|
|
self.assertIn("recovery-rollback-import-under-load", phases)
|
|
self.assertIn("post-run-evidence", phases)
|
|
|
|
commit_steps = {step["name"]: step for step in phases["concurrent-commit-stress"]["steps"]}
|
|
self.assertEqual(commit_steps["spawn-concurrent-writers"]["method"], "CLIENT-STRESS")
|
|
self.assertEqual(commit_steps["spawn-concurrent-writers"]["expected_status"], "single-winner-per-conflict-cohort")
|
|
self.assertEqual(commit_steps["load-table-after-writer-cohort"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders")
|
|
|
|
scheduler_steps = {step["name"]: step for step in phases["maintenance-scheduler-failover"]["steps"]}
|
|
self.assertEqual(scheduler_steps["queue-maintenance-job"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders/maintenance/scheduler/run")
|
|
self.assertEqual(scheduler_steps["claim-worker-job"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders/maintenance/worker/run")
|
|
self.assertEqual(scheduler_steps["recover-expired-lease"]["expected_status"], "200-or-409")
|
|
|
|
cutover_steps = {step["name"]: step for step in phases["durable-backing-cutover-under-load"]["steps"]}
|
|
self.assertEqual(cutover_steps["migration-dry-run-before-cutover"]["path"], "/iceberg/v1/lake/catalog/migration")
|
|
self.assertIn("catalog-backup", cutover_steps["capture-cutover-backup"]["expected_behavior"])
|
|
|
|
recovery_steps = {step["name"]: step for step in phases["recovery-rollback-import-under-load"]["steps"]}
|
|
self.assertEqual(recovery_steps["safe-recovery-repair-under-load"]["path"], "/iceberg/v1/lake/namespaces/sales/tables/orders/catalog/recovery")
|
|
self.assertEqual(recovery_steps["rollback-conflict-check"]["expected_status"], "200-or-409")
|
|
|
|
evidence_steps = {step["name"]: step for step in phases["post-run-evidence"]["steps"]}
|
|
self.assertEqual(evidence_steps["table-data-plane-policy-probe"]["path"], "s3://lake/tables/orders")
|
|
self.assertEqual(evidence_steps["capture-run-artifacts"]["method"], "EVIDENCE")
|
|
|
|
def test_cli_prints_scale_fault_rehearsal_plan(self) -> None:
|
|
payload = failure_coverage.cli_json(
|
|
[
|
|
"--warehouse",
|
|
"lake",
|
|
"--namespace",
|
|
"sales",
|
|
"--table",
|
|
"orders",
|
|
"--rest-path",
|
|
"/_iceberg",
|
|
"--table-warehouse-location",
|
|
"s3://lake/tables/orders",
|
|
"--writer-count",
|
|
"6",
|
|
"--maintenance-worker-count",
|
|
"2",
|
|
"--iteration-count",
|
|
"25",
|
|
"--catalog-backing",
|
|
"durable-strong",
|
|
"--print-scale-fault-rehearsal",
|
|
]
|
|
)
|
|
document = json.loads(payload)
|
|
|
|
rehearsal = document["scale_fault_rehearsal"]
|
|
self.assertEqual(rehearsal["parameters"]["writer_count"], 6)
|
|
self.assertEqual(rehearsal["parameters"]["maintenance_worker_count"], 2)
|
|
self.assertEqual(rehearsal["parameters"]["iteration_count"], 25)
|
|
first_step = rehearsal["phases"][0]["steps"][0]
|
|
self.assertEqual(first_step["path"], "/_iceberg/v1/lake/namespaces/sales/tables/orders")
|
|
|
|
def test_cli_rejects_non_positive_scale_fault_counts(self) -> None:
|
|
invalid_count_flags = [
|
|
("--writer-count", "0"),
|
|
("--maintenance-worker-count", "-1"),
|
|
("--iteration-count", "0"),
|
|
]
|
|
|
|
for flag, value in invalid_count_flags:
|
|
with self.subTest(flag=flag):
|
|
with redirect_stderr(StringIO()):
|
|
with self.assertRaises(SystemExit):
|
|
failure_coverage.cli_json([flag, value, "--print-scale-fault-rehearsal"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|