Compare commits

...

2 Commits

Author SHA1 Message Date
Hauser Zhi 8ba7ac862d Merge branch 'main' into test/internode-rpc-timeout-evidence 2026-11-17 01:30:51 +08:00
Hauser Zhi f80522ef35 test(rpc): verify timeout boundary evidence
Bind under-budget success, over-budget deadlines, recovery, fault receipts, and stable probe workload identity into one offline evidence gate.

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-17 01:19:43 +08:00
5 changed files with 359 additions and 0 deletions
+1
View File
@@ -39,6 +39,7 @@ script-tests: ## Run shell script tests
./scripts/test_entrypoint_directories.sh
./scripts/test_internode_grpc_ab_bench.sh
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_grpc_window.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_rpc_timeout_boundary.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_msgpack_peer_canary.py
./scripts/test_internode_msgpack_peer_canary_runner.sh
./scripts/test_object_batch_bench_enhanced.sh
@@ -228,6 +228,27 @@ Every offered Ping is recorded, including schedule drops, deadlines and errors.
Use identical probe source and settings on the two frozen library revisions, in separate Cargo targets, and record the harness overlay and binary identity independently from the already-built clean server binaries. Qualify repeated A cells and then A1/B1/B2/A2 before expanding to coalescing, isolation, pool or window experiments. A successful local mock regression verifies the probe's calling and accounting boundaries; acceptance against the actual authenticated four-node services is still required.
### Timeout-boundary evidence
Use `scripts/check_internode_rpc_timeout_boundary.py` after a task-owned fault injector has recorded three independent probe reports: a stall whose measured duration remains below the declared RPC timeout, a stall whose requested and measured durations both exceed it, and a no-fault recovery run. The gate does not inject faults, deploy services or infer a timeout from wall-clock labels; the manifest references separate measured fault receipts and the raw reports.
```json
{
"schema": 1,
"scope": "internode-rpc-timeout-boundary",
"rpc_timeout_ms": 10000,
"under_budget": {"fault_receipt": "under-fault.json", "report": "under.json"},
"over_budget": {"fault_receipt": "over-fault.json", "report": "over.json"},
"recovery": {"report": "recovery.json"}
}
```
```bash
python3 scripts/check_internode_rpc_timeout_boundary.py --manifest /path/to/manifest.json --out /path/to/verification.json
```
Each fault receipt is a bounded JSON object containing a non-empty `target`, `requested_stall_ms` and `actual_stall_ms`. Exit 0 requires complete bulk coverage, exact offered-control coverage, no errors below the timeout, at least one `deadline` above it, and a zero-error recovery report. `schedule_drop` and `rpc_error` never substitute for the expected deadline. Exit 1 means structurally complete evidence did not satisfy the boundary; exit 2 means the evidence is invalid. The verifier binds all reports and fault receipts by SHA-256, rejects path escape/reuse and refuses to overwrite or alias raw evidence. Deployment ownership, actual fault delivery, phase-scoped telemetry and performance attribution remain separate gates.
## Acceptance gates
Record the verdict (pass/fail plus measured delta) in the paired directory's summary.
+1
View File
@@ -130,6 +130,7 @@ their issue closes.
| `hotpath_warp_ab_gate.sh` | dev-tool | Relative-budget gate evaluated over the warp A/B results | used by `run_hotpath_warp_ab.sh`; hotpath runbook |
| `run_internode_grpc_ab_bench.sh` | dev-tool | One-click A/B driver for the internode gRPC optimization stages | `docs/operations/internode-grpc-benchmark-runbook.md` |
| `run_internode_transport_baseline.sh` | dev-tool | Internode transport baseline runner | internode runbook; `crates/io-metrics/README.md` |
| `check_internode_rpc_timeout_boundary.py` | dev-tool | Verifies under-budget success, over-budget deadlines and post-stall recovery from recorded internode RPC probe reports | internode gRPC benchmark runbook; `test_internode_rpc_timeout_boundary.py` |
| `run_four_node_cluster_failover_bench.sh` | dev-tool | Four-node cluster failover benchmark with configurable host ports and optional server-only image build | `docker/compose/README.md`; internode runbook |
| `run_object_batch_bench.sh` | dev-tool | Batch object benchmark runner with PUT-specific Warp summary extraction | internode + scanner runbooks |
| `test_object_batch_bench.sh` | dev-tool | Regression test for Warp PUT summary extraction | `scripts/test_object_batch_bench.sh` |
+163
View File
@@ -0,0 +1,163 @@
#!/usr/bin/env python3
"""Verify recorded internode RPC timeout-boundary and recovery evidence."""
import argparse
import hashlib
import json
from pathlib import Path
import sys
MAX_MANIFEST_BYTES = 1024 * 1024
MAX_REPORT_BYTES = 512 * 1024 * 1024
def require(condition, message):
if not condition:
raise ValueError(message)
def unique_object(pairs):
result = {}
for key, value in pairs:
require(key not in result, f"duplicate JSON key: {key}")
result[key] = value
return result
def uint(value, name):
require(type(value) is int and value >= 0, f"invalid {name}")
return value
def load(path, limit):
require(path.is_file() and path.stat().st_size <= limit, f"missing or oversized evidence: {path.name}")
raw = path.read_bytes()
return json.loads(raw, object_pairs_hook=unique_object), hashlib.sha256(raw).hexdigest()
def evidence_path(root, value, seen):
require(isinstance(value, str) and value, "missing evidence path")
path = (root / value).resolve()
require(path == root or root in path.parents, "evidence path escapes manifest directory")
require(path not in seen, "evidence file reused across phases")
seen.add(path)
return path
def samples(report, operation):
rows = report.get("samples")
require(isinstance(rows, list) and rows, "missing probe samples")
return [row for row in rows if row.get("operation") == operation]
def report_status(report):
require(report.get("complete") is True, "probe report is incomplete")
require(report.get("bulk_coverage_verified") is True, "probe report lacks validated bulk coverage")
duration = uint(report.get("offered_duration_secs"), "offered duration")
rate = uint(report.get("control_rps"), "control rate")
expected = uint(report.get("control_expected"), "expected control count")
require(duration > 0 and rate > 0 and expected == duration * rate, "invalid control offering contract")
ping = samples(report, "Ping")
read_all = samples(report, "ReadAll")
require(len(ping) == expected and read_all, "probe sample coverage does not match its contract")
allowed = {"ok", "deadline", "schedule_drop", "rpc_error"}
require(all(row.get("outcome") in allowed for row in ping + read_all), "unknown probe outcome")
observed_errors = sum(row.get("outcome") != "ok" for row in ping + read_all)
require(uint(report.get("errors"), "probe error count") == observed_errors, "probe error count does not match samples")
fixture = report.get("fixture_sha256")
require(isinstance(fixture, str) and len(fixture) == 64 and all(char in "0123456789abcdef" for char in fixture),
"invalid fixture identity")
targets = uint(report.get("targets"), "target count")
peers = uint(report.get("peers"), "peer count")
workers = uint(report.get("bulk_workers"), "bulk worker count")
scope = report.get("scope")
require(targets > 0 and peers > 0 and workers > 0 and isinstance(scope, str) and scope,
"invalid probe workload identity")
identity = {"fixture_sha256": fixture, "targets": targets, "peers": peers, "bulk_workers": workers,
"control_rps": rate, "scope": scope}
return ping, read_all, identity
def phase(report, expected):
ping, read_all, identity = report_status(report)
outcomes = [row["outcome"] for row in ping + read_all]
if expected == "success":
return not outcomes.count("deadline") and not outcomes.count("schedule_drop") and not outcomes.count("rpc_error"), identity
require(expected == "deadline", "unknown timeout-boundary expectation")
return outcomes.count("deadline") > 0 and not outcomes.count("schedule_drop") and not outcomes.count("rpc_error"), identity
def check(manifest, root):
root = root.resolve()
require(isinstance(manifest, dict) and manifest.get("schema") == 1, "unsupported timeout-boundary schema")
require(manifest.get("scope") == "internode-rpc-timeout-boundary", "wrong timeout-boundary scope")
timeout = uint(manifest.get("rpc_timeout_ms"), "RPC timeout")
require(timeout > 0, "RPC timeout must be positive")
seen = set()
receipts = []
verdicts = {}
workload_identity = None
phases = (("under_budget", "success"), ("over_budget", "deadline"), ("recovery", "success"))
for name, expectation in phases:
descriptor = manifest.get(name)
require(isinstance(descriptor, dict), f"missing {name} descriptor")
if name != "recovery":
fault_path = evidence_path(root, descriptor.get("fault_receipt"), seen)
fault, fault_digest = load(fault_path, MAX_MANIFEST_BYTES)
require(isinstance(fault, dict) and isinstance(fault.get("target"), str) and fault["target"],
f"invalid {name} fault target")
requested = uint(fault.get("requested_stall_ms"), f"{name} requested stall")
actual = uint(fault.get("actual_stall_ms"), f"{name} actual stall")
require(0 < requested <= actual, f"invalid {name} stall receipt")
if name == "under_budget":
require(actual < timeout, "under-budget stall crossed the RPC timeout")
else:
require(requested > timeout and actual > timeout, "over-budget stall did not cross the RPC timeout")
receipts.append({"phase": name, "kind": "fault", "path": descriptor["fault_receipt"],
"sha256": fault_digest})
path = evidence_path(root, descriptor.get("report"), seen)
report, digest = load(path, MAX_REPORT_BYTES)
verdicts[name], identity = phase(report, expectation)
if workload_identity is None:
workload_identity = identity
else:
require(identity == workload_identity, "probe workload identity changed across timeout phases")
receipts.append({"phase": name, "kind": "report", "path": descriptor["report"], "sha256": digest})
return {
"observed_pass": all(verdicts.values()),
"scope": manifest["scope"],
"rpc_timeout_ms": timeout,
"phases": verdicts,
"workload": workload_identity,
"reports": receipts,
"limits": "Recorded request outcomes only; deployment ownership, fault delivery, Prometheus windows and performance attribution remain separate evidence.",
}, seen
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
try:
require(args.manifest.is_file() and args.manifest.stat().st_size <= MAX_MANIFEST_BYTES,
"missing or oversized timeout-boundary manifest")
raw = args.manifest.read_bytes()
manifest = json.loads(raw, object_pairs_hook=unique_object)
result, inputs = check(manifest, args.manifest.resolve().parent)
output = args.out.resolve()
require(output not in inputs | {args.manifest.resolve()}, "output would overwrite raw evidence")
if args.out.exists():
require(not any(args.out.samefile(path) for path in inputs | {args.manifest.resolve()}),
"output aliases raw evidence")
result["manifest_sha256"] = hashlib.sha256(raw).hexdigest()
args.out.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
return 0 if result["observed_pass"] else 1
except (ValueError, OSError, TypeError, KeyError, AttributeError, json.JSONDecodeError) as error:
print(f"Invalid internode RPC timeout-boundary evidence: {error}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())
+173
View File
@@ -0,0 +1,173 @@
#!/usr/bin/env python3
"""Regression cases for the internode RPC timeout-boundary evidence gate."""
import copy
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
import check_internode_rpc_timeout_boundary as gate
class TimeoutBoundaryTests(unittest.TestCase):
def setUp(self):
self.directory = tempfile.TemporaryDirectory()
self.addCleanup(self.directory.cleanup)
self.root = Path(self.directory.name)
self.reports = {
"under": self.report(30, 10, []),
"over": self.report(35, 10, [("Ping", "deadline"), ("ReadAll", "deadline")]),
"recovery": self.report(15, 10, []),
}
for name, report in self.reports.items():
(self.root / f"{name}.json").write_text(json.dumps(report))
self.faults = {
"under": {"target": "node4", "requested_stall_ms": 8_000, "actual_stall_ms": 9_100},
"over": {"target": "node4", "requested_stall_ms": 12_000, "actual_stall_ms": 12_600},
}
self.manifest = {
"schema": 1,
"scope": "internode-rpc-timeout-boundary",
"rpc_timeout_ms": 10_000,
"under_budget": {"fault_receipt": "under-fault.json", "report": "under.json"},
"over_budget": {"fault_receipt": "over-fault.json", "report": "over.json"},
"recovery": {"report": "recovery.json"},
}
@staticmethod
def report(duration, rate, failures):
samples = []
for sequence in range(duration * rate):
samples.append({"operation": "Ping", "outcome": "ok", "peer": sequence % 4})
for sequence in range(8):
samples.append({"operation": "ReadAll", "outcome": "ok", "peer": sequence % 4})
for operation, outcome in failures:
row = next(sample for sample in samples if sample["operation"] == operation and sample["outcome"] == "ok")
row["outcome"] = outcome
return {
"complete": True,
"errors": len(failures),
"bulk_coverage_verified": True,
"offered_duration_secs": duration,
"control_rps": rate,
"control_expected": duration * rate,
"fixture_sha256": "0" * 64,
"targets": 4,
"peers": 4,
"bulk_workers": 2,
"scope": "Authenticated production RPC clients",
"samples": samples,
}
def write_reports(self):
for name, report in self.reports.items():
(self.root / f"{name}.json").write_text(json.dumps(report))
for name, fault in self.faults.items():
(self.root / f"{name}-fault.json").write_text(json.dumps(fault))
def check(self):
self.write_reports()
return gate.check(self.manifest, self.root)[0]
def test_valid_boundary_and_recovery_pass(self):
result = self.check()
self.assertTrue(result["observed_pass"])
self.assertEqual(result["phases"], {"under_budget": True, "over_budget": True, "recovery": True})
def test_schedule_drop_cannot_substitute_for_deadline(self):
for report in ("under", "over", "recovery"):
with self.subTest(report=report):
original = copy.deepcopy(self.reports[report])
row = next(sample for sample in self.reports[report]["samples"] if sample["operation"] == "Ping")
was_ok = row["outcome"] == "ok"
row["outcome"] = "schedule_drop"
self.reports[report]["errors"] += was_ok
self.assertFalse(self.check()["observed_pass"])
self.reports[report] = original
def test_under_budget_errors_over_budget_success_and_recovery_errors_fail(self):
cases = (("under", "deadline", 1), ("over", "ok", -2), ("recovery", "rpc_error", 1))
for report, outcome, error_delta in cases:
with self.subTest(report=report):
original = copy.deepcopy(self.reports[report])
if report == "over":
for sample in self.reports[report]["samples"]:
sample["outcome"] = "ok"
else:
next(sample for sample in self.reports[report]["samples"] if sample["operation"] == "Ping")["outcome"] = outcome
self.reports[report]["errors"] += error_delta
self.assertFalse(self.check()["observed_pass"])
self.reports[report] = original
def test_invalid_stall_receipts_and_control_contract_fail_closed(self):
edits = (
lambda: self.faults["under"].update(actual_stall_ms=10_000),
lambda: self.faults["over"].update(requested_stall_ms=9_000),
lambda: self.faults["under"].update(requested_stall_ms=9_200),
lambda: self.reports["under"].update(control_expected=1),
)
for edit in edits:
with self.subTest(edit=edit):
manifest = copy.deepcopy(self.manifest)
reports = copy.deepcopy(self.reports)
faults = copy.deepcopy(self.faults)
edit()
with self.assertRaises(ValueError):
self.check()
self.manifest, self.reports, self.faults = manifest, reports, faults
def test_paths_cannot_escape_or_reuse_evidence(self):
self.manifest["under_budget"]["report"] = "../outside.json"
with self.assertRaisesRegex(ValueError, "escapes"):
self.check()
self.manifest["under_budget"]["report"] = "under.json"
self.manifest["recovery"]["report"] = "under.json"
with self.assertRaisesRegex(ValueError, "reused"):
self.check()
def test_probe_workload_identity_cannot_change_between_phases(self):
for field, value in (("fixture_sha256", "1" * 64), ("targets", 3), ("peers", 3),
("bulk_workers", 1), ("control_rps", 9), ("scope", "different")):
with self.subTest(field=field):
self.reports["recovery"][field] = value
if field == "control_rps":
self.reports["recovery"]["control_expected"] = self.reports["recovery"]["offered_duration_secs"] * value
self.reports["recovery"]["samples"] = self.report(15, value, [])["samples"]
with self.assertRaisesRegex(ValueError, "identity changed"):
self.check()
self.reports["recovery"] = self.report(15, 10, [])
def run_cli(self, output):
self.write_reports()
manifest = self.root / "manifest.json"
manifest.write_text(json.dumps(self.manifest))
return subprocess.run([sys.executable, str(Path(gate.__file__)), "--manifest", str(manifest),
"--out", str(output)], capture_output=True, text=True)
def test_entry_point_distinguishes_pass_fail_and_invalid(self):
output = self.root / "result.json"
self.assertEqual(self.run_cli(output).returncode, 0)
self.reports["recovery"]["samples"][0]["outcome"] = "deadline"
self.reports["recovery"]["errors"] = 1
self.assertEqual(self.run_cli(output).returncode, 1)
self.manifest["scope"] = "command"
self.assertEqual(self.run_cli(output).returncode, 2)
def test_entry_point_preserves_raw_and_hard_linked_evidence(self):
raw = self.root / "under.json"
self.write_reports()
before = raw.read_bytes()
self.assertEqual(self.run_cli(raw).returncode, 2)
self.assertEqual(raw.read_bytes(), before)
alias = self.root / "alias.json"
os.link(raw, alias)
self.assertEqual(self.run_cli(alias).returncode, 2)
self.assertEqual(raw.read_bytes(), before)
if __name__ == "__main__":
unittest.main()