mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-16 21:14:45 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8ba7ac862d | |||
| f80522ef35 |
@@ -39,6 +39,7 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_entrypoint_directories.sh
|
||||
./scripts/test_internode_grpc_ab_bench.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_grpc_window.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_rpc_timeout_boundary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_msgpack_peer_canary.py
|
||||
./scripts/test_internode_msgpack_peer_canary_runner.sh
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
|
||||
@@ -228,6 +228,27 @@ Every offered Ping is recorded, including schedule drops, deadlines and errors.
|
||||
|
||||
Use identical probe source and settings on the two frozen library revisions, in separate Cargo targets, and record the harness overlay and binary identity independently from the already-built clean server binaries. Qualify repeated A cells and then A1/B1/B2/A2 before expanding to coalescing, isolation, pool or window experiments. A successful local mock regression verifies the probe's calling and accounting boundaries; acceptance against the actual authenticated four-node services is still required.
|
||||
|
||||
### Timeout-boundary evidence
|
||||
|
||||
Use `scripts/check_internode_rpc_timeout_boundary.py` after a task-owned fault injector has recorded three independent probe reports: a stall whose measured duration remains below the declared RPC timeout, a stall whose requested and measured durations both exceed it, and a no-fault recovery run. The gate does not inject faults, deploy services or infer a timeout from wall-clock labels; the manifest references separate measured fault receipts and the raw reports.
|
||||
|
||||
```json
|
||||
{
|
||||
"schema": 1,
|
||||
"scope": "internode-rpc-timeout-boundary",
|
||||
"rpc_timeout_ms": 10000,
|
||||
"under_budget": {"fault_receipt": "under-fault.json", "report": "under.json"},
|
||||
"over_budget": {"fault_receipt": "over-fault.json", "report": "over.json"},
|
||||
"recovery": {"report": "recovery.json"}
|
||||
}
|
||||
```
|
||||
|
||||
```bash
|
||||
python3 scripts/check_internode_rpc_timeout_boundary.py --manifest /path/to/manifest.json --out /path/to/verification.json
|
||||
```
|
||||
|
||||
Each fault receipt is a bounded JSON object containing a non-empty `target`, `requested_stall_ms` and `actual_stall_ms`. Exit 0 requires complete bulk coverage, exact offered-control coverage, no errors below the timeout, at least one `deadline` above it, and a zero-error recovery report. `schedule_drop` and `rpc_error` never substitute for the expected deadline. Exit 1 means structurally complete evidence did not satisfy the boundary; exit 2 means the evidence is invalid. The verifier binds all reports and fault receipts by SHA-256, rejects path escape/reuse and refuses to overwrite or alias raw evidence. Deployment ownership, actual fault delivery, phase-scoped telemetry and performance attribution remain separate gates.
|
||||
|
||||
## Acceptance gates
|
||||
|
||||
Record the verdict (pass/fail plus measured delta) in the paired directory's summary.
|
||||
|
||||
@@ -130,6 +130,7 @@ their issue closes.
|
||||
| `hotpath_warp_ab_gate.sh` | dev-tool | Relative-budget gate evaluated over the warp A/B results | used by `run_hotpath_warp_ab.sh`; hotpath runbook |
|
||||
| `run_internode_grpc_ab_bench.sh` | dev-tool | One-click A/B driver for the internode gRPC optimization stages | `docs/operations/internode-grpc-benchmark-runbook.md` |
|
||||
| `run_internode_transport_baseline.sh` | dev-tool | Internode transport baseline runner | internode runbook; `crates/io-metrics/README.md` |
|
||||
| `check_internode_rpc_timeout_boundary.py` | dev-tool | Verifies under-budget success, over-budget deadlines and post-stall recovery from recorded internode RPC probe reports | internode gRPC benchmark runbook; `test_internode_rpc_timeout_boundary.py` |
|
||||
| `run_four_node_cluster_failover_bench.sh` | dev-tool | Four-node cluster failover benchmark with configurable host ports and optional server-only image build | `docker/compose/README.md`; internode runbook |
|
||||
| `run_object_batch_bench.sh` | dev-tool | Batch object benchmark runner with PUT-specific Warp summary extraction | internode + scanner runbooks |
|
||||
| `test_object_batch_bench.sh` | dev-tool | Regression test for Warp PUT summary extraction | `scripts/test_object_batch_bench.sh` |
|
||||
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify recorded internode RPC timeout-boundary and recovery evidence."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
MAX_MANIFEST_BYTES = 1024 * 1024
|
||||
MAX_REPORT_BYTES = 512 * 1024 * 1024
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def unique_object(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
require(key not in result, f"duplicate JSON key: {key}")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def uint(value, name):
|
||||
require(type(value) is int and value >= 0, f"invalid {name}")
|
||||
return value
|
||||
|
||||
|
||||
def load(path, limit):
|
||||
require(path.is_file() and path.stat().st_size <= limit, f"missing or oversized evidence: {path.name}")
|
||||
raw = path.read_bytes()
|
||||
return json.loads(raw, object_pairs_hook=unique_object), hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def evidence_path(root, value, seen):
|
||||
require(isinstance(value, str) and value, "missing evidence path")
|
||||
path = (root / value).resolve()
|
||||
require(path == root or root in path.parents, "evidence path escapes manifest directory")
|
||||
require(path not in seen, "evidence file reused across phases")
|
||||
seen.add(path)
|
||||
return path
|
||||
|
||||
|
||||
def samples(report, operation):
|
||||
rows = report.get("samples")
|
||||
require(isinstance(rows, list) and rows, "missing probe samples")
|
||||
return [row for row in rows if row.get("operation") == operation]
|
||||
|
||||
|
||||
def report_status(report):
|
||||
require(report.get("complete") is True, "probe report is incomplete")
|
||||
require(report.get("bulk_coverage_verified") is True, "probe report lacks validated bulk coverage")
|
||||
duration = uint(report.get("offered_duration_secs"), "offered duration")
|
||||
rate = uint(report.get("control_rps"), "control rate")
|
||||
expected = uint(report.get("control_expected"), "expected control count")
|
||||
require(duration > 0 and rate > 0 and expected == duration * rate, "invalid control offering contract")
|
||||
ping = samples(report, "Ping")
|
||||
read_all = samples(report, "ReadAll")
|
||||
require(len(ping) == expected and read_all, "probe sample coverage does not match its contract")
|
||||
allowed = {"ok", "deadline", "schedule_drop", "rpc_error"}
|
||||
require(all(row.get("outcome") in allowed for row in ping + read_all), "unknown probe outcome")
|
||||
observed_errors = sum(row.get("outcome") != "ok" for row in ping + read_all)
|
||||
require(uint(report.get("errors"), "probe error count") == observed_errors, "probe error count does not match samples")
|
||||
fixture = report.get("fixture_sha256")
|
||||
require(isinstance(fixture, str) and len(fixture) == 64 and all(char in "0123456789abcdef" for char in fixture),
|
||||
"invalid fixture identity")
|
||||
targets = uint(report.get("targets"), "target count")
|
||||
peers = uint(report.get("peers"), "peer count")
|
||||
workers = uint(report.get("bulk_workers"), "bulk worker count")
|
||||
scope = report.get("scope")
|
||||
require(targets > 0 and peers > 0 and workers > 0 and isinstance(scope, str) and scope,
|
||||
"invalid probe workload identity")
|
||||
identity = {"fixture_sha256": fixture, "targets": targets, "peers": peers, "bulk_workers": workers,
|
||||
"control_rps": rate, "scope": scope}
|
||||
return ping, read_all, identity
|
||||
|
||||
|
||||
def phase(report, expected):
|
||||
ping, read_all, identity = report_status(report)
|
||||
outcomes = [row["outcome"] for row in ping + read_all]
|
||||
if expected == "success":
|
||||
return not outcomes.count("deadline") and not outcomes.count("schedule_drop") and not outcomes.count("rpc_error"), identity
|
||||
require(expected == "deadline", "unknown timeout-boundary expectation")
|
||||
return outcomes.count("deadline") > 0 and not outcomes.count("schedule_drop") and not outcomes.count("rpc_error"), identity
|
||||
|
||||
|
||||
def check(manifest, root):
|
||||
root = root.resolve()
|
||||
require(isinstance(manifest, dict) and manifest.get("schema") == 1, "unsupported timeout-boundary schema")
|
||||
require(manifest.get("scope") == "internode-rpc-timeout-boundary", "wrong timeout-boundary scope")
|
||||
timeout = uint(manifest.get("rpc_timeout_ms"), "RPC timeout")
|
||||
require(timeout > 0, "RPC timeout must be positive")
|
||||
seen = set()
|
||||
receipts = []
|
||||
verdicts = {}
|
||||
workload_identity = None
|
||||
phases = (("under_budget", "success"), ("over_budget", "deadline"), ("recovery", "success"))
|
||||
for name, expectation in phases:
|
||||
descriptor = manifest.get(name)
|
||||
require(isinstance(descriptor, dict), f"missing {name} descriptor")
|
||||
if name != "recovery":
|
||||
fault_path = evidence_path(root, descriptor.get("fault_receipt"), seen)
|
||||
fault, fault_digest = load(fault_path, MAX_MANIFEST_BYTES)
|
||||
require(isinstance(fault, dict) and isinstance(fault.get("target"), str) and fault["target"],
|
||||
f"invalid {name} fault target")
|
||||
requested = uint(fault.get("requested_stall_ms"), f"{name} requested stall")
|
||||
actual = uint(fault.get("actual_stall_ms"), f"{name} actual stall")
|
||||
require(0 < requested <= actual, f"invalid {name} stall receipt")
|
||||
if name == "under_budget":
|
||||
require(actual < timeout, "under-budget stall crossed the RPC timeout")
|
||||
else:
|
||||
require(requested > timeout and actual > timeout, "over-budget stall did not cross the RPC timeout")
|
||||
receipts.append({"phase": name, "kind": "fault", "path": descriptor["fault_receipt"],
|
||||
"sha256": fault_digest})
|
||||
path = evidence_path(root, descriptor.get("report"), seen)
|
||||
report, digest = load(path, MAX_REPORT_BYTES)
|
||||
verdicts[name], identity = phase(report, expectation)
|
||||
if workload_identity is None:
|
||||
workload_identity = identity
|
||||
else:
|
||||
require(identity == workload_identity, "probe workload identity changed across timeout phases")
|
||||
receipts.append({"phase": name, "kind": "report", "path": descriptor["report"], "sha256": digest})
|
||||
return {
|
||||
"observed_pass": all(verdicts.values()),
|
||||
"scope": manifest["scope"],
|
||||
"rpc_timeout_ms": timeout,
|
||||
"phases": verdicts,
|
||||
"workload": workload_identity,
|
||||
"reports": receipts,
|
||||
"limits": "Recorded request outcomes only; deployment ownership, fault delivery, Prometheus windows and performance attribution remain separate evidence.",
|
||||
}, seen
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
require(args.manifest.is_file() and args.manifest.stat().st_size <= MAX_MANIFEST_BYTES,
|
||||
"missing or oversized timeout-boundary manifest")
|
||||
raw = args.manifest.read_bytes()
|
||||
manifest = json.loads(raw, object_pairs_hook=unique_object)
|
||||
result, inputs = check(manifest, args.manifest.resolve().parent)
|
||||
output = args.out.resolve()
|
||||
require(output not in inputs | {args.manifest.resolve()}, "output would overwrite raw evidence")
|
||||
if args.out.exists():
|
||||
require(not any(args.out.samefile(path) for path in inputs | {args.manifest.resolve()}),
|
||||
"output aliases raw evidence")
|
||||
result["manifest_sha256"] = hashlib.sha256(raw).hexdigest()
|
||||
args.out.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
|
||||
return 0 if result["observed_pass"] else 1
|
||||
except (ValueError, OSError, TypeError, KeyError, AttributeError, json.JSONDecodeError) as error:
|
||||
print(f"Invalid internode RPC timeout-boundary evidence: {error}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression cases for the internode RPC timeout-boundary evidence gate."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import check_internode_rpc_timeout_boundary as gate
|
||||
|
||||
|
||||
class TimeoutBoundaryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.directory.cleanup)
|
||||
self.root = Path(self.directory.name)
|
||||
self.reports = {
|
||||
"under": self.report(30, 10, []),
|
||||
"over": self.report(35, 10, [("Ping", "deadline"), ("ReadAll", "deadline")]),
|
||||
"recovery": self.report(15, 10, []),
|
||||
}
|
||||
for name, report in self.reports.items():
|
||||
(self.root / f"{name}.json").write_text(json.dumps(report))
|
||||
self.faults = {
|
||||
"under": {"target": "node4", "requested_stall_ms": 8_000, "actual_stall_ms": 9_100},
|
||||
"over": {"target": "node4", "requested_stall_ms": 12_000, "actual_stall_ms": 12_600},
|
||||
}
|
||||
self.manifest = {
|
||||
"schema": 1,
|
||||
"scope": "internode-rpc-timeout-boundary",
|
||||
"rpc_timeout_ms": 10_000,
|
||||
"under_budget": {"fault_receipt": "under-fault.json", "report": "under.json"},
|
||||
"over_budget": {"fault_receipt": "over-fault.json", "report": "over.json"},
|
||||
"recovery": {"report": "recovery.json"},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def report(duration, rate, failures):
|
||||
samples = []
|
||||
for sequence in range(duration * rate):
|
||||
samples.append({"operation": "Ping", "outcome": "ok", "peer": sequence % 4})
|
||||
for sequence in range(8):
|
||||
samples.append({"operation": "ReadAll", "outcome": "ok", "peer": sequence % 4})
|
||||
for operation, outcome in failures:
|
||||
row = next(sample for sample in samples if sample["operation"] == operation and sample["outcome"] == "ok")
|
||||
row["outcome"] = outcome
|
||||
return {
|
||||
"complete": True,
|
||||
"errors": len(failures),
|
||||
"bulk_coverage_verified": True,
|
||||
"offered_duration_secs": duration,
|
||||
"control_rps": rate,
|
||||
"control_expected": duration * rate,
|
||||
"fixture_sha256": "0" * 64,
|
||||
"targets": 4,
|
||||
"peers": 4,
|
||||
"bulk_workers": 2,
|
||||
"scope": "Authenticated production RPC clients",
|
||||
"samples": samples,
|
||||
}
|
||||
|
||||
def write_reports(self):
|
||||
for name, report in self.reports.items():
|
||||
(self.root / f"{name}.json").write_text(json.dumps(report))
|
||||
for name, fault in self.faults.items():
|
||||
(self.root / f"{name}-fault.json").write_text(json.dumps(fault))
|
||||
|
||||
def check(self):
|
||||
self.write_reports()
|
||||
return gate.check(self.manifest, self.root)[0]
|
||||
|
||||
def test_valid_boundary_and_recovery_pass(self):
|
||||
result = self.check()
|
||||
self.assertTrue(result["observed_pass"])
|
||||
self.assertEqual(result["phases"], {"under_budget": True, "over_budget": True, "recovery": True})
|
||||
|
||||
def test_schedule_drop_cannot_substitute_for_deadline(self):
|
||||
for report in ("under", "over", "recovery"):
|
||||
with self.subTest(report=report):
|
||||
original = copy.deepcopy(self.reports[report])
|
||||
row = next(sample for sample in self.reports[report]["samples"] if sample["operation"] == "Ping")
|
||||
was_ok = row["outcome"] == "ok"
|
||||
row["outcome"] = "schedule_drop"
|
||||
self.reports[report]["errors"] += was_ok
|
||||
self.assertFalse(self.check()["observed_pass"])
|
||||
self.reports[report] = original
|
||||
|
||||
def test_under_budget_errors_over_budget_success_and_recovery_errors_fail(self):
|
||||
cases = (("under", "deadline", 1), ("over", "ok", -2), ("recovery", "rpc_error", 1))
|
||||
for report, outcome, error_delta in cases:
|
||||
with self.subTest(report=report):
|
||||
original = copy.deepcopy(self.reports[report])
|
||||
if report == "over":
|
||||
for sample in self.reports[report]["samples"]:
|
||||
sample["outcome"] = "ok"
|
||||
else:
|
||||
next(sample for sample in self.reports[report]["samples"] if sample["operation"] == "Ping")["outcome"] = outcome
|
||||
self.reports[report]["errors"] += error_delta
|
||||
self.assertFalse(self.check()["observed_pass"])
|
||||
self.reports[report] = original
|
||||
|
||||
def test_invalid_stall_receipts_and_control_contract_fail_closed(self):
|
||||
edits = (
|
||||
lambda: self.faults["under"].update(actual_stall_ms=10_000),
|
||||
lambda: self.faults["over"].update(requested_stall_ms=9_000),
|
||||
lambda: self.faults["under"].update(requested_stall_ms=9_200),
|
||||
lambda: self.reports["under"].update(control_expected=1),
|
||||
)
|
||||
for edit in edits:
|
||||
with self.subTest(edit=edit):
|
||||
manifest = copy.deepcopy(self.manifest)
|
||||
reports = copy.deepcopy(self.reports)
|
||||
faults = copy.deepcopy(self.faults)
|
||||
edit()
|
||||
with self.assertRaises(ValueError):
|
||||
self.check()
|
||||
self.manifest, self.reports, self.faults = manifest, reports, faults
|
||||
|
||||
def test_paths_cannot_escape_or_reuse_evidence(self):
|
||||
self.manifest["under_budget"]["report"] = "../outside.json"
|
||||
with self.assertRaisesRegex(ValueError, "escapes"):
|
||||
self.check()
|
||||
self.manifest["under_budget"]["report"] = "under.json"
|
||||
self.manifest["recovery"]["report"] = "under.json"
|
||||
with self.assertRaisesRegex(ValueError, "reused"):
|
||||
self.check()
|
||||
|
||||
def test_probe_workload_identity_cannot_change_between_phases(self):
|
||||
for field, value in (("fixture_sha256", "1" * 64), ("targets", 3), ("peers", 3),
|
||||
("bulk_workers", 1), ("control_rps", 9), ("scope", "different")):
|
||||
with self.subTest(field=field):
|
||||
self.reports["recovery"][field] = value
|
||||
if field == "control_rps":
|
||||
self.reports["recovery"]["control_expected"] = self.reports["recovery"]["offered_duration_secs"] * value
|
||||
self.reports["recovery"]["samples"] = self.report(15, value, [])["samples"]
|
||||
with self.assertRaisesRegex(ValueError, "identity changed"):
|
||||
self.check()
|
||||
self.reports["recovery"] = self.report(15, 10, [])
|
||||
|
||||
def run_cli(self, output):
|
||||
self.write_reports()
|
||||
manifest = self.root / "manifest.json"
|
||||
manifest.write_text(json.dumps(self.manifest))
|
||||
return subprocess.run([sys.executable, str(Path(gate.__file__)), "--manifest", str(manifest),
|
||||
"--out", str(output)], capture_output=True, text=True)
|
||||
|
||||
def test_entry_point_distinguishes_pass_fail_and_invalid(self):
|
||||
output = self.root / "result.json"
|
||||
self.assertEqual(self.run_cli(output).returncode, 0)
|
||||
self.reports["recovery"]["samples"][0]["outcome"] = "deadline"
|
||||
self.reports["recovery"]["errors"] = 1
|
||||
self.assertEqual(self.run_cli(output).returncode, 1)
|
||||
self.manifest["scope"] = "command"
|
||||
self.assertEqual(self.run_cli(output).returncode, 2)
|
||||
|
||||
def test_entry_point_preserves_raw_and_hard_linked_evidence(self):
|
||||
raw = self.root / "under.json"
|
||||
self.write_reports()
|
||||
before = raw.read_bytes()
|
||||
self.assertEqual(self.run_cli(raw).returncode, 2)
|
||||
self.assertEqual(raw.read_bytes(), before)
|
||||
alias = self.root / "alias.json"
|
||||
os.link(raw, alias)
|
||||
self.assertEqual(self.run_cli(alias).returncode, 2)
|
||||
self.assertEqual(raw.read_bytes(), before)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user