Files
rustfs/scripts/check_internode_msgpack_peer_canary.py
T
Hauser Zhi fb9f19c6d2 fix(rpc): scope canary errors to grpc failures (#9453)
* fix(rpc): scope canary errors to grpc failures

Exclude expected read-version remote responses without hiding transport, gRPC status, response-decode, or other gRPC operation failures.

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* fix(rpc): verify canary revisions from image history

Use immutable image history only when recognized config labels are absent, keep mismatches fail-closed, and record the provenance source in verifier output.

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* fix(audit): restore bucket config audits and metadata guards

Audit accelerate, request-payment, and website configuration mutations through the shared owned request scope, including failures and the website deletion status.

Validate the replication mutation delegation before checking peer reload notifications. Count borrowed reserved-prefix guard calls and cover missing production calls, test-only mentions, audit completion, and stale bucket incarnations.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

---------

Co-authored-by: zhi22915 <qiuzgang@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
2026-11-16 13:57:56 +08:00

374 lines
18 KiB
Python

#!/usr/bin/env python3
"""Validate measured mixed-version evidence for the internode msgpack peer gate."""
from __future__ import annotations
import argparse
import hashlib
import json
import math
from pathlib import Path
import re
import sys
from typing import Any
PHASES = ("mixed-old-peer", "all-new", "rollback-old-peer")
TOPOLOGIES = {
"mixed-old-peer": (3, 1),
"all-new": (4, 0),
"rollback-old-peer": (3, 1),
}
OLD_NODES = {
"mixed-old-peer": {"node4"},
"all-new": set(),
"rollback-old-peer": {"node2"},
}
ARTIFACTS = ("decisions", "fallback", "decode_errors", "rpc_errors")
DECISIONS = {"msgpack_only", "dual_write"}
REASONS = {"manual_fleet", "peer_capable", "peer_unknown", "peer_revoked", "not_requested"}
REVISION = re.compile(r"^[0-9a-f]{40}$")
SHA256 = re.compile(r"^[0-9a-f]{64}$")
MAX_DOCUMENT_BYTES = 1024 * 1024
class EvidenceError(ValueError):
pass
def require(condition: bool, message: str) -> None:
if not condition:
raise EvidenceError(message)
def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
result: dict[str, Any] = {}
for key, value in pairs:
require(key not in result, f"duplicate JSON key: {key}")
result[key] = value
return result
def load_json(path: Path) -> tuple[dict[str, Any], str]:
require(path.is_file(), f"missing evidence artifact: {path}")
size = path.stat().st_size
require(0 < size <= MAX_DOCUMENT_BYTES, f"invalid evidence artifact size: {path}")
raw = path.read_bytes()
document = json.loads(raw, object_pairs_hook=unique_object)
require(isinstance(document, dict), f"evidence artifact must be an object: {path}")
return document, hashlib.sha256(raw).hexdigest()
def load_raw(path: Path) -> str:
require(path.is_file(), f"missing evidence artifact: {path}")
size = path.stat().st_size
require(0 < size <= MAX_DOCUMENT_BYTES, f"invalid evidence artifact size: {path}")
return hashlib.sha256(path.read_bytes()).hexdigest()
def bounded_number(value: Any, name: str) -> float:
require(type(value) in (int, float), f"{name} must be numeric")
result = float(value)
require(math.isfinite(result) and result >= 0, f"{name} must be finite and non-negative")
return result
def uint(value: Any, name: str) -> int:
require(type(value) is int and value >= 0, f"{name} must be a non-negative integer")
return value
def timestamped_value(row: dict[str, Any], start: float, end: float, lag: float, name: str) -> float:
value = row.get("value")
require(isinstance(value, list) and len(value) == 2, f"{name} must contain one Prometheus value")
timestamp = bounded_number(value[0], f"{name} timestamp")
require(end <= timestamp <= end + lag, f"{name} timestamp is outside its phase completion window")
try:
number = float(value[1])
except (TypeError, ValueError) as error:
raise EvidenceError(f"{name} value is not numeric") from error
require(math.isfinite(number) and number >= 0, f"{name} value must be finite and non-negative")
return number
def prometheus_rows(document: dict[str, Any], name: str) -> list[dict[str, Any]]:
require(document.get("status") == "success", f"{name} Prometheus query failed")
data = document.get("data")
require(isinstance(data, dict) and data.get("resultType") == "vector", f"{name} must be a Prometheus vector")
rows = data.get("result")
require(isinstance(rows, list) and len(rows) <= 64, f"{name} has invalid vector cardinality")
require(all(isinstance(row, dict) for row in rows), f"{name} contains a non-object result")
return rows
def decision_values(document: dict[str, Any], start: float, end: float, lag: float) -> dict[tuple[str, str], float]:
result: dict[tuple[str, str], float] = {}
for index, row in enumerate(prometheus_rows(document, "decisions")):
labels = row.get("metric")
require(isinstance(labels, dict) and set(labels) == {"decision", "reason"},
"decision evidence must contain only decision/reason labels")
decision = labels.get("decision")
reason = labels.get("reason")
require(decision in DECISIONS and reason in REASONS, "decision evidence contains an unbounded label")
key = (decision, reason)
require(key not in result, f"duplicate decision series: {key}")
result[key] = timestamped_value(row, start, end, lag, f"decision[{index}]")
return result
def scalar_sum(document: dict[str, Any], start: float, end: float, lag: float, name: str) -> float:
total = 0.0
rows = prometheus_rows(document, name)
require(len(rows) == 1, f"{name} scalar query must return exactly one row")
for index, row in enumerate(rows):
labels = row.get("metric")
require(labels == {}, f"{name} scalar query must not retain labels")
total += timestamped_value(row, start, end, lag, f"{name}[{index}]")
return total
def artifact_path(root: Path, relative: Any, used: set[Path], name: str) -> Path:
require(isinstance(relative, str) and relative and not Path(relative).is_absolute(), f"invalid {name} artifact path")
path = (root / relative).resolve()
try:
path.relative_to(root)
except ValueError as error:
raise EvidenceError(f"{name} artifact escapes the evidence directory") from error
require(path not in used, f"artifact reused across phases: {relative}")
used.add(path)
return path
def bool_value(value: Any, name: str) -> bool:
require(type(value) is bool, f"{name} must be boolean")
return value
def exact_env(document: Any, name: str, old: bool) -> None:
require(isinstance(document, dict), f"{name} environment must be an object")
expected = {
"msgpack_only": not old,
"peer_capability": not old,
"fleet_confirmed": False,
}
require(document == expected, f"{name} environment does not match the peer-gate canary contract")
def expected_queries(window: int, cluster_id: str) -> dict[str, str]:
selector = f'{{rustfs_cluster_id="{cluster_id}"}}'
grpc_selector = f'{{rustfs_cluster_id="{cluster_id}",backend="grpc",operation!="grpc_read_version"}}'
read_version_selector = (
f'{{rustfs_cluster_id="{cluster_id}",backend="grpc",operation="grpc_read_version",'
'classification!="remote_response"}'
)
return {
"decisions": (
"sum by (decision, reason) "
f"(increase(rustfs_system_network_internode_msgpack_send_decisions_total{selector}[{window}s]))"
),
"fallback": f"sum(increase(rustfs_system_network_internode_msgpack_json_fallback_total{selector}[{window}s])) or vector(0)",
"decode_errors": (
f"sum(increase(rustfs_system_network_internode_msgpack_json_decode_error_total{selector}[{window}s])) or vector(0)"
),
"rpc_errors": (
f"(sum(increase(rustfs_system_network_internode_operation_errors_total{grpc_selector}[{window}s])) "
"or vector(0)) + "
"(sum(increase(rustfs_system_network_internode_operation_classified_errors_total"
f"{read_version_selector}[{window}s])) or vector(0))"
),
}
def check(manifest: dict[str, Any], root: Path) -> dict[str, Any]:
root = root.resolve()
require(type(manifest.get("schema")) is int and manifest.get("schema") == 1, "unsupported canary evidence schema")
require(manifest.get("scope") == "internode-msgpack-peer-canary", "unexpected evidence scope")
source = manifest.get("source_revision")
baseline = manifest.get("baseline_revision")
require(isinstance(source, str) and REVISION.fullmatch(source) is not None, "invalid source revision")
require(isinstance(baseline, str) and REVISION.fullmatch(baseline) is not None, "invalid baseline revision")
require(source != baseline, "candidate and baseline revisions must differ")
require(bool_value(manifest.get("stable_connection_affinity"), "stable_connection_affinity"),
"stable connection-to-peer affinity is required")
cluster_id = manifest.get("cluster_id")
require(isinstance(cluster_id, str) and re.fullmatch(r"msgpack-[0-9a-f]{32}", cluster_id) is not None,
"invalid canary cluster identity")
lag = bounded_number(manifest.get("max_metric_lag_secs"), "max_metric_lag_secs")
require(0 <= lag <= 300, "max_metric_lag_secs exceeds the bounded limit")
phases = manifest.get("phases")
require(isinstance(phases, list) and len(phases) == len(PHASES), "exactly three canary phases are required")
used: set[Path] = set()
receipts: list[dict[str, Any]] = []
summaries: list[dict[str, Any]] = []
violations: list[str] = []
image_path = artifact_path(root, manifest.get("images"), used, "images")
image_digest = load_raw(image_path)
images = json.loads(image_path.read_bytes(), object_pairs_hook=unique_object)
require(isinstance(images, list) and len(images) == 2, "expected candidate and baseline image inspections")
image_ids = []
revision_evidence = []
for image, revision in zip(images, (source, baseline)):
require(isinstance(image, dict), "invalid image inspection")
image_id = image.get("Id")
require(isinstance(image_id, str) and re.fullmatch(r"sha256:[0-9a-f]{64}", image_id) is not None,
"invalid immutable image ID")
require(image.get("RevisionEvidence") in {"config-labels", "immutable-history"},
"invalid image revision evidence source")
config = image.get("Config")
require(isinstance(config, dict), "invalid image config")
labels = config.get("Labels")
require(isinstance(labels, dict), "invalid image labels")
observed = [labels[key] for key in ("vcs-ref", "org.opencontainers.image.revision") if key in labels]
require(bool(observed) and all(value == revision for value in observed), "image revision mismatch")
image_ids.append(image_id)
revision_evidence.append(image["RevisionEvidence"])
require(image_ids[0] != image_ids[1], "candidate and baseline resolve to the same image")
receipts.append({"kind": "images", "path": manifest["images"], "sha256": image_digest})
previous_end = 0.0
for expected_name, phase in zip(PHASES, phases):
require(isinstance(phase, dict) and phase.get("name") == expected_name, "canary phases are missing or out of order")
start = uint(phase.get("start_unix"), f"{expected_name} start")
end = uint(phase.get("end_unix"), f"{expected_name} end")
require(previous_end < start < end and end - start <= 3600, f"invalid {expected_name} measurement window")
previous_end = end
topology = phase.get("topology")
require(isinstance(topology, dict), f"{expected_name} topology must be an object")
expected_new, expected_old = TOPOLOGIES[expected_name]
nodes = topology.get("nodes")
require(isinstance(nodes, dict) and set(nodes) == {"node1", "node2", "node3", "node4"},
f"{expected_name} node inventory is incomplete")
expected_nodes = {
node: baseline if node in OLD_NODES[expected_name] else source
for node in ("node1", "node2", "node3", "node4")
}
require(nodes == expected_nodes, f"{expected_name} node revisions do not match the mixed-version matrix")
running_path = artifact_path(root, phase.get("images"), used, f"{expected_name}.images")
running, running_digest = load_json(running_path)
expected_images = {
node: {"image_id": image_ids[1] if node in OLD_NODES[expected_name] else image_ids[0], "revision": revision}
for node, revision in expected_nodes.items()
}
require(running == expected_images, f"{expected_name} running images do not match verified identities")
receipts.append({"phase": expected_name, "kind": "running-images", "path": phase["images"],
"sha256": running_digest})
new_nodes = uint(topology.get("new_nodes"), f"{expected_name} new_nodes")
old_nodes = uint(topology.get("old_nodes"), f"{expected_name} old_nodes")
require(new_nodes == expected_new and old_nodes == expected_old,
f"{expected_name} topology counts do not match the four-node matrix")
environments = phase.get("environment")
require(isinstance(environments, dict), f"{expected_name} environment is missing")
exact_env(environments.get("new_nodes"), f"{expected_name} new_nodes", old=False)
exact_env(environments.get("old_nodes"), f"{expected_name} old_nodes", old=True)
queries = phase.get("queries")
require(queries == expected_queries(end - start, cluster_id), f"{expected_name} PromQL does not match its exact window")
workload_path = artifact_path(root, phase.get("workload"), used, f"{expected_name}.workload")
workload, workload_digest = load_json(workload_path)
summary_path = artifact_path(root, workload.get("summary"), used, f"{expected_name}.workload-summary")
summary_digest = load_raw(summary_path)
require(workload.get("summary_sha256") == summary_digest, f"{expected_name} workload summary hash mismatch")
workload_success = workload.get("status") == "success"
requests = uint(workload.get("requests"), f"{expected_name} workload requests")
require(requests > 0, f"{expected_name} workload has no requests")
if not workload_success:
violations.append(f"{expected_name}: workload failed")
rollback_readback = bool_value(workload.get("rollback_readback"), f"{expected_name} rollback_readback")
if expected_name != "rollback-old-peer":
require(not rollback_readback, f"{expected_name} cannot claim rollback readback")
if expected_name == "rollback-old-peer" and not rollback_readback:
violations.append(f"{expected_name}: rollback readback failed")
source_digest = workload.get("readback_source_sha256")
downloaded_digest = workload.get("readback_download_sha256")
if expected_name == "rollback-old-peer":
require(isinstance(source_digest, str) and SHA256.fullmatch(source_digest) is not None,
"rollback source digest is invalid")
require(source_digest == downloaded_digest, "rollback readback digest mismatch")
else:
require(source_digest is None and downloaded_digest is None,
f"{expected_name} cannot claim rollback content digests")
receipts.append({"phase": expected_name, "kind": "workload", "path": phase["workload"],
"sha256": workload_digest})
receipts.append({"phase": expected_name, "kind": "workload-summary", "path": workload["summary"],
"sha256": summary_digest})
artifacts = phase.get("artifacts")
require(isinstance(artifacts, dict) and set(artifacts) == set(ARTIFACTS), f"{expected_name} artifact set is incomplete")
documents: dict[str, dict[str, Any]] = {}
for artifact in ARTIFACTS:
path = artifact_path(root, artifacts[artifact], used, f"{expected_name}.{artifact}")
document, digest = load_json(path)
documents[artifact] = document
receipts.append({"phase": expected_name, "kind": artifact, "path": artifacts[artifact], "sha256": digest})
decisions = decision_values(documents["decisions"], start, end, lag)
peer_capable = decisions.get(("msgpack_only", "peer_capable"), 0.0)
compatibility = decisions.get(("dual_write", "peer_unknown"), 0.0) + decisions.get(
("dual_write", "peer_revoked"), 0.0
)
if peer_capable <= 0:
violations.append(f"{expected_name}: no peer_capable msgpack-only decision")
if expected_old > 0 and compatibility <= 0:
violations.append(f"{expected_name}: no old/unknown peer dual-write decision")
if decisions.get(("msgpack_only", "manual_fleet"), 0.0) > 0:
violations.append(f"{expected_name}: manual fleet path was unexpectedly active")
if decisions.get(("dual_write", "not_requested"), 0.0) > 0:
violations.append(f"{expected_name}: peer gate was not active on an eligible new-node request")
errors = {
"fallback": scalar_sum(documents["fallback"], start, end, lag, "fallback"),
"decode_errors": scalar_sum(documents["decode_errors"], start, end, lag, "decode_errors"),
"rpc_errors": scalar_sum(documents["rpc_errors"], start, end, lag, "rpc_errors"),
}
for name, value in errors.items():
if value != 0:
violations.append(f"{expected_name}: nonzero {name} ({value})")
summaries.append({
"name": expected_name,
"start_unix": start,
"end_unix": end,
"topology": topology,
"workload_requests": requests,
"peer_capable_decisions": peer_capable,
"compatibility_decisions": compatibility,
**errors,
})
return {
"observed_pass": not violations,
"scope": manifest["scope"],
"source_revision": source,
"baseline_revision": baseline,
"cluster_id": cluster_id,
"image_revision_evidence": {
"candidate": revision_evidence[0],
"baseline": revision_evidence[1],
},
"phases": summaries,
"violations": violations,
"artifacts": receipts,
"limits": "Measured four-node compatibility evidence only; production soak and performance drift gates remain required.",
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
try:
root = args.manifest.resolve().parent
manifest, digest = load_json(args.manifest.resolve())
result = check(manifest, root)
inputs = {args.manifest.resolve()} | {(root / item["path"]).resolve() for item in result["artifacts"]}
require(args.out.resolve() not in inputs, "output would overwrite raw canary evidence")
result["manifest_sha256"] = digest
args.out.write_text(json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + "\n", encoding="utf-8")
return 0 if result["observed_pass"] else 1
except (EvidenceError, OSError, TypeError, KeyError, AttributeError, UnicodeError, json.JSONDecodeError) as error:
print(f"Invalid internode msgpack peer canary evidence: {error}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())