mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-30 15:55:55 +00:00
Hauser
be485e6cab
* test(scanner): add memory acceptance runner and budget probes Add owned S3 workload phases with process, endpoint, cgroup and binary identity checks for scanner memory and scanner-disabled request bursts. Keep scale, restart, heap-attribution and overall-acceptance limits explicit. Characterize healthy and seeded coverage-gap cache retention through the real folder walker and production cache codec without changing runtime code. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(scanner): bind acceptance capacity to effective storage Validate the serving process's complete local disk inventory before load writes and during runtime sampling. Resolve each disk inside the owned storage root, reject nested mounts and filesystem mismatches, and bind storage observations to phase identity for container and native runs. Cover the unused-mount bypass, CLI-overridden environment, native paths, symlink escapes, named volumes and stable multi-disk inventory ordering. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * fix(scanner): fence status observations before workload writes Close each status API window with a second storage-bound process sample before Phase entry can authorize bucket creation or PUT requests. Add a fail-before regression for a process change during the initial status request, asserting no ownership or workload writes occur. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com> Co-authored-by: zhi22915 <qiuzgang@gmail.com>
955 lines
51 KiB
Python
955 lines
51 KiB
Python
#!/usr/bin/env python3
|
|
"""Run owned S3 scanner-memory acceptance phases; service lifecycle stays external."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
import fcntl
|
|
import hashlib
|
|
import ipaddress
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from urllib.parse import urlsplit
|
|
import urllib.request
|
|
import uuid
|
|
|
|
SCHEMA = 1
|
|
OWNER_KEY = ".scanner-memory-owner.json"
|
|
LABEL = "rustfs.scanner-memory.run-id"
|
|
GIB = 1024 ** 3
|
|
SAFE_ENV = frozenset({
|
|
"RUSTFS_SCANNER_ENABLED", "RUSTFS_HEAL_ENABLED", "RUSTFS_SCANNER_SPEED",
|
|
"RUSTFS_SCANNER_START_DELAY_SECS", "RUSTFS_SCANNER_CYCLE", "RUSTFS_SCANNER_IDLE_MODE",
|
|
"RUSTFS_SCANNER_CYCLE_MAX_OBJECTS", "RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS",
|
|
"RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES", "RUSTFS_SCANNER_DISK_IO_BUDGET_IOPS",
|
|
"RUSTFS_SCANNER_DISK_IO_BUDGET_BYTES_PER_SEC",
|
|
"RUSTFS_CONSOLE_ENABLE", "RUSTFS_ERASURE_SET_DRIVE_COUNT", "RUSTFS_DURABILITY_MODE",
|
|
"RUSTFS_MEMORY_OBSERVABILITY_INTERVAL_SECS", "RUSTFS_OBS_METRICS_EXPORT_ENABLED",
|
|
})
|
|
STATUS_FIELDS = (
|
|
"current_cycle", "current_cycle_active", "current_cycle_objects_scanned",
|
|
"current_cycle_directories_scanned", "last_cycle_result", "last_cycle_partial_reason",
|
|
"last_cycle_end_unix_secs", "last_cycle_objects_scanned", "failed_cycles",
|
|
"current_disk_bucket_scans_active", "last_cycle_bucket_drive_failures", "usage_publication",
|
|
)
|
|
|
|
|
|
class AcceptanceError(ValueError):
|
|
"""A public-safe diagnostic created by this runner, never by an SDK."""
|
|
|
|
|
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, *_args, **_kwargs):
|
|
raise AcceptanceError("redirected observation endpoints are not supported")
|
|
|
|
|
|
def require(condition, message):
|
|
if not condition:
|
|
raise AcceptanceError(message)
|
|
|
|
|
|
def canonical(value):
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
|
|
|
|
|
|
def digest(value):
|
|
return hashlib.sha256(canonical(value)).hexdigest()
|
|
|
|
|
|
def utc():
|
|
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def local_url(value):
|
|
parsed = urlsplit(value)
|
|
require(parsed.scheme in ("http", "https") and parsed.hostname in ("127.0.0.1", "::1")
|
|
and parsed.port is not None and not parsed.username and not parsed.password
|
|
and not parsed.query and not parsed.fragment and parsed.path in ("", "/"),
|
|
"endpoint must use literal 127.0.0.1 or ::1 with an explicit HTTP(S) port")
|
|
return value.rstrip("/")
|
|
|
|
|
|
@contextmanager
|
|
def open_owned(path, mode="r"):
|
|
require(mode in ("r", "a"), "unsupported evidence file mode")
|
|
flags = os.O_RDONLY if mode == "r" else os.O_WRONLY | os.O_CREAT | os.O_APPEND
|
|
try:
|
|
fd = os.open(path, flags | os.O_NOFOLLOW | os.O_NONBLOCK, 0o600)
|
|
except OSError:
|
|
raise AcceptanceError("evidence file is not an owned regular file") from None
|
|
try:
|
|
info = os.fstat(fd)
|
|
require(stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid() and info.st_nlink == 1
|
|
and info.st_mode & 0o077 == 0, "evidence file must be private, owned and single-link")
|
|
if mode == "a":
|
|
os.fchmod(fd, 0o600)
|
|
stream = os.fdopen(fd, mode, encoding="utf-8")
|
|
fd = None
|
|
with stream:
|
|
yield stream
|
|
finally:
|
|
if fd is not None:
|
|
os.close(fd)
|
|
|
|
|
|
def read_json(path, limit=1024 * 1024):
|
|
with open_owned(path) as stream:
|
|
text = stream.read(limit + 1)
|
|
require(len(text) <= limit, "oversized evidence file")
|
|
return json.loads(text)
|
|
|
|
|
|
def atomic_json(path, value):
|
|
temporary = path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp")
|
|
with temporary.open("x", encoding="utf-8") as stream:
|
|
os.chmod(temporary, 0o600)
|
|
json.dump(value, stream, sort_keys=True)
|
|
stream.write("\n")
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(temporary, path)
|
|
|
|
|
|
def object_key(plan, index):
|
|
require(type(index) is int and 0 <= index < plan["objects"], "object index outside manifest")
|
|
prefix = index % plan["root_prefixes"]
|
|
ordinal = index // plan["root_prefixes"]
|
|
suffix = f"source:{prefix:06d}/{ordinal:08d}.webp"
|
|
if plan["layout"] == "wide":
|
|
return suffix
|
|
return f"photos/{prefix % 256:02x}/{(prefix // 256) % 256:02x}/" + suffix
|
|
|
|
|
|
def key_index(plan, key):
|
|
pattern = r"source:([0-9]{6,})/([0-9]{8,})\.webp"
|
|
if plan["layout"] == "nested":
|
|
pattern = r"photos/[0-9a-f]{2}/[0-9a-f]{2}/" + pattern
|
|
match = re.fullmatch(pattern, key)
|
|
require(match is not None, "inventory contains a foreign object key")
|
|
prefix, ordinal = map(int, match.groups())
|
|
index = ordinal * plan["root_prefixes"] + prefix
|
|
require(prefix < plan["root_prefixes"] and index < plan["objects"] and object_key(plan, index) == key,
|
|
"inventory key does not match the immutable layout")
|
|
return index
|
|
|
|
|
|
def payload(plan):
|
|
seed = uuid.UUID(plan["run_id"]).bytes
|
|
blocks = (hashlib.sha256(seed + index.to_bytes(8, "big")).digest()
|
|
for index in range((plan["object_bytes"] + 31) // 32))
|
|
return b"".join(blocks)[:plan["object_bytes"]]
|
|
|
|
|
|
def owner_body(manifest):
|
|
return canonical({"schema": SCHEMA, "run_id": manifest["plan"]["run_id"], "plan_sha256": manifest["plan_sha256"]})
|
|
|
|
|
|
def validate_manifest(manifest):
|
|
require(type(manifest) is dict and set(manifest) == {"schema", "plan", "plan_sha256"}
|
|
and manifest["schema"] == SCHEMA, "invalid manifest envelope")
|
|
plan = manifest["plan"]
|
|
require(type(plan) is dict and set(plan) == {
|
|
"run_id", "created_at", "endpoint", "source_sha", "binary_sha256", "layout", "objects",
|
|
"object_bytes", "root_prefixes", "memory_bytes", "data_root", "data_volume", "bucket",
|
|
}, "invalid immutable plan fields")
|
|
require(digest(plan) == manifest["plan_sha256"], "manifest digest mismatch")
|
|
require(str(uuid.UUID(plan["run_id"])) == plan["run_id"], "invalid run identity")
|
|
require(plan["bucket"] == "scanner-memory-" + uuid.UUID(plan["run_id"]).hex, "bucket is not run-owned")
|
|
local_url(plan["endpoint"])
|
|
require(re.fullmatch(r"[0-9a-f]{40}", plan["source_sha"]) and re.fullmatch(r"[0-9a-f]{64}", plan["binary_sha256"]),
|
|
"invalid source or binary identity")
|
|
require(plan["layout"] in ("wide", "nested"), "invalid layout")
|
|
for name, maximum in (("objects", 20_000_000), ("object_bytes", 1024 * 1024),
|
|
("root_prefixes", 2_000_000), ("memory_bytes", 64 * GIB)):
|
|
require(type(plan[name]) is int and 0 < plan[name] <= maximum, "invalid workload bound")
|
|
require(plan["root_prefixes"] <= plan["objects"], "root prefix count exceeds object count")
|
|
require(bool(plan["data_root"]) != bool(plan["data_volume"]), "select one immutable storage source")
|
|
if plan["data_volume"]:
|
|
require(isinstance(plan["data_volume"], str) and re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}", plan["data_volume"]),
|
|
"invalid owned data volume name")
|
|
else:
|
|
path = Path(plan["data_root"])
|
|
require(path.is_absolute() and path == path.resolve() and path.is_dir(), "data root must be an existing canonical directory")
|
|
return plan
|
|
|
|
|
|
class Evidence:
|
|
def __init__(self, root):
|
|
self.root = root.resolve()
|
|
require(root == self.root and root.is_dir() and not root.is_symlink(), "run directory must be canonical")
|
|
require(root.stat().st_uid == os.getuid() and root.stat().st_mode & 0o077 == 0, "run directory must be private and owned")
|
|
self.manifest = read_json(root / "manifest.json")
|
|
self.plan = validate_manifest(self.manifest)
|
|
self.lock = threading.Lock()
|
|
|
|
@contextmanager
|
|
def exclusive(self):
|
|
fd = os.open(self.root / ".lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, "w") as stream:
|
|
info = os.fstat(stream.fileno())
|
|
require(stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid() and info.st_nlink == 1
|
|
and info.st_mode & 0o077 == 0, "invalid evidence lock ownership")
|
|
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
yield
|
|
|
|
def append(self, kind, **fields):
|
|
row = {"schema": SCHEMA, "at": utc(), "monotonic_ns": time.monotonic_ns(), "kind": kind,
|
|
"run_id": self.plan["run_id"], **fields}
|
|
with self.lock, open_owned(self.root / "observations.jsonl", "a") as stream:
|
|
stream.write(json.dumps(row, sort_keys=True) + "\n")
|
|
stream.flush()
|
|
return row
|
|
|
|
def state(self):
|
|
path = self.root / "state.json"
|
|
state = read_json(path) if path.exists() or path.is_symlink() else {"plan_sha256": self.manifest["plan_sha256"], "next_object": 0}
|
|
require(state.get("plan_sha256") == self.manifest["plan_sha256"], "resume state belongs to another plan")
|
|
require(type(state.get("next_object")) is int and 0 <= state["next_object"] <= self.plan["objects"], "invalid resume marker")
|
|
return state
|
|
|
|
def save(self, state):
|
|
atomic_json(self.root / "state.json", state)
|
|
|
|
|
|
class S3:
|
|
def __init__(self, evidence, workers):
|
|
import boto3
|
|
from botocore.config import Config
|
|
self.evidence = evidence
|
|
self.plan = evidence.plan
|
|
self.session = boto3.Session()
|
|
require(os.environ.get("AWS_ACCESS_KEY_ID") and os.environ.get("AWS_SECRET_ACCESS_KEY"),
|
|
"set disposable AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in the environment")
|
|
self.client = self.session.client("s3", endpoint_url=self.plan["endpoint"], region_name="us-east-1", config=Config(
|
|
signature_version="s3v4", s3={"addressing_style": "path"}, proxies={}, connect_timeout=5, read_timeout=30,
|
|
retries={"total_max_attempts": 2}, max_pool_connections=workers))
|
|
|
|
def admin(self, path):
|
|
from botocore.auth import S3SigV4Auth
|
|
from botocore.awsrequest import AWSRequest
|
|
request = AWSRequest(method="GET", url=self.plan["endpoint"] + path)
|
|
S3SigV4Auth(self.session.get_credentials().get_frozen_credentials(), "s3", "us-east-1").add_auth(request)
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect())
|
|
with opener.open(urllib.request.Request(request.url, headers=dict(request.headers)), timeout=15) as response:
|
|
raw = response.read(4 * 1024 * 1024 + 1)
|
|
require(len(raw) <= 4 * 1024 * 1024, "admin response exceeds bound")
|
|
return json.loads(raw)
|
|
|
|
def storage_paths(self):
|
|
# Ask the serving process for its expanded, effective disk inventory;
|
|
# Docker Config.Env and /proc/environ alone do not describe CLI overrides.
|
|
info = self.admin("/rustfs/admin/v3/storageinfo").get("info", {})
|
|
observations = info.get("observations", [])
|
|
require(len(observations) == 1 and observations[0].get("status") == "succeeded"
|
|
and observations[0].get("cached") is False,
|
|
"storage inventory must be a fresh single-node observation")
|
|
disks = info.get("disks", [])
|
|
backend = info.get("backend", {})
|
|
drives, sets = backend.get("DrivesPerSet", []), backend.get("TotalSets", [])
|
|
require(drives and len(drives) == len(sets)
|
|
and all(type(n) is int and n > 0 for n in drives + sets)
|
|
and sum(n * count for n, count in zip(drives, sets)) == len(disks),
|
|
"storage inventory does not cover the configured topology")
|
|
require(all(disk.get("local") is True and disk.get("state") == "ok" for disk in disks),
|
|
"storage inventory must contain only healthy local disks")
|
|
return [disk.get("path") for disk in disks]
|
|
|
|
def status(self, enabled):
|
|
status = self.admin("/rustfs/admin/v3/scanner/status")
|
|
require(status.get("enabled") is enabled, "scanner enabled state differs from required phase")
|
|
metrics = status.get("metrics")
|
|
require(type(metrics) is dict, "scanner metrics are unavailable")
|
|
return {"enabled": enabled, "metrics": {key: metrics.get(key) for key in STATUS_FIELDS}}
|
|
|
|
def owned(self, create=False):
|
|
from botocore.exceptions import ClientError
|
|
bucket = self.plan["bucket"]
|
|
try:
|
|
self.client.head_bucket(Bucket=bucket)
|
|
except ClientError as error:
|
|
require(error.response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404 and create,
|
|
"owned bucket is unavailable")
|
|
self.client.create_bucket(Bucket=bucket)
|
|
self.client.put_object(Bucket=bucket, Key=OWNER_KEY, Body=owner_body(self.evidence.manifest),
|
|
ContentType="application/json")
|
|
response = self.client.get_object(Bucket=bucket, Key=OWNER_KEY)
|
|
with response["Body"] as body:
|
|
marker = body.read(4097)
|
|
require(marker == owner_body(self.evidence.manifest), "bucket ownership marker mismatch")
|
|
require(self.client.get_bucket_versioning(Bucket=bucket).get("Status") is None,
|
|
"acceptance bucket must remain unversioned")
|
|
|
|
def verify(self, samples=16, deadline=None, guard=None):
|
|
deadline = deadline or time.monotonic() + 3600
|
|
def check():
|
|
require(time.monotonic() <= deadline, "inventory verification time budget exhausted")
|
|
if guard:
|
|
guard()
|
|
check()
|
|
self.owned()
|
|
plan = self.plan
|
|
seen = bytearray((plan["objects"] + 7) // 8)
|
|
count = size = markers = 0
|
|
token = None
|
|
while True:
|
|
check()
|
|
args = {"Bucket": plan["bucket"], "MaxKeys": 1000}
|
|
if token:
|
|
args["ContinuationToken"] = token
|
|
page = self.client.list_objects_v2(**args)
|
|
for entry in page.get("Contents", []):
|
|
if entry["Key"] == OWNER_KEY:
|
|
markers += 1
|
|
require(entry["Size"] == len(owner_body(self.evidence.manifest)), "ownership object size changed")
|
|
continue
|
|
index = key_index(plan, entry["Key"])
|
|
mask = 1 << (index % 8)
|
|
require(not seen[index // 8] & mask, "listing contains a duplicate object")
|
|
seen[index // 8] |= mask
|
|
require(entry["Size"] == plan["object_bytes"], "object size mismatch")
|
|
count += 1
|
|
size += entry["Size"]
|
|
if not page.get("IsTruncated"):
|
|
break
|
|
next_token = page.get("NextContinuationToken")
|
|
require(isinstance(next_token, str) and next_token and next_token != token, "listing cursor did not advance")
|
|
token = next_token
|
|
require(count == plan["objects"] and markers == 1 and size == count * plan["object_bytes"],
|
|
"exact object/byte inventory does not match manifest")
|
|
expected = payload(plan)
|
|
indices = sorted({index * (plan["objects"] - 1) // max(1, samples - 1) for index in range(samples)})
|
|
for index in indices:
|
|
check()
|
|
response = self.client.get_object(Bucket=plan["bucket"], Key=object_key(plan, index))
|
|
with response["Body"] as body:
|
|
actual = body.read(plan["object_bytes"] + 1)
|
|
require(actual == expected, "sampled payload differs from fixture")
|
|
result = {"objects": count, "logical_bytes": size, "ownership_objects": markers,
|
|
"scanner_expected_objects": count + 1,
|
|
"scanner_expected_bytes": size + len(owner_body(self.evidence.manifest)), "payload_samples": len(indices)}
|
|
check()
|
|
self.evidence.append("inventory", **result)
|
|
return result
|
|
|
|
def usage(self, expected):
|
|
account = self.admin("/rustfs/admin/v3/accountinfo")
|
|
matches = [bucket for bucket in account.get("buckets", []) if bucket.get("name") == self.plan["bucket"]]
|
|
require(len(matches) == 1, "bucket usage is unavailable")
|
|
bucket = matches[0]
|
|
return {"objects": bucket.get("objects"), "size": bucket.get("size"),
|
|
"converged": account.get("usage_snapshot_converged"),
|
|
"matches_inventory": bucket.get("objects") == expected["scanner_expected_objects"]
|
|
and bucket.get("size") == expected["scanner_expected_bytes"]}
|
|
|
|
|
|
def command(argv, timeout=30):
|
|
result = subprocess.run(argv, capture_output=True, timeout=timeout, check=False)
|
|
require(result.returncode == 0 and len(result.stdout) <= 1024 * 1024, "runtime observation command failed")
|
|
return result.stdout
|
|
|
|
|
|
def key_values(raw, colon=False):
|
|
result = {}
|
|
for line in raw.splitlines():
|
|
pieces = line.split(":" if colon else None, 1)
|
|
if len(pieces) != 2:
|
|
continue
|
|
value = pieces[1].strip().split()
|
|
if value and value[0].isdigit():
|
|
result[pieces[0]] = int(value[0]) * (1024 if len(value) > 1 and value[1] == "kB" else 1)
|
|
return result
|
|
|
|
|
|
def parse_proc_stat(raw):
|
|
end = raw.rfind(")")
|
|
require(end > 0, "invalid process identity")
|
|
fields = raw[end + 1:].split()
|
|
require(len(fields) >= 20 and fields[19].isdigit(), "invalid process start time")
|
|
return int(fields[19])
|
|
|
|
|
|
def address_matches(bound, endpoint):
|
|
try:
|
|
actual, expected = ipaddress.ip_address(bound), ipaddress.ip_address(urlsplit(endpoint).hostname)
|
|
except (ValueError, TypeError):
|
|
return False
|
|
return actual.version == expected.version and (actual == expected or actual.is_unspecified)
|
|
|
|
|
|
def docker_endpoint_matches(bindings, endpoint):
|
|
port = str(urlsplit(endpoint).port)
|
|
return any(protocol.endswith("/tcp") and binding.get("HostPort") == port
|
|
and address_matches(binding.get("HostIp"), endpoint)
|
|
for protocol, entries in bindings.items() for binding in (entries or []))
|
|
|
|
|
|
def native_endpoint_matches(tables, owned_sockets, endpoint):
|
|
port = urlsplit(endpoint).port
|
|
found = False
|
|
for protocol, table in tables.items():
|
|
for row in table.splitlines()[1:]:
|
|
fields = row.split()
|
|
if len(fields) <= 9 or fields[3] != "0A":
|
|
continue
|
|
address, encoded_port = fields[1].rsplit(":", 1)
|
|
if int(encoded_port, 16) != port:
|
|
continue
|
|
width = 8 if protocol == "tcp" else 32
|
|
require(len(address) == width, "invalid native listener address")
|
|
packed = b"".join(int(address[index:index + 8], 16).to_bytes(4, sys.byteorder)
|
|
for index in range(0, width, 8))
|
|
bound = ipaddress.ip_address(packed)
|
|
expected = ipaddress.ip_address(urlsplit(endpoint).hostname)
|
|
ambiguous_v4 = bound.version == 6 and expected.version == 4 and (bound.is_unspecified or bound.ipv4_mapped == expected)
|
|
if ambiguous_v4 and fields[9] not in owned_sockets:
|
|
return False
|
|
if not address_matches(str(bound), endpoint):
|
|
continue
|
|
if fields[9] not in owned_sockets:
|
|
return False
|
|
found = True
|
|
return found
|
|
|
|
|
|
def storage_directory(value):
|
|
require(type(value) is str and value.startswith("/") and str(Path(value)) == value
|
|
and ".." not in Path(value).parts and not any(char in value for char in "\0\n\r\t"),
|
|
"storage directory must be an unambiguous absolute path")
|
|
return Path(value)
|
|
|
|
|
|
def owned_storage_paths(owned, paths):
|
|
owned = storage_directory(owned)
|
|
require(type(paths) is list and 0 < len(paths) <= 1024, "storage disk paths are unavailable")
|
|
paths = [storage_directory(path) for path in paths]
|
|
require(all(path == owned or owned in path.parents for path in paths),
|
|
"effective storage disk is outside the owned storage root")
|
|
return sorted(paths)
|
|
|
|
|
|
def check_storage_mounts(owned, mountinfo):
|
|
lines = mountinfo.decode().splitlines()
|
|
require(lines, "storage mount inventory is unavailable")
|
|
for line in lines:
|
|
fields = line.split()
|
|
require(len(fields) >= 10 and "-" in fields[6:], "storage mount inventory is malformed")
|
|
mount = storage_directory(re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), fields[4]))
|
|
require(Path(owned) not in mount.parents, "nested mounts under owned storage are unsupported")
|
|
|
|
|
|
def native_storage(process, owned, paths, mountinfo):
|
|
# Host path resolution is valid only in the serving process's namespace/root.
|
|
require((process / "ns/mnt").samefile("/proc/self/ns/mnt") and (process / "root").samefile("/"),
|
|
"native storage requires the observer's mount namespace and root")
|
|
owned = storage_directory(owned)
|
|
require(owned.resolve(strict=True) == owned, "owned storage root changed")
|
|
paths = sorted(str(storage_directory(path).resolve(strict=True)) for path in paths)
|
|
resolved = owned_storage_paths(str(owned), paths)
|
|
check_storage_mounts(owned, mountinfo)
|
|
require(owned.is_dir() and all(path.is_dir() and path.stat().st_dev == owned.stat().st_dev for path in resolved),
|
|
"effective storage disks do not share the owned filesystem")
|
|
binding = digest({"owned": str(owned), "paths": paths, "device": owned.stat().st_dev})
|
|
return f"native 0 0 {shutil.disk_usage(owned).free // 1024} 0 /".encode(), binding
|
|
|
|
|
|
class Runtime:
|
|
def __init__(self, evidence, container=None, pid=None, metrics_url=None, storage_paths=None):
|
|
require(bool(container) != bool(pid), "select exactly one owned container or native PID")
|
|
self.evidence, self.plan = evidence, evidence.plan
|
|
self.container, self.pid = container, pid or 1
|
|
require(container is None or re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}", container), "invalid container identifier")
|
|
require(type(self.pid) is int and self.pid > 0, "invalid process PID")
|
|
self.metrics_url = metrics_url
|
|
if metrics_url:
|
|
parsed = urlsplit(metrics_url)
|
|
local_url(f"{parsed.scheme}://{parsed.netloc}")
|
|
require(not parsed.query and not parsed.fragment, "metrics URL must not contain query or fragment")
|
|
self.verified = set()
|
|
self.storage_paths = storage_paths
|
|
|
|
def read_files(self):
|
|
names = ("stat", "status", "smaps_rollup", "cgroup", "environ", "mountinfo")
|
|
if self.container:
|
|
raw = json.loads(command(["docker", "inspect", self.container]))
|
|
require(len(raw) == 1, "ambiguous container identity")
|
|
item = raw[0]
|
|
require((item["Config"].get("Labels") or {}).get(LABEL) == self.plan["run_id"], "container is not owned by this run")
|
|
require(item["State"]["Running"] and not item["State"]["OOMKilled"], "owned container stopped or was OOM-killed")
|
|
require(item["HostConfig"]["Memory"] == self.plan["memory_bytes"], "Docker memory limit differs from manifest")
|
|
bindings = item.get("NetworkSettings", {}).get("Ports", {})
|
|
require(docker_endpoint_matches(bindings, self.plan["endpoint"]),
|
|
"S3 TCP endpoint address is not published by the observed container")
|
|
if self.plan["data_volume"]:
|
|
mounts = [mount for mount in item.get("Mounts", [])
|
|
if mount.get("Type") == "volume" and mount.get("Name") == self.plan["data_volume"]]
|
|
volume = json.loads(command(["docker", "volume", "inspect", self.plan["data_volume"]]))
|
|
require(len(volume) == 1 and (volume[0].get("Labels") or {}).get(LABEL) == self.plan["run_id"],
|
|
"data volume is not owned by this run")
|
|
else:
|
|
mounts = [mount for mount in item.get("Mounts", [])
|
|
if mount.get("Type") == "bind" and Path(mount["Source"]).resolve() == Path(self.plan["data_root"])]
|
|
require(len(mounts) == 1 and mounts[0].get("RW") is True, "container storage does not match the immutable manifest")
|
|
storage_path = mounts[0]["Destination"]
|
|
require(callable(self.storage_paths), "effective storage inventory is required")
|
|
paths = [str(path) for path in owned_storage_paths(storage_path, self.storage_paths())]
|
|
identity = {"container_id": item["Id"], "started_at": item["State"]["StartedAt"],
|
|
"restart_count": item["RestartCount"], "pid": self.pid}
|
|
script = '[ "$(stat -Lc "%d:%i" /proc/$1/ns/mnt)" = "$(stat -Lc "%d:%i" /proc/self/ns/mnt)" ] || exit 1; '
|
|
script += '[ "$(stat -Lc "%d:%i" /proc/$1/root)" = "$(stat -Lc "%d:%i" /)" ] || exit 1; '
|
|
script += 'for n in stat status smaps_rollup cgroup environ mountinfo; do printf "\\n__FILE_%s__\\n" "$n"; cat "/proc/$1/$n" || exit 1; done; '
|
|
script += 'printf "\\n__FILE_exe_inode__\\n"; stat -Lc "%d:%i" "/proc/$1/exe"; '
|
|
script += 'for n in memory.current memory.max memory.peak memory.stat memory.events memory.swap.current memory.swap.max; do printf "\\n__FILE_%s__\\n" "$n"; cat "/sys/fs/cgroup/$n" 2>/dev/null || true; done'
|
|
script += '; printf "\\n__FILE_storage__\\n"; df -Pk "$2" | tail -n 1'
|
|
script += '; printf "\\n__FILE_storage_root__\\n"; readlink -f -- "$2" || exit 1'
|
|
script += '; printf "\\n__FILE_storage_devices__\\n"; storage=$2; shift 2; stat -Lc "%d" -- "$storage" "$@" || exit 1'
|
|
script += '; printf "\\n__FILE_volume_paths__\\n"; for p; do [ -d "$p" ] || exit 1; readlink -f -- "$p" || exit 1; done'
|
|
data = command(["docker", "exec", item["Id"], "sh", "-c", script, "sh", str(self.pid), storage_path, *paths])
|
|
chunks = re.split(rb"\n__FILE_([A-Za-z_.]+)__\n", data)
|
|
files = {chunks[index].decode(): chunks[index + 1].strip(b"\n") for index in range(1, len(chunks), 2)}
|
|
require(files["storage_root"].decode() == storage_path, "owned storage mount is not canonical")
|
|
resolved = files["volume_paths"].decode().splitlines()
|
|
require(len(resolved) == len(paths), "effective storage paths could not be resolved")
|
|
owned_storage_paths(storage_path, resolved)
|
|
check_storage_mounts(storage_path, files["mountinfo"])
|
|
devices = files["storage_devices"].splitlines()
|
|
require(len(devices) == len(paths) + 1 and devices[0].isdigit() and len(set(devices)) == 1,
|
|
"effective storage disks do not share the owned filesystem")
|
|
identity["storage_binding_sha256"] = digest({"mount": mounts[0], "paths": resolved, "devices": [value.decode() for value in devices]})
|
|
else:
|
|
require(self.plan["data_volume"] is None, "native PID sampling requires a host data root")
|
|
root = Path("/proc") / str(self.pid)
|
|
require(root.stat().st_uid == os.getuid(), "native process is not owned by this user")
|
|
sockets = set()
|
|
for descriptor in (root / "fd").iterdir():
|
|
try:
|
|
target = os.readlink(descriptor)
|
|
except FileNotFoundError:
|
|
continue
|
|
match = re.fullmatch(r"socket:\[([0-9]+)\]", target)
|
|
if match:
|
|
sockets.add(match[1])
|
|
tables = {protocol: (root / "net" / protocol).read_text() for protocol in ("tcp", "tcp6")
|
|
if (root / "net" / protocol).exists()}
|
|
require(native_endpoint_matches(tables, sockets, self.plan["endpoint"]),
|
|
"S3 TCP endpoint is not exclusively bound by the observed PID")
|
|
files = {name: (root / name).read_bytes() for name in names}
|
|
groups = files["cgroup"].decode().splitlines()
|
|
require(len(groups) == 1 and groups[0].startswith("0::/") and groups[0] != "0::/", "dedicated cgroup v2 is required")
|
|
relative = groups[0][3:].lstrip("/")
|
|
require(".." not in Path(relative).parts, "invalid cgroup path")
|
|
group = Path("/sys/fs/cgroup") / relative
|
|
require(set((group / "cgroup.procs").read_text().split()) == {str(self.pid)}, "native cgroup contains other processes")
|
|
for name in ("memory.current", "memory.max", "memory.peak", "memory.stat", "memory.events", "memory.swap.current", "memory.swap.max"):
|
|
files[name] = (group / name).read_bytes() if (group / name).exists() else b""
|
|
info = (root / "exe").stat()
|
|
files["exe_inode"] = f"{info.st_dev}:{info.st_ino}".encode()
|
|
require(callable(self.storage_paths), "effective storage inventory is required")
|
|
files["storage"], binding = native_storage(root, self.plan["data_root"], self.storage_paths(), files["mountinfo"])
|
|
identity = {"boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), "pid": self.pid,
|
|
"storage_binding_sha256": binding}
|
|
identity.update(start_ticks=parse_proc_stat(files["stat"].decode()), executable_inode=files["exe_inode"].decode())
|
|
key = digest(identity)
|
|
if key not in self.verified:
|
|
if self.container:
|
|
observed = command(["docker", "exec", identity["container_id"], "sha256sum", f"/proc/{self.pid}/exe"], timeout=60).decode().split()[0]
|
|
else:
|
|
with (Path("/proc") / str(self.pid) / "exe").open("rb") as stream:
|
|
observed = hashlib.file_digest(stream, "sha256").hexdigest()
|
|
require(observed == self.plan["binary_sha256"], "running binary does not match manifest")
|
|
self.verified.add(key)
|
|
environment = {}
|
|
for pair in files["environ"].split(b"\0"):
|
|
name, _, value = pair.partition(b"=")
|
|
if name.decode(errors="replace") in SAFE_ENV:
|
|
environment[name.decode()] = value.decode()
|
|
return identity, files, environment
|
|
|
|
def sample(self):
|
|
identity, files, environment = self.read_files()
|
|
def number(name):
|
|
value = files.get(name, b"").strip()
|
|
return int(value) if value.isdigit() else None
|
|
require(number("memory.max") == self.plan["memory_bytes"], "kernel memory limit differs from manifest")
|
|
status = key_values(files["status"].decode(), colon=True)
|
|
smaps = key_values(files["smaps_rollup"].decode(), colon=True)
|
|
events = key_values(files["memory.events"].decode())
|
|
require("oom" in events and "oom_kill" in events, "cgroup OOM counters are unavailable")
|
|
require(all(events.get(name, 0) == 0 for name in ("oom", "oom_kill", "oom_group_kill")), "cgroup reported an OOM event")
|
|
require(status.get("VmRSS") is not None and number("memory.current") is not None, "required memory readings are missing")
|
|
storage = files["storage"].split()
|
|
require(len(storage) >= 6 and storage[3].isdigit(), "storage free-space observation is unavailable")
|
|
result = {"storage_available_bytes": int(storage[3]) * 1024, "identity": identity, "identity_sha256": digest(identity), "configuration": environment,
|
|
"rss_bytes": status["VmRSS"], "rss_anon_bytes": status.get("RssAnon"),
|
|
"smaps_anonymous_bytes": smaps.get("Anonymous"), "cgroup_current_bytes": number("memory.current"),
|
|
"cgroup_peak_bytes": number("memory.peak"), "cgroup_limit_bytes": number("memory.max"),
|
|
"cgroup_swap_bytes": number("memory.swap.current"), "cgroup_swap_limit_bytes": number("memory.swap.max"),
|
|
"cgroup_stat": key_values(files["memory.stat"].decode()), "cgroup_events": events,
|
|
"active_heap_bytes": None, "active_heap_status": "unknown: allocator metrics not supplied",
|
|
"allocator_endpoint_attribution": "operator-supplied; process binding unverified" if self.metrics_url else None}
|
|
if self.metrics_url:
|
|
try:
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect())
|
|
with opener.open(self.metrics_url, timeout=5) as response:
|
|
text = response.read(4 * 1024 * 1024 + 1).decode()
|
|
require(len(text) <= 4 * 1024 * 1024, "oversized metrics response")
|
|
values = {}
|
|
for name in ("rustfs_memory_allocator_malloc_requested_bytes_available", "rustfs_memory_allocator_malloc_requested_bytes"):
|
|
matches = re.findall(r"^" + name + r"(?:\{[^\n]*\})?\s+([0-9.eE+-]+)\s*$", text, re.MULTILINE)
|
|
if len(matches) == 1:
|
|
values[name] = float(matches[0])
|
|
active = values.get("rustfs_memory_allocator_malloc_requested_bytes")
|
|
if values.get("rustfs_memory_allocator_malloc_requested_bytes_available") == 1 and active is not None and math.isfinite(active) and active >= 0:
|
|
result.update(active_heap_bytes=int(active), active_heap_status="observed: allocator requested current bytes")
|
|
else:
|
|
result["active_heap_status"] = "unknown: live allocator statistic unavailable"
|
|
except Exception:
|
|
result["active_heap_status"] = "unknown: allocator metrics request failed"
|
|
return result
|
|
|
|
|
|
class Phase:
|
|
"""One process identity and deadline cover work, verification and the last sample."""
|
|
|
|
def __init__(self, evidence, runtime, api, label, enabled, args):
|
|
self.evidence, self.runtime, self.api = evidence, runtime, api
|
|
self.label, self.enabled, self.interval = label, enabled, args.interval
|
|
self.phase_id = str(uuid.uuid4())
|
|
self.started = time.monotonic()
|
|
self.deadline = self.started + args.seconds
|
|
self.stop = threading.Event()
|
|
self.sample_lock = threading.Lock()
|
|
self.error = None
|
|
self.identity = None
|
|
self.latest = None
|
|
self.initial = None
|
|
self.stage = "scanner" if enabled else "workload"
|
|
|
|
def check(self):
|
|
require(self.error is None, self.error or "observation failed")
|
|
require(time.monotonic() <= self.deadline, "phase time budget exhausted; retain evidence and resume explicitly")
|
|
|
|
def once(self):
|
|
with self.sample_lock:
|
|
self.check()
|
|
memory = self.runtime.sample()
|
|
if self.identity is None:
|
|
self.identity = memory["identity_sha256"]
|
|
require(memory["identity_sha256"] == self.identity, "process changed during a phase; start a new phase after restart")
|
|
status = self.api.status(self.enabled)
|
|
# Close the API window before Phase entry can authorize S3 writes.
|
|
# Runtime identity also binds the effective storage inventory.
|
|
memory = self.runtime.sample()
|
|
require(memory["identity_sha256"] == self.identity,
|
|
"process or effective storage changed during status observation")
|
|
self.check()
|
|
row = self.evidence.append("sample", phase=self.label, phase_id=self.phase_id, stage=self.stage, memory=memory, scanner=status)
|
|
self.latest = row
|
|
return row
|
|
|
|
def __enter__(self):
|
|
self.evidence.append("phase_start", phase=self.label, phase_id=self.phase_id,
|
|
time_budget_seconds=self.deadline - self.started)
|
|
self.initial = self.once()
|
|
def sample_loop():
|
|
while not self.stop.wait(self.interval):
|
|
try:
|
|
self.once()
|
|
except Exception:
|
|
self.error = "memory/status observation failed; phase is inconclusive"
|
|
self.evidence.append("observation_error", phase=self.label, phase_id=self.phase_id)
|
|
self.stop.set()
|
|
self.thread = threading.Thread(target=sample_loop, daemon=True)
|
|
self.thread.start()
|
|
return self
|
|
|
|
def enter_stage(self, stage):
|
|
with self.sample_lock:
|
|
self.stage = stage
|
|
self.once()
|
|
|
|
def verify(self):
|
|
self.enter_stage("verification")
|
|
return self.api.verify(deadline=self.deadline, guard=self.check)
|
|
|
|
def result(self, result):
|
|
return {**result, "phase_id": self.phase_id, "process_identity": self.identity,
|
|
"elapsed_seconds": time.monotonic() - self.started}
|
|
|
|
def __exit__(self, exception_type, *_):
|
|
self.stop.set()
|
|
self.thread.join(timeout=60)
|
|
require(not self.thread.is_alive(), "sampler did not stop")
|
|
if exception_type is None:
|
|
self.check()
|
|
self.once()
|
|
|
|
|
|
def free_space(plan, remaining, available=None):
|
|
required = GIB + math.ceil(remaining * (plan["object_bytes"] + 16384) * 1.2)
|
|
if available is None:
|
|
require(plan["data_root"] is not None, "named-volume capacity must be observed inside the owned container")
|
|
available = shutil.disk_usage(plan["data_root"]).free
|
|
require(type(available) is int and available >= required, "insufficient free space for remaining objects and metadata reserve")
|
|
|
|
|
|
def run_load(evidence, api, runtime, args):
|
|
state = evidence.state()
|
|
data = payload(evidence.plan)
|
|
with Phase(evidence, runtime, api, "load", False, args) as phase, ThreadPoolExecutor(args.workers) as pool:
|
|
free_space(evidence.plan, evidence.plan["objects"] - state["next_object"], phase.latest["memory"]["storage_available_bytes"])
|
|
api.owned(create=state["next_object"] == 0)
|
|
while state["next_object"] < evidence.plan["objects"]:
|
|
phase.check()
|
|
start = state["next_object"]
|
|
end = min(start + args.batch, evidence.plan["objects"])
|
|
free_space(evidence.plan, evidence.plan["objects"] - start, phase.latest["memory"]["storage_available_bytes"])
|
|
api.owned()
|
|
def put(index):
|
|
phase.check()
|
|
api.client.put_object(Bucket=evidence.plan["bucket"], Key=object_key(evidence.plan, index), Body=data)
|
|
list(pool.map(put, range(start, end)))
|
|
state["next_object"] = end
|
|
evidence.save(state)
|
|
evidence.append("load_batch", phase_id=phase.phase_id, completed=end, expected=evidence.plan["objects"])
|
|
phase.check()
|
|
inventory = phase.verify()
|
|
state["load_complete"] = True
|
|
evidence.save(state)
|
|
return phase.result({"phase": "load", "complete": True, "inventory": inventory})
|
|
|
|
|
|
def run_probe(evidence, api, runtime, args):
|
|
require(evidence.state().get("load_complete") is True, "finish and verify load before probing")
|
|
completed = set()
|
|
with Phase(evidence, runtime, api, args.label, True, args) as phase:
|
|
api.owned()
|
|
first_end = phase.initial["scanner"]["metrics"].get("last_cycle_end_unix_secs") or 0
|
|
while True:
|
|
phase.check()
|
|
metrics = phase.latest["scanner"]["metrics"]
|
|
ended = metrics.get("last_cycle_end_unix_secs")
|
|
if type(ended) is int and ended > first_end and metrics.get("last_cycle_result") == "success":
|
|
completed.add(ended)
|
|
if args.stop_after_progress is not None:
|
|
progress = metrics.get("current_cycle_objects_scanned")
|
|
if metrics.get("current_cycle_active") is True and type(progress) is int and progress >= args.stop_after_progress:
|
|
result = {"phase": args.label, "active_before_external_restart": True,
|
|
"completed_cycles": len(completed), "complete": False}
|
|
break
|
|
elif len(completed) >= args.min_completed_cycles:
|
|
inventory = phase.verify()
|
|
usage = api.usage(inventory)
|
|
phase.check()
|
|
require(usage["matches_inventory"] and usage["converged"] is True,
|
|
"scanner usage is not converged to the exact S3 inventory")
|
|
result = {"phase": args.label, "completed_cycles": len(completed), "inventory": inventory,
|
|
"usage": usage, "complete": True}
|
|
break
|
|
require(time.monotonic() < phase.deadline, "required scanner progress was not observed within the phase budget")
|
|
phase.stop.wait(min(args.interval, max(0, phase.deadline - time.monotonic())))
|
|
return phase.result(result)
|
|
|
|
|
|
def run_burst(evidence, api, runtime, args):
|
|
require(evidence.state().get("load_complete") is True, "finish load before scanner-off burst")
|
|
state = evidence.state()
|
|
count = args.burst_objects or evidence.plan["objects"]
|
|
require(0 < count <= evidence.plan["objects"], "burst count exceeds owned fixture")
|
|
burst = state.setdefault("burst", {"id": str(uuid.uuid4()), "next": 0, "objects": count})
|
|
require(burst["objects"] == count and str(uuid.UUID(burst["id"])) == burst["id"]
|
|
and type(burst["next"]) is int and 0 <= burst["next"] <= count, "burst resume parameters changed")
|
|
initial_cursor = burst["next"]
|
|
evidence.save(state)
|
|
with Phase(evidence, runtime, api, "burst", False, args) as phase, ThreadPoolExecutor(args.workers) as pool:
|
|
api.owned()
|
|
while burst["next"] < count:
|
|
phase.check()
|
|
api.owned()
|
|
end = min(burst["next"] + args.batch, count)
|
|
free_space(evidence.plan, args.workers, phase.latest["memory"]["storage_available_bytes"])
|
|
def operation(index):
|
|
phase.check()
|
|
key = f".scanner-memory-burst/{burst['id']}/{index:08d}"
|
|
api.client.copy_object(Bucket=evidence.plan["bucket"], Key=key,
|
|
CopySource={"Bucket": evidence.plan["bucket"], "Key": object_key(evidence.plan, index)})
|
|
phase.check()
|
|
head = api.client.head_object(Bucket=evidence.plan["bucket"], Key=key)
|
|
require(head["ContentLength"] == evidence.plan["object_bytes"], "copied object size mismatch")
|
|
phase.check()
|
|
api.client.delete_object(Bucket=evidence.plan["bucket"], Key=key)
|
|
list(pool.map(operation, range(burst["next"], end)))
|
|
burst["next"] = end
|
|
evidence.save(state)
|
|
evidence.append("burst_batch", phase_id=phase.phase_id, completed=end, expected=count)
|
|
phase.check()
|
|
inventory = phase.verify()
|
|
phase.enter_stage("idle")
|
|
idle_deadline = time.monotonic() + args.idle_seconds
|
|
require(idle_deadline <= phase.deadline, "insufficient phase budget for the requested idle observation")
|
|
evidence.append("idle_start", phase="burst", phase_id=phase.phase_id, idle_seconds=args.idle_seconds)
|
|
while time.monotonic() < idle_deadline:
|
|
phase.check()
|
|
phase.stop.wait(min(args.interval, max(0, idle_deadline - time.monotonic())))
|
|
return phase.result({"phase": "burst", "complete": True, "objects": count, "burst_id": burst["id"],
|
|
"cursor_start": initial_cursor, "cursor_end": burst["next"],
|
|
"newly_completed_objects": burst["next"] - initial_cursor,
|
|
"new_logical_copy_head_delete_requests": (burst["next"] - initial_cursor) * 3,
|
|
"cumulative_completed_objects": burst["next"],
|
|
"cumulative_logical_copy_head_delete_requests": burst["next"] * 3,
|
|
"observed_http_request_count": None, "idle_seconds": args.idle_seconds, "inventory": inventory,
|
|
"reclamation_verdict": "measurement only; RSS retention alone does not prove a leak"})
|
|
|
|
|
|
def summarize(evidence):
|
|
samples = {}
|
|
results = []
|
|
errors = 0
|
|
path = evidence.root / "observations.jsonl"
|
|
if path.exists() or path.is_symlink():
|
|
with open_owned(path) as stream:
|
|
for line in stream:
|
|
require(len(line) <= 1024 * 1024, "oversized evidence row")
|
|
row = json.loads(line)
|
|
require(row.get("run_id") == evidence.plan["run_id"], "foreign run evidence")
|
|
if row["kind"] == "sample":
|
|
memory = row["memory"]
|
|
key = row["phase_id"] + ":" + memory["identity_sha256"] + ":" + row["stage"]
|
|
group = samples.setdefault(key, {"phase": row["phase"], "phase_id": row["phase_id"],
|
|
"process_identity": memory["identity_sha256"], "stage": row["stage"], "samples": 0,
|
|
"rss_peak": 0, "sampled_current_peak_bytes": 0,
|
|
"kernel_reported_peak_bytes": None, "first_rss": memory["rss_bytes"],
|
|
"active_heap_samples": 0})
|
|
group["samples"] += 1
|
|
group["rss_peak"] = max(group["rss_peak"], memory["rss_bytes"])
|
|
group["sampled_current_peak_bytes"] = max(group["sampled_current_peak_bytes"], memory["cgroup_current_bytes"])
|
|
if memory.get("cgroup_peak_bytes") is not None:
|
|
group["kernel_reported_peak_bytes"] = max(group["kernel_reported_peak_bytes"] or 0, memory["cgroup_peak_bytes"])
|
|
group["last_rss"] = memory["rss_bytes"]
|
|
group["active_heap_samples"] += memory["active_heap_bytes"] is not None
|
|
elif row["kind"] == "phase_result":
|
|
results.append(row["result"])
|
|
elif row["kind"] in ("phase_error", "observation_error"):
|
|
errors += 1
|
|
interrupted = set()
|
|
restart_observed = False
|
|
for result in results:
|
|
identity = result.get("process_identity")
|
|
if result.get("active_before_external_restart"):
|
|
interrupted.add(identity)
|
|
elif interrupted and identity and identity not in interrupted and result.get("complete") and result.get("completed_cycles", 0) > 0:
|
|
restart_observed = True
|
|
summary = {"schema": SCHEMA, "plan_sha256": evidence.manifest["plan_sha256"], "plan": evidence.plan,
|
|
"phases": samples, "results": results, "observation_errors": errors,
|
|
"restart_window_observed": restart_observed,
|
|
"full_scale": evidence.plan["objects"] == 2_000_000 and evidence.plan["object_bytes"] >= 50 * 1024,
|
|
"allocator_leak_verdict": "unknown; correlate live heap, anonymous RSS and cgroup file/slab across idle windows",
|
|
"overall_acceptance_passed": False}
|
|
atomic_json(evidence.root / "summary.json", summary)
|
|
return summary
|
|
|
|
|
|
def prepare(args):
|
|
require(not args.run_dir.exists(), "prepare requires a new run directory")
|
|
identifier = str(uuid.uuid4())
|
|
plan = {"run_id": identifier, "created_at": utc(), "endpoint": local_url(args.endpoint),
|
|
"source_sha": args.source_sha, "binary_sha256": args.binary_sha256, "layout": args.layout,
|
|
"objects": args.objects, "object_bytes": args.object_bytes, "root_prefixes": args.root_prefixes,
|
|
"memory_bytes": args.memory_bytes, "data_root": str(args.data_root.resolve()) if args.data_root else None,
|
|
"data_volume": args.data_volume,
|
|
"bucket": "scanner-memory-" + uuid.UUID(identifier).hex}
|
|
manifest = {"schema": SCHEMA, "plan": plan, "plan_sha256": digest(plan)}
|
|
validate_manifest(manifest)
|
|
if plan["data_root"]:
|
|
free_space(plan, plan["objects"])
|
|
args.run_dir.mkdir(mode=0o700)
|
|
atomic_json(args.run_dir / "manifest.json", manifest)
|
|
return {"run_id": identifier, "bucket": plan["bucket"], "container_label": LABEL + "=" + identifier,
|
|
"manifest": str(args.run_dir / "manifest.json")}
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("action", choices=("prepare", "load", "verify", "probe", "burst", "summary"))
|
|
parser.add_argument("run_dir", type=Path)
|
|
parser.add_argument("--endpoint")
|
|
parser.add_argument("--source-sha")
|
|
parser.add_argument("--binary-sha256")
|
|
parser.add_argument("--layout", choices=("wide", "nested"), default="wide")
|
|
parser.add_argument("--objects", type=int, default=2_000_000)
|
|
parser.add_argument("--object-bytes", type=int, default=50 * 1024)
|
|
parser.add_argument("--root-prefixes", type=int, default=170_000)
|
|
parser.add_argument("--memory-bytes", type=int, default=4 * GIB)
|
|
parser.add_argument("--data-root", type=Path)
|
|
parser.add_argument("--data-volume")
|
|
parser.add_argument("--container")
|
|
parser.add_argument("--pid", type=int)
|
|
parser.add_argument("--allocator-metrics-url")
|
|
parser.add_argument("--workers", type=int, default=48)
|
|
parser.add_argument("--batch", type=int, default=256)
|
|
parser.add_argument("--seconds", type=float, default=3600)
|
|
parser.add_argument("--interval", type=float, default=5)
|
|
parser.add_argument("--label", default="scanner_probe")
|
|
parser.add_argument("--min-completed-cycles", type=int, default=2)
|
|
parser.add_argument("--stop-after-progress", type=int)
|
|
parser.add_argument("--burst-objects", type=int)
|
|
parser.add_argument("--idle-seconds", type=float, default=1800)
|
|
args = parser.parse_args(argv)
|
|
evidence = None
|
|
try:
|
|
args.run_dir = args.run_dir.absolute()
|
|
require(1 <= args.workers <= 256 and 1 <= args.batch <= 4096, "invalid client concurrency or batch bound")
|
|
require(math.isfinite(args.seconds) and 0 < args.seconds <= 7 * 86400
|
|
and math.isfinite(args.interval) and 0.1 <= args.interval <= 300
|
|
and math.isfinite(args.idle_seconds) and 0 < args.idle_seconds <= 86400, "invalid time bounds")
|
|
require(re.fullmatch(r"[a-zA-Z0-9_-]{1,64}", args.label), "invalid phase label")
|
|
require(args.min_completed_cycles >= 0 and (args.stop_after_progress is None or args.stop_after_progress > 0),
|
|
"invalid cycle/progress bounds")
|
|
if args.action == "prepare":
|
|
require(args.endpoint and args.source_sha and args.binary_sha256 and (args.data_root or args.data_volume),
|
|
"prepare requires endpoint, source, binary and storage identity")
|
|
result = prepare(args)
|
|
else:
|
|
evidence = Evidence(args.run_dir)
|
|
with evidence.exclusive():
|
|
if args.action == "summary":
|
|
result = summarize(evidence)
|
|
else:
|
|
api = S3(evidence, args.workers)
|
|
if args.action == "verify":
|
|
result = api.verify(deadline=time.monotonic() + args.seconds)
|
|
else:
|
|
runtime = Runtime(evidence, args.container, args.pid, args.allocator_metrics_url, api.storage_paths)
|
|
result = {"load": run_load, "probe": run_probe, "burst": run_burst}[args.action](evidence, api, runtime, args)
|
|
evidence.append("phase_result", result=result)
|
|
summarize(evidence)
|
|
print(json.dumps(result, sort_keys=True))
|
|
return 0
|
|
except AcceptanceError as error:
|
|
message = str(error)
|
|
except Exception:
|
|
message = "acceptance phase failed; raw credentials and server errors were withheld"
|
|
if evidence:
|
|
try:
|
|
evidence.append("phase_error", phase=args.action, reason=message)
|
|
summarize(evidence)
|
|
except Exception:
|
|
pass
|
|
print(message, file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|