rustfs/scripts/test_summarize_scanner_heal_perf.py

#!/usr/bin/env python3

from __future__ import annotations

import contextlib
import hashlib
import io
import json
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
from unittest import mock

sys.path.insert(0, str(Path(__file__).resolve().parent))
import summarize_scanner_heal_perf as summary


def sha(path: Path) -> str:
    with path.open("rb") as stream:
        hasher = hashlib.sha256()
        while chunk := stream.read(1024 * 1024):
            hasher.update(chunk)
        return hasher.hexdigest()


class ScannerHealPerfSummaryTest(unittest.TestCase):
    def setUp(self):
        self.temp = tempfile.TemporaryDirectory()
        self.addCleanup(self.temp.cleanup)
        self.root = Path(self.temp.name)
        self.abba = self.root / "abba"
        self.abba.mkdir()
        self.manifest = {
            "fixed": {
                "config_sha256": "1" * 64,
                "dataset_sha256": "2" * 64,
                "release_flags": "--profile production",
                "durability": "drive-sync=on",
                "topology": "EC8+4",
                "offered_load_ops": 100,
            },
            "baseline": {"revision": "a" * 40, "sha256": "3" * 64},
            "candidate": {"revision": "b" * 40, "sha256": "4" * 64},
            "adapter_sha256": "5" * 64,
            "collector_sha256": "6" * 64,
        }
        self.comparison = {
            "scenario": "cold-hot",
            "comparison": "build",
            "round": 1,
            "status": "pass",
            "p99_regression": 0.02,
            "throughput_change": -0.01,
            "p1": {"required_reduction": 0.8, "observed_reduction": 0.82, "repeatability_drift": 0.01},
            "p2_post_stop_work_multiples": [None, 1.1, 1.0, None],
        }
        self.report = {
            "status": "pass",
            "performance": "pass",
            "evidence": "measured",
            "cells": 120,
            "comparisons": [self.comparison],
        }
        self.write_inputs()

    def write_inputs(self):
        (self.abba / "manifest.json").write_text(json.dumps(self.manifest), encoding="utf-8")
        (self.abba / "report.json").write_text(json.dumps(self.report), encoding="utf-8")

    def test_measured_pass_writes_quiet_artifacts(self):
        cache_log = self.root / "cache.log"
        cache_log.write_text(
            "compiler noise\nCACHE_COST "
            + json.dumps({
                "schema": 1,
                "scenario": "small_dirty",
                "cache_wire_bytes": 100,
                "save_body_bytes_per_sample": 200,
                "clone": {"max_ns": 10},
                "encode": {"max_ns": 20},
                "save_inclusive": {"max_ns": 30},
                "build": {"source_revision": "abc", "source_tree": "clean", "test_opt_level_override": "0"},
            })
            + "\nmore noise\n",
            encoding="utf-8",
        )
        args = type("Args", (), {
            "abba_dir": self.abba,
            "cache_cost_log": cache_log,
            "require_cache_cost": False,
            "json_out": None,
            "markdown_out": None,
        })
        result = summary.build_summary(args)
        self.assertEqual(result["verdict"], "PASS")
        self.assertEqual(result["abba"]["provenance"]["manifest_sha256"], sha(self.abba / "manifest.json"))
        self.assertEqual(result["cache_cost"]["max_save_body_amplification"], 2.0)

    def test_synthetic_report_fails_as_performance_conclusion(self):
        self.report.update(status="synthetic_validated", performance="pending", evidence="synthetic")
        self.write_inputs()
        args = type("Args", (), {
            "abba_dir": self.abba,
            "cache_cost_log": None,
            "require_cache_cost": False,
            "json_out": None,
            "markdown_out": None,
        })
        result = summary.build_summary(args)
        self.assertEqual(result["verdict"], "FAIL")
        self.assertIn("synthetic evidence", result["reason"])

    def test_requires_cache_profile_when_requested(self):
        args = type("Args", (), {
            "abba_dir": self.abba,
            "cache_cost_log": None,
            "require_cache_cost": True,
            "json_out": None,
            "markdown_out": None,
        })
        with self.assertRaisesRegex(ValueError, "cache-cost profile log is required"):
            summary.build_summary(args)

    def test_cli_prints_one_line_and_exits_nonzero_for_pending_performance(self):
        self.report.update(status="inconclusive", performance="inconclusive")
        self.write_inputs()
        stdout = io.StringIO()
        stderr = io.StringIO()
        argv = [
            "summarize_scanner_heal_perf.py",
            "--abba-dir",
            str(self.abba),
            "--json-out",
            str(self.root / "summary.json"),
            "--markdown-out",
            str(self.root / "summary.md"),
        ]
        with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
            with mock.patch.object(sys, "argv", argv):
                code = summary.main()
        self.assertEqual(code, 1)
        self.assertEqual(stdout.getvalue().count("\n"), 1)
        self.assertTrue(stdout.getvalue().startswith("FAIL scanner_heal_perf "))
        self.assertEqual(stderr.getvalue(), "")
        self.assertEqual(summary.read_json(self.root / "summary.json")["verdict"], "FAIL")
        self.assertIn("worst_p99_regression", (self.root / "summary.md").read_text(encoding="utf-8"))

    def test_invalid_cache_cost_lines_fail_closed(self):
        cache_log = self.root / "cache.log"
        cache_log.write_text("CACHE_COST {\"schema\": 2}\n", encoding="utf-8")
        with self.assertRaisesRegex(ValueError, "unsupported schema"):
            summary.summarize_cache_cost(cache_log)

    def test_script_entrypoint_is_quiet(self):
        script = Path(__file__).with_name("summarize_scanner_heal_perf.py")
        process = subprocess.run(
            [sys.executable, str(script), "--abba-dir", str(self.abba)],
            check=False,
            text=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        self.assertEqual(process.returncode, 0, process.stderr)
        self.assertEqual(process.stdout.count("\n"), 1)
        self.assertEqual(process.stderr, "")


if __name__ == "__main__":
    unittest.main()