mirror of
https://github.com/Katakate/k7.git
synced 2026-09-21 09:33:19 +00:00
G
76a8139774
HTTPS-by-default for k7-api, cluster-wide Cilium isolation, first-class --docker on Kata and k7d, and RuntimeClass k7-fc. Playbook pins k7d 0.6.0.
230 lines
9.1 KiB
Python
230 lines
9.1 KiB
Python
"""Deterministic-Markdown test for bench/docker-perf/render.py.
|
||
|
||
The point of this test is the bit-stable rendered output, not the
|
||
benchmark numbers themselves — when someone re-runs the renderer on
|
||
the same CSV, the markdown that lands in ``PERFORMANCE.md`` should be
|
||
byte-identical, so PR diffs only ever show real number changes (never
|
||
formatting drift).
|
||
|
||
We exercise:
|
||
- a 1-node CSV (only ``host`` + ``k7-fd`` + ``k7-ql-r1``) — the
|
||
``k7-ql-r2`` column disappears cleanly,
|
||
- the ``(max - min) / median > 0.30`` flag,
|
||
- and the ratio-vs-host table.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import importlib.util
|
||
import textwrap
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
BENCH_DIR = Path(__file__).resolve().parents[2] / "bench" / "docker-perf"
|
||
|
||
|
||
def _import_render():
|
||
"""Import ``bench/docker-perf/render.py`` without installing it as a package."""
|
||
spec = importlib.util.spec_from_file_location("_render_perf", BENCH_DIR / "render.py")
|
||
assert spec is not None and spec.loader is not None
|
||
module = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(module)
|
||
return module
|
||
|
||
|
||
@pytest.fixture(scope="module")
|
||
def render():
|
||
return _import_render().render
|
||
|
||
|
||
def _write_csv(path: Path, rows: list[tuple[str, str, int, float]]) -> Path:
|
||
p = path / "in.csv"
|
||
lines = ["label,operation,run,seconds"]
|
||
for label, op, run, secs in rows:
|
||
lines.append(f"{label},{op},{run},{secs}")
|
||
p.write_text("\n".join(lines) + "\n")
|
||
return p
|
||
|
||
|
||
# A tight, deterministic dataset. Numbers chosen so:
|
||
# - host is fast,
|
||
# - k7-fd is ~2× host on build,
|
||
# - k7-ql-r1 is ~4× host on build,
|
||
# - k7-ql-r2 is absent (1-node fixture).
|
||
# All ranges stay below the 30% gate so the ⚠ marker is exercised by
|
||
# the second fixture (test_range_gate_flags_noisy_cell).
|
||
SMOOTH_ROWS: list[tuple[str, str, int, float]] = []
|
||
for run, secs in enumerate([3.0, 3.05, 3.1, 3.02, 3.08], start=1):
|
||
SMOOTH_ROWS.append(("host", "pull", run, secs))
|
||
for run, secs in enumerate([60.0, 60.5, 61.0, 60.2, 60.8], start=1):
|
||
SMOOTH_ROWS.append(("host", "build_nocache", run, secs))
|
||
for run, secs in enumerate([0.5, 0.5, 0.5, 0.5, 0.5], start=1):
|
||
SMOOTH_ROWS.append(("host", "build_cached", run, secs))
|
||
for run, secs in enumerate([10.0, 10.0, 10.0, 10.0, 10.0], start=1):
|
||
SMOOTH_ROWS.append(("host", "run_cpu", run, secs))
|
||
for run, secs in enumerate([8.0, 8.2, 8.0, 8.1, 8.0], start=1):
|
||
SMOOTH_ROWS.append(("host", "run_io", run, secs))
|
||
for run, secs in enumerate([2.0, 2.1, 2.0, 2.0, 2.05], start=1):
|
||
SMOOTH_ROWS.append(("host", "run_read", run, secs))
|
||
|
||
for run, secs in enumerate([3.5, 3.55, 3.6, 3.55, 3.5], start=1):
|
||
SMOOTH_ROWS.append(("k7-fd", "pull", run, secs))
|
||
for run, secs in enumerate([120.0, 121.0, 120.5, 120.8, 121.2], start=1):
|
||
SMOOTH_ROWS.append(("k7-fd", "build_nocache", run, secs))
|
||
for run, secs in enumerate([0.6, 0.6, 0.6, 0.6, 0.6], start=1):
|
||
SMOOTH_ROWS.append(("k7-fd", "build_cached", run, secs))
|
||
for run, secs in enumerate([10.5, 10.5, 10.4, 10.6, 10.5], start=1):
|
||
SMOOTH_ROWS.append(("k7-fd", "run_cpu", run, secs))
|
||
for run, secs in enumerate([16.0, 16.2, 16.1, 16.0, 16.05], start=1):
|
||
SMOOTH_ROWS.append(("k7-fd", "run_io", run, secs))
|
||
for run, secs in enumerate([2.5, 2.55, 2.5, 2.6, 2.55], start=1):
|
||
SMOOTH_ROWS.append(("k7-fd", "run_read", run, secs))
|
||
|
||
for run, secs in enumerate([3.6, 3.7, 3.6, 3.65, 3.6], start=1):
|
||
SMOOTH_ROWS.append(("k7-ql-r1", "pull", run, secs))
|
||
for run, secs in enumerate([240.0, 241.0, 240.5, 240.8, 241.2], start=1):
|
||
SMOOTH_ROWS.append(("k7-ql-r1", "build_nocache", run, secs))
|
||
for run, secs in enumerate([0.7, 0.7, 0.7, 0.7, 0.7], start=1):
|
||
SMOOTH_ROWS.append(("k7-ql-r1", "build_cached", run, secs))
|
||
for run, secs in enumerate([10.6, 10.6, 10.7, 10.6, 10.5], start=1):
|
||
SMOOTH_ROWS.append(("k7-ql-r1", "run_cpu", run, secs))
|
||
for run, secs in enumerate([32.0, 32.2, 32.1, 32.0, 32.05], start=1):
|
||
SMOOTH_ROWS.append(("k7-ql-r1", "run_io", run, secs))
|
||
for run, secs in enumerate([3.0, 3.05, 3.0, 3.1, 3.0], start=1):
|
||
SMOOTH_ROWS.append(("k7-ql-r1", "run_read", run, secs))
|
||
|
||
|
||
def test_render_one_node_csv_is_byte_stable(tmp_path: Path, render):
|
||
csv = _write_csv(tmp_path, SMOOTH_ROWS)
|
||
out = render(csv, title="Docker workloads — fixture", hardware_note="Hardware: fixture.")
|
||
# The ``k7-ql-r2`` column must NOT appear (CSV has no rows for it).
|
||
assert "k7-ql-r2" not in out
|
||
# Headings + the two-table layout are present.
|
||
assert "## Docker workloads — fixture" in out
|
||
assert "5 runs per cell, median (range in parens)" in out
|
||
assert "Hardware: fixture." in out
|
||
# Both tables present.
|
||
assert out.count("| Operation |") == 1
|
||
assert out.count("| Cell |") == 1
|
||
# Median is selected, not mean: host.build_nocache median is 60.5
|
||
# (values 60.0 60.2 60.5 60.8 61.0 → median 60.5).
|
||
assert "60.5 s" in out
|
||
# Ratio rows use × on the second table — k7-ql-r1 vs host on build_nocache
|
||
# is 240.5 / 60.5 ≈ 3.98×.
|
||
assert "3.98×" in out
|
||
# And build_cached (excluded from the ratio table) does NOT appear there.
|
||
cells_idx = out.index("| Cell")
|
||
ratio_section = out[cells_idx:]
|
||
assert "build (cached)" not in ratio_section
|
||
|
||
|
||
def test_render_flags_noisy_cell(tmp_path: Path, render):
|
||
rows = [
|
||
("host", "run_io", 1, 1.0),
|
||
("host", "run_io", 2, 1.0),
|
||
("host", "run_io", 3, 1.0),
|
||
("host", "run_io", 4, 1.0),
|
||
("host", "run_io", 5, 1.0),
|
||
# 1.0 .. 2.0 → range/median = 1.0 > 0.30, must trip the ⚠ flag.
|
||
("k7-fd", "run_io", 1, 1.0),
|
||
("k7-fd", "run_io", 2, 1.4),
|
||
("k7-fd", "run_io", 3, 1.5),
|
||
("k7-fd", "run_io", 4, 1.7),
|
||
("k7-fd", "run_io", 5, 2.0),
|
||
]
|
||
csv = _write_csv(tmp_path, rows)
|
||
out = render(csv, title="noisy")
|
||
# The k7-fd row of run_io must carry the ⚠ flag in its cell.
|
||
io_row = next(line for line in out.splitlines() if line.startswith("| run io"))
|
||
assert "⚠" in io_row, io_row
|
||
|
||
|
||
def test_render_output_is_deterministic(tmp_path: Path, render):
|
||
csv = _write_csv(tmp_path, SMOOTH_ROWS)
|
||
a = render(csv, title="t", hardware_note="hw")
|
||
b = render(csv, title="t", hardware_note="hw")
|
||
assert a == b
|
||
|
||
|
||
def test_render_handles_missing_envs_gracefully(tmp_path: Path, render):
|
||
rows = [
|
||
("host", "pull", 1, 1.0),
|
||
("host", "pull", 2, 1.0),
|
||
("k7-fd", "pull", 1, 2.0),
|
||
("k7-fd", "pull", 2, 2.0),
|
||
]
|
||
csv = _write_csv(tmp_path, rows)
|
||
out = render(csv, title="partial")
|
||
assert "k7-ql-r1" not in out
|
||
assert "k7-ql-r2" not in out
|
||
# The single-cell rows that are missing render as ``—`` for the absent ops.
|
||
assert "—" in out
|
||
|
||
|
||
def test_render_known_byte_stable_snapshot(tmp_path: Path, render):
|
||
"""Lock down the rendered output for a small fixture.
|
||
|
||
If you change render.py's formatting deliberately, update this string.
|
||
The intent is to fail loudly when an unintended whitespace / column-
|
||
width drift sneaks in.
|
||
"""
|
||
rows = [
|
||
("host", "pull", 1, 3.0),
|
||
("host", "pull", 2, 3.0),
|
||
("host", "build_nocache", 1, 60.0),
|
||
("host", "build_nocache", 2, 60.0),
|
||
("k7-fd", "pull", 1, 3.5),
|
||
("k7-fd", "pull", 2, 3.5),
|
||
("k7-fd", "build_nocache", 1, 120.0),
|
||
("k7-fd", "build_nocache", 2, 120.0),
|
||
]
|
||
csv = _write_csv(tmp_path, rows)
|
||
out = render(csv, title="snapshot", hardware_note="hw note")
|
||
expected = textwrap.dedent(
|
||
"""\
|
||
## snapshot
|
||
|
||
hw note
|
||
|
||
5 runs per cell, median (range in parens). ⚠ marks cells whose (max−min)/median > 0.30.
|
||
|
||
| Operation | host | k7-fd |
|
||
|---------------------------|------|-------|
|
||
| pull debian:12-slim | 3.00 s (3.00 s–3.00 s) | 3.50 s (3.50 s–3.50 s) |
|
||
| build (no-cache) | 60.0 s (60.0 s–60.0 s) | 120.0 s (120.0 s–120.0 s) |
|
||
| build (cached) | — | — |
|
||
| run cpu (10s budget) | — | — |
|
||
| run io (2k small + 512 MB)| — | — |
|
||
| run read (venv tree cat) | — | — |
|
||
|
||
| Cell | k7-fd / host |
|
||
|---------------------------|--------------|
|
||
| build (no-cache) ratio | 2.00× |
|
||
| run io ratio | — |
|
||
| run cpu ratio | — |
|
||
"""
|
||
)
|
||
assert out == expected, f"--- expected ---\n{expected}\n--- got ---\n{out}"
|
||
|
||
|
||
def test_render_k7d_fc_fork_rows(tmp_path: Path, render):
|
||
"""k7d-fc column and fork ops appear only when the CSV has them."""
|
||
rows = [
|
||
("k7d", "run_io", 1, 1.2),
|
||
("k7d", "run_io", 2, 1.3),
|
||
("k7d", "fork_to_ready", 1, 2.4),
|
||
("k7d-fc", "run_io", 1, 1.4),
|
||
("k7d-fc", "run_io", 2, 1.5),
|
||
("k7d-fc", "fork_to_ready", 1, 2.6),
|
||
]
|
||
csv = _write_csv(tmp_path, rows)
|
||
out = render(csv, title="k7d-fc", hardware_note="hw")
|
||
assert "| k7d |" in out or "| k7d | k7d-fc |" in out
|
||
assert "k7d-fc" in out
|
||
assert "fork → Ready + overlay2" in out
|
||
assert "fork warm engine" not in out # not in this CSV
|
||
io_row = next(line for line in out.splitlines() if line.startswith("| run io"))
|
||
assert "1.25 s" in io_row
|
||
assert "1.45 s" in io_row
|