Files
rustfs/scripts/test_dual_stack.py
T
overtrue 704b984c1f ci: both-stack e2e diff in the source workflows and a strict mint stack gate
T3.1/T3.3 (rustfs/backlog#2772, #2774). The existing nightly lanes get a stack
dimension and their legacy leg is the legacy side: e2e-replication-nightly (repl,
cluster, protocols), e2e-distributed and the ODM minio-source job. A gateway leg
runs scripts/dual_stack.py preflight after its build and stops there when no gateway
start is proven; the summary says 'skipped: gateway stack refuses startup: <why>'.
One summary job per workflow diffs the legs of the same run with junit_diff.py.
dual-stack-nightly.yml covers only e2e-smoke and e2e-full: one build, both stacks
from that binary, gateway legs only when the preflight proved a start.
junit_diff.py reports a missing, empty or case-less side as 'no result for <label>'.
mint.yml stays weekly, builds the image once and runs it per stack; mint_stack_diff.py
gates that every case legacy passes is present and passing on gateway with no new
failure (missing, NA, unknown status and truncated lines never pass). Gateway legs and
the compare job are report-only until T2.10.

Not built: a shared-binary preflight job in the three source workflows (one preflight
step per gateway leg); harness-only filtering for their five profiles; mc/Vault/iptables
in the e2e-full leg; an s3s-e2e both-stack run; rustfs_gateway=debug.
Tests: dual_stack 30 (preflight with fake binaries), mint_stack_diff 16, junit_diff 27;
negatives outnumber positives. Mutations (red test named) are in the PR 9367 description.
Not verified: the workflows have not run in CI in this shape (composites, stack matrices,
image artifact round trip, needs result of a red continue-on-error leg).
2026-11-13 09:21:25 +08:00

336 lines
16 KiB
Python

#!/usr/bin/env python3
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contract tests for dual_stack.py: the profile table, the stack proof and the per-profile summary."""
import json
import os
import stat
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
SCRIPT = ROOT / "scripts" / "dual_stack.py"
REFUSAL = "gateway stack is not assembled yet (rustfs/backlog#2734 T2.5)"
sys.path.insert(0, str(ROOT / "scripts"))
import dual_stack # noqa: E402
def run(*args):
proc = subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, check=False)
return proc.returncode, proc.stdout, proc.stderr
def server_log(*events):
return "\n".join(json.dumps(event) for event in events) + "\n"
def started(stack=None):
event = {"level": "INFO", "event": "server_starting", "message": "Starting RustFS server"}
if stack is not None:
event["s3_stack"] = stack
return event
def junit(cases):
body = ""
for name, state in cases.items():
inner = {"pass": "", "fail": "<failure message='x'/>", "skip": "<skipped/>"}[state]
body += f"<testcase classname='e2e' name='{name}'>{inner}</testcase>"
return f"<testsuites><testsuite name='s'>{body}</testsuite></testsuites>"
class Tree:
"""A downloaded-legs directory with one profile and both of its legs."""
def __init__(self, tmp, profile="e2e-smoke"):
self.root = Path(tmp)
self.profile = profile
def leg(self, stack):
return self.root / "legs" / f"dual-stack-leg-{self.profile}-{stack}"
def write(self, stack, cases, evidence=None):
directory = self.leg(stack)
directory.mkdir(parents=True, exist_ok=True)
(directory / "junit.xml").write_text(junit(cases))
logs = self.root / f"logs-{stack}"
logs.mkdir(exist_ok=True)
(logs / "node.log").write_text(server_log(started(stack.capitalize())) if evidence is None else evidence)
code, _, _ = run("stack", "--expect", stack, "--out", str(directory / "stack.json"), str(logs))
return code
def summary(self):
code, out, err = run("summary", "--profiles", self.profile, "--legs", str(self.root / "legs"),
"--date", "2026-11-13", "--out-dir", str(self.root / "out"))
report = self.root / "out" / "2026-11-13.json"
return code, out + err, json.loads(report.read_text()) if report.is_file() else None
class StackProofTests(unittest.TestCase):
def prove(self, expect, files):
with tempfile.TemporaryDirectory() as tmp:
for name, text in files.items():
(Path(tmp) / name).write_text(text)
out = Path(tmp) / "evidence.json"
code, _, err = run("stack", "--expect", expect, "--out", str(out), tmp)
return code, json.loads(out.read_text()) if out.is_file() else None, err
def test_a_server_of_the_other_stack_fails_the_proof(self):
files = {"a.log": server_log(started("Legacy")), "b.log": server_log(started("Gateway"))}
code, evidence, _ = self.prove("legacy", files)
self.assertEqual(code, 1)
self.assertEqual(evidence["foreign_starts"], 1)
def test_the_other_direction_fails_too(self):
code, evidence, _ = self.prove("gateway", {"a.log": server_log(started("Legacy"), started("Gateway"))})
self.assertEqual((code, evidence["foreign_starts"]), (1, 1))
def test_a_refused_start_proves_nothing_and_keeps_the_message(self):
# The shape a refusing server logs: the cause is in `error`, `message` is generic.
log = server_log({"level": "INFO", "message": "obs ready"},
{"level": "ERROR", "message": "Server runtime failed", "event": "server_runtime_failed", "error": REFUSAL})
code, evidence, _ = self.prove("gateway", {"a.log": log})
self.assertEqual(code, 1)
self.assertEqual(evidence["not_started"], [{"message": REFUSAL, "logs": 1}])
def test_unstamped_starts_alone_prove_nothing(self):
code, evidence, _ = self.prove("legacy", {"old.log": server_log(started())})
self.assertEqual((code, evidence["unstamped_starts"]), (1, 1))
def test_empty_directory_is_an_input_error(self):
with tempfile.TemporaryDirectory() as tmp:
code, _, err = run("stack", "--expect", "legacy", "--out", str(Path(tmp) / "e.json"), tmp)
self.assertEqual(code, 2)
self.assertIn("nothing proves", err)
def test_missing_path_is_an_input_error(self):
code, _, _ = run("stack", "--expect", "legacy", "--out", "/dev/null", "/nonexistent-dual-stack-logs")
self.assertEqual(code, 2)
def test_runner_output_without_log_events_is_not_a_server_log(self):
files = {"a.log": server_log(started("Legacy")), "nextest.log": "PASS [ 1.0s] some::test\nSummary [ 2s] 1 test run\n"}
code, evidence, _ = self.prove("legacy", files)
self.assertEqual((code, evidence["logs_without_start"]), (0, 0))
def test_a_plain_text_prefix_does_not_hide_the_event_or_split_the_message(self):
line = "[FATAL] Server runtime failed: x" + json.dumps({"level": "ERROR", "error": REFUSAL}) + "\n"
code, evidence, _ = self.prove("gateway", {"a.log": line, "b.log": line})
self.assertEqual((code, evidence["not_started"]), (1, [{"message": REFUSAL, "logs": 2}]))
def test_every_server_on_the_requested_stack_passes(self):
files = {"a.log": server_log(started("Legacy")), "b.log": server_log(started("Legacy"), started("Legacy"))}
code, evidence, _ = self.prove("legacy", files)
self.assertEqual((code, evidence["started"]), (0, {"legacy": 3}))
def test_older_release_servers_are_counted_not_failed(self):
files = {"new.log": server_log(started("Gateway")), "old.log": server_log(started())}
code, evidence, _ = self.prove("gateway", files)
self.assertEqual((code, evidence["unstamped_starts"]), (0, 1))
class SummaryTests(unittest.TestCase):
CASES = {"alpha": "pass", "beta": "pass", "gamma": "fail", "delta": "skip"}
def tree(self, tmp, legacy=None, gateway=None):
tree = Tree(tmp)
tree.write("legacy", legacy or self.CASES)
tree.write("gateway", gateway or self.CASES)
return tree
def test_missing_gateway_report_is_no_result_and_fails(self):
with tempfile.TemporaryDirectory() as tmp:
tree = self.tree(tmp)
(tree.leg("gateway") / "junit.xml").unlink()
code, text, report = tree.summary()
self.assertEqual(code, 1)
self.assertIn("no result for gateway", text)
self.assertIsNone(report["profiles"][0]["diff"])
def test_empty_legacy_report_is_no_result_and_fails(self):
with tempfile.TemporaryDirectory() as tmp:
tree = self.tree(tmp)
(tree.leg("legacy") / "junit.xml").write_text("")
code, text, _ = tree.summary()
self.assertEqual(code, 1)
self.assertIn("no result for legacy", text)
def test_a_leg_that_never_uploaded_is_no_result(self):
with tempfile.TemporaryDirectory() as tmp:
tree = Tree(tmp)
tree.write("legacy", self.CASES)
code, text, _ = tree.summary()
self.assertEqual(code, 1)
self.assertIn("no result for gateway", text)
def test_gateway_that_never_started_is_reported_not_compared(self):
refusal = server_log({"level": "ERROR", "message": REFUSAL})
with tempfile.TemporaryDirectory() as tmp:
tree = Tree(tmp)
tree.write("legacy", self.CASES)
tree.write("gateway", {name: "fail" for name in self.CASES}, evidence=refusal)
code, text, report = tree.summary()
self.assertEqual(code, 0, text)
self.assertIn(f"server did not start: {REFUSAL}", text)
self.assertNotIn("###", text)
self.assertEqual(report["profiles"][0]["gateway"]["status"], "not_started")
self.assertIsNone(report["profiles"][0]["diff"])
def test_gateway_skipped_after_a_refused_preflight_is_reported_as_skipped(self):
with tempfile.TemporaryDirectory() as tmp:
tree = Tree(tmp)
tree.write("legacy", self.CASES)
directory = tree.leg("gateway")
directory.mkdir(parents=True)
logs = Path(tmp) / "pre"
logs.mkdir()
(logs / "server.log").write_text(server_log({"level": "ERROR", "message": "Server runtime failed", "error": REFUSAL}))
run("stack", "--expect", "gateway", "--out", str(directory / "stack.json"), str(logs))
code, text, report = tree.summary()
self.assertEqual(code, 0, text)
self.assertIn(f"skipped: gateway stack refuses startup: {REFUSAL}", text)
self.assertEqual(report["profiles"][0]["gateway"]["status"], "skipped")
def test_a_skipped_legacy_side_fails(self):
with tempfile.TemporaryDirectory() as tmp:
tree = Tree(tmp)
tree.write("legacy", self.CASES, evidence=server_log({"level": "ERROR", "message": "boom"}))
(tree.leg("legacy") / "junit.xml").unlink()
tree.write("gateway", self.CASES)
code, _, report = tree.summary()
self.assertEqual(code, 1)
self.assertEqual(report["profiles"][0]["legacy"]["status"], "not_started")
def test_a_proven_leg_states_how_many_server_logs_prove_it(self):
with tempfile.TemporaryDirectory() as tmp:
tree = self.tree(tmp)
_, text, _ = tree.summary()
self.assertIn("ran; 1 of 1 server logs prove legacy, 0 without a start, 0 unstamped starts", text)
def test_a_case_only_the_gateway_ran_is_counted(self):
with tempfile.TemporaryDirectory() as tmp:
tree = self.tree(tmp, gateway={**self.CASES, "extra": "pass"})
_, _, report = tree.summary()
diff = report["profiles"][0]["diff"]
self.assertEqual((diff["cases"], len(diff["other_differences"])), (5, 1))
def test_legacy_that_never_started_fails(self):
with tempfile.TemporaryDirectory() as tmp:
tree = Tree(tmp)
tree.write("legacy", self.CASES, evidence=server_log({"level": "ERROR", "message": "boom"}))
tree.write("gateway", self.CASES)
code, text, _ = tree.summary()
self.assertEqual(code, 1)
self.assertIn("server did not start: boom", text)
def test_a_leg_on_the_wrong_stack_fails(self):
with tempfile.TemporaryDirectory() as tmp:
tree = Tree(tmp)
tree.write("legacy", self.CASES)
tree.write("gateway", self.CASES, evidence=server_log(started("Legacy")))
code, text, report = tree.summary()
self.assertEqual(code, 1)
self.assertEqual(report["profiles"][0]["gateway"]["status"], "wrong_stack")
self.assertNotIn("No differences", text)
def test_injected_gateway_only_failure_is_named(self):
with tempfile.TemporaryDirectory() as tmp:
tree = self.tree(tmp, gateway={**self.CASES, "alpha": "fail"})
code, text, report = tree.summary()
self.assertEqual(code, 0, text)
self.assertIn("Gateway-only failures (1): `e2e::alpha`", text)
diff = report["profiles"][0]["diff"]
self.assertEqual((diff["gateway_only_failures"], diff["legacy_only_failures"]), (["e2e::alpha"], []))
self.assertEqual((diff["cases"], diff["passed_on_both"], diff["failed_on_both"]), (4, 1, 1))
def test_legacy_only_failure_is_named(self):
with tempfile.TemporaryDirectory() as tmp:
tree = self.tree(tmp, legacy={**self.CASES, "beta": "fail"})
_, text, report = tree.summary()
self.assertIn("Legacy-only failures (1): `e2e::beta`", text)
self.assertEqual(report["profiles"][0]["diff"]["gateway_only_failures"], [])
def test_pass_against_skip_and_a_missing_case_are_other_differences(self):
gateway = {"alpha": "skip", "beta": "pass", "gamma": "fail", "extra": "pass"}
with tempfile.TemporaryDirectory() as tmp:
tree = self.tree(tmp, gateway=gateway)
_, _, report = tree.summary()
diff = report["profiles"][0]["diff"]
self.assertEqual(len(diff["other_differences"]), 3)
self.assertEqual((diff["legacy_only_failures"], diff["gateway_only_failures"]), ([], []))
def test_named_failures_are_capped_and_counted(self):
legacy = {f"case{n:03}": "pass" for n in range(dual_stack.NAMED_LIMIT + 5)}
with tempfile.TemporaryDirectory() as tmp:
tree = self.tree(tmp, legacy=legacy, gateway={name: "fail" for name in legacy})
_, text, report = tree.summary()
self.assertIn("(+5 more in the JSON report)", text)
self.assertEqual(len(report["profiles"][0]["diff"]["gateway_only_failures"]), dual_stack.NAMED_LIMIT + 5)
def test_identical_legs_have_no_differences(self):
with tempfile.TemporaryDirectory() as tmp:
tree = self.tree(tmp)
code, text, report = tree.summary()
self.assertEqual(code, 0, text)
diff = report["profiles"][0]["diff"]
self.assertEqual((diff["cases"], diff["passed_on_both"], diff["failed_on_both"]), (4, 2, 1))
self.assertNotIn("###", text)
class PreflightTests(unittest.TestCase):
"""`preflight` starts a binary once on the gateway stack; the binaries here are scripts that log what a server logs."""
def preflight(self, body):
with tempfile.TemporaryDirectory() as tmp:
binary = Path(tmp) / "server"
binary.write_text(f"#!{sys.executable}\nimport json, os, sys, time\n{body}\n")
binary.chmod(binary.stat().st_mode | stat.S_IEXEC)
out = Path(tmp) / "out" / "stack.json"
code, _, _ = run("preflight", "--binary", str(binary), "--out", str(out))
return code, json.loads(out.read_text())
def test_a_refusing_binary_is_not_a_proven_start(self):
body = ('print(json.dumps({"level": "ERROR", "message": "Server runtime failed", "error": %r}), flush=True)\n'
"sys.exit(1)") % REFUSAL
code, evidence = self.preflight(body)
self.assertEqual(code, 1)
self.assertEqual(evidence["not_started"][0]["message"], REFUSAL)
def test_a_binary_that_starts_on_the_gateway_stack_passes_and_is_stopped(self):
body = ('print(json.dumps({"event": "server_starting", "s3_stack": os.environ["RUSTFS_S3_STACK"].capitalize()}), flush=True)\n'
"time.sleep(300)")
code, evidence = self.preflight(body)
self.assertEqual((code, evidence["started"]), (0, {"gateway": 1}))
def test_a_binary_that_ignores_the_request_and_starts_legacy_fails(self):
body = 'print(json.dumps({"event": "server_starting", "s3_stack": "Legacy"}), flush=True)\ntime.sleep(300)'
code, evidence = self.preflight(body)
self.assertEqual((code, evidence["foreign_starts"]), (1, 1))
def test_a_binary_that_exits_silently_proves_nothing(self):
code, evidence = self.preflight("sys.exit(0)")
self.assertEqual((code, evidence["started"]), (1, {}))
def test_a_missing_binary_is_an_input_error(self):
code, _, _ = run("preflight", "--binary", "/nonexistent-dual-stack-binary", "--out", "/dev/null")
self.assertEqual(code, 2)
if __name__ == "__main__":
unittest.main()