mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-18 05:53:13 +00:00
704b984c1f
T3.1/T3.3 (rustfs/backlog#2772, #2774). The existing nightly lanes get a stack dimension and their legacy leg is the legacy side: e2e-replication-nightly (repl, cluster, protocols), e2e-distributed and the ODM minio-source job. A gateway leg runs scripts/dual_stack.py preflight after its build and stops there when no gateway start is proven; the summary says 'skipped: gateway stack refuses startup: <why>'. One summary job per workflow diffs the legs of the same run with junit_diff.py. dual-stack-nightly.yml covers only e2e-smoke and e2e-full: one build, both stacks from that binary, gateway legs only when the preflight proved a start. junit_diff.py reports a missing, empty or case-less side as 'no result for <label>'. mint.yml stays weekly, builds the image once and runs it per stack; mint_stack_diff.py gates that every case legacy passes is present and passing on gateway with no new failure (missing, NA, unknown status and truncated lines never pass). Gateway legs and the compare job are report-only until T2.10. Not built: a shared-binary preflight job in the three source workflows (one preflight step per gateway leg); harness-only filtering for their five profiles; mc/Vault/iptables in the e2e-full leg; an s3s-e2e both-stack run; rustfs_gateway=debug. Tests: dual_stack 30 (preflight with fake binaries), mint_stack_diff 16, junit_diff 27; negatives outnumber positives. Mutations (red test named) are in the PR 9367 description. Not verified: the workflows have not run in CI in this shape (composites, stack matrices, image artifact round trip, needs result of a red continue-on-error leg).
122 lines
5.6 KiB
Python
122 lines
5.6 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright 2026 RustFS Team
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Compare the mint results of the legacy and the gateway stack (.github/workflows/mint.yml).
|
|
|
|
Each argument is the directory of one mint leg: `mint-log/log.json` and `stack.json`
|
|
(scripts/dual_stack.py stack). A case is `<suite>::<function>` and fails when any entry fails.
|
|
|
|
The gate: every case legacy passes is present and passing on gateway, and no case fails on gateway
|
|
that does not fail on legacy. A case the gateway did not run, reported `NA`, or ran to an unknown
|
|
status never passes, so a truncated or empty gateway result cannot satisfy it. A side without a
|
|
result never satisfies it either (`skipped: gateway stack refuses startup`). A `{` line of log.json
|
|
that does not parse, and a status other than PASS, FAIL or NA, are bad input.
|
|
|
|
Exit codes: 0 the gate holds; 1 it does not, or a side has no result; 2 bad input.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
import dual_stack
|
|
|
|
STACKS = dual_stack.STACKS
|
|
RANK = {"FAIL": 2, "PASS": 1, "NA": 0}
|
|
|
|
|
|
def read_cases(log):
|
|
"""Return {case: PASS|FAIL|NA} from a mint log.json, or None when it holds no result."""
|
|
try:
|
|
lines = log.read_text(encoding="utf-8").splitlines()
|
|
except FileNotFoundError:
|
|
return None
|
|
cases = {}
|
|
for number, line in enumerate(lines, 1):
|
|
if not line.startswith("{"):
|
|
continue # mint also writes plain progress lines into this file
|
|
try:
|
|
entry = json.loads(line)
|
|
except json.JSONDecodeError as exc:
|
|
raise ValueError(f"{log}:{number}: unparsable result line: {exc}") from exc
|
|
status = str(entry.get("status", "")).upper()
|
|
if status not in RANK:
|
|
raise ValueError(f"{log}:{number}: unknown status {entry.get('status')!r}")
|
|
case = f"{entry.get('name', 'unknown')}::{entry.get('function', '')}"
|
|
cases[case] = max(cases.get(case, status), status, key=RANK.get)
|
|
return cases or None
|
|
|
|
|
|
def load(directory, stack):
|
|
"""Return (cases, None) for a proven leg with a result, else (None, why there is none)."""
|
|
state = dual_stack.evidence_state(directory, stack, "mint-log/log.json")
|
|
if state["status"] != "ok":
|
|
return None, state["detail"]
|
|
cases = read_cases(directory / "mint-log" / "log.json")
|
|
if cases is None:
|
|
return None, f"no result for {stack}: mint wrote no log.json entries"
|
|
return cases, None
|
|
|
|
|
|
def compare(legacy, gateway):
|
|
"""Return the report lines and whether the gate holds."""
|
|
sides = {stack: load(Path(directory), stack) for stack, directory in zip(STACKS, (legacy, gateway))}
|
|
lines = ["Mint stack comparison"]
|
|
for stack, (cases, why) in sides.items():
|
|
if cases is None:
|
|
lines.append(f"{stack}: {why}")
|
|
else:
|
|
counts = {status: list(cases.values()).count(status) for status in RANK}
|
|
lines.append(f"{stack}: {len(cases)} cases, {counts['PASS']} pass, {counts['FAIL']} fail, {counts['NA']} n/a")
|
|
if any(cases is None for cases, _ in sides.values()):
|
|
return lines + ["Gate NOT met: a side has no result, so no failing set can be compared"], False
|
|
(legacy_cases, _), (gateway_cases, _) = sides.values()
|
|
lost = {case: gateway_cases.get(case) for case, status in legacy_cases.items() if status == "PASS" and gateway_cases.get(case) != "PASS"}
|
|
new_failures = sorted(case for case, status in gateway_cases.items() if status == "FAIL" and legacy_cases.get(case) != "FAIL" and case not in lost)
|
|
lists = {
|
|
"Passing on legacy, failing on gateway": sorted(case for case, status in lost.items() if status == "FAIL"),
|
|
"Passing on legacy, not run on gateway": sorted(case for case, status in lost.items() if status is None),
|
|
"Passing on legacy, n/a on gateway": sorted(case for case, status in lost.items() if status == "NA"),
|
|
"Failing on gateway, not run or n/a on legacy": new_failures,
|
|
}
|
|
for title, cases in lists.items():
|
|
lines.append(f"{title} ({len(cases)}):")
|
|
lines.extend(f" {case}" for case in cases)
|
|
fixed = sorted(case for case, status in legacy_cases.items() if status == "FAIL" and gateway_cases.get(case) == "PASS")
|
|
lines.append(f"Failing on legacy, passing on gateway ({len(fixed)}):")
|
|
lines.extend(f" {case}" for case in fixed)
|
|
holds = not any(lists.values())
|
|
lines.append("Gate met: every legacy pass passes on gateway and no new gateway failure" if holds
|
|
else "Gate NOT met: the gateway does not pass everything legacy passes")
|
|
return lines, holds
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
parser.add_argument("legacy", help="directory of the legacy mint leg")
|
|
parser.add_argument("gateway", help="directory of the gateway mint leg")
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
lines, holds = compare(args.legacy, args.gateway)
|
|
except (OSError, KeyError, ValueError) as exc:
|
|
print(f"error: {exc}", file=sys.stderr)
|
|
return 2
|
|
print("\n".join(lines))
|
|
return 0 if holds else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|