ci: log the selected S3 stack at start-up and prove it from server logs

.config/make/tests.mak

@@ -99,6 +99,7 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_dual_stack.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_api_coverage.py

rustfs/src/startup_server.rs

@@ -124,6 +124,7 @@ pub(crate) async fn init_startup_listen_context(
         ip = %server_addr.ip(),
         port = %server_port,
         version = %crate::version::get_version(),
+        s3_stack = ?config.s3_stack,
         "Starting RustFS server"
     );
 

scripts/README.md

@@ -59,6 +59,8 @@ their issue closes.
 | `test_object_generation_protocol_model.py` | dev-tool | Bounded single-slot, four-voter promise/accept design model and recovery/retirement examples; no production runtime evidence | `python3 scripts/test_object_generation_protocol_model.py`; [Generation contract](../docs/architecture/unified-object-generation.md#executable-design-model-and-acceptance-mapping) |
 | `junit_diff.py` | dev-tool | Case-by-case pass/fail/skip diff of two JUnit XML reports (reference vs candidate stack); exit 1 on any difference or stale `--allow` entry, 2 on unreadable input | `python3 scripts/junit_diff.py a.xml b.xml`; backlog T3.1 |
 | `test_junit_diff.py` | ci-gate | Contract tests and fixtures (`scripts/tests/junit_diff/`) for `junit_diff.py` | `make script-tests` |
+| `dual_stack.py` | ci-gate | Proves which S3 stack a leg's servers ran from their own `server_starting` line (`stack`) | `python3 scripts/dual_stack.py stack --help`; backlog T3.1 |
+| `test_dual_stack.py` | ci-gate | Contract tests for `dual_stack.py` | `make script-tests` |
 | `s3-tests/reference_parity.py` | ci-gate | Plans the reference parity targets, lists a run's failures for the single rerun, and compares every candidate's s3-tests JUnit against the pinned reference per node id with a tolerated-difference file | `.github/workflows/reference-parity.yml`; backlog T3.2 |
 | `s3-tests/test_reference_parity.py` | ci-gate | Contract tests for `reference_parity.py` | `make script-tests` |
 | `test_reference_binary.py` | ci-gate | Contract tests for `.github/actions/reference-binary/reference_binary.py` (pin resolution and verified unpacking) | `make script-tests` |

scripts/dual_stack.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Prove which S3 stack the servers of one e2e leg ran, from their own start-up output.
+
+  stack  read the `server_starting` line every server logs (`s3_stack`) in the given logs; fails on a
+         server of the other stack, on no server of the requested stack, and on no server output at all
+
+The stack is never taken from the variable a workflow set. Servers of older releases log no stack
+and are counted as unstamped. Exit codes: 0 proven; 1 not proven; 2 bad input.
+"""
+
+import argparse
+import json
+from collections import Counter
+from pathlib import Path
+import sys
+
+STACKS = ("legacy", "gateway")
+START_EVENT = "server_starting"
+MESSAGE_LIMIT = 200
+
+
+class InputError(Exception):
+    """Input that cannot be trusted; maps to exit code 2."""
+
+
+def log_files(paths):
+    files = []
+    for path in map(Path, paths):
+        if path.is_dir():
+            files.extend(sorted(item for item in path.rglob("*") if item.is_file()))
+        elif path.is_file():
+            files.append(path)
+        else:
+            raise InputError(f"{path}: not found")
+    if not files:
+        raise InputError("no server output found: nothing proves which stack ran")
+    return files
+
+
+def log_event(line):
+    """The JSON event of a log line, ignoring a plain-text prefix glued to it; None for plain text."""
+    start = line.find("{")
+    if start < 0:
+        return None
+    try:
+        event = json.loads(line[start:])
+    except json.JSONDecodeError:
+        return None
+    return event if isinstance(event, dict) else None
+
+
+def prove_stack(expect, paths):
+    started, unstamped, without_start, proven = Counter(), 0, Counter(), 0
+    files = log_files(paths)
+    for path in files:
+        starts, saw_event, last = set(), False, ""
+        for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
+            line = line.strip()
+            event = log_event(line)
+            if event is None:
+                last = line or last
+                continue
+            saw_event = True
+            if event.get("event") == START_EVENT:
+                if "s3_stack" in event:
+                    started[str(event["s3_stack"]).lower()] += 1
+                    starts.add(str(event["s3_stack"]).lower())
+                else:
+                    unstamped += 1
+                    starts.add(None)
+            elif event.get("level") == "ERROR":
+                last = str(event.get("error") or event.get("message") or line)
+        proven += expect in starts
+        # Text that is not a server log (a test runner's output) has no event and is not a server.
+        if not starts and saw_event:
+            without_start[last[:MESSAGE_LIMIT]] += 1
+    foreign = sum(count for stack, count in started.items() if stack != expect)
+    if foreign:
+        verdict = f"{foreign} server(s) started on another stack although {expect} was requested"
+    elif not started[expect]:
+        verdict = f"no server started on the {expect} stack"
+    else:
+        verdict = "ok"
+    return {
+        "expected": expect,
+        "verdict": verdict,
+        "started": dict(started),
+        "foreign_starts": foreign,
+        "unstamped_starts": unstamped,
+        "logs_scanned": len(files),
+        "logs_proven": proven,
+        "logs_without_start": sum(without_start.values()),
+        "not_started": [{"message": message, "logs": count} for message, count in without_start.most_common(5)],
+    }
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    commands = parser.add_subparsers(dest="command", required=True)
+    stack = commands.add_parser("stack")
+    stack.add_argument("--expect", choices=STACKS, required=True)
+    stack.add_argument("--out", required=True, help="write the evidence JSON here")
+    stack.add_argument("paths", nargs="+", help="server log files or directories")
+    args = parser.parse_args(argv)
+    try:
+        evidence = prove_stack(args.expect, args.paths)
+        Path(args.out).parent.mkdir(parents=True, exist_ok=True)
+        Path(args.out).write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+        print(json.dumps(evidence, indent=2, sort_keys=True))
+        return 0 if evidence["verdict"] == "ok" else 1
+    except (InputError, OSError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/test_dual_stack.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contract tests for dual_stack.py stack: which stack a leg's servers ran, proven from their own logs."""
+
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+SCRIPT = ROOT / "scripts" / "dual_stack.py"
+REFUSAL = "gateway stack is not assembled yet (rustfs/backlog#2734 T2.5)"
+
+
+def run(*args):
+    proc = subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, check=False)
+    return proc.returncode, proc.stdout, proc.stderr
+
+
+def server_log(*events):
+    return "\n".join(json.dumps(event) for event in events) + "\n"
+
+
+def started(stack=None):
+    event = {"level": "INFO", "event": "server_starting", "message": "Starting RustFS server"}
+    if stack is not None:
+        event["s3_stack"] = stack
+    return event
+
+
+class StackProofTests(unittest.TestCase):
+    def prove(self, expect, files):
+        with tempfile.TemporaryDirectory() as tmp:
+            for name, text in files.items():
+                (Path(tmp) / name).write_text(text)
+            out = Path(tmp) / "evidence.json"
+            code, _, err = run("stack", "--expect", expect, "--out", str(out), tmp)
+            return code, json.loads(out.read_text()) if out.is_file() else None, err
+
+    def test_a_server_of_the_other_stack_fails_the_proof(self):
+        files = {"a.log": server_log(started("Legacy")), "b.log": server_log(started("Gateway"))}
+        code, evidence, _ = self.prove("legacy", files)
+        self.assertEqual(code, 1)
+        self.assertEqual(evidence["foreign_starts"], 1)
+
+    def test_the_other_direction_fails_too(self):
+        code, evidence, _ = self.prove("gateway", {"a.log": server_log(started("Legacy"), started("Gateway"))})
+        self.assertEqual((code, evidence["foreign_starts"]), (1, 1))
+
+    def test_a_refused_start_proves_nothing_and_keeps_the_message(self):
+        # The shape a refusing server logs: the cause is in `error`, `message` is generic.
+        log = server_log({"level": "INFO", "message": "obs ready"},
+                         {"level": "ERROR", "message": "Server runtime failed", "event": "server_runtime_failed", "error": REFUSAL})
+        code, evidence, _ = self.prove("gateway", {"a.log": log})
+        self.assertEqual(code, 1)
+        self.assertEqual(evidence["not_started"], [{"message": REFUSAL, "logs": 1}])
+
+    def test_unstamped_starts_alone_prove_nothing(self):
+        code, evidence, _ = self.prove("legacy", {"old.log": server_log(started())})
+        self.assertEqual((code, evidence["unstamped_starts"]), (1, 1))
+
+    def test_empty_directory_is_an_input_error(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            code, _, err = run("stack", "--expect", "legacy", "--out", str(Path(tmp) / "e.json"), tmp)
+            self.assertEqual(code, 2)
+            self.assertIn("nothing proves", err)
+
+    def test_missing_path_is_an_input_error(self):
+        code, _, _ = run("stack", "--expect", "legacy", "--out", "/dev/null", "/nonexistent-dual-stack-logs")
+        self.assertEqual(code, 2)
+
+    def test_runner_output_without_log_events_is_not_a_server_log(self):
+        files = {"a.log": server_log(started("Legacy")), "nextest.log": "PASS [ 1.0s] some::test\nSummary [ 2s] 1 test run\n"}
+        code, evidence, _ = self.prove("legacy", files)
+        self.assertEqual((code, evidence["logs_without_start"]), (0, 0))
+
+    def test_a_plain_text_prefix_does_not_hide_the_event_or_split_the_message(self):
+        line = "[FATAL] Server runtime failed: x" + json.dumps({"level": "ERROR", "error": REFUSAL}) + "\n"
+        code, evidence, _ = self.prove("gateway", {"a.log": line, "b.log": line})
+        self.assertEqual((code, evidence["not_started"]), (1, [{"message": REFUSAL, "logs": 2}]))
+
+    def test_every_server_on_the_requested_stack_passes(self):
+        files = {"a.log": server_log(started("Legacy")), "b.log": server_log(started("Legacy"), started("Legacy"))}
+        code, evidence, _ = self.prove("legacy", files)
+        self.assertEqual((code, evidence["started"]), (0, {"legacy": 3}))
+
+    def test_older_release_servers_are_counted_not_failed(self):
+        files = {"new.log": server_log(started("Gateway")), "old.log": server_log(started())}
+        code, evidence, _ = self.prove("gateway", files)
+        self.assertEqual((code, evidence["unstamped_starts"]), (0, 1))
+
+
+if __name__ == "__main__":
+    unittest.main()