ci(s3-tests): add mint workflow, multi-node sweep, and API coverage tool (#4206)

Follow-ups to the compatibility harness rework:

- Weekly full sweep now runs as a matrix over both topologies: single
  node and the 4-node distributed cluster. Manual dispatch keeps the
  test-mode input.
- New mint workflow (weekly + dispatch) runs MinIO Mint against RustFS:
  functional suites of real client SDKs and tools (awscli, mc,
  aws-sdk-*, minio-*, s3cmd, ...), catching client-specific signing and
  streaming edge cases that boto3-only ceph/s3-tests cannot. Report-only
  for test failures with a per-suite summary table; fails only when mint
  produces no results.
- New scripts/s3-tests/api_coverage.py quantifies API surface coverage
  by diffing the s3s S3 trait (at the Cargo.toml pinned revision)
  against the methods overridden in impl S3 for FS, distinguishing
  NotImplemented defaults from delegating defaults (e.g. post_object).
  Current state: 76/101 operations covered (75 overridden, 1 delegated).

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

.github/workflows/e2e-s3tests.yml

@@ -63,7 +63,8 @@ on:
         required: false
         default: ""
   schedule:
-    # Weekly full sweep (Sunday 02:00 UTC): full suite, single-node topology.
+    # Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the
+    # single-node and the 4-node distributed topologies (matrix below).
     - cron: "0 2 * * 0"
 
 env:
@@ -82,8 +83,7 @@ env:
   PLATFORM: linux/amd64
   BUILDX_CACHE_SCOPE: rustfs-e2e-s3tests-source
 
-  # Scheduled runs have no inputs; default to a single-node full sweep.
-  TEST_MODE: ${{ github.event.inputs.test-mode || 'single' }}
+  # Scheduled runs have no inputs; default to a full sweep.
   TEST_SCOPE: ${{ github.event.inputs.scope || 'all' }}
   XDIST: ${{ github.event.inputs.xdist || '4' }}
   MAXFAIL: ${{ github.event.inputs.maxfail || '0' }}
@@ -101,6 +101,13 @@ jobs:
   s3tests:
     runs-on: ubicloud-standard-4
     timeout-minutes: 180
+    strategy:
+      fail-fast: false
+      matrix:
+        # Scheduled sweeps cover both topologies; manual runs use the input.
+        test-mode: ${{ github.event_name == 'schedule' && fromJSON('["single", "multi"]') || fromJSON(format('["{0}"]', github.event.inputs.test-mode || 'single')) }}
+    env:
+      TEST_MODE: ${{ matrix.test-mode }}
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
 

.github/workflows/mint.yml

@@ -0,0 +1,209 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# MinIO Mint against RustFS: multi-SDK client compatibility.
+#
+# ceph/s3-tests (see e2e-s3tests.yml and the ci.yml PR gate) exercises the S3
+# API semantics through boto3 only. Mint complements it by running the
+# functional test suites of many REAL client SDKs and tools (awscli, mc,
+# aws-sdk-java/go/php/ruby, minio-go/java/js/py, s3cmd, ...), which catches
+# client-specific signing, encoding, and streaming edge cases a single client
+# cannot.
+#
+# This job is REPORT-ONLY for test failures: RustFS does not pass every mint
+# suite yet, so failures are inventoried in the job summary instead of turning
+# the run red. The job fails only when mint produces no results at all
+# (infrastructure error). Once a suite passes reliably, tightening this into a
+# baseline gate (like implemented_tests.txt for s3-tests) is the natural next
+# step.
+
+name: mint
+
+on:
+  workflow_dispatch:
+    inputs:
+      suites:
+        description: "Space-separated mint suites to run (empty = all, e.g. 'awscli mc minio-go')"
+        required: false
+        default: ""
+      mode:
+        description: "Mint mode: core (quick) or full"
+        required: true
+        type: choice
+        default: "core"
+        options:
+          - core
+          - full
+      mint-image:
+        description: "Mint image reference"
+        required: false
+        default: "minio/mint:edge"
+  schedule:
+    # Weekly, after the Sunday s3-tests full sweep.
+    - cron: "0 4 * * 0"
+
+env:
+  S3_ACCESS_KEY: rustfsadmin
+  S3_SECRET_KEY: rustfsadmin
+  S3_REGION: us-east-1
+
+  RUST_LOG: info
+  PLATFORM: linux/amd64
+  BUILDX_CACHE_SCOPE: rustfs-e2e-s3tests-source
+
+  MINT_SUITES: ${{ github.event.inputs.suites || '' }}
+  MINT_MODE: ${{ github.event.inputs.mode || 'core' }}
+  # NOTE: minio/mint:edge is a rolling tag. If sweeps get noisy from upstream
+  # mint changes, pin this to a digest known to work.
+  MINT_IMAGE: ${{ github.event.inputs.mint-image || 'minio/mint:edge' }}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  mint:
+    runs-on: ubicloud-standard-4
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+
+      - name: Enable buildx
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+
+      - name: Build RustFS image (source, cached)
+        run: |
+          DOCKER_BUILDKIT=1 docker buildx build --load \
+            --platform ${PLATFORM} \
+            --cache-from type=gha,scope=${BUILDX_CACHE_SCOPE} \
+            --cache-to type=gha,mode=max,scope=${BUILDX_CACHE_SCOPE} \
+            -t rustfs-ci \
+            -f Dockerfile.source .
+
+      - name: Start RustFS
+        run: |
+          docker network inspect rustfs-net >/dev/null 2>&1 || docker network create rustfs-net
+          docker rm -f rustfs-mint >/dev/null 2>&1 || true
+          docker run -d --name rustfs-mint \
+            --network rustfs-net \
+            -p 9000:9000 \
+            -e RUSTFS_ADDRESS=0.0.0.0:9000 \
+            -e RUSTFS_ACCESS_KEY=${S3_ACCESS_KEY} \
+            -e RUSTFS_SECRET_KEY=${S3_SECRET_KEY} \
+            -e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
+            -v /tmp/rustfs-mint:/data \
+            rustfs-ci
+
+      - name: Wait for RustFS ready
+        run: |
+          for i in {1..60}; do
+            if curl -sf http://127.0.0.1:9000/health >/dev/null 2>&1; then
+              echo "RustFS is ready"
+              exit 0
+            fi
+            if [ "$(docker inspect -f '{{.State.Running}}' rustfs-mint 2>/dev/null)" != "true" ]; then
+              echo "RustFS container not running" >&2
+              docker logs rustfs-mint || true
+              exit 1
+            fi
+            sleep 2
+          done
+          echo "Health check timed out" >&2
+          docker logs rustfs-mint || true
+          exit 1
+
+      - name: Run mint
+        run: |
+          mkdir -p artifacts/mint-log
+          # Mint exits non-zero when any test fails; that is inventory here,
+          # not a gate (see header comment). Capture the exit code and let the
+          # summary step decide.
+          set +e
+          # shellcheck disable=SC2086  # MINT_SUITES is intentionally word-split
+          docker run --rm --network rustfs-net \
+            -e SERVER_ENDPOINT=rustfs-mint:9000 \
+            -e ACCESS_KEY=${S3_ACCESS_KEY} \
+            -e SECRET_KEY=${S3_SECRET_KEY} \
+            -e ENABLE_HTTPS=0 \
+            -e SERVER_REGION=${S3_REGION} \
+            -e MINT_MODE=${MINT_MODE} \
+            -v "${PWD}/artifacts/mint-log:/mint/log" \
+            "${MINT_IMAGE}" ${MINT_SUITES}
+          echo "MINT_RC=$?" >> "$GITHUB_ENV"
+
+      - name: Summarize results
+        run: |
+          python3 - <<'EOF'
+          import json, os, pathlib, sys
+          from collections import Counter
+
+          log = pathlib.Path("artifacts/mint-log/log.json")
+          if not log.is_file():
+              print("mint produced no log.json - infrastructure failure", file=sys.stderr)
+              sys.exit(int(os.environ.get("MINT_RC", "1")) or 1)
+
+          per_suite: dict[str, Counter] = {}
+          failures: list[tuple[str, str, str]] = []
+          for line in log.read_text(encoding="utf-8").splitlines():
+              line = line.strip()
+              if not line:
+                  continue
+              try:
+                  entry = json.loads(line)
+              except json.JSONDecodeError:
+                  continue
+              suite = entry.get("name", "unknown")
+              status = entry.get("status", "unknown").upper()
+              per_suite.setdefault(suite, Counter())[status] += 1
+              if status == "FAIL":
+                  failures.append((suite, entry.get("function", ""), entry.get("error") or entry.get("message") or ""))
+
+          lines = ["# Mint results", "", "| Suite | Pass | Fail | N/A |", "|-------|------|------|-----|"]
+          for suite in sorted(per_suite):
+              c = per_suite[suite]
+              lines.append(f"| {suite} | {c.get('PASS', 0)} | {c.get('FAIL', 0)} | {c.get('NA', 0)} |")
+          lines.append("")
+          if failures:
+              lines.append(f"## Failures ({len(failures)})")
+              lines.append("")
+              for suite, func, err in failures:
+                  first = err.strip().splitlines()[0] if err.strip() else ""
+                  lines.append(f"- `{suite}` / `{func}` {first}")
+              lines.append("")
+              lines.append("_Report-only: failures are inventory, not a gate (see workflow header)._")
+
+          report = "\n".join(lines)
+          print(report)
+          summary = os.environ.get("GITHUB_STEP_SUMMARY")
+          if summary:
+              with open(summary, "a", encoding="utf-8") as fh:
+                  fh.write(report + "\n")
+          EOF
+
+      - name: Collect RustFS logs
+        if: always()
+        run: |
+          mkdir -p artifacts/rustfs-mint
+          docker logs rustfs-mint > artifacts/rustfs-mint/rustfs.log 2>&1 || true
+
+      - name: Upload artifacts
+        if: always() && env.ACT != 'true'
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: mint
+          path: artifacts/**

scripts/s3-tests/README.md

@@ -457,6 +457,30 @@ Keeping both workflows on this script means local runs, the PR gate, and the
 scheduled sweep always execute tests the same way (same pinned s3-tests
 revision, same config template, same user provisioning).
 
+A third workflow, **mint** (`.github/workflows/mint.yml`), complements ceph
+s3-tests with MinIO Mint: the functional suites of many real client SDKs and
+tools (awscli, mc, aws-sdk-java/go/php/ruby, minio-go/java/js/py, s3cmd, ...).
+It runs weekly, is report-only for test failures, and publishes a per-suite
+pass/fail table in the job summary.
+
+## Companion Tools
+
+- `report_compat.py` — diffs a junit.xml result against the classification
+  lists; run automatically at the end of `run.sh`, and used by the weekly
+  sweep to gate on whitelist regressions only (`--fail-on-regression`).
+- `api_coverage.py` — quantifies S3 API surface coverage by comparing the
+  s3s `S3` trait (at the revision pinned in Cargo.toml) against the methods
+  RustFS overrides in `impl S3 for FS`:
+
+  ```bash
+  python3 scripts/s3-tests/api_coverage.py            # summary + missing ops
+  python3 scripts/s3-tests/api_coverage.py --output api-coverage.md
+  ```
+
+- `compare_dual_targets.py` — sends one signed S3 request to two endpoints
+  (e.g. RustFS and MinIO) and diffs status, headers, and body; useful when
+  triaging a behavioral difference found by the suites above.
+
 ## See Also
 
 - [GitHub Actions Workflow](../../.github/workflows/e2e-s3tests.yml)

scripts/s3-tests/api_coverage.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Quantify RustFS S3 API surface coverage.
+
+Compares the operations defined by the s3s `S3` trait (at the revision pinned
+in the workspace Cargo.toml) against the methods RustFS overrides in
+`impl S3 for FS` (rustfs/src/storage/ecfs.rs). Trait methods that are not
+overridden fall back to the s3s default implementation, which returns
+NotImplemented — so "overridden" is a faithful proxy for "implemented".
+
+The s3s trait source is located automatically from the local cargo git
+checkout cache; if absent (e.g. a fresh CI runner), it is fetched from the
+pinned revision on GitHub.
+"""
+
+from __future__ import annotations
+
+import argparse
+import pathlib
+import re
+import subprocess
+import sys
+import tempfile
+
+ECFS_PATH = "rustfs/src/storage/ecfs.rs"
+TRAIT_RELPATH = "crates/s3s/src/s3_trait.rs"
+
+
+def project_root() -> pathlib.Path:
+    return pathlib.Path(__file__).resolve().parents[2]
+
+
+def parse_s3s_pin(cargo_toml: pathlib.Path) -> tuple[str, str]:
+    """Return (git url, rev) of the s3s dependency from the workspace Cargo.toml."""
+    for line in cargo_toml.read_text(encoding="utf-8").splitlines():
+        if re.match(r"^s3s\s*=", line):
+            git = re.search(r'git\s*=\s*"([^"]+)"', line)
+            rev = re.search(r'rev\s*=\s*"([^"]+)"', line)
+            if git and rev:
+                return git.group(1), rev.group(1)
+    raise RuntimeError(f"s3s git dependency with rev not found in {cargo_toml}")
+
+
+def locate_trait_source(git_url: str, rev: str) -> pathlib.Path:
+    """Find s3_trait.rs in the cargo git checkout cache, or fetch the pinned rev."""
+    cargo_home = pathlib.Path.home() / ".cargo"
+    for candidate in cargo_home.glob(f"git/checkouts/s3s-*/{rev[:7]}/{TRAIT_RELPATH}"):
+        return candidate
+
+    workdir = pathlib.Path(tempfile.mkdtemp(prefix="s3s-trait-"))
+    subprocess.run(["git", "init", "-q", str(workdir)], check=True)
+    subprocess.run(["git", "-C", str(workdir), "remote", "add", "origin", git_url], check=True)
+    subprocess.run(["git", "-C", str(workdir), "fetch", "-q", "--depth", "1", "origin", rev], check=True)
+    subprocess.run(["git", "-C", str(workdir), "checkout", "-qf", "--detach", rev], check=True)
+    return workdir / TRAIT_RELPATH
+
+
+def extract_block(src: str, start: int) -> str:
+    """Return the brace-balanced block starting at the first '{' at/after start."""
+    depth = 0
+    begin = src.index("{", start)
+    for i in range(begin, len(src)):
+        if src[i] == "{":
+            depth += 1
+        elif src[i] == "}":
+            depth -= 1
+            if depth == 0:
+                return src[begin : i + 1]
+    raise RuntimeError("unbalanced braces")
+
+
+def trait_operations(trait_source: pathlib.Path) -> dict[str, str]:
+    """Return {operation: default kind} for every trait method.
+
+    Kind is "not_implemented" when the default body errors with NotImplemented,
+    or "delegated" when it forwards to another operation (e.g. post_object
+    delegates to put_object), in which case overriding is optional.
+    """
+    src = trait_source.read_text(encoding="utf-8")
+    ops: dict[str, str] = {}
+    for match in re.finditer(r"async fn (\w+)", src):
+        body = extract_block(src, match.end())
+        ops[match.group(1)] = "not_implemented" if "NotImplemented" in body else "delegated"
+    return ops
+
+
+def implemented_operations(ecfs_source: pathlib.Path) -> set[str]:
+    src = ecfs_source.read_text(encoding="utf-8")
+    marker = "impl S3 for FS {"
+    start = src.find(marker)
+    if start < 0:
+        raise RuntimeError(f"`{marker}` not found in {ecfs_source}")
+    block = extract_block(src, start)
+    return set(re.findall(r"async fn (\w+)", block))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--s3s-trait",
+        type=pathlib.Path,
+        help=f"path to s3s {TRAIT_RELPATH} (default: auto-locate from Cargo.toml pin)",
+    )
+    parser.add_argument("--output", type=pathlib.Path, help="write the markdown report to this path")
+    args = parser.parse_args()
+
+    root = project_root()
+    git_url, rev = parse_s3s_pin(root / "Cargo.toml")
+    trait_source = args.s3s_trait or locate_trait_source(git_url, rev)
+
+    all_ops = trait_operations(trait_source)
+    impl_ops = implemented_operations(root / ECFS_PATH)
+
+    stale = sorted(impl_ops - all_ops.keys())
+    implemented = sorted(op for op in all_ops if op in impl_ops)
+    delegated = sorted(op for op, kind in all_ops.items() if op not in impl_ops and kind == "delegated")
+    missing = sorted(op for op, kind in all_ops.items() if op not in impl_ops and kind == "not_implemented")
+    covered = len(implemented) + len(delegated)
+    pct = 100.0 * covered / len(all_ops) if all_ops else 0.0
+
+    lines = [
+        "# S3 API surface coverage",
+        "",
+        f"s3s trait revision: `{rev}` — {len(all_ops)} operations.",
+        "",
+        f"**{covered}/{len(all_ops)} operations covered ({pct:.0f}%)** — "
+        f"{len(implemented)} overridden by RustFS, {len(delegated)} served by s3s default delegation.",
+        "",
+        "Operations below fall back to s3s defaults that return NotImplemented.",
+        "",
+        f"## Missing operations ({len(missing)})",
+        "",
+    ]
+    lines += [f"- `{op}`" for op in missing] or ["_none_"]
+    lines.append("")
+    if delegated:
+        lines += [f"## Covered via s3s default delegation ({len(delegated)})", ""]
+        lines += [f"- `{op}`" for op in delegated]
+        lines.append("")
+    lines += [f"## Implemented operations ({len(implemented)})", ""]
+    lines += [f"- `{op}`" for op in implemented]
+    lines.append("")
+    if stale:
+        lines += [f"## Overridden but absent from the trait ({len(stale)})", ""]
+        lines += [f"- `{op}`" for op in stale]
+        lines += ["", "_These suggest the s3s pin moved; re-run after `cargo update`._", ""]
+
+    report = "\n".join(lines)
+    if args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(report, encoding="utf-8")
+        print(f"[INFO] Report written to {args.output}")
+
+    print(
+        f"[INFO] {covered}/{len(all_ops)} S3 operations covered ({pct:.0f}%): "
+        f"{len(implemented)} overridden, {len(delegated)} delegated, {len(missing)} missing"
+    )
+    for op in missing:
+        print(f"[MISSING] {op}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())