diff --git a/.github/workflows/e2e-s3tests.yml b/.github/workflows/e2e-s3tests.yml index 101a52665..fb00749a7 100644 --- a/.github/workflows/e2e-s3tests.yml +++ b/.github/workflows/e2e-s3tests.yml @@ -63,7 +63,8 @@ on: required: false default: "" schedule: - # Weekly full sweep (Sunday 02:00 UTC): full suite, single-node topology. + # Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the + # single-node and the 4-node distributed topologies (matrix below). - cron: "0 2 * * 0" env: @@ -82,8 +83,7 @@ env: PLATFORM: linux/amd64 BUILDX_CACHE_SCOPE: rustfs-e2e-s3tests-source - # Scheduled runs have no inputs; default to a single-node full sweep. - TEST_MODE: ${{ github.event.inputs.test-mode || 'single' }} + # Scheduled runs have no inputs; default to a full sweep. TEST_SCOPE: ${{ github.event.inputs.scope || 'all' }} XDIST: ${{ github.event.inputs.xdist || '4' }} MAXFAIL: ${{ github.event.inputs.maxfail || '0' }} @@ -101,6 +101,13 @@ jobs: s3tests: runs-on: ubicloud-standard-4 timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + # Scheduled sweeps cover both topologies; manual runs use the input. + test-mode: ${{ github.event_name == 'schedule' && fromJSON('["single", "multi"]') || fromJSON(format('["{0}"]', github.event.inputs.test-mode || 'single')) }} + env: + TEST_MODE: ${{ matrix.test-mode }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 diff --git a/.github/workflows/mint.yml b/.github/workflows/mint.yml new file mode 100644 index 000000000..87c1a7675 --- /dev/null +++ b/.github/workflows/mint.yml @@ -0,0 +1,209 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MinIO Mint against RustFS: multi-SDK client compatibility. +# +# ceph/s3-tests (see e2e-s3tests.yml and the ci.yml PR gate) exercises the S3 +# API semantics through boto3 only. Mint complements it by running the +# functional test suites of many REAL client SDKs and tools (awscli, mc, +# aws-sdk-java/go/php/ruby, minio-go/java/js/py, s3cmd, ...), which catches +# client-specific signing, encoding, and streaming edge cases a single client +# cannot. +# +# This job is REPORT-ONLY for test failures: RustFS does not pass every mint +# suite yet, so failures are inventoried in the job summary instead of turning +# the run red. The job fails only when mint produces no results at all +# (infrastructure error). Once a suite passes reliably, tightening this into a +# baseline gate (like implemented_tests.txt for s3-tests) is the natural next +# step. + +name: mint + +on: + workflow_dispatch: + inputs: + suites: + description: "Space-separated mint suites to run (empty = all, e.g. 'awscli mc minio-go')" + required: false + default: "" + mode: + description: "Mint mode: core (quick) or full" + required: true + type: choice + default: "core" + options: + - core + - full + mint-image: + description: "Mint image reference" + required: false + default: "minio/mint:edge" + schedule: + # Weekly, after the Sunday s3-tests full sweep. + - cron: "0 4 * * 0" + +env: + S3_ACCESS_KEY: rustfsadmin + S3_SECRET_KEY: rustfsadmin + S3_REGION: us-east-1 + + RUST_LOG: info + PLATFORM: linux/amd64 + BUILDX_CACHE_SCOPE: rustfs-e2e-s3tests-source + + MINT_SUITES: ${{ github.event.inputs.suites || '' }} + MINT_MODE: ${{ github.event.inputs.mode || 'core' }} + # NOTE: minio/mint:edge is a rolling tag. If sweeps get noisy from upstream + # mint changes, pin this to a digest known to work. + MINT_IMAGE: ${{ github.event.inputs.mint-image || 'minio/mint:edge' }} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + mint: + runs-on: ubicloud-standard-4 + timeout-minutes: 120 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + + - name: Enable buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Build RustFS image (source, cached) + run: | + DOCKER_BUILDKIT=1 docker buildx build --load \ + --platform ${PLATFORM} \ + --cache-from type=gha,scope=${BUILDX_CACHE_SCOPE} \ + --cache-to type=gha,mode=max,scope=${BUILDX_CACHE_SCOPE} \ + -t rustfs-ci \ + -f Dockerfile.source . + + - name: Start RustFS + run: | + docker network inspect rustfs-net >/dev/null 2>&1 || docker network create rustfs-net + docker rm -f rustfs-mint >/dev/null 2>&1 || true + docker run -d --name rustfs-mint \ + --network rustfs-net \ + -p 9000:9000 \ + -e RUSTFS_ADDRESS=0.0.0.0:9000 \ + -e RUSTFS_ACCESS_KEY=${S3_ACCESS_KEY} \ + -e RUSTFS_SECRET_KEY=${S3_SECRET_KEY} \ + -e RUSTFS_VOLUMES="/data/rustfs{0...3}" \ + -v /tmp/rustfs-mint:/data \ + rustfs-ci + + - name: Wait for RustFS ready + run: | + for i in {1..60}; do + if curl -sf http://127.0.0.1:9000/health >/dev/null 2>&1; then + echo "RustFS is ready" + exit 0 + fi + if [ "$(docker inspect -f '{{.State.Running}}' rustfs-mint 2>/dev/null)" != "true" ]; then + echo "RustFS container not running" >&2 + docker logs rustfs-mint || true + exit 1 + fi + sleep 2 + done + echo "Health check timed out" >&2 + docker logs rustfs-mint || true + exit 1 + + - name: Run mint + run: | + mkdir -p artifacts/mint-log + # Mint exits non-zero when any test fails; that is inventory here, + # not a gate (see header comment). Capture the exit code and let the + # summary step decide. + set +e + # shellcheck disable=SC2086 # MINT_SUITES is intentionally word-split + docker run --rm --network rustfs-net \ + -e SERVER_ENDPOINT=rustfs-mint:9000 \ + -e ACCESS_KEY=${S3_ACCESS_KEY} \ + -e SECRET_KEY=${S3_SECRET_KEY} \ + -e ENABLE_HTTPS=0 \ + -e SERVER_REGION=${S3_REGION} \ + -e MINT_MODE=${MINT_MODE} \ + -v "${PWD}/artifacts/mint-log:/mint/log" \ + "${MINT_IMAGE}" ${MINT_SUITES} + echo "MINT_RC=$?" >> "$GITHUB_ENV" + + - name: Summarize results + run: | + python3 - <<'EOF' + import json, os, pathlib, sys + from collections import Counter + + log = pathlib.Path("artifacts/mint-log/log.json") + if not log.is_file(): + print("mint produced no log.json - infrastructure failure", file=sys.stderr) + sys.exit(int(os.environ.get("MINT_RC", "1")) or 1) + + per_suite: dict[str, Counter] = {} + failures: list[tuple[str, str, str]] = [] + for line in log.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + suite = entry.get("name", "unknown") + status = entry.get("status", "unknown").upper() + per_suite.setdefault(suite, Counter())[status] += 1 + if status == "FAIL": + failures.append((suite, entry.get("function", ""), entry.get("error") or entry.get("message") or "")) + + lines = ["# Mint results", "", "| Suite | Pass | Fail | N/A |", "|-------|------|------|-----|"] + for suite in sorted(per_suite): + c = per_suite[suite] + lines.append(f"| {suite} | {c.get('PASS', 0)} | {c.get('FAIL', 0)} | {c.get('NA', 0)} |") + lines.append("") + if failures: + lines.append(f"## Failures ({len(failures)})") + lines.append("") + for suite, func, err in failures: + first = err.strip().splitlines()[0] if err.strip() else "" + lines.append(f"- `{suite}` / `{func}` {first}") + lines.append("") + lines.append("_Report-only: failures are inventory, not a gate (see workflow header)._") + + report = "\n".join(lines) + print(report) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as fh: + fh.write(report + "\n") + EOF + + - name: Collect RustFS logs + if: always() + run: | + mkdir -p artifacts/rustfs-mint + docker logs rustfs-mint > artifacts/rustfs-mint/rustfs.log 2>&1 || true + + - name: Upload artifacts + if: always() && env.ACT != 'true' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: mint + path: artifacts/** diff --git a/scripts/s3-tests/README.md b/scripts/s3-tests/README.md index 5638da283..dab4de556 100644 --- a/scripts/s3-tests/README.md +++ b/scripts/s3-tests/README.md @@ -457,6 +457,30 @@ Keeping both workflows on this script means local runs, the PR gate, and the scheduled sweep always execute tests the same way (same pinned s3-tests revision, same config template, same user provisioning). +A third workflow, **mint** (`.github/workflows/mint.yml`), complements ceph +s3-tests with MinIO Mint: the functional suites of many real client SDKs and +tools (awscli, mc, aws-sdk-java/go/php/ruby, minio-go/java/js/py, s3cmd, ...). +It runs weekly, is report-only for test failures, and publishes a per-suite +pass/fail table in the job summary. + +## Companion Tools + +- `report_compat.py` — diffs a junit.xml result against the classification + lists; run automatically at the end of `run.sh`, and used by the weekly + sweep to gate on whitelist regressions only (`--fail-on-regression`). +- `api_coverage.py` — quantifies S3 API surface coverage by comparing the + s3s `S3` trait (at the revision pinned in Cargo.toml) against the methods + RustFS overrides in `impl S3 for FS`: + + ```bash + python3 scripts/s3-tests/api_coverage.py # summary + missing ops + python3 scripts/s3-tests/api_coverage.py --output api-coverage.md + ``` + +- `compare_dual_targets.py` — sends one signed S3 request to two endpoints + (e.g. RustFS and MinIO) and diffs status, headers, and body; useful when + triaging a behavioral difference found by the suites above. + ## See Also - [GitHub Actions Workflow](../../.github/workflows/e2e-s3tests.yml) diff --git a/scripts/s3-tests/api_coverage.py b/scripts/s3-tests/api_coverage.py new file mode 100755 index 000000000..13d0098f3 --- /dev/null +++ b/scripts/s3-tests/api_coverage.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Quantify RustFS S3 API surface coverage. + +Compares the operations defined by the s3s `S3` trait (at the revision pinned +in the workspace Cargo.toml) against the methods RustFS overrides in +`impl S3 for FS` (rustfs/src/storage/ecfs.rs). Trait methods that are not +overridden fall back to the s3s default implementation, which returns +NotImplemented — so "overridden" is a faithful proxy for "implemented". + +The s3s trait source is located automatically from the local cargo git +checkout cache; if absent (e.g. a fresh CI runner), it is fetched from the +pinned revision on GitHub. +""" + +from __future__ import annotations + +import argparse +import pathlib +import re +import subprocess +import sys +import tempfile + +ECFS_PATH = "rustfs/src/storage/ecfs.rs" +TRAIT_RELPATH = "crates/s3s/src/s3_trait.rs" + + +def project_root() -> pathlib.Path: + return pathlib.Path(__file__).resolve().parents[2] + + +def parse_s3s_pin(cargo_toml: pathlib.Path) -> tuple[str, str]: + """Return (git url, rev) of the s3s dependency from the workspace Cargo.toml.""" + for line in cargo_toml.read_text(encoding="utf-8").splitlines(): + if re.match(r"^s3s\s*=", line): + git = re.search(r'git\s*=\s*"([^"]+)"', line) + rev = re.search(r'rev\s*=\s*"([^"]+)"', line) + if git and rev: + return git.group(1), rev.group(1) + raise RuntimeError(f"s3s git dependency with rev not found in {cargo_toml}") + + +def locate_trait_source(git_url: str, rev: str) -> pathlib.Path: + """Find s3_trait.rs in the cargo git checkout cache, or fetch the pinned rev.""" + cargo_home = pathlib.Path.home() / ".cargo" + for candidate in cargo_home.glob(f"git/checkouts/s3s-*/{rev[:7]}/{TRAIT_RELPATH}"): + return candidate + + workdir = pathlib.Path(tempfile.mkdtemp(prefix="s3s-trait-")) + subprocess.run(["git", "init", "-q", str(workdir)], check=True) + subprocess.run(["git", "-C", str(workdir), "remote", "add", "origin", git_url], check=True) + subprocess.run(["git", "-C", str(workdir), "fetch", "-q", "--depth", "1", "origin", rev], check=True) + subprocess.run(["git", "-C", str(workdir), "checkout", "-qf", "--detach", rev], check=True) + return workdir / TRAIT_RELPATH + + +def extract_block(src: str, start: int) -> str: + """Return the brace-balanced block starting at the first '{' at/after start.""" + depth = 0 + begin = src.index("{", start) + for i in range(begin, len(src)): + if src[i] == "{": + depth += 1 + elif src[i] == "}": + depth -= 1 + if depth == 0: + return src[begin : i + 1] + raise RuntimeError("unbalanced braces") + + +def trait_operations(trait_source: pathlib.Path) -> dict[str, str]: + """Return {operation: default kind} for every trait method. + + Kind is "not_implemented" when the default body errors with NotImplemented, + or "delegated" when it forwards to another operation (e.g. post_object + delegates to put_object), in which case overriding is optional. + """ + src = trait_source.read_text(encoding="utf-8") + ops: dict[str, str] = {} + for match in re.finditer(r"async fn (\w+)", src): + body = extract_block(src, match.end()) + ops[match.group(1)] = "not_implemented" if "NotImplemented" in body else "delegated" + return ops + + +def implemented_operations(ecfs_source: pathlib.Path) -> set[str]: + src = ecfs_source.read_text(encoding="utf-8") + marker = "impl S3 for FS {" + start = src.find(marker) + if start < 0: + raise RuntimeError(f"`{marker}` not found in {ecfs_source}") + block = extract_block(src, start) + return set(re.findall(r"async fn (\w+)", block)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--s3s-trait", + type=pathlib.Path, + help=f"path to s3s {TRAIT_RELPATH} (default: auto-locate from Cargo.toml pin)", + ) + parser.add_argument("--output", type=pathlib.Path, help="write the markdown report to this path") + args = parser.parse_args() + + root = project_root() + git_url, rev = parse_s3s_pin(root / "Cargo.toml") + trait_source = args.s3s_trait or locate_trait_source(git_url, rev) + + all_ops = trait_operations(trait_source) + impl_ops = implemented_operations(root / ECFS_PATH) + + stale = sorted(impl_ops - all_ops.keys()) + implemented = sorted(op for op in all_ops if op in impl_ops) + delegated = sorted(op for op, kind in all_ops.items() if op not in impl_ops and kind == "delegated") + missing = sorted(op for op, kind in all_ops.items() if op not in impl_ops and kind == "not_implemented") + covered = len(implemented) + len(delegated) + pct = 100.0 * covered / len(all_ops) if all_ops else 0.0 + + lines = [ + "# S3 API surface coverage", + "", + f"s3s trait revision: `{rev}` — {len(all_ops)} operations.", + "", + f"**{covered}/{len(all_ops)} operations covered ({pct:.0f}%)** — " + f"{len(implemented)} overridden by RustFS, {len(delegated)} served by s3s default delegation.", + "", + "Operations below fall back to s3s defaults that return NotImplemented.", + "", + f"## Missing operations ({len(missing)})", + "", + ] + lines += [f"- `{op}`" for op in missing] or ["_none_"] + lines.append("") + if delegated: + lines += [f"## Covered via s3s default delegation ({len(delegated)})", ""] + lines += [f"- `{op}`" for op in delegated] + lines.append("") + lines += [f"## Implemented operations ({len(implemented)})", ""] + lines += [f"- `{op}`" for op in implemented] + lines.append("") + if stale: + lines += [f"## Overridden but absent from the trait ({len(stale)})", ""] + lines += [f"- `{op}`" for op in stale] + lines += ["", "_These suggest the s3s pin moved; re-run after `cargo update`._", ""] + + report = "\n".join(lines) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(report, encoding="utf-8") + print(f"[INFO] Report written to {args.output}") + + print( + f"[INFO] {covered}/{len(all_ops)} S3 operations covered ({pct:.0f}%): " + f"{len(implemented)} overridden, {len(delegated)} delegated, {len(missing)} missing" + ) + for op in missing: + print(f"[MISSING] {op}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/s3-tests/report_compat.py b/scripts/s3-tests/report_compat.py old mode 100644 new mode 100755