mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-28 00:58:59 +00:00
Zhengchao An
3779e674a8
Follow-ups to the compatibility harness rework: - Weekly full sweep now runs as a matrix over both topologies: single node and the 4-node distributed cluster. Manual dispatch keeps the test-mode input. - New mint workflow (weekly + dispatch) runs MinIO Mint against RustFS: functional suites of real client SDKs and tools (awscli, mc, aws-sdk-*, minio-*, s3cmd, ...), catching client-specific signing and streaming edge cases that boto3-only ceph/s3-tests cannot. Report-only for test failures with a per-suite summary table; fails only when mint produces no results. - New scripts/s3-tests/api_coverage.py quantifies API surface coverage by diffing the s3s S3 trait (at the Cargo.toml pinned revision) against the methods overridden in impl S3 for FS, distinguishing NotImplemented defaults from delegating defaults (e.g. post_object). Current state: 76/101 operations covered (75 overridden, 1 delegated). Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
300 lines
10 KiB
YAML
300 lines
10 KiB
YAML
# Copyright 2024 RustFS Team
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Full ceph/s3-tests sweep against RustFS.
|
|
#
|
|
# The per-PR compatibility gate lives in ci.yml (job s3-implemented-tests) and
|
|
# runs only the implemented_tests.txt whitelist. This workflow complements it:
|
|
#
|
|
# - Scheduled weekly full sweep (TEST_SCOPE=all): runs the ENTIRE upstream
|
|
# suite and reports promotion candidates (tests that newly pass) and
|
|
# unclassified tests. The job fails only on regressions in the implemented
|
|
# whitelist or on infrastructure errors — expected failures from
|
|
# not-yet-implemented features do not turn the run red.
|
|
# - Manual runs (workflow_dispatch): same, with configurable mode/scope.
|
|
#
|
|
# All test execution is delegated to scripts/s3-tests/run.sh (single source of
|
|
# truth, also used locally and by the PR gate). This workflow only provisions
|
|
# the server topology: a single node, or a real 4-node distributed cluster
|
|
# (erasure-coded across nodes) behind an HAProxy round-robin load balancer.
|
|
|
|
name: e2e-s3tests
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
test-mode:
|
|
description: "Server topology"
|
|
required: true
|
|
type: choice
|
|
default: "single"
|
|
options:
|
|
- single
|
|
- multi
|
|
scope:
|
|
description: "Test scope: full upstream suite or implemented whitelist"
|
|
required: true
|
|
type: choice
|
|
default: "all"
|
|
options:
|
|
- all
|
|
- implemented
|
|
xdist:
|
|
description: "pytest-xdist workers. '0' to disable."
|
|
required: false
|
|
default: "4"
|
|
maxfail:
|
|
description: "Stop after N failures. '0' to run everything."
|
|
required: false
|
|
default: "0"
|
|
markexpr:
|
|
description: "Optional pytest -m expression"
|
|
required: false
|
|
default: ""
|
|
schedule:
|
|
# Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the
|
|
# single-node and the 4-node distributed topologies (matrix below).
|
|
- cron: "0 2 * * 0"
|
|
|
|
env:
|
|
# main user
|
|
S3_ACCESS_KEY: rustfsadmin
|
|
S3_SECRET_KEY: rustfsadmin
|
|
# alt user (must be different from main for many s3-tests)
|
|
S3_ALT_ACCESS_KEY: rustfsalt
|
|
S3_ALT_SECRET_KEY: rustfsalt
|
|
|
|
S3_REGION: us-east-1
|
|
S3_HOST: 127.0.0.1
|
|
S3_PORT: "9000"
|
|
|
|
RUST_LOG: info
|
|
PLATFORM: linux/amd64
|
|
BUILDX_CACHE_SCOPE: rustfs-e2e-s3tests-source
|
|
|
|
# Scheduled runs have no inputs; default to a full sweep.
|
|
TEST_SCOPE: ${{ github.event.inputs.scope || 'all' }}
|
|
XDIST: ${{ github.event.inputs.xdist || '4' }}
|
|
MAXFAIL: ${{ github.event.inputs.maxfail || '0' }}
|
|
MARKEXPR: ${{ github.event.inputs.markexpr || '' }}
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs['test-mode'] || 'single' }}
|
|
cancel-in-progress: true
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
jobs:
|
|
s3tests:
|
|
runs-on: ubicloud-standard-4
|
|
timeout-minutes: 180
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
# Scheduled sweeps cover both topologies; manual runs use the input.
|
|
test-mode: ${{ github.event_name == 'schedule' && fromJSON('["single", "multi"]') || fromJSON(format('["{0}"]', github.event.inputs.test-mode || 'single')) }}
|
|
env:
|
|
TEST_MODE: ${{ matrix.test-mode }}
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Cache pip downloads
|
|
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
|
with:
|
|
path: ~/.cache/pip
|
|
key: ${{ runner.os }}-pip-e2e-s3tests-${{ hashFiles('.github/workflows/e2e-s3tests.yml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-pip-e2e-s3tests-
|
|
|
|
- name: Install Python tools
|
|
run: |
|
|
python3 -m pip install --user --upgrade pip awscurl tox
|
|
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Enable buildx
|
|
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
|
|
|
- name: Build RustFS image (source, cached)
|
|
run: |
|
|
DOCKER_BUILDKIT=1 docker buildx build --load \
|
|
--platform ${PLATFORM} \
|
|
--cache-from type=gha,scope=${BUILDX_CACHE_SCOPE} \
|
|
--cache-to type=gha,mode=max,scope=${BUILDX_CACHE_SCOPE} \
|
|
-t rustfs-ci \
|
|
-f Dockerfile.source .
|
|
|
|
- name: Start single RustFS
|
|
if: env.TEST_MODE == 'single'
|
|
run: |
|
|
docker network inspect rustfs-net >/dev/null 2>&1 || docker network create rustfs-net
|
|
docker rm -f rustfs-single >/dev/null 2>&1 || true
|
|
docker run -d --name rustfs-single \
|
|
--network rustfs-net \
|
|
-p ${S3_PORT}:9000 \
|
|
-e RUSTFS_ADDRESS=0.0.0.0:9000 \
|
|
-e RUSTFS_ACCESS_KEY=${S3_ACCESS_KEY} \
|
|
-e RUSTFS_SECRET_KEY=${S3_SECRET_KEY} \
|
|
-e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
|
|
-v /tmp/rustfs-single:/data \
|
|
rustfs-ci
|
|
|
|
- name: Start 4-node distributed cluster
|
|
if: env.TEST_MODE == 'multi'
|
|
run: |
|
|
# A real distributed deployment: every node lists all endpoints in
|
|
# RUSTFS_VOLUMES so data is erasure-coded ACROSS nodes. Do not use
|
|
# node-local volume paths here — that would create four independent
|
|
# single-node stores behind the load balancer.
|
|
cat > compose.yml <<EOF
|
|
x-rustfs-node: &rustfs-node
|
|
image: rustfs-ci
|
|
networks: [rustfs-net]
|
|
environment:
|
|
RUSTFS_ADDRESS: "0.0.0.0:9000"
|
|
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
|
|
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
|
|
RUSTFS_VOLUMES: "http://rustfs{1...4}:9000/data/rustfs{0...3}"
|
|
|
|
services:
|
|
rustfs1:
|
|
<<: *rustfs-node
|
|
hostname: rustfs1
|
|
volumes: [rustfs1-data:/data]
|
|
rustfs2:
|
|
<<: *rustfs-node
|
|
hostname: rustfs2
|
|
volumes: [rustfs2-data:/data]
|
|
rustfs3:
|
|
<<: *rustfs-node
|
|
hostname: rustfs3
|
|
volumes: [rustfs3-data:/data]
|
|
rustfs4:
|
|
<<: *rustfs-node
|
|
hostname: rustfs4
|
|
volumes: [rustfs4-data:/data]
|
|
lb:
|
|
image: haproxy:2.9
|
|
hostname: lb
|
|
networks: [rustfs-net]
|
|
ports:
|
|
- "${S3_PORT}:9000"
|
|
volumes:
|
|
- ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
|
networks:
|
|
rustfs-net:
|
|
name: rustfs-net
|
|
volumes:
|
|
rustfs1-data:
|
|
rustfs2-data:
|
|
rustfs3-data:
|
|
rustfs4-data:
|
|
EOF
|
|
|
|
cat > haproxy.cfg <<'EOF'
|
|
defaults
|
|
mode http
|
|
timeout connect 5s
|
|
timeout client 30s
|
|
timeout server 30s
|
|
|
|
frontend fe_s3
|
|
bind *:9000
|
|
default_backend be_s3
|
|
|
|
backend be_s3
|
|
balance roundrobin
|
|
server s1 rustfs1:9000 check
|
|
server s2 rustfs2:9000 check
|
|
server s3 rustfs3:9000 check
|
|
server s4 rustfs4:9000 check
|
|
EOF
|
|
|
|
docker compose -f compose.yml up -d
|
|
|
|
- name: Wait for RustFS ready
|
|
run: |
|
|
for i in {1..120}; do
|
|
if curl -sf "http://${S3_HOST}:${S3_PORT}/health" >/dev/null 2>&1; then
|
|
echo "RustFS is ready"
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "Health check timed out" >&2
|
|
if [ "${TEST_MODE}" = "single" ]; then
|
|
docker logs rustfs-single || true
|
|
else
|
|
docker compose -f compose.yml logs --tail=200 || true
|
|
fi
|
|
exit 1
|
|
|
|
- name: Run ceph s3-tests
|
|
run: |
|
|
set +e
|
|
DEPLOY_MODE=existing \
|
|
TEST_MODE="${TEST_MODE}" \
|
|
TEST_SCOPE="${TEST_SCOPE}" \
|
|
XDIST="${XDIST}" \
|
|
MAXFAIL="${MAXFAIL}" \
|
|
MARKEXPR="${MARKEXPR}" \
|
|
./scripts/s3-tests/run.sh
|
|
RC=$?
|
|
set -e
|
|
|
|
if [ "${TEST_SCOPE}" = "implemented" ]; then
|
|
# Whitelist run: every failure is a regression.
|
|
exit "${RC}"
|
|
fi
|
|
|
|
# Full sweep: failures outside the implemented whitelist are
|
|
# inventory (promotion candidates / unimplemented features), not a
|
|
# gate. Fail only on whitelist regressions or infrastructure errors.
|
|
JUNIT="artifacts/s3tests-${TEST_MODE}/junit.xml"
|
|
if [ ! -f "${JUNIT}" ]; then
|
|
echo "No junit.xml produced — infrastructure failure (exit ${RC})" >&2
|
|
exit "${RC}"
|
|
fi
|
|
python3 scripts/s3-tests/report_compat.py \
|
|
--junit "${JUNIT}" \
|
|
--lists-dir scripts/s3-tests \
|
|
--fail-on-regression
|
|
|
|
- name: Publish compatibility report
|
|
if: always()
|
|
run: |
|
|
REPORT="artifacts/s3tests-${TEST_MODE}/compat-report.md"
|
|
if [ -f "${REPORT}" ]; then
|
|
cat "${REPORT}" >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
|
|
- name: Collect RustFS logs
|
|
if: always()
|
|
run: |
|
|
mkdir -p "artifacts/rustfs-${TEST_MODE}"
|
|
if [ "${TEST_MODE}" = "single" ]; then
|
|
docker logs rustfs-single > "artifacts/rustfs-${TEST_MODE}/rustfs.log" 2>&1 || true
|
|
docker inspect rustfs-single > "artifacts/rustfs-${TEST_MODE}/inspect.json" || true
|
|
else
|
|
docker compose -f compose.yml logs --no-color > "artifacts/rustfs-${TEST_MODE}/cluster.log" 2>&1 || true
|
|
fi
|
|
|
|
- name: Upload artifacts
|
|
if: always() && env.ACT != 'true'
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: s3tests-${{ env.TEST_MODE }}
|
|
path: artifacts/**
|