mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
Zhengchao An
4310b55238
Scheduled workflow failures previously went unnoticed (15 consecutive red weekly s3-tests sweeps, silent perf nightly failures). Add a reusable composite action that opens a tracking issue titled "[scheduled-failure] <workflow name>" — or appends a comment to the existing open one (dedupe key = workflow name) — with the run URL, run attempt, and failed job names, labeled "infrastructure". Wire it into the scheduled paths of e2e-s3tests, mint, fuzz (nightly) and performance-ab via a final alert-on-failure job gated by "always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')", with issues:write scoped to that job only. e2e-s3tests and mint previously had no permissions key; they now get workflow-level contents:read, reducing every other job's token. Add a dispatch-only drill workflow that forces a job failure and runs the action through the exact consumer wiring, for end-to-end verification. The ci-7 e2e nightly does not exist yet; it is recorded as a pending consumer in the action README. Refs: rustfs/backlog#1149 (ci-8), rustfs/backlog#1155
352 lines
13 KiB
YAML
352 lines
13 KiB
YAML
# Copyright 2024 RustFS Team
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Full ceph/s3-tests sweep against RustFS.
|
|
#
|
|
# The per-PR compatibility gate lives in ci.yml (job s3-implemented-tests) and
|
|
# runs only the implemented_tests.txt whitelist. This workflow complements it:
|
|
#
|
|
# - Scheduled weekly full sweep (TEST_SCOPE=all): runs the ENTIRE upstream
|
|
# suite and reports promotion candidates (tests that newly pass) and
|
|
# unclassified tests. The job fails only on regressions in the implemented
|
|
# whitelist or on infrastructure errors — expected failures from
|
|
# not-yet-implemented features do not turn the run red.
|
|
# - Manual runs (workflow_dispatch): same, with configurable mode/scope.
|
|
#
|
|
# All test execution is delegated to scripts/s3-tests/run.sh (single source of
|
|
# truth, also used locally and by the PR gate). This workflow only provisions
|
|
# the server topology: a single node, or a real 4-node distributed cluster
|
|
# (erasure-coded across nodes) behind an HAProxy round-robin load balancer.
|
|
#
|
|
# Runner infrastructure (ci-1, backlog#1149): this sweep needs BOTH a working
|
|
# Docker daemon (to build the source image and run the single/multi topologies)
|
|
# AND a Python toolchain with pip (for awscurl/tox and the pytest harness).
|
|
# It previously ran on the self-hosted `sm-standard-4` label, which is served
|
|
# by heterogeneous runners: some scheduled runs landed on minimal pods that
|
|
# had neither `pip` (bare python3, `No module named pip`) nor `docker.sock`,
|
|
# so "Install Python tools" died before any test executed (all 15 recorded
|
|
# runs failed). Two changes make the infra assumptions explicit instead of
|
|
# trusting drifting runner state:
|
|
# 1. Pin to GitHub-hosted `ubuntu-latest`, which reliably ships Docker,
|
|
# docker compose, python3 and pip (no self-hosted fleet drift).
|
|
# 2. Set up Python explicitly (actions/setup-python) before any pip usage,
|
|
# so the workflow stays correct even if the runner image changes.
|
|
# The PR gate (ci.yml s3-implemented-tests) is unaffected: it avoids Docker
|
|
# via DEPLOY_MODE=binary and defers all pip setup to run.sh's self-bootstrap.
|
|
|
|
name: e2e-s3tests
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
test-mode:
|
|
description: "Server topology"
|
|
required: true
|
|
type: choice
|
|
default: "single"
|
|
options:
|
|
- single
|
|
- multi
|
|
scope:
|
|
description: "Test scope: full upstream suite or implemented whitelist"
|
|
required: true
|
|
type: choice
|
|
default: "all"
|
|
options:
|
|
- all
|
|
- implemented
|
|
xdist:
|
|
description: "pytest-xdist workers. '0' to disable."
|
|
required: false
|
|
default: "4"
|
|
maxfail:
|
|
description: "Stop after N failures. '0' to run everything."
|
|
required: false
|
|
default: "0"
|
|
markexpr:
|
|
description: "Optional pytest -m expression"
|
|
required: false
|
|
default: ""
|
|
schedule:
|
|
# Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the
|
|
# single-node and the 4-node distributed topologies (matrix below).
|
|
- cron: "0 2 * * 0"
|
|
|
|
env:
|
|
# main user
|
|
S3_ACCESS_KEY: rustfsadmin
|
|
S3_SECRET_KEY: rustfsadmin
|
|
# alt user (must be different from main for many s3-tests)
|
|
S3_ALT_ACCESS_KEY: rustfsalt
|
|
S3_ALT_SECRET_KEY: rustfsalt
|
|
|
|
S3_REGION: us-east-1
|
|
S3_HOST: 127.0.0.1
|
|
S3_PORT: "9000"
|
|
|
|
RUST_LOG: info
|
|
PLATFORM: linux/amd64
|
|
BUILDX_CACHE_SCOPE: rustfs-e2e-s3tests-source
|
|
|
|
# Scheduled runs have no inputs; default to a full sweep.
|
|
TEST_SCOPE: ${{ github.event.inputs.scope || 'all' }}
|
|
XDIST: ${{ github.event.inputs.xdist || '4' }}
|
|
MAXFAIL: ${{ github.event.inputs.maxfail || '0' }}
|
|
MARKEXPR: ${{ github.event.inputs.markexpr || '' }}
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs['test-mode'] || 'single' }}
|
|
cancel-in-progress: true
|
|
|
|
# Only alert-on-failure needs more than read access; it declares its own
|
|
# job-level `issues: write`.
|
|
permissions:
|
|
contents: read
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
jobs:
|
|
s3tests:
|
|
# GitHub-hosted: reliably provides Docker + docker compose + python3/pip.
|
|
# See the header note (ci-1) for why the self-hosted sm-standard-4 label
|
|
# was abandoned. TODO(ci-8): scheduled-failure alerting (auto-open issue)
|
|
# is added by the ci-8 composite action; do not implement it here.
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 180
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
# Scheduled sweeps cover both topologies; manual runs use the input.
|
|
test-mode: ${{ github.event_name == 'schedule' && fromJSON('["single", "multi"]') || fromJSON(format('["{0}"]', github.event.inputs.test-mode || 'single')) }}
|
|
env:
|
|
TEST_MODE: ${{ matrix.test-mode }}
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
# Provision Python explicitly rather than trusting the runner image to
|
|
# ship a working pip (ci-1: a bare python3 without pip is what broke the
|
|
# scheduled sweep). Keeps the workflow correct across runner drift.
|
|
- name: Set up Python
|
|
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- name: Cache pip downloads
|
|
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
|
with:
|
|
path: ~/.cache/pip
|
|
key: ${{ runner.os }}-pip-e2e-s3tests-${{ hashFiles('.github/workflows/e2e-s3tests.yml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-pip-e2e-s3tests-
|
|
|
|
- name: Install Python tools
|
|
run: |
|
|
python3 -m pip install --user --upgrade pip awscurl tox
|
|
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Enable buildx
|
|
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
|
|
|
- name: Build RustFS image (source, cached)
|
|
run: |
|
|
DOCKER_BUILDKIT=1 docker buildx build --load \
|
|
--platform "${PLATFORM}" \
|
|
--cache-from "type=gha,scope=${BUILDX_CACHE_SCOPE}" \
|
|
--cache-to "type=gha,mode=max,scope=${BUILDX_CACHE_SCOPE}" \
|
|
-t rustfs-ci \
|
|
-f Dockerfile.source .
|
|
|
|
- name: Start single RustFS
|
|
if: env.TEST_MODE == 'single'
|
|
run: |
|
|
docker network inspect rustfs-net >/dev/null 2>&1 || docker network create rustfs-net
|
|
docker rm -f rustfs-single >/dev/null 2>&1 || true
|
|
docker run -d --name rustfs-single \
|
|
--network rustfs-net \
|
|
-p "${S3_PORT}:9000" \
|
|
-e RUSTFS_ADDRESS=0.0.0.0:9000 \
|
|
-e RUSTFS_ACCESS_KEY="${S3_ACCESS_KEY}" \
|
|
-e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
|
|
-e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
|
|
-v /tmp/rustfs-single:/data \
|
|
rustfs-ci
|
|
|
|
- name: Start 4-node distributed cluster
|
|
if: env.TEST_MODE == 'multi'
|
|
run: |
|
|
# A real distributed deployment: every node lists all endpoints in
|
|
# RUSTFS_VOLUMES so data is erasure-coded ACROSS nodes. Do not use
|
|
# node-local volume paths here — that would create four independent
|
|
# single-node stores behind the load balancer.
|
|
cat > compose.yml <<EOF
|
|
x-rustfs-node: &rustfs-node
|
|
image: rustfs-ci
|
|
networks: [rustfs-net]
|
|
environment:
|
|
RUSTFS_ADDRESS: "0.0.0.0:9000"
|
|
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
|
|
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
|
|
RUSTFS_VOLUMES: "http://rustfs{1...4}:9000/data/rustfs{0...3}"
|
|
|
|
services:
|
|
rustfs1:
|
|
<<: *rustfs-node
|
|
hostname: rustfs1
|
|
volumes: [rustfs1-data:/data]
|
|
rustfs2:
|
|
<<: *rustfs-node
|
|
hostname: rustfs2
|
|
volumes: [rustfs2-data:/data]
|
|
rustfs3:
|
|
<<: *rustfs-node
|
|
hostname: rustfs3
|
|
volumes: [rustfs3-data:/data]
|
|
rustfs4:
|
|
<<: *rustfs-node
|
|
hostname: rustfs4
|
|
volumes: [rustfs4-data:/data]
|
|
lb:
|
|
image: haproxy:2.9
|
|
hostname: lb
|
|
networks: [rustfs-net]
|
|
ports:
|
|
- "${S3_PORT}:9000"
|
|
volumes:
|
|
- ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
|
networks:
|
|
rustfs-net:
|
|
name: rustfs-net
|
|
volumes:
|
|
rustfs1-data:
|
|
rustfs2-data:
|
|
rustfs3-data:
|
|
rustfs4-data:
|
|
EOF
|
|
|
|
cat > haproxy.cfg <<'EOF'
|
|
defaults
|
|
mode http
|
|
timeout connect 5s
|
|
timeout client 30s
|
|
timeout server 30s
|
|
|
|
frontend fe_s3
|
|
bind *:9000
|
|
default_backend be_s3
|
|
|
|
backend be_s3
|
|
balance roundrobin
|
|
server s1 rustfs1:9000 check
|
|
server s2 rustfs2:9000 check
|
|
server s3 rustfs3:9000 check
|
|
server s4 rustfs4:9000 check
|
|
EOF
|
|
|
|
docker compose -f compose.yml up -d
|
|
|
|
- name: Wait for RustFS ready
|
|
run: |
|
|
for _ in {1..120}; do
|
|
if curl -sf "http://${S3_HOST}:${S3_PORT}/health" >/dev/null 2>&1; then
|
|
echo "RustFS is ready"
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "Health check timed out" >&2
|
|
if [ "${TEST_MODE}" = "single" ]; then
|
|
docker logs rustfs-single || true
|
|
else
|
|
docker compose -f compose.yml logs --tail=200 || true
|
|
fi
|
|
exit 1
|
|
|
|
- name: Run ceph s3-tests
|
|
run: |
|
|
set +e
|
|
DEPLOY_MODE=existing \
|
|
TEST_MODE="${TEST_MODE}" \
|
|
TEST_SCOPE="${TEST_SCOPE}" \
|
|
XDIST="${XDIST}" \
|
|
MAXFAIL="${MAXFAIL}" \
|
|
MARKEXPR="${MARKEXPR}" \
|
|
./scripts/s3-tests/run.sh
|
|
RC=$?
|
|
set -e
|
|
|
|
if [ "${TEST_SCOPE}" = "implemented" ]; then
|
|
# Whitelist run: every failure is a regression.
|
|
exit "${RC}"
|
|
fi
|
|
|
|
# Full sweep: failures outside the implemented whitelist are
|
|
# inventory (promotion candidates / unimplemented features), not a
|
|
# gate. Fail only on whitelist regressions or infrastructure errors.
|
|
JUNIT="artifacts/s3tests-${TEST_MODE}/junit.xml"
|
|
if [ ! -f "${JUNIT}" ]; then
|
|
echo "No junit.xml produced — infrastructure failure (exit ${RC})" >&2
|
|
exit "${RC}"
|
|
fi
|
|
python3 scripts/s3-tests/report_compat.py \
|
|
--junit "${JUNIT}" \
|
|
--lists-dir scripts/s3-tests \
|
|
--fail-on-regression
|
|
|
|
- name: Publish compatibility report
|
|
if: always()
|
|
run: |
|
|
REPORT="artifacts/s3tests-${TEST_MODE}/compat-report.md"
|
|
if [ -f "${REPORT}" ]; then
|
|
cat "${REPORT}" >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
|
|
- name: Collect RustFS logs
|
|
if: always()
|
|
run: |
|
|
mkdir -p "artifacts/rustfs-${TEST_MODE}"
|
|
if [ "${TEST_MODE}" = "single" ]; then
|
|
docker logs rustfs-single > "artifacts/rustfs-${TEST_MODE}/rustfs.log" 2>&1 || true
|
|
docker inspect rustfs-single > "artifacts/rustfs-${TEST_MODE}/inspect.json" || true
|
|
else
|
|
docker compose -f compose.yml logs --no-color > "artifacts/rustfs-${TEST_MODE}/cluster.log" 2>&1 || true
|
|
fi
|
|
|
|
- name: Upload artifacts
|
|
if: always() && env.ACT != 'true'
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: s3tests-${{ env.TEST_MODE }}
|
|
path: artifacts/**
|
|
|
|
alert-on-failure:
|
|
name: Alert on scheduled failure
|
|
needs: [s3tests]
|
|
# `always()` is required: without it this job is skipped when a needed
|
|
# job fails. Alerts only for scheduled runs (backlog#1149 ci-8) — manual
|
|
# dispatch failures are already watched by a human.
|
|
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
issues: write
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
- name: Open or update failure-tracking issue
|
|
uses: ./.github/actions/schedule-failure-issue
|
|
with:
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|