rustfs/.github/workflows/e2e-s3tests.yml

# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Full ceph/s3-tests sweep against RustFS.
#
# The per-PR compatibility gate lives in ci.yml (job s3-implemented-tests) and
# runs only the implemented_tests.txt whitelist. This workflow complements it:
#
#   - Scheduled weekly full sweep (TEST_SCOPE=all): runs the ENTIRE upstream
#     suite and reports promotion candidates (tests that newly pass) and
#     unclassified tests. The job fails only on regressions in the implemented
#     whitelist or on infrastructure errors — expected failures from
#     not-yet-implemented features do not turn the run red.
#   - Manual runs (workflow_dispatch): same, with configurable mode/scope.
#
# All test execution is delegated to scripts/s3-tests/run.sh (single source of
# truth, also used locally and by the PR gate). This workflow only provisions
# the server topology: a single node, or a real 4-node distributed cluster
# (erasure-coded across nodes) behind an HAProxy round-robin load balancer.
#
# Runner infrastructure (ci-1, backlog#1149): this sweep needs BOTH a working
# Docker daemon (to build the source image and run the single/multi topologies)
# AND a Python toolchain with pip (for awscurl/tox and the pytest harness).
# It previously ran on the self-hosted `sm-standard-4` label, which is served
# by heterogeneous runners: some scheduled runs landed on minimal pods that
# had neither `pip` (bare python3, `No module named pip`) nor `docker.sock`,
# so "Install Python tools" died before any test executed (all 15 recorded
# runs failed). Two changes make the infra assumptions explicit instead of
# trusting drifting runner state:
#   1. Pin to GitHub-hosted `ubuntu-latest`, which reliably ships Docker,
#      docker compose, python3 and pip (no self-hosted fleet drift).
#   2. Set up Python explicitly (actions/setup-python) before any pip usage,
#      so the workflow stays correct even if the runner image changes.
# The PR gate (ci.yml s3-implemented-tests) is unaffected: it avoids Docker
# via DEPLOY_MODE=binary and defers all pip setup to run.sh's self-bootstrap.

name: e2e-s3tests

on:
  workflow_dispatch:
    inputs:
      test-mode:
        description: "Server topology"
        required: true
        type: choice
        default: "single"
        options:
          - single
          - multi
      scope:
        description: "Test scope: full upstream suite or implemented whitelist"
        required: true
        type: choice
        default: "all"
        options:
          - all
          - implemented
      xdist:
        description: "pytest-xdist workers. '0' to disable."
        required: false
        default: "4"
      maxfail:
        description: "Stop after N failures. '0' to run everything."
        required: false
        default: "0"
      markexpr:
        description: "Optional pytest -m expression"
        required: false
        default: ""
  schedule:
    # Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the
    # single-node and the 4-node distributed topologies (matrix below).
    - cron: "0 2 * * 0"

env:
  # main user
  S3_ACCESS_KEY: rustfsadmin-ci
  S3_SECRET_KEY: rustfssecret-ci
  # alt user (must be different from main for many s3-tests)
  S3_ALT_ACCESS_KEY: rustfsalt
  S3_ALT_SECRET_KEY: rustfsalt

  S3_REGION: us-east-1
  S3_HOST: 127.0.0.1
  S3_PORT: "9000"

  RUST_LOG: info
  PLATFORM: linux/amd64
  BUILDX_CACHE_SCOPE: rustfs-e2e-s3tests-source

  # Scheduled runs have no inputs; default to a full sweep.
  TEST_SCOPE: ${{ github.event.inputs.scope || 'all' }}
  XDIST: ${{ github.event.inputs.xdist || '4' }}
  MAXFAIL: ${{ github.event.inputs.maxfail || '0' }}
  MARKEXPR: ${{ github.event.inputs.markexpr || '' }}

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs['test-mode'] || 'single' }}
  cancel-in-progress: true

# Only alert-on-failure needs more than read access; it declares its own
# job-level `issues: write`.
permissions:
  contents: read

defaults:
  run:
    shell: bash

jobs:
  s3tests:
    # GitHub-hosted: reliably provides Docker + docker compose + python3/pip.
    # See the header note (ci-1) for why the self-hosted sm-standard-4 label
    # was abandoned. TODO(ci-8): scheduled-failure alerting (auto-open issue)
    # is added by the ci-8 composite action; do not implement it here.
    runs-on: ubuntu-latest
    timeout-minutes: 180
    strategy:
      fail-fast: false
      matrix:
        # Scheduled sweeps cover both topologies; manual runs use the input.
        test-mode: ${{ github.event_name == 'schedule' && fromJSON('["single", "multi"]') || fromJSON(format('["{0}"]', github.event.inputs.test-mode || 'single')) }}
    env:
      TEST_MODE: ${{ matrix.test-mode }}
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7

      # Provision Python explicitly rather than trusting the runner image to
      # ship a working pip (ci-1: a bare python3 without pip is what broke the
      # scheduled sweep). Keeps the workflow correct across runner drift.
      - name: Set up Python
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
        with:
          python-version: "3.12"

      - name: Cache pip downloads
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-e2e-s3tests-${{ hashFiles('.github/workflows/e2e-s3tests.yml') }}
          restore-keys: |
            ${{ runner.os }}-pip-e2e-s3tests-

      - name: Install Python tools
        run: |
          python3 -m pip install --user --upgrade pip awscurl tox
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"

      - name: Enable buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3

      - name: Build RustFS image (source, cached)
        run: |
          DOCKER_BUILDKIT=1 docker buildx build --load \
            --platform "${PLATFORM}" \
            --cache-from "type=gha,scope=${BUILDX_CACHE_SCOPE}" \
            --cache-to "type=gha,mode=max,scope=${BUILDX_CACHE_SCOPE}" \
            -t rustfs-ci \
            -f Dockerfile.source .

      - name: Start single RustFS
        if: env.TEST_MODE == 'single'
        run: |
          docker network inspect rustfs-net >/dev/null 2>&1 || docker network create rustfs-net
          docker rm -f rustfs-single >/dev/null 2>&1 || true
          # The four disks share one physical device on the runner (a single
          # loopback filesystem), so the local physical-disk-independence guard
          # would refuse to start. Bypass it — this is the CI use case the guard
          # explicitly sanctions via RUSTFS_UNSAFE_BYPASS_DISK_CHECK.
          docker run -d --name rustfs-single \
            --network rustfs-net \
            -p "${S3_PORT}:9000" \
            -e RUSTFS_ADDRESS=0.0.0.0:9000 \
            -e RUSTFS_ACCESS_KEY="${S3_ACCESS_KEY}" \
            -e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
            -e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
            -e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
            -v /tmp/rustfs-single:/data \
            rustfs-ci

      - name: Start 4-node distributed cluster
        if: env.TEST_MODE == 'multi'
        run: |
          # A real distributed deployment: every node lists all endpoints in
          # RUSTFS_VOLUMES so data is erasure-coded ACROSS nodes. Do not use
          # node-local volume paths here — that would create four independent
          # single-node stores behind the load balancer.
          cat > compose.yml <<EOF
          x-rustfs-node: &rustfs-node
            image: rustfs-ci
            networks: [rustfs-net]
            environment:
              RUSTFS_ADDRESS: "0.0.0.0:9000"
              RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
              RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
              RUSTFS_VOLUMES: "http://rustfs{1...4}:9000/data/rustfs{0...3}"
              # Each node's four disks share one physical device inside its
              # container, so bypass the local physical-disk-independence guard
              # (the CI use case it explicitly sanctions).
              RUSTFS_UNSAFE_BYPASS_DISK_CHECK: "true"

          services:
            rustfs1:
              <<: *rustfs-node
              hostname: rustfs1
              volumes: [rustfs1-data:/data]
            rustfs2:
              <<: *rustfs-node
              hostname: rustfs2
              volumes: [rustfs2-data:/data]
            rustfs3:
              <<: *rustfs-node
              hostname: rustfs3
              volumes: [rustfs3-data:/data]
            rustfs4:
              <<: *rustfs-node
              hostname: rustfs4
              volumes: [rustfs4-data:/data]
            lb:
              image: haproxy:2.9
              hostname: lb
              networks: [rustfs-net]
              ports:
                - "${S3_PORT}:9000"
              volumes:
                - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
          networks:
            rustfs-net:
              name: rustfs-net
          volumes:
            rustfs1-data:
            rustfs2-data:
            rustfs3-data:
            rustfs4-data:
          EOF

          cat > haproxy.cfg <<'EOF'
          defaults
            mode http
            timeout connect 5s
            timeout client  30s
            timeout server  30s

          frontend fe_s3
            bind *:9000
            default_backend be_s3

          backend be_s3
            balance roundrobin
            server s1 rustfs1:9000 check
            server s2 rustfs2:9000 check
            server s3 rustfs3:9000 check
            server s4 rustfs4:9000 check
          EOF

          docker compose -f compose.yml up -d

      - name: Wait for RustFS ready
        run: |
          for _ in {1..120}; do
            if curl -sf "http://${S3_HOST}:${S3_PORT}/health" >/dev/null 2>&1; then
              echo "RustFS is ready"
              exit 0
            fi
            sleep 2
          done
          echo "Health check timed out" >&2
          if [ "${TEST_MODE}" = "single" ]; then
            docker logs rustfs-single || true
          else
            docker compose -f compose.yml logs --tail=200 || true
          fi
          exit 1

      - name: Run ceph s3-tests
        run: |
          set +e
          DEPLOY_MODE=existing \
          TEST_MODE="${TEST_MODE}" \
          TEST_SCOPE="${TEST_SCOPE}" \
          XDIST="${XDIST}" \
          MAXFAIL="${MAXFAIL}" \
          MARKEXPR="${MARKEXPR}" \
          ./scripts/s3-tests/run.sh
          RC=$?
          set -e

          if [ "${TEST_SCOPE}" = "implemented" ]; then
            # Whitelist run: every failure is a regression.
            exit "${RC}"
          fi

          # Full sweep: failures outside the implemented whitelist are
          # inventory (promotion candidates / unimplemented features), not a
          # gate. Fail only on whitelist regressions or infrastructure errors.
          JUNIT="artifacts/s3tests-${TEST_MODE}/junit.xml"
          if [ ! -f "${JUNIT}" ]; then
            echo "No junit.xml produced — infrastructure failure (exit ${RC})" >&2
            exit "${RC}"
          fi
          python3 scripts/s3-tests/report_compat.py \
            --junit "${JUNIT}" \
            --lists-dir scripts/s3-tests \
            --fail-on-regression

      - name: Publish compatibility report
        if: always()
        run: |
          REPORT="artifacts/s3tests-${TEST_MODE}/compat-report.md"
          if [ -f "${REPORT}" ]; then
            cat "${REPORT}" >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Collect RustFS logs
        if: always()
        run: |
          mkdir -p "artifacts/rustfs-${TEST_MODE}"
          if [ "${TEST_MODE}" = "single" ]; then
            docker logs rustfs-single > "artifacts/rustfs-${TEST_MODE}/rustfs.log" 2>&1 || true
            docker inspect rustfs-single > "artifacts/rustfs-${TEST_MODE}/inspect.json" || true
          else
            docker compose -f compose.yml logs --no-color > "artifacts/rustfs-${TEST_MODE}/cluster.log" 2>&1 || true
          fi

      - name: Upload artifacts
        if: always() && env.ACT != 'true'
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
          name: s3tests-${{ env.TEST_MODE }}
          path: artifacts/**

  alert-on-failure:
    name: Alert on scheduled failure
    needs: [s3tests]
    # `always()` is required: without it this job is skipped when a needed
    # job fails. Alerts only for scheduled runs (backlog#1149 ci-8) — manual
    # dispatch failures are already watched by a human.
    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
      - name: Open or update failure-tracking issue
        uses: ./.github/actions/schedule-failure-issue
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}