rustfs/.github/workflows/e2e-replication-nightly.yml

# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Consolidated nightly e2e lane for replication, cluster faults, and protocols.
#
# The per-PR gate (ci.yml `e2e-tests` job, `--profile e2e-smoke`) runs the
# FAST replication tests. This scheduled lane runs the remaining heavier
# replication e2e tests that are unfit for a per-PR gate: remote-target TLS
# validation, bucket-replication data-plane/helper tests (PUT/delete + poll
# for convergence, HTTPS targets, active SSE failure contracts, event/history
# observers), and the `_real_dual_node` / `_real_three_node` /
# `_real_single_node` site-replication tests that each spawn full rustfs
# server processes.
#
# The selection is the [profile.e2e-repl-nightly] default-filter in
# .config/nextest.toml — the single wiring mechanism (repl-1 / ci-4). Do NOT
# add ad-hoc cargo-test steps here; change the filterset instead. The
# authoritative membership and count come from
# `cargo nextest list -p e2e_test --profile e2e-repl-nightly`; the PR/nightly
# selection digest is committed under .config/.
#
# Explicit division of labor: these subsets run only here and never double-run
# in the e2e-full merge gate.

name: e2e-nightly

on:
  workflow_dispatch:
  schedule:
    # 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00),
    # stale (01:30) and performance-ab (06:00).
    - cron: "29 4 * * *"

# Only alert-on-failure needs more than read access; it declares its own
# job-level `issues: write`.
permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: false

jobs:
  repl-nightly:
    name: Replication e2e (nightly)
    # dual-node tests spawn two full rustfs servers each; the extra headroom
    # over the per-PR sm-standard-2 e2e job keeps the parallel process fan-out
    # from starving the runner. Mirrors the ILM serial lane's runner class.
    runs-on: sm-standard-4
    timeout-minutes: 45
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: ci-e2e-repl
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
          install-build-packaging-tools: 'false'

      # The STS dual-node test requires awscurl and fails if it is unavailable.
      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Install awscurl
        run: |
          python3 -m pip install --user --upgrade pip "awscurl==0.44"
          echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"

      - name: Verify awscurl
        run: test -x "$AWSCURL_PATH"

      # Build once and carry its source/binary identity into the test invocation.
      - name: Build rustfs binary
        run: |
          python3 scripts/e2e_binary.py build --bins

      - name: Verify replication e2e membership
        env:
          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
        run: |
          cargo nextest list --profile e2e-repl-nightly -p e2e_test --message-format json > "${NEXTEST_LISTING}"
          python3 ./scripts/check_test_wiring.py --check-profile e2e-repl-nightly "${NEXTEST_LISTING}"

      - name: Run replication e2e nightly suite
        env:
          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
        run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test

      - name: Upload nextest junit report
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: e2e-replication-nightly-junit-${{ github.run_number }}
          path: |
            target/nextest/e2e-repl-nightly/junit.xml
            ${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
            ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs/
          retention-days: 7
          if-no-files-found: ignore

  cluster-nightly:
    name: Cluster fault e2e (nightly)
    runs-on: sm-standard-4
    timeout-minutes: 90
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: ci-e2e-nightly
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'

      - name: Build rustfs binary
        run: |
          python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks

      - name: Verify cluster fault e2e membership
        env:
          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-nightly-list.json
        run: |
          cargo nextest list --profile e2e-nightly -p e2e_test --message-format json > "${NEXTEST_LISTING}"
          python3 ./scripts/check_test_wiring.py --check-profile e2e-nightly "${NEXTEST_LISTING}"

      - name: Run cluster fault e2e nightly suite
        env:
          CARGO_BIN_EXE_rustfs: ${{ github.workspace }}/target/debug/rustfs
          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
        run: python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test

      - name: Upload cluster fault diagnostics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: e2e-cluster-nightly-${{ github.run_number }}
          path: |
            target/nextest/e2e-nightly/junit.xml
            ${{ runner.temp }}/rustfs-e2e-nightly-list.json
            ${{ runner.temp }}/rustfs-e2e-nightly-logs/
          retention-days: 7
          if-no-files-found: warn

  protocols-nightly:
    name: Protocol e2e (nightly)
    runs-on: sm-standard-4
    timeout-minutes: 90
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
      RUSTFS_BUILD_FEATURES: ftps,webdav,sftp
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: ci-e2e-protocols
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'

      - name: Install and verify protocol socket oracle
        run: |
          sudo apt-get update -qq
          sudo apt-get install -y -qq iproute2
          ss -tn state CLOSE-WAIT >/dev/null

      - name: Build protocol server
        run: python3 scripts/e2e_binary.py build --features "$RUSTFS_BUILD_FEATURES"

      # The suite owns fixed protocol ports and serializes its internal cases.
      - name: Verify protocol e2e membership
        env:
          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-protocols-list.json
        run: |
          cargo nextest list --profile e2e-protocols -p e2e_test --message-format json > "${NEXTEST_LISTING}"
          python3 ./scripts/check_test_wiring.py --check-profile e2e-protocols "${NEXTEST_LISTING}"

      - name: Run protocol e2e nightly suite
        env:
          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
        run: >-
          python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture

      - name: Upload protocol diagnostics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: e2e-protocol-nightly-${{ github.run_number }}
          path: |
            target/nextest/e2e-protocols/junit.xml
            ${{ runner.temp }}/rustfs-e2e-protocols-list.json
            ${{ runner.temp }}/rustfs-protocol-e2e-logs/
          retention-days: 7
          if-no-files-found: warn

  alert-on-failure:
    name: Alert on scheduled failure
    needs: [repl-nightly, cluster-nightly, protocols-nightly]
    # Only scheduled runs open/append the tracking issue (backlog#1149 ci-8);
    # manual workflow_dispatch runs stay quiet so a debugging run never files a
    # spurious alert.
    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false
      - name: Open or update failure-tracking issue
        uses: ./.github/actions/schedule-failure-issue
        with:
          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}