# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# 4-node 4-disk distributed e2e lane.
#
# Each selected test starts a real localhost cluster via
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
# nightly runs, and manual dispatches all execute the same fail-closed suite.
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.

name: e2e-distributed

on:
  pull_request:
    paths:
      - "Cargo.lock"
      - "Cargo.toml"
      - ".config/nextest.toml"
      - ".github/workflows/e2e-distributed.yml"
      - "crates/audit/**"
      - "crates/common/**"
      - "crates/config/**"
      - "crates/e2e_test/**"
      - "crates/ecstore/**"
      - "crates/filemeta/**"
      - "crates/heal/**"
      - "crates/iam/**"
      - "crates/lock/**"
      - "crates/madmin/**"
      - "crates/notify/**"
      - "crates/replication/**"
      - "crates/s3-client/**"
      - "crates/s3-ops/**"
      - "crates/s3-types/**"
      - "crates/scanner/**"
      - "crates/storage-api/**"
      - "crates/utils/**"
      - "rustfs/**"
  workflow_dispatch:
    inputs:
      filter:
        description: "Optional nextest -E filter (default: the whole e2e-distributed profile)"
        required: false
        default: ""
  schedule:
    # 05:53 UTC nightly — clear of e2e-nightly (04:29) and ODM interop (05:23).
    - cron: "53 5 * * *"

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name != 'schedule' }}

jobs:
  distributed:
    name: Distributed 4-node 4-disk e2e
    runs-on: sm-standard-4
    timeout-minutes: 180
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
      NO_PROXY: 127.0.0.1,localhost
      HTTP_PROXY: ""
      HTTPS_PROXY: ""
      # Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml).
      UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
      UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
      UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
    steps:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: ci-e2e-distributed
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
          install-build-packaging-tools: 'false'

      - name: Prepare isolated filesystems for pool movement
        run: |
          set -euo pipefail
          mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
          mkdir -p "${mount_base}"
          roots=()
          for pool in 0 1 2 3; do
            image="${mount_base}/pool-${pool}.img"
            mountpoint="${mount_base}/pool-${pool}"
            truncate -s 1G "${image}"
            mkfs.ext4 -q -F "${image}"
            mkdir -p "${mountpoint}"
            sudo mount -o loop,nosuid,nodev "${image}" "${mountpoint}"
            sudo chmod 1777 "${mountpoint}"
            roots+=("${mountpoint}")
          done
          printf -v joined_roots '%s:' "${roots[@]}"
          echo "RUSTFS_E2E_POOL_ROOTS=${joined_roots%:}" >> "${GITHUB_ENV}"
          findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[0]}"
          findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[1]}"
          findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[2]}"
          findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[3]}"

      - name: Download pinned previous release
        env:
          SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source
        run: |
          set -euo pipefail
          mkdir -p "$SOURCE_DIR"
          archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET"
          curl --fail --location --retry 3 --output "$archive" \
            "https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
          echo "$UPGRADE_SOURCE_SHA256  $archive" | sha256sum --check --strict
          unzip -q "$archive" -d "$SOURCE_DIR"
          chmod +x "$SOURCE_DIR/rustfs"
          test -x "$SOURCE_DIR/rustfs"
          echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"

      - name: Build rustfs binary
        run: |
          cargo build -p rustfs --bins
          : > target/debug/rustfs.features

      - name: Verify distributed e2e membership
        env:
          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
        run: |
          cargo nextest list --profile e2e-distributed -p e2e_test --message-format json > "${NEXTEST_LISTING}"
          python3 ./scripts/check_test_wiring.py --check-profile e2e-distributed "${NEXTEST_LISTING}"

      - name: Run distributed 4-node e2e suite
        env:
          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
          FILTER: ${{ inputs.filter }}
        run: |
          set -euo pipefail
          if [ -n "${FILTER}" ]; then
            cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
          else
            cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
          fi

      - name: Upload distributed e2e diagnostics
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
          name: e2e-distributed-${{ github.run_number }}
          path: |
            target/nextest/e2e-distributed/junit.xml
            ${{ runner.temp }}/rustfs-e2e-distributed-list.json
            ${{ runner.temp }}/rustfs-e2e-distributed-logs/
          retention-days: 7
          if-no-files-found: warn

      - name: Unmount isolated pool filesystems
        if: always()
        run: |
          set -euo pipefail
          mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
          for pool in 0 1 2 3; do
            mountpoint="${mount_base}/pool-${pool}"
            if mountpoint --quiet "${mountpoint}"; then
              sudo umount "${mountpoint}"
            fi
          done

  alert-on-failure:
    name: Alert on scheduled failure
    needs: [distributed]
    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          persist-credentials: false
      - name: Open or update failure-tracking issue
        uses: ./.github/actions/schedule-failure-issue
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}