rustfs/.github/workflows/e2e-upgrade.yml

# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Upgrade Compatibility

on:
  pull_request:
    paths:
      - ".github/workflows/e2e-upgrade.yml"
      - "crates/e2e_test/src/common.rs"
      - "crates/e2e_test/src/fake_s3_target/**"
      - "crates/e2e_test/src/lib.rs"
      - "crates/e2e_test/src/replication_extension_test.rs"
      - "crates/e2e_test/src/upgrade_compatibility_test.rs"
      - "crates/ecstore/**"
      - "crates/filemeta/**"
      - "crates/heal/**"
      - "crates/kms/**"
      - "crates/storage-api/**"
      - "rustfs/**"
      - "Cargo.lock"
  push:
    tags:
      - "[0-9]*.[0-9]*.[0-9]*"
  schedule:
    - cron: "17 3 * * 1"
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1

jobs:
  build:
    name: Build upgrade candidate
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          cache-shared-key: e2e-upgrade-server-hosted
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
          install-build-packaging-tools: "false"
          install-test-tools: "false"

      - name: Build current RustFS binary
        run: python3 scripts/e2e_binary.py build

      - name: Upload verified upgrade candidate
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: upgrade-candidate-${{ github.run_id }}
          path: |
            target/debug/rustfs
            target/debug/rustfs.e2e.json
          if-no-files-found: error
          retention-days: 3
          compression-level: 0
          overwrite: true

  upgrade:
    name: Upgrade compatibility (${{ matrix.source.version }})
    needs: build
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        source:
          - version: 1.0.0-rc.5
            asset: rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip
            sha256: 3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d
          - version: 1.0.1
            asset: rustfs-linux-x86_64-gnu-v1.0.1.zip
            sha256: 9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
      UPGRADE_SOURCE_VERSION: ${{ matrix.source.version }}
      UPGRADE_SOURCE_ASSET: ${{ matrix.source.asset }}
      UPGRADE_SOURCE_SHA256: ${{ matrix.source.sha256 }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          cache-shared-key: e2e-upgrade-tests-hosted
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
          install-build-packaging-tools: "false"
          install-test-tools: "false"

      - name: Download verified upgrade candidate
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: upgrade-candidate-${{ github.run_id }}
          path: target/debug

      - name: Restore candidate executable bit
        run: chmod +x target/debug/rustfs

      - name: Verify and unpack pinned previous release
        env:
          SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source
        run: |
          set -euo pipefail
          mkdir -p "$SOURCE_DIR"
          archive="target/debug/$UPGRADE_SOURCE_ASSET"
          curl --fail --location --retry 3 --output "$archive" \
            "https://github.com/rustfs/rustfs/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
          echo "$UPGRADE_SOURCE_SHA256  $archive" | sha256sum --check --strict
          unzip -q "$archive" -d "$SOURCE_DIR"
          chmod +x "$SOURCE_DIR/rustfs"
          test -x "$SOURCE_DIR/rustfs"
          echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
          echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV"

      # Reuse one compiled harness, while keeping each case in its own process.
      # Continue after a failure to retain the former matrix's complete verdict.
      - name: Run upgrade compatibility tests
        shell: bash
        env:
          NO_PROXY: 127.0.0.1,localhost,::1
        run: |
          export no_proxy="$NO_PROXY"
          cases=(
            direct-upgrade:direct_upgrade_from_previous_release_preserves_object_contracts
            mixed-version-upgrade:rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
            bucket-config-upgrade:direct_upgrade_from_previous_release_preserves_bucket_configuration
            bucket-config-rollback:rollback_to_previous_release_reads_current_bucket_metadata
            multipart-layout-upgrade:direct_upgrade_from_previous_release_preserves_multipart_layouts
            multipart-layout-baseline:previous_release_baseline_replicates_multipart_layouts
          )
          if [[ "$UPGRADE_SOURCE_VERSION" == "1.0.0-rc.5" ]]; then
            cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
            cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects)
          else
            cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects)
          fi
          status=0
          for entry in "${cases[@]}"; do
            artifact="${entry%%:*}"
            test="${entry#*:}"
            export RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR="$RUNNER_TEMP/rustfs-upgrade-g09-evidence/$artifact"
            export RUSTFS_E2E_LOG_DIR="$RUNNER_TEMP/rustfs-upgrade-logs/$artifact"
            mkdir -p "$RUSTFS_E2E_LOG_DIR"
            echo "::group::$test"
            if python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test \
              "upgrade_compatibility_test::$test" -- --ignored --exact --nocapture \
              2>&1 | tee "$RUSTFS_E2E_LOG_DIR/test.log"; then
              echo "PASS: $test"
            else
              echo "::error::Upgrade compatibility failed: $test"
              status=1
            fi
            echo "::endgroup::"
          done
          exit "$status"

      - name: Upload scanner/heal G09 evidence
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: upgrade-g09-evidence-${{ matrix.source.version }}-${{ github.run_number }}
          path: ${{ runner.temp }}/rustfs-upgrade-g09-evidence
          if-no-files-found: ignore
          retention-days: 14

      - name: Upload server logs
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: upgrade-server-logs-${{ matrix.source.version }}-${{ github.run_number }}
          path: ${{ runner.temp }}/rustfs-upgrade-logs
          if-no-files-found: warn
          retention-days: 14