fix(ci): split workflows and add perf version reporting (#6848)

* ci: pass package selector to test run steps (fix rc.3 fallback)

* fix(ci): split workflows and add perf version reporting

* fix(ci): enforce strict shared workflow order

---------

Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: houseme <housemecn@gmail.com>

.github/workflows/package.yml

@@ -224,7 +224,7 @@ jobs:
           ZIP_FILE=$(find ./binary-artifact -name "*.zip" -type f | head -1)
           if [[ -z "$ZIP_FILE" ]]; then
             echo "❌ No binary artifact found"
-            ls -la ./binary-artifact/ || true
+            find ./binary-artifact -mindepth 1 -maxdepth 1 -print 2>/dev/null || true
             exit 1
           fi
 
@@ -239,7 +239,7 @@ jobs:
           fi
 
           chmod +x ./bin/rustfs
-          ls -lh ./bin/rustfs
+          stat --printf='%n %s bytes\n' ./bin/rustfs
           echo "✅ Binary extracted"
 
       - name: Build DEB package
@@ -336,7 +336,7 @@ jobs:
           fakeroot dpkg-deb --build "${PKG_DIR}"
 
           DEB_FILE="${PKG_DIR}.deb"
-          ls -lh "$DEB_FILE"
+          stat --printf='%n %s bytes\n' "$DEB_FILE"
           echo "deb_file=$DEB_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ DEB built: $DEB_FILE"
 
@@ -410,13 +410,14 @@ jobs:
             LICENSE=/usr/share/doc/rustfs/LICENSE \
             README.md=/usr/share/doc/rustfs/README.md
 
-          RPM_FILE=$(ls -1 rustfs-*.rpm 2>/dev/null | head -1)
+            RPM_FILE=$(find . -maxdepth 1 -type f -name 'rustfs-*.rpm' -print | head -1)
+            RPM_FILE="${RPM_FILE#./}"
           if [[ -z "$RPM_FILE" ]]; then
             echo "❌ RPM build failed"
             exit 1
           fi
 
-          ls -lh "$RPM_FILE"
+          stat --printf='%n %s bytes\n' "$RPM_FILE"
           echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ RPM built: $RPM_FILE"
 
@@ -552,11 +553,13 @@ jobs:
       - name: Print summary
         shell: bash
         run: |
-          echo "## 📦 Package Summary" >> "$GITHUB_STEP_SUMMARY"
-          echo "" >> "$GITHUB_STEP_SUMMARY"
-          echo "| Item | Value |" >> "$GITHUB_STEP_SUMMARY"
-          echo "|------|-------|" >> "$GITHUB_STEP_SUMMARY"
-          echo "| Version | \`${{ needs.resolve.outputs.version }}\` |" >> "$GITHUB_STEP_SUMMARY"
-          echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |" >> "$GITHUB_STEP_SUMMARY"
-          echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |" >> "$GITHUB_STEP_SUMMARY"
-          echo "| Package Status | ${{ needs.package.result }} |" >> "$GITHUB_STEP_SUMMARY"
+          {
+            echo "## 📦 Package Summary"
+            echo ""
+            echo "| Item | Value |"
+            echo "|------|-------|"
+            echo "| Version | \`${{ needs.resolve.outputs.version }}\` |"
+            echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |"
+            echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |"
+            echo "| Package Status | ${{ needs.package.result }} |"
+          } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/rustfs-heal-test.yml

@@ -23,12 +23,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  workflow_run:
-    # Run after the nightly build completes: the nightly deb is what the test
-    # installs. Heal also runs inside the functional suite; this standalone
-    # workflow enables manual single-suite runs as well.
-    workflows: ["Nightly GNU Build"]
-    types: [completed]
 
 permissions:
   contents: read
@@ -36,7 +30,7 @@ permissions:
 # Only one test at a time: both this and the pool-expansion workflow mutate
 # the same test environment, so they share one concurrency group.
 concurrency:
-  group: rustfs-pool-expansion-test
+  group: rustfs-shared-functional-tests
   cancel-in-progress: false
 
 defaults:
@@ -55,9 +49,9 @@ jobs:
   heal-test:
     runs-on: smoke-testing
     timeout-minutes: 480
-    # Run on manual dispatch, or when the nightly build completed successfully.
-    # Skipped when nightly failed.
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    # Manual-only standalone run. Nightly chain already runs heal in
+    # rustfs-pool-expand-test.yml to avoid duplicate heal executions.
+    if: ${{ github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout auto-testing scripts
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7

.github/workflows/rustfs-kms-test.yml

@@ -0,0 +1,169 @@
+name: RustFS KMS Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      rustfs_version:
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        required: false
+        default: '1.0.0-rc.4-preview.1'
+      package_url:
+        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
+        required: false
+        type: string
+  workflow_run:
+    # Strict shared-environment order: run after S3 compatibility test succeeds.
+    workflows: ["RustFS S3 Compatibility Test"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-shared-functional-tests
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  kms-test:
+    runs-on: smoke-testing
+    timeout-minutes: 420
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          openssl version
+          docker --version || true
+
+      - name: Cleanup environment (before)
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
+            '
+          done
+
+      - name: Ensure docker (Vault container)
+        run: |
+          if ! command -v docker >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y docker.io
+          fi
+          sudo systemctl enable --now docker
+          docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1
+
+      - name: Run KMS suite
+        id: test
+        env:
+          LOG_FILE: /tmp/rustfs-kms.log
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-kms-test.sh
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          ARGS=(--all-topologies --backends "local,vault-kv2" -y --log-file "${LOG_FILE}")
+          if [ -n "${PACKAGE_URL}" ]; then
+            ARGS+=(--package-url "${PACKAGE_URL}")
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            ARGS+=(--version "${RUSTFS_VERSION}")
+          else
+            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-kms.log
+          REPORT_FILE: /tmp/rustfs-kms-report.md
+        run: |
+          set -euo pipefail
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          {
+            echo "# RustFS KMS test report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Package: ${PACKAGE_SOURCE}"
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload report and logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-kms-test-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-kms.log
+            /tmp/rustfs-kms-report.md
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        if: always()
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
+            '
+          done
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS KMS suite failed"
+          echo "See the uploaded report and log artifacts for details."

.github/workflows/rustfs-performance-test.yml

@@ -149,11 +149,30 @@ jobs:
         run: |
           ./auto-testing/rustfs_performance_test.sh --step 6 -y
 
+      - name: Collect RustFS version info
+        if: ${{ steps.benchmark.conclusion == 'success' }}
+        env:
+          VERSION_FILE: /tmp/rustfs-version.txt
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES}"
+          [ "${#NODES[@]}" -gt 0 ] || { echo "RUSTFS_NODES is empty"; exit 1; }
+          NODE="${NODES[0]}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          {
+            echo "Node: ${NODE}"
+            echo "Command: rustfs --version"
+            echo ""
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
+              "${SSH_USER}@${NODE}" 'rustfs --version'
+          } > "${VERSION_FILE}"
+
       - name: Upload report to dashboard (reports/YYYY-MM-DD.md)
         if: ${{ steps.benchmark.conclusion == 'success' }}
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
           RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
+          VERSION_FILE: /tmp/rustfs-version.txt
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
@@ -173,6 +192,11 @@ jobs:
             echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
             echo ""
             cat "${SUMMARY}"
+            echo ""
+            echo "## RustFS version"
+            echo '```text'
+            cat "${VERSION_FILE}"
+            echo '```'
           } > /tmp/rustfs-perf-report.md
           CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
           SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
@@ -196,6 +220,7 @@ jobs:
           path: |
             /tmp/rustfs-perf-test*.log
             /tmp/rustfs-perf-results/**
+            /tmp/rustfs-version.txt
           if-no-files-found: warn
 
       - name: Reset test environment (after)

.github/workflows/rustfs-pool-expand-test.yml

@@ -1,4 +1,4 @@
-name: RustFS Functional Test Suite (S3/KMS/tier/pool/heal)
+name: RustFS Pool Expansion / Heal Test
 
 on:
   workflow_dispatch:
@@ -46,8 +46,8 @@ on:
         type: boolean
         default: true
   workflow_run:
-    # Run after the nightly build completes: S3 -> KMS -> tier -> pool -> heal.
-    workflows: ["Nightly GNU Build"]
+    # Strict shared-environment order: run after tier test succeeds.
+    workflows: ["RustFS Tier Test"]
     types: [completed]
 
 permissions:
@@ -57,7 +57,7 @@ permissions:
 # environment (vm000/vm001/vm002), so concurrent runs must not clobber each
 # other. Jobs inside a run are chained with needs to serialize them.
 concurrency:
-  group: rustfs-pool-expansion-test
+  group: rustfs-shared-functional-tests
   cancel-in-progress: false
 
 defaults:
@@ -75,181 +75,11 @@ env:
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
 
 jobs:
-  # All test scripts live in rustfs/auto-testing; rustfs stores only this
-  # workflow. Each job checks out auto-testing before running.
-
-  s3-compat-test:
-    name: S3 compatibility test
-    runs-on: smoke-testing
-    timeout-minutes: 240
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
-    steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-      - name: Run S3 compatibility suite
-        run: |
-          chmod +x auto-testing/rustfs-s3-compat-test.sh
-          ARGS=(--all-topologies -y --log-file /tmp/rustfs-s3-compat.log)
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          elif [ -n "${{ inputs.rustfs_version }}" ]; then
-            ARGS+=(--version "${{ inputs.rustfs_version }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
-
-      - name: Upload test logs
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: rustfs-s3-compat-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-s3-compat*.log
-          if-no-files-found: warn
-
-      - name: Notify on failure
-        if: failure()
-        run: |
-          echo "RustFS S3 compatibility suite failed"
-          echo "See the uploaded log artifact for details."
-
-  kms-test:
-    name: KMS test (after S3)
-    runs-on: smoke-testing
-    timeout-minutes: 360
-    needs: s3-compat-test
-    if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
-    steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-      - name: Ensure docker (Vault container)
-        run: |
-          if ! command -v docker >/dev/null 2>&1; then
-            sudo apt-get update
-            sudo apt-get install -y docker.io
-          fi
-          sudo systemctl enable --now docker
-          docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1
-
-      - name: Run KMS suite
-        run: |
-          chmod +x auto-testing/rustfs-kms-test.sh
-          ARGS=(--all-topologies --backends local,vault-kv2 -y --log-file /tmp/rustfs-kms.log)
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          elif [ -n "${{ inputs.rustfs_version }}" ]; then
-            ARGS+=(--version "${{ inputs.rustfs_version }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
-
-      - name: Upload test logs
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: rustfs-kms-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-kms*.log
-          if-no-files-found: warn
-
-      - name: Notify on failure
-        if: failure()
-        run: |
-          echo "RustFS KMS suite failed"
-          echo "See the uploaded log artifact for details."
-
-  tier-test:
-    name: Tier / event / audit test (after KMS)
-    runs-on: smoke-testing
-    timeout-minutes: 360
-    needs: kms-test
-    if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
-    steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-      - name: Ensure MQTT broker + clients (event notification tests)
-        run: |
-          if ! command -v mosquitto_sub >/dev/null 2>&1; then
-            sudo apt-get update
-            sudo apt-get install -y mosquitto mosquitto-clients
-          fi
-          # Make restart deterministic on shared runners: stop any previous
-          # mosquitto instance/process and drop stale config first.
-          sudo systemctl stop mosquitto 2>/dev/null || true
-          sudo pkill -f mosquitto 2>/dev/null || true
-          sleep 1
-          sudo rm -f /etc/mosquitto/conf.d/rustfs-test.conf
-          sudo mkdir -p /etc/mosquitto/conf.d
-          printf 'listener 1883 0.0.0.0\nallow_anonymous true\n' | sudo tee /etc/mosquitto/conf.d/rustfs-test.conf >/dev/null
-          sudo systemctl reset-failed mosquitto 2>/dev/null || true
-          sudo systemctl start mosquitto
-          sleep 2
-          if ! ss -tln 2>/dev/null | grep -q ':1883'; then
-            echo "=== mosquitto status ==="
-            sudo systemctl status mosquitto --no-pager | head -20 || true
-            echo "=== mosquitto journal ==="
-            sudo journalctl -u mosquitto -n 30 --no-pager || true
-            echo "mosquitto not listening on 1883"
-            exit 1
-          fi
-
-      - name: Run tier / event / audit suite
-        run: |
-          chmod +x auto-testing/rustfs-tier-test.sh
-          ARGS=(--all-topologies -y --log-file /tmp/rustfs-tier.log)
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          elif [ -n "${{ inputs.rustfs_version }}" ]; then
-            ARGS+=(--version "${{ inputs.rustfs_version }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs-tier-test.sh "${ARGS[@]}"
-
-      - name: Upload test logs
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: rustfs-tier-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-tier*.log
-          if-no-files-found: warn
-
-      - name: Notify on failure
-        if: failure()
-        run: |
-          echo "RustFS tier/event/audit suite failed"
-          echo "See the uploaded log artifact for details."
-
   pool-expansion-test:
-    name: Pool expansion / decommission test (after tier)
+    name: Pool expansion / decommission test
     runs-on: smoke-testing
     timeout-minutes: 360
-    needs: tier-test
-    if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     steps:
       - name: Checkout auto-testing scripts
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -391,7 +221,7 @@ jobs:
 
       - name: Run heal test (write -> outage -> heal -> verify)
         run: |
-          ARGS=(--steps 3,4,5,6,7 -y \
+          ARGS=(--steps "3,4,5,6,7" -y \
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
             --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
             --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \

.github/workflows/rustfs-s3-compat-test.yml

@@ -0,0 +1,159 @@
+name: RustFS S3 Compatibility Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      rustfs_version:
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        required: false
+        default: '1.0.0-rc.4-preview.1'
+      package_url:
+        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
+        required: false
+        type: string
+  workflow_run:
+    # Run after the nightly build completes; the nightly deb is what the test installs.
+    workflows: ["Nightly GNU Build"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-shared-functional-tests
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+
+jobs:
+  s3-compat-test:
+    runs-on: smoke-testing
+    timeout-minutes: 360
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          openssl version
+          df -h /data | tail -1
+
+      - name: Cleanup environment (before)
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs
+            '
+          done
+
+      - name: Run S3 compatibility suite
+        id: test
+        env:
+          LOG_FILE: /tmp/rustfs-s3-compat.log
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-s3-compat-test.sh
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          ARGS=(--all-topologies -y --log-file "${LOG_FILE}")
+          if [ -n "${PACKAGE_URL}" ]; then
+            ARGS+=(--package-url "${PACKAGE_URL}")
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            ARGS+=(--version "${RUSTFS_VERSION}")
+          else
+            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-s3-compat.log
+          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
+        run: |
+          set -euo pipefail
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          {
+            echo "# RustFS S3 compatibility test report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Package: ${PACKAGE_SOURCE}"
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload report and logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-s3-compat-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-s3-compat.log
+            /tmp/rustfs-s3-compat-report.md
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        if: always()
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs
+            '
+          done
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS S3 compatibility suite failed"
+          echo "See the uploaded report and log artifacts for details."

.github/workflows/rustfs-tier-test.yml

@@ -0,0 +1,172 @@
+name: RustFS Tier Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      rustfs_version:
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        required: false
+        default: '1.0.0-rc.4-preview.1'
+      package_url:
+        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
+        required: false
+        type: string
+  workflow_run:
+    # Strict shared-environment order: run after KMS test succeeds.
+    workflows: ["RustFS KMS Test"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-shared-functional-tests
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  tier-test:
+    runs-on: smoke-testing
+    timeout-minutes: 420
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          openssl version
+          df -h /data | tail -1
+
+      - name: Cleanup environment (before)
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
+            '
+          done
+
+      - name: Ensure MQTT broker + clients
+        run: |
+          if ! command -v mosquitto_sub >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y mosquitto mosquitto-clients
+          fi
+          sudo mkdir -p /etc/mosquitto/conf.d
+          printf 'listener 1883 0.0.0.0\nallow_anonymous true\n' | sudo tee /etc/mosquitto/conf.d/rustfs-test.conf >/dev/null
+          sudo systemctl restart mosquitto
+          sleep 2
+          ss -tln 2>/dev/null | grep -q ':1883' || { echo 'mosquitto not listening on 1883'; exit 1; }
+
+      - name: Run tier suite
+        id: test
+        env:
+          LOG_FILE: /tmp/rustfs-tier.log
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-tier-test.sh
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          ARGS=(--all-topologies -y --log-file "${LOG_FILE}")
+          if [ -n "${PACKAGE_URL}" ]; then
+            ARGS+=(--package-url "${PACKAGE_URL}")
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            ARGS+=(--version "${RUSTFS_VERSION}")
+          else
+            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-tier-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-tier.log
+          REPORT_FILE: /tmp/rustfs-tier-report.md
+        run: |
+          set -euo pipefail
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          {
+            echo "# RustFS tier test report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Package: ${PACKAGE_SOURCE}"
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload report and logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-tier-test-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-tier.log
+            /tmp/rustfs-tier-report.md
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        if: always()
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
+            '
+          done
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS tier suite failed"
+          echo "See the uploaded report and log artifacts for details."