Extend functional test workflow with S3/KMS/tier suites (#6806)

.github/workflows/rustfs-pool-expand-test.yml

@@ -1,4 +1,4 @@
-name: RustFS Pool Expansion / Decommission Test
+name: RustFS Functional Test Suite (S3/KMS/tier/pool/heal)
 
 on:
   workflow_dispatch:
@@ -47,15 +47,16 @@ on:
         type: boolean
         default: true
   workflow_run:
-    # Run after the nightly build completes: pool expansion first, then heal.
+    # Run after the nightly build completes: S3 -> KMS -> tier -> pool -> heal.
     workflows: ["Nightly GNU Build"]
     types: [completed]
 
 permissions:
   contents: read
 
-# Only one pool-expansion test at a time: the workflow mutates a shared
-# test environment, so concurrent runs must not clobber each other.
+# Only one test run at a time: every job mutates the same shared test
+# environment (vm000/vm001/vm002), so concurrent runs must not clobber each
+# other. Jobs inside a run are chained with needs to serialize them.
 concurrency:
   group: rustfs-pool-expansion-test
   cancel-in-progress: false
@@ -75,18 +76,172 @@ env:
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
 
 jobs:
-  pool-expansion-test:
+  # All test scripts live in rustfs/auto-testing; rustfs stores only this
+  # workflow. Each job checks out auto-testing before running.
+
+  s3-compat-test:
+    name: S3 compatibility test
     runs-on: smoke-testing
-    timeout-minutes: 360
-    # Run on manual dispatch, or when the nightly build completed successfully
-    # (its deb is what the tests install). Skipped when nightly failed.
+    timeout-minutes: 240
     if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     steps:
-      - name: Checkout
+      - name: Checkout auto-testing scripts
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Run S3 compatibility suite
+        run: |
+          chmod +x auto-testing/rustfs-s3-compat-test.sh
+          ARGS=(--all-topologies -y --log-file /tmp/rustfs-s3-compat.log)
+          if [ -n "${{ inputs.package_url }}" ]; then
+            ARGS+=(--package-url "${{ inputs.package_url }}")
+          elif [ -n "${{ inputs.rustfs_version }}" ]; then
+            ARGS+=(--version "${{ inputs.rustfs_version }}")
+          else
+            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
+          fi
+          ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
+
+      - name: Upload test logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-s3-compat-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-s3-compat*.log
+          if-no-files-found: warn
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS S3 compatibility suite failed"
+          echo "See the uploaded log artifact for details."
+
+  kms-test:
+    name: KMS test (after S3)
+    runs-on: smoke-testing
+    timeout-minutes: 360
+    needs: s3-compat-test
+    if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Ensure docker (Vault container)
+        run: |
+          if ! command -v docker >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y docker.io
+          fi
+          sudo systemctl enable --now docker
+          docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1
+
+      - name: Run KMS suite
+        run: |
+          chmod +x auto-testing/rustfs-kms-test.sh
+          ARGS=(--all-topologies --backends local,vault-kv2 -y --log-file /tmp/rustfs-kms.log)
+          if [ -n "${{ inputs.package_url }}" ]; then
+            ARGS+=(--package-url "${{ inputs.package_url }}")
+          elif [ -n "${{ inputs.rustfs_version }}" ]; then
+            ARGS+=(--version "${{ inputs.rustfs_version }}")
+          else
+            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
+          fi
+          ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
+
+      - name: Upload test logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-kms-test-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-kms*.log
+          if-no-files-found: warn
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS KMS suite failed"
+          echo "See the uploaded log artifact for details."
+
+  tier-test:
+    name: Tier / event / audit test (after KMS)
+    runs-on: smoke-testing
+    timeout-minutes: 360
+    needs: kms-test
+    if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Ensure MQTT broker + clients (event notification tests)
+        run: |
+          if ! command -v mosquitto_sub >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y mosquitto mosquitto-clients
+          fi
+          sudo mkdir -p /etc/mosquitto/conf.d
+          printf 'listener 1883 0.0.0.0\nallow_anonymous true\n' | sudo tee /etc/mosquitto/conf.d/rustfs-test.conf >/dev/null
+          sudo systemctl restart mosquitto
+          sleep 2
+          ss -tln 2>/dev/null | grep -q ':1883' || { echo "mosquitto not listening on 1883"; exit 1; }
+
+      - name: Run tier / event / audit suite
+        run: |
+          chmod +x auto-testing/rustfs-tier-test.sh
+          ARGS=(--all-topologies -y --log-file /tmp/rustfs-tier.log)
+          if [ -n "${{ inputs.package_url }}" ]; then
+            ARGS+=(--package-url "${{ inputs.package_url }}")
+          elif [ -n "${{ inputs.rustfs_version }}" ]; then
+            ARGS+=(--version "${{ inputs.rustfs_version }}")
+          else
+            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
+          fi
+          ./auto-testing/rustfs-tier-test.sh "${ARGS[@]}"
+
+      - name: Upload test logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-tier-test-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-tier*.log
+          if-no-files-found: warn
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS tier/event/audit suite failed"
+          echo "See the uploaded log artifact for details."
+
+  pool-expansion-test:
+    name: Pool expansion / decommission test (after tier)
+    runs-on: smoke-testing
+    timeout-minutes: 360
+    needs: tier-test
+    if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
           persist-credentials: false
-          ref: ${{ github.event.workflow_run.head_sha || github.ref }}
 
       - name: Show environment
         run: |
@@ -99,8 +254,8 @@ jobs:
       - name: Reset test environment (before)
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
-          chmod +x scripts/test/rustfs_pool_expand.sh
-          ./scripts/test/rustfs_pool_expand.sh --reset -y
+          chmod +x auto-testing/rustfs_pool_expand.sh
+          ./auto-testing/rustfs_pool_expand.sh --reset -y
 
       - name: Install RustFS package & start first pool
         run: |
@@ -112,7 +267,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./scripts/test/rustfs_pool_expand.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
 
       - name: Preflight checks
         run: |
@@ -124,7 +279,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./scripts/test/rustfs_pool_expand.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
 
       - name: Run pool expansion & decommission test
         id: pool_test
@@ -137,7 +292,7 @@ jobs:
               STEPS="$STEPS,9"
             fi
           fi
-          ./scripts/test/rustfs_pool_expand.sh \
+          ./auto-testing/rustfs_pool_expand.sh \
             --steps "$STEPS" --with-warp -y \
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
             --storage-threshold "${{ inputs.storage_threshold || '50' }}" \
@@ -157,7 +312,7 @@ jobs:
       - name: Reset test environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}
         run: |
-          ./scripts/test/rustfs_pool_expand.sh --reset -y
+          ./auto-testing/rustfs_pool_expand.sh --reset -y
 
       - name: Notify on failure
         if: failure()
@@ -175,17 +330,19 @@ jobs:
     needs: pool-expansion-test
     if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
     steps:
-      - name: Checkout
+      - name: Checkout auto-testing scripts
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
           persist-credentials: false
-          ref: ${{ github.event.workflow_run.head_sha || github.ref }}
 
       - name: Reset test environment (before)
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
-          chmod +x scripts/test/rustfs_heal_test.sh
-          ./scripts/test/rustfs_heal_test.sh --reset -y
+          chmod +x auto-testing/rustfs_heal_test.sh
+          ./auto-testing/rustfs_heal_test.sh --reset -y
 
       - name: Install RustFS package & start cluster
         run: |
@@ -195,7 +352,7 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./scripts/test/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
 
       - name: Preflight checks
         run: |
@@ -205,11 +362,11 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          ./scripts/test/rustfs_heal_test.sh "${ARGS[@]}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
 
       - name: Run heal test (write -> outage -> heal -> verify)
         run: |
-          ./scripts/test/rustfs_heal_test.sh \
+          ./auto-testing/rustfs_heal_test.sh \
             --steps 3,4,5,6,7 -y \
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
             --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
@@ -229,7 +386,7 @@ jobs:
       - name: Reset test environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}
         run: |
-          ./scripts/test/rustfs_heal_test.sh --reset -y
+          ./auto-testing/rustfs_heal_test.sh --reset -y
 
       - name: Notify on failure
         if: failure()