diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 5f07463dc..2a8d87093 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -6,3 +6,4 @@ self-hosted-runner: - sm-standard-4 - dind-sm-standard-2 - smoke-testing + - pf-testing diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 20b1ebcf8..f9680cf3b 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -224,7 +224,7 @@ jobs: ZIP_FILE=$(find ./binary-artifact -name "*.zip" -type f | head -1) if [[ -z "$ZIP_FILE" ]]; then echo "❌ No binary artifact found" - ls -la ./binary-artifact/ || true + find ./binary-artifact -mindepth 1 -maxdepth 1 -print 2>/dev/null || true exit 1 fi @@ -239,7 +239,7 @@ jobs: fi chmod +x ./bin/rustfs - ls -lh ./bin/rustfs + stat --printf='%n %s bytes\n' ./bin/rustfs echo "✅ Binary extracted" - name: Build DEB package @@ -336,7 +336,7 @@ jobs: fakeroot dpkg-deb --build "${PKG_DIR}" DEB_FILE="${PKG_DIR}.deb" - ls -lh "$DEB_FILE" + stat --printf='%n %s bytes\n' "$DEB_FILE" echo "deb_file=$DEB_FILE" >> "$GITHUB_OUTPUT" echo "✅ DEB built: $DEB_FILE" @@ -410,13 +410,14 @@ jobs: LICENSE=/usr/share/doc/rustfs/LICENSE \ README.md=/usr/share/doc/rustfs/README.md - RPM_FILE=$(ls -1 rustfs-*.rpm 2>/dev/null | head -1) + RPM_FILE=$(find . -maxdepth 1 -type f -name 'rustfs-*.rpm' -print | head -1) + RPM_FILE="${RPM_FILE#./}" if [[ -z "$RPM_FILE" ]]; then echo "❌ RPM build failed" exit 1 fi - ls -lh "$RPM_FILE" + stat --printf='%n %s bytes\n' "$RPM_FILE" echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT" echo "✅ RPM built: $RPM_FILE" @@ -552,11 +553,13 @@ jobs: - name: Print summary shell: bash run: | - echo "## 📦 Package Summary" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "| Item | Value |" >> "$GITHUB_STEP_SUMMARY" - echo "|------|-------|" >> "$GITHUB_STEP_SUMMARY" - echo "| Version | \`${{ needs.resolve.outputs.version }}\` |" >> "$GITHUB_STEP_SUMMARY" - echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |" >> "$GITHUB_STEP_SUMMARY" - echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |" >> "$GITHUB_STEP_SUMMARY" - echo "| Package Status | ${{ needs.package.result }} |" >> "$GITHUB_STEP_SUMMARY" + { + echo "## 📦 Package Summary" + echo "" + echo "| Item | Value |" + echo "|------|-------|" + echo "| Version | \`${{ needs.resolve.outputs.version }}\` |" + echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |" + echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |" + echo "| Package Status | ${{ needs.package.result }} |" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml index 8a9e90bb4..d890335e5 100644 --- a/.github/workflows/rustfs-heal-test.yml +++ b/.github/workflows/rustfs-heal-test.yml @@ -23,12 +23,6 @@ on: description: 'Reset the nodes after the test (DESTROYS test data/config)' type: boolean default: true - workflow_run: - # Run after the nightly build completes: the nightly deb is what the test - # installs. Heal also runs inside the functional suite; this standalone - # workflow enables manual single-suite runs as well. - workflows: ["Nightly GNU Build"] - types: [completed] permissions: contents: read @@ -36,7 +30,7 @@ permissions: # Only one test at a time: both this and the pool-expansion workflow mutate # the same test environment, so they share one concurrency group. concurrency: - group: rustfs-pool-expansion-test + group: rustfs-shared-functional-tests cancel-in-progress: false defaults: @@ -55,9 +49,9 @@ jobs: heal-test: runs-on: smoke-testing timeout-minutes: 480 - # Run on manual dispatch, or when the nightly build completed successfully. - # Skipped when nightly failed. - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + # Manual-only standalone run. Nightly chain already runs heal in + # rustfs-pool-expand-test.yml to avoid duplicate heal executions. + if: ${{ github.event_name == 'workflow_dispatch' }} steps: - name: Checkout auto-testing scripts uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml new file mode 100644 index 000000000..07f80d520 --- /dev/null +++ b/.github/workflows/rustfs-kms-test.yml @@ -0,0 +1,169 @@ +name: RustFS KMS Test + +on: + workflow_dispatch: + inputs: + rustfs_version: + description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)' + required: false + default: '1.0.0-rc.4-preview.1' + package_url: + description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.' + required: false + type: string + workflow_run: + # Strict shared-environment order: run after S3 compatibility test succeeds. + workflows: ["RustFS S3 Compatibility Test"] + types: [completed] + +permissions: + contents: read + +concurrency: + group: rustfs-shared-functional-tests + cancel-in-progress: false + +defaults: + run: + shell: bash + +env: + RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }} + RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }} + RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }} + RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }} + RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }} + PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + +jobs: + kms-test: + runs-on: smoke-testing + timeout-minutes: 420 + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + steps: + - name: Checkout auto-testing scripts + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + repository: rustfs/auto-testing + ref: main + path: auto-testing + persist-credentials: false + token: ${{ secrets.PF_TESTING_GH_TOKEN }} + + - name: Show environment + run: | + uname -a + jq --version + openssl version + docker --version || true + + - name: Cleanup environment (before) + run: | + set -euo pipefail + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + for node in "${NODES[@]}"; do + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup + ' + done + + - name: Ensure docker (Vault container) + run: | + if ! command -v docker >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y docker.io + fi + sudo systemctl enable --now docker + docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1 + + - name: Run KMS suite + id: test + env: + LOG_FILE: /tmp/rustfs-kms.log + run: | + set -euo pipefail + chmod +x auto-testing/rustfs-kms-test.sh + PACKAGE_URL='${{ inputs.package_url }}' + RUSTFS_VERSION='${{ inputs.rustfs_version }}' + ARGS=(--all-topologies --backends "local,vault-kv2" -y --log-file "${LOG_FILE}") + if [ -n "${PACKAGE_URL}" ]; then + ARGS+=(--package-url "${PACKAGE_URL}") + elif [ -n "${RUSTFS_VERSION}" ]; then + ARGS+=(--version "${RUSTFS_VERSION}") + else + ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}") + fi + ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}" + + - name: Generate report + if: always() + env: + LOG_FILE: /tmp/rustfs-kms.log + REPORT_FILE: /tmp/rustfs-kms-report.md + run: | + set -euo pipefail + PACKAGE_URL='${{ inputs.package_url }}' + RUSTFS_VERSION='${{ inputs.rustfs_version }}' + if [ -n "${PACKAGE_URL}" ]; then + PACKAGE_SOURCE="${PACKAGE_URL}" + elif [ -n "${RUSTFS_VERSION}" ]; then + PACKAGE_SOURCE="version ${RUSTFS_VERSION}" + else + PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" + fi + { + echo "# RustFS KMS test report" + echo "" + echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Trigger: ${{ github.event_name }}" + echo "- Package: ${PACKAGE_SOURCE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" || true + echo '```' + } | tee "${REPORT_FILE}" + cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload report and logs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: rustfs-kms-test-${{ github.run_id }} + path: | + /tmp/rustfs-kms.log + /tmp/rustfs-kms-report.md + if-no-files-found: warn + + - name: Cleanup environment (after) + if: always() + run: | + set -euo pipefail + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + for node in "${NODES[@]}"; do + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup + ' + done + + - name: Notify on failure + if: failure() + run: | + echo "RustFS KMS suite failed" + echo "See the uploaded report and log artifacts for details." diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml index bb20a704d..ec64f7480 100644 --- a/.github/workflows/rustfs-performance-test.yml +++ b/.github/workflows/rustfs-performance-test.yml @@ -149,11 +149,30 @@ jobs: run: | ./auto-testing/rustfs_performance_test.sh --step 6 -y + - name: Collect RustFS version info + if: ${{ steps.benchmark.conclusion == 'success' }} + env: + VERSION_FILE: /tmp/rustfs-version.txt + run: | + set -euo pipefail + read -r -a NODES <<< "${RUSTFS_NODES}" + [ "${#NODES[@]}" -gt 0 ] || { echo "RUSTFS_NODES is empty"; exit 1; } + NODE="${NODES[0]}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + { + echo "Node: ${NODE}" + echo "Command: rustfs --version" + echo "" + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \ + "${SSH_USER}@${NODE}" 'rustfs --version' + } > "${VERSION_FILE}" + - name: Upload report to dashboard (reports/YYYY-MM-DD.md) if: ${{ steps.benchmark.conclusion == 'success' }} env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }} + VERSION_FILE: /tmp/rustfs-version.txt run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -173,6 +192,11 @@ jobs: echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}" echo "" cat "${SUMMARY}" + echo "" + echo "## RustFS version" + echo '```text' + cat "${VERSION_FILE}" + echo '```' } > /tmp/rustfs-perf-report.md CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')" SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" @@ -196,6 +220,7 @@ jobs: path: | /tmp/rustfs-perf-test*.log /tmp/rustfs-perf-results/** + /tmp/rustfs-version.txt if-no-files-found: warn - name: Reset test environment (after) diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml index 7e647211b..7dedb7787 100644 --- a/.github/workflows/rustfs-pool-expand-test.yml +++ b/.github/workflows/rustfs-pool-expand-test.yml @@ -1,4 +1,4 @@ -name: RustFS Functional Test Suite (S3/KMS/tier/pool/heal) +name: RustFS Pool Expansion / Heal Test on: workflow_dispatch: @@ -46,8 +46,8 @@ on: type: boolean default: true workflow_run: - # Run after the nightly build completes: S3 -> KMS -> tier -> pool -> heal. - workflows: ["Nightly GNU Build"] + # Strict shared-environment order: run after tier test succeeds. + workflows: ["RustFS Tier Test"] types: [completed] permissions: @@ -57,7 +57,7 @@ permissions: # environment (vm000/vm001/vm002), so concurrent runs must not clobber each # other. Jobs inside a run are chained with needs to serialize them. concurrency: - group: rustfs-pool-expansion-test + group: rustfs-shared-functional-tests cancel-in-progress: false defaults: @@ -75,181 +75,11 @@ env: RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }} jobs: - # All test scripts live in rustfs/auto-testing; rustfs stores only this - # workflow. Each job checks out auto-testing before running. - - s3-compat-test: - name: S3 compatibility test - runs-on: smoke-testing - timeout-minutes: 240 - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} - steps: - - name: Checkout auto-testing scripts - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - repository: rustfs/auto-testing - ref: main - path: auto-testing - persist-credentials: false - token: ${{ secrets.PF_TESTING_GH_TOKEN }} - - - name: Run S3 compatibility suite - run: | - chmod +x auto-testing/rustfs-s3-compat-test.sh - ARGS=(--all-topologies -y --log-file /tmp/rustfs-s3-compat.log) - if [ -n "${{ inputs.package_url }}" ]; then - ARGS+=(--package-url "${{ inputs.package_url }}") - elif [ -n "${{ inputs.rustfs_version }}" ]; then - ARGS+=(--version "${{ inputs.rustfs_version }}") - else - ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") - fi - ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}" - - - name: Upload test logs - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: rustfs-s3-compat-${{ github.run_id }} - path: | - /tmp/rustfs-s3-compat*.log - if-no-files-found: warn - - - name: Notify on failure - if: failure() - run: | - echo "RustFS S3 compatibility suite failed" - echo "See the uploaded log artifact for details." - - kms-test: - name: KMS test (after S3) - runs-on: smoke-testing - timeout-minutes: 360 - needs: s3-compat-test - if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }} - steps: - - name: Checkout auto-testing scripts - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - repository: rustfs/auto-testing - ref: main - path: auto-testing - persist-credentials: false - token: ${{ secrets.PF_TESTING_GH_TOKEN }} - - - name: Ensure docker (Vault container) - run: | - if ! command -v docker >/dev/null 2>&1; then - sudo apt-get update - sudo apt-get install -y docker.io - fi - sudo systemctl enable --now docker - docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1 - - - name: Run KMS suite - run: | - chmod +x auto-testing/rustfs-kms-test.sh - ARGS=(--all-topologies --backends local,vault-kv2 -y --log-file /tmp/rustfs-kms.log) - if [ -n "${{ inputs.package_url }}" ]; then - ARGS+=(--package-url "${{ inputs.package_url }}") - elif [ -n "${{ inputs.rustfs_version }}" ]; then - ARGS+=(--version "${{ inputs.rustfs_version }}") - else - ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") - fi - ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}" - - - name: Upload test logs - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: rustfs-kms-test-${{ github.run_id }} - path: | - /tmp/rustfs-kms*.log - if-no-files-found: warn - - - name: Notify on failure - if: failure() - run: | - echo "RustFS KMS suite failed" - echo "See the uploaded log artifact for details." - - tier-test: - name: Tier / event / audit test (after KMS) - runs-on: smoke-testing - timeout-minutes: 360 - needs: kms-test - if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }} - steps: - - name: Checkout auto-testing scripts - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - repository: rustfs/auto-testing - ref: main - path: auto-testing - persist-credentials: false - token: ${{ secrets.PF_TESTING_GH_TOKEN }} - - - name: Ensure MQTT broker + clients (event notification tests) - run: | - if ! command -v mosquitto_sub >/dev/null 2>&1; then - sudo apt-get update - sudo apt-get install -y mosquitto mosquitto-clients - fi - # Make restart deterministic on shared runners: stop any previous - # mosquitto instance/process and drop stale config first. - sudo systemctl stop mosquitto 2>/dev/null || true - sudo pkill -f mosquitto 2>/dev/null || true - sleep 1 - sudo rm -f /etc/mosquitto/conf.d/rustfs-test.conf - sudo mkdir -p /etc/mosquitto/conf.d - printf 'listener 1883 0.0.0.0\nallow_anonymous true\n' | sudo tee /etc/mosquitto/conf.d/rustfs-test.conf >/dev/null - sudo systemctl reset-failed mosquitto 2>/dev/null || true - sudo systemctl start mosquitto - sleep 2 - if ! ss -tln 2>/dev/null | grep -q ':1883'; then - echo "=== mosquitto status ===" - sudo systemctl status mosquitto --no-pager | head -20 || true - echo "=== mosquitto journal ===" - sudo journalctl -u mosquitto -n 30 --no-pager || true - echo "mosquitto not listening on 1883" - exit 1 - fi - - - name: Run tier / event / audit suite - run: | - chmod +x auto-testing/rustfs-tier-test.sh - ARGS=(--all-topologies -y --log-file /tmp/rustfs-tier.log) - if [ -n "${{ inputs.package_url }}" ]; then - ARGS+=(--package-url "${{ inputs.package_url }}") - elif [ -n "${{ inputs.rustfs_version }}" ]; then - ARGS+=(--version "${{ inputs.rustfs_version }}") - else - ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") - fi - ./auto-testing/rustfs-tier-test.sh "${ARGS[@]}" - - - name: Upload test logs - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: rustfs-tier-test-${{ github.run_id }} - path: | - /tmp/rustfs-tier*.log - if-no-files-found: warn - - - name: Notify on failure - if: failure() - run: | - echo "RustFS tier/event/audit suite failed" - echo "See the uploaded log artifact for details." - pool-expansion-test: - name: Pool expansion / decommission test (after tier) + name: Pool expansion / decommission test runs-on: smoke-testing timeout-minutes: 360 - needs: tier-test - if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }} + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} steps: - name: Checkout auto-testing scripts uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 @@ -391,7 +221,7 @@ jobs: - name: Run heal test (write -> outage -> heal -> verify) run: | - ARGS=(--steps 3,4,5,6,7 -y \ + ARGS=(--steps "3,4,5,6,7" -y \ --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \ --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \ --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \ diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml new file mode 100644 index 000000000..2efe2986b --- /dev/null +++ b/.github/workflows/rustfs-s3-compat-test.yml @@ -0,0 +1,159 @@ +name: RustFS S3 Compatibility Test + +on: + workflow_dispatch: + inputs: + rustfs_version: + description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)' + required: false + default: '1.0.0-rc.4-preview.1' + package_url: + description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.' + required: false + type: string + workflow_run: + # Run after the nightly build completes; the nightly deb is what the test installs. + workflows: ["Nightly GNU Build"] + types: [completed] + +permissions: + contents: read + +concurrency: + group: rustfs-shared-functional-tests + cancel-in-progress: false + +defaults: + run: + shell: bash + +env: + RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }} + RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }} + RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }} + RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }} + RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }} + +jobs: + s3-compat-test: + runs-on: smoke-testing + timeout-minutes: 360 + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + steps: + - name: Checkout auto-testing scripts + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + repository: rustfs/auto-testing + ref: main + path: auto-testing + persist-credentials: false + token: ${{ secrets.PF_TESTING_GH_TOKEN }} + + - name: Show environment + run: | + uname -a + jq --version + openssl version + df -h /data | tail -1 + + - name: Cleanup environment (before) + run: | + set -euo pipefail + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + for node in "${NODES[@]}"; do + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -rf /var/log/rustfs + ' + done + + - name: Run S3 compatibility suite + id: test + env: + LOG_FILE: /tmp/rustfs-s3-compat.log + run: | + set -euo pipefail + chmod +x auto-testing/rustfs-s3-compat-test.sh + PACKAGE_URL='${{ inputs.package_url }}' + RUSTFS_VERSION='${{ inputs.rustfs_version }}' + ARGS=(--all-topologies -y --log-file "${LOG_FILE}") + if [ -n "${PACKAGE_URL}" ]; then + ARGS+=(--package-url "${PACKAGE_URL}") + elif [ -n "${RUSTFS_VERSION}" ]; then + ARGS+=(--version "${RUSTFS_VERSION}") + else + ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}") + fi + ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}" + + - name: Generate report + if: always() + env: + LOG_FILE: /tmp/rustfs-s3-compat.log + REPORT_FILE: /tmp/rustfs-s3-compat-report.md + run: | + set -euo pipefail + PACKAGE_URL='${{ inputs.package_url }}' + RUSTFS_VERSION='${{ inputs.rustfs_version }}' + if [ -n "${PACKAGE_URL}" ]; then + PACKAGE_SOURCE="${PACKAGE_URL}" + elif [ -n "${RUSTFS_VERSION}" ]; then + PACKAGE_SOURCE="version ${RUSTFS_VERSION}" + else + PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" + fi + { + echo "# RustFS S3 compatibility test report" + echo "" + echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Trigger: ${{ github.event_name }}" + echo "- Package: ${PACKAGE_SOURCE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" || true + echo '```' + } | tee "${REPORT_FILE}" + cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload report and logs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: rustfs-s3-compat-${{ github.run_id }} + path: | + /tmp/rustfs-s3-compat.log + /tmp/rustfs-s3-compat-report.md + if-no-files-found: warn + + - name: Cleanup environment (after) + if: always() + run: | + set -euo pipefail + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + for node in "${NODES[@]}"; do + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -rf /var/log/rustfs + ' + done + + - name: Notify on failure + if: failure() + run: | + echo "RustFS S3 compatibility suite failed" + echo "See the uploaded report and log artifacts for details." diff --git a/.github/workflows/rustfs-tier-test.yml b/.github/workflows/rustfs-tier-test.yml new file mode 100644 index 000000000..60e05e2ea --- /dev/null +++ b/.github/workflows/rustfs-tier-test.yml @@ -0,0 +1,172 @@ +name: RustFS Tier Test + +on: + workflow_dispatch: + inputs: + rustfs_version: + description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)' + required: false + default: '1.0.0-rc.4-preview.1' + package_url: + description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.' + required: false + type: string + workflow_run: + # Strict shared-environment order: run after KMS test succeeds. + workflows: ["RustFS KMS Test"] + types: [completed] + +permissions: + contents: read + +concurrency: + group: rustfs-shared-functional-tests + cancel-in-progress: false + +defaults: + run: + shell: bash + +env: + RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }} + RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }} + RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }} + RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }} + RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }} + PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + +jobs: + tier-test: + runs-on: smoke-testing + timeout-minutes: 420 + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + steps: + - name: Checkout auto-testing scripts + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + repository: rustfs/auto-testing + ref: main + path: auto-testing + persist-credentials: false + token: ${{ secrets.PF_TESTING_GH_TOKEN }} + + - name: Show environment + run: | + uname -a + jq --version + openssl version + df -h /data | tail -1 + + - name: Cleanup environment (before) + run: | + set -euo pipefail + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + for node in "${NODES[@]}"; do + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms + ' + done + + - name: Ensure MQTT broker + clients + run: | + if ! command -v mosquitto_sub >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y mosquitto mosquitto-clients + fi + sudo mkdir -p /etc/mosquitto/conf.d + printf 'listener 1883 0.0.0.0\nallow_anonymous true\n' | sudo tee /etc/mosquitto/conf.d/rustfs-test.conf >/dev/null + sudo systemctl restart mosquitto + sleep 2 + ss -tln 2>/dev/null | grep -q ':1883' || { echo 'mosquitto not listening on 1883'; exit 1; } + + - name: Run tier suite + id: test + env: + LOG_FILE: /tmp/rustfs-tier.log + run: | + set -euo pipefail + chmod +x auto-testing/rustfs-tier-test.sh + PACKAGE_URL='${{ inputs.package_url }}' + RUSTFS_VERSION='${{ inputs.rustfs_version }}' + ARGS=(--all-topologies -y --log-file "${LOG_FILE}") + if [ -n "${PACKAGE_URL}" ]; then + ARGS+=(--package-url "${PACKAGE_URL}") + elif [ -n "${RUSTFS_VERSION}" ]; then + ARGS+=(--version "${RUSTFS_VERSION}") + else + ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}") + fi + ./auto-testing/rustfs-tier-test.sh "${ARGS[@]}" + + - name: Generate report + if: always() + env: + LOG_FILE: /tmp/rustfs-tier.log + REPORT_FILE: /tmp/rustfs-tier-report.md + run: | + set -euo pipefail + PACKAGE_URL='${{ inputs.package_url }}' + RUSTFS_VERSION='${{ inputs.rustfs_version }}' + if [ -n "${PACKAGE_URL}" ]; then + PACKAGE_SOURCE="${PACKAGE_URL}" + elif [ -n "${RUSTFS_VERSION}" ]; then + PACKAGE_SOURCE="version ${RUSTFS_VERSION}" + else + PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" + fi + { + echo "# RustFS tier test report" + echo "" + echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Trigger: ${{ github.event_name }}" + echo "- Package: ${PACKAGE_SOURCE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" || true + echo '```' + } | tee "${REPORT_FILE}" + cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload report and logs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: rustfs-tier-test-${{ github.run_id }} + path: | + /tmp/rustfs-tier.log + /tmp/rustfs-tier-report.md + if-no-files-found: warn + + - name: Cleanup environment (after) + if: always() + run: | + set -euo pipefail + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + for node in "${NODES[@]}"; do + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms + ' + done + + - name: Notify on failure + if: failure() + run: | + echo "RustFS tier suite failed" + echo "See the uploaded report and log artifacts for details."