diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml index 3ed565d51..e9aad23c2 100644 --- a/.github/workflows/rustfs-heal-test.yml +++ b/.github/workflows/rustfs-heal-test.yml @@ -23,6 +23,12 @@ on: description: 'Reset the nodes after the test (DESTROYS test data/config)' type: boolean default: true + workflow_run: + # Run after the nightly build completes: the nightly deb is what the test + # installs. Heal also runs inside the functional suite; this standalone + # workflow enables manual single-suite runs as well. + workflows: ["Nightly GNU Build"] + types: [completed] permissions: contents: read @@ -49,10 +55,16 @@ jobs: heal-test: runs-on: smoke-testing timeout-minutes: 480 + # Run on manual dispatch, or when the nightly build completed successfully. + # Skipped when nightly failed. + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} steps: - - name: Checkout + - name: Checkout auto-testing scripts uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: + repository: rustfs/auto-testing + ref: main + path: auto-testing persist-credentials: false - name: Show environment @@ -66,8 +78,8 @@ jobs: - name: Reset test environment (before) if: ${{ inputs.cleanup_before != 'false' }} run: | - chmod +x scripts/test/rustfs_heal_test.sh - ./scripts/test/rustfs_heal_test.sh --reset -y + chmod +x auto-testing/rustfs_heal_test.sh + ./auto-testing/rustfs_heal_test.sh --reset -y - name: Install RustFS package & start cluster run: | @@ -77,7 +89,7 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./scripts/test/rustfs_heal_test.sh "${ARGS[@]}" + ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" - name: Preflight checks run: | @@ -87,11 +99,11 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./scripts/test/rustfs_heal_test.sh "${ARGS[@]}" + ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" - name: Run heal test (write -> outage -> heal -> verify) run: | - ./scripts/test/rustfs_heal_test.sh \ + ./auto-testing/rustfs_heal_test.sh \ --steps "3,4,5,6,7" -y \ --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \ --stop-node-gb "${{ inputs.stop_node_gb }}" \ @@ -111,7 +123,7 @@ jobs: - name: Reset test environment (after) if: ${{ always() && inputs.cleanup_after != 'false' }} run: | - ./scripts/test/rustfs_heal_test.sh --reset -y + ./auto-testing/rustfs_heal_test.sh --reset -y - name: Notify on failure if: failure() diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml index c9ab9764f..2a9947f60 100644 --- a/.github/workflows/rustfs-performance-test.yml +++ b/.github/workflows/rustfs-performance-test.yml @@ -89,11 +89,13 @@ jobs: # Skipped when nightly failed. if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} steps: - - name: Checkout + - name: Checkout auto-testing scripts uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: + repository: rustfs/auto-testing + ref: main + path: auto-testing persist-credentials: false - ref: ${{ github.event.workflow_run.head_sha || github.ref }} - name: Show environment run: | @@ -105,8 +107,8 @@ jobs: - name: Reset test environment (before) if: ${{ inputs.cleanup_before != 'false' }} run: | - chmod +x scripts/test/rustfs_performance_test.sh - ./scripts/test/rustfs_performance_test.sh --step 1 -y + chmod +x auto-testing/rustfs_performance_test.sh + ./auto-testing/rustfs_performance_test.sh --step 1 -y - name: Install RustFS package & start cluster (4x4) run: | @@ -116,7 +118,7 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./scripts/test/rustfs_performance_test.sh "${ARGS[@]}" + ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" - name: Preflight checks run: | @@ -126,7 +128,7 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./scripts/test/rustfs_performance_test.sh "${ARGS[@]}" + ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" - name: Run benchmark (GET/PUT/MIXED) id: benchmark @@ -135,7 +137,7 @@ jobs: # Manual dispatch can restrict method(s)/size(s). export WARP_METHODS="${{ inputs.test_method }}" export WARP_SIZES="${{ inputs.object_size }}" - ./scripts/test/rustfs_performance_test.sh \ + ./auto-testing/rustfs_performance_test.sh \ --step 5 -y \ --warp-duration "${{ inputs.warp_duration || '5m' }}" \ --warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \ @@ -144,7 +146,7 @@ jobs: - name: Analyze results if: ${{ steps.benchmark.conclusion == 'success' }} run: | - ./scripts/test/rustfs_performance_test.sh --step 6 -y + ./auto-testing/rustfs_performance_test.sh --step 6 -y - name: Post results to backlog issue if: ${{ steps.benchmark.conclusion == 'success' }} @@ -193,7 +195,7 @@ jobs: - name: Reset test environment (after) if: ${{ always() && inputs.cleanup_after != 'false' }} run: | - ./scripts/test/rustfs_performance_test.sh --step 7 -y + ./auto-testing/rustfs_performance_test.sh --step 7 -y - name: Notify on failure if: failure() diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml index 18814b697..1f526c8ef 100644 --- a/.github/workflows/rustfs-pool-expand-test.yml +++ b/.github/workflows/rustfs-pool-expand-test.yml @@ -1,4 +1,4 @@ -name: RustFS Pool Expansion / Decommission Test +name: RustFS Functional Test Suite (S3/KMS/tier/pool/heal) on: workflow_dispatch: @@ -47,15 +47,16 @@ on: type: boolean default: true workflow_run: - # Run after the nightly build completes: pool expansion first, then heal. + # Run after the nightly build completes: S3 -> KMS -> tier -> pool -> heal. workflows: ["Nightly GNU Build"] types: [completed] permissions: contents: read -# Only one pool-expansion test at a time: the workflow mutates a shared -# test environment, so concurrent runs must not clobber each other. +# Only one test run at a time: every job mutates the same shared test +# environment (vm000/vm001/vm002), so concurrent runs must not clobber each +# other. Jobs inside a run are chained with needs to serialize them. concurrency: group: rustfs-pool-expansion-test cancel-in-progress: false @@ -75,18 +76,172 @@ env: RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }} jobs: - pool-expansion-test: + # All test scripts live in rustfs/auto-testing; rustfs stores only this + # workflow. Each job checks out auto-testing before running. + + s3-compat-test: + name: S3 compatibility test runs-on: smoke-testing - timeout-minutes: 360 - # Run on manual dispatch, or when the nightly build completed successfully - # (its deb is what the tests install). Skipped when nightly failed. + timeout-minutes: 240 if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} steps: - - name: Checkout + - name: Checkout auto-testing scripts uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: + repository: rustfs/auto-testing + ref: main + path: auto-testing + persist-credentials: false + + - name: Run S3 compatibility suite + run: | + chmod +x auto-testing/rustfs-s3-compat-test.sh + ARGS=(--all-topologies -y --log-file /tmp/rustfs-s3-compat.log) + if [ -n "${{ inputs.package_url }}" ]; then + ARGS+=(--package-url "${{ inputs.package_url }}") + elif [ -n "${{ inputs.rustfs_version }}" ]; then + ARGS+=(--version "${{ inputs.rustfs_version }}") + else + ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") + fi + ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}" + + - name: Upload test logs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: rustfs-s3-compat-${{ github.run_id }} + path: | + /tmp/rustfs-s3-compat*.log + if-no-files-found: warn + + - name: Notify on failure + if: failure() + run: | + echo "RustFS S3 compatibility suite failed" + echo "See the uploaded log artifact for details." + + kms-test: + name: KMS test (after S3) + runs-on: smoke-testing + timeout-minutes: 360 + needs: s3-compat-test + if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }} + steps: + - name: Checkout auto-testing scripts + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + repository: rustfs/auto-testing + ref: main + path: auto-testing + persist-credentials: false + + - name: Ensure docker (Vault container) + run: | + if ! command -v docker >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y docker.io + fi + sudo systemctl enable --now docker + docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1 + + - name: Run KMS suite + run: | + chmod +x auto-testing/rustfs-kms-test.sh + ARGS=(--all-topologies --backends local,vault-kv2 -y --log-file /tmp/rustfs-kms.log) + if [ -n "${{ inputs.package_url }}" ]; then + ARGS+=(--package-url "${{ inputs.package_url }}") + elif [ -n "${{ inputs.rustfs_version }}" ]; then + ARGS+=(--version "${{ inputs.rustfs_version }}") + else + ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") + fi + ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}" + + - name: Upload test logs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: rustfs-kms-test-${{ github.run_id }} + path: | + /tmp/rustfs-kms*.log + if-no-files-found: warn + + - name: Notify on failure + if: failure() + run: | + echo "RustFS KMS suite failed" + echo "See the uploaded log artifact for details." + + tier-test: + name: Tier / event / audit test (after KMS) + runs-on: smoke-testing + timeout-minutes: 360 + needs: kms-test + if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }} + steps: + - name: Checkout auto-testing scripts + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + repository: rustfs/auto-testing + ref: main + path: auto-testing + persist-credentials: false + + - name: Ensure MQTT broker + clients (event notification tests) + run: | + if ! command -v mosquitto_sub >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y mosquitto mosquitto-clients + fi + sudo mkdir -p /etc/mosquitto/conf.d + printf 'listener 1883 0.0.0.0\nallow_anonymous true\n' | sudo tee /etc/mosquitto/conf.d/rustfs-test.conf >/dev/null + sudo systemctl restart mosquitto + sleep 2 + ss -tln 2>/dev/null | grep -q ':1883' || { echo "mosquitto not listening on 1883"; exit 1; } + + - name: Run tier / event / audit suite + run: | + chmod +x auto-testing/rustfs-tier-test.sh + ARGS=(--all-topologies -y --log-file /tmp/rustfs-tier.log) + if [ -n "${{ inputs.package_url }}" ]; then + ARGS+=(--package-url "${{ inputs.package_url }}") + elif [ -n "${{ inputs.rustfs_version }}" ]; then + ARGS+=(--version "${{ inputs.rustfs_version }}") + else + ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") + fi + ./auto-testing/rustfs-tier-test.sh "${ARGS[@]}" + + - name: Upload test logs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: rustfs-tier-test-${{ github.run_id }} + path: | + /tmp/rustfs-tier*.log + if-no-files-found: warn + + - name: Notify on failure + if: failure() + run: | + echo "RustFS tier/event/audit suite failed" + echo "See the uploaded log artifact for details." + + pool-expansion-test: + name: Pool expansion / decommission test (after tier) + runs-on: smoke-testing + timeout-minutes: 360 + needs: tier-test + if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }} + steps: + - name: Checkout auto-testing scripts + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + repository: rustfs/auto-testing + ref: main + path: auto-testing persist-credentials: false - ref: ${{ github.event.workflow_run.head_sha || github.ref }} - name: Show environment run: | @@ -99,8 +254,8 @@ jobs: - name: Reset test environment (before) if: ${{ inputs.cleanup_before != 'false' }} run: | - chmod +x scripts/test/rustfs_pool_expand.sh - ./scripts/test/rustfs_pool_expand.sh --reset -y + chmod +x auto-testing/rustfs_pool_expand.sh + ./auto-testing/rustfs_pool_expand.sh --reset -y - name: Install RustFS package & start first pool run: | @@ -112,7 +267,7 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./scripts/test/rustfs_pool_expand.sh "${ARGS[@]}" + ./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}" - name: Preflight checks run: | @@ -124,7 +279,7 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./scripts/test/rustfs_pool_expand.sh "${ARGS[@]}" + ./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}" - name: Run pool expansion & decommission test id: pool_test @@ -137,7 +292,7 @@ jobs: STEPS="$STEPS,9" fi fi - ./scripts/test/rustfs_pool_expand.sh \ + ./auto-testing/rustfs_pool_expand.sh \ --steps "$STEPS" --with-warp -y \ --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \ --storage-threshold "${{ inputs.storage_threshold || '50' }}" \ @@ -157,7 +312,7 @@ jobs: - name: Reset test environment (after) if: ${{ always() && inputs.cleanup_after != 'false' }} run: | - ./scripts/test/rustfs_pool_expand.sh --reset -y + ./auto-testing/rustfs_pool_expand.sh --reset -y - name: Notify on failure if: failure() @@ -175,17 +330,19 @@ jobs: needs: pool-expansion-test if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }} steps: - - name: Checkout + - name: Checkout auto-testing scripts uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: + repository: rustfs/auto-testing + ref: main + path: auto-testing persist-credentials: false - ref: ${{ github.event.workflow_run.head_sha || github.ref }} - name: Reset test environment (before) if: ${{ inputs.cleanup_before != 'false' }} run: | - chmod +x scripts/test/rustfs_heal_test.sh - ./scripts/test/rustfs_heal_test.sh --reset -y + chmod +x auto-testing/rustfs_heal_test.sh + ./auto-testing/rustfs_heal_test.sh --reset -y - name: Install RustFS package & start cluster run: | @@ -195,7 +352,7 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./scripts/test/rustfs_heal_test.sh "${ARGS[@]}" + ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" - name: Preflight checks run: | @@ -205,11 +362,11 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./scripts/test/rustfs_heal_test.sh "${ARGS[@]}" + ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" - name: Run heal test (write -> outage -> heal -> verify) run: | - ./scripts/test/rustfs_heal_test.sh \ + ./auto-testing/rustfs_heal_test.sh \ --steps 3,4,5,6,7 -y \ --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \ --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \ @@ -229,7 +386,7 @@ jobs: - name: Reset test environment (after) if: ${{ always() && inputs.cleanup_after != 'false' }} run: | - ./scripts/test/rustfs_heal_test.sh --reset -y + ./auto-testing/rustfs_heal_test.sh --reset -y - name: Notify on failure if: failure() diff --git a/scripts/test/rustfs_heal_test.md b/scripts/test/rustfs_heal_test.md deleted file mode 100644 index a0b4c0e54..000000000 --- a/scripts/test/rustfs_heal_test.md +++ /dev/null @@ -1,106 +0,0 @@ -# RustFS Heal Test - -Node-outage heal test driven by -[`scripts/test/rustfs_heal_test.sh`](rustfs_heal_test.sh), based on the -Obsidian note "RustFS Heal 测试步骤". Uses the same 3-node test environment as -the pool expansion test (`vm000 vm001 vm002`). - -All status checks talk to the RustFS admin API directly (SigV4-signed, -`jq` assertions), no `rc` required. - -## What it does - -1. Downloads the `.deb` package on all nodes (release tag or a direct URL such - as the nightly/R2 package). -2. Installs it, writes the 3x4 config - (`http://rustfs-node{1...3}:9000/data/rustfs{1...4}/mnmd`), starts all - three nodes simultaneously, verifies the cluster is up. -3. Writes data with `warp` while monitoring disk usage on the surviving nodes - (`df -B1G | grep /data/rustfs`): - - when both surviving nodes reach `STOP_NODE_AT_GB` (default 15 GiB), stop - the outage node (`vm002`, `OUTAGE_NODE_INDEX=2`); - - keep writing until both surviving nodes reach `WARP_STOP_AT_GB` - (default 40 GiB), then stop warp. -4. Restarts the outage node. -5. Starts cluster heal: `POST /rustfs/admin/v3/heal/` with body - `{"recursive":true}` (retried, returns a `clientToken`). -6. Monitors the heal task via `POST /rustfs/admin/v3/heal/?clientToken=` - until the server verdict is a terminal success (`finished`/`completed`) with - `objects_failed == 0`. -7. Result analysis: heal stats (scanned/healed/failed), an **S3 read-back - verification** of the written objects (list the test bucket and GET a - sample — every read must succeed), per-node disk usage (observability), - pass/fail verdict. - -Success is the server's own scan/repair verdict (heal finished, 0 failed) -**plus** an end-to-end data read-back; per-node disk usage is logged as -observability, not a pass gate (EC distributes different shards per node, so a -fixed per-node GB target is not a meaningful invariant). - -## Self-hosted runner prerequisites - -- Register the admin host (e.g. `heal`) as a runner with the - `smoke-testing` label. -- Install `jq`, `openssl`, `curl` and `warp` on the runner. `rc` is **not** - required. -- The runner user must be able to SSH to `vm000/vm001/vm002` without a - password prompt; nodes need passwordless `sudo` for the SSH user and - resolvable `rustfs-node*` hostnames. -- Admin API credentials need the `admin:server-info`, `admin:heal` and - `admin:rebalance` actions. - -## Configuration - -Same repository secrets/variables as the pool expansion workflow: - -| Kind | Name | Purpose | -| ------ | --------------------- | ---------------------------------------------- | -| Secret | `RUSTFS_ACCESS_KEY` | RustFS access key (default `rustfs@test`) | -| Secret | `RUSTFS_SECRET_KEY` | RustFS secret key (default `rustfs@test`) | -| Var | `RUSTFS_API_ENDPOINT` | Admin API endpoint, e.g. `http://127.0.0.1:9000` (`RUSTFS_RC_ENDPOINT` fallback) | -| Var | `RUSTFS_NODES` | `vm000 vm001 vm002` | -| Var | `RUSTFS_SSH_USER` | `azureuser` | -| Var | `RUSTFS_NIGHTLY_PACKAGE_URL` | Default nightly deb URL (defaults to the R2 `latest` alias) | - -## Workflow inputs - -| Input | Default | Meaning | -| ---------------- | ------- | ----------------------------------------- | -| `package_url` | nightly | Direct `.deb` URL; empty = latest nightly | -| `stop_node_gb` | `15` | Stop outage node at N GiB on survivors | -| `warp_stop_gb` | `40` | Stop warp at N GiB on survivors | -| `cleanup_before` | `true` | Reset nodes before the test | -| `cleanup_after` | `true` | Reset nodes after the test | - -> ⚠️ `--reset` purges the `rustfs` package and deletes the data directories on -> all nodes. Only run against a dedicated test environment. - -## Manual usage - -```bash -./scripts/test/rustfs_heal_test.sh --all -y \ - --package-url https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb \ - --endpoint http://127.0.0.1:9000 - -./scripts/test/rustfs_heal_test.sh --steps 5,6,7 -./scripts/test/rustfs_heal_test.sh --reset -y -``` - -## Known issues - -- Nightly builds gate pool/rebalance activation on a live fleet capability - proof (rustfs/backlog#2031); the script retries heal/rebalance starts and - prints a hint when the signature appears. -- The cluster-level `GET /rustfs/admin/v3/background-heal/status` aggregator - returns 501 in the single-pool 3x4 topology (no notification system), so the - script monitors the started heal task via its `clientToken` instead. -- The heal task may report `progress: null` while running; the script logs - this as evidence (rustfs/backlog#2035) rather than coercing it to zero, and - reads the canonical camelCase progress fields - (`objectsScanned`/`objectsHealed`/`objectsFailed`/`progressPercentage`) with - a snake_case fallback. -- The server-side per-task heal timeout defaults to 5 minutes; the script - writes `RUSTFS_HEAL_TASK_TIMEOUT_SECS=21600` (6h) into the node config so a - multi-tens-of-GiB heal can finish. The background scanner is disabled - (`RUSTFS_HEAL_AUTO_HEAL_ENABLE=false`) so the explicit heal is the only - repair mechanism and the outage effect stays observable. diff --git a/scripts/test/rustfs_heal_test.sh b/scripts/test/rustfs_heal_test.sh deleted file mode 100755 index b35dbca2a..000000000 --- a/scripts/test/rustfs_heal_test.sh +++ /dev/null @@ -1,1158 +0,0 @@ -#!/usr/bin/env bash -# -# rustfs-heal-test.sh -# RustFS node-outage heal test script -# -# Based on the Obsidian note "RustFS Heal 测试步骤". Full workflow: -# 1. Download the RustFS package on all nodes -# 2. Install RustFS on all nodes (dpkg -i), write the 3x4 single-pool -# config, start all nodes in parallel and verify the topology via the -# admin API -# 3. Write data (warp); when the surviving nodes reach STOP_NODE_AT_GB -# stop vm002 (simulated node outage), keep writing until the surviving -# nodes reach WARP_STOP_AT_GB -# 4. Restart vm002 (the node that was offline while data was written) -# 5. Start cluster heal (POST /rustfs/admin/v3/heal/ {"recursive":true}) -# 6. Monitor the heal task (POST /rustfs/admin/v3/heal/?clientToken=...) -# until the server verdict is a terminal success (finished, 0 failed), -# then verify the data is readable back from the cluster -# 7. Result analysis: heal stats, per-node disk usage, success verdict -# -# The script is driven from an admin host (e.g. a jumpbox or a GitHub -# self-hosted runner) and operates on the target nodes over SSH. -# -# Usage: -# ./rustfs-heal-test.sh --all # run all steps 1-7 -# ./rustfs-heal-test.sh --step 5 # run a single step -# ./rustfs-heal-test.sh --steps 3,4,5,6,7 # run selected steps -# ./rustfs-heal-test.sh --all --dry-run # preview only -# ./rustfs-heal-test.sh --all -y --package-url -# -# Notes: -# - SSH user defaults to azureuser (passwordless sudo on the nodes); -# pass --ssh-user root if your nodes accept root login. -# - The script talks to the RustFS admin API directly with SigV4-signed -# requests (no rc required). jq, openssl and curl must be installed on the -# admin host; warp is needed for the data-write step. -# - Credentials default to rustfs@test / rustfs@test (the config written on -# the nodes); override via RUSTFS_ACCESS_KEY / RUSTFS_SECRET_KEY. -# -set -Eeuo pipefail - -# ==================== Configuration (adjust to your environment) ==================== - -# Target nodes; pool N is enabled by NODES[N-1] (index order matters) -if [ -n "${RUSTFS_NODES:-}" ]; then - read -r -a NODES <<<"${RUSTFS_NODES}" -else - NODES=(vm000 vm001 vm002) -fi - -# SSH user configured on the nodes (azureuser has passwordless sudo on heal's nodes) -SSH_USER="${RUSTFS_SSH_USER:-azureuser}" -SSH_PORT="${RUSTFS_SSH_PORT:-22}" -SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new -p "${SSH_PORT}") - -# Package: version is a GitHub release tag such as "1.0.0-rc.3". -# PACKAGE_URL is derived from RUSTFS_VERSION unless --package-url / PACKAGE_URL is given. -RUSTFS_VERSION="${RUSTFS_VERSION:-1.0.0-rc.3}" -PACKAGE_URL="${PACKAGE_URL:-}" -ARCH="${RUSTFS_ARCH:-amd64}" -PACKAGES_DIR="/home/rustfs/packages" -PACKAGE_FILE="rustfs.deb" -PACKAGE_SHA256="${PACKAGE_SHA256:-}" - -# Admin API endpoint and credentials (SigV4-signed requests, no rc needed). -# RUSTFS_RC_ENDPOINT is honoured as a fallback for existing setups. -API_ENDPOINT="${RUSTFS_API_ENDPOINT:-${RUSTFS_RC_ENDPOINT:-http://10.0.0.7:9000}}" -ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfs@test}" -SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfs@test}" -SIGV4_REGION="us-east-1" -SIGV4_SERVICE="s3" -API_REQUEST_TIMEOUT=60 -# admin_api writes the HTTP status here so callers can read it after the -# command-substitution subshell exits ($$ is identical inside the subshell). -ADMIN_API_CODE_FILE="${TMPDIR:-/tmp}/rustfs-heal-test-api-code.$$" - -# 3x4 topology (3 nodes x 4 disks each), same expression on every node: -# http://rustfs-node{1...3}:9000/data/rustfs{1...4}/mnmd -VOLUMES="http://rustfs-node{1...3}:9000/data/rustfs{1...4}/mnmd" - -# Node that is stopped during the write phase (index into NODES; 2 = vm002) -OUTAGE_NODE_INDEX="${RUSTFS_OUTAGE_NODE_INDEX:-2}" - -# RustFS service configuration (written to /etc/default/rustfs) -RUSTFS_CONFIG_FILE="/etc/default/rustfs" -RUSTFS_SERVICE="rustfs" -RUSTFS_PACKAGE_NAME="rustfs" -RUSTFS_USER="rustfs" -RUSTFS_ADDRESS=":9000" -RUSTFS_CONSOLE_ADDRESS=":9001" -RUSTFS_CONSOLE_ENABLE=true -RUSTFS_OBS_LOGGER_LEVEL=error -RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/" - -# Data writing & monitoring (step 3) -WARP_BUCKET="test-10mb" -WARP_OBJ_SIZE="100MiB" -WARP_CONCURRENT=32 -# Warp log path; empty = auto-created unique temp file (the runner user may -# not be able to write a shared /tmp path owned by another user). -WARP_LOG_FILE="${RUSTFS_WARP_LOG_FILE:-}" -# Disk-usage thresholds (per surviving node, GiB, via df -B1G | grep /data/rustfs) -STOP_NODE_AT_GB="${RUSTFS_STOP_NODE_AT_GB:-15}" # stop the outage node when surviving nodes reach this -WARP_STOP_AT_GB="${RUSTFS_WARP_STOP_AT_GB:-40}" # stop warp when surviving nodes reach this -POLL_INTERVAL=15 # status polling interval (seconds) - -# Timeouts (seconds) -SERVICE_TIMEOUT=300 -WARP_TIMEOUT=3600 # max time waiting for the write phase thresholds -HEAL_TIMEOUT=86400 # max time waiting for heal to complete -HEAL_START_RETRIES=6 # heal start retries (fleet capability proof timing) -HEAL_START_RETRY_DELAY=20 # delay between heal start retries (seconds) -# Per-task heal timeout on the server (default is 5 minutes, far too short -# for healing tens of GiB); written into /etc/default/rustfs. -HEAL_TASK_TIMEOUT_SECS="${RUSTFS_HEAL_TASK_TIMEOUT_SECS:-21600}" -# Disable the background scanner so the explicit heal is the only repair -# mechanism (otherwise automatic repairs can mask the outage effect). -HEAL_AUTO_HEAL_ENABLE="${RUSTFS_HEAL_AUTO_HEAL_ENABLE:-false}" - -# ==================== Runtime options (set by CLI) ==================== -DRY_RUN=0 -ASSUME_YES=0 -SKIP_DOWNLOAD=0 -PREFLIGHT=0 -RESET=0 -LOG_FILE="" -SELECTED_STEPS=() -HEAL_CLIENT_TOKEN="" - -# ==================== Helpers ==================== - -log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; } -warn() { printf '\033[1;33m[WARN]\033[0m %s\n' "$*"; } -die() { printf '\033[1;31m[ERROR]\033[0m %s\n' "$*" >&2; exit 1; } - -confirm() { - # $1: prompt text; bypassed with --yes - if [ "${ASSUME_YES}" -eq 1 ]; then - return 0 - fi - printf '\033[1;33m[CONFIRM]\033[0m %s (y/N) ' "$1" - read -r answer - case "${answer}" in - y|Y|yes|YES) return 0 ;; - *) die "cancelled" ;; - esac -} - -need_cmd() { - # $1: command name; $2: description - if [ "${DRY_RUN}" -eq 1 ]; then return 0; fi - command -v "$1" >/dev/null 2>&1 || die "missing command: $1 ($2); install it first" -} - -# ==================== Admin API (SigV4-signed) ==================== - -sha256_hex() { - # $1: ascii data - printf '%s' "$1" | openssl dgst -sha256 -hex 2>/dev/null | awk '{print $NF}' -} - -hmac_sha256_hex() { - # $1: key in hex, $2: ascii data - printf '%s' "$2" | openssl dgst -sha256 -mac HMAC -macopt "hexkey:$1" -hex 2>/dev/null | awk '{print $NF}' -} - -hex_of_ascii() { - # $1: ascii string -> hex - printf '%s' "$1" | od -An -vtx1 | tr -d ' \n' -} - -# Sort a "k=v&k2=v2" query string by key (values are used verbatim, matching -# the RustFS signer). The same canonical form is used in the URL and in the -# SigV4 canonical request so the server-side verification always agrees. -canonical_query() { - local q="$1" - [ -z "${q}" ] && return 0 - # The trailing newline matters: `while read` drops the last item when the - # input has no final newline (e.g. "a=1&b=2" without a trailing '&'). - printf '%s\n' "${q}" | tr '&' '\n' | while IFS= read -r pair; do - printf '%s=%s\n' "${pair%%=*}" "${pair#*=}" - done | sort | paste -sd '&' - -} - -# Issue an admin API request. Prints the response body on stdout and writes the -# HTTP status (000 on transport failure) to ${ADMIN_API_CODE_FILE}. -admin_api() { - # $1: method, $2: path, $3: query string, $4: optional JSON body, - # $5: "discard" to skip body capture (only the status code matters) - local method="$1" path="$2" query="$3" body="${4:-}" discard="${5:-}" - local amz_date date_stamp host_port - local canonical_headers signed_headers canonical_request string_to_sign - local scope k_date k_region k_service k_signing signature auth - local url tmp code - - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: admin API ${method} ${API_ENDPOINT}${path}${query:+?${query}}" - printf '200' > "${ADMIN_API_CODE_FILE}" - return 0 - fi - - host_port="${API_ENDPOINT#*://}" - host_port="${host_port%%/*}" - - amz_date="$(date -u +%Y%m%dT%H%M%SZ)" - date_stamp="${amz_date:0:8}" - query="$(canonical_query "${query}")" - - canonical_headers="host:${host_port} -x-amz-content-sha256:UNSIGNED-PAYLOAD -x-amz-date:${amz_date} -" - signed_headers="host;x-amz-content-sha256;x-amz-date" - canonical_request="${method} -${path} -${query} -${canonical_headers} -${signed_headers} -UNSIGNED-PAYLOAD" - - scope="${date_stamp}/${SIGV4_REGION}/${SIGV4_SERVICE}/aws4_request" - string_to_sign="AWS4-HMAC-SHA256 -${amz_date} -${scope} -$(sha256_hex "${canonical_request}")" - - k_date="$(hmac_sha256_hex "$(hex_of_ascii "AWS4${SECRET_KEY}")" "${date_stamp}")" - k_region="$(hmac_sha256_hex "${k_date}" "${SIGV4_REGION}")" - k_service="$(hmac_sha256_hex "${k_region}" "${SIGV4_SERVICE}")" - k_signing="$(hmac_sha256_hex "${k_service}" "aws4_request")" - signature="$(hmac_sha256_hex "${k_signing}" "${string_to_sign}")" - auth="AWS4-HMAC-SHA256 Credential=${ACCESS_KEY}/${scope}, SignedHeaders=${signed_headers}, Signature=${signature}" - - url="http://${host_port}${path}${query:+?${query}}" - tmp="$(mktemp)" - local curl_body=() - if [ -n "${body}" ]; then - curl_body=(-d "${body}" -H "Content-Type: application/json") - fi - local out_file="${tmp}" - [ "${discard}" = "discard" ] && out_file="/dev/null" - code="$(curl -sS --max-time "${API_REQUEST_TIMEOUT}" -o "${out_file}" -w '%{http_code}' \ - -H "Host: ${host_port}" \ - -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ - -H "x-amz-date: ${amz_date}" \ - -H "Authorization: ${auth}" \ - "${curl_body[@]}" -X "${method}" "${url}")" || code="000" - printf '%s' "${code}" > "${ADMIN_API_CODE_FILE}" - if [ "${discard}" != "discard" ]; then - cat "${tmp}" - fi - rm -f "${tmp}" -} - -# Read the HTTP status written by the last admin_api call -admin_api_code() { - cat "${ADMIN_API_CODE_FILE}" -} - -# Run an admin API request and fail loudly (with the full response body) on a -# non-success status. Prints the body on stdout. -admin_api_assert() { - # $1: method, $2: path, $3: query, $4: description - local method="$1" path="$2" query="$3" desc="$4" body code - body="$(admin_api "${method}" "${path}" "${query}")" - code="$(admin_api_code)" - if [ "${code}" != "200" ] && [ "${code}" != "201" ] && [ "${code}" != "204" ]; then - printf '\033[1;31m[ERROR]\033[0m %s failed (HTTP %s)\n' "${desc}" "${code}" >&2 - printf '%s\n' "${body}" >&2 - die "admin API ${method} ${path} -> HTTP ${code}" - fi - printf '%s' "${body}" -} - -# Build the GitHub release download URL from a release tag. -# "1.0.0-rc.3" -> https://github.com/rustfs/rustfs/releases/download/1.0.0-rc.3/rustfs_1.0.0.rc.3_amd64.deb -build_package_url() { - local tag="${RUSTFS_VERSION#v}" asset - asset="${tag//-/.}" - printf 'https://github.com/rustfs/rustfs/releases/download/%s/rustfs_%s_%s.deb' "${tag}" "${asset}" "${ARCH}" -} - -resolve_package_url() { - if [ -n "${PACKAGE_URL}" ]; then - printf '%s' "${PACKAGE_URL}" - else - build_package_url - fi -} - -# Run a remote script on a single node (script is read from stdin) -run_remote() { - local node="$1" - local script - script="$(cat)" - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: ssh ${SSH_USER}@${node} <<'REMOTE'" - printf '%s\n' "${script}" | sed 's/^/ | /' - log "DRY-RUN: ----" - return 0 - fi - log "==> ${node}: executing remote script" - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'bash -s' <<<"${script}" -} - -# Run the same remote script on all nodes in parallel (script from stdin) -run_remote_all() { - local script - script="$(cat)" - local pids=() i=0 fail=0 - for node in "${NODES[@]}"; do - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: ssh ${SSH_USER}@${node} <<'REMOTE'" - printf '%s\n' "${script}" | sed 's/^/ | /' - log "DRY-RUN: ----" - else - log "==> ${node}: executing remote script" - ( ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'bash -s' <<<"${script}" ) & - pids[$i]=$! - i=$((i+1)) - fi - done - if [ "${#pids[@]}" -gt 0 ]; then - for pid in "${pids[@]}"; do - wait "${pid}" || fail=1 - done - fi - [ "${fail}" -eq 0 ] || die "one or more nodes failed" -} - -# Wait for the systemd service to become active on a node -wait_service_active() { - local node="$1" waited=0 - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: waiting for ${RUSTFS_SERVICE} on ${node} to become active" - return 0 - fi - while [ "${waited}" -lt "${SERVICE_TIMEOUT}" ]; do - if ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \ - "systemctl is-active --quiet ${RUSTFS_SERVICE}" 2>/dev/null; then - log "${node}: ${RUSTFS_SERVICE} is active" - return 0 - fi - sleep 5 - waited=$((waited + 5)) - done - diagnose_node_start_failure "${node}" - die "${node}: timed out waiting for ${RUSTFS_SERVICE} (${SERVICE_TIMEOUT}s)" -} - -# Known server-side issues the test can hit. Format: -# "||" -KNOWN_SERVER_ISSUES=( - "pool activation requires a live fleet capability proof|rustfs/backlog#2031|multi-pool cold start with rebalance metadata fails on nightly builds; server fix pending, no script workaround" -) - -# Print a hint when $1 matches a known server-side issue signature. -hint_server_issue() { - local text="$1" entry sig tracking hint - for entry in "${KNOWN_SERVER_ISSUES[@]}"; do - sig="${entry%%|*}" - tracking="${entry#*|}" - hint="${tracking#*|}" - tracking="${tracking%%|*}" - if printf '%s' "${text}" | grep -qiF "${sig}"; then - printf '\033[1;33m[KNOWN SERVER ISSUE]\033[0m %s (%s): %s\n' "${sig}" "${tracking}" "${hint}" >&2 - return 0 - fi - done - return 1 -} - -# Fetch the journal tail from a node whose service failed to start and -# annotate known server-side issues. -diagnose_node_start_failure() { - local node="$1" journal - journal="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \ - "SUDO=\"\"; [ \"\$(id -u)\" -ne 0 ] && SUDO=\"sudo -n\"; \${SUDO} journalctl -u ${RUSTFS_SERVICE} --no-pager -n 60 2>/dev/null || true")" - printf '%s\n' "--- ${node}: ${RUSTFS_SERVICE} journal (last 60 lines) ---" >&2 - printf '%s\n' "${journal}" >&2 - hint_server_issue "${journal}" || true -} - -# Generate the /etc/default/rustfs content -rustfs_config_body() { - local volumes="$1" - cat < /data/rustfs1/mnmd /data/rustfs2/mnmd /data/rustfs3/mnmd /data/rustfs4/mnmd -volume_dirs() { - local volumes="$1" expr path prefix suffix i start end - for expr in ${volumes}; do - path="${expr#*://}" - path="${path#*/}" - path="/${path}" - if [[ "${path}" =~ \{([0-9]+)\.\.\.([0-9]+)\} ]]; then - start="${BASH_REMATCH[1]}" - end="${BASH_REMATCH[2]}" - prefix="${path%%\{*}" - suffix="${path#*\}}" - for ((i=start; i<=end; i++)); do - printf '%s%s%s\n' "${prefix}" "${i}" "${suffix}" - done - else - printf '%s\n' "${path}" - fi - done -} - -# Ensure the volume directories exist on a node and are owned by the service user -ensure_volume_dirs() { - local node="$1" volumes="$2" - local -a dirs - dirs=() - while IFS= read -r d; do dirs+=("${d}"); done < <(volume_dirs "${volumes}" | sort -u) - log "${node}: ensuring data dirs exist and are owned by ${RUSTFS_USER} (${dirs[*]})" - { - printf 'set -euo pipefail\n' - printf 'SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"\n' - for d in "${dirs[@]}"; do - printf '${SUDO} mkdir -p %s\n' "${d}" - printf '${SUDO} chown -R %s:%s %s\n' "${RUSTFS_USER}" "${RUSTFS_USER}" "${d}" - done - } | run_remote "${node}" -} - -# Write the RustFS config on a single node (with a timestamped backup first) -write_rustfs_config() { - local node="$1" volumes="$2" - local body - body="$(rustfs_config_body "${volumes}")" - log "${node}: writing config ${RUSTFS_CONFIG_FILE} (volumes=${volumes})" - { - printf 'set -euo pipefail\n' - printf 'SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"\n' - printf 'if [ -f %s ]; then ${SUDO} cp -a %s %s.bak.$(date +%%Y%%m%%d%%H%%M%%S); fi\n' \ - "${RUSTFS_CONFIG_FILE}" "${RUSTFS_CONFIG_FILE}" "${RUSTFS_CONFIG_FILE}" - printf '%s tee %s >/dev/null </dev/null) - printf '%s' "${total}" -} - -# Read a heal-progress counter. The heal API serializes progress in camelCase -# (objectsScanned / objectsHealed / objectsFailed / progressPercentage); keep a -# snake_case fallback for older builds. Prints "null" when the field is absent. -heal_progress_field() { - local body="$1" camel="$2" snake="$3" - printf '%s' "${body}" | jq -r --arg c "${camel}" --arg s "${snake}" ' - (.progress // null) as $p - | if $p == null then "null" else (($p[$c] // $p[$s] // null) | if . == null then "null" else tostring end) end' -} - -# Sample data verification after heal: list the test bucket and GET a sample of -# objects. Every GET must return 200 — this is the end-to-end proof that the -# cluster can still reconstruct the data after repair. -verify_data_readable() { - local list body code keys key count checked ok - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: S3 read-back verification of ${WARP_BUCKET}" - return 0 - fi - list="$(admin_api GET "/${WARP_BUCKET}" "list-type=2&max-keys=1000")" - code="$(admin_api_code)" - if [ "${code}" != "200" ]; then - printf '\033[1;31m[ERROR]\033[0m bucket list failed (HTTP %s): %s\n' "${code}" "${list}" >&2 - return 1 - fi - # sed -n '1,20p' reads the whole stream (unlike head, which closes the pipe - # early and SIGPIPEs grep/sed under pipefail). - keys="$(printf '%s' "${list}" | grep -oE '[^<]+' | sed 's###g' | sed -n '1,20p')" - count="$(printf '%s\n' "${keys}" | sed '/^$/d' | wc -l | tr -d ' ')" - log "data verification: ${count} object(s) sampled from the bucket; reading each (status-code check)..." - checked=0; ok=0 - while IFS= read -r key; do - [ -z "${key}" ] && continue - admin_api GET "/${WARP_BUCKET}/${key}" "" "" discard - code="$(admin_api_code)" - checked=$((checked + 1)) - if [ "${code}" = "200" ]; then - ok=$((ok + 1)) - else - printf '\033[1;31m[ERROR]\033[0m GET %s failed (HTTP %s)\n' "${key}" "${code}" >&2 - fi - done <<<"${keys}" - log "data verification: ${ok}/${checked} objects read successfully" - [ "${checked}" -gt 0 ] && [ "${ok}" -eq "${checked}" ] -} - -# Verify the expected number of pools via the admin API (JSON + jq assertions) -verify_pools() { - local expected="$1" - local body code count nonactive - log "verifying pools via admin API (expect at least ${expected} active): GET ${API_ENDPOINT}/rustfs/admin/v3/pools/list" - if [ "${DRY_RUN}" -eq 1 ]; then return 0; fi - body="$(admin_api GET /rustfs/admin/v3/pools/list "")" - code="$(admin_api_code)" - if [ "${code}" != "200" ]; then - printf '\033[1;31m[ERROR]\033[0m pools/list returned HTTP %s\n' "${code}" >&2 - printf '%s\n' "${body}" >&2 - die "pools/list failed" - fi - if ! count="$(printf '%s' "${body}" | jq -e 'length' 2>/dev/null)"; then - printf '\033[1;31m[ERROR]\033[0m pools/list returned an unexpected (non-JSON) body:\n%s\n' "${body}" >&2 - die "pools/list response could not be parsed as JSON" - fi - nonactive="$(printf '%s' "${body}" | jq '[.[] | select(.status != "active")] | length')" - if [ "${count}" -lt "${expected}" ] || [ "${nonactive}" -ne 0 ]; then - printf '\033[1;31m[ERROR]\033[0m pool assertion failed: expected >=%s active pools, got %s (non-active: %s)\n' \ - "${expected}" "${count}" "${nonactive}" >&2 - printf '%s\n' "--- pool detail ---" >&2 - printf '%s' "${body}" | jq -r '.[] | " pool id=\(.id) status=\(.status) decommission=\(.decommissionStatus) rebalance=\(.rebalanceStatus) used=\(.usedSize)/\(.totalSize) cmdline=\(.cmdline)"' >&2 - printf '%s\n' "--- full JSON ---" >&2 - printf '%s\n' "${body}" >&2 - die "pool verification failed" - fi - log "pools OK: ${count} pools, all active" - printf '%s' "${body}" | jq -r '.[] | " pool id=\(.id) status=\(.status) decommission=\(.decommissionStatus) rebalance=\(.rebalanceStatus)"' -} - -# Preflight checks before running the workflow -preflight() { - log "preflight checks" - need_cmd ssh "SSH client" - need_cmd curl "HTTP client" - need_cmd jq "JSON processor" - need_cmd openssl "OpenSSL (SigV4 signing)" - local url - url="$(resolve_package_url)" - log "package URL: ${url}" - if [ "${DRY_RUN}" -eq 0 ]; then - curl -fsSI --max-time 20 "${url}" >/dev/null || die "package URL not reachable: ${url}" - for node in "${NODES[@]}"; do - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" "hostname" >/dev/null \ - || die "cannot SSH to ${node}" - log "${node}: SSH OK" - done - log "admin API connectivity: GET ${API_ENDPOINT}/rustfs/admin/v3/pools/list" - local body code - body="$(admin_api GET /rustfs/admin/v3/pools/list "")" - code="$(admin_api_code)" - if [ "${code}" != "200" ]; then - # In the automated workflow the cluster is stopped before preflight - # (reset), so an unreachable admin API is expected there; the test - # starts the cluster and verifies the API in step 3. Only fail hard - # when the service is actually running but the API is broken. - local service_state - service_state="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${NODES[0]}" \ - "systemctl is-active ${RUSTFS_SERVICE} 2>/dev/null || true")" - if [ "${service_state}" = "active" ]; then - printf '\033[1;31m[ERROR]\033[0m admin API check failed (HTTP %s) while ${RUSTFS_SERVICE} is active on %s\n' \ - "${code}" "${NODES[0]}" >&2 - printf '%s\n' "${body}" >&2 - die "cannot reach the admin API at ${API_ENDPOINT} with the configured credentials" - fi - printf '\033[1;33m[WARN]\033[0m admin API not reachable (HTTP %s) — ${RUSTFS_SERVICE} is not active on %s; the test will start the cluster and verify the API in step 3\n' \ - "${code}" "${NODES[0]}" >&2 - printf '%s\n' "${body}" >&2 - else - log "admin API OK ($(printf '%s' "${body}" | jq 'length') pool(s) listed)" - fi - # Check that each node resolves the rustfs-node* hostnames used by the volumes - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${NODES[0]}" \ - "grep -q rustfs-node /etc/hosts" || warn "rustfs-node* hostnames not found in /etc/hosts on ${NODES[0]}" - log "node disk space (/data):" - for node in "${NODES[@]}"; do - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \ - "df -h /data | tail -1 | awk '{print \" ${node}: \" \$2 \" total, \" \$4 \" avail\"}'" || true - done - fi - need_cmd warp "warp benchmark tool" - log "preflight OK" -} - -# Reset the test environment: purge the rustfs package (if installed) and -# recreate the data directories on all nodes. Intended for CI so every run -# starts from a clean slate. Destructive! -step0_reset() { - log "reset: purge rustfs package and recreate data dirs on all nodes" - confirm "This DESTROYS the RustFS install and ALL data on ${NODES[*]} (irreversible). Continue?" - for node in "${NODES[@]}"; do - { - printf 'set -euo pipefail\n' - printf 'SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"\n' - # Stop the service if it is still running, then purge the package - # (skipped when rustfs is not installed). - printf '${SUDO} systemctl stop %s 2>/dev/null || true\n' "${RUSTFS_SERVICE}" - printf 'if ${SUDO} dpkg -l %s 2>/dev/null | grep -q "^ii"; then\n' "${RUSTFS_PACKAGE_NAME}" - printf ' ${SUDO} dpkg -P %s\n' "${RUSTFS_PACKAGE_NAME}" - printf ' echo "purged %s"\n' "${RUSTFS_PACKAGE_NAME}" - printf 'else\n' - printf ' echo "%s not installed, skip purge"\n' "${RUSTFS_PACKAGE_NAME}" - printf 'fi\n' - # Ensure the service user exists (created by the package postinst on - # install; a purge keeps it, but a never-installed node needs it for chown). - printf 'id -u %s >/dev/null 2>&1 || ${SUDO} useradd -r -s /bin/false -d /opt/%s %s\n' \ - "${RUSTFS_USER}" "${RUSTFS_USER}" "${RUSTFS_USER}" - # Recreate the volume directories with the service user as owner. - printf 'for i in 1 2 3 4; do\n' - printf ' ${SUDO} rm -rf /data/rustfs${i}/mnmd\n' - printf ' ${SUDO} mkdir -p /data/rustfs${i}/mnmd\n' - printf ' ${SUDO} chown -R %s:%s /data/rustfs${i}/mnmd\n' "${RUSTFS_USER}" "${RUSTFS_USER}" - printf 'done\n' - } | run_remote "${node}" - done - log "reset complete" -} - -# ==================== Steps ==================== - -step1_download() { - log "step 1: download the package on all nodes" - local script url - url="$(resolve_package_url)" - script="$(cat <"${warp_log}" 2>&1 & - warp_pid=$! - log "warp PID=${warp_pid}" - trap 'kill "${warp_pid:-}" 2>/dev/null || true' EXIT - - # Surviving nodes: everything except the outage node - survived_a="${NODES[0]}" - survived_b="${NODES[1]}" - if [ "${OUTAGE_NODE_INDEX}" = "0" ]; then - survived_a="${NODES[1]}" - survived_b="${NODES[2]}" - elif [ "${OUTAGE_NODE_INDEX}" = "1" ]; then - survived_b="${NODES[2]}" - fi - log "surviving nodes: ${survived_a} ${survived_b}; outage node: ${outage_node}" - - log "monitoring disk usage every ${POLL_INTERVAL}s (stop node @ ${STOP_NODE_AT_GB}GB, stop warp @ ${WARP_STOP_AT_GB}GB)" - while [ "${waited}" -lt "${WARP_TIMEOUT}" ]; do - used_a="$(node_used_gb "${survived_a}")" - used_b="$(node_used_gb "${survived_b}")" - used_c="$(node_used_gb "${outage_node}")" - log "used: ${survived_a}=${used_a}GB ${survived_b}=${used_b}GB ${outage_node}=${used_c}GB (node_stopped=${node_stopped})" - - if [ "${node_stopped}" -eq 0 ] \ - && [ "${used_a}" -ge "${STOP_NODE_AT_GB}" ] && [ "${used_b}" -ge "${STOP_NODE_AT_GB}" ]; then - log "surviving nodes reached ${STOP_NODE_AT_GB}GB; stopping ${outage_node}" - service_action stop "${outage_node}" - node_stopped=1 - # Fail closed: the outage node must actually be down before continuing. - if ssh "${SSH_OPTS[@]}" "${SSH_USER}@${outage_node}" \ - "systemctl is-active ${RUSTFS_SERVICE} 2>/dev/null" | grep -q active; then - die "${outage_node} is still active after systemctl stop; cannot continue the outage scenario" - fi - log "${outage_node} service is down" - fi - - if [ "${used_a}" -ge "${WARP_STOP_AT_GB}" ] && [ "${used_b}" -ge "${WARP_STOP_AT_GB}" ]; then - log "surviving nodes reached ${WARP_STOP_AT_GB}GB; stopping warp" - target_reached=1 - break - fi - - if ! kill -0 "${warp_pid}" 2>/dev/null; then - die "warp exited early at ${used_a}/${used_b}GB before reaching ${WARP_STOP_AT_GB}GB (log: ${warp_log})" - break - fi - - sleep "${POLL_INTERVAL}" - waited=$((waited + POLL_INTERVAL)) - done - kill "${warp_pid}" 2>/dev/null || true - trap - EXIT - wait "${warp_pid}" 2>/dev/null || true - log "warp stopped (log: ${warp_log})" - - if [ "${node_stopped}" -eq 0 ]; then - die "never reached ${STOP_NODE_AT_GB}GB on both surviving nodes within ${WARP_TIMEOUT}s; the outage node was NOT stopped — test invalid" - fi - if [ "${target_reached}" -eq 0 ]; then - die "warp did not reach ${WARP_STOP_AT_GB}GB on the surviving nodes within ${WARP_TIMEOUT}s — test invalid" - fi - used_a="$(node_used_gb "${survived_a}")" - used_b="$(node_used_gb "${survived_b}")" - log "final used before heal: ${survived_a}=${used_a}GB ${survived_b}=${used_b}GB ${outage_node}=$(node_used_gb "${outage_node}")GB" -} - -step4_restart_node() { - local node="${NODES[${OUTAGE_NODE_INDEX}]}" - log "step 4: restart ${node} after the outage" - confirm "About to start rustfs on ${node}. Continue?" - if [ "${DRY_RUN}" -eq 1 ]; then - service_action start "${node}" - log "DRY-RUN: waiting for ${node} to become active" - return 0 - fi - service_action start "${node}" - wait_service_active "${node}" - # Bounded readiness: wait for the admin API to report an active pool. - local attempts=0 - while ! verify_pools 1 >/dev/null 2>&1; do - attempts=$((attempts + 1)) - if [ "${attempts}" -ge 12 ]; then - verify_pools 1 # final call fails loudly - fi - log "cluster not ready yet (attempt ${attempts}/12); waiting 5s" - sleep 5 - done - log "${node} is back; cluster reports an active pool" -} - -step5_start_heal() { - log "step 5: start cluster heal (POST /rustfs/admin/v3/heal/ {\"recursive\":true,...})" - confirm "About to start heal on the cluster. Continue?" - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: POST /rustfs/admin/v3/heal/ {\"recursive\":true,...}" - return 0 - fi - local attempts="${HEAL_START_RETRIES}" delay="${HEAL_START_RETRY_DELAY}" attempt=1 body code - while :; do - body="$(admin_api POST /rustfs/admin/v3/heal/ "" \ - '{"recursive":true,"dryRun":false,"remove":false,"recreate":false,"scanMode":"normal","updateParity":false,"nolock":false}')" - code="$(admin_api_code)" - if [ "${code}" = "200" ]; then - if [ -n "${body}" ]; then - HEAL_CLIENT_TOKEN="$(printf '%s' "${body}" | jq -r '.clientToken // empty')" - log "heal started: clientToken=${HEAL_CLIENT_TOKEN}" - else - log "heal start accepted (empty response)" - fi - return 0 - fi - if [ "${code}" = "400" ] || [ "${code}" = "403" ]; then - printf '\033[1;31m[ERROR]\033[0m heal start rejected (HTTP %s): %s\n' "${code}" "${body}" >&2 - die "heal start rejected (HTTP ${code}); fix the request, not the retry" - fi - warn "heal start attempt ${attempt}/${attempts} failed (HTTP ${code}): ${body}" - hint_server_issue "${body}" || true - if [ "${attempt}" -ge "${attempts}" ]; then - die "heal start failed after ${attempts} attempts (see last error above)" - fi - attempt=$((attempt + 1)) - sleep "${delay}" - done -} - -step6_monitor_heal() { - log "step 6: monitor heal task until the server verdict is done" - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: waiting for heal to complete" - return 0 - fi - if [ -z "${HEAL_CLIENT_TOKEN}" ]; then - die "no heal client token (run step 5 first, or pass --heal-token)" - fi - local waited=0 body code summary failed healed scanned pct prog_present - local failed_n healed_n scanned_n pct_n vm002_used warned501=0 pre_outage_used - pre_outage_used="$(node_used_gb "${NODES[${OUTAGE_NODE_INDEX}]}")" - log "outage node usage before heal: ${pre_outage_used}GB" - while [ "${waited}" -lt "${HEAL_TIMEOUT}" ]; do - body="$(admin_api POST /rustfs/admin/v3/heal/ "clientToken=${HEAL_CLIENT_TOKEN}" "")" - code="$(admin_api_code)" - if [ "${code}" != "200" ]; then - if [ "${code}" = "501" ] && [ "${warned501}" -eq 0 ]; then - # background task status should work; keep polling if the cluster's - # background-heal aggregator is unavailable (501 on some topologies). - warn "heal task status returned HTTP 501 (${body}); retrying" - warned501=1 - fi - sleep "${POLL_INTERVAL}" - waited=$((waited + POLL_INTERVAL)) - continue - fi - summary="$(printf '%s' "${body}" | jq -r '.summary // "running"')" - prog_present="$(printf '%s' "${body}" | jq -r '.progress != null')" - failed="$(heal_progress_field "${body}" objectsFailed objects_failed)" - healed="$(heal_progress_field "${body}" objectsHealed objects_healed)" - scanned="$(heal_progress_field "${body}" objectsScanned objects_scanned)" - pct="$(heal_progress_field "${body}" progressPercentage progress_percentage)" - failed_n="${failed}"; [ "${failed_n}" = "null" ] && failed_n=0 - healed_n="${healed}"; [ "${healed_n}" = "null" ] && healed_n=0 - scanned_n="${scanned}"; [ "${scanned_n}" = "null" ] && scanned_n=0 - pct_n="${pct}"; [ "${pct_n}" = "null" ] && pct_n=0 - vm002_used="$(node_used_gb "${NODES[${OUTAGE_NODE_INDEX}]}")" - log "heal: summary=${summary} scanned=${scanned_n} healed=${healed_n} failed=${failed_n} pct=${pct_n} ${NODES[${OUTAGE_NODE_INDEX}]}_used=${vm002_used}GB" - - if [ "${prog_present}" = "false" ] && [ "${summary}" = "running" ]; then - warn "heal progress is null while the task is running (server-side reporting gap; see rustfs/backlog#2035)" - fi - - if [ "${failed_n}" -gt 0 ] || printf '%s' "${summary}" | grep -qiE 'fail|error|stopped'; then - printf '\033[1;31m[ERROR]\033[0m heal reported failed objects (%s)\n' "${failed_n}" >&2 - printf '%s\n' "--- full heal status JSON ---" >&2 - printf '%s\n' "${body}" >&2 - die "heal failed (summary=${summary}, objects_failed=${failed_n})" - fi - - # Success only for a real terminal summary; "running"/"notFound"/"" mean - # the task is still going (or lives on another node) — keep polling. - if printf '%s' "${summary}" | grep -qiE '^(finished|completed|success|done)$'; then - log "heal done: summary=${summary} failed=0 (server verdict)" - if [ "${vm002_used}" -le "${pre_outage_used}" ]; then - warn "heal finished but ${NODES[${OUTAGE_NODE_INDEX}]} usage did not grow (${pre_outage_used}GB -> ${vm002_used}GB); the repair may not have landed on its disks" - fi - final_status_file="$(mktemp "${TMPDIR:-/tmp}/rustfs-heal-final-status.XXXXXX.json" 2>/dev/null \ - || printf '%s' "${TMPDIR:-/tmp}/rustfs-heal-final-status.$$.json")" - printf '%s\n' "${body}" > "${final_status_file}" 2>/dev/null \ - && log "final heal status saved: ${final_status_file}" \ - || warn "could not save final heal status to ${final_status_file}" - return 0 - fi - - sleep "${POLL_INTERVAL}" - waited=$((waited + POLL_INTERVAL)) - done - printf '\033[1;31m[ERROR]\033[0m timed out waiting for heal (${HEAL_TIMEOUT}s); final status:\n' >&2 - printf '%s\n' "${body}" >&2 - die "timed out waiting for heal (${HEAL_TIMEOUT}s)" -} - -step7_analyze_results() { - log "step 7: result analysis" - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: analyzing heal results" - return 0 - fi - if [ -z "${HEAL_CLIENT_TOKEN}" ]; then - die "no heal client token (run step 5 first, or pass --heal-token)" - fi - local body summary failed healed scanned pct prog_present - local failed_n healed_n scanned_n pct_n i used - body="$(admin_api POST /rustfs/admin/v3/heal/ "clientToken=${HEAL_CLIENT_TOKEN}" "")" - if [ "$(admin_api_code)" = "200" ]; then - summary="$(printf '%s' "${body}" | jq -r '.summary // "unknown"')" - prog_present="$(printf '%s' "${body}" | jq -r '.progress != null')" - failed="$(heal_progress_field "${body}" objectsFailed objects_failed)" - healed="$(heal_progress_field "${body}" objectsHealed objects_healed)" - scanned="$(heal_progress_field "${body}" objectsScanned objects_scanned)" - pct="$(heal_progress_field "${body}" progressPercentage progress_percentage)" - failed_n="${failed}"; [ "${failed_n}" = "null" ] && failed_n=0 - healed_n="${healed}"; [ "${healed_n}" = "null" ] && healed_n=0 - scanned_n="${scanned}"; [ "${scanned_n}" = "null" ] && scanned_n=0 - pct_n="${pct}"; [ "${pct_n}" = "null" ] && pct_n=0 - fi - printf '%s\n' "--- heal status ---" - printf ' summary=%s scanned=%s healed=%s failed=%s progress=%s%%\n' \ - "${summary}" "${scanned_n}" "${healed_n}" "${failed_n}" "${pct_n}" - printf '%s\n' "--- per-node disk usage (GiB) ---" - for i in "${!NODES[@]}"; do - used="$(node_used_gb "${NODES[$i]}")" - printf ' %s: %sGB\n' "${NODES[$i]}" "${used}" - done - - local outage_used - outage_used="$(node_used_gb "${NODES[${OUTAGE_NODE_INDEX}]}")" - - if ! verify_data_readable; then - die "heal test FAILED: data read-back verification failed (see errors above)" - fi - - if printf '%s' "${summary}" | grep -qiE '^(finished|completed|success|done)$' \ - && [ "${failed_n}" -eq 0 ]; then - log "heal test PASSED: cluster heal complete, 0 failed, data read-back OK, ${NODES[${OUTAGE_NODE_INDEX}]}_used=${outage_used}GB" - return 0 - fi - die "heal test FAILED: summary=${summary} failed=${failed_n} ${NODES[${OUTAGE_NODE_INDEX}]}_used=${outage_used}GB" -} -# ==================== CLI parsing ==================== - -usage() { - cat <<'USAGE' -Usage: rustfs-heal-test.sh [options] - -Options: - --all Run the full workflow (steps 1-7) - --step N Run a single step (repeatable) - --steps 1,3,5-7 Run steps in order - --skip-download Skip download when the package already exists - --version VER RustFS release tag to test, e.g. 1.0.0-rc.3 (default from config) - --package-url URL Full package download URL (overrides --version) - --sha256 HEX Verify the downloaded package checksum - --preflight Run preflight checks before the selected steps - --reset Reset all nodes (stop services, wipe data dirs + config). Destructive! - --ssh-user USER SSH user for the nodes (default azureuser) - --ssh-port PORT SSH port for the nodes (default 22) - --endpoint URL Cluster admin API endpoint, e.g. http://10.0.0.7:9000 (default from config) - --rc-endpoint URL Deprecated alias for --endpoint - --stop-node-gb N Stop the outage node when surviving nodes reach N GiB (default 15) - --warp-stop-gb N Stop warp when surviving nodes reach N GiB (default 40) - --heal-token TOKEN clientToken of a heal started earlier (for steps 6/7 reruns) - --warp-timeout N Write phase timeout in seconds (default 3600) - --heal-timeout N Heal wait timeout in seconds (default 86400) - --service-timeout N Service start wait timeout in seconds (default 300) - --poll-interval N Status polling interval in seconds (default 15) - --log-file FILE Append all output to FILE - --dry-run Preview commands without executing them - -y, --yes Skip all confirmation prompts - -h, --help Show this help - -Examples: - ./rustfs-heal-test.sh --all - ./rustfs-heal-test.sh --all -y --package-url https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb - ./rustfs-heal-test.sh --steps 5,6,7 - ./rustfs-heal-test.sh --all --dry-run -USAGE -} - -expand_steps() { - # Expand a "1,3,5-7" step spec into SELECTED_STEPS - local spec="$1" part start end i - IFS=',' read -ra parts <<<"${spec}" - for part in "${parts[@]}"; do - if [[ "${part}" =~ ^([0-9]+)-([0-9]+)$ ]]; then - start="${BASH_REMATCH[1]}"; end="${BASH_REMATCH[2]}" - for ((i=start; i<=end; i++)); do SELECTED_STEPS+=("${i}"); done - elif [[ "${part}" =~ ^[0-9]+$ ]]; then - SELECTED_STEPS+=("${part}") - else - die "cannot parse step spec: ${part}" - fi - done -} - -run_steps() { - local step - for step in "${SELECTED_STEPS[@]}"; do - case "${step}" in - 1) step1_download ;; - 2) step2_install_and_start ;; - 3) step3_write_data_with_node_outage ;; - 4) step4_restart_node ;; - 5) step5_start_heal ;; - 6) step6_monitor_heal ;; - 7) step7_analyze_results ;; - *) die "unknown step: ${step}" ;; - esac - log "step ${step} completed" - done -} - -main() { - [ "$#" -eq 0 ] && { usage; exit 0; } - local opt all=0 - while [ "$#" -gt 0 ]; do - opt="$1"; shift - case "${opt}" in - --all) all=1 ;; - --step) SELECTED_STEPS+=("$1"); shift ;; - --steps) expand_steps "$1"; shift ;; - --skip-download) SKIP_DOWNLOAD=1 ;; - --version) RUSTFS_VERSION="$1"; shift ;; - --package-url) PACKAGE_URL="$1"; shift ;; - --sha256) PACKAGE_SHA256="$1"; shift ;; - --preflight) PREFLIGHT=1 ;; - --reset) RESET=1 ;; - --ssh-user) SSH_USER="$1"; shift ;; - --ssh-port) SSH_PORT="$1"; shift ;; - --endpoint) API_ENDPOINT="$1"; shift ;; - --rc-endpoint) API_ENDPOINT="$1"; shift ;; - --stop-node-gb) STOP_NODE_AT_GB="$1"; shift ;; - --warp-stop-gb) WARP_STOP_AT_GB="$1"; shift ;; - --heal-token) HEAL_CLIENT_TOKEN="$1"; shift ;; - --warp-timeout) WARP_TIMEOUT="$1"; shift ;; - --heal-timeout) HEAL_TIMEOUT="$1"; shift ;; - --service-timeout) SERVICE_TIMEOUT="$1"; shift ;; - --poll-interval) POLL_INTERVAL="$1"; shift ;; - --log-file) LOG_FILE="$1"; shift ;; - --dry-run) DRY_RUN=1 ;; - -y|--yes) ASSUME_YES=1 ;; - -h|--help) usage; exit 0 ;; - *) die "unknown option: ${opt} (see --help)" ;; - esac - done - trap 'rm -f "${ADMIN_API_CODE_FILE}"' EXIT - if [ -n "${LOG_FILE}" ]; then - mkdir -p "$(dirname "${LOG_FILE}")" - if ! touch "${LOG_FILE}" 2>/dev/null; then - # A fixed /tmp path may be owned by another user (e.g. a previous root - # run); fall back to a unique, always-writable temp file. - LOG_FILE="$(mktemp "${TMPDIR:-/tmp}/rustfs-heal-test.XXXXXX.log")" - warn "log file not writable; using ${LOG_FILE}" - fi - exec > >(tee -a "${LOG_FILE}") 2>&1 - fi - if [ "${RESET}" -eq 1 ]; then - step0_reset - log "reset finished" - exit 0 - fi - if [ "${all}" -eq 1 ]; then - SELECTED_STEPS=(1 2 3 4 5 6 7) - fi - PACKAGE_URL="$(resolve_package_url)" - if [ "${PREFLIGHT}" -eq 1 ]; then - preflight - if [ "${#SELECTED_STEPS[@]}" -eq 0 ]; then - log "preflight only (no steps selected); done" - exit 0 - fi - fi - [ "${#SELECTED_STEPS[@]}" -gt 0 ] || die "no steps selected (--all / --step / --steps)" - if [ "${DRY_RUN}" -eq 0 ]; then - log "nodes: ${NODES[*]} ssh user: ${SSH_USER} version: ${RUSTFS_VERSION}" - log "package: ${PACKAGE_URL}" - else - warn "DRY-RUN mode: only printing the commands that would run" - fi - run_steps - log "all done" -} - -# Allow sourcing the file for unit tests without running main. -if [ "${RUSTFS_HEAL_SCRIPT_SOURCE_ONLY:-0}" != "1" ]; then - main "$@" -fi diff --git a/scripts/test/rustfs_performance_test.sh b/scripts/test/rustfs_performance_test.sh deleted file mode 100755 index 12878772d..000000000 --- a/scripts/test/rustfs_performance_test.sh +++ /dev/null @@ -1,543 +0,0 @@ -#!/usr/bin/env bash -# -# rustfs-performance-test.sh -# RustFS 4x4 集群性能压测全流程脚本 -# -# Based on the Obsidian note "RustFS 性能测试". Full workflow: -# 1. Cleanup: stop & purge rustfs, remove data dirs on all nodes -# 2. Download the RustFS package on all nodes -# 3. Install RustFS on all nodes (dpkg -i), recreate volume dirs -# 4. Write /etc/default/rustfs (4-node x 4-drive MNMD), start all nodes -# in parallel and verify the service is Running -# 5. Run the benchmark (warp GET/PUT/MIXED via rustfs-performance-testing.sh) -# 6. Analyze results (summary.tsv / summary.md) -# 7. Final cleanup: stop & purge rustfs, remove data dirs -# -# The script is driven from an admin host (e.g. a jumpbox) and operates on -# the target nodes over SSH, mirroring scripts/test/rustfs_*_test.sh. -# -# Usage: -# ./rustfs-performance-test.sh --all # run all steps 1-7 -# ./rustfs-performance-test.sh --step 5 # run a single step -# ./rustfs-performance-test.sh --steps 2,3,4 # run selected steps -# ./rustfs-performance-test.sh --all --dry-run # preview only -# ./rustfs-performance-test.sh --all -y --package-url -# -# Notes: -# - SSH user defaults to azureuser (passwordless sudo on the nodes); -# pass --ssh-user root if your nodes accept root login. -# - The benchmark runner defaults to -# ~/Documents/Obsidian Vault/rustfs-performance-testing.sh; override with -# --bench-script / RUSTFS_BENCH_SCRIPT. warp must be installed on the -# admin host. -# - Steps 1 and 7 destroy the RustFS install and all data (confirmed). -# -set -Eeuo pipefail - -# ==================== Configuration (adjust to your environment) ==================== - -# Target nodes (4x4: 4 nodes x 4 drives each) -if [ -n "${RUSTFS_NODES:-}" ]; then - read -r -a NODES <<<"${RUSTFS_NODES}" -else - NODES=(vm000 vm001 vm002 vm003) -fi - -SSH_USER="${RUSTFS_SSH_USER:-azureuser}" -SSH_PORT="${RUSTFS_SSH_PORT:-22}" -SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new -p "${SSH_PORT}") - -# Package: GitHub release tag, e.g. "1.0.0-rc.3". PACKAGE_URL is derived from -# RUSTFS_VERSION unless --package-url / RUSTFS_PACKAGE_URL is given. -RUSTFS_VERSION="${RUSTFS_VERSION:-1.0.0-rc.3}" -PACKAGE_URL="${RUSTFS_PACKAGE_URL:-}" -ARCH="${RUSTFS_ARCH:-amd64}" -PACKAGES_DIR="/home/rustfs/packages" -PACKAGE_FILE="rustfs.deb" -PACKAGE_SHA256="${RUSTFS_PACKAGE_SHA256:-}" - -# 4x4 topology: 4 nodes x 4 drives each, same expression on every node -DRIVES_PER_NODE="${RUSTFS_DRIVES_PER_NODE:-4}" -VOLUMES="http://rustfs-node{1...4}:9000/data/rustfs{1...4}/mnmd" - -# RustFS service configuration (written to /etc/default/rustfs) -RUSTFS_CONFIG_FILE="/etc/default/rustfs" -RUSTFS_SERVICE="rustfs" -RUSTFS_PACKAGE_NAME="rustfs" -RUSTFS_USER="rustfs" -ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfs@test}" -SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfs@test}" -RUSTFS_ADDRESS=":9000" -RUSTFS_CONSOLE_ADDRESS=":9001" -RUSTFS_CONSOLE_ENABLE=true -RUSTFS_OBS_LOGGER_LEVEL=error -RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/" - -# Benchmark runner (step 5/6): prefer the default Obsidian location, fall back -# to a rustfs-performance-testing.sh next to this script (e.g. in the repo or -# on a jumpbox). -_DEFAULT_BENCH="${HOME}/Documents/Obsidian Vault/rustfs-performance-testing.sh" -_SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -if [ -x "${_DEFAULT_BENCH}" ]; then - _BENCH_RESOLVED="${_DEFAULT_BENCH}" -elif [ -x "${_SCRIPT_DIR}/rustfs-performance-testing.sh" ]; then - _BENCH_RESOLVED="${_SCRIPT_DIR}/rustfs-performance-testing.sh" -elif [ -x "${_SCRIPT_DIR}/rustfs_performance_testing.sh" ]; then - _BENCH_RESOLVED="${_SCRIPT_DIR}/rustfs_performance_testing.sh" -else - _BENCH_RESOLVED="${_DEFAULT_BENCH}" -fi -BENCH_SCRIPT="${RUSTFS_BENCH_SCRIPT:-${_BENCH_RESOLVED}}" -RESULT_DIR="${RUSTFS_RESULT_DIR:-$(pwd)/warp-bench-results-$(date +%Y%m%d-%H%M%S)}" -WARP_HOST="${RUSTFS_WARP_HOST:-rustfs-node1:9000,rustfs-node2:9000,rustfs-node3:9000,rustfs-node4:9000}" -WARP_BUCKET="${RUSTFS_WARP_BUCKET:-warp-benchmark-bucket}" -WARP_CONCURRENCY="${RUSTFS_WARP_CONCURRENCY:-64}" -WARP_DURATION="${RUSTFS_WARP_DURATION:-5m}" -WARP_GET_OBJECTS="${RUSTFS_WARP_GET_OBJECTS:-2500}" -WARP_SLEEP="${RUSTFS_WARP_SLEEP:-60}" -# Manual method/size selection (passed through to the benchmark runner; empty = full run) -WARP_METHODS="${RUSTFS_WARP_METHODS:-}" -WARP_SIZES="${RUSTFS_WARP_SIZES:-}" - -# Timeouts (seconds) -SERVICE_TIMEOUT="${RUSTFS_SERVICE_TIMEOUT:-300}" -POLL_INTERVAL="${RUSTFS_POLL_INTERVAL:-10}" - -# ==================== Runtime options (set by CLI) ==================== -DRY_RUN=0 -ASSUME_YES=0 -SKIP_DOWNLOAD=0 -PREFLIGHT=0 -LOG_FILE="" -SELECTED_STEPS=() - -# ==================== Helpers ==================== - -log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; } -warn() { printf '\033[1;33m[WARN]\033[0m %s\n' "$*"; } -die() { printf '\033[1;31m[ERROR]\033[0m %s\n' "$*" >&2; exit 1; } - -confirm() { - if [ "${ASSUME_YES}" -eq 1 ] || [ "${DRY_RUN}" -eq 1 ]; then return 0; fi - printf '\033[1;33m[CONFIRM]\033[0m %s (y/N) ' "$1" - read -r answer - case "${answer}" in - y|Y|yes|YES) return 0 ;; - *) die "cancelled" ;; - esac -} - -need_cmd() { - [ "${DRY_RUN}" -eq 1 ] && return 0 - command -v "$1" >/dev/null 2>&1 || die "missing command: $1 ($2); install it first" -} - -# Run a remote script on a single node (script is read from stdin) -run_remote() { - local node="$1" script - script="$(cat)" - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: ssh ${SSH_USER}@${node} <<'REMOTE'" - printf '%s\n' "${script}" | sed 's/^/ | /' - log "DRY-RUN: ----" - return 0 - fi - log "==> ${node}: executing remote script" - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'bash -s' <<<"${script}" -} - -# Run the same remote script on all nodes in parallel (script from stdin) -run_remote_all() { - local script pids=() i=0 fail=0 - script="$(cat)" - for node in "${NODES[@]}"; do - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: ssh ${SSH_USER}@${node} <<'REMOTE'" - printf '%s\n' "${script}" | sed 's/^/ | /' - log "DRY-RUN: ----" - else - log "==> ${node}: executing remote script" - ( ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'bash -s' <<<"${script}" ) & - pids[$i]=$! - i=$((i+1)) - fi - done - if [ "${#pids[@]}" -gt 0 ]; then - for pid in "${pids[@]}"; do - wait "${pid}" || fail=1 - done - fi - [ "${fail}" -eq 0 ] || die "one or more remote executions failed" -} - -rustfs_config_body() { - cat </dev/null </dev/null" | grep -q active; then - log "${node}: service active" - return 0 - fi - sleep "${POLL_INTERVAL}" - elapsed=$((elapsed + POLL_INTERVAL)) - done - die "${node}: ${RUSTFS_SERVICE} did not become active within ${SERVICE_TIMEOUT}s" -} - -verify_service_running() { - local node="$1" - log "${node}: checking service status" - if [ "${DRY_RUN}" -eq 0 ]; then - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \ - "systemctl status ${RUSTFS_SERVICE} --no-pager | head -n 12" || true - fi -} - -build_package_url() { - local asset - asset="rustfs_$(printf '%s' "${RUSTFS_VERSION}" | tr '-' '.')_${ARCH}.deb" - printf 'https://github.com/rustfs/rustfs/releases/download/%s/%s' "${RUSTFS_VERSION}" "${asset}" -} - -resolve_package_url() { - if [ -n "${PACKAGE_URL}" ]; then printf '%s' "${PACKAGE_URL}"; else build_package_url; fi -} - -preflight() { - log "preflight checks" - need_cmd ssh "openssh client" - need_cmd curl "http client" - need_cmd warp "warp benchmark tool (for step 5)" - if [ ! -x "${BENCH_SCRIPT}" ]; then - die "benchmark script not found or not executable: ${BENCH_SCRIPT}" - fi - if [ "${DRY_RUN}" -eq 0 ]; then - for node in "${NODES[@]}"; do - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'echo ok' >/dev/null \ - || die "cannot ssh to ${node}" - done - log "all nodes reachable: ${NODES[*]}" - fi - log "preflight OK" -} - -# ==================== Steps ==================== - -step1_cleanup() { - log "step 1: cleanup environment on all nodes (stop & purge rustfs, remove data dirs)" - confirm "This DESTROYS the RustFS install and ALL data on ${NODES[*]} (irreversible). Continue?" - local script - script="$(cat </dev/null || true -if \${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then - \${SUDO} dpkg -P rustfs - echo "purged rustfs" -else - echo "rustfs not installed, skip purge" -fi -for i in \$(seq 1 ${DRIVES_PER_NODE}); do - \${SUDO} rm -rf /data/rustfs\${i}/mnmd - \${SUDO} mkdir -p /data/rustfs\${i}/mnmd - \${SUDO} chown -R rustfs:rustfs /data/rustfs\${i}/mnmd -done -echo "cleanup done on \$(hostname)" -EOF -)" - printf '%s\n' "${script}" | run_remote_all - log "step 1 complete" -} - -step2_download() { - log "step 2: download the package on all nodes" - local url script - url="$(resolve_package_url)" - script="$(cat </dev/null || true -if \${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then - \${SUDO} dpkg -P rustfs - echo "purged rustfs" -fi -for i in \$(seq 1 ${DRIVES_PER_NODE}); do - \${SUDO} rm -rf /data/rustfs\${i}/mnmd -done -echo "cleanup done on \$(hostname)" -EOF -)" - printf '%s\n' "${script}" | run_remote_all - log "step 7 complete" -} - -# ==================== CLI ==================== - -usage() { - cat <<'USAGE' -Usage: ./rustfs-performance-test.sh [options] - -Steps: - 1 cleanup environment (purge rustfs, remove data dirs) [destructive] - 2 download the RustFS package on all nodes - 3 install RustFS (dpkg -i) - 4 write config, start service, verify Running - 5 run benchmark (warp GET/PUT/MIXED) - 6 analyze results (summary.tsv / summary.md) - 7 final cleanup (purge rustfs, remove data dirs) [destructive] - -Options: - --all Run all steps 1-7 - --step N Run a single step - --steps 1,3,5-7 Run selected steps - --version VERSION GitHub release tag (default 1.0.0-rc.3) - --package-url URL Direct deb URL (overrides --version) - --sha256 HASH Verify package checksum - --skip-download Keep an existing package file - --bench-script PATH Benchmark runner (default: Obsidian Vault rustfs-performance-testing.sh) - --result-dir DIR Benchmark result directory - --warp-duration DUR warp duration per round (default 5m) - --warp-concurrency N warp concurrency (default 64) - --ssh-user USER SSH user (default azureuser) - --ssh-port PORT SSH port (default 22) - --preflight Check environment and exit - --log-file FILE Append all output to FILE - --dry-run Preview commands without executing them - -y, --yes Skip all confirmation prompts - -h, --help Show this help - -Examples: - ./rustfs-performance-test.sh --all - ./rustfs-performance-test.sh --all --dry-run - ./rustfs-performance-test.sh --all -y --package-url https://dl.rustfs.com/...deb - ./rustfs-performance-test.sh --step 5 -USAGE -} - -expand_steps() { - local spec="$1" part start end i - IFS=',' read -ra parts <<<"${spec}" - for part in "${parts[@]}"; do - if [[ "${part}" =~ ^([0-9]+)-([0-9]+)$ ]]; then - start="${BASH_REMATCH[1]}"; end="${BASH_REMATCH[2]}" - for ((i=start; i<=end; i++)); do SELECTED_STEPS+=("${i}"); done - elif [[ "${part}" =~ ^[0-9]+$ ]]; then - SELECTED_STEPS+=("${part}") - else - die "cannot parse step spec: ${part}" - fi - done -} - -run_steps() { - local step - for step in "${SELECTED_STEPS[@]}"; do - case "${step}" in - 1) step1_cleanup ;; - 2) step2_download ;; - 3) step3_install ;; - 4) step4_configure_start ;; - 5) step5_benchmark ;; - 6) step6_analyze ;; - 7) step7_cleanup ;; - *) die "unknown step: ${step}" ;; - esac - log "step ${step} completed" - done -} - -main() { - [ "$#" -eq 0 ] && { usage; exit 0; } - local opt all=0 - while [ "$#" -gt 0 ]; do - opt="$1"; shift - case "${opt}" in - --all) all=1 ;; - --step) SELECTED_STEPS+=("$1"); shift ;; - --steps) expand_steps "$1"; shift ;; - --version) RUSTFS_VERSION="$1"; shift ;; - --package-url) PACKAGE_URL="$1"; shift ;; - --sha256) PACKAGE_SHA256="$1"; shift ;; - --skip-download) SKIP_DOWNLOAD=1 ;; - --bench-script) BENCH_SCRIPT="$1"; shift ;; - --result-dir) RESULT_DIR="$1"; shift ;; - --warp-duration) WARP_DURATION="$1"; shift ;; - --warp-concurrency) WARP_CONCURRENCY="$1"; shift ;; - --ssh-user) SSH_USER="$1"; shift ;; - --ssh-port) SSH_PORT="$1"; shift ;; - --preflight) PREFLIGHT=1 ;; - --log-file) LOG_FILE="$1"; shift ;; - --dry-run) DRY_RUN=1 ;; - -y|--yes) ASSUME_YES=1 ;; - -h|--help) usage; exit 0 ;; - *) die "unknown option: ${opt} (see --help)" ;; - esac - done - if [ -n "${LOG_FILE}" ]; then - mkdir -p "$(dirname "${LOG_FILE}")" - exec > >(tee -a "${LOG_FILE}") 2>&1 - fi - if [ "${all}" -eq 1 ]; then - SELECTED_STEPS=(1 2 3 4 5 6 7) - fi - if [ "${PREFLIGHT}" -eq 1 ]; then - preflight - if [ "${#SELECTED_STEPS[@]}" -eq 0 ]; then - log "preflight only; done" - exit 0 - fi - fi - [ "${#SELECTED_STEPS[@]}" -gt 0 ] || die "no steps selected (--all / --step / --steps)" - log "nodes: ${NODES[*]} ssh user: ${SSH_USER} version: ${RUSTFS_VERSION}" - log "package: $(resolve_package_url)" - log "result dir: ${RESULT_DIR}" - [ "${DRY_RUN}" -eq 1 ] && warn "DRY-RUN mode: only printing the commands that would run" - run_steps - log "all done" -} - -# Allow sourcing the file for unit tests without running main. -if [ "${RUSTFS_PERF_SCRIPT_SOURCE_ONLY:-0}" != "1" ]; then - main "$@" -fi diff --git a/scripts/test/rustfs_performance_testing.sh b/scripts/test/rustfs_performance_testing.sh deleted file mode 100755 index c53e2a812..000000000 --- a/scripts/test/rustfs_performance_testing.sh +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/env bash -# -# rustfs-performance-testing.sh -# RustFS 对象存储压测脚本(固定版):测试方法 + 执行 + 结果解析 -# -# 测试方法 -# 1) 方法:GET / PUT / MIXED(warp 默认混合负载 45% GET + 55% PUT) -# 2) 对象尺寸:1KiB 4KiB 16KiB 128KiB 1MiB 4MiB 8MiB 16MiB 32MiB 64MiB -# 3) 并发:64;单轮时长:5m;轮间 sleep:60s;GET 对象数:2500 -# 4) 顺序:GET 全部尺寸 -> PUT 全部尺寸 -> MIXED 全部尺寸 -# 5) 结果解析:每轮结束后自动解析 warp 输出,写入 -# summary.tsv(机器可读)与 summary.md(Markdown 汇总表) -# -# 依赖:warp >= v1.6(MinIO warp),bash,awk/sed/grep -# 说明:warp v1.6.1 的 put 不支持 --objects,脚本已自动处理(仅 get/mixed 传该参数) -# -# 环境变量覆盖(不传时使用固定默认值): -# WARP_HOST WARP_ACCESS_KEY WARP_SECRET_KEY WARP_BUCKET -# WARP_CONCURRENCY WARP_DURATION WARP_GET_OBJECTS WARP_SLEEP_BETWEEN_ROUNDS -# WARP_RESULT_DIR -# WARP_METHODS WARP_SIZES # 手动指定方法/尺寸(逗号或空格分隔),不传则全量 - -set -u -o pipefail - -HOST="${WARP_HOST:-rustfs-node1:9000,rustfs-node2:9000,rustfs-node3:9000,rustfs-node4:9000}" -ACCESS_KEY="${WARP_ACCESS_KEY:-rustfs@test}" -SECRET_KEY="${WARP_SECRET_KEY:-rustfs@test}" -BUCKET="${WARP_BUCKET:-warp-benchmark-bucket}" -CONCURRENCY="${WARP_CONCURRENCY:-64}" -DURATION="${WARP_DURATION:-5m}" -GET_OBJECTS="${WARP_GET_OBJECTS:-2500}" -SLEEP_BETWEEN_ROUNDS="${WARP_SLEEP_BETWEEN_ROUNDS:-60}" -RESULT_DIR="${WARP_RESULT_DIR:-$(pwd)/warp-bench-results-$(date +%Y%m%d-%H%M%S)}" - -if [ -n "${WARP_SIZES:-}" ] && [ "${WARP_SIZES}" != "all" ] && [ "${WARP_SIZES}" != "ALL" ]; then - read -r -a SIZES <<<"${WARP_SIZES//,/ }" -else - SIZES=(1KiB 4KiB 16KiB 128KiB 1MiB 4MiB 8MiB 16MiB 32MiB 64MiB) -fi -if [ -n "${WARP_METHODS:-}" ] && [ "${WARP_METHODS}" != "all" ] && [ "${WARP_METHODS}" != "ALL" ]; then - read -r -a METHODS <<<"${WARP_METHODS//,/ }" -else - METHODS=(get put mixed) -fi -TOTAL_ROUNDS=$(( ${#METHODS[@]} * ${#SIZES[@]} )) -ROUND=0 - -# --parse-only :只解析已有结果目录(${method}_${size}.txt),不执行压测 -if [[ "${1:-}" == "--parse-only" && -n "${2:-}" ]]; then - RESULT_DIR="$2" -fi - -LOG_FILE="${RESULT_DIR}/master.log" -SUMMARY_TSV="${RESULT_DIR}/summary.tsv" -SUMMARY_MD="${RESULT_DIR}/summary.md" - -if [[ "${1:-}" != "--parse-only" ]] && ! command -v warp >/dev/null 2>&1; then - echo "错误:未找到 warp 命令,请先安装 MinIO warp。" >&2 - exit 1 -fi - -log() { - echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') $*" | tee -a "${LOG_FILE}" -} - -# ---- 结果解析 ---- - -# 提取指定 section(GET/PUT/Total 等)的 Average / Reqs / TTFB 原始行 -section_lines() { - awk -v sec="$2" ' - /^Report: / { cur = $2; sub(/\.$/, "", cur) } - cur == sec && /^ *\* Average:/ { avg = $0 } - cur == sec && /^ *\* Reqs:/ { reqs = $0 } - cur == sec && /^ *\* TTFB:/ { ttfb = $0 } - END { - if (avg != "") print avg - if (reqs != "") print reqs - if (ttfb != "") print ttfb - } - ' "$1" -} - -# 从统计行中取字段:tp objs avg p50 p90 p99 ttfb_avg ttfb_p99 ttfb_worst -field() { - case "$2" in - tp) echo "$1" | sed -n 's/^ *\* Average: \(.*\), \([0-9.]*\) obj\/s.*/\1/p' ;; - objs) echo "$1" | sed -n 's/^ *\* Average: .*, \([0-9.]*\) obj\/s.*/\1/p' ;; - avg) echo "$1" | sed -n 's/^ *\* Reqs: Avg: \([^,]*\),.*/\1/p' ;; - p50) echo "$1" | sed -n 's/^ *\* Reqs: Avg: [^,]*, 50%: \([^,]*\),.*/\1/p' ;; - p90) echo "$1" | sed -n 's/^ *\* Reqs: Avg: [^,]*, 50%: [^,]*, 90%: \([^,]*\),.*/\1/p' ;; - p99) echo "$1" | sed -n 's/^ *\* Reqs: Avg: [^,]*, 50%: [^,]*, 90%: [^,]*, 99%: \([^,]*\),.*/\1/p' ;; - ttfb_avg) echo "$1" | sed -n 's/^ *\* TTFB: Avg: \([^,]*\),.*/\1/p' ;; - ttfb_p99) echo "$1" | sed -n 's/^ *\* TTFB: .*99th: \([^,]*\),.*/\1/p' ;; - ttfb_worst) echo "$1" | sed -n 's/^ *\* TTFB: .*Worst: \([^ ]*\).*/\1/p' ;; - *) echo "" ;; - esac -} - -# 解析一轮输出,追加一行到 summary.tsv -parse_round() { - local method="$1" size="$2" file="$3" - local line - if [[ "${method}" == "mixed" ]]; then - local total get put - total=$(section_lines "$file" Total) - get=$(section_lines "$file" GET) - put=$(section_lines "$file" PUT) - line=$(printf 'mixed\t%s\t%s\t%s\t%s\t%s' \ - "$size" \ - "$(field "${total}" tp)" \ - "$(field "${total}" objs)" \ - "$(field "${get}" avg)" \ - "$(field "${put}" avg)") - else - local sec - sec=$(printf '%s' "${method}" | tr '[:lower:]' '[:upper:]') - local stats - stats=$(section_lines "$file" "${sec}") - line=$(printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s' \ - "${method}" "${size}" \ - "$(field "${stats}" tp)" \ - "$(field "${stats}" objs)" \ - "$(field "${stats}" avg)" \ - "$(field "${stats}" p50)" \ - "$(field "${stats}" p90)" \ - "$(field "${stats}" p99)" \ - "$(field "${stats}" ttfb_avg)" \ - "$(field "${stats}" ttfb_p99)" \ - "$(field "${stats}" ttfb_worst)") - fi - printf '%s\n' "${line}" >> "${SUMMARY_TSV}" -} - -# 汇总 summary.tsv -> summary.md(Markdown 表格) -gen_summary_md() { - { - echo "# RustFS 性能压测结果" - echo "" - echo "- 日期:$(date -u '+%Y-%m-%d %H:%M:%S UTC')" - echo "- 目标:${HOST}" - echo "- 并发:${CONCURRENCY};单轮:${DURATION};sleep:${SLEEP_BETWEEN_ROUNDS}s;GET objects:${GET_OBJECTS}" - echo "- 方法:GET / PUT / MIXED(warp 默认混合负载);尺寸:${SIZES[*]}" - echo "" - } > "${SUMMARY_MD}" - - for m in get put; do - { - echo "## $(printf '%s' "$m" | tr '[:lower:]' '[:upper:]') 结果" - echo "" - echo "| 对象尺寸 | 平均吞吐 | 平均 obj/s | Avg Latency | P50 | P90 | P99 | TTFB Avg | TTFB P99 | TTFB 最差 |" - echo "|----------|----------|-----------|-------------|-----|-----|-----|----------|----------|-----------|" - } >> "${SUMMARY_MD}" - while IFS=$'\t' read -r method size tp objs avg p50 p90 p99 ttfb_avg ttfb_p99 ttfb_worst; do - [[ "${method}" == "${m}" ]] && \ - echo "| ${size} | ${tp} | ${objs} | ${avg} | ${p50} | ${p90} | ${p99} | ${ttfb_avg} | ${ttfb_p99} | ${ttfb_worst} |" >> "${SUMMARY_MD}" - done < "${SUMMARY_TSV}" - echo "" >> "${SUMMARY_MD}" - done - - { - echo "## MIXED 结果(Total 口径)" - echo "" - echo "| 对象尺寸 | Total 平均吞吐 | Total 平均 obj/s | Mixed-GET Avg | Mixed-PUT Avg |" - echo "|----------|---------------|------------------|----------------|----------------|" - } >> "${SUMMARY_MD}" - while IFS=$'\t' read -r method size tp objs gavg pavg rest; do - [[ "${method}" == "mixed" ]] && \ - echo "| ${size} | ${tp} | ${objs} | ${gavg} | ${pavg} |" >> "${SUMMARY_MD}" - done < "${SUMMARY_TSV}" - echo "" >> "${SUMMARY_MD}" -} - -# ---- 主流程 ---- - -mkdir -p "${RESULT_DIR}" -log "CONFIG host=${HOST} bucket=${BUCKET} concurrency=${CONCURRENCY} duration=${DURATION} get_objects=${GET_OBJECTS} sleep_between_rounds=${SLEEP_BETWEEN_ROUNDS}s" -printf 'method\tsize\tthroughput\tobj_per_s\treq_avg\treq_p50\treq_p90\treq_p99\tttfb_avg\tttfb_p99\tttfb_worst\n' > "${SUMMARY_TSV}" - -if [[ "${1:-}" == "--parse-only" ]]; then - for method in "${METHODS[@]}"; do - for size in "${SIZES[@]}"; do - outfile="${RESULT_DIR}/${method}_${size}.txt" - if [[ -s "${outfile}" ]]; then - parse_round "${method}" "${size}" "${outfile}" - fi - done - done - gen_summary_md - echo "parsed from ${RESULT_DIR}" - echo "" - cat "${SUMMARY_MD}" - exit 0 -fi - -for method in "${METHODS[@]}"; do - for size in "${SIZES[@]}"; do - ROUND=$((ROUND + 1)) - outfile="${RESULT_DIR}/${method}_${size}.txt" - - log "START round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} concurrency=${CONCURRENCY} duration=${DURATION}" - - extra_args=() - if [[ "${method}" != "put" ]]; then - extra_args=(--objects "${GET_OBJECTS}") - fi - - start_epoch=$(date +%s) - warp "${method}" \ - --host "${HOST}" \ - --access-key "${ACCESS_KEY}" \ - --secret-key "${SECRET_KEY}" \ - --bucket "${BUCKET}" \ - --concurrent "${CONCURRENCY}" \ - --duration "${DURATION}" \ - --obj.size "${size}" \ - "${extra_args[@]}" \ - --no-color 2>&1 | tee "${outfile}" - rc=${PIPESTATUS[0]} - end_epoch=$(date +%s) - - if [[ ${rc} -eq 0 ]]; then - parse_round "${method}" "${size}" "${outfile}" - log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=ok" - else - log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=skipped" - fi - - if [[ ${ROUND} -lt ${TOTAL_ROUNDS} ]]; then - log "SLEEP ${SLEEP_BETWEEN_ROUNDS}s before next round" - sleep "${SLEEP_BETWEEN_ROUNDS}" - fi - done -done - -gen_summary_md -log "ALL_ROUNDS_COMPLETE summary_tsv=${SUMMARY_TSV} summary_md=${SUMMARY_MD}" -echo "" -echo "==== 结果汇总 ====" -cat "${SUMMARY_MD}" diff --git a/scripts/test/rustfs_pool_expand.md b/scripts/test/rustfs_pool_expand.md deleted file mode 100644 index b73f3c0c5..000000000 --- a/scripts/test/rustfs_pool_expand.md +++ /dev/null @@ -1,106 +0,0 @@ -# RustFS Pool Expansion / Decommission Test - -End-to-end test for **storage pool expansion**, **data rebalancing** and -**pool decommission**, driven by -[`scripts/test/rustfs_pool_expand.sh`](rustfs_pool_expand.sh). - -All pool / rebalance / decommission checks talk to the RustFS **admin API -directly** with SigV4-signed HTTP requests and assert on the JSON responses -(`jq`), so the results are exact and independent of any client CLI output -formatting. **`rc` is not required.** - -## What it does - -1. Downloads the RustFS `.deb` package on all nodes (a release tag, or a - direct URL such as a nightly/R2 package). -2. Installs it (`dpkg -i`). -3. Starts the first pool and verifies it via - `GET /rustfs/admin/v3/pools/list`. -4. Writes data with `warp` and monitors storage usage - (`GET /rustfs/admin/v3/storageinfo`) until the threshold. -5. Expands to a second pool (nodes started in parallel). -6. Starts rebalance (`POST /rustfs/admin/v3/rebalance/start`) and waits for - **all** pools to report `Completed`. -7. (3-pool mode) Expands to a third pool, rebalances again. -8. (optional) Decommissions pool 0 - (`POST /rustfs/admin/v3/pools/decommission`), with automatic clear + retry - on failure, and waits for `decommissionInfo.complete == true`. - -When an assertion fails, the script prints a per-pool summary **and the full -JSON response** (pool state, progress, failure counters, `waitingReason`, -`unresolvedEntries`, last rebalance error), so the GitHub Actions log shows -exactly where the test stopped. Credentials never appear in the logs. - -## Self-hosted runner prerequisites - -- Register the admin host (e.g. `heal`) as a runner with the - `smoke-testing` label. -- Install `jq`, `openssl`, `curl` and `warp` (only needed for `--with-warp`) - on the runner. `rc` is **not** required. -- The runner user must be able to SSH to all nodes without a password prompt - (`~/.ssh/config` with keys). -- The nodes need passwordless `sudo` for the SSH user, resolvable - `rustfs-node*` hostnames in `/etc/hosts`, and writable data directories. -- The admin API credentials must have the `admin:server-info`, - `admin:decommission` and `admin:rebalance` actions. - -## Configuration - -Set these in the repository (secrets/variables): - -| Kind | Name | Purpose | -| ------ | --------------------- | ---------------------------------------------- | -| Secret | `RUSTFS_ACCESS_KEY` | RustFS access key | -| Secret | `RUSTFS_SECRET_KEY` | RustFS secret key | -| Var | `RUSTFS_API_ENDPOINT` | Admin API endpoint, e.g. `http://10.0.0.7:9000` (`RUSTFS_RC_ENDPOINT` is used as a fallback) | -| Var | `RUSTFS_NODES` | Space-separated node names, e.g. `vm000 vm001 vm002` | -| Var | `RUSTFS_SSH_USER` | SSH user for the nodes, e.g. `azureuser` | - -## Workflow inputs - -| Input | Default | Meaning | -| ------------------- | ------------- | ---------------------------------------- | -| `rustfs_version` | `1.0.0-rc.3` | GitHub release tag to test | -| `package_url` | *(empty)* | Direct `.deb` URL (e.g. nightly/R2); overrides `rustfs_version` | -| `pools` | `3` | Expand to 2 or 3 pools | -| `storage_threshold` | `50` | Stop warp writes at N% usage | -| `warp_duration` | `10m` | warp write duration | -| `run_decommission` | `true` | Run decommission (3-pool mode only) | -| `cleanup_before` | `true` | Reset nodes before the test | -| `cleanup_after` | `true` | Reset nodes after the test | - -> ⚠️ `cleanup_before` / `cleanup_after` run the script's `--reset` mode, which -> **stops the services and deletes the data directories and config** on all -> nodes. Only use this workflow against a dedicated test environment. - -## Manual usage - -```bash -# Full workflow with a release tag -./scripts/test/rustfs_pool_expand.sh --all --with-warp -y \ - --version 1.0.0-rc.3 --endpoint http://10.0.0.7:9000 - -# Use a direct .deb URL (e.g. nightly package on R2) -./scripts/test/rustfs_pool_expand.sh --all --with-warp -y \ - --package-url https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb \ - --endpoint http://10.0.0.7:9000 - -# Preflight / reset / single step -./scripts/test/rustfs_pool_expand.sh --preflight --version 1.0.0-rc.3 -./scripts/test/rustfs_pool_expand.sh --reset -y -./scripts/test/rustfs_pool_expand.sh --step 9 --finalize-decommission -y -``` - -## Known issues and caveats - -- RustFS `1.0.0-rc.3` fails decommission with a large `warp`-written bucket - ("metacache listing quorum failed / timeout"). If decommission repeatedly - fails, reduce the written data (lower `storage_threshold`) or remove the test - bucket, then re-run step 9. The script detects the failure, clears metadata - and retries `DECOMMISSION_RETRIES` times before giving up. -- Multi-pool nodes must start **simultaneously** (the script does this) or the - first node dies with `not first disk`. -- The admin API is SigV4-signed (`host`, `x-amz-content-sha256: - UNSIGNED-PAYLOAD`, `x-amz-date`), matching the signer the RustFS server - itself trusts. If the cluster requires a non-default region, set - `SIGV4_REGION` in the script. diff --git a/scripts/test/rustfs_pool_expand.sh b/scripts/test/rustfs_pool_expand.sh deleted file mode 100755 index ad2e0bbb6..000000000 --- a/scripts/test/rustfs_pool_expand.sh +++ /dev/null @@ -1,1199 +0,0 @@ -#!/usr/bin/env bash -# -# rustfs-pool-expand.sh -# RustFS multi-pool expansion & pool decommission test script -# -# Based on the Obsidian note "RustFS pool 扩容脚本". Full workflow: -# 1. Download the RustFS package on all nodes -# 2. Install the RustFS service on all nodes (dpkg -i) -# 3. Start the first pool on node 0 and verify it -# 4. Write data (warp) and monitor storage usage up to a threshold -# 5. Expand to a second pool (node 0 + node 1), start rebalance -# 6. Wait for rebalance to finish -# 7. Expand to a third pool (all nodes), start rebalance again -# 8. Wait for rebalance to finish -# 9. Decommission pool 0 and wait for it to complete -# -# The script is driven from an admin host (e.g. a jumpbox or a GitHub -# self-hosted runner) and operates on the target nodes over SSH. -# -# Usage: -# ./rustfs-pool-expand.sh --all # run all steps 1-9 -# ./rustfs-pool-expand.sh --step 5 # run a single step -# ./rustfs-pool-expand.sh --steps 5,7,9 # run selected steps -# ./rustfs-pool-expand.sh --all --dry-run # preview only -# ./rustfs-pool-expand.sh --all -y --version 1.0.0-rc.3 -# -# Notes: -# - SSH user defaults to azureuser (passwordless sudo on the nodes); -# pass --ssh-user root if your nodes accept root login. -# - The script talks to the RustFS admin API directly with SigV4-signed -# requests (no rc required). jq, openssl and curl must be installed on the -# admin host; warp is only needed for --with-warp. -# - RUSTFS_VOLUMES must keep existing pool expressions unchanged and in -# order when expanding. -# -set -Eeuo pipefail - -# ==================== Configuration (adjust to your environment) ==================== - -# Target nodes; pool N is enabled by NODES[N-1] (index order matters) -if [ -n "${RUSTFS_NODES:-}" ]; then - read -r -a NODES <<<"${RUSTFS_NODES}" -else - NODES=(vm000 vm001 vm002) -fi - -# SSH user configured on the nodes (azureuser has passwordless sudo on heal's nodes) -SSH_USER="${RUSTFS_SSH_USER:-azureuser}" -SSH_PORT="${RUSTFS_SSH_PORT:-22}" -SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new -p "${SSH_PORT}") - -# Package: version is a GitHub release tag such as "1.0.0-rc.3". -# PACKAGE_URL is derived from RUSTFS_VERSION unless --package-url / PACKAGE_URL is given. -RUSTFS_VERSION="${RUSTFS_VERSION:-1.0.0-rc.3}" -PACKAGE_URL="${PACKAGE_URL:-}" -ARCH="${RUSTFS_ARCH:-amd64}" -PACKAGES_DIR="/home/rustfs/packages" -PACKAGE_FILE="rustfs.deb" -PACKAGE_SHA256="${PACKAGE_SHA256:-}" - -# Admin API endpoint and credentials (SigV4-signed requests, no rc needed). -# RUSTFS_RC_ENDPOINT is honoured as a fallback for existing setups. -API_ENDPOINT="${RUSTFS_API_ENDPOINT:-${RUSTFS_RC_ENDPOINT:-http://10.0.0.7:9000}}" -ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfs@test}" -SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfs@test}" -SIGV4_REGION="us-east-1" -SIGV4_SERVICE="s3" -API_REQUEST_TIMEOUT=60 -# admin_api writes the HTTP status here so callers can read it after the -# command-substitution subshell exits ($$ is identical inside the subshell). -ADMIN_API_CODE_FILE="${TMPDIR:-/tmp}/rustfs-pool-expand-api-code.$$" - -# Complete RUSTFS_VOLUMES value per stage (one space-separated expression per pool) -VOLUMES_1="http://rustfs-node1:9000/data/rustfs{1...4}/mnmd" -VOLUMES_2="http://rustfs-node1:9000/data/rustfs{1...4}/mnmd http://rustfs-node2:9000/data/rustfs{1...4}/mnmd" -VOLUMES_3="http://rustfs-node1:9000/data/rustfs{1...4}/mnmd http://rustfs-node2:9000/data/rustfs{1...4}/mnmd http://rustfs-node3:9000/data/rustfs{1...4}/mnmd" -# Topology after decommissioning pool 0 (used by --finalize-decommission) -VOLUMES_AFTER_DECOMMISSION="http://rustfs-node2:9000/data/rustfs{1...4}/mnmd http://rustfs-node3:9000/data/rustfs{1...4}/mnmd" - -# RustFS service configuration (written to /etc/default/rustfs) -RUSTFS_CONFIG_FILE="/etc/default/rustfs" -RUSTFS_SERVICE="rustfs" -RUSTFS_PACKAGE_NAME="rustfs" -RUSTFS_USER="rustfs" -RUSTFS_ADDRESS=":9000" -RUSTFS_CONSOLE_ADDRESS=":9001" -RUSTFS_CONSOLE_ENABLE=true -RUSTFS_OBS_LOGGER_LEVEL=error -RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/" - -# Data writing & monitoring (step 4) -WARP_BUCKET="test-10mb" -WARP_OBJ_SIZE="100MiB" -WARP_CONCURRENT=32 -WARP_DURATION="5m" -# Warp log path; empty = auto-created unique temp file (the runner user may -# not be able to write a shared /tmp path owned by another user). -WARP_LOG_FILE="${RUSTFS_WARP_LOG_FILE:-}" -STORAGE_THRESHOLD=85 # stop writing when usage reaches N% (note suggests 80-85) -POLL_INTERVAL=30 # status polling interval (seconds) - -# Timeouts (seconds) -REBALANCE_TIMEOUT=86400 -DECOMMISSION_TIMEOUT=86400 -SERVICE_TIMEOUT=300 -DECOMMISSION_RETRIES=3 # auto clear+retry attempts after a failed decommission -DECOMMISSION_RETRY_DELAY=30 # delay between retries (seconds) -REBALANCE_START_RETRIES=6 # rebalance start retries (fleet proof may take ~10-20s after a topology change) -REBALANCE_START_RETRY_DELAY=20 # delay between rebalance start retries (seconds) - -# Pool to decommission (zero-based; 0 in the note) -DECOMMISSION_POOL_ID=0 - -# ==================== Runtime options (set by CLI) ==================== -DRY_RUN=0 -ASSUME_YES=0 -WITH_WARP=0 -SKIP_DOWNLOAD=0 -FINALIZE_DECOMMISSION=0 -PREFLIGHT=0 -RESET=0 -LOG_FILE="" -SELECTED_STEPS=() - -# ==================== Helpers ==================== - -log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; } -warn() { printf '\033[1;33m[WARN]\033[0m %s\n' "$*"; } -die() { printf '\033[1;31m[ERROR]\033[0m %s\n' "$*" >&2; exit 1; } - -confirm() { - # $1: prompt text; bypassed with --yes - if [ "${ASSUME_YES}" -eq 1 ]; then - return 0 - fi - printf '\033[1;33m[CONFIRM]\033[0m %s (y/N) ' "$1" - read -r answer - case "${answer}" in - y|Y|yes|YES) return 0 ;; - *) die "cancelled" ;; - esac -} - -need_cmd() { - # $1: command name; $2: description - if [ "${DRY_RUN}" -eq 1 ]; then return 0; fi - command -v "$1" >/dev/null 2>&1 || die "missing command: $1 ($2); install it first" -} - -# ==================== Admin API (SigV4-signed) ==================== - -sha256_hex() { - # $1: ascii data - printf '%s' "$1" | openssl dgst -sha256 -hex 2>/dev/null | awk '{print $NF}' -} - -hmac_sha256_hex() { - # $1: key in hex, $2: ascii data - printf '%s' "$2" | openssl dgst -sha256 -mac HMAC -macopt "hexkey:$1" -hex 2>/dev/null | awk '{print $NF}' -} - -hex_of_ascii() { - # $1: ascii string -> hex - printf '%s' "$1" | od -An -vtx1 | tr -d ' \n' -} - -# Sort a "k=v&k2=v2" query string by key (values are used verbatim, matching -# the RustFS signer). The same canonical form is used in the URL and in the -# SigV4 canonical request so the server-side verification always agrees. -canonical_query() { - local q="$1" - [ -z "${q}" ] && return 0 - # The trailing newline matters: `while read` drops the last item when the - # input has no final newline (e.g. "a=1&b=2" without a trailing '&'). - printf '%s\n' "${q}" | tr '&' '\n' | while IFS= read -r pair; do - printf '%s=%s\n' "${pair%%=*}" "${pair#*=}" - done | sort | paste -sd '&' - -} - -# Issue an admin API request. Prints the response body on stdout and writes the -# HTTP status (000 on transport failure) to ${ADMIN_API_CODE_FILE}. -admin_api() { - # $1: method, $2: path, $3: query string - local method="$1" path="$2" query="$3" - local amz_date date_stamp host_port - local canonical_headers signed_headers canonical_request string_to_sign - local scope k_date k_region k_service k_signing signature auth - local url tmp code - - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: admin API ${method} ${API_ENDPOINT}${path}${query:+?${query}}" - printf '200' > "${ADMIN_API_CODE_FILE}" - return 0 - fi - - host_port="${API_ENDPOINT#*://}" - host_port="${host_port%%/*}" - - amz_date="$(date -u +%Y%m%dT%H%M%SZ)" - date_stamp="${amz_date:0:8}" - query="$(canonical_query "${query}")" - - canonical_headers="host:${host_port} -x-amz-content-sha256:UNSIGNED-PAYLOAD -x-amz-date:${amz_date} -" - signed_headers="host;x-amz-content-sha256;x-amz-date" - canonical_request="${method} -${path} -${query} -${canonical_headers} -${signed_headers} -UNSIGNED-PAYLOAD" - - scope="${date_stamp}/${SIGV4_REGION}/${SIGV4_SERVICE}/aws4_request" - string_to_sign="AWS4-HMAC-SHA256 -${amz_date} -${scope} -$(sha256_hex "${canonical_request}")" - - k_date="$(hmac_sha256_hex "$(hex_of_ascii "AWS4${SECRET_KEY}")" "${date_stamp}")" - k_region="$(hmac_sha256_hex "${k_date}" "${SIGV4_REGION}")" - k_service="$(hmac_sha256_hex "${k_region}" "${SIGV4_SERVICE}")" - k_signing="$(hmac_sha256_hex "${k_service}" "aws4_request")" - signature="$(hmac_sha256_hex "${k_signing}" "${string_to_sign}")" - auth="AWS4-HMAC-SHA256 Credential=${ACCESS_KEY}/${scope}, SignedHeaders=${signed_headers}, Signature=${signature}" - - url="http://${host_port}${path}${query:+?${query}}" - tmp="$(mktemp)" - code="$(curl -sS --max-time "${API_REQUEST_TIMEOUT}" -o "${tmp}" -w '%{http_code}' \ - -H "Host: ${host_port}" \ - -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ - -H "x-amz-date: ${amz_date}" \ - -H "Authorization: ${auth}" \ - -X "${method}" "${url}")" || code="000" - printf '%s' "${code}" > "${ADMIN_API_CODE_FILE}" - cat "${tmp}" - rm -f "${tmp}" -} - -# Read the HTTP status written by the last admin_api call -admin_api_code() { - cat "${ADMIN_API_CODE_FILE}" -} - -# Run an admin API request and fail loudly (with the full response body) on a -# non-success status. Prints the body on stdout. -admin_api_assert() { - # $1: method, $2: path, $3: query, $4: description - local method="$1" path="$2" query="$3" desc="$4" body code - body="$(admin_api "${method}" "${path}" "${query}")" - code="$(admin_api_code)" - if [ "${code}" != "200" ] && [ "${code}" != "201" ] && [ "${code}" != "204" ]; then - printf '\033[1;31m[ERROR]\033[0m %s failed (HTTP %s)\n' "${desc}" "${code}" >&2 - printf '%s\n' "${body}" >&2 - die "admin API ${method} ${path} -> HTTP ${code}" - fi - printf '%s' "${body}" -} - -# Build the GitHub release download URL from a release tag. -# "1.0.0-rc.3" -> https://github.com/rustfs/rustfs/releases/download/1.0.0-rc.3/rustfs_1.0.0.rc.3_amd64.deb -build_package_url() { - local tag="${RUSTFS_VERSION#v}" asset - asset="${tag//-/.}" - printf 'https://github.com/rustfs/rustfs/releases/download/%s/rustfs_%s_%s.deb' "${tag}" "${asset}" "${ARCH}" -} - -resolve_package_url() { - if [ -n "${PACKAGE_URL}" ]; then - printf '%s' "${PACKAGE_URL}" - else - build_package_url - fi -} - -# Run a remote script on a single node (script is read from stdin) -run_remote() { - local node="$1" - local script - script="$(cat)" - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: ssh ${SSH_USER}@${node} <<'REMOTE'" - printf '%s\n' "${script}" | sed 's/^/ | /' - log "DRY-RUN: ----" - return 0 - fi - log "==> ${node}: executing remote script" - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'bash -s' <<<"${script}" -} - -# Run the same remote script on all nodes in parallel (script from stdin) -run_remote_all() { - local script - script="$(cat)" - local pids=() i=0 fail=0 - for node in "${NODES[@]}"; do - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: ssh ${SSH_USER}@${node} <<'REMOTE'" - printf '%s\n' "${script}" | sed 's/^/ | /' - log "DRY-RUN: ----" - else - log "==> ${node}: executing remote script" - ( ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'bash -s' <<<"${script}" ) & - pids[$i]=$! - i=$((i+1)) - fi - done - if [ "${#pids[@]}" -gt 0 ]; then - for pid in "${pids[@]}"; do - wait "${pid}" || fail=1 - done - fi - [ "${fail}" -eq 0 ] || die "one or more nodes failed" -} - -# Wait for the systemd service to become active on a node -wait_service_active() { - local node="$1" waited=0 - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: waiting for ${RUSTFS_SERVICE} on ${node} to become active" - return 0 - fi - while [ "${waited}" -lt "${SERVICE_TIMEOUT}" ]; do - if ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \ - "systemctl is-active --quiet ${RUSTFS_SERVICE}" 2>/dev/null; then - log "${node}: ${RUSTFS_SERVICE} is active" - return 0 - fi - sleep 5 - waited=$((waited + 5)) - done - diagnose_node_start_failure "${node}" - die "${node}: timed out waiting for ${RUSTFS_SERVICE} (${SERVICE_TIMEOUT}s)" -} - -# Known server-side issues the test can hit. Format: -# "||" -KNOWN_SERVER_ISSUES=( - "pool activation requires a live fleet capability proof|rustfs/backlog#2031|server-side cold-start recovery is covered by this PR; if this appears, collect node journals and treat it as a regression" -) - -# Print a hint when $1 matches a known server-side issue signature. -hint_server_issue() { - local text="$1" entry sig tracking hint - for entry in "${KNOWN_SERVER_ISSUES[@]}"; do - sig="${entry%%|*}" - tracking="${entry#*|}" - hint="${tracking#*|}" - tracking="${tracking%%|*}" - if printf '%s' "${text}" | grep -qiF "${sig}"; then - printf '\033[1;33m[KNOWN SERVER ISSUE]\033[0m %s (%s): %s\n' "${sig}" "${tracking}" "${hint}" >&2 - return 0 - fi - done - return 1 -} - -# Fetch the journal tail from a node whose service failed to start and -# annotate known server-side issues. -diagnose_node_start_failure() { - local node="$1" journal - if ! journal="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \ - "SUDO=\"\"; [ \"\$(id -u)\" -ne 0 ] && SUDO=\"sudo -n\"; \${SUDO} journalctl -u ${RUSTFS_SERVICE} --no-pager -n 60 2>/dev/null || true")"; then - journal="unable to collect journal (SSH command failed)" - fi - printf '%s\n' "--- ${node}: ${RUSTFS_SERVICE} journal (last 60 lines) ---" >&2 - printf '%s\n' "${journal}" >&2 - hint_server_issue "${journal}" || true -} - -# Generate the /etc/default/rustfs content -rustfs_config_body() { - local volumes="$1" - cat < /data/rustfs1/mnmd /data/rustfs2/mnmd /data/rustfs3/mnmd /data/rustfs4/mnmd -volume_dirs() { - local volumes="$1" expr path prefix suffix i start end - for expr in ${volumes}; do - path="${expr#*://}" - path="${path#*/}" - path="/${path}" - if [[ "${path}" =~ \{([0-9]+)\.\.\.([0-9]+)\} ]]; then - start="${BASH_REMATCH[1]}" - end="${BASH_REMATCH[2]}" - prefix="${path%%\{*}" - suffix="${path#*\}}" - for ((i=start; i<=end; i++)); do - printf '%s%s%s\n' "${prefix}" "${i}" "${suffix}" - done - else - printf '%s\n' "${path}" - fi - done -} - -# Ensure the volume directories exist on a node and are owned by the service user -ensure_volume_dirs() { - local node="$1" volumes="$2" - local -a dirs - dirs=() - while IFS= read -r d; do dirs+=("${d}"); done < <(volume_dirs "${volumes}" | sort -u) - log "${node}: ensuring data dirs exist and are owned by ${RUSTFS_USER} (${dirs[*]})" - { - printf 'set -euo pipefail\n' - printf 'SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"\n' - for d in "${dirs[@]}"; do - printf '${SUDO} mkdir -p %s\n' "${d}" - printf '${SUDO} chown -R %s:%s %s\n' "${RUSTFS_USER}" "${RUSTFS_USER}" "${d}" - done - } | run_remote "${node}" -} - -# Write the RustFS config on a single node (with a timestamped backup first) -write_rustfs_config() { - local node="$1" volumes="$2" - local body - body="$(rustfs_config_body "${volumes}")" - log "${node}: writing config ${RUSTFS_CONFIG_FILE} (volumes=${volumes})" - { - printf 'set -euo pipefail\n' - printf 'SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"\n' - printf 'if [ -f %s ]; then ${SUDO} cp -a %s %s.bak.$(date +%%Y%%m%%d%%H%%M%%S); fi\n' \ - "${RUSTFS_CONFIG_FILE}" "${RUSTFS_CONFIG_FILE}" "${RUSTFS_CONFIG_FILE}" - printf '%s tee %s >/dev/null </dev/null 2>&1; then - printf '\033[1;31m[ERROR]\033[0m storageinfo response unexpected (expected JSON with .info.disks or .disks):\n%s\n' "${body}" >&2 - die "storageinfo response could not be parsed" - fi - printf '%s' "${body}" | jq -r ' - ([.info.disks // .disks // [] | .[] | select(.state != "offline" and .totalspace > 0) | .usedspace] | add // 0) as $used - | ([.info.disks // .disks // [] | .[] | select(.state != "offline" and .totalspace > 0) | .totalspace] | add // 0) as $total - | if $total > 0 then ($used / $total * 100) else 0 end - ' -} - -# Poll storage usage until it reaches the threshold -monitor_storage() { - local threshold="${1:-${STORAGE_THRESHOLD}}" warp_pid="${2:-}" usage - log "monitoring storage usage until ${threshold}% (every ${POLL_INTERVAL}s)" - while :; do - usage="$(storage_usage_percent)" - [ -n "${usage}" ] || usage="0" - log "current storage usage: ${usage}%" - awk -v u="${usage}" -v t="${threshold}" 'BEGIN{exit !(u >= t)}' && break - if [ -n "${warp_pid}" ] && ! kill -0 "${warp_pid}" 2>/dev/null; then - warn "warp finished at ${usage}%, below the threshold of ${threshold}%" - warn "re-run step 4 to write more data; the data already written is enough to exercise the remaining steps" - return 0 - fi - sleep "${POLL_INTERVAL}" - done - log "storage usage reached ${usage}%, stopping writes" -} - -# Verify the expected number of pools via the admin API (JSON + jq assertions) -verify_pools() { - local expected="$1" - local body code count nonactive - log "verifying pools via admin API (expect at least ${expected} active): GET ${API_ENDPOINT}/rustfs/admin/v3/pools/list" - if [ "${DRY_RUN}" -eq 1 ]; then return 0; fi - body="$(admin_api GET /rustfs/admin/v3/pools/list "")" - code="$(admin_api_code)" - if [ "${code}" != "200" ]; then - printf '\033[1;31m[ERROR]\033[0m pools/list returned HTTP %s\n' "${code}" >&2 - printf '%s\n' "${body}" >&2 - die "pools/list failed" - fi - if ! count="$(printf '%s' "${body}" | jq -e 'length' 2>/dev/null)"; then - printf '\033[1;31m[ERROR]\033[0m pools/list returned an unexpected (non-JSON) body:\n%s\n' "${body}" >&2 - die "pools/list response could not be parsed as JSON" - fi - nonactive="$(printf '%s' "${body}" | jq '[.[] | select(.status != "active")] | length')" - if [ "${count}" -lt "${expected}" ] || [ "${nonactive}" -ne 0 ]; then - printf '\033[1;31m[ERROR]\033[0m pool assertion failed: expected >=%s active pools, got %s (non-active: %s)\n' \ - "${expected}" "${count}" "${nonactive}" >&2 - printf '%s\n' "--- pool detail ---" >&2 - printf '%s' "${body}" | jq -r '.[] | " pool id=\(.id) status=\(.status) decommission=\(.decommissionStatus) rebalance=\(.rebalanceStatus) used=\(.usedSize)/\(.totalSize) cmdline=\(.cmdline)"' >&2 - printf '%s\n' "--- full JSON ---" >&2 - printf '%s\n' "${body}" >&2 - die "pool verification failed" - fi - log "pools OK: ${count} pools, all active" - printf '%s' "${body}" | jq -r '.[] | " pool id=\(.id) status=\(.status) decommission=\(.decommissionStatus) rebalance=\(.rebalanceStatus)"' -} - -# Print a detailed rebalance failure report (never includes credentials) -print_rebalance_failure() { - local body="$1" reason="$2" - printf '\033[1;31m[ERROR]\033[0m %s\n' "${reason}" >&2 - printf '%s\n' "--- rebalance summary ---" >&2 - printf '%s' "${body}" | jq -r ' - " rebalance id=\(.id) stoppedAt=\(.stoppedAt // "null")", - " stopPropagation: pendingTerminalReload=\(.stopPropagation.pendingTerminalReload // false) failedPeers=[\(.stopPropagation.failedPeers // [] | join(", "))]", - (.pools[] | - " pool id=\(.id) status=\(.status) stopping=\(.stopping) used=\(.used) lastError=\(.lastError // "null")" - + (if .progress != null then - " progress: objects=\(.progress.objects) bytes=\(.progress.bytes) remainingBuckets=\(.progress.remainingBuckets) bucket=\(.progress.bucket) object=\(.progress.object) elapsed=\(.progress.elapsed)s eta=\(.progress.eta)s" - else " progress: null" end))' >&2 - printf '%s\n' "--- full JSON ---" >&2 - printf '%s\n' "${body}" >&2 -} - -# Wait for rebalance to complete (all pools must finish, not just one) -wait_rebalance() { - local waited=0 body code active failed completed total last_error remaining - log "waiting for rebalance to complete via admin API (timeout ${REBALANCE_TIMEOUT}s)..." - while [ "${waited}" -lt "${REBALANCE_TIMEOUT}" ]; do - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: waiting for rebalance to complete" - return 0 - fi - body="$(admin_api GET /rustfs/admin/v3/rebalance/status "")" - code="$(admin_api_code)" - if [ "${code}" != "200" ]; then - # 404 means rebalance metadata has not been persisted yet (e.g. right - # after start); keep polling. 000 is a transport error; also retry. - if [ "${code}" = "404" ] || [ "${code}" = "000" ]; then - sleep "${POLL_INTERVAL}" - waited=$((waited + POLL_INTERVAL)) - continue - fi - printf '\033[1;31m[ERROR]\033[0m rebalance/status returned HTTP %s\n' "${code}" >&2 - printf '%s\n' "${body}" >&2 - die "rebalance/status failed" - fi - failed="$(printf '%s' "${body}" | jq '[.pools[] | select(.status == "Failed" or .status == "Stopped")] | length')" - last_error="$(printf '%s' "${body}" | jq '[.pools[] | select(.lastError != null)] | length')" - active="$(printf '%s' "${body}" | jq '[.pools[] | select(.status == "Started")] | length')" - completed="$(printf '%s' "${body}" | jq '[.pools[] | select(.status == "Completed")] | length')" - total="$(printf '%s' "${body}" | jq '.pools | length')" - remaining="$(printf '%s' "${body}" | jq '[.pools[] | select(.progress != null) | .progress.remainingBuckets] | add // 0')" - log "rebalance: completed=${completed}/${total} active=${active} remainingBuckets=${remaining}" - if [ "${failed}" -gt 0 ] || [ "${last_error}" -gt 0 ]; then - print_rebalance_failure "${body}" "rebalance entered a failure state (failed=${failed}, lastError=${last_error})" - die "rebalance failed (see detail above)" - fi - if [ "${active}" -eq 0 ] && [ "${completed}" -gt 0 ]; then - log "rebalance completed: ${completed}/${total} pools completed" - printf '%s' "${body}" | jq -r '.pools[] | " pool id=\(.id) status=\(.status) stopping=\(.stopping) used=\(.used)"' - return 0 - fi - sleep "${POLL_INTERVAL}" - waited=$((waited + POLL_INTERVAL)) - done - body="$(admin_api GET /rustfs/admin/v3/rebalance/status "")" - print_rebalance_failure "${body}" "timed out waiting for rebalance (${REBALANCE_TIMEOUT}s)" - die "timed out waiting for rebalance (${REBALANCE_TIMEOUT}s)" -} - -# Start rebalance via the admin API. Nightly builds gate rebalance activation -# on a live cross-pool fence fleet capability proof that is re-established -# shortly after a pool joins, so retry a few times before failing. -start_rebalance_with_retry() { - local attempts="${REBALANCE_START_RETRIES}" delay="${REBALANCE_START_RETRY_DELAY}" attempt=1 body code id - while :; do - body="$(admin_api POST /rustfs/admin/v3/rebalance/start "")" - code="$(admin_api_code)" - if [ "${code}" = "200" ]; then - id="$(printf '%s' "${body}" | jq -r '.id // empty')" - log "rebalance started: id=${id}" - return 0 - fi - warn "rebalance start attempt ${attempt}/${attempts} failed (HTTP ${code}): ${body}" - if [ "${attempt}" -ge "${attempts}" ]; then - hint_server_issue "${body}" || true - die "rebalance start failed after ${attempts} attempts (see last error above)" - fi - attempt=$((attempt + 1)) - sleep "${delay}" - done -} - -# Print a detailed decommission failure/progress report for one pool -print_decommission_detail() { - local body="$1" pool_id="$2" label="$3" - printf '%s\n' "--- decommission detail (pool ${pool_id}) ${label} ---" >&2 - printf '%s' "${body}" | jq -r --argjson id "${pool_id}" ' - .pools[] | select(.id == $id) | - " pool id=\(.id) status=\(.status) poolStatus=\(.poolStatus)" - + (if .decommissionInfo != null then - " complete=\(.decommissionInfo.complete) failed=\(.decommissionInfo.failed) canceled=\(.decommissionInfo.canceled) queued=\(.decommissionInfo.queued)", - " stage=\(.decommissionInfo.stage // "null") bucket=\(.decommissionInfo.bucket // "null") prefix=\(.decommissionInfo.prefix // "null") object=\(.decommissionInfo.object // "null")", - " waitingReason=\(.decommissionInfo.waitingReason // "null")", - " objectsDecommissioned=\(.decommissionInfo.objectsDecommissioned) objectsDecommissionedFailed=\(.decommissionInfo.objectsDecommissionedFailed)", - " bytesDecommissioned=\(.decommissionInfo.bytesDecommissioned) bytesDecommissionedFailed=\(.decommissionInfo.bytesDecommissionedFailed)", - " size current=\(.decommissionInfo.currentSize)/\(.decommissionInfo.totalSize)", - " queuedBuckets=[\(.decommissionInfo.queuedBuckets | join(", "))]", - " decommissionedBuckets=[\(.decommissionInfo.decommissionedBuckets | join(", "))]", - (if (.decommissionInfo.unresolvedEntries // [] | length) > 0 then - " unresolvedEntries=" + ([.decommissionInfo.unresolvedEntries[] | "bucket=" + .bucket + " object=" + .object + " reason=" + .reason] | join("; ")) - else empty end) - else " decommissionInfo=null (no decommission state recorded)" end)' >&2 - printf '%s\n' "--- full JSON ---" >&2 - printf '%s\n' "${body}" >&2 -} - -# Wait for a pool decommission to complete -wait_decommission() { - local pool_id="$1" waited=0 body code - local complete failed canceled queued objects_failed bytes_failed remaining done - log "waiting for pool ${pool_id} decommission to complete via admin API (timeout ${DECOMMISSION_TIMEOUT}s)..." - while [ "${waited}" -lt "${DECOMMISSION_TIMEOUT}" ]; do - if [ "${DRY_RUN}" -eq 1 ]; then - log "DRY-RUN: waiting for decommission to complete" - return 0 - fi - body="$(admin_api GET /rustfs/admin/v3/decommission/status "")" - code="$(admin_api_code)" - if [ "${code}" != "200" ]; then - printf '\033[1;31m[ERROR]\033[0m decommission/status returned HTTP %s\n' "${code}" >&2 - printf '%s\n' "${body}" >&2 - die "decommission/status failed" - fi - if ! printf '%s' "${body}" | jq -e --argjson id "${pool_id}" '.pools[] | select(.id == $id)' >/dev/null 2>&1; then - print_decommission_detail "${body}" "${pool_id}" "pool missing" - die "pool ${pool_id} not found in decommission status" - fi - complete="$(printf '%s' "${body}" | jq -r --argjson id "${pool_id}" '.pools[] | select(.id == $id) | .decommissionInfo.complete // false')" - failed="$(printf '%s' "${body}" | jq -r --argjson id "${pool_id}" '.pools[] | select(.id == $id) | .decommissionInfo.failed // false')" - canceled="$(printf '%s' "${body}" | jq -r --argjson id "${pool_id}" '.pools[] | select(.id == $id) | .decommissionInfo.canceled // false')" - queued="$(printf '%s' "${body}" | jq -r --argjson id "${pool_id}" '.pools[] | select(.id == $id) | .decommissionInfo.queued // false')" - objects_failed="$(printf '%s' "${body}" | jq -r --argjson id "${pool_id}" '.pools[] | select(.id == $id) | .decommissionInfo.objectsDecommissionedFailed // 0')" - bytes_failed="$(printf '%s' "${body}" | jq -r --argjson id "${pool_id}" '.pools[] | select(.id == $id) | .decommissionInfo.bytesDecommissionedFailed // 0')" - remaining="$(printf '%s' "${body}" | jq -r --argjson id "${pool_id}" '.pools[] | select(.id == $id) | (.decommissionInfo.queuedBuckets // [] | length)')" - done="$(printf '%s' "${body}" | jq -r --argjson id "${pool_id}" '.pools[] | select(.id == $id) | (.decommissionInfo.decommissionedBuckets // [] | length)')" - if [ "${failed}" = "true" ] || [ "${canceled}" = "true" ] \ - || [ "${objects_failed}" -gt 0 ] || [ "${bytes_failed}" -gt 0 ]; then - print_decommission_detail "${body}" "${pool_id}" "FAILED" - warn "pool ${pool_id} decommission failed/canceled (objects_failed=${objects_failed}, bytes_failed=${bytes_failed}); see detail above" - return 1 - fi - if [ "${complete}" = "true" ]; then - log "pool ${pool_id} decommission completed" - printf '%s' "${body}" | jq -r --argjson id "${pool_id}" '.pools[] | select(.id == $id) | " pool id=\(.id) status=\(.status) complete=\(.decommissionInfo.complete) objects=\(.decommissionInfo.objectsDecommissioned) bytes=\(.decommissionInfo.bytesDecommissioned) waitingReason=\(.decommissionInfo.waitingReason // "null")"' - return 0 - fi - log "pool ${pool_id} decommission in progress: doneBuckets=${done} queuedBuckets=${remaining} objects_failed=${objects_failed} queued=${queued}" - sleep "${POLL_INTERVAL}" - waited=$((waited + POLL_INTERVAL)) - done - body="$(admin_api GET /rustfs/admin/v3/decommission/status "")" - print_decommission_detail "${body}" "${pool_id}" "TIMED OUT" - die "timed out waiting for pool ${pool_id} decommission (${DECOMMISSION_TIMEOUT}s)" -} - -# Preflight checks before running the workflow -preflight() { - log "preflight checks" - need_cmd ssh "SSH client" - need_cmd curl "HTTP client" - need_cmd jq "JSON processor" - need_cmd openssl "OpenSSL (SigV4 signing)" - local url - url="$(resolve_package_url)" - log "package URL: ${url}" - if [ "${DRY_RUN}" -eq 0 ]; then - curl -fsSI --max-time 20 "${url}" >/dev/null || die "package URL not reachable: ${url}" - for node in "${NODES[@]}"; do - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" "hostname" >/dev/null \ - || die "cannot SSH to ${node}" - log "${node}: SSH OK" - done - log "admin API connectivity: GET ${API_ENDPOINT}/rustfs/admin/v3/pools/list" - local body code - body="$(admin_api GET /rustfs/admin/v3/pools/list "")" - code="$(admin_api_code)" - if [ "${code}" != "200" ]; then - # In the automated workflow the cluster is stopped before preflight - # (reset), so an unreachable admin API is expected there; the test - # starts the cluster and verifies the API in step 3. Only fail hard - # when the service is actually running but the API is broken. - local service_state - service_state="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${NODES[0]}" \ - "systemctl is-active ${RUSTFS_SERVICE} 2>/dev/null || true")" - if [ "${service_state}" = "active" ]; then - printf '\033[1;31m[ERROR]\033[0m admin API check failed (HTTP %s) while ${RUSTFS_SERVICE} is active on %s\n' \ - "${code}" "${NODES[0]}" >&2 - printf '%s\n' "${body}" >&2 - die "cannot reach the admin API at ${API_ENDPOINT} with the configured credentials" - fi - printf '\033[1;33m[WARN]\033[0m admin API not reachable (HTTP %s) — ${RUSTFS_SERVICE} is not active on %s; the test will start the cluster and verify the API in step 3\n' \ - "${code}" "${NODES[0]}" >&2 - printf '%s\n' "${body}" >&2 - else - log "admin API OK ($(printf '%s' "${body}" | jq 'length') pool(s) listed)" - fi - # Check that each node resolves the rustfs-node* hostnames used by the volumes - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${NODES[0]}" \ - "grep -q rustfs-node /etc/hosts" || warn "rustfs-node* hostnames not found in /etc/hosts on ${NODES[0]}" - log "node disk space (/data):" - for node in "${NODES[@]}"; do - ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \ - "df -h /data | tail -1 | awk '{print \" ${node}: \" \$2 \" total, \" \$4 \" avail\"}'" || true - done - fi - if [ "${WITH_WARP}" -eq 1 ]; then - need_cmd warp "warp benchmark tool" - fi - log "preflight OK" -} - -# Reset the test environment: purge the rustfs package (if installed) and -# recreate the data directories on all nodes. Intended for CI so every run -# starts from a clean slate. Destructive! -step0_reset() { - log "reset: purge rustfs package and recreate data dirs on all nodes" - confirm "This DESTROYS the RustFS install and ALL data on ${NODES[*]} (irreversible). Continue?" - for node in "${NODES[@]}"; do - { - printf 'set -euo pipefail\n' - printf 'SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"\n' - # Stop the service if it is still running, then purge the package - # (skipped when rustfs is not installed). - printf '${SUDO} systemctl stop %s 2>/dev/null || true\n' "${RUSTFS_SERVICE}" - printf 'if ${SUDO} dpkg -l %s 2>/dev/null | grep -q "^ii"; then\n' "${RUSTFS_PACKAGE_NAME}" - printf ' ${SUDO} dpkg -P %s\n' "${RUSTFS_PACKAGE_NAME}" - printf ' echo "purged %s"\n' "${RUSTFS_PACKAGE_NAME}" - printf 'else\n' - printf ' echo "%s not installed, skip purge"\n' "${RUSTFS_PACKAGE_NAME}" - printf 'fi\n' - # Ensure the service user exists (created by the package postinst on - # install; a purge keeps it, but a never-installed node needs it for chown). - printf 'id -u %s >/dev/null 2>&1 || ${SUDO} useradd -r -s /bin/false -d /opt/%s %s\n' \ - "${RUSTFS_USER}" "${RUSTFS_USER}" "${RUSTFS_USER}" - # Recreate the volume directories with the service user as owner. - printf 'for i in 1 2 3 4; do\n' - printf ' ${SUDO} rm -rf /data/rustfs${i}/mnmd\n' - printf ' ${SUDO} mkdir -p /data/rustfs${i}/mnmd\n' - printf ' ${SUDO} chown -R %s:%s /data/rustfs${i}/mnmd\n' "${RUSTFS_USER}" "${RUSTFS_USER}" - printf 'done\n' - } | run_remote "${node}" - done - log "reset complete" -} - -# ==================== Steps ==================== - -step1_download() { - log "step 1: download the package on all nodes" - local script url - url="$(resolve_package_url)" - script="$(cat <"${warp_log}" 2>&1 & - warp_pid=$! - log "warp PID=${warp_pid}" - trap 'kill "${warp_pid:-}" 2>/dev/null || true' EXIT - monitor_storage "${STORAGE_THRESHOLD}" "${warp_pid}" - kill "${warp_pid}" 2>/dev/null || true - trap - EXIT - wait "${warp_pid}" 2>/dev/null || true - log "warp stopped" - else - cat <<'NOTE' -Write data manually (or re-run with --with-warp to automate): - - warp put --host "rustfs-node1:9000" --bucket "test-10mb" \ - --access-key "rustfs@test" --secret-key "rustfs@test" \ - --obj.size "100MiB" --concurrent 32 --noprefix --duration 5m --noclear - -While writing, monitor storage usage: - GET ${API_ENDPOINT}/rustfs/admin/v3/storageinfo (SigV4-signed) - e.g. Storage: 30.88 GiB / 1.56 TiB (1%) - -Stop writing once usage reaches 80%-85%, then continue. -NOTE - confirm "Data has been written and usage reached 80%-85%? Continue?" - fi -} - -step5_expand_pool2() { - log "step 5: expand to the second pool (${NODES[0]} + ${NODES[1]})" - confirm "About to stop ${NODES[0]} and update its config. Continue?" - service_action stop "${NODES[0]}" - write_rustfs_config "${NODES[0]}" "${VOLUMES_2}" - write_rustfs_config "${NODES[1]}" "${VOLUMES_2}" - ensure_volume_dirs "${NODES[0]}" "${VOLUMES_2}" - ensure_volume_dirs "${NODES[1]}" "${VOLUMES_2}" - start_and_wait_nodes "${NODES[0]}" "${NODES[1]}" - verify_pools 2 -} - -step6_rebalance() { - log "step 6: start data rebalance (admin API)" - confirm "About to start rebalance (POST ${API_ENDPOINT}/rustfs/admin/v3/rebalance/start). Continue?" - if [ "${DRY_RUN}" -eq 0 ]; then - start_rebalance_with_retry - fi - wait_rebalance -} - -step7_expand_pool3() { - log "step 7: expand to the third pool (all nodes)" - confirm "About to stop ${NODES[0]} ${NODES[1]} and update all node configs. Continue?" - service_action stop "${NODES[0]}" - service_action stop "${NODES[1]}" - for node in "${NODES[@]}"; do - write_rustfs_config "${node}" "${VOLUMES_3}" - done - for node in "${NODES[@]}"; do - ensure_volume_dirs "${node}" "${VOLUMES_3}" - done - start_and_wait_nodes "${NODES[@]}" - verify_pools 3 -} - -step8_rebalance() { - log "step 8: start data rebalance (admin API)" - confirm "About to start rebalance (POST ${API_ENDPOINT}/rustfs/admin/v3/rebalance/start). Continue?" - if [ "${DRY_RUN}" -eq 0 ]; then - start_rebalance_with_retry - fi - wait_rebalance -} - -# Abort any stale in-flight multipart uploads in a bucket. warp is killed at -# the write threshold and can leave incomplete uploads behind; decommission's -# post-check refuses to finalize a pool that still contains them ("resolve it -# before retrying decommission"). -abort_stale_multipart_uploads() { - local bucket="$1" body code key id - body="$(admin_api GET "/${bucket}" "uploads")" - code="$(admin_api_code)" - if [ "${code}" != "200" ]; then - warn "list multipart uploads failed (HTTP ${code}); continuing without cleanup" - return 0 - fi - # Split the one-line XML into one ... block per line. - while IFS= read -r block; do - [ -z "${block}" ] && continue - key="$(printf '%s' "${block}" | grep -oE '[^<]+' | sed 's###g')" - id="$(printf '%s' "${block}" | grep -oE '[^<]+' | sed 's###g')" - if [ -n "${key}" ] && [ -n "${id}" ]; then - log "aborting stale multipart upload ${id} for ${key}" - admin_api DELETE "/${bucket}/${key}" "uploadId=${id}" "" - [ "$(admin_api_code)" = "200" ] || [ "$(admin_api_code)" = "204" ] \ - || warn "abort multipart upload returned HTTP $(admin_api_code)" - fi - done < <(printf '%s' "${body}" | sed 's##\n#g' | grep '') - log "multipart upload cleanup done for ${bucket}" -} - -step9_decommission() { - log "step 9: decommission pool ${DECOMMISSION_POOL_ID} (admin API)" - confirm "About to decommission pool ${DECOMMISSION_POOL_ID} (up to ${DECOMMISSION_RETRIES} automatic retries). Continue?" - local attempt=1 body - if [ "${DRY_RUN}" -eq 0 ]; then - abort_stale_multipart_uploads "${WARP_BUCKET}" - fi - # If the previous decommission is in a failed/cancelled state, clear its metadata first - if [ "${DRY_RUN}" -eq 0 ]; then - body="$(admin_api POST /rustfs/admin/v3/pools/clear "by-id=true&pool=${DECOMMISSION_POOL_ID}")" - [ "$(admin_api_code)" = "200" ] || warn "initial decommission clear returned HTTP $(admin_api_code): ${body}" - fi - while :; do - log "decommission attempt ${attempt}/${DECOMMISSION_RETRIES}" - if [ "${DRY_RUN}" -eq 0 ]; then - body="$(admin_api POST /rustfs/admin/v3/pools/decommission "by-id=true&pool=${DECOMMISSION_POOL_ID}")" - if [ "$(admin_api_code)" != "200" ]; then - printf '\033[1;31m[ERROR]\033[0m decommission start returned HTTP %s\n' "$(admin_api_code)" >&2 - printf '%s\n' "${body}" >&2 - die "decommission start failed" - fi - log "decommission start accepted" - fi - if wait_decommission "${DECOMMISSION_POOL_ID}"; then - break - fi - if [ "${attempt}" -ge "${DECOMMISSION_RETRIES}" ]; then - warn "if the source bucket has many objects and the tested version is 1.0.0-rc.3, this is the known metacache-listing decommission bug; remove the test bucket (rc rb --force rustfs/${WARP_BUCKET}) or lower --storage-threshold, then re-run step 9" - die "pool ${DECOMMISSION_POOL_ID} still failed after ${attempt} attempts; investigate manually (POST ${API_ENDPOINT}/rustfs/admin/v3/pools/clear?by-id=true&pool=${DECOMMISSION_POOL_ID} to reset)" - fi - warn "attempt ${attempt} failed; clearing metadata and retrying in ${DECOMMISSION_RETRY_DELAY}s" - if [ "${DRY_RUN}" -eq 0 ]; then - body="$(admin_api POST /rustfs/admin/v3/pools/clear "by-id=true&pool=${DECOMMISSION_POOL_ID}")" - [ "$(admin_api_code)" = "200" ] || warn "decommission clear failed (HTTP $(admin_api_code)): ${body}; retrying anyway" - fi - attempt=$((attempt + 1)) - sleep "${DECOMMISSION_RETRY_DELAY}" - done - if [ "${FINALIZE_DECOMMISSION}" -eq 1 ]; then - log "decommission done; removing pool ${DECOMMISSION_POOL_ID} from all node configs and restarting" - confirm "About to update all node configs (remove the decommissioned pool) and restart services. Continue?" - for node in "${NODES[@]}"; do - write_rustfs_config "${node}" "${VOLUMES_AFTER_DECOMMISSION}" - done - service_action_all restart - for node in "${NODES[@]}"; do - wait_service_active "${node}" - done - verify_pools 2 - fi -} - -# ==================== CLI parsing ==================== - -usage() { - cat <<'USAGE' -Usage: rustfs-pool-expand.sh [options] - -Options: - --all Run the full workflow (steps 1-9) - --step N Run a single step (repeatable) - --steps 1,3,5-7 Run steps in order - --with-warp Automate step 4 (warp writes + usage monitoring) - --finalize-decommission After decommission, remove the pool from the topology and restart - --skip-download Skip download when the package already exists - --version VER RustFS release tag to test, e.g. 1.0.0-rc.3 (default from config) - --package-url URL Full package download URL (overrides --version) - --sha256 HEX Verify the downloaded package checksum - --preflight Run preflight checks before the selected steps - --reset Reset all nodes (stop services, wipe data dirs + config). Destructive! - --ssh-user USER SSH user for the nodes (default azureuser) - --ssh-port PORT SSH port for the nodes (default 22) - --endpoint URL Cluster admin API endpoint, e.g. http://10.0.0.7:9000 (default from config) - --rc-endpoint URL Deprecated alias for --endpoint - --storage-threshold N Stop writing when usage reaches N% (default 85) - --warp-duration DUR warp write duration, e.g. 5m, 20s (default 5m) - --rebalance-timeout N Rebalance wait timeout in seconds (default 86400) - --decommission-timeout N Decommission wait timeout in seconds (default 86400) - --service-timeout N Service start wait timeout in seconds (default 300) - --poll-interval N Status polling interval in seconds (default 30) - --log-file FILE Append all output to FILE - --dry-run Preview commands without executing them - -y, --yes Skip all confirmation prompts - -h, --help Show this help - -Examples: - ./rustfs-pool-expand.sh --all - ./rustfs-pool-expand.sh --all --with-warp --yes --version 1.0.0-rc.3 - ./rustfs-pool-expand.sh --step 5 - ./rustfs-pool-expand.sh --all --dry-run -USAGE -} - -expand_steps() { - # Expand a "1,3,5-7" step spec into SELECTED_STEPS - local spec="$1" part start end i - IFS=',' read -ra parts <<<"${spec}" - for part in "${parts[@]}"; do - if [[ "${part}" =~ ^([0-9]+)-([0-9]+)$ ]]; then - start="${BASH_REMATCH[1]}"; end="${BASH_REMATCH[2]}" - for ((i=start; i<=end; i++)); do SELECTED_STEPS+=("${i}"); done - elif [[ "${part}" =~ ^[0-9]+$ ]]; then - SELECTED_STEPS+=("${part}") - else - die "cannot parse step spec: ${part}" - fi - done -} - -run_steps() { - local step - for step in "${SELECTED_STEPS[@]}"; do - case "${step}" in - 1) step1_download ;; - 2) step2_install ;; - 3) step3_start_first_pool ;; - 4) step4_write_data ;; - 5) step5_expand_pool2 ;; - 6) step6_rebalance ;; - 7) step7_expand_pool3 ;; - 8) step8_rebalance ;; - 9) step9_decommission ;; - *) die "unknown step: ${step}" ;; - esac - log "step ${step} completed" - done -} - -main() { - [ "$#" -eq 0 ] && { usage; exit 0; } - local opt all=0 - while [ "$#" -gt 0 ]; do - opt="$1"; shift - case "${opt}" in - --all) all=1 ;; - --step) SELECTED_STEPS+=("$1"); shift ;; - --steps) expand_steps "$1"; shift ;; - --with-warp) WITH_WARP=1 ;; - --finalize-decommission) FINALIZE_DECOMMISSION=1 ;; - --skip-download) SKIP_DOWNLOAD=1 ;; - --version) RUSTFS_VERSION="$1"; shift ;; - --package-url) PACKAGE_URL="$1"; shift ;; - --sha256) PACKAGE_SHA256="$1"; shift ;; - --preflight) PREFLIGHT=1 ;; - --reset) RESET=1 ;; - --ssh-user) SSH_USER="$1"; shift ;; - --ssh-port) SSH_PORT="$1"; shift ;; - --endpoint) API_ENDPOINT="$1"; shift ;; - --rc-endpoint) API_ENDPOINT="$1"; shift ;; - --storage-threshold) STORAGE_THRESHOLD="$1"; shift ;; - --warp-duration) WARP_DURATION="$1"; shift ;; - --rebalance-timeout) REBALANCE_TIMEOUT="$1"; shift ;; - --decommission-timeout) DECOMMISSION_TIMEOUT="$1"; shift ;; - --service-timeout) SERVICE_TIMEOUT="$1"; shift ;; - --poll-interval) POLL_INTERVAL="$1"; shift ;; - --log-file) LOG_FILE="$1"; shift ;; - --dry-run) DRY_RUN=1 ;; - -y|--yes) ASSUME_YES=1 ;; - -h|--help) usage; exit 0 ;; - *) die "unknown option: ${opt} (see --help)" ;; - esac - done - trap 'rm -f "${ADMIN_API_CODE_FILE}"' EXIT - if [ -n "${LOG_FILE}" ]; then - mkdir -p "$(dirname "${LOG_FILE}")" - if ! touch "${LOG_FILE}" 2>/dev/null; then - # A fixed /tmp path may be owned by another user (e.g. a previous root - # run); fall back to a unique, always-writable temp file. - LOG_FILE="$(mktemp "${TMPDIR:-/tmp}/rustfs-pool-test.XXXXXX.log")" - warn "log file not writable; using ${LOG_FILE}" - fi - exec > >(tee -a "${LOG_FILE}") 2>&1 - fi - if [ "${RESET}" -eq 1 ]; then - step0_reset - log "reset finished" - exit 0 - fi - if [ "${all}" -eq 1 ]; then - SELECTED_STEPS=(1 2 3 4 5 6 7 8 9) - fi - PACKAGE_URL="$(resolve_package_url)" - if [ "${PREFLIGHT}" -eq 1 ]; then - preflight - if [ "${#SELECTED_STEPS[@]}" -eq 0 ]; then - log "preflight only (no steps selected); done" - exit 0 - fi - fi - [ "${#SELECTED_STEPS[@]}" -gt 0 ] || die "no steps selected (--all / --step / --steps)" - if [ "${DRY_RUN}" -eq 0 ]; then - log "nodes: ${NODES[*]} ssh user: ${SSH_USER} version: ${RUSTFS_VERSION}" - log "package: ${PACKAGE_URL}" - else - warn "DRY-RUN mode: only printing the commands that would run" - fi - run_steps - log "all done" -} - -# Allow sourcing the file for unit tests without running main. -if [ "${RUSTFS_POOL_SCRIPT_SOURCE_ONLY:-0}" != "1" ]; then - main "$@" -fi