gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-28 16:07:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: add RustFS 4x4 performance test workflow and scripts (#6752)

Browse Source 
* ci: add RustFS 4x4 performance test workflow and scripts

* ci: run performance test on dedicated pf-testing runner
This commit is contained in:
hector
2026-08-27 22:24:57 +08:00
committed by GitHub
parent 44f3f0e73e
commit d48dda5bdc
3 changed files with 983 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/rustfs-performance-test.yml
+201
View File
@@ -0,0 +1,201 @@
name: RustFS Performance Test
on:
workflow_dispatch:
inputs:
package_url:
description: 'Direct .deb URL (nightly/R2). Defaults to the latest nightly deb.'
required: false
type: string
test_method:
description: 'Benchmark method(s) to run (manual runs only; "all" = GET+PUT+MIXED)'
type: choice
options:
- all
- get
- put
- mixed
default: 'all'
object_size:
description: 'Object size(s) to test (manual runs only; "all" = all 10 sizes)'
type: choice
options:
- all
- 1KiB
- 4KiB
- 16KiB
- 128KiB
- 1MiB
- 4MiB
- 8MiB
- 16MiB
- 32MiB
- 64MiB
default: 'all'
warp_duration:
description: 'warp duration per round (e.g. 5m, 30s)'
required: false
default: '5m'
warp_concurrency:
description: 'warp concurrency'
required: false
default: '64'
cleanup_before:
description: 'Reset the nodes before the test (DESTROYS existing data/config)'
type: boolean
default: true
cleanup_after:
description: 'Reset the nodes after the test (DESTROYS test data/config)'
type: boolean
default: true
workflow_run:
# Run after the nightly build completes; the nightly deb is what the test installs.
workflows: ["Nightly GNU Build"]
types: [completed]
permissions:
contents: read
# Dedicated pf-testing runner/environment: own concurrency group so perf runs
# never block (or are blocked by) the pool-expansion / heal tests.
concurrency:
group: rustfs-performance-test
cancel-in-progress: false
defaults:
run:
shell: bash
env:
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
# Package used by the nightly run (workflow_dispatch inputs are empty for
# workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
# Fixed benchmark result directory so later steps can read summary.md
RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
# Cross-repo token for writing to rustfs/backlog (set in repo settings)
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
jobs:
performance-test:
runs-on: pf-testing
timeout-minutes: 900
# Run on manual dispatch, or when the nightly build completed successfully.
# Skipped when nightly failed.
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
ref: ${{ github.event.workflow_run.head_sha || github.ref }}
- name: Show environment
run: |
uname -a
jq --version
warp --version || true
df -h /data | tail -1
- name: Reset test environment (before)
if: ${{ inputs.cleanup_before != 'false' }}
run: |
chmod +x scripts/test/rustfs_performance_test.sh
./scripts/test/rustfs_performance_test.sh --step 1 -y
- name: Install RustFS package & start cluster (4x4)
run: |
ARGS=(--steps "2,3,4" -y)
if [ -n "${{ inputs.package_url }}" ]; then
ARGS+=(--package-url "${{ inputs.package_url }}")
else
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
fi
./scripts/test/rustfs_performance_test.sh "${ARGS[@]}"
- name: Preflight checks
run: |
ARGS=(--preflight)
if [ -n "${{ inputs.package_url }}" ]; then
ARGS+=(--package-url "${{ inputs.package_url }}")
else
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
fi
./scripts/test/rustfs_performance_test.sh "${ARGS[@]}"
- name: Run benchmark (GET/PUT/MIXED)
id: benchmark
run: |
# Empty on automatic (workflow_run) runs -> full 30 rounds.
# Manual dispatch can restrict method(s)/size(s).
export WARP_METHODS="${{ inputs.test_method }}"
export WARP_SIZES="${{ inputs.object_size }}"
./scripts/test/rustfs_performance_test.sh \
--step 5 -y \
--warp-duration "${{ inputs.warp_duration || '5m' }}" \
--warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
--log-file /tmp/rustfs-perf-test.log
- name: Analyze results
if: ${{ steps.benchmark.conclusion == 'success' }}
run: |
./scripts/test/rustfs_performance_test.sh --step 6 -y
- name: Post results to backlog issue
if: ${{ steps.benchmark.conclusion == 'success' }}
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
echo "PF_TESTING_GH_TOKEN is not configured; skipping issue post"
exit 0
fi
SUMMARY="${RESULT_DIR}/summary.md"
[ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
DATE="$(date -u +%Y-%m-%d)"
{
echo "## RustFS nightly build performance testing report"
echo ""
echo "- **日期**: ${DATE}"
echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "- **触发方式**: ${{ github.event_name }}"
echo ""
cat "${SUMMARY}"
} > /tmp/rustfs-perf-issue-body.md
TITLE="RustFS nightly build performance testing report"
EXISTING="$(gh issue list --repo rustfs/backlog \
--search "in:title \"${TITLE}\"" --state all --limit 5 \
--json number --jq '.[0].number // empty')"
if [ -n "${EXISTING}" ]; then
gh issue comment "${EXISTING}" --repo rustfs/backlog --body-file /tmp/rustfs-perf-issue-body.md
echo "commented on existing issue #${EXISTING}"
else
gh issue create --repo rustfs/backlog --title "${TITLE}" --body-file /tmp/rustfs-perf-issue-body.md
fi
- name: Upload test logs & results
if: always()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-perf-test-${{ github.run_id }}
path: |
/tmp/rustfs-perf-test*.log
/tmp/rustfs-perf-results/**
if-no-files-found: warn
- name: Reset test environment (after)
if: ${{ always() && inputs.cleanup_after != 'false' }}
run: |
./scripts/test/rustfs_performance_test.sh --step 7 -y
- name: Notify on failure
if: failure()
run: |
echo "RustFS performance test failed"
echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
echo "See the uploaded log artifact for details."
scripts/test/rustfs_performance_test.sh
Executable
+543
View File
@@ -0,0 +1,543 @@
#!/usr/bin/env bash
#
# rustfs-performance-test.sh
# RustFS 4x4 集群性能压测全流程脚本
#
# Based on the Obsidian note "RustFS 性能测试". Full workflow:
# 1. Cleanup: stop & purge rustfs, remove data dirs on all nodes
# 2. Download the RustFS package on all nodes
# 3. Install RustFS on all nodes (dpkg -i), recreate volume dirs
# 4. Write /etc/default/rustfs (4-node x 4-drive MNMD), start all nodes
# in parallel and verify the service is Running
# 5. Run the benchmark (warp GET/PUT/MIXED via rustfs-performance-testing.sh)
# 6. Analyze results (summary.tsv / summary.md)
# 7. Final cleanup: stop & purge rustfs, remove data dirs
#
# The script is driven from an admin host (e.g. a jumpbox) and operates on
# the target nodes over SSH, mirroring scripts/test/rustfs_*_test.sh.
#
# Usage:
# ./rustfs-performance-test.sh --all # run all steps 1-7
# ./rustfs-performance-test.sh --step 5 # run a single step
# ./rustfs-performance-test.sh --steps 2,3,4 # run selected steps
# ./rustfs-performance-test.sh --all --dry-run # preview only
# ./rustfs-performance-test.sh --all -y --package-url <deb URL>
#
# Notes:
# - SSH user defaults to azureuser (passwordless sudo on the nodes);
# pass --ssh-user root if your nodes accept root login.
# - The benchmark runner defaults to
# ~/Documents/Obsidian Vault/rustfs-performance-testing.sh; override with
# --bench-script / RUSTFS_BENCH_SCRIPT. warp must be installed on the
# admin host.
# - Steps 1 and 7 destroy the RustFS install and all data (confirmed).
#
set -Eeuo pipefail
# ==================== Configuration (adjust to your environment) ====================
# Target nodes (4x4: 4 nodes x 4 drives each)
if [ -n "${RUSTFS_NODES:-}" ]; then
read -r -a NODES <<<"${RUSTFS_NODES}"
else
NODES=(vm000 vm001 vm002 vm003)
fi
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
SSH_PORT="${RUSTFS_SSH_PORT:-22}"
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new -p "${SSH_PORT}")
# Package: GitHub release tag, e.g. "1.0.0-rc.3". PACKAGE_URL is derived from
# RUSTFS_VERSION unless --package-url / RUSTFS_PACKAGE_URL is given.
RUSTFS_VERSION="${RUSTFS_VERSION:-1.0.0-rc.3}"
PACKAGE_URL="${RUSTFS_PACKAGE_URL:-}"
ARCH="${RUSTFS_ARCH:-amd64}"
PACKAGES_DIR="/home/rustfs/packages"
PACKAGE_FILE="rustfs.deb"
PACKAGE_SHA256="${RUSTFS_PACKAGE_SHA256:-}"
# 4x4 topology: 4 nodes x 4 drives each, same expression on every node
DRIVES_PER_NODE="${RUSTFS_DRIVES_PER_NODE:-4}"
VOLUMES="http://rustfs-node{1...4}:9000/data/rustfs{1...4}/mnmd"
# RustFS service configuration (written to /etc/default/rustfs)
RUSTFS_CONFIG_FILE="/etc/default/rustfs"
RUSTFS_SERVICE="rustfs"
RUSTFS_PACKAGE_NAME="rustfs"
RUSTFS_USER="rustfs"
ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfs@test}"
SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfs@test}"
RUSTFS_ADDRESS=":9000"
RUSTFS_CONSOLE_ADDRESS=":9001"
RUSTFS_CONSOLE_ENABLE=true
RUSTFS_OBS_LOGGER_LEVEL=error
RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
# Benchmark runner (step 5/6): prefer the default Obsidian location, fall back
# to a rustfs-performance-testing.sh next to this script (e.g. in the repo or
# on a jumpbox).
_DEFAULT_BENCH="${HOME}/Documents/Obsidian Vault/rustfs-performance-testing.sh"
_SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
if [ -x "${_DEFAULT_BENCH}" ]; then
_BENCH_RESOLVED="${_DEFAULT_BENCH}"
elif [ -x "${_SCRIPT_DIR}/rustfs-performance-testing.sh" ]; then
_BENCH_RESOLVED="${_SCRIPT_DIR}/rustfs-performance-testing.sh"
elif [ -x "${_SCRIPT_DIR}/rustfs_performance_testing.sh" ]; then
_BENCH_RESOLVED="${_SCRIPT_DIR}/rustfs_performance_testing.sh"
else
_BENCH_RESOLVED="${_DEFAULT_BENCH}"
fi
BENCH_SCRIPT="${RUSTFS_BENCH_SCRIPT:-${_BENCH_RESOLVED}}"
RESULT_DIR="${RUSTFS_RESULT_DIR:-$(pwd)/warp-bench-results-$(date +%Y%m%d-%H%M%S)}"
WARP_HOST="${RUSTFS_WARP_HOST:-rustfs-node1:9000,rustfs-node2:9000,rustfs-node3:9000,rustfs-node4:9000}"
WARP_BUCKET="${RUSTFS_WARP_BUCKET:-warp-benchmark-bucket}"
WARP_CONCURRENCY="${RUSTFS_WARP_CONCURRENCY:-64}"
WARP_DURATION="${RUSTFS_WARP_DURATION:-5m}"
WARP_GET_OBJECTS="${RUSTFS_WARP_GET_OBJECTS:-2500}"
WARP_SLEEP="${RUSTFS_WARP_SLEEP:-60}"
# Manual method/size selection (passed through to the benchmark runner; empty = full run)
WARP_METHODS="${RUSTFS_WARP_METHODS:-}"
WARP_SIZES="${RUSTFS_WARP_SIZES:-}"
# Timeouts (seconds)
SERVICE_TIMEOUT="${RUSTFS_SERVICE_TIMEOUT:-300}"
POLL_INTERVAL="${RUSTFS_POLL_INTERVAL:-10}"
# ==================== Runtime options (set by CLI) ====================
DRY_RUN=0
ASSUME_YES=0
SKIP_DOWNLOAD=0
PREFLIGHT=0
LOG_FILE=""
SELECTED_STEPS=()
# ==================== Helpers ====================
log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; }
warn() { printf '\033[1;33m[WARN]\033[0m %s\n' "$*"; }
die() { printf '\033[1;31m[ERROR]\033[0m %s\n' "$*" >&2; exit 1; }
confirm() {
if [ "${ASSUME_YES}" -eq 1 ] || [ "${DRY_RUN}" -eq 1 ]; then return 0; fi
printf '\033[1;33m[CONFIRM]\033[0m %s (y/N) ' "$1"
read -r answer
case "${answer}" in
y|Y|yes|YES) return 0 ;;
*) die "cancelled" ;;
esac
}
need_cmd() {
[ "${DRY_RUN}" -eq 1 ] && return 0
command -v "$1" >/dev/null 2>&1 || die "missing command: $1 ($2); install it first"
}
# Run a remote script on a single node (script is read from stdin)
run_remote() {
local node="$1" script
script="$(cat)"
if [ "${DRY_RUN}" -eq 1 ]; then
log "DRY-RUN: ssh ${SSH_USER}@${node} <<'REMOTE'"
printf '%s\n' "${script}" | sed 's/^/ | /'
log "DRY-RUN: ----"
return 0
fi
log "==> ${node}: executing remote script"
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'bash -s' <<<"${script}"
}
# Run the same remote script on all nodes in parallel (script from stdin)
run_remote_all() {
local script pids=() i=0 fail=0
script="$(cat)"
for node in "${NODES[@]}"; do
if [ "${DRY_RUN}" -eq 1 ]; then
log "DRY-RUN: ssh ${SSH_USER}@${node} <<'REMOTE'"
printf '%s\n' "${script}" | sed 's/^/ | /'
log "DRY-RUN: ----"
else
log "==> ${node}: executing remote script"
( ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'bash -s' <<<"${script}" ) &
pids[$i]=$!
i=$((i+1))
fi
done
if [ "${#pids[@]}" -gt 0 ]; then
for pid in "${pids[@]}"; do
wait "${pid}" || fail=1
done
fi
[ "${fail}" -eq 0 ] || die "one or more remote executions failed"
}
rustfs_config_body() {
cat <<EOF
RUSTFS_ACCESS_KEY=${ACCESS_KEY}
RUSTFS_SECRET_KEY=${SECRET_KEY}
RUSTFS_VOLUMES="${VOLUMES}"
RUSTFS_ADDRESS="${RUSTFS_ADDRESS}"
RUSTFS_CONSOLE_ADDRESS="${RUSTFS_CONSOLE_ADDRESS}"
RUSTFS_CONSOLE_ENABLE=${RUSTFS_CONSOLE_ENABLE}
RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL}
RUSTFS_OBS_LOG_DIRECTORY="${RUSTFS_OBS_LOG_DIRECTORY}"
EOF
}
write_rustfs_config() {
local node="$1" body
body="$(rustfs_config_body)"
log "${node}: writing config ${RUSTFS_CONFIG_FILE}"
{
printf 'set -euo pipefail\n'
printf 'SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"\n'
printf '%s tee %s >/dev/null <<RUSTFS_EOF\n' '${SUDO}' "${RUSTFS_CONFIG_FILE}"
printf '%s' "${body}"
printf '\nRUSTFS_EOF\n'
printf '${SUDO} systemctl daemon-reload\n'
} | run_remote "${node}"
}
service_action() {
local action="$1" node="$2"
log "${node}: systemctl ${action} ${RUSTFS_SERVICE}"
[ "${DRY_RUN}" -eq 1 ] && return 0
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \
"if [ \"\$(id -u)\" -ne 0 ]; then sudo -n systemctl ${action} ${RUSTFS_SERVICE}; else systemctl ${action} ${RUSTFS_SERVICE}; fi" \
|| die "${node}: systemctl ${action} failed"
}
wait_service_active() {
local node="$1" elapsed=0
log "${node}: waiting for ${RUSTFS_SERVICE} to become active"
[ "${DRY_RUN}" -eq 1 ] && { log "${node}: (dry-run) skip wait"; return 0; }
while [ "${elapsed}" -lt "${SERVICE_TIMEOUT}" ]; do
if ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \
"systemctl is-active ${RUSTFS_SERVICE} 2>/dev/null" | grep -q active; then
log "${node}: service active"
return 0
fi
sleep "${POLL_INTERVAL}"
elapsed=$((elapsed + POLL_INTERVAL))
done
die "${node}: ${RUSTFS_SERVICE} did not become active within ${SERVICE_TIMEOUT}s"
}
verify_service_running() {
local node="$1"
log "${node}: checking service status"
if [ "${DRY_RUN}" -eq 0 ]; then
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \
"systemctl status ${RUSTFS_SERVICE} --no-pager | head -n 12" || true
fi
}
build_package_url() {
local asset
asset="rustfs_$(printf '%s' "${RUSTFS_VERSION}" | tr '-' '.')_${ARCH}.deb"
printf 'https://github.com/rustfs/rustfs/releases/download/%s/%s' "${RUSTFS_VERSION}" "${asset}"
}
resolve_package_url() {
if [ -n "${PACKAGE_URL}" ]; then printf '%s' "${PACKAGE_URL}"; else build_package_url; fi
}
preflight() {
log "preflight checks"
need_cmd ssh "openssh client"
need_cmd curl "http client"
need_cmd warp "warp benchmark tool (for step 5)"
if [ ! -x "${BENCH_SCRIPT}" ]; then
die "benchmark script not found or not executable: ${BENCH_SCRIPT}"
fi
if [ "${DRY_RUN}" -eq 0 ]; then
for node in "${NODES[@]}"; do
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'echo ok' >/dev/null \
|| die "cannot ssh to ${node}"
done
log "all nodes reachable: ${NODES[*]}"
fi
log "preflight OK"
}
# ==================== Steps ====================
step1_cleanup() {
log "step 1: cleanup environment on all nodes (stop & purge rustfs, remove data dirs)"
confirm "This DESTROYS the RustFS install and ALL data on ${NODES[*]} (irreversible). Continue?"
local script
script="$(cat <<EOF
set -euo pipefail
SUDO=""; [ "\$(id -u)" -ne 0 ] && SUDO="sudo -n"
\${SUDO} systemctl stop rustfs 2>/dev/null || true
if \${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
\${SUDO} dpkg -P rustfs
echo "purged rustfs"
else
echo "rustfs not installed, skip purge"
fi
for i in \$(seq 1 ${DRIVES_PER_NODE}); do
\${SUDO} rm -rf /data/rustfs\${i}/mnmd
\${SUDO} mkdir -p /data/rustfs\${i}/mnmd
\${SUDO} chown -R rustfs:rustfs /data/rustfs\${i}/mnmd
done
echo "cleanup done on \$(hostname)"
EOF
)"
printf '%s\n' "${script}" | run_remote_all
log "step 1 complete"
}
step2_download() {
log "step 2: download the package on all nodes"
local url script
url="$(resolve_package_url)"
script="$(cat <<EOF
set -euo pipefail
SUDO=""; [ "\$(id -u)" -ne 0 ] && SUDO="sudo -n"
if [ -f "${PACKAGES_DIR}/${PACKAGE_FILE}" ] && [ "${SKIP_DOWNLOAD}" -eq 1 ]; then
echo "already exists: ${PACKAGES_DIR}/${PACKAGE_FILE}, skipping download"
else
echo "downloading ${url} ..."
curl -fSL --retry 3 -o "/tmp/${PACKAGE_FILE}" "${url}"
\${SUDO} mkdir -p "${PACKAGES_DIR}"
\${SUDO} install -m 0644 "/tmp/${PACKAGE_FILE}" "${PACKAGES_DIR}/${PACKAGE_FILE}"
\${SUDO} rm -f "/tmp/${PACKAGE_FILE}"
fi
if [ -n "${PACKAGE_SHA256}" ]; then
echo "${PACKAGE_SHA256} ${PACKAGES_DIR}/${PACKAGE_FILE}" | sha256sum -c - || { echo "checksum verification failed"; exit 1; }
fi
ls -lh "${PACKAGES_DIR}/${PACKAGE_FILE}"
EOF
)"
printf '%s\n' "${script}" | run_remote_all
log "step 2 complete"
}
step3_install() {
log "step 3: install the RustFS service on all nodes"
confirm "About to run dpkg -i ${PACKAGE_FILE} on all nodes. Continue?"
local script
script="$(cat <<'EOF'
set -euo pipefail
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
${SUDO} dpkg -i /home/rustfs/packages/rustfs.deb
${SUDO} systemctl daemon-reload
echo "--- installed package ---"
dpkg -l rustfs | tail -n 1
EOF
)"
printf '%s\n' "${script}" | run_remote_all
log "step 3 complete"
}
step4_configure_start() {
log "step 4: write config, start and verify the service on all nodes"
local node
for node in "${NODES[@]}"; do
write_rustfs_config "${node}"
done
for node in "${NODES[@]}"; do
service_action start "${node}" &
done
wait
for node in "${NODES[@]}"; do
wait_service_active "${node}"
verify_service_running "${node}"
done
log "step 4 complete"
}
step5_benchmark() {
log "step 5: run the benchmark (${BENCH_SCRIPT})"
need_cmd warp "warp benchmark tool"
confirm "About to run the full GET/PUT/MIXED benchmark (~6-10 hours). Continue?"
if [ "${DRY_RUN}" -eq 1 ]; then
log "DRY-RUN: WARP_HOST=${WARP_HOST} WARP_RESULT_DIR=${RESULT_DIR} bash ${BENCH_SCRIPT}"
return 0
fi
WARP_HOST="${WARP_HOST}" \
WARP_ACCESS_KEY="${ACCESS_KEY}" \
WARP_SECRET_KEY="${SECRET_KEY}" \
WARP_BUCKET="${WARP_BUCKET}" \
WARP_CONCURRENCY="${WARP_CONCURRENCY}" \
WARP_DURATION="${WARP_DURATION}" \
WARP_GET_OBJECTS="${WARP_GET_OBJECTS}" \
WARP_SLEEP_BETWEEN_ROUNDS="${WARP_SLEEP}" \
WARP_METHODS="${WARP_METHODS}" \
WARP_SIZES="${WARP_SIZES}" \
WARP_RESULT_DIR="${RESULT_DIR}" \
bash "${BENCH_SCRIPT}"
log "step 5 complete (results in ${RESULT_DIR})"
}
step6_analyze() {
log "step 6: analyze results from ${RESULT_DIR}"
if [ "${DRY_RUN}" -eq 1 ]; then
log "DRY-RUN: bash ${BENCH_SCRIPT} --parse-only ${RESULT_DIR}"
return 0
fi
if [ ! -d "${RESULT_DIR}" ]; then
die "result directory not found: ${RESULT_DIR}"
fi
if [ -f "${RESULT_DIR}/summary.md" ]; then
log "summary already generated: ${RESULT_DIR}/summary.md"
else
log "generating summary with --parse-only"
bash "${BENCH_SCRIPT}" --parse-only "${RESULT_DIR}"
fi
log "----- summary.md -----"
cat "${RESULT_DIR}/summary.md"
log "step 6 complete"
}
step7_cleanup() {
log "step 7: final cleanup on all nodes (stop & purge rustfs, remove data dirs)"
confirm "This DESTROYS the RustFS install and ALL data on ${NODES[*]} (irreversible). Continue?"
local script
script="$(cat <<EOF
set -euo pipefail
SUDO=""; [ "\$(id -u)" -ne 0 ] && SUDO="sudo -n"
\${SUDO} systemctl stop rustfs 2>/dev/null || true
if \${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
\${SUDO} dpkg -P rustfs
echo "purged rustfs"
fi
for i in \$(seq 1 ${DRIVES_PER_NODE}); do
\${SUDO} rm -rf /data/rustfs\${i}/mnmd
done
echo "cleanup done on \$(hostname)"
EOF
)"
printf '%s\n' "${script}" | run_remote_all
log "step 7 complete"
}
# ==================== CLI ====================
usage() {
cat <<'USAGE'
Usage: ./rustfs-performance-test.sh [options]
Steps:
1 cleanup environment (purge rustfs, remove data dirs) [destructive]
2 download the RustFS package on all nodes
3 install RustFS (dpkg -i)
4 write config, start service, verify Running
5 run benchmark (warp GET/PUT/MIXED)
6 analyze results (summary.tsv / summary.md)
7 final cleanup (purge rustfs, remove data dirs) [destructive]
Options:
--all Run all steps 1-7
--step N Run a single step
--steps 1,3,5-7 Run selected steps
--version VERSION GitHub release tag (default 1.0.0-rc.3)
--package-url URL Direct deb URL (overrides --version)
--sha256 HASH Verify package checksum
--skip-download Keep an existing package file
--bench-script PATH Benchmark runner (default: Obsidian Vault rustfs-performance-testing.sh)
--result-dir DIR Benchmark result directory
--warp-duration DUR warp duration per round (default 5m)
--warp-concurrency N warp concurrency (default 64)
--ssh-user USER SSH user (default azureuser)
--ssh-port PORT SSH port (default 22)
--preflight Check environment and exit
--log-file FILE Append all output to FILE
--dry-run Preview commands without executing them
-y, --yes Skip all confirmation prompts
-h, --help Show this help
Examples:
./rustfs-performance-test.sh --all
./rustfs-performance-test.sh --all --dry-run
./rustfs-performance-test.sh --all -y --package-url https://dl.rustfs.com/...deb
./rustfs-performance-test.sh --step 5
USAGE
}
expand_steps() {
local spec="$1" part start end i
IFS=',' read -ra parts <<<"${spec}"
for part in "${parts[@]}"; do
if [[ "${part}" =~ ^([0-9]+)-([0-9]+)$ ]]; then
start="${BASH_REMATCH[1]}"; end="${BASH_REMATCH[2]}"
for ((i=start; i<=end; i++)); do SELECTED_STEPS+=("${i}"); done
elif [[ "${part}" =~ ^[0-9]+$ ]]; then
SELECTED_STEPS+=("${part}")
else
die "cannot parse step spec: ${part}"
fi
done
}
run_steps() {
local step
for step in "${SELECTED_STEPS[@]}"; do
case "${step}" in
1) step1_cleanup ;;
2) step2_download ;;
3) step3_install ;;
4) step4_configure_start ;;
5) step5_benchmark ;;
6) step6_analyze ;;
7) step7_cleanup ;;
*) die "unknown step: ${step}" ;;
esac
log "step ${step} completed"
done
}
main() {
[ "$#" -eq 0 ] && { usage; exit 0; }
local opt all=0
while [ "$#" -gt 0 ]; do
opt="$1"; shift
case "${opt}" in
--all) all=1 ;;
--step) SELECTED_STEPS+=("$1"); shift ;;
--steps) expand_steps "$1"; shift ;;
--version) RUSTFS_VERSION="$1"; shift ;;
--package-url) PACKAGE_URL="$1"; shift ;;
--sha256) PACKAGE_SHA256="$1"; shift ;;
--skip-download) SKIP_DOWNLOAD=1 ;;
--bench-script) BENCH_SCRIPT="$1"; shift ;;
--result-dir) RESULT_DIR="$1"; shift ;;
--warp-duration) WARP_DURATION="$1"; shift ;;
--warp-concurrency) WARP_CONCURRENCY="$1"; shift ;;
--ssh-user) SSH_USER="$1"; shift ;;
--ssh-port) SSH_PORT="$1"; shift ;;
--preflight) PREFLIGHT=1 ;;
--log-file) LOG_FILE="$1"; shift ;;
--dry-run) DRY_RUN=1 ;;
-y|--yes) ASSUME_YES=1 ;;
-h|--help) usage; exit 0 ;;
*) die "unknown option: ${opt} (see --help)" ;;
esac
done
if [ -n "${LOG_FILE}" ]; then
mkdir -p "$(dirname "${LOG_FILE}")"
exec > >(tee -a "${LOG_FILE}") 2>&1
fi
if [ "${all}" -eq 1 ]; then
SELECTED_STEPS=(1 2 3 4 5 6 7)
fi
if [ "${PREFLIGHT}" -eq 1 ]; then
preflight
if [ "${#SELECTED_STEPS[@]}" -eq 0 ]; then
log "preflight only; done"
exit 0
fi
fi
[ "${#SELECTED_STEPS[@]}" -gt 0 ] || die "no steps selected (--all / --step / --steps)"
log "nodes: ${NODES[*]} ssh user: ${SSH_USER} version: ${RUSTFS_VERSION}"
log "package: $(resolve_package_url)"
log "result dir: ${RESULT_DIR}"
[ "${DRY_RUN}" -eq 1 ] && warn "DRY-RUN mode: only printing the commands that would run"
run_steps
log "all done"
}
# Allow sourcing the file for unit tests without running main.
if [ "${RUSTFS_PERF_SCRIPT_SOURCE_ONLY:-0}" != "1" ]; then
main "$@"
fi
scripts/test/rustfs_performance_testing.sh
Executable
+239
View File
@@ -0,0 +1,239 @@
#!/usr/bin/env bash
#
# rustfs-performance-testing.sh
# RustFS 对象存储压测脚本（固定版）：测试方法 + 执行 + 结果解析
#
# 测试方法
# 1) 方法：GET / PUT / MIXED（warp 默认混合负载 45% GET + 55% PUT）
# 2) 对象尺寸：1KiB 4KiB 16KiB 128KiB 1MiB 4MiB 8MiB 16MiB 32MiB 64MiB
# 3) 并发：64；单轮时长：5m；轮间 sleep：60s；GET 对象数：2500
# 4) 顺序：GET 全部尺寸 -> PUT 全部尺寸 -> MIXED 全部尺寸
# 5) 结果解析：每轮结束后自动解析 warp 输出，写入
# summary.tsv（机器可读）与 summary.md（Markdown 汇总表）
#
# 依赖：warp >= v1.6（MinIO warp），bash，awk/sed/grep
# 说明：warp v1.6.1 的 put 不支持 --objects，脚本已自动处理（仅 get/mixed 传该参数）
#
# 环境变量覆盖（不传时使用固定默认值）：
# WARP_HOST WARP_ACCESS_KEY WARP_SECRET_KEY WARP_BUCKET
# WARP_CONCURRENCY WARP_DURATION WARP_GET_OBJECTS WARP_SLEEP_BETWEEN_ROUNDS
# WARP_RESULT_DIR
# WARP_METHODS WARP_SIZES # 手动指定方法/尺寸（逗号或空格分隔），不传则全量
set -u -o pipefail
HOST="${WARP_HOST:-rustfs-node1:9000,rustfs-node2:9000,rustfs-node3:9000,rustfs-node4:9000}"
ACCESS_KEY="${WARP_ACCESS_KEY:-rustfs@test}"
SECRET_KEY="${WARP_SECRET_KEY:-rustfs@test}"
BUCKET="${WARP_BUCKET:-warp-benchmark-bucket}"
CONCURRENCY="${WARP_CONCURRENCY:-64}"
DURATION="${WARP_DURATION:-5m}"
GET_OBJECTS="${WARP_GET_OBJECTS:-2500}"
SLEEP_BETWEEN_ROUNDS="${WARP_SLEEP_BETWEEN_ROUNDS:-60}"
RESULT_DIR="${WARP_RESULT_DIR:-$(pwd)/warp-bench-results-$(date +%Y%m%d-%H%M%S)}"
if [ -n "${WARP_SIZES:-}" ] && [ "${WARP_SIZES}" != "all" ] && [ "${WARP_SIZES}" != "ALL" ]; then
read -r -a SIZES <<<"${WARP_SIZES//,/ }"
else
SIZES=(1KiB 4KiB 16KiB 128KiB 1MiB 4MiB 8MiB 16MiB 32MiB 64MiB)
fi
if [ -n "${WARP_METHODS:-}" ] && [ "${WARP_METHODS}" != "all" ] && [ "${WARP_METHODS}" != "ALL" ]; then
read -r -a METHODS <<<"${WARP_METHODS//,/ }"
else
METHODS=(get put mixed)
fi
TOTAL_ROUNDS=$(( ${#METHODS[@]} * ${#SIZES[@]} ))
ROUND=0
# --parse-only <result-dir>：只解析已有结果目录（${method}_${size}.txt），不执行压测
if [[ "${1:-}" == "--parse-only" && -n "${2:-}" ]]; then
RESULT_DIR="$2"
fi
LOG_FILE="${RESULT_DIR}/master.log"
SUMMARY_TSV="${RESULT_DIR}/summary.tsv"
SUMMARY_MD="${RESULT_DIR}/summary.md"
if [[ "${1:-}" != "--parse-only" ]] && ! command -v warp >/dev/null 2>&1; then
echo "错误：未找到 warp 命令，请先安装 MinIO warp。" >&2
exit 1
fi
log() {
echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') $*" | tee -a "${LOG_FILE}"
}
# ---- 结果解析 ----
# 提取指定 section（GET/PUT/Total 等）的 Average / Reqs / TTFB 原始行
section_lines() {
awk -v sec="$2" '
/^Report: / { cur = $2; sub(/\.$/, "", cur) }
cur == sec && /^ *\* Average:/ { avg = $0 }
cur == sec && /^ *\* Reqs:/ { reqs = $0 }
cur == sec && /^ *\* TTFB:/ { ttfb = $0 }
END {
if (avg != "") print avg
if (reqs != "") print reqs
if (ttfb != "") print ttfb
}
' "$1"
}
# 从统计行中取字段：tp objs avg p50 p90 p99 ttfb_avg ttfb_p99 ttfb_worst
field() {
case "$2" in
tp) echo "$1" | sed -n 's/^ *\* Average: \(.*\), \([0-9.]*\) obj\/s.*/\1/p' ;;
objs) echo "$1" | sed -n 's/^ *\* Average: .*, \([0-9.]*\) obj\/s.*/\1/p' ;;
avg) echo "$1" | sed -n 's/^ *\* Reqs: Avg: \([^,]*\),.*/\1/p' ;;
p50) echo "$1" | sed -n 's/^ *\* Reqs: Avg: [^,]*, 50%: \([^,]*\),.*/\1/p' ;;
p90) echo "$1" | sed -n 's/^ *\* Reqs: Avg: [^,]*, 50%: [^,]*, 90%: \([^,]*\),.*/\1/p' ;;
p99) echo "$1" | sed -n 's/^ *\* Reqs: Avg: [^,]*, 50%: [^,]*, 90%: [^,]*, 99%: \([^,]*\),.*/\1/p' ;;
ttfb_avg) echo "$1" | sed -n 's/^ *\* TTFB: Avg: \([^,]*\),.*/\1/p' ;;
ttfb_p99) echo "$1" | sed -n 's/^ *\* TTFB: .*99th: \([^,]*\),.*/\1/p' ;;
ttfb_worst) echo "$1" | sed -n 's/^ *\* TTFB: .*Worst: \([^ ]*\).*/\1/p' ;;
*) echo "" ;;
esac
}
# 解析一轮输出，追加一行到 summary.tsv
parse_round() {
local method="$1" size="$2" file="$3"
local line
if [[ "${method}" == "mixed" ]]; then
local total get put
total=$(section_lines "$file" Total)
get=$(section_lines "$file" GET)
put=$(section_lines "$file" PUT)
line=$(printf 'mixed\t%s\t%s\t%s\t%s\t%s' \
"$size" \
"$(field "${total}" tp)" \
"$(field "${total}" objs)" \
"$(field "${get}" avg)" \
"$(field "${put}" avg)")
else
local sec
sec=$(printf '%s' "${method}" | tr '[:lower:]' '[:upper:]')
local stats
stats=$(section_lines "$file" "${sec}")
line=$(printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s' \
"${method}" "${size}" \
"$(field "${stats}" tp)" \
"$(field "${stats}" objs)" \
"$(field "${stats}" avg)" \
"$(field "${stats}" p50)" \
"$(field "${stats}" p90)" \
"$(field "${stats}" p99)" \
"$(field "${stats}" ttfb_avg)" \
"$(field "${stats}" ttfb_p99)" \
"$(field "${stats}" ttfb_worst)")
fi
printf '%s\n' "${line}" >> "${SUMMARY_TSV}"
}
# 汇总 summary.tsv -> summary.md（Markdown 表格）
gen_summary_md() {
{
echo "# RustFS 性能压测结果"
echo ""
echo "- 日期：$(date -u '+%Y-%m-%d %H:%M:%S UTC')"
echo "- 目标：${HOST}"
echo "- 并发：${CONCURRENCY}；单轮：${DURATION}；sleep：${SLEEP_BETWEEN_ROUNDS}s；GET objects：${GET_OBJECTS}"
echo "- 方法：GET / PUT / MIXED（warp 默认混合负载）；尺寸：${SIZES[*]}"
echo ""
} > "${SUMMARY_MD}"
for m in get put; do
{
echo "## $(printf '%s' "$m" | tr '[:lower:]' '[:upper:]') 结果"
echo ""
echo "| 对象尺寸 | 平均吞吐 | 平均 obj/s | Avg Latency | P50 | P90 | P99 | TTFB Avg | TTFB P99 | TTFB 最差 |"
echo "|----------|----------|-----------|-------------|-----|-----|-----|----------|----------|-----------|"
} >> "${SUMMARY_MD}"
while IFS=$'\t' read -r method size tp objs avg p50 p90 p99 ttfb_avg ttfb_p99 ttfb_worst; do
[[ "${method}" == "${m}" ]] && \
echo "| ${size} | ${tp} | ${objs} | ${avg} | ${p50} | ${p90} | ${p99} | ${ttfb_avg} | ${ttfb_p99} | ${ttfb_worst} |" >> "${SUMMARY_MD}"
done < "${SUMMARY_TSV}"
echo "" >> "${SUMMARY_MD}"
done
{
echo "## MIXED 结果（Total 口径）"
echo ""
echo "| 对象尺寸 | Total 平均吞吐 | Total 平均 obj/s | Mixed-GET Avg | Mixed-PUT Avg |"
echo "|----------|---------------|------------------|----------------|----------------|"
} >> "${SUMMARY_MD}"
while IFS=$'\t' read -r method size tp objs gavg pavg rest; do
[[ "${method}" == "mixed" ]] && \
echo "| ${size} | ${tp} | ${objs} | ${gavg} | ${pavg} |" >> "${SUMMARY_MD}"
done < "${SUMMARY_TSV}"
echo "" >> "${SUMMARY_MD}"
}
# ---- 主流程 ----
mkdir -p "${RESULT_DIR}"
log "CONFIG host=${HOST} bucket=${BUCKET} concurrency=${CONCURRENCY} duration=${DURATION} get_objects=${GET_OBJECTS} sleep_between_rounds=${SLEEP_BETWEEN_ROUNDS}s"
printf 'method\tsize\tthroughput\tobj_per_s\treq_avg\treq_p50\treq_p90\treq_p99\tttfb_avg\tttfb_p99\tttfb_worst\n' > "${SUMMARY_TSV}"
if [[ "${1:-}" == "--parse-only" ]]; then
for method in "${METHODS[@]}"; do
for size in "${SIZES[@]}"; do
outfile="${RESULT_DIR}/${method}_${size}.txt"
if [[ -s "${outfile}" ]]; then
parse_round "${method}" "${size}" "${outfile}"
fi
done
done
gen_summary_md
echo "parsed from ${RESULT_DIR}"
echo ""
cat "${SUMMARY_MD}"
exit 0
fi
for method in "${METHODS[@]}"; do
for size in "${SIZES[@]}"; do
ROUND=$((ROUND + 1))
outfile="${RESULT_DIR}/${method}_${size}.txt"
log "START round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} concurrency=${CONCURRENCY} duration=${DURATION}"
extra_args=()
if [[ "${method}" != "put" ]]; then
extra_args=(--objects "${GET_OBJECTS}")
fi
start_epoch=$(date +%s)
warp "${method}" \
--host "${HOST}" \
--access-key "${ACCESS_KEY}" \
--secret-key "${SECRET_KEY}" \
--bucket "${BUCKET}" \
--concurrent "${CONCURRENCY}" \
--duration "${DURATION}" \
--obj.size "${size}" \
"${extra_args[@]}" \
--no-color 2>&1 | tee "${outfile}"
rc=${PIPESTATUS[0]}
end_epoch=$(date +%s)
if [[ ${rc} -eq 0 ]]; then
parse_round "${method}" "${size}" "${outfile}"
log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=ok"
else
log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=skipped"
fi
if [[ ${ROUND} -lt ${TOTAL_ROUNDS} ]]; then
log "SLEEP ${SLEEP_BETWEEN_ROUNDS}s before next round"
sleep "${SLEEP_BETWEEN_ROUNDS}"
fi
done
done
gen_summary_md
log "ALL_ROUNDS_COMPLETE summary_tsv=${SUMMARY_TSV} summary_md=${SUMMARY_MD}"
echo ""
echo "==== 结果汇总 ===="
cat "${SUMMARY_MD}"