From d48dda5bdcfaed567c423f640e5961fa0689ce72 Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:24:57 +0800 Subject: [PATCH] ci: add RustFS 4x4 performance test workflow and scripts (#6752) * ci: add RustFS 4x4 performance test workflow and scripts * ci: run performance test on dedicated pf-testing runner --- .github/workflows/rustfs-performance-test.yml | 201 +++++++ scripts/test/rustfs_performance_test.sh | 543 ++++++++++++++++++ scripts/test/rustfs_performance_testing.sh | 239 ++++++++ 3 files changed, 983 insertions(+) create mode 100644 .github/workflows/rustfs-performance-test.yml create mode 100755 scripts/test/rustfs_performance_test.sh create mode 100755 scripts/test/rustfs_performance_testing.sh diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml new file mode 100644 index 000000000..4cfc4d99b --- /dev/null +++ b/.github/workflows/rustfs-performance-test.yml @@ -0,0 +1,201 @@ +name: RustFS Performance Test + +on: + workflow_dispatch: + inputs: + package_url: + description: 'Direct .deb URL (nightly/R2). Defaults to the latest nightly deb.' + required: false + type: string + test_method: + description: 'Benchmark method(s) to run (manual runs only; "all" = GET+PUT+MIXED)' + type: choice + options: + - all + - get + - put + - mixed + default: 'all' + object_size: + description: 'Object size(s) to test (manual runs only; "all" = all 10 sizes)' + type: choice + options: + - all + - 1KiB + - 4KiB + - 16KiB + - 128KiB + - 1MiB + - 4MiB + - 8MiB + - 16MiB + - 32MiB + - 64MiB + default: 'all' + warp_duration: + description: 'warp duration per round (e.g. 5m, 30s)' + required: false + default: '5m' + warp_concurrency: + description: 'warp concurrency' + required: false + default: '64' + cleanup_before: + description: 'Reset the nodes before the test (DESTROYS existing data/config)' + type: boolean + default: true + cleanup_after: + description: 'Reset the nodes after the test (DESTROYS test data/config)' + type: boolean + default: true + workflow_run: + # Run after the nightly build completes; the nightly deb is what the test installs. + workflows: ["Nightly GNU Build"] + types: [completed] + +permissions: + contents: read + +# Dedicated pf-testing runner/environment: own concurrency group so perf runs +# never block (or are blocked by) the pool-expansion / heal tests. +concurrency: + group: rustfs-performance-test + cancel-in-progress: false + +defaults: + run: + shell: bash + +env: + RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }} + RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }} + RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }} + RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }} + # Package used by the nightly run (workflow_dispatch inputs are empty for + # workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml. + RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }} + # Fixed benchmark result directory so later steps can read summary.md + RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results + # Cross-repo token for writing to rustfs/backlog (set in repo settings) + PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + +jobs: + performance-test: + runs-on: pf-testing + timeout-minutes: 900 + # Run on manual dispatch, or when the nightly build completed successfully. + # Skipped when nightly failed. + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + ref: ${{ github.event.workflow_run.head_sha || github.ref }} + + - name: Show environment + run: | + uname -a + jq --version + warp --version || true + df -h /data | tail -1 + + - name: Reset test environment (before) + if: ${{ inputs.cleanup_before != 'false' }} + run: | + chmod +x scripts/test/rustfs_performance_test.sh + ./scripts/test/rustfs_performance_test.sh --step 1 -y + + - name: Install RustFS package & start cluster (4x4) + run: | + ARGS=(--steps "2,3,4" -y) + if [ -n "${{ inputs.package_url }}" ]; then + ARGS+=(--package-url "${{ inputs.package_url }}") + else + ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") + fi + ./scripts/test/rustfs_performance_test.sh "${ARGS[@]}" + + - name: Preflight checks + run: | + ARGS=(--preflight) + if [ -n "${{ inputs.package_url }}" ]; then + ARGS+=(--package-url "${{ inputs.package_url }}") + else + ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") + fi + ./scripts/test/rustfs_performance_test.sh "${ARGS[@]}" + + - name: Run benchmark (GET/PUT/MIXED) + id: benchmark + run: | + # Empty on automatic (workflow_run) runs -> full 30 rounds. + # Manual dispatch can restrict method(s)/size(s). + export WARP_METHODS="${{ inputs.test_method }}" + export WARP_SIZES="${{ inputs.object_size }}" + ./scripts/test/rustfs_performance_test.sh \ + --step 5 -y \ + --warp-duration "${{ inputs.warp_duration || '5m' }}" \ + --warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \ + --log-file /tmp/rustfs-perf-test.log + + - name: Analyze results + if: ${{ steps.benchmark.conclusion == 'success' }} + run: | + ./scripts/test/rustfs_performance_test.sh --step 6 -y + + - name: Post results to backlog issue + if: ${{ steps.benchmark.conclusion == 'success' }} + env: + GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} + RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }} + run: | + set -euo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "PF_TESTING_GH_TOKEN is not configured; skipping issue post" + exit 0 + fi + SUMMARY="${RESULT_DIR}/summary.md" + [ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; } + DATE="$(date -u +%Y-%m-%d)" + { + echo "## RustFS nightly build performance testing report" + echo "" + echo "- **日期**: ${DATE}" + echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- **触发方式**: ${{ github.event_name }}" + echo "" + cat "${SUMMARY}" + } > /tmp/rustfs-perf-issue-body.md + TITLE="RustFS nightly build performance testing report" + EXISTING="$(gh issue list --repo rustfs/backlog \ + --search "in:title \"${TITLE}\"" --state all --limit 5 \ + --json number --jq '.[0].number // empty')" + if [ -n "${EXISTING}" ]; then + gh issue comment "${EXISTING}" --repo rustfs/backlog --body-file /tmp/rustfs-perf-issue-body.md + echo "commented on existing issue #${EXISTING}" + else + gh issue create --repo rustfs/backlog --title "${TITLE}" --body-file /tmp/rustfs-perf-issue-body.md + fi + + - name: Upload test logs & results + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: rustfs-perf-test-${{ github.run_id }} + path: | + /tmp/rustfs-perf-test*.log + /tmp/rustfs-perf-results/** + if-no-files-found: warn + + - name: Reset test environment (after) + if: ${{ always() && inputs.cleanup_after != 'false' }} + run: | + ./scripts/test/rustfs_performance_test.sh --step 7 -y + + - name: Notify on failure + if: failure() + run: | + echo "RustFS performance test failed" + echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}" + echo "See the uploaded log artifact for details." diff --git a/scripts/test/rustfs_performance_test.sh b/scripts/test/rustfs_performance_test.sh new file mode 100755 index 000000000..12878772d --- /dev/null +++ b/scripts/test/rustfs_performance_test.sh @@ -0,0 +1,543 @@ +#!/usr/bin/env bash +# +# rustfs-performance-test.sh +# RustFS 4x4 集群性能压测全流程脚本 +# +# Based on the Obsidian note "RustFS 性能测试". Full workflow: +# 1. Cleanup: stop & purge rustfs, remove data dirs on all nodes +# 2. Download the RustFS package on all nodes +# 3. Install RustFS on all nodes (dpkg -i), recreate volume dirs +# 4. Write /etc/default/rustfs (4-node x 4-drive MNMD), start all nodes +# in parallel and verify the service is Running +# 5. Run the benchmark (warp GET/PUT/MIXED via rustfs-performance-testing.sh) +# 6. Analyze results (summary.tsv / summary.md) +# 7. Final cleanup: stop & purge rustfs, remove data dirs +# +# The script is driven from an admin host (e.g. a jumpbox) and operates on +# the target nodes over SSH, mirroring scripts/test/rustfs_*_test.sh. +# +# Usage: +# ./rustfs-performance-test.sh --all # run all steps 1-7 +# ./rustfs-performance-test.sh --step 5 # run a single step +# ./rustfs-performance-test.sh --steps 2,3,4 # run selected steps +# ./rustfs-performance-test.sh --all --dry-run # preview only +# ./rustfs-performance-test.sh --all -y --package-url +# +# Notes: +# - SSH user defaults to azureuser (passwordless sudo on the nodes); +# pass --ssh-user root if your nodes accept root login. +# - The benchmark runner defaults to +# ~/Documents/Obsidian Vault/rustfs-performance-testing.sh; override with +# --bench-script / RUSTFS_BENCH_SCRIPT. warp must be installed on the +# admin host. +# - Steps 1 and 7 destroy the RustFS install and all data (confirmed). +# +set -Eeuo pipefail + +# ==================== Configuration (adjust to your environment) ==================== + +# Target nodes (4x4: 4 nodes x 4 drives each) +if [ -n "${RUSTFS_NODES:-}" ]; then + read -r -a NODES <<<"${RUSTFS_NODES}" +else + NODES=(vm000 vm001 vm002 vm003) +fi + +SSH_USER="${RUSTFS_SSH_USER:-azureuser}" +SSH_PORT="${RUSTFS_SSH_PORT:-22}" +SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new -p "${SSH_PORT}") + +# Package: GitHub release tag, e.g. "1.0.0-rc.3". PACKAGE_URL is derived from +# RUSTFS_VERSION unless --package-url / RUSTFS_PACKAGE_URL is given. +RUSTFS_VERSION="${RUSTFS_VERSION:-1.0.0-rc.3}" +PACKAGE_URL="${RUSTFS_PACKAGE_URL:-}" +ARCH="${RUSTFS_ARCH:-amd64}" +PACKAGES_DIR="/home/rustfs/packages" +PACKAGE_FILE="rustfs.deb" +PACKAGE_SHA256="${RUSTFS_PACKAGE_SHA256:-}" + +# 4x4 topology: 4 nodes x 4 drives each, same expression on every node +DRIVES_PER_NODE="${RUSTFS_DRIVES_PER_NODE:-4}" +VOLUMES="http://rustfs-node{1...4}:9000/data/rustfs{1...4}/mnmd" + +# RustFS service configuration (written to /etc/default/rustfs) +RUSTFS_CONFIG_FILE="/etc/default/rustfs" +RUSTFS_SERVICE="rustfs" +RUSTFS_PACKAGE_NAME="rustfs" +RUSTFS_USER="rustfs" +ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfs@test}" +SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfs@test}" +RUSTFS_ADDRESS=":9000" +RUSTFS_CONSOLE_ADDRESS=":9001" +RUSTFS_CONSOLE_ENABLE=true +RUSTFS_OBS_LOGGER_LEVEL=error +RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/" + +# Benchmark runner (step 5/6): prefer the default Obsidian location, fall back +# to a rustfs-performance-testing.sh next to this script (e.g. in the repo or +# on a jumpbox). +_DEFAULT_BENCH="${HOME}/Documents/Obsidian Vault/rustfs-performance-testing.sh" +_SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +if [ -x "${_DEFAULT_BENCH}" ]; then + _BENCH_RESOLVED="${_DEFAULT_BENCH}" +elif [ -x "${_SCRIPT_DIR}/rustfs-performance-testing.sh" ]; then + _BENCH_RESOLVED="${_SCRIPT_DIR}/rustfs-performance-testing.sh" +elif [ -x "${_SCRIPT_DIR}/rustfs_performance_testing.sh" ]; then + _BENCH_RESOLVED="${_SCRIPT_DIR}/rustfs_performance_testing.sh" +else + _BENCH_RESOLVED="${_DEFAULT_BENCH}" +fi +BENCH_SCRIPT="${RUSTFS_BENCH_SCRIPT:-${_BENCH_RESOLVED}}" +RESULT_DIR="${RUSTFS_RESULT_DIR:-$(pwd)/warp-bench-results-$(date +%Y%m%d-%H%M%S)}" +WARP_HOST="${RUSTFS_WARP_HOST:-rustfs-node1:9000,rustfs-node2:9000,rustfs-node3:9000,rustfs-node4:9000}" +WARP_BUCKET="${RUSTFS_WARP_BUCKET:-warp-benchmark-bucket}" +WARP_CONCURRENCY="${RUSTFS_WARP_CONCURRENCY:-64}" +WARP_DURATION="${RUSTFS_WARP_DURATION:-5m}" +WARP_GET_OBJECTS="${RUSTFS_WARP_GET_OBJECTS:-2500}" +WARP_SLEEP="${RUSTFS_WARP_SLEEP:-60}" +# Manual method/size selection (passed through to the benchmark runner; empty = full run) +WARP_METHODS="${RUSTFS_WARP_METHODS:-}" +WARP_SIZES="${RUSTFS_WARP_SIZES:-}" + +# Timeouts (seconds) +SERVICE_TIMEOUT="${RUSTFS_SERVICE_TIMEOUT:-300}" +POLL_INTERVAL="${RUSTFS_POLL_INTERVAL:-10}" + +# ==================== Runtime options (set by CLI) ==================== +DRY_RUN=0 +ASSUME_YES=0 +SKIP_DOWNLOAD=0 +PREFLIGHT=0 +LOG_FILE="" +SELECTED_STEPS=() + +# ==================== Helpers ==================== + +log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33m[WARN]\033[0m %s\n' "$*"; } +die() { printf '\033[1;31m[ERROR]\033[0m %s\n' "$*" >&2; exit 1; } + +confirm() { + if [ "${ASSUME_YES}" -eq 1 ] || [ "${DRY_RUN}" -eq 1 ]; then return 0; fi + printf '\033[1;33m[CONFIRM]\033[0m %s (y/N) ' "$1" + read -r answer + case "${answer}" in + y|Y|yes|YES) return 0 ;; + *) die "cancelled" ;; + esac +} + +need_cmd() { + [ "${DRY_RUN}" -eq 1 ] && return 0 + command -v "$1" >/dev/null 2>&1 || die "missing command: $1 ($2); install it first" +} + +# Run a remote script on a single node (script is read from stdin) +run_remote() { + local node="$1" script + script="$(cat)" + if [ "${DRY_RUN}" -eq 1 ]; then + log "DRY-RUN: ssh ${SSH_USER}@${node} <<'REMOTE'" + printf '%s\n' "${script}" | sed 's/^/ | /' + log "DRY-RUN: ----" + return 0 + fi + log "==> ${node}: executing remote script" + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'bash -s' <<<"${script}" +} + +# Run the same remote script on all nodes in parallel (script from stdin) +run_remote_all() { + local script pids=() i=0 fail=0 + script="$(cat)" + for node in "${NODES[@]}"; do + if [ "${DRY_RUN}" -eq 1 ]; then + log "DRY-RUN: ssh ${SSH_USER}@${node} <<'REMOTE'" + printf '%s\n' "${script}" | sed 's/^/ | /' + log "DRY-RUN: ----" + else + log "==> ${node}: executing remote script" + ( ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'bash -s' <<<"${script}" ) & + pids[$i]=$! + i=$((i+1)) + fi + done + if [ "${#pids[@]}" -gt 0 ]; then + for pid in "${pids[@]}"; do + wait "${pid}" || fail=1 + done + fi + [ "${fail}" -eq 0 ] || die "one or more remote executions failed" +} + +rustfs_config_body() { + cat </dev/null </dev/null" | grep -q active; then + log "${node}: service active" + return 0 + fi + sleep "${POLL_INTERVAL}" + elapsed=$((elapsed + POLL_INTERVAL)) + done + die "${node}: ${RUSTFS_SERVICE} did not become active within ${SERVICE_TIMEOUT}s" +} + +verify_service_running() { + local node="$1" + log "${node}: checking service status" + if [ "${DRY_RUN}" -eq 0 ]; then + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \ + "systemctl status ${RUSTFS_SERVICE} --no-pager | head -n 12" || true + fi +} + +build_package_url() { + local asset + asset="rustfs_$(printf '%s' "${RUSTFS_VERSION}" | tr '-' '.')_${ARCH}.deb" + printf 'https://github.com/rustfs/rustfs/releases/download/%s/%s' "${RUSTFS_VERSION}" "${asset}" +} + +resolve_package_url() { + if [ -n "${PACKAGE_URL}" ]; then printf '%s' "${PACKAGE_URL}"; else build_package_url; fi +} + +preflight() { + log "preflight checks" + need_cmd ssh "openssh client" + need_cmd curl "http client" + need_cmd warp "warp benchmark tool (for step 5)" + if [ ! -x "${BENCH_SCRIPT}" ]; then + die "benchmark script not found or not executable: ${BENCH_SCRIPT}" + fi + if [ "${DRY_RUN}" -eq 0 ]; then + for node in "${NODES[@]}"; do + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'echo ok' >/dev/null \ + || die "cannot ssh to ${node}" + done + log "all nodes reachable: ${NODES[*]}" + fi + log "preflight OK" +} + +# ==================== Steps ==================== + +step1_cleanup() { + log "step 1: cleanup environment on all nodes (stop & purge rustfs, remove data dirs)" + confirm "This DESTROYS the RustFS install and ALL data on ${NODES[*]} (irreversible). Continue?" + local script + script="$(cat </dev/null || true +if \${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + \${SUDO} dpkg -P rustfs + echo "purged rustfs" +else + echo "rustfs not installed, skip purge" +fi +for i in \$(seq 1 ${DRIVES_PER_NODE}); do + \${SUDO} rm -rf /data/rustfs\${i}/mnmd + \${SUDO} mkdir -p /data/rustfs\${i}/mnmd + \${SUDO} chown -R rustfs:rustfs /data/rustfs\${i}/mnmd +done +echo "cleanup done on \$(hostname)" +EOF +)" + printf '%s\n' "${script}" | run_remote_all + log "step 1 complete" +} + +step2_download() { + log "step 2: download the package on all nodes" + local url script + url="$(resolve_package_url)" + script="$(cat </dev/null || true +if \${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + \${SUDO} dpkg -P rustfs + echo "purged rustfs" +fi +for i in \$(seq 1 ${DRIVES_PER_NODE}); do + \${SUDO} rm -rf /data/rustfs\${i}/mnmd +done +echo "cleanup done on \$(hostname)" +EOF +)" + printf '%s\n' "${script}" | run_remote_all + log "step 7 complete" +} + +# ==================== CLI ==================== + +usage() { + cat <<'USAGE' +Usage: ./rustfs-performance-test.sh [options] + +Steps: + 1 cleanup environment (purge rustfs, remove data dirs) [destructive] + 2 download the RustFS package on all nodes + 3 install RustFS (dpkg -i) + 4 write config, start service, verify Running + 5 run benchmark (warp GET/PUT/MIXED) + 6 analyze results (summary.tsv / summary.md) + 7 final cleanup (purge rustfs, remove data dirs) [destructive] + +Options: + --all Run all steps 1-7 + --step N Run a single step + --steps 1,3,5-7 Run selected steps + --version VERSION GitHub release tag (default 1.0.0-rc.3) + --package-url URL Direct deb URL (overrides --version) + --sha256 HASH Verify package checksum + --skip-download Keep an existing package file + --bench-script PATH Benchmark runner (default: Obsidian Vault rustfs-performance-testing.sh) + --result-dir DIR Benchmark result directory + --warp-duration DUR warp duration per round (default 5m) + --warp-concurrency N warp concurrency (default 64) + --ssh-user USER SSH user (default azureuser) + --ssh-port PORT SSH port (default 22) + --preflight Check environment and exit + --log-file FILE Append all output to FILE + --dry-run Preview commands without executing them + -y, --yes Skip all confirmation prompts + -h, --help Show this help + +Examples: + ./rustfs-performance-test.sh --all + ./rustfs-performance-test.sh --all --dry-run + ./rustfs-performance-test.sh --all -y --package-url https://dl.rustfs.com/...deb + ./rustfs-performance-test.sh --step 5 +USAGE +} + +expand_steps() { + local spec="$1" part start end i + IFS=',' read -ra parts <<<"${spec}" + for part in "${parts[@]}"; do + if [[ "${part}" =~ ^([0-9]+)-([0-9]+)$ ]]; then + start="${BASH_REMATCH[1]}"; end="${BASH_REMATCH[2]}" + for ((i=start; i<=end; i++)); do SELECTED_STEPS+=("${i}"); done + elif [[ "${part}" =~ ^[0-9]+$ ]]; then + SELECTED_STEPS+=("${part}") + else + die "cannot parse step spec: ${part}" + fi + done +} + +run_steps() { + local step + for step in "${SELECTED_STEPS[@]}"; do + case "${step}" in + 1) step1_cleanup ;; + 2) step2_download ;; + 3) step3_install ;; + 4) step4_configure_start ;; + 5) step5_benchmark ;; + 6) step6_analyze ;; + 7) step7_cleanup ;; + *) die "unknown step: ${step}" ;; + esac + log "step ${step} completed" + done +} + +main() { + [ "$#" -eq 0 ] && { usage; exit 0; } + local opt all=0 + while [ "$#" -gt 0 ]; do + opt="$1"; shift + case "${opt}" in + --all) all=1 ;; + --step) SELECTED_STEPS+=("$1"); shift ;; + --steps) expand_steps "$1"; shift ;; + --version) RUSTFS_VERSION="$1"; shift ;; + --package-url) PACKAGE_URL="$1"; shift ;; + --sha256) PACKAGE_SHA256="$1"; shift ;; + --skip-download) SKIP_DOWNLOAD=1 ;; + --bench-script) BENCH_SCRIPT="$1"; shift ;; + --result-dir) RESULT_DIR="$1"; shift ;; + --warp-duration) WARP_DURATION="$1"; shift ;; + --warp-concurrency) WARP_CONCURRENCY="$1"; shift ;; + --ssh-user) SSH_USER="$1"; shift ;; + --ssh-port) SSH_PORT="$1"; shift ;; + --preflight) PREFLIGHT=1 ;; + --log-file) LOG_FILE="$1"; shift ;; + --dry-run) DRY_RUN=1 ;; + -y|--yes) ASSUME_YES=1 ;; + -h|--help) usage; exit 0 ;; + *) die "unknown option: ${opt} (see --help)" ;; + esac + done + if [ -n "${LOG_FILE}" ]; then + mkdir -p "$(dirname "${LOG_FILE}")" + exec > >(tee -a "${LOG_FILE}") 2>&1 + fi + if [ "${all}" -eq 1 ]; then + SELECTED_STEPS=(1 2 3 4 5 6 7) + fi + if [ "${PREFLIGHT}" -eq 1 ]; then + preflight + if [ "${#SELECTED_STEPS[@]}" -eq 0 ]; then + log "preflight only; done" + exit 0 + fi + fi + [ "${#SELECTED_STEPS[@]}" -gt 0 ] || die "no steps selected (--all / --step / --steps)" + log "nodes: ${NODES[*]} ssh user: ${SSH_USER} version: ${RUSTFS_VERSION}" + log "package: $(resolve_package_url)" + log "result dir: ${RESULT_DIR}" + [ "${DRY_RUN}" -eq 1 ] && warn "DRY-RUN mode: only printing the commands that would run" + run_steps + log "all done" +} + +# Allow sourcing the file for unit tests without running main. +if [ "${RUSTFS_PERF_SCRIPT_SOURCE_ONLY:-0}" != "1" ]; then + main "$@" +fi diff --git a/scripts/test/rustfs_performance_testing.sh b/scripts/test/rustfs_performance_testing.sh new file mode 100755 index 000000000..c53e2a812 --- /dev/null +++ b/scripts/test/rustfs_performance_testing.sh @@ -0,0 +1,239 @@ +#!/usr/bin/env bash +# +# rustfs-performance-testing.sh +# RustFS 对象存储压测脚本（固定版）：测试方法 + 执行 + 结果解析 +# +# 测试方法 +# 1) 方法：GET / PUT / MIXED（warp 默认混合负载 45% GET + 55% PUT） +# 2) 对象尺寸：1KiB 4KiB 16KiB 128KiB 1MiB 4MiB 8MiB 16MiB 32MiB 64MiB +# 3) 并发：64；单轮时长：5m；轮间 sleep：60s；GET 对象数：2500 +# 4) 顺序：GET 全部尺寸 -> PUT 全部尺寸 -> MIXED 全部尺寸 +# 5) 结果解析：每轮结束后自动解析 warp 输出，写入 +# summary.tsv（机器可读）与 summary.md（Markdown 汇总表） +# +# 依赖：warp >= v1.6（MinIO warp），bash，awk/sed/grep +# 说明：warp v1.6.1 的 put 不支持 --objects，脚本已自动处理（仅 get/mixed 传该参数） +# +# 环境变量覆盖（不传时使用固定默认值）： +# WARP_HOST WARP_ACCESS_KEY WARP_SECRET_KEY WARP_BUCKET +# WARP_CONCURRENCY WARP_DURATION WARP_GET_OBJECTS WARP_SLEEP_BETWEEN_ROUNDS +# WARP_RESULT_DIR +# WARP_METHODS WARP_SIZES # 手动指定方法/尺寸（逗号或空格分隔），不传则全量 + +set -u -o pipefail + +HOST="${WARP_HOST:-rustfs-node1:9000,rustfs-node2:9000,rustfs-node3:9000,rustfs-node4:9000}" +ACCESS_KEY="${WARP_ACCESS_KEY:-rustfs@test}" +SECRET_KEY="${WARP_SECRET_KEY:-rustfs@test}" +BUCKET="${WARP_BUCKET:-warp-benchmark-bucket}" +CONCURRENCY="${WARP_CONCURRENCY:-64}" +DURATION="${WARP_DURATION:-5m}" +GET_OBJECTS="${WARP_GET_OBJECTS:-2500}" +SLEEP_BETWEEN_ROUNDS="${WARP_SLEEP_BETWEEN_ROUNDS:-60}" +RESULT_DIR="${WARP_RESULT_DIR:-$(pwd)/warp-bench-results-$(date +%Y%m%d-%H%M%S)}" + +if [ -n "${WARP_SIZES:-}" ] && [ "${WARP_SIZES}" != "all" ] && [ "${WARP_SIZES}" != "ALL" ]; then + read -r -a SIZES <<<"${WARP_SIZES//,/ }" +else + SIZES=(1KiB 4KiB 16KiB 128KiB 1MiB 4MiB 8MiB 16MiB 32MiB 64MiB) +fi +if [ -n "${WARP_METHODS:-}" ] && [ "${WARP_METHODS}" != "all" ] && [ "${WARP_METHODS}" != "ALL" ]; then + read -r -a METHODS <<<"${WARP_METHODS//,/ }" +else + METHODS=(get put mixed) +fi +TOTAL_ROUNDS=$(( ${#METHODS[@]} * ${#SIZES[@]} )) +ROUND=0 + +# --parse-only ：只解析已有结果目录（${method}_${size}.txt），不执行压测 +if [[ "${1:-}" == "--parse-only" && -n "${2:-}" ]]; then + RESULT_DIR="$2" +fi + +LOG_FILE="${RESULT_DIR}/master.log" +SUMMARY_TSV="${RESULT_DIR}/summary.tsv" +SUMMARY_MD="${RESULT_DIR}/summary.md" + +if [[ "${1:-}" != "--parse-only" ]] && ! command -v warp >/dev/null 2>&1; then + echo "错误：未找到 warp 命令，请先安装 MinIO warp。" >&2 + exit 1 +fi + +log() { + echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') $*" | tee -a "${LOG_FILE}" +} + +# ---- 结果解析 ---- + +# 提取指定 section（GET/PUT/Total 等）的 Average / Reqs / TTFB 原始行 +section_lines() { + awk -v sec="$2" ' + /^Report: / { cur = $2; sub(/\.$/, "", cur) } + cur == sec && /^ *\* Average:/ { avg = $0 } + cur == sec && /^ *\* Reqs:/ { reqs = $0 } + cur == sec && /^ *\* TTFB:/ { ttfb = $0 } + END { + if (avg != "") print avg + if (reqs != "") print reqs + if (ttfb != "") print ttfb + } + ' "$1" +} + +# 从统计行中取字段：tp objs avg p50 p90 p99 ttfb_avg ttfb_p99 ttfb_worst +field() { + case "$2" in + tp) echo "$1" | sed -n 's/^ *\* Average: \(.*\), \([0-9.]*\) obj\/s.*/\1/p' ;; + objs) echo "$1" | sed -n 's/^ *\* Average: .*, \([0-9.]*\) obj\/s.*/\1/p' ;; + avg) echo "$1" | sed -n 's/^ *\* Reqs: Avg: \([^,]*\),.*/\1/p' ;; + p50) echo "$1" | sed -n 's/^ *\* Reqs: Avg: [^,]*, 50%: \([^,]*\),.*/\1/p' ;; + p90) echo "$1" | sed -n 's/^ *\* Reqs: Avg: [^,]*, 50%: [^,]*, 90%: \([^,]*\),.*/\1/p' ;; + p99) echo "$1" | sed -n 's/^ *\* Reqs: Avg: [^,]*, 50%: [^,]*, 90%: [^,]*, 99%: \([^,]*\),.*/\1/p' ;; + ttfb_avg) echo "$1" | sed -n 's/^ *\* TTFB: Avg: \([^,]*\),.*/\1/p' ;; + ttfb_p99) echo "$1" | sed -n 's/^ *\* TTFB: .*99th: \([^,]*\),.*/\1/p' ;; + ttfb_worst) echo "$1" | sed -n 's/^ *\* TTFB: .*Worst: \([^ ]*\).*/\1/p' ;; + *) echo "" ;; + esac +} + +# 解析一轮输出，追加一行到 summary.tsv +parse_round() { + local method="$1" size="$2" file="$3" + local line + if [[ "${method}" == "mixed" ]]; then + local total get put + total=$(section_lines "$file" Total) + get=$(section_lines "$file" GET) + put=$(section_lines "$file" PUT) + line=$(printf 'mixed\t%s\t%s\t%s\t%s\t%s' \ + "$size" \ + "$(field "${total}" tp)" \ + "$(field "${total}" objs)" \ + "$(field "${get}" avg)" \ + "$(field "${put}" avg)") + else + local sec + sec=$(printf '%s' "${method}" | tr '[:lower:]' '[:upper:]') + local stats + stats=$(section_lines "$file" "${sec}") + line=$(printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s' \ + "${method}" "${size}" \ + "$(field "${stats}" tp)" \ + "$(field "${stats}" objs)" \ + "$(field "${stats}" avg)" \ + "$(field "${stats}" p50)" \ + "$(field "${stats}" p90)" \ + "$(field "${stats}" p99)" \ + "$(field "${stats}" ttfb_avg)" \ + "$(field "${stats}" ttfb_p99)" \ + "$(field "${stats}" ttfb_worst)") + fi + printf '%s\n' "${line}" >> "${SUMMARY_TSV}" +} + +# 汇总 summary.tsv -> summary.md（Markdown 表格） +gen_summary_md() { + { + echo "# RustFS 性能压测结果" + echo "" + echo "- 日期：$(date -u '+%Y-%m-%d %H:%M:%S UTC')" + echo "- 目标：${HOST}" + echo "- 并发：${CONCURRENCY}；单轮：${DURATION}；sleep：${SLEEP_BETWEEN_ROUNDS}s；GET objects：${GET_OBJECTS}" + echo "- 方法：GET / PUT / MIXED（warp 默认混合负载）；尺寸：${SIZES[*]}" + echo "" + } > "${SUMMARY_MD}" + + for m in get put; do + { + echo "## $(printf '%s' "$m" | tr '[:lower:]' '[:upper:]') 结果" + echo "" + echo "| 对象尺寸 | 平均吞吐 | 平均 obj/s | Avg Latency | P50 | P90 | P99 | TTFB Avg | TTFB P99 | TTFB 最差 |" + echo "|----------|----------|-----------|-------------|-----|-----|-----|----------|----------|-----------|" + } >> "${SUMMARY_MD}" + while IFS=$'\t' read -r method size tp objs avg p50 p90 p99 ttfb_avg ttfb_p99 ttfb_worst; do + [[ "${method}" == "${m}" ]] && \ + echo "| ${size} | ${tp} | ${objs} | ${avg} | ${p50} | ${p90} | ${p99} | ${ttfb_avg} | ${ttfb_p99} | ${ttfb_worst} |" >> "${SUMMARY_MD}" + done < "${SUMMARY_TSV}" + echo "" >> "${SUMMARY_MD}" + done + + { + echo "## MIXED 结果（Total 口径）" + echo "" + echo "| 对象尺寸 | Total 平均吞吐 | Total 平均 obj/s | Mixed-GET Avg | Mixed-PUT Avg |" + echo "|----------|---------------|------------------|----------------|----------------|" + } >> "${SUMMARY_MD}" + while IFS=$'\t' read -r method size tp objs gavg pavg rest; do + [[ "${method}" == "mixed" ]] && \ + echo "| ${size} | ${tp} | ${objs} | ${gavg} | ${pavg} |" >> "${SUMMARY_MD}" + done < "${SUMMARY_TSV}" + echo "" >> "${SUMMARY_MD}" +} + +# ---- 主流程 ---- + +mkdir -p "${RESULT_DIR}" +log "CONFIG host=${HOST} bucket=${BUCKET} concurrency=${CONCURRENCY} duration=${DURATION} get_objects=${GET_OBJECTS} sleep_between_rounds=${SLEEP_BETWEEN_ROUNDS}s" +printf 'method\tsize\tthroughput\tobj_per_s\treq_avg\treq_p50\treq_p90\treq_p99\tttfb_avg\tttfb_p99\tttfb_worst\n' > "${SUMMARY_TSV}" + +if [[ "${1:-}" == "--parse-only" ]]; then + for method in "${METHODS[@]}"; do + for size in "${SIZES[@]}"; do + outfile="${RESULT_DIR}/${method}_${size}.txt" + if [[ -s "${outfile}" ]]; then + parse_round "${method}" "${size}" "${outfile}" + fi + done + done + gen_summary_md + echo "parsed from ${RESULT_DIR}" + echo "" + cat "${SUMMARY_MD}" + exit 0 +fi + +for method in "${METHODS[@]}"; do + for size in "${SIZES[@]}"; do + ROUND=$((ROUND + 1)) + outfile="${RESULT_DIR}/${method}_${size}.txt" + + log "START round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} concurrency=${CONCURRENCY} duration=${DURATION}" + + extra_args=() + if [[ "${method}" != "put" ]]; then + extra_args=(--objects "${GET_OBJECTS}") + fi + + start_epoch=$(date +%s) + warp "${method}" \ + --host "${HOST}" \ + --access-key "${ACCESS_KEY}" \ + --secret-key "${SECRET_KEY}" \ + --bucket "${BUCKET}" \ + --concurrent "${CONCURRENCY}" \ + --duration "${DURATION}" \ + --obj.size "${size}" \ + "${extra_args[@]}" \ + --no-color 2>&1 | tee "${outfile}" + rc=${PIPESTATUS[0]} + end_epoch=$(date +%s) + + if [[ ${rc} -eq 0 ]]; then + parse_round "${method}" "${size}" "${outfile}" + log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=ok" + else + log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=skipped" + fi + + if [[ ${ROUND} -lt ${TOTAL_ROUNDS} ]]; then + log "SLEEP ${SLEEP_BETWEEN_ROUNDS}s before next round" + sleep "${SLEEP_BETWEEN_ROUNDS}" + fi + done +done + +gen_summary_md +log "ALL_ROUNDS_COMPLETE summary_tsv=${SUMMARY_TSV} summary_md=${SUMMARY_MD}" +echo "" +echo "==== 结果汇总 ====" +cat "${SUMMARY_MD}"