mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-01 17:58:22 +00:00
Extend functional test workflow with S3/KMS/tier suites (#6806)
This commit is contained in:
hector
@@ -23,6 +23,12 @@ on:
|
|||||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||||
type: boolean
|
type: boolean
|
||||||
default: true
|
default: true
|
||||||
|
workflow_run:
|
||||||
|
# Run after the nightly build completes: the nightly deb is what the test
|
||||||
|
# installs. Heal also runs inside the functional suite; this standalone
|
||||||
|
# workflow enables manual single-suite runs as well.
|
||||||
|
workflows: ["Nightly GNU Build"]
|
||||||
|
types: [completed]
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
@@ -49,10 +55,16 @@ jobs:
|
|||||||
heal-test:
|
heal-test:
|
||||||
runs-on: smoke-testing
|
runs-on: smoke-testing
|
||||||
timeout-minutes: 480
|
timeout-minutes: 480
|
||||||
|
# Run on manual dispatch, or when the nightly build completed successfully.
|
||||||
|
# Skipped when nightly failed.
|
||||||
|
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout auto-testing scripts
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
with:
|
with:
|
||||||
|
repository: rustfs/auto-testing
|
||||||
|
ref: main
|
||||||
|
path: auto-testing
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
|
||||||
- name: Show environment
|
- name: Show environment
|
||||||
@@ -66,8 +78,8 @@ jobs:
|
|||||||
- name: Reset test environment (before)
|
- name: Reset test environment (before)
|
||||||
if: ${{ inputs.cleanup_before != 'false' }}
|
if: ${{ inputs.cleanup_before != 'false' }}
|
||||||
run: |
|
run: |
|
||||||
chmod +x scripts/test/rustfs_heal_test.sh
|
chmod +x auto-testing/rustfs_heal_test.sh
|
||||||
./scripts/test/rustfs_heal_test.sh --reset -y
|
./auto-testing/rustfs_heal_test.sh --reset -y
|
||||||
|
|
||||||
- name: Install RustFS package & start cluster
|
- name: Install RustFS package & start cluster
|
||||||
run: |
|
run: |
|
||||||
@@ -77,7 +89,7 @@ jobs:
|
|||||||
else
|
else
|
||||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||||
fi
|
fi
|
||||||
./scripts/test/rustfs_heal_test.sh "${ARGS[@]}"
|
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
|
||||||
|
|
||||||
- name: Preflight checks
|
- name: Preflight checks
|
||||||
run: |
|
run: |
|
||||||
@@ -87,11 +99,11 @@ jobs:
|
|||||||
else
|
else
|
||||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||||
fi
|
fi
|
||||||
./scripts/test/rustfs_heal_test.sh "${ARGS[@]}"
|
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
|
||||||
|
|
||||||
- name: Run heal test (write -> outage -> heal -> verify)
|
- name: Run heal test (write -> outage -> heal -> verify)
|
||||||
run: |
|
run: |
|
||||||
./scripts/test/rustfs_heal_test.sh \
|
./auto-testing/rustfs_heal_test.sh \
|
||||||
--steps "3,4,5,6,7" -y \
|
--steps "3,4,5,6,7" -y \
|
||||||
--endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
|
--endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
|
||||||
--stop-node-gb "${{ inputs.stop_node_gb }}" \
|
--stop-node-gb "${{ inputs.stop_node_gb }}" \
|
||||||
@@ -111,7 +123,7 @@ jobs:
|
|||||||
- name: Reset test environment (after)
|
- name: Reset test environment (after)
|
||||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||||
run: |
|
run: |
|
||||||
./scripts/test/rustfs_heal_test.sh --reset -y
|
./auto-testing/rustfs_heal_test.sh --reset -y
|
||||||
|
|
||||||
- name: Notify on failure
|
- name: Notify on failure
|
||||||
if: failure()
|
if: failure()
|
||||||
|
|||||||
@@ -89,11 +89,13 @@ jobs:
|
|||||||
# Skipped when nightly failed.
|
# Skipped when nightly failed.
|
||||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout auto-testing scripts
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
with:
|
with:
|
||||||
|
repository: rustfs/auto-testing
|
||||||
|
ref: main
|
||||||
|
path: auto-testing
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
ref: ${{ github.event.workflow_run.head_sha || github.ref }}
|
|
||||||
|
|
||||||
- name: Show environment
|
- name: Show environment
|
||||||
run: |
|
run: |
|
||||||
@@ -105,8 +107,8 @@ jobs:
|
|||||||
- name: Reset test environment (before)
|
- name: Reset test environment (before)
|
||||||
if: ${{ inputs.cleanup_before != 'false' }}
|
if: ${{ inputs.cleanup_before != 'false' }}
|
||||||
run: |
|
run: |
|
||||||
chmod +x scripts/test/rustfs_performance_test.sh
|
chmod +x auto-testing/rustfs_performance_test.sh
|
||||||
./scripts/test/rustfs_performance_test.sh --step 1 -y
|
./auto-testing/rustfs_performance_test.sh --step 1 -y
|
||||||
|
|
||||||
- name: Install RustFS package & start cluster (4x4)
|
- name: Install RustFS package & start cluster (4x4)
|
||||||
run: |
|
run: |
|
||||||
@@ -116,7 +118,7 @@ jobs:
|
|||||||
else
|
else
|
||||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||||
fi
|
fi
|
||||||
./scripts/test/rustfs_performance_test.sh "${ARGS[@]}"
|
./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
|
||||||
|
|
||||||
- name: Preflight checks
|
- name: Preflight checks
|
||||||
run: |
|
run: |
|
||||||
@@ -126,7 +128,7 @@ jobs:
|
|||||||
else
|
else
|
||||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||||
fi
|
fi
|
||||||
./scripts/test/rustfs_performance_test.sh "${ARGS[@]}"
|
./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
|
||||||
|
|
||||||
- name: Run benchmark (GET/PUT/MIXED)
|
- name: Run benchmark (GET/PUT/MIXED)
|
||||||
id: benchmark
|
id: benchmark
|
||||||
@@ -135,7 +137,7 @@ jobs:
|
|||||||
# Manual dispatch can restrict method(s)/size(s).
|
# Manual dispatch can restrict method(s)/size(s).
|
||||||
export WARP_METHODS="${{ inputs.test_method }}"
|
export WARP_METHODS="${{ inputs.test_method }}"
|
||||||
export WARP_SIZES="${{ inputs.object_size }}"
|
export WARP_SIZES="${{ inputs.object_size }}"
|
||||||
./scripts/test/rustfs_performance_test.sh \
|
./auto-testing/rustfs_performance_test.sh \
|
||||||
--step 5 -y \
|
--step 5 -y \
|
||||||
--warp-duration "${{ inputs.warp_duration || '5m' }}" \
|
--warp-duration "${{ inputs.warp_duration || '5m' }}" \
|
||||||
--warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
|
--warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
|
||||||
@@ -144,7 +146,7 @@ jobs:
|
|||||||
- name: Analyze results
|
- name: Analyze results
|
||||||
if: ${{ steps.benchmark.conclusion == 'success' }}
|
if: ${{ steps.benchmark.conclusion == 'success' }}
|
||||||
run: |
|
run: |
|
||||||
./scripts/test/rustfs_performance_test.sh --step 6 -y
|
./auto-testing/rustfs_performance_test.sh --step 6 -y
|
||||||
|
|
||||||
- name: Post results to backlog issue
|
- name: Post results to backlog issue
|
||||||
if: ${{ steps.benchmark.conclusion == 'success' }}
|
if: ${{ steps.benchmark.conclusion == 'success' }}
|
||||||
@@ -193,7 +195,7 @@ jobs:
|
|||||||
- name: Reset test environment (after)
|
- name: Reset test environment (after)
|
||||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||||
run: |
|
run: |
|
||||||
./scripts/test/rustfs_performance_test.sh --step 7 -y
|
./auto-testing/rustfs_performance_test.sh --step 7 -y
|
||||||
|
|
||||||
- name: Notify on failure
|
- name: Notify on failure
|
||||||
if: failure()
|
if: failure()
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
name: RustFS Pool Expansion / Decommission Test
|
name: RustFS Functional Test Suite (S3/KMS/tier/pool/heal)
|
||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
@@ -47,15 +47,16 @@ on:
|
|||||||
type: boolean
|
type: boolean
|
||||||
default: true
|
default: true
|
||||||
workflow_run:
|
workflow_run:
|
||||||
# Run after the nightly build completes: pool expansion first, then heal.
|
# Run after the nightly build completes: S3 -> KMS -> tier -> pool -> heal.
|
||||||
workflows: ["Nightly GNU Build"]
|
workflows: ["Nightly GNU Build"]
|
||||||
types: [completed]
|
types: [completed]
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
# Only one pool-expansion test at a time: the workflow mutates a shared
|
# Only one test run at a time: every job mutates the same shared test
|
||||||
# test environment, so concurrent runs must not clobber each other.
|
# environment (vm000/vm001/vm002), so concurrent runs must not clobber each
|
||||||
|
# other. Jobs inside a run are chained with needs to serialize them.
|
||||||
concurrency:
|
concurrency:
|
||||||
group: rustfs-pool-expansion-test
|
group: rustfs-pool-expansion-test
|
||||||
cancel-in-progress: false
|
cancel-in-progress: false
|
||||||
@@ -75,18 +76,172 @@ env:
|
|||||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
pool-expansion-test:
|
# All test scripts live in rustfs/auto-testing; rustfs stores only this
|
||||||
|
# workflow. Each job checks out auto-testing before running.
|
||||||
|
|
||||||
|
s3-compat-test:
|
||||||
|
name: S3 compatibility test
|
||||||
runs-on: smoke-testing
|
runs-on: smoke-testing
|
||||||
timeout-minutes: 360
|
timeout-minutes: 240
|
||||||
# Run on manual dispatch, or when the nightly build completed successfully
|
|
||||||
# (its deb is what the tests install). Skipped when nightly failed.
|
|
||||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout auto-testing scripts
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
with:
|
with:
|
||||||
|
repository: rustfs/auto-testing
|
||||||
|
ref: main
|
||||||
|
path: auto-testing
|
||||||
|
persist-credentials: false
|
||||||
|
|
||||||
|
- name: Run S3 compatibility suite
|
||||||
|
run: |
|
||||||
|
chmod +x auto-testing/rustfs-s3-compat-test.sh
|
||||||
|
ARGS=(--all-topologies -y --log-file /tmp/rustfs-s3-compat.log)
|
||||||
|
if [ -n "${{ inputs.package_url }}" ]; then
|
||||||
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
||||||
|
elif [ -n "${{ inputs.rustfs_version }}" ]; then
|
||||||
|
ARGS+=(--version "${{ inputs.rustfs_version }}")
|
||||||
|
else
|
||||||
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||||
|
fi
|
||||||
|
./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
|
||||||
|
|
||||||
|
- name: Upload test logs
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||||
|
with:
|
||||||
|
name: rustfs-s3-compat-${{ github.run_id }}
|
||||||
|
path: |
|
||||||
|
/tmp/rustfs-s3-compat*.log
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- name: Notify on failure
|
||||||
|
if: failure()
|
||||||
|
run: |
|
||||||
|
echo "RustFS S3 compatibility suite failed"
|
||||||
|
echo "See the uploaded log artifact for details."
|
||||||
|
|
||||||
|
kms-test:
|
||||||
|
name: KMS test (after S3)
|
||||||
|
runs-on: smoke-testing
|
||||||
|
timeout-minutes: 360
|
||||||
|
needs: s3-compat-test
|
||||||
|
if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout auto-testing scripts
|
||||||
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
|
with:
|
||||||
|
repository: rustfs/auto-testing
|
||||||
|
ref: main
|
||||||
|
path: auto-testing
|
||||||
|
persist-credentials: false
|
||||||
|
|
||||||
|
- name: Ensure docker (Vault container)
|
||||||
|
run: |
|
||||||
|
if ! command -v docker >/dev/null 2>&1; then
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y docker.io
|
||||||
|
fi
|
||||||
|
sudo systemctl enable --now docker
|
||||||
|
docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1
|
||||||
|
|
||||||
|
- name: Run KMS suite
|
||||||
|
run: |
|
||||||
|
chmod +x auto-testing/rustfs-kms-test.sh
|
||||||
|
ARGS=(--all-topologies --backends local,vault-kv2 -y --log-file /tmp/rustfs-kms.log)
|
||||||
|
if [ -n "${{ inputs.package_url }}" ]; then
|
||||||
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
||||||
|
elif [ -n "${{ inputs.rustfs_version }}" ]; then
|
||||||
|
ARGS+=(--version "${{ inputs.rustfs_version }}")
|
||||||
|
else
|
||||||
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||||
|
fi
|
||||||
|
./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
|
||||||
|
|
||||||
|
- name: Upload test logs
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||||
|
with:
|
||||||
|
name: rustfs-kms-test-${{ github.run_id }}
|
||||||
|
path: |
|
||||||
|
/tmp/rustfs-kms*.log
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- name: Notify on failure
|
||||||
|
if: failure()
|
||||||
|
run: |
|
||||||
|
echo "RustFS KMS suite failed"
|
||||||
|
echo "See the uploaded log artifact for details."
|
||||||
|
|
||||||
|
tier-test:
|
||||||
|
name: Tier / event / audit test (after KMS)
|
||||||
|
runs-on: smoke-testing
|
||||||
|
timeout-minutes: 360
|
||||||
|
needs: kms-test
|
||||||
|
if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout auto-testing scripts
|
||||||
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
|
with:
|
||||||
|
repository: rustfs/auto-testing
|
||||||
|
ref: main
|
||||||
|
path: auto-testing
|
||||||
|
persist-credentials: false
|
||||||
|
|
||||||
|
- name: Ensure MQTT broker + clients (event notification tests)
|
||||||
|
run: |
|
||||||
|
if ! command -v mosquitto_sub >/dev/null 2>&1; then
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y mosquitto mosquitto-clients
|
||||||
|
fi
|
||||||
|
sudo mkdir -p /etc/mosquitto/conf.d
|
||||||
|
printf 'listener 1883 0.0.0.0\nallow_anonymous true\n' | sudo tee /etc/mosquitto/conf.d/rustfs-test.conf >/dev/null
|
||||||
|
sudo systemctl restart mosquitto
|
||||||
|
sleep 2
|
||||||
|
ss -tln 2>/dev/null | grep -q ':1883' || { echo "mosquitto not listening on 1883"; exit 1; }
|
||||||
|
|
||||||
|
- name: Run tier / event / audit suite
|
||||||
|
run: |
|
||||||
|
chmod +x auto-testing/rustfs-tier-test.sh
|
||||||
|
ARGS=(--all-topologies -y --log-file /tmp/rustfs-tier.log)
|
||||||
|
if [ -n "${{ inputs.package_url }}" ]; then
|
||||||
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
||||||
|
elif [ -n "${{ inputs.rustfs_version }}" ]; then
|
||||||
|
ARGS+=(--version "${{ inputs.rustfs_version }}")
|
||||||
|
else
|
||||||
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||||
|
fi
|
||||||
|
./auto-testing/rustfs-tier-test.sh "${ARGS[@]}"
|
||||||
|
|
||||||
|
- name: Upload test logs
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||||
|
with:
|
||||||
|
name: rustfs-tier-test-${{ github.run_id }}
|
||||||
|
path: |
|
||||||
|
/tmp/rustfs-tier*.log
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- name: Notify on failure
|
||||||
|
if: failure()
|
||||||
|
run: |
|
||||||
|
echo "RustFS tier/event/audit suite failed"
|
||||||
|
echo "See the uploaded log artifact for details."
|
||||||
|
|
||||||
|
pool-expansion-test:
|
||||||
|
name: Pool expansion / decommission test (after tier)
|
||||||
|
runs-on: smoke-testing
|
||||||
|
timeout-minutes: 360
|
||||||
|
needs: tier-test
|
||||||
|
if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout auto-testing scripts
|
||||||
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
|
with:
|
||||||
|
repository: rustfs/auto-testing
|
||||||
|
ref: main
|
||||||
|
path: auto-testing
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
ref: ${{ github.event.workflow_run.head_sha || github.ref }}
|
|
||||||
|
|
||||||
- name: Show environment
|
- name: Show environment
|
||||||
run: |
|
run: |
|
||||||
@@ -99,8 +254,8 @@ jobs:
|
|||||||
- name: Reset test environment (before)
|
- name: Reset test environment (before)
|
||||||
if: ${{ inputs.cleanup_before != 'false' }}
|
if: ${{ inputs.cleanup_before != 'false' }}
|
||||||
run: |
|
run: |
|
||||||
chmod +x scripts/test/rustfs_pool_expand.sh
|
chmod +x auto-testing/rustfs_pool_expand.sh
|
||||||
./scripts/test/rustfs_pool_expand.sh --reset -y
|
./auto-testing/rustfs_pool_expand.sh --reset -y
|
||||||
|
|
||||||
- name: Install RustFS package & start first pool
|
- name: Install RustFS package & start first pool
|
||||||
run: |
|
run: |
|
||||||
@@ -112,7 +267,7 @@ jobs:
|
|||||||
else
|
else
|
||||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||||
fi
|
fi
|
||||||
./scripts/test/rustfs_pool_expand.sh "${ARGS[@]}"
|
./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
|
||||||
|
|
||||||
- name: Preflight checks
|
- name: Preflight checks
|
||||||
run: |
|
run: |
|
||||||
@@ -124,7 +279,7 @@ jobs:
|
|||||||
else
|
else
|
||||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||||
fi
|
fi
|
||||||
./scripts/test/rustfs_pool_expand.sh "${ARGS[@]}"
|
./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
|
||||||
|
|
||||||
- name: Run pool expansion & decommission test
|
- name: Run pool expansion & decommission test
|
||||||
id: pool_test
|
id: pool_test
|
||||||
@@ -137,7 +292,7 @@ jobs:
|
|||||||
STEPS="$STEPS,9"
|
STEPS="$STEPS,9"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
./scripts/test/rustfs_pool_expand.sh \
|
./auto-testing/rustfs_pool_expand.sh \
|
||||||
--steps "$STEPS" --with-warp -y \
|
--steps "$STEPS" --with-warp -y \
|
||||||
--endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
|
--endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
|
||||||
--storage-threshold "${{ inputs.storage_threshold || '50' }}" \
|
--storage-threshold "${{ inputs.storage_threshold || '50' }}" \
|
||||||
@@ -157,7 +312,7 @@ jobs:
|
|||||||
- name: Reset test environment (after)
|
- name: Reset test environment (after)
|
||||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||||
run: |
|
run: |
|
||||||
./scripts/test/rustfs_pool_expand.sh --reset -y
|
./auto-testing/rustfs_pool_expand.sh --reset -y
|
||||||
|
|
||||||
- name: Notify on failure
|
- name: Notify on failure
|
||||||
if: failure()
|
if: failure()
|
||||||
@@ -175,17 +330,19 @@ jobs:
|
|||||||
needs: pool-expansion-test
|
needs: pool-expansion-test
|
||||||
if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
|
if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout auto-testing scripts
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
with:
|
with:
|
||||||
|
repository: rustfs/auto-testing
|
||||||
|
ref: main
|
||||||
|
path: auto-testing
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
ref: ${{ github.event.workflow_run.head_sha || github.ref }}
|
|
||||||
|
|
||||||
- name: Reset test environment (before)
|
- name: Reset test environment (before)
|
||||||
if: ${{ inputs.cleanup_before != 'false' }}
|
if: ${{ inputs.cleanup_before != 'false' }}
|
||||||
run: |
|
run: |
|
||||||
chmod +x scripts/test/rustfs_heal_test.sh
|
chmod +x auto-testing/rustfs_heal_test.sh
|
||||||
./scripts/test/rustfs_heal_test.sh --reset -y
|
./auto-testing/rustfs_heal_test.sh --reset -y
|
||||||
|
|
||||||
- name: Install RustFS package & start cluster
|
- name: Install RustFS package & start cluster
|
||||||
run: |
|
run: |
|
||||||
@@ -195,7 +352,7 @@ jobs:
|
|||||||
else
|
else
|
||||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||||
fi
|
fi
|
||||||
./scripts/test/rustfs_heal_test.sh "${ARGS[@]}"
|
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
|
||||||
|
|
||||||
- name: Preflight checks
|
- name: Preflight checks
|
||||||
run: |
|
run: |
|
||||||
@@ -205,11 +362,11 @@ jobs:
|
|||||||
else
|
else
|
||||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||||
fi
|
fi
|
||||||
./scripts/test/rustfs_heal_test.sh "${ARGS[@]}"
|
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
|
||||||
|
|
||||||
- name: Run heal test (write -> outage -> heal -> verify)
|
- name: Run heal test (write -> outage -> heal -> verify)
|
||||||
run: |
|
run: |
|
||||||
./scripts/test/rustfs_heal_test.sh \
|
./auto-testing/rustfs_heal_test.sh \
|
||||||
--steps 3,4,5,6,7 -y \
|
--steps 3,4,5,6,7 -y \
|
||||||
--endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
|
--endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
|
||||||
--stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
|
--stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
|
||||||
@@ -229,7 +386,7 @@ jobs:
|
|||||||
- name: Reset test environment (after)
|
- name: Reset test environment (after)
|
||||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||||
run: |
|
run: |
|
||||||
./scripts/test/rustfs_heal_test.sh --reset -y
|
./auto-testing/rustfs_heal_test.sh --reset -y
|
||||||
|
|
||||||
- name: Notify on failure
|
- name: Notify on failure
|
||||||
if: failure()
|
if: failure()
|
||||||
|
|||||||
@@ -1,106 +0,0 @@
|
|||||||
# RustFS Heal Test
|
|
||||||
|
|
||||||
Node-outage heal test driven by
|
|
||||||
[`scripts/test/rustfs_heal_test.sh`](rustfs_heal_test.sh), based on the
|
|
||||||
Obsidian note "RustFS Heal 测试步骤". Uses the same 3-node test environment as
|
|
||||||
the pool expansion test (`vm000 vm001 vm002`).
|
|
||||||
|
|
||||||
All status checks talk to the RustFS admin API directly (SigV4-signed,
|
|
||||||
`jq` assertions), no `rc` required.
|
|
||||||
|
|
||||||
## What it does
|
|
||||||
|
|
||||||
1. Downloads the `.deb` package on all nodes (release tag or a direct URL such
|
|
||||||
as the nightly/R2 package).
|
|
||||||
2. Installs it, writes the 3x4 config
|
|
||||||
(`http://rustfs-node{1...3}:9000/data/rustfs{1...4}/mnmd`), starts all
|
|
||||||
three nodes simultaneously, verifies the cluster is up.
|
|
||||||
3. Writes data with `warp` while monitoring disk usage on the surviving nodes
|
|
||||||
(`df -B1G | grep /data/rustfs`):
|
|
||||||
- when both surviving nodes reach `STOP_NODE_AT_GB` (default 15 GiB), stop
|
|
||||||
the outage node (`vm002`, `OUTAGE_NODE_INDEX=2`);
|
|
||||||
- keep writing until both surviving nodes reach `WARP_STOP_AT_GB`
|
|
||||||
(default 40 GiB), then stop warp.
|
|
||||||
4. Restarts the outage node.
|
|
||||||
5. Starts cluster heal: `POST /rustfs/admin/v3/heal/` with body
|
|
||||||
`{"recursive":true}` (retried, returns a `clientToken`).
|
|
||||||
6. Monitors the heal task via `POST /rustfs/admin/v3/heal/?clientToken=<token>`
|
|
||||||
until the server verdict is a terminal success (`finished`/`completed`) with
|
|
||||||
`objects_failed == 0`.
|
|
||||||
7. Result analysis: heal stats (scanned/healed/failed), an **S3 read-back
|
|
||||||
verification** of the written objects (list the test bucket and GET a
|
|
||||||
sample — every read must succeed), per-node disk usage (observability),
|
|
||||||
pass/fail verdict.
|
|
||||||
|
|
||||||
Success is the server's own scan/repair verdict (heal finished, 0 failed)
|
|
||||||
**plus** an end-to-end data read-back; per-node disk usage is logged as
|
|
||||||
observability, not a pass gate (EC distributes different shards per node, so a
|
|
||||||
fixed per-node GB target is not a meaningful invariant).
|
|
||||||
|
|
||||||
## Self-hosted runner prerequisites
|
|
||||||
|
|
||||||
- Register the admin host (e.g. `heal`) as a runner with the
|
|
||||||
`smoke-testing` label.
|
|
||||||
- Install `jq`, `openssl`, `curl` and `warp` on the runner. `rc` is **not**
|
|
||||||
required.
|
|
||||||
- The runner user must be able to SSH to `vm000/vm001/vm002` without a
|
|
||||||
password prompt; nodes need passwordless `sudo` for the SSH user and
|
|
||||||
resolvable `rustfs-node*` hostnames.
|
|
||||||
- Admin API credentials need the `admin:server-info`, `admin:heal` and
|
|
||||||
`admin:rebalance` actions.
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
Same repository secrets/variables as the pool expansion workflow:
|
|
||||||
|
|
||||||
| Kind | Name | Purpose |
|
|
||||||
| ------ | --------------------- | ---------------------------------------------- |
|
|
||||||
| Secret | `RUSTFS_ACCESS_KEY` | RustFS access key (default `rustfs@test`) |
|
|
||||||
| Secret | `RUSTFS_SECRET_KEY` | RustFS secret key (default `rustfs@test`) |
|
|
||||||
| Var | `RUSTFS_API_ENDPOINT` | Admin API endpoint, e.g. `http://127.0.0.1:9000` (`RUSTFS_RC_ENDPOINT` fallback) |
|
|
||||||
| Var | `RUSTFS_NODES` | `vm000 vm001 vm002` |
|
|
||||||
| Var | `RUSTFS_SSH_USER` | `azureuser` |
|
|
||||||
| Var | `RUSTFS_NIGHTLY_PACKAGE_URL` | Default nightly deb URL (defaults to the R2 `latest` alias) |
|
|
||||||
|
|
||||||
## Workflow inputs
|
|
||||||
|
|
||||||
| Input | Default | Meaning |
|
|
||||||
| ---------------- | ------- | ----------------------------------------- |
|
|
||||||
| `package_url` | nightly | Direct `.deb` URL; empty = latest nightly |
|
|
||||||
| `stop_node_gb` | `15` | Stop outage node at N GiB on survivors |
|
|
||||||
| `warp_stop_gb` | `40` | Stop warp at N GiB on survivors |
|
|
||||||
| `cleanup_before` | `true` | Reset nodes before the test |
|
|
||||||
| `cleanup_after` | `true` | Reset nodes after the test |
|
|
||||||
|
|
||||||
> ⚠️ `--reset` purges the `rustfs` package and deletes the data directories on
|
|
||||||
> all nodes. Only run against a dedicated test environment.
|
|
||||||
|
|
||||||
## Manual usage
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/test/rustfs_heal_test.sh --all -y \
|
|
||||||
--package-url https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb \
|
|
||||||
--endpoint http://127.0.0.1:9000
|
|
||||||
|
|
||||||
./scripts/test/rustfs_heal_test.sh --steps 5,6,7
|
|
||||||
./scripts/test/rustfs_heal_test.sh --reset -y
|
|
||||||
```
|
|
||||||
|
|
||||||
## Known issues
|
|
||||||
|
|
||||||
- Nightly builds gate pool/rebalance activation on a live fleet capability
|
|
||||||
proof (rustfs/backlog#2031); the script retries heal/rebalance starts and
|
|
||||||
prints a hint when the signature appears.
|
|
||||||
- The cluster-level `GET /rustfs/admin/v3/background-heal/status` aggregator
|
|
||||||
returns 501 in the single-pool 3x4 topology (no notification system), so the
|
|
||||||
script monitors the started heal task via its `clientToken` instead.
|
|
||||||
- The heal task may report `progress: null` while running; the script logs
|
|
||||||
this as evidence (rustfs/backlog#2035) rather than coercing it to zero, and
|
|
||||||
reads the canonical camelCase progress fields
|
|
||||||
(`objectsScanned`/`objectsHealed`/`objectsFailed`/`progressPercentage`) with
|
|
||||||
a snake_case fallback.
|
|
||||||
- The server-side per-task heal timeout defaults to 5 minutes; the script
|
|
||||||
writes `RUSTFS_HEAL_TASK_TIMEOUT_SECS=21600` (6h) into the node config so a
|
|
||||||
multi-tens-of-GiB heal can finish. The background scanner is disabled
|
|
||||||
(`RUSTFS_HEAL_AUTO_HEAL_ENABLE=false`) so the explicit heal is the only
|
|
||||||
repair mechanism and the outage effect stays observable.
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,543 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
#
|
|
||||||
# rustfs-performance-test.sh
|
|
||||||
# RustFS 4x4 集群性能压测全流程脚本
|
|
||||||
#
|
|
||||||
# Based on the Obsidian note "RustFS 性能测试". Full workflow:
|
|
||||||
# 1. Cleanup: stop & purge rustfs, remove data dirs on all nodes
|
|
||||||
# 2. Download the RustFS package on all nodes
|
|
||||||
# 3. Install RustFS on all nodes (dpkg -i), recreate volume dirs
|
|
||||||
# 4. Write /etc/default/rustfs (4-node x 4-drive MNMD), start all nodes
|
|
||||||
# in parallel and verify the service is Running
|
|
||||||
# 5. Run the benchmark (warp GET/PUT/MIXED via rustfs-performance-testing.sh)
|
|
||||||
# 6. Analyze results (summary.tsv / summary.md)
|
|
||||||
# 7. Final cleanup: stop & purge rustfs, remove data dirs
|
|
||||||
#
|
|
||||||
# The script is driven from an admin host (e.g. a jumpbox) and operates on
|
|
||||||
# the target nodes over SSH, mirroring scripts/test/rustfs_*_test.sh.
|
|
||||||
#
|
|
||||||
# Usage:
|
|
||||||
# ./rustfs-performance-test.sh --all # run all steps 1-7
|
|
||||||
# ./rustfs-performance-test.sh --step 5 # run a single step
|
|
||||||
# ./rustfs-performance-test.sh --steps 2,3,4 # run selected steps
|
|
||||||
# ./rustfs-performance-test.sh --all --dry-run # preview only
|
|
||||||
# ./rustfs-performance-test.sh --all -y --package-url <deb URL>
|
|
||||||
#
|
|
||||||
# Notes:
|
|
||||||
# - SSH user defaults to azureuser (passwordless sudo on the nodes);
|
|
||||||
# pass --ssh-user root if your nodes accept root login.
|
|
||||||
# - The benchmark runner defaults to
|
|
||||||
# ~/Documents/Obsidian Vault/rustfs-performance-testing.sh; override with
|
|
||||||
# --bench-script / RUSTFS_BENCH_SCRIPT. warp must be installed on the
|
|
||||||
# admin host.
|
|
||||||
# - Steps 1 and 7 destroy the RustFS install and all data (confirmed).
|
|
||||||
#
|
|
||||||
set -Eeuo pipefail
|
|
||||||
|
|
||||||
# ==================== Configuration (adjust to your environment) ====================
|
|
||||||
|
|
||||||
# Target nodes (4x4: 4 nodes x 4 drives each)
|
|
||||||
if [ -n "${RUSTFS_NODES:-}" ]; then
|
|
||||||
read -r -a NODES <<<"${RUSTFS_NODES}"
|
|
||||||
else
|
|
||||||
NODES=(vm000 vm001 vm002 vm003)
|
|
||||||
fi
|
|
||||||
|
|
||||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
|
||||||
SSH_PORT="${RUSTFS_SSH_PORT:-22}"
|
|
||||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new -p "${SSH_PORT}")
|
|
||||||
|
|
||||||
# Package: GitHub release tag, e.g. "1.0.0-rc.3". PACKAGE_URL is derived from
|
|
||||||
# RUSTFS_VERSION unless --package-url / RUSTFS_PACKAGE_URL is given.
|
|
||||||
RUSTFS_VERSION="${RUSTFS_VERSION:-1.0.0-rc.3}"
|
|
||||||
PACKAGE_URL="${RUSTFS_PACKAGE_URL:-}"
|
|
||||||
ARCH="${RUSTFS_ARCH:-amd64}"
|
|
||||||
PACKAGES_DIR="/home/rustfs/packages"
|
|
||||||
PACKAGE_FILE="rustfs.deb"
|
|
||||||
PACKAGE_SHA256="${RUSTFS_PACKAGE_SHA256:-}"
|
|
||||||
|
|
||||||
# 4x4 topology: 4 nodes x 4 drives each, same expression on every node
|
|
||||||
DRIVES_PER_NODE="${RUSTFS_DRIVES_PER_NODE:-4}"
|
|
||||||
VOLUMES="http://rustfs-node{1...4}:9000/data/rustfs{1...4}/mnmd"
|
|
||||||
|
|
||||||
# RustFS service configuration (written to /etc/default/rustfs)
|
|
||||||
RUSTFS_CONFIG_FILE="/etc/default/rustfs"
|
|
||||||
RUSTFS_SERVICE="rustfs"
|
|
||||||
RUSTFS_PACKAGE_NAME="rustfs"
|
|
||||||
RUSTFS_USER="rustfs"
|
|
||||||
ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfs@test}"
|
|
||||||
SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfs@test}"
|
|
||||||
RUSTFS_ADDRESS=":9000"
|
|
||||||
RUSTFS_CONSOLE_ADDRESS=":9001"
|
|
||||||
RUSTFS_CONSOLE_ENABLE=true
|
|
||||||
RUSTFS_OBS_LOGGER_LEVEL=error
|
|
||||||
RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
|
|
||||||
|
|
||||||
# Benchmark runner (step 5/6): prefer the default Obsidian location, fall back
|
|
||||||
# to a rustfs-performance-testing.sh next to this script (e.g. in the repo or
|
|
||||||
# on a jumpbox).
|
|
||||||
_DEFAULT_BENCH="${HOME}/Documents/Obsidian Vault/rustfs-performance-testing.sh"
|
|
||||||
_SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
||||||
if [ -x "${_DEFAULT_BENCH}" ]; then
|
|
||||||
_BENCH_RESOLVED="${_DEFAULT_BENCH}"
|
|
||||||
elif [ -x "${_SCRIPT_DIR}/rustfs-performance-testing.sh" ]; then
|
|
||||||
_BENCH_RESOLVED="${_SCRIPT_DIR}/rustfs-performance-testing.sh"
|
|
||||||
elif [ -x "${_SCRIPT_DIR}/rustfs_performance_testing.sh" ]; then
|
|
||||||
_BENCH_RESOLVED="${_SCRIPT_DIR}/rustfs_performance_testing.sh"
|
|
||||||
else
|
|
||||||
_BENCH_RESOLVED="${_DEFAULT_BENCH}"
|
|
||||||
fi
|
|
||||||
BENCH_SCRIPT="${RUSTFS_BENCH_SCRIPT:-${_BENCH_RESOLVED}}"
|
|
||||||
RESULT_DIR="${RUSTFS_RESULT_DIR:-$(pwd)/warp-bench-results-$(date +%Y%m%d-%H%M%S)}"
|
|
||||||
WARP_HOST="${RUSTFS_WARP_HOST:-rustfs-node1:9000,rustfs-node2:9000,rustfs-node3:9000,rustfs-node4:9000}"
|
|
||||||
WARP_BUCKET="${RUSTFS_WARP_BUCKET:-warp-benchmark-bucket}"
|
|
||||||
WARP_CONCURRENCY="${RUSTFS_WARP_CONCURRENCY:-64}"
|
|
||||||
WARP_DURATION="${RUSTFS_WARP_DURATION:-5m}"
|
|
||||||
WARP_GET_OBJECTS="${RUSTFS_WARP_GET_OBJECTS:-2500}"
|
|
||||||
WARP_SLEEP="${RUSTFS_WARP_SLEEP:-60}"
|
|
||||||
# Manual method/size selection (passed through to the benchmark runner; empty = full run)
|
|
||||||
WARP_METHODS="${RUSTFS_WARP_METHODS:-}"
|
|
||||||
WARP_SIZES="${RUSTFS_WARP_SIZES:-}"
|
|
||||||
|
|
||||||
# Timeouts (seconds)
|
|
||||||
SERVICE_TIMEOUT="${RUSTFS_SERVICE_TIMEOUT:-300}"
|
|
||||||
POLL_INTERVAL="${RUSTFS_POLL_INTERVAL:-10}"
|
|
||||||
|
|
||||||
# ==================== Runtime options (set by CLI) ====================
|
|
||||||
DRY_RUN=0
|
|
||||||
ASSUME_YES=0
|
|
||||||
SKIP_DOWNLOAD=0
|
|
||||||
PREFLIGHT=0
|
|
||||||
LOG_FILE=""
|
|
||||||
SELECTED_STEPS=()
|
|
||||||
|
|
||||||
# ==================== Helpers ====================
|
|
||||||
|
|
||||||
log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; }
|
|
||||||
warn() { printf '\033[1;33m[WARN]\033[0m %s\n' "$*"; }
|
|
||||||
die() { printf '\033[1;31m[ERROR]\033[0m %s\n' "$*" >&2; exit 1; }
|
|
||||||
|
|
||||||
confirm() {
|
|
||||||
if [ "${ASSUME_YES}" -eq 1 ] || [ "${DRY_RUN}" -eq 1 ]; then return 0; fi
|
|
||||||
printf '\033[1;33m[CONFIRM]\033[0m %s (y/N) ' "$1"
|
|
||||||
read -r answer
|
|
||||||
case "${answer}" in
|
|
||||||
y|Y|yes|YES) return 0 ;;
|
|
||||||
*) die "cancelled" ;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
need_cmd() {
|
|
||||||
[ "${DRY_RUN}" -eq 1 ] && return 0
|
|
||||||
command -v "$1" >/dev/null 2>&1 || die "missing command: $1 ($2); install it first"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Run a remote script on a single node (script is read from stdin)
|
|
||||||
run_remote() {
|
|
||||||
local node="$1" script
|
|
||||||
script="$(cat)"
|
|
||||||
if [ "${DRY_RUN}" -eq 1 ]; then
|
|
||||||
log "DRY-RUN: ssh ${SSH_USER}@${node} <<'REMOTE'"
|
|
||||||
printf '%s\n' "${script}" | sed 's/^/ | /'
|
|
||||||
log "DRY-RUN: ----"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
log "==> ${node}: executing remote script"
|
|
||||||
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'bash -s' <<<"${script}"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Run the same remote script on all nodes in parallel (script from stdin)
|
|
||||||
run_remote_all() {
|
|
||||||
local script pids=() i=0 fail=0
|
|
||||||
script="$(cat)"
|
|
||||||
for node in "${NODES[@]}"; do
|
|
||||||
if [ "${DRY_RUN}" -eq 1 ]; then
|
|
||||||
log "DRY-RUN: ssh ${SSH_USER}@${node} <<'REMOTE'"
|
|
||||||
printf '%s\n' "${script}" | sed 's/^/ | /'
|
|
||||||
log "DRY-RUN: ----"
|
|
||||||
else
|
|
||||||
log "==> ${node}: executing remote script"
|
|
||||||
( ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'bash -s' <<<"${script}" ) &
|
|
||||||
pids[$i]=$!
|
|
||||||
i=$((i+1))
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
if [ "${#pids[@]}" -gt 0 ]; then
|
|
||||||
for pid in "${pids[@]}"; do
|
|
||||||
wait "${pid}" || fail=1
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
[ "${fail}" -eq 0 ] || die "one or more remote executions failed"
|
|
||||||
}
|
|
||||||
|
|
||||||
rustfs_config_body() {
|
|
||||||
cat <<EOF
|
|
||||||
RUSTFS_ACCESS_KEY=${ACCESS_KEY}
|
|
||||||
RUSTFS_SECRET_KEY=${SECRET_KEY}
|
|
||||||
RUSTFS_VOLUMES="${VOLUMES}"
|
|
||||||
RUSTFS_ADDRESS="${RUSTFS_ADDRESS}"
|
|
||||||
RUSTFS_CONSOLE_ADDRESS="${RUSTFS_CONSOLE_ADDRESS}"
|
|
||||||
RUSTFS_CONSOLE_ENABLE=${RUSTFS_CONSOLE_ENABLE}
|
|
||||||
RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL}
|
|
||||||
RUSTFS_OBS_LOG_DIRECTORY="${RUSTFS_OBS_LOG_DIRECTORY}"
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
write_rustfs_config() {
|
|
||||||
local node="$1" body
|
|
||||||
body="$(rustfs_config_body)"
|
|
||||||
log "${node}: writing config ${RUSTFS_CONFIG_FILE}"
|
|
||||||
{
|
|
||||||
printf 'set -euo pipefail\n'
|
|
||||||
printf 'SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"\n'
|
|
||||||
printf '%s tee %s >/dev/null <<RUSTFS_EOF\n' '${SUDO}' "${RUSTFS_CONFIG_FILE}"
|
|
||||||
printf '%s' "${body}"
|
|
||||||
printf '\nRUSTFS_EOF\n'
|
|
||||||
printf '${SUDO} systemctl daemon-reload\n'
|
|
||||||
} | run_remote "${node}"
|
|
||||||
}
|
|
||||||
|
|
||||||
service_action() {
|
|
||||||
local action="$1" node="$2"
|
|
||||||
log "${node}: systemctl ${action} ${RUSTFS_SERVICE}"
|
|
||||||
[ "${DRY_RUN}" -eq 1 ] && return 0
|
|
||||||
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \
|
|
||||||
"if [ \"\$(id -u)\" -ne 0 ]; then sudo -n systemctl ${action} ${RUSTFS_SERVICE}; else systemctl ${action} ${RUSTFS_SERVICE}; fi" \
|
|
||||||
|| die "${node}: systemctl ${action} failed"
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_service_active() {
|
|
||||||
local node="$1" elapsed=0
|
|
||||||
log "${node}: waiting for ${RUSTFS_SERVICE} to become active"
|
|
||||||
[ "${DRY_RUN}" -eq 1 ] && { log "${node}: (dry-run) skip wait"; return 0; }
|
|
||||||
while [ "${elapsed}" -lt "${SERVICE_TIMEOUT}" ]; do
|
|
||||||
if ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \
|
|
||||||
"systemctl is-active ${RUSTFS_SERVICE} 2>/dev/null" | grep -q active; then
|
|
||||||
log "${node}: service active"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
sleep "${POLL_INTERVAL}"
|
|
||||||
elapsed=$((elapsed + POLL_INTERVAL))
|
|
||||||
done
|
|
||||||
die "${node}: ${RUSTFS_SERVICE} did not become active within ${SERVICE_TIMEOUT}s"
|
|
||||||
}
|
|
||||||
|
|
||||||
verify_service_running() {
|
|
||||||
local node="$1"
|
|
||||||
log "${node}: checking service status"
|
|
||||||
if [ "${DRY_RUN}" -eq 0 ]; then
|
|
||||||
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \
|
|
||||||
"systemctl status ${RUSTFS_SERVICE} --no-pager | head -n 12" || true
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
build_package_url() {
|
|
||||||
local asset
|
|
||||||
asset="rustfs_$(printf '%s' "${RUSTFS_VERSION}" | tr '-' '.')_${ARCH}.deb"
|
|
||||||
printf 'https://github.com/rustfs/rustfs/releases/download/%s/%s' "${RUSTFS_VERSION}" "${asset}"
|
|
||||||
}
|
|
||||||
|
|
||||||
resolve_package_url() {
|
|
||||||
if [ -n "${PACKAGE_URL}" ]; then printf '%s' "${PACKAGE_URL}"; else build_package_url; fi
|
|
||||||
}
|
|
||||||
|
|
||||||
preflight() {
|
|
||||||
log "preflight checks"
|
|
||||||
need_cmd ssh "openssh client"
|
|
||||||
need_cmd curl "http client"
|
|
||||||
need_cmd warp "warp benchmark tool (for step 5)"
|
|
||||||
if [ ! -x "${BENCH_SCRIPT}" ]; then
|
|
||||||
die "benchmark script not found or not executable: ${BENCH_SCRIPT}"
|
|
||||||
fi
|
|
||||||
if [ "${DRY_RUN}" -eq 0 ]; then
|
|
||||||
for node in "${NODES[@]}"; do
|
|
||||||
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" 'echo ok' >/dev/null \
|
|
||||||
|| die "cannot ssh to ${node}"
|
|
||||||
done
|
|
||||||
log "all nodes reachable: ${NODES[*]}"
|
|
||||||
fi
|
|
||||||
log "preflight OK"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==================== Steps ====================
|
|
||||||
|
|
||||||
step1_cleanup() {
|
|
||||||
log "step 1: cleanup environment on all nodes (stop & purge rustfs, remove data dirs)"
|
|
||||||
confirm "This DESTROYS the RustFS install and ALL data on ${NODES[*]} (irreversible). Continue?"
|
|
||||||
local script
|
|
||||||
script="$(cat <<EOF
|
|
||||||
set -euo pipefail
|
|
||||||
SUDO=""; [ "\$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
|
||||||
\${SUDO} systemctl stop rustfs 2>/dev/null || true
|
|
||||||
if \${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
|
||||||
\${SUDO} dpkg -P rustfs
|
|
||||||
echo "purged rustfs"
|
|
||||||
else
|
|
||||||
echo "rustfs not installed, skip purge"
|
|
||||||
fi
|
|
||||||
for i in \$(seq 1 ${DRIVES_PER_NODE}); do
|
|
||||||
\${SUDO} rm -rf /data/rustfs\${i}/mnmd
|
|
||||||
\${SUDO} mkdir -p /data/rustfs\${i}/mnmd
|
|
||||||
\${SUDO} chown -R rustfs:rustfs /data/rustfs\${i}/mnmd
|
|
||||||
done
|
|
||||||
echo "cleanup done on \$(hostname)"
|
|
||||||
EOF
|
|
||||||
)"
|
|
||||||
printf '%s\n' "${script}" | run_remote_all
|
|
||||||
log "step 1 complete"
|
|
||||||
}
|
|
||||||
|
|
||||||
step2_download() {
|
|
||||||
log "step 2: download the package on all nodes"
|
|
||||||
local url script
|
|
||||||
url="$(resolve_package_url)"
|
|
||||||
script="$(cat <<EOF
|
|
||||||
set -euo pipefail
|
|
||||||
SUDO=""; [ "\$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
|
||||||
if [ -f "${PACKAGES_DIR}/${PACKAGE_FILE}" ] && [ "${SKIP_DOWNLOAD}" -eq 1 ]; then
|
|
||||||
echo "already exists: ${PACKAGES_DIR}/${PACKAGE_FILE}, skipping download"
|
|
||||||
else
|
|
||||||
echo "downloading ${url} ..."
|
|
||||||
curl -fSL --retry 3 -o "/tmp/${PACKAGE_FILE}" "${url}"
|
|
||||||
\${SUDO} mkdir -p "${PACKAGES_DIR}"
|
|
||||||
\${SUDO} install -m 0644 "/tmp/${PACKAGE_FILE}" "${PACKAGES_DIR}/${PACKAGE_FILE}"
|
|
||||||
\${SUDO} rm -f "/tmp/${PACKAGE_FILE}"
|
|
||||||
fi
|
|
||||||
if [ -n "${PACKAGE_SHA256}" ]; then
|
|
||||||
echo "${PACKAGE_SHA256} ${PACKAGES_DIR}/${PACKAGE_FILE}" | sha256sum -c - || { echo "checksum verification failed"; exit 1; }
|
|
||||||
fi
|
|
||||||
ls -lh "${PACKAGES_DIR}/${PACKAGE_FILE}"
|
|
||||||
EOF
|
|
||||||
)"
|
|
||||||
printf '%s\n' "${script}" | run_remote_all
|
|
||||||
log "step 2 complete"
|
|
||||||
}
|
|
||||||
|
|
||||||
step3_install() {
|
|
||||||
log "step 3: install the RustFS service on all nodes"
|
|
||||||
confirm "About to run dpkg -i ${PACKAGE_FILE} on all nodes. Continue?"
|
|
||||||
local script
|
|
||||||
script="$(cat <<'EOF'
|
|
||||||
set -euo pipefail
|
|
||||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
|
||||||
${SUDO} dpkg -i /home/rustfs/packages/rustfs.deb
|
|
||||||
${SUDO} systemctl daemon-reload
|
|
||||||
echo "--- installed package ---"
|
|
||||||
dpkg -l rustfs | tail -n 1
|
|
||||||
EOF
|
|
||||||
)"
|
|
||||||
printf '%s\n' "${script}" | run_remote_all
|
|
||||||
log "step 3 complete"
|
|
||||||
}
|
|
||||||
|
|
||||||
step4_configure_start() {
|
|
||||||
log "step 4: write config, start and verify the service on all nodes"
|
|
||||||
local node
|
|
||||||
for node in "${NODES[@]}"; do
|
|
||||||
write_rustfs_config "${node}"
|
|
||||||
done
|
|
||||||
for node in "${NODES[@]}"; do
|
|
||||||
service_action start "${node}" &
|
|
||||||
done
|
|
||||||
wait
|
|
||||||
for node in "${NODES[@]}"; do
|
|
||||||
wait_service_active "${node}"
|
|
||||||
verify_service_running "${node}"
|
|
||||||
done
|
|
||||||
log "step 4 complete"
|
|
||||||
}
|
|
||||||
|
|
||||||
step5_benchmark() {
|
|
||||||
log "step 5: run the benchmark (${BENCH_SCRIPT})"
|
|
||||||
need_cmd warp "warp benchmark tool"
|
|
||||||
confirm "About to run the full GET/PUT/MIXED benchmark (~6-10 hours). Continue?"
|
|
||||||
if [ "${DRY_RUN}" -eq 1 ]; then
|
|
||||||
log "DRY-RUN: WARP_HOST=${WARP_HOST} WARP_RESULT_DIR=${RESULT_DIR} bash ${BENCH_SCRIPT}"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
WARP_HOST="${WARP_HOST}" \
|
|
||||||
WARP_ACCESS_KEY="${ACCESS_KEY}" \
|
|
||||||
WARP_SECRET_KEY="${SECRET_KEY}" \
|
|
||||||
WARP_BUCKET="${WARP_BUCKET}" \
|
|
||||||
WARP_CONCURRENCY="${WARP_CONCURRENCY}" \
|
|
||||||
WARP_DURATION="${WARP_DURATION}" \
|
|
||||||
WARP_GET_OBJECTS="${WARP_GET_OBJECTS}" \
|
|
||||||
WARP_SLEEP_BETWEEN_ROUNDS="${WARP_SLEEP}" \
|
|
||||||
WARP_METHODS="${WARP_METHODS}" \
|
|
||||||
WARP_SIZES="${WARP_SIZES}" \
|
|
||||||
WARP_RESULT_DIR="${RESULT_DIR}" \
|
|
||||||
bash "${BENCH_SCRIPT}"
|
|
||||||
log "step 5 complete (results in ${RESULT_DIR})"
|
|
||||||
}
|
|
||||||
|
|
||||||
step6_analyze() {
|
|
||||||
log "step 6: analyze results from ${RESULT_DIR}"
|
|
||||||
if [ "${DRY_RUN}" -eq 1 ]; then
|
|
||||||
log "DRY-RUN: bash ${BENCH_SCRIPT} --parse-only ${RESULT_DIR}"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
if [ ! -d "${RESULT_DIR}" ]; then
|
|
||||||
die "result directory not found: ${RESULT_DIR}"
|
|
||||||
fi
|
|
||||||
if [ -f "${RESULT_DIR}/summary.md" ]; then
|
|
||||||
log "summary already generated: ${RESULT_DIR}/summary.md"
|
|
||||||
else
|
|
||||||
log "generating summary with --parse-only"
|
|
||||||
bash "${BENCH_SCRIPT}" --parse-only "${RESULT_DIR}"
|
|
||||||
fi
|
|
||||||
log "----- summary.md -----"
|
|
||||||
cat "${RESULT_DIR}/summary.md"
|
|
||||||
log "step 6 complete"
|
|
||||||
}
|
|
||||||
|
|
||||||
step7_cleanup() {
|
|
||||||
log "step 7: final cleanup on all nodes (stop & purge rustfs, remove data dirs)"
|
|
||||||
confirm "This DESTROYS the RustFS install and ALL data on ${NODES[*]} (irreversible). Continue?"
|
|
||||||
local script
|
|
||||||
script="$(cat <<EOF
|
|
||||||
set -euo pipefail
|
|
||||||
SUDO=""; [ "\$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
|
||||||
\${SUDO} systemctl stop rustfs 2>/dev/null || true
|
|
||||||
if \${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
|
||||||
\${SUDO} dpkg -P rustfs
|
|
||||||
echo "purged rustfs"
|
|
||||||
fi
|
|
||||||
for i in \$(seq 1 ${DRIVES_PER_NODE}); do
|
|
||||||
\${SUDO} rm -rf /data/rustfs\${i}/mnmd
|
|
||||||
done
|
|
||||||
echo "cleanup done on \$(hostname)"
|
|
||||||
EOF
|
|
||||||
)"
|
|
||||||
printf '%s\n' "${script}" | run_remote_all
|
|
||||||
log "step 7 complete"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==================== CLI ====================
|
|
||||||
|
|
||||||
usage() {
|
|
||||||
cat <<'USAGE'
|
|
||||||
Usage: ./rustfs-performance-test.sh [options]
|
|
||||||
|
|
||||||
Steps:
|
|
||||||
1 cleanup environment (purge rustfs, remove data dirs) [destructive]
|
|
||||||
2 download the RustFS package on all nodes
|
|
||||||
3 install RustFS (dpkg -i)
|
|
||||||
4 write config, start service, verify Running
|
|
||||||
5 run benchmark (warp GET/PUT/MIXED)
|
|
||||||
6 analyze results (summary.tsv / summary.md)
|
|
||||||
7 final cleanup (purge rustfs, remove data dirs) [destructive]
|
|
||||||
|
|
||||||
Options:
|
|
||||||
--all Run all steps 1-7
|
|
||||||
--step N Run a single step
|
|
||||||
--steps 1,3,5-7 Run selected steps
|
|
||||||
--version VERSION GitHub release tag (default 1.0.0-rc.3)
|
|
||||||
--package-url URL Direct deb URL (overrides --version)
|
|
||||||
--sha256 HASH Verify package checksum
|
|
||||||
--skip-download Keep an existing package file
|
|
||||||
--bench-script PATH Benchmark runner (default: Obsidian Vault rustfs-performance-testing.sh)
|
|
||||||
--result-dir DIR Benchmark result directory
|
|
||||||
--warp-duration DUR warp duration per round (default 5m)
|
|
||||||
--warp-concurrency N warp concurrency (default 64)
|
|
||||||
--ssh-user USER SSH user (default azureuser)
|
|
||||||
--ssh-port PORT SSH port (default 22)
|
|
||||||
--preflight Check environment and exit
|
|
||||||
--log-file FILE Append all output to FILE
|
|
||||||
--dry-run Preview commands without executing them
|
|
||||||
-y, --yes Skip all confirmation prompts
|
|
||||||
-h, --help Show this help
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
./rustfs-performance-test.sh --all
|
|
||||||
./rustfs-performance-test.sh --all --dry-run
|
|
||||||
./rustfs-performance-test.sh --all -y --package-url https://dl.rustfs.com/...deb
|
|
||||||
./rustfs-performance-test.sh --step 5
|
|
||||||
USAGE
|
|
||||||
}
|
|
||||||
|
|
||||||
expand_steps() {
|
|
||||||
local spec="$1" part start end i
|
|
||||||
IFS=',' read -ra parts <<<"${spec}"
|
|
||||||
for part in "${parts[@]}"; do
|
|
||||||
if [[ "${part}" =~ ^([0-9]+)-([0-9]+)$ ]]; then
|
|
||||||
start="${BASH_REMATCH[1]}"; end="${BASH_REMATCH[2]}"
|
|
||||||
for ((i=start; i<=end; i++)); do SELECTED_STEPS+=("${i}"); done
|
|
||||||
elif [[ "${part}" =~ ^[0-9]+$ ]]; then
|
|
||||||
SELECTED_STEPS+=("${part}")
|
|
||||||
else
|
|
||||||
die "cannot parse step spec: ${part}"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
run_steps() {
|
|
||||||
local step
|
|
||||||
for step in "${SELECTED_STEPS[@]}"; do
|
|
||||||
case "${step}" in
|
|
||||||
1) step1_cleanup ;;
|
|
||||||
2) step2_download ;;
|
|
||||||
3) step3_install ;;
|
|
||||||
4) step4_configure_start ;;
|
|
||||||
5) step5_benchmark ;;
|
|
||||||
6) step6_analyze ;;
|
|
||||||
7) step7_cleanup ;;
|
|
||||||
*) die "unknown step: ${step}" ;;
|
|
||||||
esac
|
|
||||||
log "step ${step} completed"
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
[ "$#" -eq 0 ] && { usage; exit 0; }
|
|
||||||
local opt all=0
|
|
||||||
while [ "$#" -gt 0 ]; do
|
|
||||||
opt="$1"; shift
|
|
||||||
case "${opt}" in
|
|
||||||
--all) all=1 ;;
|
|
||||||
--step) SELECTED_STEPS+=("$1"); shift ;;
|
|
||||||
--steps) expand_steps "$1"; shift ;;
|
|
||||||
--version) RUSTFS_VERSION="$1"; shift ;;
|
|
||||||
--package-url) PACKAGE_URL="$1"; shift ;;
|
|
||||||
--sha256) PACKAGE_SHA256="$1"; shift ;;
|
|
||||||
--skip-download) SKIP_DOWNLOAD=1 ;;
|
|
||||||
--bench-script) BENCH_SCRIPT="$1"; shift ;;
|
|
||||||
--result-dir) RESULT_DIR="$1"; shift ;;
|
|
||||||
--warp-duration) WARP_DURATION="$1"; shift ;;
|
|
||||||
--warp-concurrency) WARP_CONCURRENCY="$1"; shift ;;
|
|
||||||
--ssh-user) SSH_USER="$1"; shift ;;
|
|
||||||
--ssh-port) SSH_PORT="$1"; shift ;;
|
|
||||||
--preflight) PREFLIGHT=1 ;;
|
|
||||||
--log-file) LOG_FILE="$1"; shift ;;
|
|
||||||
--dry-run) DRY_RUN=1 ;;
|
|
||||||
-y|--yes) ASSUME_YES=1 ;;
|
|
||||||
-h|--help) usage; exit 0 ;;
|
|
||||||
*) die "unknown option: ${opt} (see --help)" ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
if [ -n "${LOG_FILE}" ]; then
|
|
||||||
mkdir -p "$(dirname "${LOG_FILE}")"
|
|
||||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
|
||||||
fi
|
|
||||||
if [ "${all}" -eq 1 ]; then
|
|
||||||
SELECTED_STEPS=(1 2 3 4 5 6 7)
|
|
||||||
fi
|
|
||||||
if [ "${PREFLIGHT}" -eq 1 ]; then
|
|
||||||
preflight
|
|
||||||
if [ "${#SELECTED_STEPS[@]}" -eq 0 ]; then
|
|
||||||
log "preflight only; done"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
[ "${#SELECTED_STEPS[@]}" -gt 0 ] || die "no steps selected (--all / --step / --steps)"
|
|
||||||
log "nodes: ${NODES[*]} ssh user: ${SSH_USER} version: ${RUSTFS_VERSION}"
|
|
||||||
log "package: $(resolve_package_url)"
|
|
||||||
log "result dir: ${RESULT_DIR}"
|
|
||||||
[ "${DRY_RUN}" -eq 1 ] && warn "DRY-RUN mode: only printing the commands that would run"
|
|
||||||
run_steps
|
|
||||||
log "all done"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Allow sourcing the file for unit tests without running main.
|
|
||||||
if [ "${RUSTFS_PERF_SCRIPT_SOURCE_ONLY:-0}" != "1" ]; then
|
|
||||||
main "$@"
|
|
||||||
fi
|
|
||||||
@@ -1,239 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
#
|
|
||||||
# rustfs-performance-testing.sh
|
|
||||||
# RustFS 对象存储压测脚本（固定版）：测试方法 + 执行 + 结果解析
|
|
||||||
#
|
|
||||||
# 测试方法
|
|
||||||
# 1) 方法：GET / PUT / MIXED（warp 默认混合负载 45% GET + 55% PUT）
|
|
||||||
# 2) 对象尺寸：1KiB 4KiB 16KiB 128KiB 1MiB 4MiB 8MiB 16MiB 32MiB 64MiB
|
|
||||||
# 3) 并发：64；单轮时长：5m；轮间 sleep：60s；GET 对象数：2500
|
|
||||||
# 4) 顺序：GET 全部尺寸 -> PUT 全部尺寸 -> MIXED 全部尺寸
|
|
||||||
# 5) 结果解析：每轮结束后自动解析 warp 输出，写入
|
|
||||||
# summary.tsv（机器可读）与 summary.md（Markdown 汇总表）
|
|
||||||
#
|
|
||||||
# 依赖：warp >= v1.6（MinIO warp），bash，awk/sed/grep
|
|
||||||
# 说明：warp v1.6.1 的 put 不支持 --objects，脚本已自动处理（仅 get/mixed 传该参数）
|
|
||||||
#
|
|
||||||
# 环境变量覆盖（不传时使用固定默认值）：
|
|
||||||
# WARP_HOST WARP_ACCESS_KEY WARP_SECRET_KEY WARP_BUCKET
|
|
||||||
# WARP_CONCURRENCY WARP_DURATION WARP_GET_OBJECTS WARP_SLEEP_BETWEEN_ROUNDS
|
|
||||||
# WARP_RESULT_DIR
|
|
||||||
# WARP_METHODS WARP_SIZES # 手动指定方法/尺寸（逗号或空格分隔），不传则全量
|
|
||||||
|
|
||||||
set -u -o pipefail
|
|
||||||
|
|
||||||
HOST="${WARP_HOST:-rustfs-node1:9000,rustfs-node2:9000,rustfs-node3:9000,rustfs-node4:9000}"
|
|
||||||
ACCESS_KEY="${WARP_ACCESS_KEY:-rustfs@test}"
|
|
||||||
SECRET_KEY="${WARP_SECRET_KEY:-rustfs@test}"
|
|
||||||
BUCKET="${WARP_BUCKET:-warp-benchmark-bucket}"
|
|
||||||
CONCURRENCY="${WARP_CONCURRENCY:-64}"
|
|
||||||
DURATION="${WARP_DURATION:-5m}"
|
|
||||||
GET_OBJECTS="${WARP_GET_OBJECTS:-2500}"
|
|
||||||
SLEEP_BETWEEN_ROUNDS="${WARP_SLEEP_BETWEEN_ROUNDS:-60}"
|
|
||||||
RESULT_DIR="${WARP_RESULT_DIR:-$(pwd)/warp-bench-results-$(date +%Y%m%d-%H%M%S)}"
|
|
||||||
|
|
||||||
if [ -n "${WARP_SIZES:-}" ] && [ "${WARP_SIZES}" != "all" ] && [ "${WARP_SIZES}" != "ALL" ]; then
|
|
||||||
read -r -a SIZES <<<"${WARP_SIZES//,/ }"
|
|
||||||
else
|
|
||||||
SIZES=(1KiB 4KiB 16KiB 128KiB 1MiB 4MiB 8MiB 16MiB 32MiB 64MiB)
|
|
||||||
fi
|
|
||||||
if [ -n "${WARP_METHODS:-}" ] && [ "${WARP_METHODS}" != "all" ] && [ "${WARP_METHODS}" != "ALL" ]; then
|
|
||||||
read -r -a METHODS <<<"${WARP_METHODS//,/ }"
|
|
||||||
else
|
|
||||||
METHODS=(get put mixed)
|
|
||||||
fi
|
|
||||||
TOTAL_ROUNDS=$(( ${#METHODS[@]} * ${#SIZES[@]} ))
|
|
||||||
ROUND=0
|
|
||||||
|
|
||||||
# --parse-only <result-dir>：只解析已有结果目录（${method}_${size}.txt），不执行压测
|
|
||||||
if [[ "${1:-}" == "--parse-only" && -n "${2:-}" ]]; then
|
|
||||||
RESULT_DIR="$2"
|
|
||||||
fi
|
|
||||||
|
|
||||||
LOG_FILE="${RESULT_DIR}/master.log"
|
|
||||||
SUMMARY_TSV="${RESULT_DIR}/summary.tsv"
|
|
||||||
SUMMARY_MD="${RESULT_DIR}/summary.md"
|
|
||||||
|
|
||||||
if [[ "${1:-}" != "--parse-only" ]] && ! command -v warp >/dev/null 2>&1; then
|
|
||||||
echo "错误：未找到 warp 命令，请先安装 MinIO warp。" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
log() {
|
|
||||||
echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') $*" | tee -a "${LOG_FILE}"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ---- 结果解析 ----
|
|
||||||
|
|
||||||
# 提取指定 section（GET/PUT/Total 等）的 Average / Reqs / TTFB 原始行
|
|
||||||
section_lines() {
|
|
||||||
awk -v sec="$2" '
|
|
||||||
/^Report: / { cur = $2; sub(/\.$/, "", cur) }
|
|
||||||
cur == sec && /^ *\* Average:/ { avg = $0 }
|
|
||||||
cur == sec && /^ *\* Reqs:/ { reqs = $0 }
|
|
||||||
cur == sec && /^ *\* TTFB:/ { ttfb = $0 }
|
|
||||||
END {
|
|
||||||
if (avg != "") print avg
|
|
||||||
if (reqs != "") print reqs
|
|
||||||
if (ttfb != "") print ttfb
|
|
||||||
}
|
|
||||||
' "$1"
|
|
||||||
}
|
|
||||||
|
|
||||||
# 从统计行中取字段：tp objs avg p50 p90 p99 ttfb_avg ttfb_p99 ttfb_worst
|
|
||||||
field() {
|
|
||||||
case "$2" in
|
|
||||||
tp) echo "$1" | sed -n 's/^ *\* Average: \(.*\), \([0-9.]*\) obj\/s.*/\1/p' ;;
|
|
||||||
objs) echo "$1" | sed -n 's/^ *\* Average: .*, \([0-9.]*\) obj\/s.*/\1/p' ;;
|
|
||||||
avg) echo "$1" | sed -n 's/^ *\* Reqs: Avg: \([^,]*\),.*/\1/p' ;;
|
|
||||||
p50) echo "$1" | sed -n 's/^ *\* Reqs: Avg: [^,]*, 50%: \([^,]*\),.*/\1/p' ;;
|
|
||||||
p90) echo "$1" | sed -n 's/^ *\* Reqs: Avg: [^,]*, 50%: [^,]*, 90%: \([^,]*\),.*/\1/p' ;;
|
|
||||||
p99) echo "$1" | sed -n 's/^ *\* Reqs: Avg: [^,]*, 50%: [^,]*, 90%: [^,]*, 99%: \([^,]*\),.*/\1/p' ;;
|
|
||||||
ttfb_avg) echo "$1" | sed -n 's/^ *\* TTFB: Avg: \([^,]*\),.*/\1/p' ;;
|
|
||||||
ttfb_p99) echo "$1" | sed -n 's/^ *\* TTFB: .*99th: \([^,]*\),.*/\1/p' ;;
|
|
||||||
ttfb_worst) echo "$1" | sed -n 's/^ *\* TTFB: .*Worst: \([^ ]*\).*/\1/p' ;;
|
|
||||||
*) echo "" ;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
# 解析一轮输出，追加一行到 summary.tsv
|
|
||||||
parse_round() {
|
|
||||||
local method="$1" size="$2" file="$3"
|
|
||||||
local line
|
|
||||||
if [[ "${method}" == "mixed" ]]; then
|
|
||||||
local total get put
|
|
||||||
total=$(section_lines "$file" Total)
|
|
||||||
get=$(section_lines "$file" GET)
|
|
||||||
put=$(section_lines "$file" PUT)
|
|
||||||
line=$(printf 'mixed\t%s\t%s\t%s\t%s\t%s' \
|
|
||||||
"$size" \
|
|
||||||
"$(field "${total}" tp)" \
|
|
||||||
"$(field "${total}" objs)" \
|
|
||||||
"$(field "${get}" avg)" \
|
|
||||||
"$(field "${put}" avg)")
|
|
||||||
else
|
|
||||||
local sec
|
|
||||||
sec=$(printf '%s' "${method}" | tr '[:lower:]' '[:upper:]')
|
|
||||||
local stats
|
|
||||||
stats=$(section_lines "$file" "${sec}")
|
|
||||||
line=$(printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s' \
|
|
||||||
"${method}" "${size}" \
|
|
||||||
"$(field "${stats}" tp)" \
|
|
||||||
"$(field "${stats}" objs)" \
|
|
||||||
"$(field "${stats}" avg)" \
|
|
||||||
"$(field "${stats}" p50)" \
|
|
||||||
"$(field "${stats}" p90)" \
|
|
||||||
"$(field "${stats}" p99)" \
|
|
||||||
"$(field "${stats}" ttfb_avg)" \
|
|
||||||
"$(field "${stats}" ttfb_p99)" \
|
|
||||||
"$(field "${stats}" ttfb_worst)")
|
|
||||||
fi
|
|
||||||
printf '%s\n' "${line}" >> "${SUMMARY_TSV}"
|
|
||||||
}
|
|
||||||
|
|
||||||
# 汇总 summary.tsv -> summary.md（Markdown 表格）
|
|
||||||
gen_summary_md() {
|
|
||||||
{
|
|
||||||
echo "# RustFS 性能压测结果"
|
|
||||||
echo ""
|
|
||||||
echo "- 日期：$(date -u '+%Y-%m-%d %H:%M:%S UTC')"
|
|
||||||
echo "- 目标：${HOST}"
|
|
||||||
echo "- 并发：${CONCURRENCY}；单轮：${DURATION}；sleep：${SLEEP_BETWEEN_ROUNDS}s；GET objects：${GET_OBJECTS}"
|
|
||||||
echo "- 方法：GET / PUT / MIXED（warp 默认混合负载）；尺寸：${SIZES[*]}"
|
|
||||||
echo ""
|
|
||||||
} > "${SUMMARY_MD}"
|
|
||||||
|
|
||||||
for m in get put; do
|
|
||||||
{
|
|
||||||
echo "## $(printf '%s' "$m" | tr '[:lower:]' '[:upper:]') 结果"
|
|
||||||
echo ""
|
|
||||||
echo "| 对象尺寸 | 平均吞吐 | 平均 obj/s | Avg Latency | P50 | P90 | P99 | TTFB Avg | TTFB P99 | TTFB 最差 |"
|
|
||||||
echo "|----------|----------|-----------|-------------|-----|-----|-----|----------|----------|-----------|"
|
|
||||||
} >> "${SUMMARY_MD}"
|
|
||||||
while IFS=$'\t' read -r method size tp objs avg p50 p90 p99 ttfb_avg ttfb_p99 ttfb_worst; do
|
|
||||||
[[ "${method}" == "${m}" ]] && \
|
|
||||||
echo "| ${size} | ${tp} | ${objs} | ${avg} | ${p50} | ${p90} | ${p99} | ${ttfb_avg} | ${ttfb_p99} | ${ttfb_worst} |" >> "${SUMMARY_MD}"
|
|
||||||
done < "${SUMMARY_TSV}"
|
|
||||||
echo "" >> "${SUMMARY_MD}"
|
|
||||||
done
|
|
||||||
|
|
||||||
{
|
|
||||||
echo "## MIXED 结果（Total 口径）"
|
|
||||||
echo ""
|
|
||||||
echo "| 对象尺寸 | Total 平均吞吐 | Total 平均 obj/s | Mixed-GET Avg | Mixed-PUT Avg |"
|
|
||||||
echo "|----------|---------------|------------------|----------------|----------------|"
|
|
||||||
} >> "${SUMMARY_MD}"
|
|
||||||
while IFS=$'\t' read -r method size tp objs gavg pavg rest; do
|
|
||||||
[[ "${method}" == "mixed" ]] && \
|
|
||||||
echo "| ${size} | ${tp} | ${objs} | ${gavg} | ${pavg} |" >> "${SUMMARY_MD}"
|
|
||||||
done < "${SUMMARY_TSV}"
|
|
||||||
echo "" >> "${SUMMARY_MD}"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ---- 主流程 ----
|
|
||||||
|
|
||||||
mkdir -p "${RESULT_DIR}"
|
|
||||||
log "CONFIG host=${HOST} bucket=${BUCKET} concurrency=${CONCURRENCY} duration=${DURATION} get_objects=${GET_OBJECTS} sleep_between_rounds=${SLEEP_BETWEEN_ROUNDS}s"
|
|
||||||
printf 'method\tsize\tthroughput\tobj_per_s\treq_avg\treq_p50\treq_p90\treq_p99\tttfb_avg\tttfb_p99\tttfb_worst\n' > "${SUMMARY_TSV}"
|
|
||||||
|
|
||||||
if [[ "${1:-}" == "--parse-only" ]]; then
|
|
||||||
for method in "${METHODS[@]}"; do
|
|
||||||
for size in "${SIZES[@]}"; do
|
|
||||||
outfile="${RESULT_DIR}/${method}_${size}.txt"
|
|
||||||
if [[ -s "${outfile}" ]]; then
|
|
||||||
parse_round "${method}" "${size}" "${outfile}"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
done
|
|
||||||
gen_summary_md
|
|
||||||
echo "parsed from ${RESULT_DIR}"
|
|
||||||
echo ""
|
|
||||||
cat "${SUMMARY_MD}"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
for method in "${METHODS[@]}"; do
|
|
||||||
for size in "${SIZES[@]}"; do
|
|
||||||
ROUND=$((ROUND + 1))
|
|
||||||
outfile="${RESULT_DIR}/${method}_${size}.txt"
|
|
||||||
|
|
||||||
log "START round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} concurrency=${CONCURRENCY} duration=${DURATION}"
|
|
||||||
|
|
||||||
extra_args=()
|
|
||||||
if [[ "${method}" != "put" ]]; then
|
|
||||||
extra_args=(--objects "${GET_OBJECTS}")
|
|
||||||
fi
|
|
||||||
|
|
||||||
start_epoch=$(date +%s)
|
|
||||||
warp "${method}" \
|
|
||||||
--host "${HOST}" \
|
|
||||||
--access-key "${ACCESS_KEY}" \
|
|
||||||
--secret-key "${SECRET_KEY}" \
|
|
||||||
--bucket "${BUCKET}" \
|
|
||||||
--concurrent "${CONCURRENCY}" \
|
|
||||||
--duration "${DURATION}" \
|
|
||||||
--obj.size "${size}" \
|
|
||||||
"${extra_args[@]}" \
|
|
||||||
--no-color 2>&1 | tee "${outfile}"
|
|
||||||
rc=${PIPESTATUS[0]}
|
|
||||||
end_epoch=$(date +%s)
|
|
||||||
|
|
||||||
if [[ ${rc} -eq 0 ]]; then
|
|
||||||
parse_round "${method}" "${size}" "${outfile}"
|
|
||||||
log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=ok"
|
|
||||||
else
|
|
||||||
log "END round=${ROUND}/${TOTAL_ROUNDS} method=${method} size=${size} rc=${rc} elapsed=$((end_epoch - start_epoch))s parsed=skipped"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ ${ROUND} -lt ${TOTAL_ROUNDS} ]]; then
|
|
||||||
log "SLEEP ${SLEEP_BETWEEN_ROUNDS}s before next round"
|
|
||||||
sleep "${SLEEP_BETWEEN_ROUNDS}"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
done
|
|
||||||
|
|
||||||
gen_summary_md
|
|
||||||
log "ALL_ROUNDS_COMPLETE summary_tsv=${SUMMARY_TSV} summary_md=${SUMMARY_MD}"
|
|
||||||
echo ""
|
|
||||||
echo "==== 结果汇总 ===="
|
|
||||||
cat "${SUMMARY_MD}"
|
|
||||||
@@ -1,106 +0,0 @@
|
|||||||
# RustFS Pool Expansion / Decommission Test
|
|
||||||
|
|
||||||
End-to-end test for **storage pool expansion**, **data rebalancing** and
|
|
||||||
**pool decommission**, driven by
|
|
||||||
[`scripts/test/rustfs_pool_expand.sh`](rustfs_pool_expand.sh).
|
|
||||||
|
|
||||||
All pool / rebalance / decommission checks talk to the RustFS **admin API
|
|
||||||
directly** with SigV4-signed HTTP requests and assert on the JSON responses
|
|
||||||
(`jq`), so the results are exact and independent of any client CLI output
|
|
||||||
formatting. **`rc` is not required.**
|
|
||||||
|
|
||||||
## What it does
|
|
||||||
|
|
||||||
1. Downloads the RustFS `.deb` package on all nodes (a release tag, or a
|
|
||||||
direct URL such as a nightly/R2 package).
|
|
||||||
2. Installs it (`dpkg -i`).
|
|
||||||
3. Starts the first pool and verifies it via
|
|
||||||
`GET /rustfs/admin/v3/pools/list`.
|
|
||||||
4. Writes data with `warp` and monitors storage usage
|
|
||||||
(`GET /rustfs/admin/v3/storageinfo`) until the threshold.
|
|
||||||
5. Expands to a second pool (nodes started in parallel).
|
|
||||||
6. Starts rebalance (`POST /rustfs/admin/v3/rebalance/start`) and waits for
|
|
||||||
**all** pools to report `Completed`.
|
|
||||||
7. (3-pool mode) Expands to a third pool, rebalances again.
|
|
||||||
8. (optional) Decommissions pool 0
|
|
||||||
(`POST /rustfs/admin/v3/pools/decommission`), with automatic clear + retry
|
|
||||||
on failure, and waits for `decommissionInfo.complete == true`.
|
|
||||||
|
|
||||||
When an assertion fails, the script prints a per-pool summary **and the full
|
|
||||||
JSON response** (pool state, progress, failure counters, `waitingReason`,
|
|
||||||
`unresolvedEntries`, last rebalance error), so the GitHub Actions log shows
|
|
||||||
exactly where the test stopped. Credentials never appear in the logs.
|
|
||||||
|
|
||||||
## Self-hosted runner prerequisites
|
|
||||||
|
|
||||||
- Register the admin host (e.g. `heal`) as a runner with the
|
|
||||||
`smoke-testing` label.
|
|
||||||
- Install `jq`, `openssl`, `curl` and `warp` (only needed for `--with-warp`)
|
|
||||||
on the runner. `rc` is **not** required.
|
|
||||||
- The runner user must be able to SSH to all nodes without a password prompt
|
|
||||||
(`~/.ssh/config` with keys).
|
|
||||||
- The nodes need passwordless `sudo` for the SSH user, resolvable
|
|
||||||
`rustfs-node*` hostnames in `/etc/hosts`, and writable data directories.
|
|
||||||
- The admin API credentials must have the `admin:server-info`,
|
|
||||||
`admin:decommission` and `admin:rebalance` actions.
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
Set these in the repository (secrets/variables):
|
|
||||||
|
|
||||||
| Kind | Name | Purpose |
|
|
||||||
| ------ | --------------------- | ---------------------------------------------- |
|
|
||||||
| Secret | `RUSTFS_ACCESS_KEY` | RustFS access key |
|
|
||||||
| Secret | `RUSTFS_SECRET_KEY` | RustFS secret key |
|
|
||||||
| Var | `RUSTFS_API_ENDPOINT` | Admin API endpoint, e.g. `http://10.0.0.7:9000` (`RUSTFS_RC_ENDPOINT` is used as a fallback) |
|
|
||||||
| Var | `RUSTFS_NODES` | Space-separated node names, e.g. `vm000 vm001 vm002` |
|
|
||||||
| Var | `RUSTFS_SSH_USER` | SSH user for the nodes, e.g. `azureuser` |
|
|
||||||
|
|
||||||
## Workflow inputs
|
|
||||||
|
|
||||||
| Input | Default | Meaning |
|
|
||||||
| ------------------- | ------------- | ---------------------------------------- |
|
|
||||||
| `rustfs_version` | `1.0.0-rc.3` | GitHub release tag to test |
|
|
||||||
| `package_url` | *(empty)* | Direct `.deb` URL (e.g. nightly/R2); overrides `rustfs_version` |
|
|
||||||
| `pools` | `3` | Expand to 2 or 3 pools |
|
|
||||||
| `storage_threshold` | `50` | Stop warp writes at N% usage |
|
|
||||||
| `warp_duration` | `10m` | warp write duration |
|
|
||||||
| `run_decommission` | `true` | Run decommission (3-pool mode only) |
|
|
||||||
| `cleanup_before` | `true` | Reset nodes before the test |
|
|
||||||
| `cleanup_after` | `true` | Reset nodes after the test |
|
|
||||||
|
|
||||||
> ⚠️ `cleanup_before` / `cleanup_after` run the script's `--reset` mode, which
|
|
||||||
> **stops the services and deletes the data directories and config** on all
|
|
||||||
> nodes. Only use this workflow against a dedicated test environment.
|
|
||||||
|
|
||||||
## Manual usage
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Full workflow with a release tag
|
|
||||||
./scripts/test/rustfs_pool_expand.sh --all --with-warp -y \
|
|
||||||
--version 1.0.0-rc.3 --endpoint http://10.0.0.7:9000
|
|
||||||
|
|
||||||
# Use a direct .deb URL (e.g. nightly package on R2)
|
|
||||||
./scripts/test/rustfs_pool_expand.sh --all --with-warp -y \
|
|
||||||
--package-url https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb \
|
|
||||||
--endpoint http://10.0.0.7:9000
|
|
||||||
|
|
||||||
# Preflight / reset / single step
|
|
||||||
./scripts/test/rustfs_pool_expand.sh --preflight --version 1.0.0-rc.3
|
|
||||||
./scripts/test/rustfs_pool_expand.sh --reset -y
|
|
||||||
./scripts/test/rustfs_pool_expand.sh --step 9 --finalize-decommission -y
|
|
||||||
```
|
|
||||||
|
|
||||||
## Known issues and caveats
|
|
||||||
|
|
||||||
- RustFS `1.0.0-rc.3` fails decommission with a large `warp`-written bucket
|
|
||||||
("metacache listing quorum failed / timeout"). If decommission repeatedly
|
|
||||||
fails, reduce the written data (lower `storage_threshold`) or remove the test
|
|
||||||
bucket, then re-run step 9. The script detects the failure, clears metadata
|
|
||||||
and retries `DECOMMISSION_RETRIES` times before giving up.
|
|
||||||
- Multi-pool nodes must start **simultaneously** (the script does this) or the
|
|
||||||
first node dies with `not first disk`.
|
|
||||||
- The admin API is SigV4-signed (`host`, `x-amz-content-sha256:
|
|
||||||
UNSIGNED-PAYLOAD`, `x-amz-date`), matching the signer the RustFS server
|
|
||||||
itself trusts. If the cluster requires a non-default region, set
|
|
||||||
`SIGV4_REGION` in the script.
|
|
||||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user