mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-30 16:59:52 +00:00
hector
426 lines
16 KiB
YAML
426 lines
16 KiB
YAML
name: RustFS Functional Test Suite (S3/KMS/tier/pool/heal)
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
rustfs_version:
|
|
description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
|
|
required: false
|
|
package_url:
|
|
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
|
required: false
|
|
type: string
|
|
pools:
|
|
description: 'Number of pools to expand to (2 = first rebalance only)'
|
|
type: choice
|
|
options:
|
|
- '2'
|
|
- '3'
|
|
default: '3'
|
|
storage_threshold:
|
|
description: 'Stop writing when storage usage reaches N%'
|
|
required: false
|
|
default: '50'
|
|
warp_duration:
|
|
description: 'warp write duration (e.g. 5m, 10m)'
|
|
required: false
|
|
default: '10m'
|
|
run_decommission:
|
|
description: 'Run the pool decommission step (3-pool topology only)'
|
|
type: boolean
|
|
default: true
|
|
stop_node_gb:
|
|
description: 'Heal: stop the outage node when surviving nodes reach N GiB'
|
|
required: false
|
|
default: '15'
|
|
warp_stop_gb:
|
|
description: 'Heal: stop warp when surviving nodes reach N GiB'
|
|
required: false
|
|
default: '40'
|
|
cleanup_before:
|
|
description: 'Reset the nodes before the test (DESTROYS existing data/config)'
|
|
type: boolean
|
|
default: true
|
|
cleanup_after:
|
|
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
|
type: boolean
|
|
default: true
|
|
workflow_run:
|
|
# Run after the nightly build completes: S3 -> KMS -> tier -> pool -> heal.
|
|
workflows: ["Nightly GNU Build"]
|
|
types: [completed]
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Only one test run at a time: every job mutates the same shared test
|
|
# environment (vm000/vm001/vm002), so concurrent runs must not clobber each
|
|
# other. Jobs inside a run are chained with needs to serialize them.
|
|
concurrency:
|
|
group: rustfs-pool-expansion-test
|
|
cancel-in-progress: false
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
env:
|
|
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
|
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
|
RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
|
|
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
|
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
|
# Package used by the nightly run (workflow_dispatch inputs are empty for
|
|
# workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
|
|
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
|
|
|
jobs:
|
|
# All test scripts live in rustfs/auto-testing; rustfs stores only this
|
|
# workflow. Each job checks out auto-testing before running.
|
|
|
|
s3-compat-test:
|
|
name: S3 compatibility test
|
|
runs-on: smoke-testing
|
|
timeout-minutes: 240
|
|
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
|
steps:
|
|
- name: Checkout auto-testing scripts
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
with:
|
|
repository: rustfs/auto-testing
|
|
ref: main
|
|
path: auto-testing
|
|
persist-credentials: false
|
|
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
|
|
- name: Run S3 compatibility suite
|
|
run: |
|
|
chmod +x auto-testing/rustfs-s3-compat-test.sh
|
|
ARGS=(--all-topologies -y --log-file /tmp/rustfs-s3-compat.log)
|
|
if [ -n "${{ inputs.package_url }}" ]; then
|
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
|
elif [ -n "${{ inputs.rustfs_version }}" ]; then
|
|
ARGS+=(--version "${{ inputs.rustfs_version }}")
|
|
else
|
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
|
fi
|
|
./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
|
|
|
|
- name: Upload test logs
|
|
if: always()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: rustfs-s3-compat-${{ github.run_id }}
|
|
path: |
|
|
/tmp/rustfs-s3-compat*.log
|
|
if-no-files-found: warn
|
|
|
|
- name: Notify on failure
|
|
if: failure()
|
|
run: |
|
|
echo "RustFS S3 compatibility suite failed"
|
|
echo "See the uploaded log artifact for details."
|
|
|
|
kms-test:
|
|
name: KMS test (after S3)
|
|
runs-on: smoke-testing
|
|
timeout-minutes: 360
|
|
needs: s3-compat-test
|
|
if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
|
|
steps:
|
|
- name: Checkout auto-testing scripts
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
with:
|
|
repository: rustfs/auto-testing
|
|
ref: main
|
|
path: auto-testing
|
|
persist-credentials: false
|
|
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
|
|
- name: Ensure docker (Vault container)
|
|
run: |
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
sudo apt-get update
|
|
sudo apt-get install -y docker.io
|
|
fi
|
|
sudo systemctl enable --now docker
|
|
docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1
|
|
|
|
- name: Run KMS suite
|
|
run: |
|
|
chmod +x auto-testing/rustfs-kms-test.sh
|
|
ARGS=(--all-topologies --backends local,vault-kv2 -y --log-file /tmp/rustfs-kms.log)
|
|
if [ -n "${{ inputs.package_url }}" ]; then
|
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
|
elif [ -n "${{ inputs.rustfs_version }}" ]; then
|
|
ARGS+=(--version "${{ inputs.rustfs_version }}")
|
|
else
|
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
|
fi
|
|
./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
|
|
|
|
- name: Upload test logs
|
|
if: always()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: rustfs-kms-test-${{ github.run_id }}
|
|
path: |
|
|
/tmp/rustfs-kms*.log
|
|
if-no-files-found: warn
|
|
|
|
- name: Notify on failure
|
|
if: failure()
|
|
run: |
|
|
echo "RustFS KMS suite failed"
|
|
echo "See the uploaded log artifact for details."
|
|
|
|
tier-test:
|
|
name: Tier / event / audit test (after KMS)
|
|
runs-on: smoke-testing
|
|
timeout-minutes: 360
|
|
needs: kms-test
|
|
if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
|
|
steps:
|
|
- name: Checkout auto-testing scripts
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
with:
|
|
repository: rustfs/auto-testing
|
|
ref: main
|
|
path: auto-testing
|
|
persist-credentials: false
|
|
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
|
|
- name: Ensure MQTT broker + clients (event notification tests)
|
|
run: |
|
|
if ! command -v mosquitto_sub >/dev/null 2>&1; then
|
|
sudo apt-get update
|
|
sudo apt-get install -y mosquitto mosquitto-clients
|
|
fi
|
|
# Make restart deterministic on shared runners: stop any previous
|
|
# mosquitto instance/process and drop stale config first.
|
|
sudo systemctl stop mosquitto 2>/dev/null || true
|
|
sudo pkill -f mosquitto 2>/dev/null || true
|
|
sleep 1
|
|
sudo rm -f /etc/mosquitto/conf.d/rustfs-test.conf
|
|
sudo mkdir -p /etc/mosquitto/conf.d
|
|
printf 'listener 1883 0.0.0.0\nallow_anonymous true\n' | sudo tee /etc/mosquitto/conf.d/rustfs-test.conf >/dev/null
|
|
sudo systemctl reset-failed mosquitto 2>/dev/null || true
|
|
sudo systemctl start mosquitto
|
|
sleep 2
|
|
if ! ss -tln 2>/dev/null | grep -q ':1883'; then
|
|
echo "=== mosquitto status ==="
|
|
sudo systemctl status mosquitto --no-pager | head -20 || true
|
|
echo "=== mosquitto journal ==="
|
|
sudo journalctl -u mosquitto -n 30 --no-pager || true
|
|
echo "mosquitto not listening on 1883"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Run tier / event / audit suite
|
|
run: |
|
|
chmod +x auto-testing/rustfs-tier-test.sh
|
|
ARGS=(--all-topologies -y --log-file /tmp/rustfs-tier.log)
|
|
if [ -n "${{ inputs.package_url }}" ]; then
|
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
|
elif [ -n "${{ inputs.rustfs_version }}" ]; then
|
|
ARGS+=(--version "${{ inputs.rustfs_version }}")
|
|
else
|
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
|
fi
|
|
./auto-testing/rustfs-tier-test.sh "${ARGS[@]}"
|
|
|
|
- name: Upload test logs
|
|
if: always()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: rustfs-tier-test-${{ github.run_id }}
|
|
path: |
|
|
/tmp/rustfs-tier*.log
|
|
if-no-files-found: warn
|
|
|
|
- name: Notify on failure
|
|
if: failure()
|
|
run: |
|
|
echo "RustFS tier/event/audit suite failed"
|
|
echo "See the uploaded log artifact for details."
|
|
|
|
pool-expansion-test:
|
|
name: Pool expansion / decommission test (after tier)
|
|
runs-on: smoke-testing
|
|
timeout-minutes: 360
|
|
needs: tier-test
|
|
if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
|
|
steps:
|
|
- name: Checkout auto-testing scripts
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
with:
|
|
repository: rustfs/auto-testing
|
|
ref: main
|
|
path: auto-testing
|
|
persist-credentials: false
|
|
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
|
|
- name: Show environment
|
|
run: |
|
|
uname -a
|
|
jq --version
|
|
openssl version
|
|
warp --version || true
|
|
df -h /data | tail -1
|
|
|
|
- name: Reset test environment (before)
|
|
if: ${{ inputs.cleanup_before != 'false' }}
|
|
run: |
|
|
chmod +x auto-testing/rustfs_pool_expand.sh
|
|
./auto-testing/rustfs_pool_expand.sh --reset -y
|
|
|
|
- name: Install RustFS package & start first pool
|
|
run: |
|
|
ARGS=(--steps "1,2,3" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
|
|
if [ -n "${{ inputs.package_url }}" ]; then
|
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
|
elif [ -n "${{ inputs.rustfs_version }}" ]; then
|
|
ARGS+=(--version "${{ inputs.rustfs_version }}")
|
|
else
|
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
|
fi
|
|
./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
|
|
|
|
- name: Preflight checks
|
|
run: |
|
|
ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
|
|
if [ -n "${{ inputs.package_url }}" ]; then
|
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
|
elif [ -n "${{ inputs.rustfs_version }}" ]; then
|
|
ARGS+=(--version "${{ inputs.rustfs_version }}")
|
|
else
|
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
|
fi
|
|
./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
|
|
|
|
- name: Run pool expansion & decommission test
|
|
id: pool_test
|
|
run: |
|
|
set -o pipefail
|
|
STEPS="4,5,6"
|
|
if [ "${{ inputs.pools || '3' }}" = "3" ]; then
|
|
STEPS="$STEPS,7,8"
|
|
if [ "${{ inputs.run_decommission != 'false' }}" = "true" ]; then
|
|
STEPS="$STEPS,9"
|
|
fi
|
|
fi
|
|
ARGS=(--steps "$STEPS" --with-warp -y \
|
|
--endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
|
|
--storage-threshold "${{ inputs.storage_threshold || '50' }}" \
|
|
--warp-duration "${{ inputs.warp_duration || '10m' }}" \
|
|
--log-file /tmp/rustfs-pool-test.log)
|
|
if [ -n "${{ inputs.package_url }}" ]; then
|
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
|
elif [ -n "${{ inputs.rustfs_version }}" ]; then
|
|
ARGS+=(--version "${{ inputs.rustfs_version }}")
|
|
else
|
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
|
fi
|
|
./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
|
|
|
|
- name: Upload test logs
|
|
if: always()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: rustfs-pool-test-${{ github.run_id }}
|
|
path: |
|
|
/tmp/rustfs-pool-test*.log
|
|
/tmp/rustfs-warp.*.log
|
|
if-no-files-found: warn
|
|
|
|
- name: Reset test environment (after)
|
|
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
|
run: |
|
|
./auto-testing/rustfs_pool_expand.sh --reset -y
|
|
|
|
- name: Notify on failure
|
|
if: failure()
|
|
run: |
|
|
echo "RustFS pool expansion test failed"
|
|
echo "Package source: ${{ inputs.package_url || inputs.rustfs_version || 'nightly (R2 latest)' }}"
|
|
echo "See the uploaded log artifact for details."
|
|
|
|
# Heal regression runs after the pool test regardless of its outcome: a pool
|
|
# failure must be reported (it makes the run red) but must not block heal.
|
|
heal-test:
|
|
name: Heal test (after pool test)
|
|
runs-on: smoke-testing
|
|
timeout-minutes: 480
|
|
needs: pool-expansion-test
|
|
if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
|
|
steps:
|
|
- name: Checkout auto-testing scripts
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
with:
|
|
repository: rustfs/auto-testing
|
|
ref: main
|
|
path: auto-testing
|
|
persist-credentials: false
|
|
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
|
|
- name: Reset test environment (before)
|
|
if: ${{ inputs.cleanup_before != 'false' }}
|
|
run: |
|
|
chmod +x auto-testing/rustfs_heal_test.sh
|
|
./auto-testing/rustfs_heal_test.sh --reset -y
|
|
|
|
- name: Install RustFS package & start cluster
|
|
run: |
|
|
ARGS=(--steps "1,2" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
|
|
if [ -n "${{ inputs.package_url }}" ]; then
|
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
|
else
|
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
|
fi
|
|
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
|
|
|
|
- name: Preflight checks
|
|
run: |
|
|
ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
|
|
if [ -n "${{ inputs.package_url }}" ]; then
|
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
|
else
|
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
|
fi
|
|
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
|
|
|
|
- name: Run heal test (write -> outage -> heal -> verify)
|
|
run: |
|
|
ARGS=(--steps 3,4,5,6,7 -y \
|
|
--endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
|
|
--stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
|
|
--warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
|
|
--log-file /tmp/rustfs-heal-test.log)
|
|
if [ -n "${{ inputs.package_url }}" ]; then
|
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
|
else
|
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
|
fi
|
|
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
|
|
|
|
- name: Upload test logs
|
|
if: always()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: rustfs-heal-test-${{ github.run_id }}
|
|
path: |
|
|
/tmp/rustfs-heal-test.log
|
|
/tmp/rustfs-warp.*.log
|
|
if-no-files-found: warn
|
|
|
|
- name: Reset test environment (after)
|
|
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
|
run: |
|
|
./auto-testing/rustfs_heal_test.sh --reset -y
|
|
|
|
- name: Notify on failure
|
|
if: failure()
|
|
run: |
|
|
echo "RustFS heal test failed"
|
|
echo "See the uploaded log artifact for details."
|