mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-13 16:11:56 +00:00
hector
45f97bccf9
timeout-minutes requires a numeric value, but inputs.pool_timeout_minutes
is a string, so '${{ inputs.pool_timeout_minutes || 240 }}' evaluated to
the string '240' and the runner rejected the template: 'An error occurred
when attempting to determine the step timeout ... Unexpected value 240'
(chain run 36505625402 job at 02:25:52, standalone run 36393453114 at
07:46:37). Both pool runs since #8155 merged have therefore executed
without the intended step budget. Wrap the expression in fromJSON and
update the policy test that pinned the broken literal.
790 lines
36 KiB
YAML
790 lines
36 KiB
YAML
name: RustFS Pool Expansion Test
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
chain_manifest:
|
|
description: Verified candidate and chain attempt from the chain driver
|
|
type: string
|
|
required: true
|
|
pool_timeout_minutes:
|
|
description: Wall-clock budget (minutes) for the pool run step
|
|
type: string
|
|
required: false
|
|
default: '240'
|
|
workflow_dispatch:
|
|
inputs:
|
|
rustfs_version:
|
|
description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
|
|
required: false
|
|
package_url:
|
|
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
|
required: false
|
|
type: string
|
|
pools:
|
|
description: 'Number of pools to expand to (2 = first rebalance only)'
|
|
type: choice
|
|
options:
|
|
- '2'
|
|
- '3'
|
|
default: '3'
|
|
storage_threshold:
|
|
description: 'Stop writing when storage usage reaches N%'
|
|
required: false
|
|
default: '50'
|
|
warp_duration:
|
|
description: 'warp write duration (e.g. 5m, 10m)'
|
|
required: false
|
|
default: '10m'
|
|
warp_concurrent:
|
|
description: 'Pool fill: concurrent warp operations'
|
|
required: false
|
|
default: '32'
|
|
run_decommission:
|
|
description: 'Run the pool decommission step (3-pool topology only)'
|
|
type: boolean
|
|
default: true
|
|
pool_timeout_minutes:
|
|
description: 'Wall-clock budget (minutes) for the whole pool run: fill + rebalance wait + decommission'
|
|
required: false
|
|
default: '240'
|
|
cleanup_before:
|
|
description: 'Reset the nodes before the test (DESTROYS existing data/config)'
|
|
type: boolean
|
|
default: true
|
|
cleanup_after:
|
|
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
|
type: boolean
|
|
default: true
|
|
repository_dispatch:
|
|
# Chain handoff: dispatched when the heal suite finishes.
|
|
types: [rustfs-chain-pool]
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Only one test run at a time: the job mutates the same shared test
|
|
# environment (vm000/vm001/vm002), so concurrent runs must not clobber each
|
|
# other.
|
|
concurrency:
|
|
group: rustfs-shared-functional-tests-v2
|
|
cancel-in-progress: false
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
env:
|
|
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
|
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
|
RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
|
|
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002' }}
|
|
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
|
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
# Package used by the nightly run (workflow_dispatch inputs are empty for
|
|
# workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
|
|
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
|
|
|
jobs:
|
|
# Pool expansion: dispatched by the heal suite's chain handoff. Heal
|
|
# itself lives in rustfs-heal-test.yml and runs exactly once per chain.
|
|
pool-expansion-test:
|
|
name: Pool expansion / decommission test
|
|
runs-on: smoke-testing
|
|
# Leave setup and finalizer headroom beyond the default 240-minute test.
|
|
timeout-minutes: 360
|
|
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
|
env:
|
|
RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
|
|
RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
|
|
RUSTFS_POOL_WARP_ENDPOINT: http://127.0.0.1:19000
|
|
RUSTFS_SHARED_PROXY_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
|
|
RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000 http://rustfs-node4:9000' }}
|
|
steps:
|
|
- name: Checkout chain tooling
|
|
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Bind functional candidate
|
|
id: chain
|
|
if: ${{ inputs.chain_manifest != '' }}
|
|
env:
|
|
CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
|
|
run: python3 scripts/functional_chain_evidence.py consume
|
|
|
|
- name: Checkout auto-testing scripts
|
|
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
|
with:
|
|
repository: rustfs/auto-testing
|
|
ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
|
|
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
path: auto-testing
|
|
persist-credentials: false
|
|
|
|
- name: Initialize pool test artifacts
|
|
id: evidence
|
|
run: |
|
|
set -euo pipefail
|
|
ARTIFACT_DIR="${RUNNER_TEMP}/rustfs-pool-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
|
mkdir -p "${ARTIFACT_DIR}"
|
|
echo "POOL_ARTIFACT_DIR=${ARTIFACT_DIR}" >> "${GITHUB_ENV}"
|
|
|
|
- name: Show environment
|
|
run: |
|
|
uname -a
|
|
jq --version
|
|
openssl version
|
|
warp --version || true
|
|
df -h /data | tail -1
|
|
|
|
- name: Validate pool topology before destructive cleanup
|
|
id: topology
|
|
run: |
|
|
set -euo pipefail
|
|
read -r -a NODES <<< "${RUSTFS_NODES}"
|
|
read -r -a ENDPOINTS <<< "${RUSTFS_POOL_NODE_ENDPOINTS}"
|
|
if [ "${#NODES[@]}" -lt 3 ] || [ "${#ENDPOINTS[@]}" -lt "${#NODES[@]}" ]; then
|
|
echo "Pool tests require at least three nodes and a direct endpoint for every node" >&2
|
|
exit 1
|
|
fi
|
|
|
|
- name: Cleanup environment (before)
|
|
timeout-minutes: 5
|
|
if: ${{ inputs.cleanup_before != 'false' }}
|
|
run: |
|
|
set -euo pipefail
|
|
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
|
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
|
for node in "${NODES[@]}"; do
|
|
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
|
set -euo pipefail
|
|
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
|
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
|
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
|
${SUDO} dpkg -P rustfs
|
|
fi
|
|
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
|
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
|
'
|
|
done
|
|
|
|
- name: Install RustFS package & start cluster
|
|
timeout-minutes: 5
|
|
run: |
|
|
ARGS=(--steps "1,2,3" -y \
|
|
--admin-endpoint "${RUSTFS_POOL_ADMIN_ENDPOINT}" \
|
|
--warp-endpoint "${RUSTFS_POOL_WARP_ENDPOINT}" \
|
|
--node-endpoints "${RUSTFS_POOL_NODE_ENDPOINTS}" \
|
|
--log-file "${POOL_ARTIFACT_DIR}/pool-test.log")
|
|
if [ -n "${{ inputs.package_url }}" ]; then
|
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
|
elif [ -n "${{ inputs.rustfs_version }}" ]; then
|
|
ARGS+=(--version "${{ inputs.rustfs_version }}")
|
|
else
|
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
|
fi
|
|
./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
|
|
|
|
- name: Preflight checks
|
|
timeout-minutes: 5
|
|
run: |
|
|
ARGS=(--preflight \
|
|
--admin-endpoint "${RUSTFS_POOL_ADMIN_ENDPOINT}" \
|
|
--warp-endpoint "${RUSTFS_POOL_WARP_ENDPOINT}" \
|
|
--node-endpoints "${RUSTFS_POOL_NODE_ENDPOINTS}" \
|
|
--log-file "${POOL_ARTIFACT_DIR}/pool-test.log")
|
|
if [ -n "${{ inputs.package_url }}" ]; then
|
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
|
elif [ -n "${{ inputs.rustfs_version }}" ]; then
|
|
ARGS+=(--version "${{ inputs.rustfs_version }}")
|
|
else
|
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
|
fi
|
|
./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
|
|
|
|
- name: Reset dedicated pool proxy
|
|
timeout-minutes: 2
|
|
run: |
|
|
set -euo pipefail
|
|
RUSTFS_POOL_NGINX_CONFIG_PATH=/etc/nginx/conf.d/rustfs-pool-test.conf \
|
|
RUSTFS_POOL_NGINX_LISTEN="${RUSTFS_POOL_PROXY_ENDPOINT#http://}" \
|
|
RUSTFS_POOL_NGINX_ACCESS_LOG=/var/log/nginx/rustfs-pool-test-access.log \
|
|
RUSTFS_POOL_NGINX_ERROR_LOG=/var/log/nginx/rustfs-pool-test-error.log \
|
|
./auto-testing/rustfs_pool_nginx_stage.sh cleanup
|
|
|
|
- name: Capture pool test baseline
|
|
timeout-minutes: 2
|
|
run: |
|
|
set -uo pipefail
|
|
BASELINE_FILE="${POOL_ARTIFACT_DIR}/pool-baseline.log"
|
|
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
|
read -r -a DIRECT_ENDPOINTS <<< "${RUSTFS_POOL_NODE_ENDPOINTS}"
|
|
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
|
failed=0
|
|
: > "${BASELINE_FILE}"
|
|
|
|
if [ "${#DIRECT_ENDPOINTS[@]}" -lt "${#NODES[@]}" ]; then
|
|
echo "not enough direct endpoints for the configured nodes" | tee -a "${BASELINE_FILE}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
for index in "${!NODES[@]}"; do
|
|
node="${NODES[$index]}"
|
|
endpoint="${DIRECT_ENDPOINTS[$index]}"
|
|
body_file="${POOL_ARTIFACT_DIR}/ready-baseline-$((index + 1)).body"
|
|
{
|
|
echo "--- node=${node} endpoint=${endpoint} ---"
|
|
if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
|
|
"${SSH_USER}@${node}" '
|
|
set -euo pipefail
|
|
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
|
echo "--- rustfs version ---"
|
|
rustfs --version
|
|
echo "--- systemd state ---"
|
|
${SUDO} systemctl show rustfs --no-pager \
|
|
--property=ActiveState,SubState,Result,ExecMainPID,ExecMainStartTimestamp,NRestarts
|
|
'; then
|
|
echo "baseline collection failed for ${node}"
|
|
failed=1
|
|
fi
|
|
curl -sS --connect-timeout 5 --max-time 15 -o "${body_file}" \
|
|
-w "baseline_ready=${endpoint} http=%{http_code} connect=%{time_connect} ttfb=%{time_starttransfer} total=%{time_total}\n" \
|
|
"${endpoint%/}/health/ready" || true
|
|
echo "--- readiness body ---"
|
|
cat "${body_file}" 2>/dev/null || true
|
|
echo
|
|
} >> "${BASELINE_FILE}" 2>&1
|
|
done
|
|
|
|
[ "${failed}" -eq 0 ] || exit 1
|
|
|
|
- name: Run pool expansion & decommission test
|
|
id: pool_test
|
|
# The suite budgets up to 24h each for rebalance and decommission
|
|
# (REBALANCE_TIMEOUT / DECOMMISSION_TIMEOUT in rustfs_pool_expand.sh);
|
|
# the previous 45m step cap killed four consecutive runs mid-rebalance
|
|
# with steps 1-7 green. 240m covers the observed pace (rebalance 1/4
|
|
# in ~21m) plus one decommission pass; override per dispatch.
|
|
# fromJSON coerces the string dispatch input to the number that
|
|
# timeout-minutes requires; the bare || form yielded the string '240'
|
|
# and the runner rejected the template (Unexpected value '240').
|
|
timeout-minutes: ${{ fromJSON(inputs.pool_timeout_minutes || '240') }}
|
|
# Case failures keep the run green: the report and the backlog
|
|
# issue manager carry the product signal.
|
|
continue-on-error: true
|
|
run: |
|
|
set -o pipefail
|
|
STEPS="4,5,6"
|
|
if [ "${{ inputs.pools || '3' }}" = "3" ]; then
|
|
STEPS="$STEPS,7,8"
|
|
if [ "${{ inputs.run_decommission != 'false' }}" = "true" ]; then
|
|
STEPS="$STEPS,9"
|
|
fi
|
|
fi
|
|
ARGS=(--steps "$STEPS" --with-warp -y \
|
|
--admin-endpoint "${RUSTFS_POOL_ADMIN_ENDPOINT}" \
|
|
--warp-endpoint "${RUSTFS_POOL_WARP_ENDPOINT}" \
|
|
--node-endpoints "${RUSTFS_POOL_NODE_ENDPOINTS}" \
|
|
--storage-threshold "${{ inputs.storage_threshold || '50' }}" \
|
|
--warp-duration "${{ inputs.warp_duration || '10m' }}" \
|
|
--warp-concurrent "${{ inputs.warp_concurrent || '32' }}" \
|
|
--log-file "${POOL_ARTIFACT_DIR}/pool-test.log")
|
|
if [ -n "${RUSTFS_POOL_PROXY_ENDPOINT}" ]; then
|
|
ARGS+=(--proxy-endpoint "${RUSTFS_POOL_PROXY_ENDPOINT}")
|
|
fi
|
|
if [ -n "${{ inputs.package_url }}" ]; then
|
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
|
elif [ -n "${{ inputs.rustfs_version }}" ]; then
|
|
ARGS+=(--version "${{ inputs.rustfs_version }}")
|
|
else
|
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
|
fi
|
|
RUSTFS_WARP_LOG_FILE="${POOL_ARTIFACT_DIR}/warp.log" \
|
|
RUSTFS_PROXY_STAGE_HOOK=./auto-testing/rustfs_pool_nginx_stage.sh \
|
|
RUSTFS_POOL_NGINX_CONFIG_PATH=/etc/nginx/conf.d/rustfs-pool-test.conf \
|
|
RUSTFS_POOL_NGINX_LISTEN="${RUSTFS_POOL_PROXY_ENDPOINT#http://}" \
|
|
RUSTFS_POOL_NGINX_ACCESS_LOG=/var/log/nginx/rustfs-pool-test-access.log \
|
|
RUSTFS_POOL_NGINX_ERROR_LOG=/var/log/nginx/rustfs-pool-test-error.log \
|
|
./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
|
|
|
|
- name: Collect pool test diagnostics
|
|
timeout-minutes: 2
|
|
if: always()
|
|
run: |
|
|
set -uo pipefail
|
|
ARTIFACT_DIR="${POOL_ARTIFACT_DIR:-${RUNNER_TEMP}/rustfs-pool-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}}"
|
|
mkdir -p "${ARTIFACT_DIR}"
|
|
echo "POOL_ARTIFACT_DIR=${ARTIFACT_DIR}" >> "${GITHUB_ENV}"
|
|
|
|
redact() {
|
|
sed -E \
|
|
-e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)=).*/\1[REDACTED]/Ig' \
|
|
-e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
|
|
-e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
|
|
-e 's/(proxy_set_header[[:space:]]+Authorization[[:space:]]+).*/\1[REDACTED];/Ig' \
|
|
-e 's/^.*(password|secret|token).*/[REDACTED SENSITIVE LINE]/Ig'
|
|
}
|
|
|
|
if [ "$(id -u)" -eq 0 ]; then
|
|
SUDO=()
|
|
else
|
|
SUDO=(sudo -n)
|
|
fi
|
|
|
|
{
|
|
echo "captured_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
echo "run_id=${GITHUB_RUN_ID}"
|
|
echo "run_attempt=${GITHUB_RUN_ATTEMPT}"
|
|
if command -v nginx >/dev/null 2>&1; then
|
|
"${SUDO[@]}" nginx -T 2>&1 || echo "nginx -T failed"
|
|
else
|
|
echo "nginx is not installed on the runner"
|
|
fi
|
|
} | redact > "${ARTIFACT_DIR}/nginx-config-redacted.txt"
|
|
|
|
for log_path in \
|
|
/var/log/nginx/access.log \
|
|
/var/log/nginx/error.log \
|
|
/var/log/nginx/rustfs-pool-test-access.log \
|
|
/var/log/nginx/rustfs-pool-test-error.log; do
|
|
log_name="$(basename "${log_path}")"
|
|
if "${SUDO[@]}" test -r "${log_path}" 2>/dev/null; then
|
|
"${SUDO[@]}" cat "${log_path}" 2>&1 | redact \
|
|
> "${ARTIFACT_DIR}/nginx-${log_name%.log}-redacted.log"
|
|
else
|
|
echo "unavailable: ${log_path}" > "${ARTIFACT_DIR}/nginx-${log_name%.log}-redacted.log"
|
|
fi
|
|
done
|
|
"${SUDO[@]}" journalctl -u nginx --no-pager -n 5000 2>&1 | redact \
|
|
> "${ARTIFACT_DIR}/nginx-journal-redacted.log" || true
|
|
|
|
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
|
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
|
for node in "${NODES[@]}"; do
|
|
safe_node="${node//[^A-Za-z0-9_.-]/_}"
|
|
{
|
|
if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
|
|
"${SSH_USER}@${node}" '
|
|
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
|
echo "--- rustfs version ---"
|
|
rustfs --version 2>&1 || true
|
|
echo "--- systemd state ---"
|
|
${SUDO} systemctl show rustfs --no-pager \
|
|
--property=ActiveState,SubState,Result,ExecMainPID,ExecMainStartTimestamp,NRestarts 2>&1 || true
|
|
echo "--- rustfs journal ---"
|
|
${SUDO} journalctl -u rustfs --no-pager -n 10000 2>&1 || true
|
|
echo "--- rustfs file logs ---"
|
|
if ${SUDO} test -d /var/log/rustfs; then
|
|
${SUDO} find /var/log/rustfs -maxdepth 2 -type f -print 2>/dev/null | while IFS= read -r file; do
|
|
echo "--- ${file} (last 5000 lines) ---"
|
|
${SUDO} tail -n 5000 "${file}" 2>&1 || true
|
|
done
|
|
else
|
|
echo "/var/log/rustfs is unavailable"
|
|
fi
|
|
'; then
|
|
echo "SSH diagnostics failed for ${node}"
|
|
fi
|
|
} 2>&1 | redact > "${ARTIFACT_DIR}/${safe_node}-rustfs-redacted.log"
|
|
done
|
|
|
|
: > "${ARTIFACT_DIR}/endpoint-ready-probes.log"
|
|
read -r -a DIRECT_ENDPOINTS <<< "${RUSTFS_POOL_NODE_ENDPOINTS}"
|
|
probe_index=0
|
|
for endpoint in "${DIRECT_ENDPOINTS[@]}"; do
|
|
probe_index=$((probe_index + 1))
|
|
curl -sS --connect-timeout 5 --max-time 15 -o "${ARTIFACT_DIR}/ready-direct-${probe_index}.body" \
|
|
-w "direct[${probe_index}]=${endpoint} http=%{http_code} connect=%{time_connect} ttfb=%{time_starttransfer} total=%{time_total}\n" \
|
|
"${endpoint%/}/health/ready" >> "${ARTIFACT_DIR}/endpoint-ready-probes.log" 2>&1 || true
|
|
done
|
|
if [ -n "${RUSTFS_POOL_PROXY_ENDPOINT}" ]; then
|
|
curl -sS --connect-timeout 5 --max-time 15 -o "${ARTIFACT_DIR}/ready-proxy.body" \
|
|
-w "proxy=${RUSTFS_POOL_PROXY_ENDPOINT} http=%{http_code} connect=%{time_connect} ttfb=%{time_starttransfer} total=%{time_total}\n" \
|
|
"${RUSTFS_POOL_PROXY_ENDPOINT%/}/health/ready" >> "${ARTIFACT_DIR}/endpoint-ready-probes.log" 2>&1 || true
|
|
fi
|
|
if [ -n "${RUSTFS_SHARED_PROXY_ENDPOINT}" ]; then
|
|
curl -sS --connect-timeout 5 --max-time 15 -o "${ARTIFACT_DIR}/ready-shared-proxy.body" \
|
|
-w "shared_proxy=${RUSTFS_SHARED_PROXY_ENDPOINT} http=%{http_code} connect=%{time_connect} ttfb=%{time_starttransfer} total=%{time_total}\n" \
|
|
"${RUSTFS_SHARED_PROXY_ENDPOINT%/}/health/ready" >> "${ARTIFACT_DIR}/endpoint-ready-probes.log" 2>&1 || true
|
|
fi
|
|
|
|
- name: Generate report
|
|
id: chain_report
|
|
timeout-minutes: 2
|
|
if: always()
|
|
run: |
|
|
set -euo pipefail
|
|
LOG_FILE="${POOL_ARTIFACT_DIR}/pool-test.log"
|
|
REPORT_FILE="${POOL_ARTIFACT_DIR}/pool-report.md"
|
|
PACKAGE_URL='${{ inputs.package_url }}'
|
|
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
|
if [ -n "${PACKAGE_URL}" ]; then
|
|
PACKAGE_SOURCE="${PACKAGE_URL}"
|
|
elif [ -n "${RUSTFS_VERSION}" ]; then
|
|
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
|
else
|
|
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
|
fi
|
|
STEPS_TABLE="${POOL_ARTIFACT_DIR}/pool-steps.md"
|
|
python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
|
|
import re
|
|
import sys
|
|
|
|
log_file, out_file = sys.argv[1], sys.argv[2]
|
|
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
|
step_re = re.compile(r'^\[POOL-STEP\]\s+(\d+)\s+(.+?)\s+(PASS|FAIL|SKIP)\s*$')
|
|
ver_re = re.compile(r'^\[POOL-VERSION\]\s+(\S+)(?:\s+\(node\s+(\S+)\))?\s*$')
|
|
result_re = re.compile(r'^\[POOL-RESULT\]\s+(PASS|FAIL)\s+(.*)$')
|
|
|
|
steps = {}
|
|
order = []
|
|
version = None
|
|
version_node = None
|
|
verdict = None
|
|
verdict_detail = ''
|
|
try:
|
|
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
|
for raw in fh:
|
|
line = ansi.sub('', raw).strip()
|
|
m = step_re.match(line)
|
|
if m:
|
|
n, desc, status = m.group(1), m.group(2), m.group(3)
|
|
if n not in steps:
|
|
order.append(n)
|
|
steps[n] = (desc, status) # later lines win (fail after pass)
|
|
continue
|
|
m = ver_re.match(line)
|
|
if m:
|
|
version, version_node = m.group(1), m.group(2)
|
|
continue
|
|
m = result_re.match(line)
|
|
if m:
|
|
verdict, verdict_detail = m.group(1), m.group(2)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
with open(out_file, 'w', encoding='utf-8') as out:
|
|
out.write('## Step Results\n\n')
|
|
if version:
|
|
node_note = f' (captured via `rustfs --version` on {version_node})' if version_node else ''
|
|
out.write(f'- Version under test: **{version}**{node_note}\n')
|
|
if verdict:
|
|
out.write(f'- Overall result: **{verdict}** — {verdict_detail}\n')
|
|
out.write('\n')
|
|
out.write('| Step | Description | Result |\n')
|
|
out.write('| --- | --- | --- |\n')
|
|
for n in sorted(order, key=int):
|
|
desc, status = steps[n]
|
|
out.write(f'| {n} | {desc} | {status} |\n')
|
|
if not order:
|
|
out.write('| - | - | NOT RUN (no step result lines found) |\n')
|
|
PY
|
|
{
|
|
echo "# RustFS pool expansion test report"
|
|
echo ""
|
|
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
|
echo "- Trigger: ${{ github.event_name }}"
|
|
echo "- Package: ${PACKAGE_SOURCE}"
|
|
echo "- Warp concurrent: ${{ inputs.warp_concurrent || '32' }}"
|
|
echo "- Test Step Outcome: ${{ steps.pool_test.outcome }}"
|
|
echo ""
|
|
cat "${STEPS_TABLE}" || true
|
|
echo ""
|
|
echo "## Log tail"
|
|
echo '```text'
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
echo '```'
|
|
} | tee "${REPORT_FILE}"
|
|
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
|
|
|
- name: Validate pool diagnostic completeness
|
|
timeout-minutes: 2
|
|
if: always()
|
|
run: |
|
|
set -euo pipefail
|
|
failed=0
|
|
require_nonempty() {
|
|
if [ ! -s "$1" ]; then
|
|
echo "required diagnostic is missing or empty: $1" >&2
|
|
failed=1
|
|
fi
|
|
}
|
|
require_available() {
|
|
if [ ! -e "$1" ]; then
|
|
echo "required diagnostic is missing: $1" >&2
|
|
failed=1
|
|
elif grep -Fq 'unavailable:' "$1" 2>/dev/null; then
|
|
echo "required diagnostic could not be collected: $1" >&2
|
|
failed=1
|
|
fi
|
|
}
|
|
|
|
require_nonempty "${POOL_ARTIFACT_DIR}/pool-test.log"
|
|
# warp is stopped early (SIGINT/SIGTERM) once the storage threshold
|
|
# is reached, and it only writes its final report on a clean exit —
|
|
# an empty warp.log is the expected shape of a healthy run, so only
|
|
# require that the redirect target exists.
|
|
require_available "${POOL_ARTIFACT_DIR}/warp.log"
|
|
require_nonempty "${POOL_ARTIFACT_DIR}/pool-report.md"
|
|
require_nonempty "${POOL_ARTIFACT_DIR}/pool-baseline.log"
|
|
require_nonempty "${POOL_ARTIFACT_DIR}/nginx-config-redacted.txt"
|
|
require_nonempty "${POOL_ARTIFACT_DIR}/nginx-rustfs-pool-test-access-redacted.log"
|
|
require_available "${POOL_ARTIFACT_DIR}/nginx-rustfs-pool-test-access-redacted.log"
|
|
require_available "${POOL_ARTIFACT_DIR}/nginx-rustfs-pool-test-error-redacted.log"
|
|
require_nonempty "${POOL_ARTIFACT_DIR}/endpoint-ready-probes.log"
|
|
|
|
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
|
if grep -Fq 'baseline collection failed' "${POOL_ARTIFACT_DIR}/pool-baseline.log" 2>/dev/null; then
|
|
echo "one or more node baselines could not be collected" >&2
|
|
failed=1
|
|
fi
|
|
for node in "${NODES[@]}"; do
|
|
safe_node="${node//[^A-Za-z0-9_.-]/_}"
|
|
node_log="${POOL_ARTIFACT_DIR}/${safe_node}-rustfs-redacted.log"
|
|
require_nonempty "${node_log}"
|
|
if grep -Fq "SSH diagnostics failed for ${node}" "${node_log}" 2>/dev/null; then
|
|
echo "node diagnostics failed: ${node_log}" >&2
|
|
failed=1
|
|
fi
|
|
if ! grep -Eq '^rustfs @' "${node_log}" 2>/dev/null \
|
|
|| ! grep -Eq '^NRestarts=[0-9]+$' "${node_log}" 2>/dev/null; then
|
|
echo "node version or restart evidence is incomplete: ${node_log}" >&2
|
|
failed=1
|
|
elif grep -Eq '^NRestarts=[1-9][0-9]*$' "${node_log}"; then
|
|
echo "RustFS restarted unexpectedly during the run: ${node_log}" >&2
|
|
failed=1
|
|
fi
|
|
done
|
|
|
|
if ! grep -Fq "upstream_status=\"\$upstream_status\"" \
|
|
"${POOL_ARTIFACT_DIR}/nginx-config-redacted.txt"; then
|
|
echo "Nginx config does not expose upstream status fields" >&2
|
|
failed=1
|
|
fi
|
|
if ! grep -Eq '^proxy=.* http=200([[:space:]]|$)' "${POOL_ARTIFACT_DIR}/endpoint-ready-probes.log"; then
|
|
echo "dedicated proxy readiness probe did not return HTTP 200" >&2
|
|
failed=1
|
|
fi
|
|
if grep -Eq 'status=50(2|4)|upstream_status="[^"]*50(2|4)' \
|
|
"${POOL_ARTIFACT_DIR}/nginx-rustfs-pool-test-access-redacted.log"; then
|
|
echo "dedicated proxy access log contains a 502/504 response" >&2
|
|
failed=1
|
|
fi
|
|
if grep -Eiq 'upstream prematurely closed connection|upstream timed out|(connect\(\)|recv\(\)|send\(\)) failed.*upstream|connection reset by peer.*upstream' \
|
|
"${POOL_ARTIFACT_DIR}/nginx-rustfs-pool-test-error-redacted.log"; then
|
|
echo "dedicated proxy error log contains an upstream timeout or connection failure" >&2
|
|
failed=1
|
|
fi
|
|
# A mid-script abort is a product/harness signal: the run step's
|
|
# continue-on-error would otherwise keep the workflow green, so the
|
|
# validator must turn it red. Aborts go through die(), which prints
|
|
# an [ERROR] line; a step that dies never emits its [POOL-STEP]
|
|
# verdict, so "any FAIL verdict" cannot be the trigger.
|
|
if grep -q '^\[POOL-STEP\] .* FAIL' "${POOL_ARTIFACT_DIR}/pool-test.log" 2>/dev/null \
|
|
|| grep -q '\[ERROR\]' "${POOL_ARTIFACT_DIR}/pool-test.log" 2>/dev/null; then
|
|
echo "pool test script reported a failure (step FAIL verdict or die() abort)" >&2
|
|
failed=1
|
|
fi
|
|
[ "${failed}" -eq 0 ] || exit 1
|
|
|
|
- name: Upload functional report to dashboard
|
|
timeout-minutes: 2
|
|
if: always()
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
|
SUITE: pool
|
|
run: |
|
|
set -euo pipefail
|
|
REPORT_FILE="${POOL_ARTIFACT_DIR}/pool-report.md"
|
|
if [ -z "${GH_TOKEN:-}" ]; then
|
|
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
|
exit 0
|
|
fi
|
|
DATE="$(date -u +%Y-%m-%d)"
|
|
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
|
# Base64-encode the report into a temp file and feed it to jq via
|
|
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
|
|
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
|
|
B64_FILE="$(mktemp)"
|
|
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
|
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
|
if [ -n "${SHA}" ]; then
|
|
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
|
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
|
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
|
else
|
|
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
|
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
|
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
|
fi
|
|
rm -f "${B64_FILE}"
|
|
|
|
- name: Manage backlog issues (dedup / label / auto-close)
|
|
timeout-minutes: 2
|
|
# Replaces the old per-run failure filing. One entry point that:
|
|
# - dedups by signal: failing cases are matched against open backlog
|
|
# issues by label (category + case ID); covered cases become a
|
|
# comment on the existing issue, only uncovered cases file a new one
|
|
# - labels new issues (functional-test, category, case IDs, env)
|
|
# - closes fixed issues after a fully green run
|
|
# - never files or closes on cancelled runs
|
|
if: ${{ always() }}
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
SUITE: 'pool'
|
|
SUITE_LABEL: 'Pool expansion'
|
|
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [ -z "${GH_TOKEN:-}" ]; then
|
|
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
|
exit 0
|
|
fi
|
|
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
|
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
|
exit 0
|
|
fi
|
|
PACKAGE_URL='${{ inputs.package_url }}'
|
|
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
|
PACKAGE_SOURCE=""
|
|
if [ -n "${PACKAGE_URL}" ]; then
|
|
PACKAGE_SOURCE="${PACKAGE_URL}"
|
|
elif [ -n "${RUSTFS_VERSION}" ]; then
|
|
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
|
else
|
|
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
|
fi
|
|
python3 auto-testing/scripts/issue_manager.py handle \
|
|
--repo rustfs/backlog \
|
|
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
|
--outcome "${{ steps.pool_test.outcome }}" \
|
|
--report-file "${POOL_ARTIFACT_DIR}/pool-report.md" \
|
|
--log "${POOL_ARTIFACT_DIR}/pool-test.log" \
|
|
--run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
|
|
--run-id "${GITHUB_RUN_ID}" \
|
|
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
|
--commit "${GITHUB_SHA}" \
|
|
--trigger "${{ github.event_name }}" \
|
|
--package-source "${PACKAGE_SOURCE}" \
|
|
--date "$(date -u +%Y-%m-%d)"
|
|
|
|
- name: Upload test logs
|
|
timeout-minutes: 2
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: rustfs-pool-test-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: ${{ runner.temp }}/rustfs-pool-${{ github.run_id }}-${{ github.run_attempt }}
|
|
if-no-files-found: warn
|
|
|
|
- name: Restore dedicated pool proxy
|
|
timeout-minutes: 2
|
|
if: always()
|
|
run: |
|
|
set -euo pipefail
|
|
RUSTFS_POOL_NGINX_CONFIG_PATH=/etc/nginx/conf.d/rustfs-pool-test.conf \
|
|
RUSTFS_POOL_NGINX_LISTEN="${RUSTFS_POOL_PROXY_ENDPOINT#http://}" \
|
|
RUSTFS_POOL_NGINX_ACCESS_LOG=/var/log/nginx/rustfs-pool-test-access.log \
|
|
RUSTFS_POOL_NGINX_ERROR_LOG=/var/log/nginx/rustfs-pool-test-error.log \
|
|
./auto-testing/rustfs_pool_nginx_stage.sh cleanup
|
|
|
|
- name: Cleanup environment (after)
|
|
timeout-minutes: 5
|
|
if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}
|
|
run: |
|
|
set -euo pipefail
|
|
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
|
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
|
for node in "${NODES[@]}"; do
|
|
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
|
set -euo pipefail
|
|
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
|
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
|
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
|
${SUDO} dpkg -P rustfs
|
|
fi
|
|
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
|
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
|
'
|
|
done
|
|
|
|
- name: "Continue functional chain (next: Security)"
|
|
# Only chain-triggered runs forward to the next suite; standalone
|
|
# workflow_dispatch runs stop after their own cleanup. A failed
|
|
# handoff must never pass silently: it retries, then files an alert
|
|
# issue in rustfs/backlog so a stalled chain is visible.
|
|
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
run: |
|
|
set -uo pipefail
|
|
if [ -z "${GH_TOKEN:-}" ]; then
|
|
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
|
exit 1
|
|
fi
|
|
DISPATCHED=0
|
|
for attempt in 1 2 3; do
|
|
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
|
-f event_type='rustfs-chain-security' \
|
|
-F 'client_payload[from_suite]=pool'; then
|
|
echo "dispatched next suite Security (attempt ${attempt})"
|
|
DISPATCHED=1
|
|
break
|
|
fi
|
|
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
|
sleep "${attempt}0"
|
|
done
|
|
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
|
echo "ERROR: functional chain stalled: could not dispatch Security after 3 attempts" >&2
|
|
TITLE="[functional][chain] stalled after pool (run ${GITHUB_RUN_ID})"
|
|
BODY_FILE="$(mktemp)"
|
|
{
|
|
echo "The functional chain could not hand off from **pool** to **Security** after 3 attempts."
|
|
echo ""
|
|
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
|
echo "- Expected next event: 'rustfs-chain-security'"
|
|
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
|
echo "- Recovery: re-dispatch manually with"
|
|
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
|
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-security'"
|
|
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
|
} > "${BODY_FILE}"
|
|
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
|
--body-file "${BODY_FILE}" --label functional-test \
|
|
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
|
|| echo "could not file the stall alert issue either; check the token" >&2
|
|
exit 1
|
|
fi
|
|
|
|
- name: Notify on failure
|
|
if: failure()
|
|
run: |
|
|
echo "RustFS pool expansion test failed"
|
|
echo "Package source: ${{ inputs.package_url || inputs.rustfs_version || 'nightly (R2 latest)' }}"
|
|
echo "See the uploaded log artifact for details."
|
|
|
|
- name: Record chain evidence
|
|
id: chain_record
|
|
if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
|
|
env:
|
|
CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
|
|
CHAIN_JOB_STATUS: ${{ job.status }}
|
|
CHAIN_TEST_OUTCOME: ${{ steps.pool_test.outcome }}
|
|
CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
|
run: >-
|
|
python3 scripts/functional_chain_evidence.py record --suite pool
|
|
--report "${POOL_ARTIFACT_DIR}/pool-steps.md"
|
|
--output "${RUNNER_TEMP}/chain-pool-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/pool.json"
|
|
|
|
- name: Upload chain evidence
|
|
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: functional-chain-pool-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: ${{ runner.temp }}/chain-pool-${{ github.run_id }}-${{ github.run_attempt }}/pool.json
|
|
if-no-files-found: error
|