mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-29 08:27:06 +00:00
test(pool): cover rebalance retry and cold-start recovery (#6720)
* test(pool): fix warp log path and retry rebalance start - warp writes now use a unique mktemp log file instead of a fixed /tmp/rustfs-warp.log: the runner user could not write the stale root-owned file, which made the background warp process die instantly (warp never ran). The workflow uploads /tmp/rustfs-warp.*.log. - rebalance start is retried (6x, 20s apart): nightly builds gate rebalance activation on a live cross-pool fence fleet capability proof that takes ~10-20s to re-establish after a pool joins. Verified live: attempt 1 fails with 500 'pool activation requires a live fleet capability proof', attempt 2 succeeds. * test(pool): annotate known server-side issues in failure output When a node fails to start, grab the rustfs journal tail and match known server-side error signatures (e.g. the fleet capability proof cold-start regression, rustfs/backlog#2031), printing a hint with the tracking issue. Also annotate the rebalance-start retry exhaustion and the rc.3 decommission metacache-listing failure with actionable guidance. * fix(ecstore): defer rebalance activation without fleet proof --------- Co-authored-by: 马登山 <cxymds@qq.com> Co-authored-by: cxymds <cxymds@gmail.com>
This commit is contained in:
hector
@@ -93,6 +93,9 @@ WARP_BUCKET="test-10mb"
|
||||
WARP_OBJ_SIZE="100MiB"
|
||||
WARP_CONCURRENT=32
|
||||
WARP_DURATION="5m"
|
||||
# Warp log path; empty = auto-created unique temp file (the runner user may
|
||||
# not be able to write a shared /tmp path owned by another user).
|
||||
WARP_LOG_FILE="${RUSTFS_WARP_LOG_FILE:-}"
|
||||
STORAGE_THRESHOLD=85 # stop writing when usage reaches N% (note suggests 80-85)
|
||||
POLL_INTERVAL=30 # status polling interval (seconds)
|
||||
|
||||
@@ -102,6 +105,8 @@ DECOMMISSION_TIMEOUT=86400
|
||||
SERVICE_TIMEOUT=300
|
||||
DECOMMISSION_RETRIES=3 # auto clear+retry attempts after a failed decommission
|
||||
DECOMMISSION_RETRY_DELAY=30 # delay between retries (seconds)
|
||||
REBALANCE_START_RETRIES=6 # rebalance start retries (fleet proof may take ~10-20s after a topology change)
|
||||
REBALANCE_START_RETRY_DELAY=20 # delay between rebalance start retries (seconds)
|
||||
|
||||
# Pool to decommission (zero-based; 0 in the note)
|
||||
DECOMMISSION_POOL_ID=0
|
||||
@@ -325,9 +330,45 @@ wait_service_active() {
|
||||
sleep 5
|
||||
waited=$((waited + 5))
|
||||
done
|
||||
diagnose_node_start_failure "${node}"
|
||||
die "${node}: timed out waiting for ${RUSTFS_SERVICE} (${SERVICE_TIMEOUT}s)"
|
||||
}
|
||||
|
||||
# Known server-side issues the test can hit. Format:
|
||||
# "<error signature>|<tracking>|<hint>"
|
||||
KNOWN_SERVER_ISSUES=(
|
||||
"pool activation requires a live fleet capability proof|rustfs/backlog#2031|server-side cold-start recovery is covered by this PR; if this appears, collect node journals and treat it as a regression"
|
||||
)
|
||||
|
||||
# Print a hint when $1 matches a known server-side issue signature.
|
||||
hint_server_issue() {
|
||||
local text="$1" entry sig tracking hint
|
||||
for entry in "${KNOWN_SERVER_ISSUES[@]}"; do
|
||||
sig="${entry%%|*}"
|
||||
tracking="${entry#*|}"
|
||||
hint="${tracking#*|}"
|
||||
tracking="${tracking%%|*}"
|
||||
if printf '%s' "${text}" | grep -qiF "${sig}"; then
|
||||
printf '\033[1;33m[KNOWN SERVER ISSUE]\033[0m %s (%s): %s\n' "${sig}" "${tracking}" "${hint}" >&2
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Fetch the journal tail from a node whose service failed to start and
|
||||
# annotate known server-side issues.
|
||||
diagnose_node_start_failure() {
|
||||
local node="$1" journal
|
||||
if ! journal="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \
|
||||
"SUDO=\"\"; [ \"\$(id -u)\" -ne 0 ] && SUDO=\"sudo -n\"; \${SUDO} journalctl -u ${RUSTFS_SERVICE} --no-pager -n 60 2>/dev/null || true")"; then
|
||||
journal="unable to collect journal (SSH command failed)"
|
||||
fi
|
||||
printf '%s\n' "--- ${node}: ${RUSTFS_SERVICE} journal (last 60 lines) ---" >&2
|
||||
printf '%s\n' "${journal}" >&2
|
||||
hint_server_issue "${journal}" || true
|
||||
}
|
||||
|
||||
# Generate the /etc/default/rustfs content
|
||||
rustfs_config_body() {
|
||||
local volumes="$1"
|
||||
@@ -406,9 +447,13 @@ service_action() {
|
||||
local action="$1" node="$2"
|
||||
log "${node}: systemctl ${action} ${RUSTFS_SERVICE}"
|
||||
if [ "${DRY_RUN}" -eq 1 ]; then return 0; fi
|
||||
ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \
|
||||
"if [ \"\$(id -u)\" -ne 0 ]; then sudo -n systemctl ${action} ${RUSTFS_SERVICE}; else systemctl ${action} ${RUSTFS_SERVICE}; fi" \
|
||||
|| die "${node}: systemctl ${action} failed"
|
||||
if ! ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \
|
||||
"if [ \"\$(id -u)\" -ne 0 ]; then sudo -n systemctl ${action} ${RUSTFS_SERVICE}; else systemctl ${action} ${RUSTFS_SERVICE}; fi"; then
|
||||
if [ "${action}" = "start" ]; then
|
||||
diagnose_node_start_failure "${node}"
|
||||
fi
|
||||
die "${node}: systemctl ${action} failed"
|
||||
fi
|
||||
}
|
||||
|
||||
service_action_all() {
|
||||
@@ -587,6 +632,29 @@ wait_rebalance() {
|
||||
die "timed out waiting for rebalance (${REBALANCE_TIMEOUT}s)"
|
||||
}
|
||||
|
||||
# Start rebalance via the admin API. Nightly builds gate rebalance activation
|
||||
# on a live cross-pool fence fleet capability proof that is re-established
|
||||
# shortly after a pool joins, so retry a few times before failing.
|
||||
start_rebalance_with_retry() {
|
||||
local attempts="${REBALANCE_START_RETRIES}" delay="${REBALANCE_START_RETRY_DELAY}" attempt=1 body code id
|
||||
while :; do
|
||||
body="$(admin_api POST /rustfs/admin/v3/rebalance/start "")"
|
||||
code="$(admin_api_code)"
|
||||
if [ "${code}" = "200" ]; then
|
||||
id="$(printf '%s' "${body}" | jq -r '.id // empty')"
|
||||
log "rebalance started: id=${id}"
|
||||
return 0
|
||||
fi
|
||||
warn "rebalance start attempt ${attempt}/${attempts} failed (HTTP ${code}): ${body}"
|
||||
if [ "${attempt}" -ge "${attempts}" ]; then
|
||||
hint_server_issue "${body}" || true
|
||||
die "rebalance start failed after ${attempts} attempts (see last error above)"
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
sleep "${delay}"
|
||||
done
|
||||
}
|
||||
|
||||
# Print a detailed decommission failure/progress report for one pool
|
||||
print_decommission_detail() {
|
||||
local body="$1" pool_id="$2" label="$3"
|
||||
@@ -817,16 +885,18 @@ step4_write_data() {
|
||||
log "DRY-RUN: monitoring storage usage until ${STORAGE_THRESHOLD}%"
|
||||
return 0
|
||||
fi
|
||||
log "starting warp writes (background)..."
|
||||
local warp_log warp_pid
|
||||
warp_log="${WARP_LOG_FILE:-$(mktemp "${TMPDIR:-/tmp}/rustfs-warp.XXXXXX.log")}"
|
||||
log "starting warp writes (background), log: ${warp_log}"
|
||||
warp put --host "${API_ENDPOINT#http://}" \
|
||||
--bucket "${WARP_BUCKET}" \
|
||||
--access-key "${ACCESS_KEY}" \
|
||||
--secret-key "${SECRET_KEY}" \
|
||||
--obj.size "${WARP_OBJ_SIZE}" \
|
||||
--concurrent "${WARP_CONCURRENT}" \
|
||||
--noprefix --duration "${WARP_DURATION}" --noclear >/tmp/rustfs-warp.log 2>&1 &
|
||||
local warp_pid=$!
|
||||
log "warp PID=${warp_pid}, log /tmp/rustfs-warp.log"
|
||||
--noprefix --duration "${WARP_DURATION}" --noclear >"${warp_log}" 2>&1 &
|
||||
warp_pid=$!
|
||||
log "warp PID=${warp_pid}"
|
||||
trap 'kill "${warp_pid:-}" 2>/dev/null || true' EXIT
|
||||
monitor_storage "${STORAGE_THRESHOLD}" "${warp_pid}"
|
||||
kill "${warp_pid}" 2>/dev/null || true
|
||||
@@ -866,12 +936,8 @@ step5_expand_pool2() {
|
||||
step6_rebalance() {
|
||||
log "step 6: start data rebalance (admin API)"
|
||||
confirm "About to start rebalance (POST ${API_ENDPOINT}/rustfs/admin/v3/rebalance/start). Continue?"
|
||||
local body id
|
||||
if [ "${DRY_RUN}" -eq 0 ]; then
|
||||
body="$(admin_api POST /rustfs/admin/v3/rebalance/start "")"
|
||||
[ "$(admin_api_code)" = "200" ] || die "rebalance start failed (HTTP $(admin_api_code)): ${body}"
|
||||
id="$(printf '%s' "${body}" | jq -r '.id // empty')"
|
||||
log "rebalance started: id=${id}"
|
||||
start_rebalance_with_retry
|
||||
fi
|
||||
wait_rebalance
|
||||
}
|
||||
@@ -894,12 +960,8 @@ step7_expand_pool3() {
|
||||
step8_rebalance() {
|
||||
log "step 8: start data rebalance (admin API)"
|
||||
confirm "About to start rebalance (POST ${API_ENDPOINT}/rustfs/admin/v3/rebalance/start). Continue?"
|
||||
local body id
|
||||
if [ "${DRY_RUN}" -eq 0 ]; then
|
||||
body="$(admin_api POST /rustfs/admin/v3/rebalance/start "")"
|
||||
[ "$(admin_api_code)" = "200" ] || die "rebalance start failed (HTTP $(admin_api_code)): ${body}"
|
||||
id="$(printf '%s' "${body}" | jq -r '.id // empty')"
|
||||
log "rebalance started: id=${id}"
|
||||
start_rebalance_with_retry
|
||||
fi
|
||||
wait_rebalance
|
||||
}
|
||||
@@ -928,6 +990,7 @@ step9_decommission() {
|
||||
break
|
||||
fi
|
||||
if [ "${attempt}" -ge "${DECOMMISSION_RETRIES}" ]; then
|
||||
warn "if the source bucket has many objects and the tested version is 1.0.0-rc.3, this is the known metacache-listing decommission bug; remove the test bucket (rc rb --force rustfs/${WARP_BUCKET}) or lower --storage-threshold, then re-run step 9"
|
||||
die "pool ${DECOMMISSION_POOL_ID} still failed after ${attempt} attempts; investigate manually (POST ${API_ENDPOINT}/rustfs/admin/v3/pools/clear?by-id=true&pool=${DECOMMISSION_POOL_ID} to reset)"
|
||||
fi
|
||||
warn "attempt ${attempt} failed; clearing metadata and retrying in ${DECOMMISSION_RETRY_DELAY}s"
|
||||
|
||||
Reference in New Issue
Block a user