mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-05 12:57:42 +00:00
feat(ecstore): refresh issue-787 phase-3 validation track (#4106)
* feat: isolate list objects quorum config * perf: add list gather observability for issue-785 * feat: add list-path observability for storage layers * chore: add issue 785 list observability acceptance runner * fix: avoid moved fields in list_path logging * test: add marker replay idempotence checks for list pagination * feat(ecstore): add list marker v2 cache id * Revert "feat(ecstore): add list marker v2 cache id" This reverts commit980c364e02. * Reapply "feat(ecstore): add list marker v2 cache id" This reverts commit766d2cee65. * feat(ecstore): propagate list cache cursor in list layers * test: add versioned continuation fallback tests * test: add live pagination continuity smoke mode * chore: run cargo fmt on issue-4003 branch * build: fix bash3 compatibility in live smoke * test(ecstore): simplify bool marker test assertion * refactor(ecstore): unify list pagination state machine * test: add issue 787 quorum validation runner * build: support bash32 in issue-787 runner * docs: add issue-787 benchmark evidence
This commit is contained in:
houseme
Executable
+348
@@ -0,0 +1,348 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Validation runner for rustfs/backlog#787: list quorum tuning and index-readiness baseline
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/issue-787-list-quorum-$(date +%Y%m%d-%H%M%S)}"
|
||||
SKIP_LIVE="${SKIP_LIVE:-true}"
|
||||
ENDPOINT="${ENDPOINT:-http://127.0.0.1:9000}"
|
||||
LIVE_LIST_BUCKET="${LIVE_LIST_BUCKET:-}"
|
||||
LIVE_LIST_PREFIX="${LIVE_LIST_PREFIX:-}"
|
||||
LIVE_LIST_MAX_KEYS="${LIVE_LIST_MAX_KEYS:-1000}"
|
||||
LIVE_LIST_REGION="${LIVE_LIST_REGION:-us-east-1}"
|
||||
MODE_LIST="${MODE_LIST:-strict,optimal,reduced}"
|
||||
SERVER_RESTART_CMD="${SERVER_RESTART_CMD:-}"
|
||||
SERVER_WAIT_SECONDS="${SERVER_WAIT_SECONDS:-20}"
|
||||
|
||||
log_info() { printf '[INFO] %s\n' "$*"; }
|
||||
log_warn() { printf '[WARN] %s\n' "$*"; }
|
||||
log_error() { printf '[ERROR] %s\n' "$*" >&2; }
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/validate_issue_787_list_quorum.sh [--full] [--no-skip-live]
|
||||
|
||||
Environment:
|
||||
OUT_DIR output directory (default: target/issue-787-list-quorum-<ts>)
|
||||
SKIP_LIVE skip live S3 endpoint checks when true (default: true)
|
||||
ENDPOINT rustfs endpoint for live checks (default: http://127.0.0.1:9000)
|
||||
LIVE_LIST_BUCKET bucket used for live pagination smoke (required when running live)
|
||||
LIVE_LIST_PREFIX optional prefix for list smoke (default: empty)
|
||||
LIVE_LIST_MAX_KEYS max-keys for live smoke page samples (default: 1000)
|
||||
LIVE_LIST_REGION signing region for curl (default: us-east-1)
|
||||
MODE_LIST comma-separated list of list quorum modes (default: strict,optimal,reduced)
|
||||
SERVER_RESTART_CMD optional command to restart server with RUSTFS_TUNING_ENV_FILE
|
||||
SERVER_WAIT_SECONDS seconds to wait after restart command before next checks (default: 20)
|
||||
|
||||
Run modes:
|
||||
--full or --no-skip-live run live checks
|
||||
--help this help
|
||||
USAGE
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
log_error "command not found: $1"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
timestamp_ms() {
|
||||
python3 - <<'PY'
|
||||
import time
|
||||
print(int(time.time() * 1000))
|
||||
PY
|
||||
}
|
||||
|
||||
extract_xml_keys() {
|
||||
local xml_text="$1"
|
||||
printf '%s\n' "$xml_text" | sed -n 's:.*<Key>\(.*\)</Key>.*:\1:p'
|
||||
}
|
||||
|
||||
wait_for_health() {
|
||||
local waited=0
|
||||
while (( waited < SERVER_WAIT_SECONDS )); do
|
||||
if curl -fsS "${ENDPOINT}/health/ready" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
waited=$(( waited + 1 ))
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
run_unit_checks() {
|
||||
log_info "Running focused rustfs-ecstore tests for #787 list quorum tuning"
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
local test_log="$OUT_DIR/issue-787-ecstore-tests.log"
|
||||
: > "$test_log"
|
||||
|
||||
local tests=(
|
||||
list_path_gather_results_returns_after_limit_without_waiting_for_input_close
|
||||
list_path_marker_parser_uses_trailing_cache_tag
|
||||
normalize_list_quorum_accepts_supported_values
|
||||
normalize_list_quorum_falls_back_to_strict
|
||||
list_quorum_from_env_defaults_to_strict
|
||||
list_quorum_from_env_honors_supported_value
|
||||
list_objects_quorum_from_env_defaults_to_optimal
|
||||
list_objects_quorum_from_env_honors_supported_value
|
||||
)
|
||||
|
||||
local test
|
||||
for test in "${tests[@]}"; do
|
||||
echo "[TEST] $test" | tee -a "$test_log"
|
||||
(cd "$PROJECT_ROOT" && cargo test -p rustfs-ecstore "$test" -- --nocapture | tee -a "$test_log")
|
||||
done
|
||||
|
||||
log_info "Unit test log: $test_log"
|
||||
}
|
||||
|
||||
run_static_checks() {
|
||||
log_info "Running #787 static checks for list-quorum and list path hot path"
|
||||
local static_log="$OUT_DIR/static-checks.log"
|
||||
: > "$static_log"
|
||||
|
||||
if rg -n "ENV_API_LIST_OBJECTS_QUORUM|list_objects_quorum_from_env\(|ask_disks: list_objects_quorum_from_env\(\)" "$PROJECT_ROOT/crates/ecstore/src/store/list_objects.rs" | tee -a "$static_log"; then
|
||||
log_info "Required list quorum symbols found in list_objects.rs"
|
||||
else
|
||||
log_error "list quorum symbols missing in list_objects.rs"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if rg -n "store_list_objects_list_merged|sets_list_objects_list_merged|set_disks_list_objects_list_path|store list_merged finished|sets list_merged finished|set_disks list_path selected listing quorum" "$PROJECT_ROOT/crates/ecstore/src/store/list_objects.rs" | tee -a "$static_log"; then
|
||||
log_info "List path merged/path logging symbols found"
|
||||
else
|
||||
log_error "List path merged symbols missing, static guard failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if rg -n "index-backed|list index|list index path|list_index" "$PROJECT_ROOT/crates/ecstore/src/store/list_objects.rs" >/dev/null 2>&1; then
|
||||
log_info "Index-backed scaffold symbols detected in list_objects.rs"
|
||||
else
|
||||
log_warn "Index-backed symbols not yet present in list_objects.rs (expected for current phase)"
|
||||
fi
|
||||
|
||||
log_info "Static check log: $static_log"
|
||||
}
|
||||
|
||||
run_metrics_context() {
|
||||
log_info "Collecting local metric symbol index for list-quorum review"
|
||||
local metric_log="$OUT_DIR/list-quorum-metrics-context.log"
|
||||
|
||||
if rg -n "record_stage_duration\(\"(store|sets|set_disks)_list_objects|store_list_objects_gather|set_disks_list_objects_list_path\"" "$PROJECT_ROOT/crates/ecstore/src/store/list_objects.rs" > "$metric_log"; then
|
||||
log_info "Metric context saved: $metric_log"
|
||||
else
|
||||
log_error "No metric symbols found in list_objects.rs"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_live_listing_two_page_smoke() {
|
||||
local mode="$1"
|
||||
local log_file="$OUT_DIR/live-${mode}.log"
|
||||
: > "$log_file"
|
||||
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
log_warn "python3 missing, skip live smoke"
|
||||
echo "live pagination smoke: skipped (missing python3)" | tee -a "$log_file"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local access_key="${RUSTFS_ACCESS_KEY:-${WARP_ACCESS_KEY:-}}"
|
||||
local secret_key="${RUSTFS_SECRET_KEY:-${WARP_SECRET_KEY:-}}"
|
||||
if [[ -z "$access_key" || -z "$secret_key" ]]; then
|
||||
log_warn "Missing credentials, skip live smoke"
|
||||
echo "live pagination smoke: skipped (missing credentials)" | tee -a "$log_file"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ -z "$LIVE_LIST_BUCKET" ]]; then
|
||||
log_warn "LIVE_LIST_BUCKET is empty, skip live smoke"
|
||||
echo "live pagination smoke: skipped (bucket not configured)" | tee -a "$log_file"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! curl -fsS "${ENDPOINT}/health/ready" >/dev/null 2>&1; then
|
||||
log_error "health check failed: ${ENDPOINT}/health/ready"
|
||||
echo "live pagination smoke: failed (health check)" | tee -a "$log_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local base_path="$LIVE_LIST_BUCKET"
|
||||
if [[ -n "$LIVE_LIST_PREFIX" ]]; then
|
||||
base_path="${LIVE_LIST_BUCKET}/${LIVE_LIST_PREFIX}"
|
||||
fi
|
||||
|
||||
local base_url="${ENDPOINT}/${base_path}?list-type=2&max-keys=${LIVE_LIST_MAX_KEYS}"
|
||||
local sig_target="aws:amz:${LIVE_LIST_REGION}:s3"
|
||||
|
||||
local start_ms page1 page2 token1 page1_key_count page2_key_count
|
||||
start_ms="$(timestamp_ms)"
|
||||
page1=$(curl -fsS \
|
||||
--user "${access_key}:${secret_key}" \
|
||||
--aws-sigv4 "$sig_target" \
|
||||
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
||||
"$base_url")
|
||||
page1_duration_ms="$(( $(timestamp_ms) - start_ms ))"
|
||||
|
||||
page1_key_count=$(extract_xml_keys "$page1" | grep -cv '^$' || true)
|
||||
token1=$(printf '%s' "$page1" | sed -n 's:.*<NextContinuationToken>\(.*\)</NextContinuationToken>.*:\1:p' | head -n 1)
|
||||
local -a page1_keys=()
|
||||
while IFS= read -r key; do
|
||||
[[ -n "$key" ]] || continue
|
||||
page1_keys+=("$key")
|
||||
done < <(extract_xml_keys "$page1")
|
||||
|
||||
log_info "Mode=${mode} first page keys=${page1_key_count} duration=${page1_duration_ms}ms" | tee -a "$log_file"
|
||||
|
||||
if [[ -z "$token1" ]]; then
|
||||
log_info "Mode=${mode} live pagination smoke: single page only" | tee -a "$log_file"
|
||||
echo "live pagination smoke: pass (single page, key_count=${page1_key_count}, duration_ms=${page1_duration_ms})" | tee -a "$log_file"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local encoded_token
|
||||
encoded_token=$(printf '%s' "$token1" | python3 - <<'PY'
|
||||
import urllib.parse,sys
|
||||
print(urllib.parse.quote(sys.stdin.read().strip(), safe=""))
|
||||
PY)
|
||||
|
||||
start_ms="$(timestamp_ms)"
|
||||
page2=$(curl -fsS \
|
||||
--user "${access_key}:${secret_key}" \
|
||||
--aws-sigv4 "$sig_target" \
|
||||
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
||||
"${base_url}&continuation-token=${encoded_token}")
|
||||
page2_duration_ms="$(( $(timestamp_ms) - start_ms ))"
|
||||
|
||||
page2_key_count=$(extract_xml_keys "$page2" | grep -cv '^$' || true)
|
||||
|
||||
if (( page2_key_count == 0 )); then
|
||||
log_error "Mode=${mode} live smoke: page2 empty"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local duplicate=0
|
||||
local -a dkeys=()
|
||||
local -a page2_keys=()
|
||||
while IFS= read -r key; do
|
||||
[[ -n "$key" ]] || continue
|
||||
page2_keys+=("$key")
|
||||
done < <(extract_xml_keys "$page2")
|
||||
|
||||
for key in "${page2_keys[@]}"; do
|
||||
if printf '%s\n' "${page1_keys[@]}" | grep -Fx -- "$key" >/dev/null 2>&1; then
|
||||
duplicate=1
|
||||
dkeys+=("$key")
|
||||
fi
|
||||
done
|
||||
|
||||
if (( duplicate )); then
|
||||
log_error "Mode=${mode} live pagination duplicates detected: ${dkeys[*]}"
|
||||
echo "live pagination smoke: fail (duplicate keys)" | tee -a "$log_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Mode=${mode}" | tee -a "$log_file"
|
||||
echo "first-page-count=${page1_key_count}" | tee -a "$log_file"
|
||||
echo "second-page-count=${page2_key_count}" | tee -a "$log_file"
|
||||
echo "page1-duration-ms=${page1_duration_ms}" | tee -a "$log_file"
|
||||
echo "page2-duration-ms=${page2_duration_ms}" | tee -a "$log_file"
|
||||
echo "live pagination smoke: pass (two-page continuity)" | tee -a "$log_file"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
run_mode_series() {
|
||||
if [[ -z "${MODE_LIST}" ]]; then
|
||||
log_warn "MODE_LIST is empty; skip mode series"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local mode
|
||||
local mode_count=0
|
||||
IFS=',' read -r -a modes <<< "$MODE_LIST"
|
||||
|
||||
for mode in "${modes[@]}"; do
|
||||
[[ -z "$mode" ]] && continue
|
||||
log_info "Live check for list quorum mode=${mode}"
|
||||
mode=$(printf '%s' "$mode" | tr -d '[:space:]')
|
||||
|
||||
if [[ -n "$SERVER_RESTART_CMD" ]]; then
|
||||
local mode_cmd=""
|
||||
local mode_env_file=""
|
||||
|
||||
if [[ "$mode" == "default" ]]; then
|
||||
mode_cmd="$SERVER_RESTART_CMD"
|
||||
else
|
||||
mode_env_file="$OUT_DIR/.issue787-${mode}.env"
|
||||
printf 'RUSTFS_LIST_OBJECTS_QUORUM=%s\n' "$mode" > "$mode_env_file"
|
||||
mode_cmd="RUSTFS_TUNING_ENV_FILE=${mode_env_file} $SERVER_RESTART_CMD"
|
||||
fi
|
||||
|
||||
log_info "Restarting server for mode=${mode}"
|
||||
(cd "$PROJECT_ROOT" && eval "$mode_cmd")
|
||||
|
||||
if ! wait_for_health; then
|
||||
log_error "health check timeout during mode=${mode} restart"
|
||||
return 1
|
||||
fi
|
||||
elif (( mode_count > 0 )); then
|
||||
log_warn "SERVER_RESTART_CMD unset, mode=${mode} does not auto-switch. Skip remaining modes."
|
||||
break
|
||||
fi
|
||||
|
||||
mode_count=$(( mode_count + 1 ))
|
||||
|
||||
run_live_listing_two_page_smoke "$mode"
|
||||
done
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--full|--no-skip-live)
|
||||
SKIP_LIVE="false"
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
log_error "unknown arg: $1"
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
|
||||
require_cmd cargo
|
||||
require_cmd rg
|
||||
require_cmd tee
|
||||
require_cmd curl
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
run_unit_checks
|
||||
run_static_checks
|
||||
run_metrics_context
|
||||
|
||||
if [[ "$SKIP_LIVE" != "true" ]]; then
|
||||
require_cmd python3
|
||||
run_mode_series
|
||||
fi
|
||||
|
||||
log_info "Validation summary:"
|
||||
log_info " - logs: $OUT_DIR"
|
||||
log_info " - status: pass"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user