mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
perf: add list objects quorum tuning knob (#4072)
This commit is contained in:
houseme
File diff suppressed because it is too large
Load Diff
Executable
+152
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Acceptance runner for rustfs/backlog#785 and PR #4072
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/issue-785-acceptance-$(date +%Y%m%d-%H%M%S)}"
|
||||
SKIP_LIVE="${SKIP_LIVE:-true}"
|
||||
ENDPOINT="${ENDPOINT:-http://127.0.0.1:9000}"
|
||||
|
||||
log_info() { printf '[INFO] %s\n' "$*"; }
|
||||
log_warn() { printf '[WARN] %s\n' "$*"; }
|
||||
log_error() { printf '[ERROR] %s\n' "$*" >&2; }
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/validate_issue_785_list_objects.sh [--full] [--no-skip-live]
|
||||
|
||||
Environment:
|
||||
OUT_DIR output directory (default: target/issue-785-acceptance-<ts>)
|
||||
SKIP_LIVE skip live S3 endpoint checks when true (default: true)
|
||||
ENDPOINT rustfs endpoint used by optional live checks (default: http://127.0.0.1:9000)
|
||||
USAGE
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
log_error "command not found: $1"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_unit_checks() {
|
||||
log_info "Running focused rustfs-ecstore tests for list_objects hot path"
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
local test_log="$OUT_DIR/issue-785-ecstore-tests.log"
|
||||
: > "$test_log"
|
||||
|
||||
local tests=(
|
||||
list_path_gather_results_returns_after_limit_without_waiting_for_input_close
|
||||
list_path_gather_results_keeps_marker_entry_for_version_marker_listing
|
||||
list_path_gather_results_skips_marker_entry_by_default
|
||||
list_path_forward_past_is_idempotent_for_same_marker
|
||||
list_path_parse_marker_replay_still_stable
|
||||
normalize_list_quorum_falls_back_to_strict
|
||||
list_objects_quorum_from_env_defaults_to_optimal
|
||||
)
|
||||
|
||||
local test
|
||||
for test in "${tests[@]}"; do
|
||||
echo "[TEST] $test" | tee -a "$test_log"
|
||||
(cd "$PROJECT_ROOT" && cargo test -p rustfs-ecstore "$test" -- --nocapture | tee -a "$test_log")
|
||||
done
|
||||
|
||||
log_info "Unit test log: $test_log"
|
||||
}
|
||||
|
||||
run_static_checks() {
|
||||
log_info "Running acceptance-focused static checks"
|
||||
local static_log="$OUT_DIR/static-checks.log"
|
||||
|
||||
: > "$static_log"
|
||||
|
||||
if rg -n "store_list_objects_list_path|store_list_objects_list_merged|sets_list_objects_list_path|sets_list_objects_list_merged|set_disks_list_objects_list_path|store_list_objects_gather" \
|
||||
"$PROJECT_ROOT/crates/ecstore/src/store/list_objects.rs" | tee -a "$static_log"; then
|
||||
log_info "Metric and stage names exist in list_objects.rs"
|
||||
else
|
||||
log_error "Required metric symbols not found in list_objects.rs"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "Static check log: $static_log"
|
||||
}
|
||||
|
||||
run_live_smoke() {
|
||||
if [[ "$SKIP_LIVE" == "true" ]]; then
|
||||
log_warn "SKIP_LIVE=true, skipping live endpoint checks"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "Running optional live readiness check: $ENDPOINT"
|
||||
if ! curl -fsS "${ENDPOINT}/health/ready" >/dev/null 2>&1; then
|
||||
log_error "Health ready endpoint check failed: ${ENDPOINT}/health/ready"
|
||||
return 1
|
||||
fi
|
||||
log_info "Live endpoint reachable"
|
||||
|
||||
log_warn "Optional live S3 pagination/candidates check is intentionally not enforced in this runner."
|
||||
log_warn "Please run your preferred client workload (mc/warp/rclone) and verify page continuity + duplicates manually."
|
||||
}
|
||||
|
||||
run_metrics_context() {
|
||||
log_info "Collecting local metric symbol index for review"
|
||||
local metric_log="$OUT_DIR/issue-785-metrics-context.log"
|
||||
if rg -n "record_stage_duration\(" "$PROJECT_ROOT/crates/ecstore/src/store/list_objects.rs" > "$metric_log"; then
|
||||
log_info "Metric context saved: $metric_log"
|
||||
else
|
||||
log_error "No metric stage context found"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--full)
|
||||
SKIP_LIVE="false"
|
||||
shift
|
||||
;;
|
||||
--no-skip-live)
|
||||
SKIP_LIVE="false"
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
log_error "unknown arg: $1"
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
:
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
require_cmd cargo
|
||||
require_cmd rg
|
||||
require_cmd tee
|
||||
require_cmd curl
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
run_unit_checks
|
||||
run_static_checks
|
||||
run_metrics_context
|
||||
|
||||
if [[ "$SKIP_LIVE" != "true" ]]; then
|
||||
run_live_smoke
|
||||
fi
|
||||
|
||||
log_info "Validation summary:"
|
||||
log_info " - logs: $OUT_DIR"
|
||||
log_info " - status: pass"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Executable
+232
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Acceptance runner for rustfs/backlog#786 and PR #4072 follow-up
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/issue-786-acceptance-$(date +%Y%m%d-%H%M%S)}"
|
||||
SKIP_LIVE="${SKIP_LIVE:-true}"
|
||||
ENDPOINT="${ENDPOINT:-http://127.0.0.1:9000}"
|
||||
LIVE_LIST_BUCKET="${LIVE_LIST_BUCKET:-}"
|
||||
LIVE_LIST_MAX_KEYS="${LIVE_LIST_MAX_KEYS:-2}"
|
||||
LIVE_LIST_REGION="${LIVE_LIST_REGION:-us-east-1}"
|
||||
|
||||
log_info() { printf '[INFO] %s\n' "$*"; }
|
||||
log_warn() { printf '[WARN] %s\n' "$*"; }
|
||||
log_error() { printf '[ERROR] %s\n' "$*" >&2; }
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/validate_issue_786_list_objects.sh [--full] [--no-skip-live]
|
||||
|
||||
Environment:
|
||||
OUT_DIR output directory (default: target/issue-786-acceptance-<ts>)
|
||||
SKIP_LIVE skip live S3 endpoint checks when true (default: true)
|
||||
ENDPOINT rustfs endpoint used by optional live checks (default: http://127.0.0.1:9000)
|
||||
LIVE_LIST_BUCKET S3 bucket used for pagination smoke (required)
|
||||
LIVE_LIST_MAX_KEYS maximum keys used per page for smoke (default: 2)
|
||||
LIVE_LIST_REGION signing region for curl --aws-sigv4 (default: us-east-1)
|
||||
USAGE
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
log_error "command not found: $1"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_unit_checks() {
|
||||
log_info "Running focused rustfs-ecstore tests for issue-786 cursor fallback"
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
local test_log="$OUT_DIR/issue-786-ecstore-tests.log"
|
||||
: > "$test_log"
|
||||
|
||||
local tests=(
|
||||
list_path_marker_round_trip_preserves_set_index
|
||||
list_path_marker_parser_uses_trailing_cache_tag
|
||||
list_path_marker_parser_accepts_legacy_v1_tag
|
||||
list_path_marker_parser_return_tag_forces_cache_refresh
|
||||
list_path_marker_parser_recovers_from_corrupt_set_index
|
||||
)
|
||||
|
||||
local test
|
||||
for test in "${tests[@]}"; do
|
||||
echo "[TEST] $test" | tee -a "$test_log"
|
||||
(cd "$PROJECT_ROOT" && cargo test -p rustfs-ecstore "$test" -- --nocapture | tee -a "$test_log")
|
||||
done
|
||||
|
||||
log_info "Unit test log: $test_log"
|
||||
}
|
||||
|
||||
run_static_checks() {
|
||||
log_info "Running issue-786 static marker checks"
|
||||
local static_log="$OUT_DIR/static-checks.log"
|
||||
|
||||
: > "$static_log"
|
||||
|
||||
if rg -n "struct ListContinuationV2|return:\\]|rustfs_cache:v2|MARKER_TAG_VERSION" "$PROJECT_ROOT/crates/ecstore/src/store/list_objects.rs" | tee -a "$static_log"; then
|
||||
log_info "ListContinuationV2 symbols found in list_objects.rs"
|
||||
else
|
||||
log_error "Required cursor symbols not found in list_objects.rs"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "Static check log: $static_log"
|
||||
}
|
||||
|
||||
extract_xml_keys() {
|
||||
local xml_text="$1"
|
||||
printf '%s\n' "$xml_text" | sed -n 's:.*<Key>\(.*\)</Key>.*:\1:p'
|
||||
}
|
||||
|
||||
run_live_smoke() {
|
||||
if [[ "$SKIP_LIVE" == "true" ]]; then
|
||||
log_warn "SKIP_LIVE=true, skipping live endpoint checks"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local live_log="$OUT_DIR/live-pagination-smoke.log"
|
||||
: > "$live_log"
|
||||
|
||||
if ! curl -fsS "${ENDPOINT}/health/ready" >/dev/null 2>&1; then
|
||||
log_warn "Live endpoint not reachable: ${ENDPOINT}/health/ready"
|
||||
echo "live pagination smoke: skipped (endpoint not reachable)" | tee -a "$live_log"
|
||||
return 0
|
||||
fi
|
||||
log_info "Live endpoint reachable"
|
||||
|
||||
local access_key="${RUSTFS_ACCESS_KEY:-${WARP_ACCESS_KEY:-}}"
|
||||
local secret_key="${RUSTFS_SECRET_KEY:-${WARP_SECRET_KEY:-}}"
|
||||
if [[ -z "$access_key" || -z "$secret_key" ]]; then
|
||||
log_warn "Live pagination smoke skipped: missing RUSTFS_ACCESS_KEY / RUSTFS_SECRET_KEY or WARP_* alternatives"
|
||||
echo "live pagination smoke: skipped (missing credentials)" | tee -a "$live_log"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ -z "$LIVE_LIST_BUCKET" ]]; then
|
||||
log_warn "Live pagination smoke skipped: LIVE_LIST_BUCKET is empty"
|
||||
echo "live pagination smoke: skipped (bucket not configured)" | tee -a "$live_log"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
log_warn "Live pagination smoke skipped: python3 not available"
|
||||
echo "live pagination smoke: skipped (missing python3 parser)" | tee -a "$live_log"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local sigv4_target="aws:amz:${LIVE_LIST_REGION}:s3"
|
||||
local base_url="${ENDPOINT}/${LIVE_LIST_BUCKET}?list-type=2&max-keys=${LIVE_LIST_MAX_KEYS}"
|
||||
|
||||
local page1
|
||||
page1=$(curl -fsS \
|
||||
--user "${access_key}:${secret_key}" \
|
||||
--aws-sigv4 "$sigv4_target" \
|
||||
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
||||
"$base_url")
|
||||
|
||||
local token1
|
||||
token1=$(printf '%s' "$page1" | sed -n 's:.*<NextContinuationToken>\(.*\)</NextContinuationToken>.*:\1:p' | head -n 1)
|
||||
|
||||
local page1_key_count
|
||||
page1_key_count=$(printf '%s' "$page1" | sed -n 's:.*<Key>\(.*\)</Key>.*:\1:p' | grep -cv '^$')
|
||||
|
||||
local -a keys1 keys2
|
||||
while IFS= read -r key; do
|
||||
keys1+=("$key")
|
||||
done < <(extract_xml_keys "$page1")
|
||||
|
||||
if [[ -z "$token1" ]]; then
|
||||
log_info "Live pagination smoke: single page only (key count=${page1_key_count})"
|
||||
echo "live pagination smoke: pass (single page, key_count=${page1_key_count})" | tee -a "$live_log"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local token1_encoded
|
||||
token1_encoded=$(printf '%s' "$token1" | python3 -c 'import urllib.parse,sys; print(urllib.parse.quote(sys.stdin.read().strip(), safe=""))')
|
||||
|
||||
local page2
|
||||
page2=$(curl -fsS \
|
||||
--user "${access_key}:${secret_key}" \
|
||||
--aws-sigv4 "$sigv4_target" \
|
||||
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
||||
"${base_url}&continuation-token=${token1_encoded}")
|
||||
|
||||
local page2_key_count
|
||||
page2_key_count=$(printf '%s' "$page2" | sed -n 's:.*<Key>\(.*\)</Key>.*:\1:p' | grep -cv '^$')
|
||||
|
||||
if (( page2_key_count == 0 )); then
|
||||
log_error "Live pagination smoke: second page returned zero keys for token ${token1}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local duplicate
|
||||
local dkeys=()
|
||||
duplicate=0
|
||||
local key
|
||||
|
||||
while IFS= read -r key; do
|
||||
keys2+=("$key")
|
||||
if printf '%s\n' "${keys1[@]}" | grep -Fx -- "$key" >/dev/null 2>&1; then
|
||||
duplicate=1
|
||||
dkeys+=("$key")
|
||||
fi
|
||||
done < <(extract_xml_keys "$page2")
|
||||
|
||||
if (( duplicate )); then
|
||||
log_error "Live pagination smoke detected duplicated keys across page transitions: ${dkeys[*]}"
|
||||
printf '%s\n' "${dkeys[@]}" | tee -a "$live_log"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "Live pagination smoke: first page=${page1_key_count}, second page=${page2_key_count}, token=${token1}"
|
||||
echo "live pagination smoke: pass (continuous two-page check)" | tee -a "$live_log"
|
||||
echo "page1=${page1_key_count}" | tee -a "$live_log"
|
||||
echo "page2=${page2_key_count}" | tee -a "$live_log"
|
||||
log_info "Live pagination smoke log: $live_log"
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--full|--no-skip-live)
|
||||
SKIP_LIVE="false"
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
log_error "unknown arg: $1"
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
require_cmd cargo
|
||||
require_cmd rg
|
||||
require_cmd tee
|
||||
require_cmd curl
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
run_unit_checks
|
||||
run_static_checks
|
||||
|
||||
if [[ "$SKIP_LIVE" != "true" ]]; then
|
||||
run_live_smoke
|
||||
fi
|
||||
|
||||
log_info "Validation summary:"
|
||||
log_info " - logs: $OUT_DIR"
|
||||
log_info " - status: pass"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user