mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-31 09:18:28 +00:00
perf: add list objects quorum tuning knob (#4072)
This commit is contained in:
houseme
File diff suppressed because it is too large
Load Diff
Executable
+152
@@ -0,0 +1,152 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Acceptance runner for rustfs/backlog#785 and PR #4072
|
||||||
|
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/issue-785-acceptance-$(date +%Y%m%d-%H%M%S)}"
|
||||||
|
SKIP_LIVE="${SKIP_LIVE:-true}"
|
||||||
|
ENDPOINT="${ENDPOINT:-http://127.0.0.1:9000}"
|
||||||
|
|
||||||
|
log_info() { printf '[INFO] %s\n' "$*"; }
|
||||||
|
log_warn() { printf '[WARN] %s\n' "$*"; }
|
||||||
|
log_error() { printf '[ERROR] %s\n' "$*" >&2; }
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Usage:
|
||||||
|
scripts/validate_issue_785_list_objects.sh [--full] [--no-skip-live]
|
||||||
|
|
||||||
|
Environment:
|
||||||
|
OUT_DIR output directory (default: target/issue-785-acceptance-<ts>)
|
||||||
|
SKIP_LIVE skip live S3 endpoint checks when true (default: true)
|
||||||
|
ENDPOINT rustfs endpoint used by optional live checks (default: http://127.0.0.1:9000)
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
require_cmd() {
|
||||||
|
if ! command -v "$1" >/dev/null 2>&1; then
|
||||||
|
log_error "command not found: $1"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
run_unit_checks() {
|
||||||
|
log_info "Running focused rustfs-ecstore tests for list_objects hot path"
|
||||||
|
mkdir -p "$OUT_DIR"
|
||||||
|
|
||||||
|
local test_log="$OUT_DIR/issue-785-ecstore-tests.log"
|
||||||
|
: > "$test_log"
|
||||||
|
|
||||||
|
local tests=(
|
||||||
|
list_path_gather_results_returns_after_limit_without_waiting_for_input_close
|
||||||
|
list_path_gather_results_keeps_marker_entry_for_version_marker_listing
|
||||||
|
list_path_gather_results_skips_marker_entry_by_default
|
||||||
|
list_path_forward_past_is_idempotent_for_same_marker
|
||||||
|
list_path_parse_marker_replay_still_stable
|
||||||
|
normalize_list_quorum_falls_back_to_strict
|
||||||
|
list_objects_quorum_from_env_defaults_to_optimal
|
||||||
|
)
|
||||||
|
|
||||||
|
local test
|
||||||
|
for test in "${tests[@]}"; do
|
||||||
|
echo "[TEST] $test" | tee -a "$test_log"
|
||||||
|
(cd "$PROJECT_ROOT" && cargo test -p rustfs-ecstore "$test" -- --nocapture | tee -a "$test_log")
|
||||||
|
done
|
||||||
|
|
||||||
|
log_info "Unit test log: $test_log"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_static_checks() {
|
||||||
|
log_info "Running acceptance-focused static checks"
|
||||||
|
local static_log="$OUT_DIR/static-checks.log"
|
||||||
|
|
||||||
|
: > "$static_log"
|
||||||
|
|
||||||
|
if rg -n "store_list_objects_list_path|store_list_objects_list_merged|sets_list_objects_list_path|sets_list_objects_list_merged|set_disks_list_objects_list_path|store_list_objects_gather" \
|
||||||
|
"$PROJECT_ROOT/crates/ecstore/src/store/list_objects.rs" | tee -a "$static_log"; then
|
||||||
|
log_info "Metric and stage names exist in list_objects.rs"
|
||||||
|
else
|
||||||
|
log_error "Required metric symbols not found in list_objects.rs"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "Static check log: $static_log"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_live_smoke() {
|
||||||
|
if [[ "$SKIP_LIVE" == "true" ]]; then
|
||||||
|
log_warn "SKIP_LIVE=true, skipping live endpoint checks"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "Running optional live readiness check: $ENDPOINT"
|
||||||
|
if ! curl -fsS "${ENDPOINT}/health/ready" >/dev/null 2>&1; then
|
||||||
|
log_error "Health ready endpoint check failed: ${ENDPOINT}/health/ready"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
log_info "Live endpoint reachable"
|
||||||
|
|
||||||
|
log_warn "Optional live S3 pagination/candidates check is intentionally not enforced in this runner."
|
||||||
|
log_warn "Please run your preferred client workload (mc/warp/rclone) and verify page continuity + duplicates manually."
|
||||||
|
}
|
||||||
|
|
||||||
|
run_metrics_context() {
|
||||||
|
log_info "Collecting local metric symbol index for review"
|
||||||
|
local metric_log="$OUT_DIR/issue-785-metrics-context.log"
|
||||||
|
if rg -n "record_stage_duration\(" "$PROJECT_ROOT/crates/ecstore/src/store/list_objects.rs" > "$metric_log"; then
|
||||||
|
log_info "Metric context saved: $metric_log"
|
||||||
|
else
|
||||||
|
log_error "No metric stage context found"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_args() {
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--full)
|
||||||
|
SKIP_LIVE="false"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--no-skip-live)
|
||||||
|
SKIP_LIVE="false"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
-h|--help)
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
log_error "unknown arg: $1"
|
||||||
|
usage
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
:
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
parse_args "$@"
|
||||||
|
require_cmd cargo
|
||||||
|
require_cmd rg
|
||||||
|
require_cmd tee
|
||||||
|
require_cmd curl
|
||||||
|
|
||||||
|
mkdir -p "$OUT_DIR"
|
||||||
|
|
||||||
|
run_unit_checks
|
||||||
|
run_static_checks
|
||||||
|
run_metrics_context
|
||||||
|
|
||||||
|
if [[ "$SKIP_LIVE" != "true" ]]; then
|
||||||
|
run_live_smoke
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "Validation summary:"
|
||||||
|
log_info " - logs: $OUT_DIR"
|
||||||
|
log_info " - status: pass"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Executable
+232
@@ -0,0 +1,232 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Acceptance runner for rustfs/backlog#786 and PR #4072 follow-up
|
||||||
|
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/issue-786-acceptance-$(date +%Y%m%d-%H%M%S)}"
|
||||||
|
SKIP_LIVE="${SKIP_LIVE:-true}"
|
||||||
|
ENDPOINT="${ENDPOINT:-http://127.0.0.1:9000}"
|
||||||
|
LIVE_LIST_BUCKET="${LIVE_LIST_BUCKET:-}"
|
||||||
|
LIVE_LIST_MAX_KEYS="${LIVE_LIST_MAX_KEYS:-2}"
|
||||||
|
LIVE_LIST_REGION="${LIVE_LIST_REGION:-us-east-1}"
|
||||||
|
|
||||||
|
log_info() { printf '[INFO] %s\n' "$*"; }
|
||||||
|
log_warn() { printf '[WARN] %s\n' "$*"; }
|
||||||
|
log_error() { printf '[ERROR] %s\n' "$*" >&2; }
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Usage:
|
||||||
|
scripts/validate_issue_786_list_objects.sh [--full] [--no-skip-live]
|
||||||
|
|
||||||
|
Environment:
|
||||||
|
OUT_DIR output directory (default: target/issue-786-acceptance-<ts>)
|
||||||
|
SKIP_LIVE skip live S3 endpoint checks when true (default: true)
|
||||||
|
ENDPOINT rustfs endpoint used by optional live checks (default: http://127.0.0.1:9000)
|
||||||
|
LIVE_LIST_BUCKET S3 bucket used for pagination smoke (required)
|
||||||
|
LIVE_LIST_MAX_KEYS maximum keys used per page for smoke (default: 2)
|
||||||
|
LIVE_LIST_REGION signing region for curl --aws-sigv4 (default: us-east-1)
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
require_cmd() {
|
||||||
|
if ! command -v "$1" >/dev/null 2>&1; then
|
||||||
|
log_error "command not found: $1"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
run_unit_checks() {
|
||||||
|
log_info "Running focused rustfs-ecstore tests for issue-786 cursor fallback"
|
||||||
|
mkdir -p "$OUT_DIR"
|
||||||
|
|
||||||
|
local test_log="$OUT_DIR/issue-786-ecstore-tests.log"
|
||||||
|
: > "$test_log"
|
||||||
|
|
||||||
|
local tests=(
|
||||||
|
list_path_marker_round_trip_preserves_set_index
|
||||||
|
list_path_marker_parser_uses_trailing_cache_tag
|
||||||
|
list_path_marker_parser_accepts_legacy_v1_tag
|
||||||
|
list_path_marker_parser_return_tag_forces_cache_refresh
|
||||||
|
list_path_marker_parser_recovers_from_corrupt_set_index
|
||||||
|
)
|
||||||
|
|
||||||
|
local test
|
||||||
|
for test in "${tests[@]}"; do
|
||||||
|
echo "[TEST] $test" | tee -a "$test_log"
|
||||||
|
(cd "$PROJECT_ROOT" && cargo test -p rustfs-ecstore "$test" -- --nocapture | tee -a "$test_log")
|
||||||
|
done
|
||||||
|
|
||||||
|
log_info "Unit test log: $test_log"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_static_checks() {
|
||||||
|
log_info "Running issue-786 static marker checks"
|
||||||
|
local static_log="$OUT_DIR/static-checks.log"
|
||||||
|
|
||||||
|
: > "$static_log"
|
||||||
|
|
||||||
|
if rg -n "struct ListContinuationV2|return:\\]|rustfs_cache:v2|MARKER_TAG_VERSION" "$PROJECT_ROOT/crates/ecstore/src/store/list_objects.rs" | tee -a "$static_log"; then
|
||||||
|
log_info "ListContinuationV2 symbols found in list_objects.rs"
|
||||||
|
else
|
||||||
|
log_error "Required cursor symbols not found in list_objects.rs"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "Static check log: $static_log"
|
||||||
|
}
|
||||||
|
|
||||||
|
extract_xml_keys() {
|
||||||
|
local xml_text="$1"
|
||||||
|
printf '%s\n' "$xml_text" | sed -n 's:.*<Key>\(.*\)</Key>.*:\1:p'
|
||||||
|
}
|
||||||
|
|
||||||
|
run_live_smoke() {
|
||||||
|
if [[ "$SKIP_LIVE" == "true" ]]; then
|
||||||
|
log_warn "SKIP_LIVE=true, skipping live endpoint checks"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local live_log="$OUT_DIR/live-pagination-smoke.log"
|
||||||
|
: > "$live_log"
|
||||||
|
|
||||||
|
if ! curl -fsS "${ENDPOINT}/health/ready" >/dev/null 2>&1; then
|
||||||
|
log_warn "Live endpoint not reachable: ${ENDPOINT}/health/ready"
|
||||||
|
echo "live pagination smoke: skipped (endpoint not reachable)" | tee -a "$live_log"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
log_info "Live endpoint reachable"
|
||||||
|
|
||||||
|
local access_key="${RUSTFS_ACCESS_KEY:-${WARP_ACCESS_KEY:-}}"
|
||||||
|
local secret_key="${RUSTFS_SECRET_KEY:-${WARP_SECRET_KEY:-}}"
|
||||||
|
if [[ -z "$access_key" || -z "$secret_key" ]]; then
|
||||||
|
log_warn "Live pagination smoke skipped: missing RUSTFS_ACCESS_KEY / RUSTFS_SECRET_KEY or WARP_* alternatives"
|
||||||
|
echo "live pagination smoke: skipped (missing credentials)" | tee -a "$live_log"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z "$LIVE_LIST_BUCKET" ]]; then
|
||||||
|
log_warn "Live pagination smoke skipped: LIVE_LIST_BUCKET is empty"
|
||||||
|
echo "live pagination smoke: skipped (bucket not configured)" | tee -a "$live_log"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! command -v python3 >/dev/null 2>&1; then
|
||||||
|
log_warn "Live pagination smoke skipped: python3 not available"
|
||||||
|
echo "live pagination smoke: skipped (missing python3 parser)" | tee -a "$live_log"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local sigv4_target="aws:amz:${LIVE_LIST_REGION}:s3"
|
||||||
|
local base_url="${ENDPOINT}/${LIVE_LIST_BUCKET}?list-type=2&max-keys=${LIVE_LIST_MAX_KEYS}"
|
||||||
|
|
||||||
|
local page1
|
||||||
|
page1=$(curl -fsS \
|
||||||
|
--user "${access_key}:${secret_key}" \
|
||||||
|
--aws-sigv4 "$sigv4_target" \
|
||||||
|
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
||||||
|
"$base_url")
|
||||||
|
|
||||||
|
local token1
|
||||||
|
token1=$(printf '%s' "$page1" | sed -n 's:.*<NextContinuationToken>\(.*\)</NextContinuationToken>.*:\1:p' | head -n 1)
|
||||||
|
|
||||||
|
local page1_key_count
|
||||||
|
page1_key_count=$(printf '%s' "$page1" | sed -n 's:.*<Key>\(.*\)</Key>.*:\1:p' | grep -cv '^$')
|
||||||
|
|
||||||
|
local -a keys1 keys2
|
||||||
|
while IFS= read -r key; do
|
||||||
|
keys1+=("$key")
|
||||||
|
done < <(extract_xml_keys "$page1")
|
||||||
|
|
||||||
|
if [[ -z "$token1" ]]; then
|
||||||
|
log_info "Live pagination smoke: single page only (key count=${page1_key_count})"
|
||||||
|
echo "live pagination smoke: pass (single page, key_count=${page1_key_count})" | tee -a "$live_log"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local token1_encoded
|
||||||
|
token1_encoded=$(printf '%s' "$token1" | python3 -c 'import urllib.parse,sys; print(urllib.parse.quote(sys.stdin.read().strip(), safe=""))')
|
||||||
|
|
||||||
|
local page2
|
||||||
|
page2=$(curl -fsS \
|
||||||
|
--user "${access_key}:${secret_key}" \
|
||||||
|
--aws-sigv4 "$sigv4_target" \
|
||||||
|
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
||||||
|
"${base_url}&continuation-token=${token1_encoded}")
|
||||||
|
|
||||||
|
local page2_key_count
|
||||||
|
page2_key_count=$(printf '%s' "$page2" | sed -n 's:.*<Key>\(.*\)</Key>.*:\1:p' | grep -cv '^$')
|
||||||
|
|
||||||
|
if (( page2_key_count == 0 )); then
|
||||||
|
log_error "Live pagination smoke: second page returned zero keys for token ${token1}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local duplicate
|
||||||
|
local dkeys=()
|
||||||
|
duplicate=0
|
||||||
|
local key
|
||||||
|
|
||||||
|
while IFS= read -r key; do
|
||||||
|
keys2+=("$key")
|
||||||
|
if printf '%s\n' "${keys1[@]}" | grep -Fx -- "$key" >/dev/null 2>&1; then
|
||||||
|
duplicate=1
|
||||||
|
dkeys+=("$key")
|
||||||
|
fi
|
||||||
|
done < <(extract_xml_keys "$page2")
|
||||||
|
|
||||||
|
if (( duplicate )); then
|
||||||
|
log_error "Live pagination smoke detected duplicated keys across page transitions: ${dkeys[*]}"
|
||||||
|
printf '%s\n' "${dkeys[@]}" | tee -a "$live_log"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "Live pagination smoke: first page=${page1_key_count}, second page=${page2_key_count}, token=${token1}"
|
||||||
|
echo "live pagination smoke: pass (continuous two-page check)" | tee -a "$live_log"
|
||||||
|
echo "page1=${page1_key_count}" | tee -a "$live_log"
|
||||||
|
echo "page2=${page2_key_count}" | tee -a "$live_log"
|
||||||
|
log_info "Live pagination smoke log: $live_log"
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_args() {
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--full|--no-skip-live)
|
||||||
|
SKIP_LIVE="false"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
-h|--help)
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
log_error "unknown arg: $1"
|
||||||
|
usage
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
parse_args "$@"
|
||||||
|
require_cmd cargo
|
||||||
|
require_cmd rg
|
||||||
|
require_cmd tee
|
||||||
|
require_cmd curl
|
||||||
|
|
||||||
|
mkdir -p "$OUT_DIR"
|
||||||
|
|
||||||
|
run_unit_checks
|
||||||
|
run_static_checks
|
||||||
|
|
||||||
|
if [[ "$SKIP_LIVE" != "true" ]]; then
|
||||||
|
run_live_smoke
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "Validation summary:"
|
||||||
|
log_info " - logs: $OUT_DIR"
|
||||||
|
log_info " - status: pass"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Reference in New Issue
Block a user