mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
Zhengchao An
eb392f24d6
chore(scripts): index scripts/ and archive 29 one-shot scripts backlog#1153 infra-13. scripts/ had 80+ unlabelled top-level entries mixing CI gates with finished one-shot issue-validation scripts. - scripts/README.md — one index row per entry with status (ci-gate / dev-tool / archived), purpose, and wiring; subdirectories get one row each. run_scanner_benchmarks.sh is annotated "disposition owned by backlog perf-10" and deliberately untouched. - git mv 29 confirmed-stale one-shot entries to scripts/archive/: 11 issue-scoped validation/perf-capture scripts, the 5-script backlog#706 large-PUT breakdown family, the 4-file GET-optimization stress suite, 2 gt1g one-shots, and 7 other orphaned one-shots. Evidence: a whole-tree boundary-aware reference census showed zero references from CI/Makefiles/docs/code for every moved entry (or references only from other scripts inside the same archived set); re-run after the move shows zero dangling references. - docs/testing/README.md links the index. Moves only — no script content changed.
1325 lines
50 KiB
Bash
Executable File
1325 lines
50 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
|
|
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/bench/listobjects-verified-$(date +%Y%m%d-%H%M%S)}"
|
|
DATA_ROOT="${DATA_ROOT:-/private/tmp/listobjects-verified-$(date +%Y%m%d-%H%M%S)}"
|
|
BUILD_PROFILE="${BUILD_PROFILE:-debug}"
|
|
RUSTFS_BIN_USER_SET="false"
|
|
if [[ -n "${RUSTFS_BIN:-}" ]]; then
|
|
RUSTFS_BIN_USER_SET="true"
|
|
else
|
|
RUSTFS_BIN="${PROJECT_ROOT}/target/${BUILD_PROFILE}/rustfs"
|
|
fi
|
|
WARP_BIN="${WARP_BIN:-warp}"
|
|
ADDRESS="${ADDRESS:-127.0.0.1:9000}"
|
|
ENDPOINT="${ENDPOINT:-http://${ADDRESS}}"
|
|
ACCESS_KEY="${ACCESS_KEY:-minioadmin}"
|
|
SECRET_KEY="${SECRET_KEY:-minioadmin}"
|
|
REGION="${REGION:-us-east-1}"
|
|
OBJECTS="${OBJECTS:-10000}"
|
|
OBJECT_SIZE="${OBJECT_SIZE:-1KB}"
|
|
CONCURRENCY="${CONCURRENCY:-16}"
|
|
MAX_KEYS="${MAX_KEYS:-1000}"
|
|
DURATION="${DURATION:-20s}"
|
|
WARMUP_DURATION="${WARMUP_DURATION:-5s}"
|
|
METER_INTERVAL="${METER_INTERVAL:-1}"
|
|
RESOURCE_SAMPLE_INTERVAL="${RESOURCE_SAMPLE_INTERVAL:-1}"
|
|
SKIP_BUILD="${SKIP_BUILD:-false}"
|
|
INDEX_PROVIDER="${INDEX_PROVIDER:-persistent_key_only}"
|
|
INDEX_GENERATION="${INDEX_GENERATION:-bench-$(date +%Y%m%d%H%M%S)}"
|
|
KEY_INDEX_PATH="${KEY_INDEX_PATH:-${OUT_DIR}/persistent-key-only.index}"
|
|
NAMESPACE_JOURNAL_PATH="${NAMESPACE_JOURNAL_PATH:-}"
|
|
PREBUILD_KEY_INDEX="${PREBUILD_KEY_INDEX:-false}"
|
|
PREPARE_ONLY="${PREPARE_ONLY:-false}"
|
|
REUSE_PREPARED_DATA="${REUSE_PREPARED_DATA:-false}"
|
|
RUN_METADATA_FAST="${RUN_METADATA_FAST:-true}"
|
|
RUN_CHAOS="${RUN_CHAOS:-true}"
|
|
RUN_QUORUM_JOURNAL_CHAOS="${RUN_QUORUM_JOURNAL_CHAOS:-true}"
|
|
CHAOS_MAX_KEYS="${CHAOS_MAX_KEYS:-3}"
|
|
METADATA_FAST_STALENESS_MS="${METADATA_FAST_STALENESS_MS:-5000}"
|
|
PROMETHEUS_QUERY_URL="${PROMETHEUS_QUERY_URL:-http://localhost:9090/api/v1/query}"
|
|
PROMETHEUS_JOB="${PROMETHEUS_JOB:-rustfs-app-metrics}"
|
|
JOURNAL_CHAOS_ENABLED=""
|
|
JOURNAL_CHAOS_BUCKET=""
|
|
JOURNAL_CHAOS_SEQUENCE=""
|
|
JOURNAL_CHAOS_STATUS=""
|
|
|
|
SERVER_PID=""
|
|
RESOURCE_SAMPLER_PID=""
|
|
|
|
log_info() { printf '[INFO] %s\n' "$*"; }
|
|
log_warn() { printf '[WARN] %s\n' "$*" >&2; }
|
|
die() {
|
|
printf '[ERROR] %s\n' "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage:
|
|
scripts/run_listobjects_verified_bench.sh [options]
|
|
|
|
Options:
|
|
--out-dir <path> Output directory.
|
|
--data-root <path> RustFS data root for d1..d4.
|
|
--rustfs-bin <path> RustFS binary path.
|
|
--warp-bin <path> warp binary path.
|
|
--objects <n> Objects prepared by warp list (default: 10000).
|
|
--duration <dur> Timed benchmark duration (default: 20s).
|
|
--warmup-duration <d> Warmup duration before timed run (default: 5s).
|
|
--concurrency <n> warp --concurrent value (default: 16).
|
|
--max-keys <n> warp --max-keys value (default: 1000).
|
|
--release Build and run target/release/rustfs.
|
|
--skip-build Use existing RustFS binary.
|
|
--index-provider <v> Opt-in provider (default: persistent_key_only).
|
|
--key-index-path <p> Persistent key-only index path.
|
|
--journal-path <p> Optional local namespace mutation journal override path.
|
|
--prebuild-key-index Build persistent key-only index from the bench script.
|
|
--prepare-only Prepare bucket data and exit without running modes.
|
|
--reuse-prepared-data Reuse --data-root and skip warp data preparation.
|
|
--skip-metadata-fast Skip metadata-fast benchmark mode.
|
|
--skip-chaos Skip metadata-fast stale/fallback chaos probes.
|
|
--skip-quorum-journal-chaos
|
|
Skip quorum-backed .rustfs.sys degraded/restore probe.
|
|
--chaos-max-keys <n> Metadata-fast chaos page size (default: 3).
|
|
--metadata-fast-staleness-ms <n>
|
|
Metadata-fast staleness SLA in milliseconds (default: 5000).
|
|
--prometheus-query-url <url>
|
|
Prometheus API query URL or UI /query URL.
|
|
-h, --help Show this help.
|
|
|
|
Environment:
|
|
OUT_DIR, DATA_ROOT, RUSTFS_BIN, WARP_BIN, ADDRESS, ACCESS_KEY, SECRET_KEY,
|
|
REGION, OBJECTS, OBJECT_SIZE, CONCURRENCY, MAX_KEYS, DURATION,
|
|
WARMUP_DURATION, METER_INTERVAL, RESOURCE_SAMPLE_INTERVAL, SKIP_BUILD,
|
|
BUILD_PROFILE, INDEX_PROVIDER, INDEX_GENERATION, KEY_INDEX_PATH,
|
|
NAMESPACE_JOURNAL_PATH, PREBUILD_KEY_INDEX, PREPARE_ONLY, REUSE_PREPARED_DATA,
|
|
RUN_METADATA_FAST, RUN_CHAOS, RUN_QUORUM_JOURNAL_CHAOS, CHAOS_MAX_KEYS,
|
|
METADATA_FAST_STALENESS_MS, PROMETHEUS_QUERY_URL, PROMETHEUS_JOB.
|
|
USAGE
|
|
}
|
|
|
|
arg_value() {
|
|
local flag="$1"
|
|
local value="${2:-}"
|
|
[[ -n "$value" ]] || die "missing value for $flag"
|
|
printf '%s\n' "$value"
|
|
}
|
|
|
|
parse_args() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
|
--data-root) DATA_ROOT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
|
--rustfs-bin) RUSTFS_BIN="$(arg_value "$1" "${2:-}")"; RUSTFS_BIN_USER_SET="true"; shift 2 ;;
|
|
--warp-bin) WARP_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
|
--objects) OBJECTS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
|
--duration) DURATION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
|
--warmup-duration) WARMUP_DURATION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
|
--concurrency) CONCURRENCY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
|
--max-keys) MAX_KEYS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
|
--release)
|
|
BUILD_PROFILE="release"
|
|
if [[ "$RUSTFS_BIN_USER_SET" == "false" ]]; then
|
|
RUSTFS_BIN="${PROJECT_ROOT}/target/release/rustfs"
|
|
fi
|
|
shift
|
|
;;
|
|
--skip-build) SKIP_BUILD="true"; shift ;;
|
|
--index-provider) INDEX_PROVIDER="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
|
--key-index-path) KEY_INDEX_PATH="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
|
--journal-path) NAMESPACE_JOURNAL_PATH="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
|
--prebuild-key-index) PREBUILD_KEY_INDEX="true"; shift ;;
|
|
--prepare-only) PREPARE_ONLY="true"; shift ;;
|
|
--reuse-prepared-data) REUSE_PREPARED_DATA="true"; shift ;;
|
|
--skip-metadata-fast) RUN_METADATA_FAST="false"; shift ;;
|
|
--skip-chaos) RUN_CHAOS="false"; shift ;;
|
|
--skip-quorum-journal-chaos) RUN_QUORUM_JOURNAL_CHAOS="false"; shift ;;
|
|
--chaos-max-keys) CHAOS_MAX_KEYS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
|
--metadata-fast-staleness-ms) METADATA_FAST_STALENESS_MS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
|
--prometheus-query-url) PROMETHEUS_QUERY_URL="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) die "unknown arg: $1" ;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
require_cmds() {
|
|
command -v "$WARP_BIN" >/dev/null 2>&1 || die "warp binary not found: $WARP_BIN"
|
|
command -v curl >/dev/null 2>&1 || die "curl not found"
|
|
command -v python3 >/dev/null 2>&1 || die "python3 not found"
|
|
command -v awk >/dev/null 2>&1 || die "awk not found"
|
|
command -v sed >/dev/null 2>&1 || die "sed not found"
|
|
}
|
|
|
|
build_rustfs() {
|
|
if [[ "$SKIP_BUILD" == "true" ]]; then
|
|
[[ -x "$RUSTFS_BIN" ]] || die "RustFS binary is not executable: $RUSTFS_BIN"
|
|
return
|
|
fi
|
|
if [[ "$BUILD_PROFILE" == "release" ]]; then
|
|
(cd "$PROJECT_ROOT" && cargo build -p rustfs --release)
|
|
else
|
|
(cd "$PROJECT_ROOT" && cargo build -p rustfs)
|
|
fi
|
|
[[ -x "$RUSTFS_BIN" ]] || die "RustFS binary is not executable after build: $RUSTFS_BIN"
|
|
}
|
|
|
|
stop_server() {
|
|
stop_resource_sampler
|
|
if [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" >/dev/null 2>&1; then
|
|
kill "$SERVER_PID" >/dev/null 2>&1 || true
|
|
wait "$SERVER_PID" >/dev/null 2>&1 || true
|
|
fi
|
|
SERVER_PID=""
|
|
}
|
|
|
|
stop_resource_sampler() {
|
|
if [[ -n "$RESOURCE_SAMPLER_PID" ]] && kill -0 "$RESOURCE_SAMPLER_PID" >/dev/null 2>&1; then
|
|
kill "$RESOURCE_SAMPLER_PID" >/dev/null 2>&1 || true
|
|
wait "$RESOURCE_SAMPLER_PID" >/dev/null 2>&1 || true
|
|
fi
|
|
RESOURCE_SAMPLER_PID=""
|
|
}
|
|
|
|
start_resource_sampler() {
|
|
local out_file="$1"
|
|
local pid="$SERVER_PID"
|
|
echo "unix_ts,pid,cpu_percent,rss_kib" >"$out_file"
|
|
(
|
|
while kill -0 "$pid" >/dev/null 2>&1; do
|
|
ps -p "$pid" -o pid=,%cpu=,rss= | awk -v ts="$(date +%s)" '{printf "%s,%s,%s,%s\n", ts, $1, $2, $3}'
|
|
sleep "$RESOURCE_SAMPLE_INTERVAL"
|
|
done
|
|
) >>"$out_file" 2>/dev/null &
|
|
RESOURCE_SAMPLER_PID="$!"
|
|
}
|
|
|
|
wait_for_health() {
|
|
local log_file="$1"
|
|
local deadline=$((SECONDS + 60))
|
|
while (( SECONDS < deadline )); do
|
|
if curl -fsS "${ENDPOINT}/health" >/dev/null 2>&1 || curl -fsS "${ENDPOINT}/health/ready" >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
if [[ -n "$SERVER_PID" ]] && ! kill -0 "$SERVER_PID" >/dev/null 2>&1; then
|
|
tail -n 80 "$log_file" >&2 || true
|
|
die "RustFS exited before health check passed"
|
|
fi
|
|
sleep 1
|
|
done
|
|
tail -n 80 "$log_file" >&2 || true
|
|
die "RustFS health check timed out"
|
|
}
|
|
|
|
start_server() {
|
|
local mode="$1"
|
|
local log_file="$2"
|
|
local reset_data="${3:-false}"
|
|
|
|
stop_server
|
|
if [[ "$reset_data" == "true" ]]; then
|
|
rm -rf "$DATA_ROOT"
|
|
fi
|
|
mkdir -p "$DATA_ROOT"/d{1,2,3,4}
|
|
|
|
(
|
|
export RUST_LOG=off
|
|
export RUSTFS_OBS_LOGGER_LEVEL=off
|
|
export RUSTFS_OBS_TRACES_EXPORT_ENABLED=false
|
|
export RUSTFS_OBS_METRICS_EXPORT_ENABLED=true
|
|
export RUSTFS_OBS_LOGS_EXPORT_ENABLED=false
|
|
export RUSTFS_OBS_PROFILING_EXPORT_ENABLED=false
|
|
export RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318
|
|
export RUSTFS_OBS_METRIC_ENDPOINT=http://127.0.0.1:4318/v1/metrics
|
|
export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://127.0.0.1:4318/v1/metrics
|
|
export RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS=200
|
|
export RUSTFS_OBS_USE_STDOUT=true
|
|
export RUSTFS_OBS_METER_INTERVAL="$METER_INTERVAL"
|
|
export RUSTFS_OBS_LOG_STDOUT_ENABLED=false
|
|
export RUSTFS_SCANNER_ENABLED=false
|
|
export RUSTFS_CONSOLE_ENABLE=false
|
|
export RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true
|
|
export RUSTFS_ADDRESS="$ADDRESS"
|
|
export RUSTFS_ACCESS_KEY="$ACCESS_KEY"
|
|
export RUSTFS_SECRET_KEY="$SECRET_KEY"
|
|
export RUSTFS_RPC_SECRET="rustfs-listobjects-verified-rpc-secret"
|
|
export RUSTFS_REGION="$REGION"
|
|
unset RUSTFS_LIST_OBJECTS_INDEX_MODE
|
|
unset RUSTFS_LIST_OBJECTS_INDEX_PROVIDER
|
|
unset RUSTFS_LIST_OBJECTS_INDEX_PROVIDER_PATH
|
|
unset RUSTFS_LIST_OBJECTS_INDEX_PROVIDER_GENERATION
|
|
unset RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_PATH
|
|
unset RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED
|
|
unset RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET
|
|
unset RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_SEQUENCE
|
|
unset RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS
|
|
unset RUSTFS_LIST_OBJECTS_METADATA_FAST_ENABLED
|
|
unset RUSTFS_LIST_OBJECTS_METADATA_FAST_STALENESS_MS
|
|
if [[ "$mode" == "opt_in_verified" || "$mode" == "metadata_fast" ]]; then
|
|
export RUSTFS_LIST_OBJECTS_INDEX_MODE=index_key_only
|
|
if [[ "$mode" == "metadata_fast" ]]; then
|
|
export RUSTFS_LIST_OBJECTS_INDEX_MODE=index_metadata_fast
|
|
export RUSTFS_LIST_OBJECTS_METADATA_FAST_ENABLED=true
|
|
export RUSTFS_LIST_OBJECTS_METADATA_FAST_STALENESS_MS="$METADATA_FAST_STALENESS_MS"
|
|
fi
|
|
export RUSTFS_LIST_OBJECTS_INDEX_PROVIDER="$INDEX_PROVIDER"
|
|
if [[ "$INDEX_PROVIDER" == "persistent_key_only" || "$INDEX_PROVIDER" == "persisted_key_only" ]]; then
|
|
export RUSTFS_LIST_OBJECTS_INDEX_PROVIDER_PATH="$KEY_INDEX_PATH"
|
|
export RUSTFS_LIST_OBJECTS_INDEX_PROVIDER_GENERATION="$INDEX_GENERATION"
|
|
if [[ -n "$NAMESPACE_JOURNAL_PATH" ]]; then
|
|
export RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_PATH="$NAMESPACE_JOURNAL_PATH"
|
|
fi
|
|
if [[ -n "$JOURNAL_CHAOS_BUCKET" && -n "$JOURNAL_CHAOS_STATUS" ]]; then
|
|
if [[ -n "$JOURNAL_CHAOS_ENABLED" ]]; then
|
|
export RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED="$JOURNAL_CHAOS_ENABLED"
|
|
fi
|
|
export RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET="$JOURNAL_CHAOS_BUCKET"
|
|
export RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS="$JOURNAL_CHAOS_STATUS"
|
|
if [[ -n "$JOURNAL_CHAOS_SEQUENCE" ]]; then
|
|
export RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_SEQUENCE="$JOURNAL_CHAOS_SEQUENCE"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
exec "$RUSTFS_BIN" server \
|
|
"${DATA_ROOT}/d1" \
|
|
"${DATA_ROOT}/d2" \
|
|
"${DATA_ROOT}/d3" \
|
|
"${DATA_ROOT}/d4"
|
|
) >"$log_file" 2>&1 &
|
|
|
|
SERVER_PID="$!"
|
|
wait_for_health "$log_file"
|
|
}
|
|
|
|
prepare_bucket() {
|
|
local duration="$2"
|
|
local log_file="$3"
|
|
local bucket="$1"
|
|
|
|
"$WARP_BIN" list \
|
|
--host "$ADDRESS" \
|
|
--access-key "$ACCESS_KEY" \
|
|
--secret-key "$SECRET_KEY" \
|
|
--region "$REGION" \
|
|
--bucket "$bucket" \
|
|
--objects "$OBJECTS" \
|
|
--obj.size "$OBJECT_SIZE" \
|
|
--concurrent "$CONCURRENCY" \
|
|
--max-keys "$MAX_KEYS" \
|
|
--duration "$duration" \
|
|
--noclear \
|
|
--no-color \
|
|
>"$log_file" 2>&1
|
|
}
|
|
|
|
write_key_index() {
|
|
local bucket="$1"
|
|
local index_path="$2"
|
|
local index_dir tmp_keys tmp_objects sorted_keys tmp_index token page response_file next_token key_count
|
|
index_dir="$(dirname "$index_path")"
|
|
tmp_keys="${index_path}.keys.tmp"
|
|
tmp_objects="${index_path}.objects.tmp"
|
|
sorted_keys="${index_path}.keys.sorted.tmp"
|
|
tmp_index="${index_path}.tmp"
|
|
mkdir -p "$index_dir"
|
|
: >"$tmp_keys"
|
|
: >"$tmp_objects"
|
|
|
|
token=""
|
|
page=0
|
|
while true; do
|
|
local url="${ENDPOINT}/${bucket}?list-type=2&max-keys=${MAX_KEYS}"
|
|
if [[ -n "$token" ]]; then
|
|
local encoded_token
|
|
encoded_token="$(printf '%s' "$token" | urlencode)"
|
|
url="${url}&continuation-token=${encoded_token}"
|
|
fi
|
|
|
|
response_file="${index_path}.page-${page}.xml"
|
|
curl -fsS \
|
|
--user "${ACCESS_KEY}:${SECRET_KEY}" \
|
|
--aws-sigv4 "aws:amz:${REGION}:s3" \
|
|
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
|
-o "$response_file" \
|
|
"$url"
|
|
extract_xml_keys <"$response_file" >>"$tmp_keys" || true
|
|
extract_metadata_object_rows <"$response_file" >>"$tmp_objects" || true
|
|
next_token="$(extract_next_token <"$response_file" || true)"
|
|
rm -f "$response_file"
|
|
if [[ -z "$next_token" ]]; then
|
|
break
|
|
fi
|
|
token="$next_token"
|
|
page=$((page + 1))
|
|
done
|
|
|
|
LC_ALL=C sort -u "$tmp_keys" >"$sorted_keys"
|
|
key_count="$(awk 'END{print NR + 0}' "$sorted_keys")"
|
|
{
|
|
echo "# rustfs-listobjects-key-only-v1"
|
|
echo "# bucket=${bucket}"
|
|
echo "# generation=${INDEX_GENERATION}"
|
|
echo "# checkpoint_high_water_mark=${key_count}"
|
|
cat "$sorted_keys"
|
|
cat "$tmp_objects"
|
|
} >"$tmp_index"
|
|
mv "$tmp_index" "$index_path"
|
|
rm -f "$tmp_keys" "$tmp_objects" "$sorted_keys"
|
|
}
|
|
|
|
extract_xml_keys() {
|
|
python3 -c 'import sys, xml.etree.ElementTree as ET
|
|
root = ET.fromstring(sys.stdin.read())
|
|
for elem in root.iter():
|
|
if elem.tag.rsplit("}", 1)[-1] == "Key" and elem.text:
|
|
print(elem.text)'
|
|
}
|
|
|
|
extract_next_token() {
|
|
python3 -c 'import sys, xml.etree.ElementTree as ET
|
|
root = ET.fromstring(sys.stdin.read())
|
|
for elem in root.iter():
|
|
if elem.tag.rsplit("}", 1)[-1] == "NextContinuationToken" and elem.text:
|
|
print(elem.text)
|
|
break'
|
|
}
|
|
|
|
xml_tag_text() {
|
|
local tag="$1"
|
|
python3 -c 'import sys, xml.etree.ElementTree as ET
|
|
tag = sys.argv[1]
|
|
root = ET.fromstring(sys.stdin.read())
|
|
for elem in root.iter():
|
|
if elem.tag.rsplit("}", 1)[-1] == tag and elem.text:
|
|
print(elem.text)
|
|
break' "$tag"
|
|
}
|
|
|
|
extract_metadata_object_rows() {
|
|
python3 -c 'import base64, datetime, sys, xml.etree.ElementTree as ET
|
|
def local_name(tag):
|
|
return tag.rsplit("}", 1)[-1]
|
|
def text(parent, name):
|
|
for child in parent:
|
|
if local_name(child.tag) == name:
|
|
return child.text or ""
|
|
return ""
|
|
def enc(value):
|
|
if not value:
|
|
return "-"
|
|
return base64.b64encode(value.encode("utf-8")).decode("ascii")
|
|
def nanos(value):
|
|
if not value:
|
|
return "-"
|
|
try:
|
|
dt = datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
return str(int(dt.timestamp() * 1_000_000_000))
|
|
except ValueError:
|
|
return "-"
|
|
root = ET.fromstring(sys.stdin.read())
|
|
for contents in root.iter():
|
|
if local_name(contents.tag) != "Contents":
|
|
continue
|
|
key = text(contents, "Key")
|
|
if not key:
|
|
continue
|
|
size = text(contents, "Size") or "0"
|
|
etag = text(contents, "ETag").strip("\"")
|
|
storage_class = text(contents, "StorageClass")
|
|
print("# object\t{}\t{}\t{}\t{}\t{}".format(
|
|
enc(key),
|
|
size,
|
|
nanos(text(contents, "LastModified")),
|
|
enc(etag),
|
|
enc(storage_class),
|
|
))'
|
|
}
|
|
|
|
signed_list_once() {
|
|
local bucket="$1"
|
|
local response_file="$2"
|
|
local max_keys="${3:-1}"
|
|
signed_list_page "$bucket" "$response_file" "$max_keys" ""
|
|
}
|
|
|
|
signed_list_page() {
|
|
local bucket="$1"
|
|
local response_file="$2"
|
|
local max_keys="${3:-1}"
|
|
local token="${4:-}"
|
|
local url="${ENDPOINT}/${bucket}?list-type=2&max-keys=${max_keys}"
|
|
if [[ -n "$token" ]]; then
|
|
local encoded_token
|
|
encoded_token="$(printf '%s' "$token" | urlencode)"
|
|
url="${url}&continuation-token=${encoded_token}"
|
|
fi
|
|
curl -fsS \
|
|
--user "${ACCESS_KEY}:${SECRET_KEY}" \
|
|
--aws-sigv4 "aws:amz:${REGION}:s3" \
|
|
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
|
-o "$response_file" \
|
|
"$url"
|
|
}
|
|
|
|
urlencode_path() {
|
|
python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe="/-_.~"))' "$1"
|
|
}
|
|
|
|
signed_put_object() {
|
|
local bucket="$1"
|
|
local key="$2"
|
|
local body_file="$3"
|
|
local out_file="$4"
|
|
local encoded_key
|
|
encoded_key="$(urlencode_path "$key")"
|
|
curl -fsS \
|
|
--request PUT \
|
|
--user "${ACCESS_KEY}:${SECRET_KEY}" \
|
|
--aws-sigv4 "aws:amz:${REGION}:s3" \
|
|
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
|
--upload-file "$body_file" \
|
|
-o "$out_file" \
|
|
"${ENDPOINT}/${bucket}/${encoded_key}"
|
|
}
|
|
|
|
signed_delete_object() {
|
|
local bucket="$1"
|
|
local key="$2"
|
|
local out_file="$3"
|
|
local encoded_key
|
|
encoded_key="$(urlencode_path "$key")"
|
|
curl -fsS \
|
|
--request DELETE \
|
|
--user "${ACCESS_KEY}:${SECRET_KEY}" \
|
|
--aws-sigv4 "aws:amz:${REGION}:s3" \
|
|
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
|
-o "$out_file" \
|
|
"${ENDPOINT}/${bucket}/${encoded_key}"
|
|
}
|
|
|
|
response_etag() {
|
|
awk 'BEGIN{IGNORECASE=1} /^etag:/ {gsub("\r", "", $0); sub("^[^:]*:[[:space:]]*", "", $0); print $0; exit}' "$1"
|
|
}
|
|
|
|
signed_multipart_complete() {
|
|
local bucket="$1"
|
|
local key="$2"
|
|
local work_dir="$3"
|
|
local encoded_key upload_xml upload_id encoded_upload_id part1_file part2_file part1_headers part2_headers etag1 etag2 complete_xml
|
|
encoded_key="$(urlencode_path "$key")"
|
|
upload_xml="$work_dir/multipart-init.xml"
|
|
|
|
curl -fsS \
|
|
--request POST \
|
|
--user "${ACCESS_KEY}:${SECRET_KEY}" \
|
|
--aws-sigv4 "aws:amz:${REGION}:s3" \
|
|
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
|
-o "$upload_xml" \
|
|
"${ENDPOINT}/${bucket}/${encoded_key}?uploads"
|
|
upload_id="$(xml_tag_text UploadId <"$upload_xml")"
|
|
[[ -n "$upload_id" ]] || die "multipart initiate did not return UploadId"
|
|
encoded_upload_id="$(printf '%s' "$upload_id" | urlencode)"
|
|
|
|
part1_file="$work_dir/multipart-part-1.bin"
|
|
part2_file="$work_dir/multipart-part-2.bin"
|
|
part1_headers="$work_dir/multipart-part-1.headers"
|
|
part2_headers="$work_dir/multipart-part-2.headers"
|
|
python3 -c 'from pathlib import Path; Path(__import__("sys").argv[1]).write_bytes(b"a" * (5 * 1024 * 1024)); Path(__import__("sys").argv[2]).write_bytes(b"b")' "$part1_file" "$part2_file"
|
|
|
|
curl -fsS \
|
|
--request PUT \
|
|
--user "${ACCESS_KEY}:${SECRET_KEY}" \
|
|
--aws-sigv4 "aws:amz:${REGION}:s3" \
|
|
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
|
--upload-file "$part1_file" \
|
|
-D "$part1_headers" \
|
|
-o "$work_dir/multipart-part-1.out" \
|
|
"${ENDPOINT}/${bucket}/${encoded_key}?partNumber=1&uploadId=${encoded_upload_id}"
|
|
curl -fsS \
|
|
--request PUT \
|
|
--user "${ACCESS_KEY}:${SECRET_KEY}" \
|
|
--aws-sigv4 "aws:amz:${REGION}:s3" \
|
|
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
|
--upload-file "$part2_file" \
|
|
-D "$part2_headers" \
|
|
-o "$work_dir/multipart-part-2.out" \
|
|
"${ENDPOINT}/${bucket}/${encoded_key}?partNumber=2&uploadId=${encoded_upload_id}"
|
|
|
|
etag1="$(response_etag "$part1_headers")"
|
|
etag2="$(response_etag "$part2_headers")"
|
|
[[ -n "$etag1" && -n "$etag2" ]] || die "multipart part upload did not return ETags"
|
|
complete_xml="$work_dir/multipart-complete.xml"
|
|
{
|
|
echo "<CompleteMultipartUpload>"
|
|
echo " <Part><PartNumber>1</PartNumber><ETag>${etag1}</ETag></Part>"
|
|
echo " <Part><PartNumber>2</PartNumber><ETag>${etag2}</ETag></Part>"
|
|
echo "</CompleteMultipartUpload>"
|
|
} >"$complete_xml"
|
|
|
|
curl -fsS \
|
|
--request POST \
|
|
--user "${ACCESS_KEY}:${SECRET_KEY}" \
|
|
--aws-sigv4 "aws:amz:${REGION}:s3" \
|
|
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
|
-H "Content-Type: application/xml" \
|
|
--data-binary "@${complete_xml}" \
|
|
-o "$work_dir/multipart-complete.out" \
|
|
"${ENDPOINT}/${bucket}/${encoded_key}?uploadId=${encoded_upload_id}"
|
|
}
|
|
|
|
urlencode() {
|
|
python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.stdin.read().strip(), safe=""))'
|
|
}
|
|
|
|
base64_token() {
|
|
python3 -c 'import base64, sys; print(base64.b64encode(sys.stdin.read().encode("utf-8")).decode("ascii"))'
|
|
}
|
|
|
|
percentile_summary() {
|
|
local values_file="$1"
|
|
python3 - "$values_file" <<'PY'
|
|
import math
|
|
import statistics
|
|
import sys
|
|
|
|
path = sys.argv[1]
|
|
values = []
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
values.append(float(line))
|
|
|
|
if not values:
|
|
print("0,N/A,N/A,N/A,N/A")
|
|
raise SystemExit(0)
|
|
|
|
values.sort()
|
|
def percentile(p: float) -> float:
|
|
if len(values) == 1:
|
|
return values[0]
|
|
rank = math.ceil((p / 100.0) * len(values)) - 1
|
|
rank = min(max(rank, 0), len(values) - 1)
|
|
return values[rank]
|
|
|
|
print(
|
|
f"{len(values)},"
|
|
f"{statistics.fmean(values):.6f},"
|
|
f"{percentile(50):.6f},"
|
|
f"{percentile(95):.6f},"
|
|
f"{percentile(99):.6f}"
|
|
)
|
|
PY
|
|
}
|
|
|
|
run_signed_list_bench() {
|
|
local mode="$1"
|
|
local bucket="$2"
|
|
local duration="$3"
|
|
local out_dir="$4"
|
|
local deadline=$((SECONDS + ${duration%s}))
|
|
local latencies_file="$out_dir/list-latencies-ms.txt"
|
|
local request_log="$out_dir/list-requests.csv"
|
|
local total_keys=0
|
|
local total_requests=0
|
|
local iterations=0
|
|
|
|
: >"$latencies_file"
|
|
echo "iteration,request,keys,latency_ms,truncated" >"$request_log"
|
|
|
|
while (( SECONDS < deadline )); do
|
|
local token=""
|
|
local request_in_iteration=0
|
|
iterations=$((iterations + 1))
|
|
|
|
while (( SECONDS < deadline )); do
|
|
local url="${ENDPOINT}/${bucket}?list-type=2&max-keys=${MAX_KEYS}"
|
|
if [[ -n "$token" ]]; then
|
|
local encoded_token
|
|
encoded_token="$(printf '%s' "$token" | urlencode)"
|
|
url="${url}&continuation-token=${encoded_token}"
|
|
fi
|
|
|
|
local response_file time_total status key_count next_token truncated
|
|
response_file="$out_dir/response-${mode}-${iterations}-${request_in_iteration}.xml"
|
|
time_total="$(curl -fsS \
|
|
--user "${ACCESS_KEY}:${SECRET_KEY}" \
|
|
--aws-sigv4 "aws:amz:${REGION}:s3" \
|
|
-H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
|
|
-w '%{time_total}' \
|
|
-o "$response_file" \
|
|
"$url")"
|
|
status=$?
|
|
if [[ "$status" -ne 0 ]]; then
|
|
log_warn "curl list failed for mode=$mode status=$status url=$url"
|
|
return 1
|
|
fi
|
|
|
|
key_count="$(extract_xml_keys <"$response_file" | grep -cv '^$' || true)"
|
|
next_token="$(extract_next_token <"$response_file" || true)"
|
|
truncated="false"
|
|
[[ -n "$next_token" ]] && truncated="true"
|
|
|
|
awk -v sec="$time_total" 'BEGIN{printf "%.6f\n", sec * 1000.0}' >>"$latencies_file"
|
|
echo "$iterations,$request_in_iteration,$key_count,$(awk -v sec="$time_total" 'BEGIN{printf "%.6f", sec * 1000.0}'),$truncated" >>"$request_log"
|
|
|
|
total_keys=$((total_keys + key_count))
|
|
total_requests=$((total_requests + 1))
|
|
request_in_iteration=$((request_in_iteration + 1))
|
|
|
|
if [[ -z "$next_token" ]]; then
|
|
break
|
|
fi
|
|
token="$next_token"
|
|
done
|
|
done
|
|
|
|
local percentile_line req_count avg_ms p50_ms p95_ms p99_ms objps
|
|
percentile_line="$(percentile_summary "$latencies_file")"
|
|
IFS=',' read -r req_count avg_ms p50_ms p95_ms p99_ms <<<"$percentile_line"
|
|
objps="$(awk -v keys="$total_keys" -v duration="${duration%s}" 'BEGIN{if(duration<=0) print "N/A"; else printf "%.6f", keys/duration}')"
|
|
|
|
{
|
|
echo "mode,objects,object_size,max_keys,duration,requests,scan_iterations,total_keys,objps,latency_avg_ms,p50_ms,p95_ms,p99_ms"
|
|
echo "$mode,$OBJECTS,$OBJECT_SIZE,$MAX_KEYS,$duration,$req_count,$iterations,$total_keys,$objps,$avg_ms,$p50_ms,$p95_ms,$p99_ms"
|
|
} >"$out_dir/list-summary.csv"
|
|
}
|
|
|
|
last_metric_field_sum() {
|
|
local metric="$1"
|
|
local field="$2"
|
|
local log_file="$3"
|
|
awk -v metric="$metric" -v field="$field" '
|
|
function finish_block() {
|
|
if (found) {
|
|
snapshot_total += sum
|
|
snapshot_seen = 1
|
|
found=0
|
|
sum=0
|
|
}
|
|
}
|
|
function finish_snapshot() {
|
|
finish_block()
|
|
if (snapshot_seen) {
|
|
value=snapshot_total
|
|
}
|
|
snapshot_total=0
|
|
snapshot_seen=0
|
|
}
|
|
/^Metric #0$/ {
|
|
finish_snapshot()
|
|
}
|
|
/^Metric #[0-9]+/ {
|
|
finish_block()
|
|
}
|
|
$0 ~ "Name[[:space:]]*:[[:space:]]*" metric "$" { found=1; sum=0; next }
|
|
found && $0 ~ "^[[:space:]]*" field "[[:space:]]*:" {
|
|
line=$0
|
|
sub("^[[:space:]]*" field "[[:space:]]*:[[:space:]]*", "", line)
|
|
gsub(/[^0-9.eE+-]/, "", line)
|
|
sum += line + 0
|
|
}
|
|
END {
|
|
finish_snapshot()
|
|
if (value == "") print "0"; else print value
|
|
}
|
|
' "$log_file"
|
|
}
|
|
|
|
last_metric_value() {
|
|
local metric="$1"
|
|
local log_file="$2"
|
|
last_metric_field_sum "$metric" "Value" "$log_file"
|
|
}
|
|
|
|
last_metric_sum() {
|
|
local metric="$1"
|
|
local log_file="$2"
|
|
last_metric_field_sum "$metric" "Sum" "$log_file"
|
|
}
|
|
|
|
last_metric_count() {
|
|
local metric="$1"
|
|
local log_file="$2"
|
|
last_metric_field_sum "$metric" "Count" "$log_file"
|
|
}
|
|
|
|
write_ratio_summary() {
|
|
local mode="$1"
|
|
local log_file="$2"
|
|
local csv_file="$3"
|
|
local attempt fallback served verify_attempts verify_misses returned_objects amplification_sum amplification_count amplification stale_ratio fallback_ratio
|
|
|
|
attempt="$(last_metric_value rustfs_s3_list_objects_index_attempt_total "$log_file")"
|
|
fallback="$(last_metric_value rustfs_s3_list_objects_index_fallback_total "$log_file")"
|
|
served="$(last_metric_value rustfs_s3_list_objects_index_served_total "$log_file")"
|
|
verify_attempts="$(last_metric_sum rustfs_s3_list_objects_index_live_verify_attempts "$log_file")"
|
|
verify_misses="$(last_metric_sum rustfs_s3_list_objects_index_live_verify_misses "$log_file")"
|
|
returned_objects="$(last_metric_sum rustfs_s3_list_objects_index_returned_objects "$log_file")"
|
|
amplification_sum="$(last_metric_sum rustfs_s3_list_objects_index_verification_io_amplification "$log_file")"
|
|
amplification_count="$(last_metric_count rustfs_s3_list_objects_index_verification_io_amplification "$log_file")"
|
|
amplification="$(awk -v sum="$amplification_sum" -v count="$amplification_count" 'BEGIN{if(count+0==0) print "0"; else printf "%.6f", (sum+0)/(count+0)}')"
|
|
|
|
fallback_ratio="$(awk -v f="$fallback" -v a="$attempt" 'BEGIN{if(a+0==0) print "N/A"; else printf "%.6f", (f+0)/(a+0)}')"
|
|
stale_ratio="$(awk -v m="$verify_misses" -v a="$verify_attempts" 'BEGIN{if(a+0==0) print "N/A"; else printf "%.6f", (m+0)/(a+0)}')"
|
|
|
|
{
|
|
echo "mode,index_attempt_total,index_fallback_total,index_served_total,live_verify_attempts,live_verify_misses,returned_objects,verification_io_amplification,fallback_ratio,stale_missing_candidate_ratio"
|
|
echo "$mode,$attempt,$fallback,$served,$verify_attempts,$verify_misses,$returned_objects,$amplification,$fallback_ratio,$stale_ratio"
|
|
} >"$csv_file"
|
|
}
|
|
|
|
write_fallback_reason_summary() {
|
|
local mode="$1"
|
|
local log_file="$2"
|
|
local csv_file="$3"
|
|
|
|
awk -v mode="$mode" '
|
|
BEGIN {
|
|
print "mode,reason,index_fallback_total"
|
|
}
|
|
/Name[[:space:]]*: rustfs_s3_list_objects_index_fallback_total/ {
|
|
in_metric = 1
|
|
value = ""
|
|
next
|
|
}
|
|
in_metric && /Name[[:space:]]*:/ && $0 !~ /rustfs_s3_list_objects_index_fallback_total/ {
|
|
in_metric = 0
|
|
value = ""
|
|
next
|
|
}
|
|
in_metric && /Value[[:space:]]*:/ {
|
|
line = $0
|
|
sub(/^.*Value[[:space:]]*:[[:space:]]*/, "", line)
|
|
split(line, parts, /[[:space:]]+/)
|
|
value = parts[1]
|
|
next
|
|
}
|
|
in_metric && /->[[:space:]]*reason:/ {
|
|
line = $0
|
|
sub(/^.*->[[:space:]]*reason:[[:space:]]*/, "", line)
|
|
split(line, parts, /[[:space:]]+/)
|
|
reason = parts[1]
|
|
last[reason] = value + 0
|
|
seen[reason] = 1
|
|
value = ""
|
|
next
|
|
}
|
|
END {
|
|
count = 0
|
|
for (reason in seen) {
|
|
printf "%s,%s,%.0f\n", mode, reason, last[reason]
|
|
count += 1
|
|
}
|
|
if (count == 0) {
|
|
printf "%s,N/A,0\n", mode
|
|
}
|
|
}
|
|
' "$log_file" >"$csv_file"
|
|
}
|
|
|
|
prometheus_api_query_url() {
|
|
local url="$PROMETHEUS_QUERY_URL"
|
|
if [[ "$url" == */api/v1/query ]]; then
|
|
printf '%s\n' "$url"
|
|
elif [[ "$url" == */query ]]; then
|
|
printf '%s/api/v1/query\n' "${url%/query}"
|
|
else
|
|
printf '%s\n' "$url"
|
|
fi
|
|
}
|
|
|
|
query_prometheus_expr() {
|
|
local expr="$1"
|
|
local out_file="$2"
|
|
local api_url
|
|
api_url="$(prometheus_api_query_url)"
|
|
curl -fsS --get --data-urlencode "query=${expr}" "$api_url" -o "$out_file"
|
|
}
|
|
|
|
write_prometheus_snapshot() {
|
|
local mode="$1"
|
|
local out_dir="$2"
|
|
local prom_dir="$out_dir/prometheus"
|
|
mkdir -p "$prom_dir"
|
|
|
|
query_prometheus_expr "sum(rustfs_s3_list_objects_index_attempt_total)" "$prom_dir/index_attempt_total.json" || {
|
|
log_warn "Prometheus query failed; skipping Prometheus snapshot for mode=$mode"
|
|
return 0
|
|
}
|
|
query_prometheus_expr "sum(rustfs_s3_list_objects_index_fallback_total)" "$prom_dir/index_fallback_total.json" || true
|
|
query_prometheus_expr "sum(rustfs_s3_list_objects_index_served_total)" "$prom_dir/index_served_total.json" || true
|
|
query_prometheus_expr "sum(rustfs_s3_list_objects_index_live_verify_attempts_sum)" "$prom_dir/live_verify_attempts_sum.json" || true
|
|
query_prometheus_expr "sum(rustfs_s3_list_objects_index_live_verify_misses_sum)" "$prom_dir/live_verify_misses_sum.json" || true
|
|
query_prometheus_expr "sum(rustfs_s3_list_objects_index_returned_objects_sum)" "$prom_dir/returned_objects_sum.json" || true
|
|
query_prometheus_expr "sum(rustfs_s3_list_objects_index_verification_io_amplification_sum) / sum(rustfs_s3_list_objects_index_verification_io_amplification_count)" "$prom_dir/verification_io_amplification_avg.json" || true
|
|
query_prometheus_expr "rate(process_cpu_seconds_total{job=\"${PROMETHEUS_JOB}\"}[1m])" "$prom_dir/process_cpu_rate_1m.json" || true
|
|
query_prometheus_expr "process_resident_memory_bytes{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/process_resident_memory_bytes.json" || true
|
|
query_prometheus_expr "rustfs_memory_allocator_reserved_bytes{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/allocator_reserved_bytes.json" || true
|
|
query_prometheus_expr "rustfs_memory_allocator_committed_bytes{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/allocator_committed_bytes.json" || true
|
|
query_prometheus_expr "rustfs_memory_allocator_page_committed_bytes{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/allocator_page_committed_bytes.json" || true
|
|
query_prometheus_expr "rustfs_memory_allocator_malloc_requested_bytes{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/allocator_malloc_requested_bytes.json" || true
|
|
query_prometheus_expr "rustfs_memory_allocator_malloc_requested_peak_bytes{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/allocator_malloc_requested_peak_bytes.json" || true
|
|
query_prometheus_expr "rustfs_memory_allocator_malloc_requested_total_bytes{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/allocator_malloc_requested_total_bytes.json" || true
|
|
query_prometheus_expr "rustfs_memory_allocator_heap_count{job=\"${PROMETHEUS_JOB}\"}" "$prom_dir/allocator_heap_count.json" || true
|
|
}
|
|
|
|
write_resource_summary() {
|
|
local samples_file="$1"
|
|
local summary_file="$2"
|
|
awk -F',' '
|
|
NR == 1 { next }
|
|
NF >= 4 {
|
|
cpu += $3
|
|
rss += $4
|
|
if ($3 > max_cpu) max_cpu = $3
|
|
if ($4 > max_rss) max_rss = $4
|
|
count += 1
|
|
}
|
|
END {
|
|
print "samples,avg_cpu_percent,max_cpu_percent,avg_rss_mib,max_rss_mib"
|
|
if (count == 0) {
|
|
print "0,N/A,N/A,N/A,N/A"
|
|
} else {
|
|
printf "%d,%.6f,%.6f,%.6f,%.6f\n", count, cpu / count, max_cpu, (rss / count) / 1024.0, max_rss / 1024.0
|
|
}
|
|
}
|
|
' "$samples_file" >"$summary_file"
|
|
}
|
|
|
|
run_metadata_fast_chaos() {
|
|
local bucket="$1"
|
|
local out_dir="$2"
|
|
mkdir -p "$out_dir"
|
|
|
|
local summary="$out_dir/chaos-summary.csv"
|
|
local fallback_summary="$out_dir/fallback-probes.csv"
|
|
local page1="$out_dir/page1-before-mutation.xml"
|
|
local page2="$out_dir/page2-after-mutation.xml"
|
|
local fresh_after="$out_dir/fresh-after-mutation.xml"
|
|
local generation_mismatch="$out_dir/generation-mismatch.xml"
|
|
local degraded_response="$out_dir/degraded-fallback.xml"
|
|
local page1_keys="$out_dir/page1-before-mutation.keys"
|
|
local page2_keys="$out_dir/page2-after-mutation.keys"
|
|
local fresh_after_keys="$out_dir/fresh-after-mutation.keys"
|
|
local expected_page2_keys="$out_dir/expected-page2-after-mutation.keys"
|
|
local body_file="$out_dir/overwrite-body.bin"
|
|
local delete_out="$out_dir/delete.out"
|
|
local overwrite_out="$out_dir/overwrite.out"
|
|
local token first_key second_key multipart_key overlap last_key monotonic_status no_skip_status page1_count page2_count fresh_max_keys forged_token journal_sequence
|
|
|
|
echo "probe,status,detail" >"$summary"
|
|
echo "probe,status,detail" >"$fallback_summary"
|
|
|
|
signed_list_page "$bucket" "$page1" "$CHAOS_MAX_KEYS" ""
|
|
extract_xml_keys <"$page1" >"$page1_keys"
|
|
token="$(extract_next_token <"$page1" || true)"
|
|
page1_count="$(awk 'END{print NR + 0}' "$page1_keys")"
|
|
if [[ -z "$token" || "$page1_count" -lt 2 ]]; then
|
|
echo "initial_page,skipped,requires at least two keys and a continuation token" >>"$summary"
|
|
return 0
|
|
fi
|
|
echo "initial_page,ok,keys=${page1_count}; continuation_token=yes" >>"$summary"
|
|
|
|
first_key="$(sed -n '1p' "$page1_keys")"
|
|
second_key="$(sed -n '2p' "$page1_keys")"
|
|
printf 'metadata-fast overwrite probe %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$body_file"
|
|
|
|
signed_put_object "$bucket" "$first_key" "$body_file" "$overwrite_out"
|
|
echo "overwrite,ok,key=${first_key}" >>"$summary"
|
|
|
|
signed_delete_object "$bucket" "$second_key" "$delete_out"
|
|
echo "delete,ok,key=${second_key}" >>"$summary"
|
|
|
|
multipart_key="zzzz-metadata-fast-chaos-multipart-$(date +%s).bin"
|
|
signed_multipart_complete "$bucket" "$multipart_key" "$out_dir"
|
|
echo "multipart_complete,ok,key=${multipart_key}" >>"$summary"
|
|
|
|
signed_list_page "$bucket" "$page2" "$CHAOS_MAX_KEYS" "$token"
|
|
extract_xml_keys <"$page2" >"$page2_keys"
|
|
page2_count="$(awk 'END{print NR + 0}' "$page2_keys")"
|
|
|
|
overlap="$(comm -12 <(LC_ALL=C sort "$page1_keys") <(LC_ALL=C sort "$page2_keys") | awk 'END{print NR + 0}')"
|
|
if [[ "$overlap" -eq 0 ]]; then
|
|
echo "continuation_no_duplicate,ok,overlap=0" >>"$summary"
|
|
else
|
|
echo "continuation_no_duplicate,failed,overlap=${overlap}" >>"$summary"
|
|
return 1
|
|
fi
|
|
|
|
last_key="$(tail -n 1 "$page1_keys")"
|
|
monotonic_status="ok"
|
|
if [[ -n "$last_key" ]] && ! awk -v last="$last_key" 'NF && $0 <= last { bad = 1 } END { exit bad }' "$page2_keys"; then
|
|
monotonic_status="failed"
|
|
fi
|
|
echo "continuation_monotonic,${monotonic_status},page2_keys=${page2_count}; last_page1=${last_key}" >>"$summary"
|
|
[[ "$monotonic_status" == "ok" ]] || return 1
|
|
|
|
fresh_max_keys=$((page1_count + page2_count + CHAOS_MAX_KEYS))
|
|
signed_list_page "$bucket" "$fresh_after" "$fresh_max_keys" ""
|
|
extract_xml_keys <"$fresh_after" >"$fresh_after_keys"
|
|
awk -v last="$last_key" 'NF && $0 > last { print }' "$fresh_after_keys" | head -n "$page2_count" >"$expected_page2_keys"
|
|
no_skip_status="ok"
|
|
if ! cmp -s "$expected_page2_keys" "$page2_keys"; then
|
|
no_skip_status="failed"
|
|
fi
|
|
echo "continuation_no_skip,${no_skip_status},expected_next_keys=$(awk 'END{print NR + 0}' "$expected_page2_keys"); actual_page2_keys=${page2_count}" >>"$summary"
|
|
[[ "$no_skip_status" == "ok" ]] || return 1
|
|
|
|
forged_token="$(printf '%s[rustfs_cache:v2,return:,src:index_metadata_fast,gen:%s-mismatch]' "$last_key" "$INDEX_GENERATION" | base64_token)"
|
|
if signed_list_page "$bucket" "$generation_mismatch" "$CHAOS_MAX_KEYS" "$forged_token"; then
|
|
echo "generation_mismatch,ok,forged_generation=${INDEX_GENERATION}-mismatch" >>"$fallback_summary"
|
|
else
|
|
echo "generation_mismatch,failed,forged_generation=${INDEX_GENERATION}-mismatch" >>"$fallback_summary"
|
|
fi
|
|
|
|
if [[ -z "$NAMESPACE_JOURNAL_PATH" ]]; then
|
|
echo "degraded,skipped,requires --journal-path to avoid mutating quorum-backed journal state" >>"$fallback_summary"
|
|
return 0
|
|
fi
|
|
|
|
journal_sequence="$(journal_high_water_mark "$NAMESPACE_JOURNAL_PATH" "$bucket")"
|
|
write_local_journal_state "$NAMESPACE_JOURNAL_PATH" "$bucket" "$journal_sequence" "degraded"
|
|
if signed_list_page "$bucket" "$degraded_response" "$CHAOS_MAX_KEYS" ""; then
|
|
echo "degraded,ok,journal_path=${NAMESPACE_JOURNAL_PATH}; high_water_mark=${journal_sequence}" >>"$fallback_summary"
|
|
else
|
|
echo "degraded,failed,journal_path=${NAMESPACE_JOURNAL_PATH}; high_water_mark=${journal_sequence}" >>"$fallback_summary"
|
|
fi
|
|
write_local_journal_state "$NAMESPACE_JOURNAL_PATH" "$bucket" "$journal_sequence" "healthy"
|
|
echo "degraded_restore,ok,journal_path=${NAMESPACE_JOURNAL_PATH}; high_water_mark=${journal_sequence}" >>"$fallback_summary"
|
|
}
|
|
|
|
journal_high_water_mark() {
|
|
local root="$1"
|
|
local bucket="$2"
|
|
local state_file="${root}/${bucket}.state"
|
|
if [[ -f "$state_file" ]]; then
|
|
awk -F= '/^# high_water_mark=/ {print $2; found=1; exit} END {if(!found) print "0"}' "$state_file"
|
|
else
|
|
echo "0"
|
|
fi
|
|
}
|
|
|
|
write_local_journal_state() {
|
|
local root="$1"
|
|
local bucket="$2"
|
|
local high_water_mark="$3"
|
|
local status="$4"
|
|
local state_file="${root}/${bucket}.state"
|
|
local tmp_file="${state_file}.tmp"
|
|
mkdir -p "$root"
|
|
{
|
|
echo "# rustfs-listobjects-namespace-journal-v1"
|
|
echo "# bucket=${bucket}"
|
|
echo "# high_water_mark=${high_water_mark}"
|
|
echo "# status=${status}"
|
|
} >"$tmp_file"
|
|
mv "$tmp_file" "$state_file"
|
|
}
|
|
|
|
run_metadata_fast_quorum_journal_chaos() {
|
|
local bucket="$1"
|
|
local out_dir="$2"
|
|
mkdir -p "$out_dir"
|
|
|
|
local summary="$out_dir/quorum-journal-probes.csv"
|
|
local degraded_response="$out_dir/quorum-degraded-fallback.xml"
|
|
local disabled_gate_rebuild_response="$out_dir/quorum-disabled-gate-rebuild.xml"
|
|
local disabled_gate_response="$out_dir/quorum-disabled-gate.xml"
|
|
local restore_state_response="$out_dir/quorum-restore-state.xml"
|
|
local restore_rebuild_response="$out_dir/quorum-restore-rebuild.xml"
|
|
local restore_response="$out_dir/quorum-restore.xml"
|
|
local disabled_gate_fallbacks
|
|
|
|
echo "probe,status,detail" >"$summary"
|
|
if [[ "$RUN_QUORUM_JOURNAL_CHAOS" != "true" ]]; then
|
|
echo "quorum_degraded,skipped,RUN_QUORUM_JOURNAL_CHAOS=false" >>"$summary"
|
|
return 0
|
|
fi
|
|
if [[ -n "$NAMESPACE_JOURNAL_PATH" ]]; then
|
|
echo "quorum_degraded,skipped,local namespace journal override is active" >>"$summary"
|
|
return 0
|
|
fi
|
|
|
|
stop_server
|
|
|
|
JOURNAL_CHAOS_ENABLED=""
|
|
JOURNAL_CHAOS_BUCKET=""
|
|
JOURNAL_CHAOS_SEQUENCE=""
|
|
JOURNAL_CHAOS_STATUS=""
|
|
start_server opt_in_verified "$out_dir/quorum-disabled-gate-rebuild-rustfs.log" false
|
|
if signed_list_once "$bucket" "$disabled_gate_rebuild_response" 1; then
|
|
echo "quorum_disabled_gate_rebuild,ok,bucket=${bucket}; provider=rebuild" >>"$summary"
|
|
else
|
|
echo "quorum_disabled_gate_rebuild,failed,bucket=${bucket}; provider=rebuild" >>"$summary"
|
|
fi
|
|
stop_server
|
|
|
|
JOURNAL_CHAOS_ENABLED=""
|
|
JOURNAL_CHAOS_BUCKET="$bucket"
|
|
JOURNAL_CHAOS_SEQUENCE=""
|
|
JOURNAL_CHAOS_STATUS="degraded"
|
|
start_server metadata_fast "$out_dir/quorum-disabled-gate-rustfs.log" false
|
|
signed_list_page "$bucket" "$disabled_gate_response" "$CHAOS_MAX_KEYS" ""
|
|
sleep "$((METER_INTERVAL + 2))"
|
|
write_ratio_summary "metadata_fast_quorum_disabled_gate" "$out_dir/quorum-disabled-gate-rustfs.log" "$out_dir/quorum-disabled-gate-index-ratios.csv"
|
|
write_fallback_reason_summary \
|
|
"metadata_fast_quorum_disabled_gate" \
|
|
"$out_dir/quorum-disabled-gate-rustfs.log" \
|
|
"$out_dir/quorum-disabled-gate-index-fallback-reasons.csv"
|
|
disabled_gate_fallbacks="$(awk -F',' 'NR == 2 {print $3 + 0}' "$out_dir/quorum-disabled-gate-index-ratios.csv")"
|
|
if [[ "$disabled_gate_fallbacks" -eq 0 ]]; then
|
|
echo "quorum_disabled_gate,ok,bucket=${bucket}; status=degraded; enabled=unset" >>"$summary"
|
|
else
|
|
echo "quorum_disabled_gate,failed,bucket=${bucket}; status=degraded; enabled=unset; fallbacks=${disabled_gate_fallbacks}" >>"$summary"
|
|
fi
|
|
stop_server
|
|
|
|
JOURNAL_CHAOS_ENABLED="true"
|
|
JOURNAL_CHAOS_BUCKET="$bucket"
|
|
JOURNAL_CHAOS_SEQUENCE=""
|
|
JOURNAL_CHAOS_STATUS="degraded"
|
|
start_server metadata_fast "$out_dir/quorum-degraded-rustfs.log" false
|
|
if signed_list_page "$bucket" "$degraded_response" "$CHAOS_MAX_KEYS" ""; then
|
|
echo "quorum_degraded,ok,bucket=${bucket}; status=degraded" >>"$summary"
|
|
else
|
|
echo "quorum_degraded,failed,bucket=${bucket}; status=degraded" >>"$summary"
|
|
fi
|
|
sleep "$((METER_INTERVAL + 2))"
|
|
write_ratio_summary "metadata_fast_quorum_degraded" "$out_dir/quorum-degraded-rustfs.log" "$out_dir/quorum-degraded-index-ratios.csv"
|
|
write_fallback_reason_summary \
|
|
"metadata_fast_quorum_degraded" \
|
|
"$out_dir/quorum-degraded-rustfs.log" \
|
|
"$out_dir/quorum-degraded-index-fallback-reasons.csv"
|
|
stop_server
|
|
|
|
JOURNAL_CHAOS_ENABLED="true"
|
|
JOURNAL_CHAOS_STATUS="healthy"
|
|
start_server metadata_fast "$out_dir/quorum-restore-state-rustfs.log" false
|
|
if signed_list_page "$bucket" "$restore_state_response" "$CHAOS_MAX_KEYS" ""; then
|
|
echo "quorum_restore_state,ok,bucket=${bucket}; status=healthy" >>"$summary"
|
|
else
|
|
echo "quorum_restore_state,failed,bucket=${bucket}; status=healthy" >>"$summary"
|
|
fi
|
|
stop_server
|
|
|
|
JOURNAL_CHAOS_ENABLED=""
|
|
JOURNAL_CHAOS_BUCKET=""
|
|
JOURNAL_CHAOS_SEQUENCE=""
|
|
JOURNAL_CHAOS_STATUS=""
|
|
|
|
start_server opt_in_verified "$out_dir/quorum-restore-rebuild-rustfs.log" false
|
|
if signed_list_once "$bucket" "$restore_rebuild_response" 1; then
|
|
echo "quorum_restore_rebuild,ok,bucket=${bucket}; provider=rebuild" >>"$summary"
|
|
else
|
|
echo "quorum_restore_rebuild,failed,bucket=${bucket}; provider=rebuild" >>"$summary"
|
|
fi
|
|
stop_server
|
|
|
|
start_server metadata_fast "$out_dir/quorum-restore-rustfs.log" false
|
|
if signed_list_page "$bucket" "$restore_response" "$CHAOS_MAX_KEYS" ""; then
|
|
echo "quorum_restore,ok,bucket=${bucket}; status=healthy; provider=metadata_fast" >>"$summary"
|
|
else
|
|
echo "quorum_restore,failed,bucket=${bucket}; status=healthy; provider=metadata_fast" >>"$summary"
|
|
fi
|
|
sleep "$((METER_INTERVAL + 2))"
|
|
write_ratio_summary "metadata_fast_quorum_restore" "$out_dir/quorum-restore-rustfs.log" "$out_dir/quorum-restore-index-ratios.csv"
|
|
write_fallback_reason_summary \
|
|
"metadata_fast_quorum_restore" \
|
|
"$out_dir/quorum-restore-rustfs.log" \
|
|
"$out_dir/quorum-restore-index-fallback-reasons.csv"
|
|
stop_server
|
|
}
|
|
|
|
run_mode() {
|
|
local mode="$1"
|
|
local bucket="$2"
|
|
local reset_data="${3:-false}"
|
|
local mode_dir="$OUT_DIR/$mode"
|
|
mkdir -p "$mode_dir"
|
|
|
|
log_info "Starting RustFS for mode=$mode"
|
|
start_server "$mode" "$mode_dir/rustfs.log" "$reset_data"
|
|
start_resource_sampler "$mode_dir/resource-samples.csv"
|
|
|
|
if [[ "$mode" == "opt_in_verified" && "$PREBUILD_KEY_INDEX" != "true" ]] \
|
|
&& [[ "$INDEX_PROVIDER" == "persistent_key_only" || "$INDEX_PROVIDER" == "persisted_key_only" ]]; then
|
|
log_info "Warming persistent key-only provider through production rebuild path"
|
|
signed_list_once "$bucket" "$mode_dir/provider-warmup.xml" 1
|
|
fi
|
|
|
|
log_info "Signed ListObjectsV2 bench for mode=$mode duration=$DURATION"
|
|
run_signed_list_bench "$mode" "$bucket" "$DURATION" "$mode_dir"
|
|
stop_resource_sampler
|
|
sleep "$((METER_INTERVAL + 2))"
|
|
write_ratio_summary "$mode" "$mode_dir/rustfs.log" "$mode_dir/index-ratios.csv"
|
|
write_fallback_reason_summary "$mode" "$mode_dir/rustfs.log" "$mode_dir/index-fallback-reasons.csv"
|
|
write_resource_summary "$mode_dir/resource-samples.csv" "$mode_dir/resource-summary.csv"
|
|
write_prometheus_snapshot "$mode" "$mode_dir"
|
|
|
|
if [[ "$mode" == "metadata_fast" && "$RUN_CHAOS" == "true" ]]; then
|
|
log_info "Running metadata-fast chaos probes"
|
|
run_metadata_fast_chaos "$bucket" "$mode_dir/chaos"
|
|
sleep "$((METER_INTERVAL + 2))"
|
|
write_ratio_summary "${mode}_after_chaos" "$mode_dir/rustfs.log" "$mode_dir/chaos/index-ratios-after-chaos.csv"
|
|
write_fallback_reason_summary "${mode}_after_chaos" "$mode_dir/rustfs.log" "$mode_dir/chaos/index-fallback-reasons-after-chaos.csv"
|
|
log_info "Running metadata-fast quorum-backed namespace journal degraded probe"
|
|
run_metadata_fast_quorum_journal_chaos "$bucket" "$mode_dir/chaos"
|
|
fi
|
|
|
|
stop_server
|
|
}
|
|
|
|
write_manifest() {
|
|
mkdir -p "$OUT_DIR"
|
|
{
|
|
echo "out_dir=$OUT_DIR"
|
|
echo "data_root=$DATA_ROOT"
|
|
echo "build_profile=$BUILD_PROFILE"
|
|
echo "rustfs_bin=$RUSTFS_BIN"
|
|
echo "warp_bin=$WARP_BIN"
|
|
echo "address=$ADDRESS"
|
|
echo "objects=$OBJECTS"
|
|
echo "object_size=$OBJECT_SIZE"
|
|
echo "concurrency=$CONCURRENCY"
|
|
echo "max_keys=$MAX_KEYS"
|
|
echo "duration=$DURATION"
|
|
echo "warmup_duration=$WARMUP_DURATION"
|
|
echo "meter_interval=$METER_INTERVAL"
|
|
echo "resource_sample_interval=$RESOURCE_SAMPLE_INTERVAL"
|
|
echo "index_provider=$INDEX_PROVIDER"
|
|
echo "index_generation=$INDEX_GENERATION"
|
|
echo "key_index_path=$KEY_INDEX_PATH"
|
|
echo "namespace_journal_path=${NAMESPACE_JOURNAL_PATH:-.rustfs.sys/listobjects/ns-journal/v1}"
|
|
echo "prebuild_key_index=$PREBUILD_KEY_INDEX"
|
|
echo "prepare_only=$PREPARE_ONLY"
|
|
echo "reuse_prepared_data=$REUSE_PREPARED_DATA"
|
|
echo "run_metadata_fast=$RUN_METADATA_FAST"
|
|
echo "run_chaos=$RUN_CHAOS"
|
|
echo "run_quorum_journal_chaos=$RUN_QUORUM_JOURNAL_CHAOS"
|
|
echo "chaos_max_keys=$CHAOS_MAX_KEYS"
|
|
echo "metadata_fast_staleness_ms=$METADATA_FAST_STALENESS_MS"
|
|
echo "prometheus_query_url=$PROMETHEUS_QUERY_URL"
|
|
echo "prometheus_job=$PROMETHEUS_JOB"
|
|
} >"$OUT_DIR/manifest.env"
|
|
}
|
|
|
|
summary_csv_section() {
|
|
local title="$1"
|
|
local file="$2"
|
|
if [[ -f "$file" ]]; then
|
|
echo "### $title"
|
|
echo
|
|
cat "$file"
|
|
echo
|
|
fi
|
|
}
|
|
|
|
write_combined_summary() {
|
|
local summary="$OUT_DIR/summary.md"
|
|
{
|
|
echo "# ListObjects Verified Benchmark Summary"
|
|
echo
|
|
echo "## Manifest"
|
|
echo
|
|
sed 's/^/- /' "$OUT_DIR/manifest.env"
|
|
echo
|
|
echo "## Warp"
|
|
echo
|
|
summary_csv_section "Walker" "$OUT_DIR/walker/list-summary.csv"
|
|
summary_csv_section "Opt-in verified" "$OUT_DIR/opt_in_verified/list-summary.csv"
|
|
summary_csv_section "Metadata-fast" "$OUT_DIR/metadata_fast/list-summary.csv"
|
|
echo
|
|
echo "## Index Ratios"
|
|
echo
|
|
summary_csv_section "Walker" "$OUT_DIR/walker/index-ratios.csv"
|
|
summary_csv_section "Opt-in verified" "$OUT_DIR/opt_in_verified/index-ratios.csv"
|
|
summary_csv_section "Metadata-fast" "$OUT_DIR/metadata_fast/index-ratios.csv"
|
|
echo
|
|
echo "## Fallback Reasons"
|
|
echo
|
|
summary_csv_section "Walker" "$OUT_DIR/walker/index-fallback-reasons.csv"
|
|
summary_csv_section "Opt-in verified" "$OUT_DIR/opt_in_verified/index-fallback-reasons.csv"
|
|
summary_csv_section "Metadata-fast" "$OUT_DIR/metadata_fast/index-fallback-reasons.csv"
|
|
echo
|
|
echo "## Resource Samples"
|
|
echo
|
|
summary_csv_section "Walker" "$OUT_DIR/walker/resource-summary.csv"
|
|
summary_csv_section "Opt-in verified" "$OUT_DIR/opt_in_verified/resource-summary.csv"
|
|
summary_csv_section "Metadata-fast" "$OUT_DIR/metadata_fast/resource-summary.csv"
|
|
if [[ -f "$OUT_DIR/metadata_fast/chaos/chaos-summary.csv" ]]; then
|
|
echo
|
|
echo "## Metadata-fast Chaos"
|
|
echo
|
|
summary_csv_section "Chaos probes" "$OUT_DIR/metadata_fast/chaos/chaos-summary.csv"
|
|
summary_csv_section "Fallback probes" "$OUT_DIR/metadata_fast/chaos/fallback-probes.csv"
|
|
summary_csv_section "Ratios after chaos" "$OUT_DIR/metadata_fast/chaos/index-ratios-after-chaos.csv"
|
|
summary_csv_section "Fallback reasons after chaos" "$OUT_DIR/metadata_fast/chaos/index-fallback-reasons-after-chaos.csv"
|
|
summary_csv_section "Quorum journal probes" "$OUT_DIR/metadata_fast/chaos/quorum-journal-probes.csv"
|
|
summary_csv_section "Quorum disabled-gate ratios" "$OUT_DIR/metadata_fast/chaos/quorum-disabled-gate-index-ratios.csv"
|
|
summary_csv_section "Quorum disabled-gate fallback reasons" "$OUT_DIR/metadata_fast/chaos/quorum-disabled-gate-index-fallback-reasons.csv"
|
|
summary_csv_section "Quorum degraded ratios" "$OUT_DIR/metadata_fast/chaos/quorum-degraded-index-ratios.csv"
|
|
summary_csv_section "Quorum degraded fallback reasons" "$OUT_DIR/metadata_fast/chaos/quorum-degraded-index-fallback-reasons.csv"
|
|
summary_csv_section "Quorum restore ratios" "$OUT_DIR/metadata_fast/chaos/quorum-restore-index-ratios.csv"
|
|
summary_csv_section "Quorum restore fallback reasons" "$OUT_DIR/metadata_fast/chaos/quorum-restore-index-fallback-reasons.csv"
|
|
fi
|
|
} >"$summary"
|
|
}
|
|
|
|
main() {
|
|
parse_args "$@"
|
|
require_cmds
|
|
write_manifest
|
|
build_rustfs
|
|
trap stop_server EXIT INT TERM
|
|
|
|
local bucket="rustfs-listobjects-bench"
|
|
local prep_dir="$OUT_DIR/prepare"
|
|
mkdir -p "$prep_dir"
|
|
|
|
if [[ "$REUSE_PREPARED_DATA" == "true" ]]; then
|
|
log_info "Reusing prepared data root: $DATA_ROOT"
|
|
else
|
|
log_info "Starting RustFS for data preparation"
|
|
start_server walker "$prep_dir/rustfs.log" true
|
|
log_info "Preparing bucket=$bucket objects=$OBJECTS with warp list"
|
|
prepare_bucket "$bucket" "$WARMUP_DURATION" "$prep_dir/warp-prepare.log" || {
|
|
tail -n 80 "$prep_dir/warp-prepare.log" >&2 || true
|
|
return 1
|
|
}
|
|
fi
|
|
|
|
if [[ "$PREBUILD_KEY_INDEX" == "true" ]]; then
|
|
if [[ "$REUSE_PREPARED_DATA" == "true" ]]; then
|
|
log_info "Starting RustFS to build persistent key-only index from reused data"
|
|
start_server walker "$prep_dir/rustfs.log" false
|
|
fi
|
|
log_info "Writing persistent key-only index from bench script: $KEY_INDEX_PATH"
|
|
write_key_index "$bucket" "$KEY_INDEX_PATH"
|
|
stop_server
|
|
else
|
|
log_info "Skipping bench-side key index generation; opt-in mode will rebuild through RustFS lifecycle"
|
|
stop_server
|
|
fi
|
|
|
|
if [[ "$PREPARE_ONLY" == "true" ]]; then
|
|
log_info "Prepare-only mode complete: $DATA_ROOT"
|
|
return 0
|
|
fi
|
|
|
|
run_mode walker "$bucket" false
|
|
run_mode opt_in_verified "$bucket" false
|
|
if [[ "$RUN_METADATA_FAST" == "true" ]]; then
|
|
run_mode metadata_fast "$bucket" false
|
|
fi
|
|
write_combined_summary
|
|
|
|
log_info "Summary: $OUT_DIR/summary.md"
|
|
}
|
|
|
|
main "$@"
|