diff --git a/scripts/run_hotpath_warp_abba.sh b/scripts/run_hotpath_warp_abba.sh index af77230f7..8a57fee59 100755 --- a/scripts/run_hotpath_warp_abba.sh +++ b/scripts/run_hotpath_warp_abba.sh @@ -43,6 +43,14 @@ EXEMPTION_REASON="deliberate correctness tradeoff" OUT_DIR="${PROJECT_ROOT}/target/hotpath-abba/$(date -u +%Y%m%dT%H%M%SZ 2>/dev/null || echo run)" DRY_RUN=false ALLOW_UNMANAGED_EXTERNAL=false +AFTER_PROBE=false +NODE_SSH_TARGETS=() +NODE_SSH_IDENTITY_FILE="" +NODE_SSH_TIMEOUT_SECS=30 +REQUIRE_NODE_TELEMETRY=false +SERVICE_PROMETHEUS_QUERY_URL="" +SERVICE_PROMETHEUS_QUERY="" +SERVICE_METRICS_SERVICE_NAME="" WORKLOADS=( "put-4kib|put|4KiB" @@ -52,7 +60,9 @@ WORKLOADS=( "get-10mib|get|10MiB" "mixed-256k|mixed|256KiB" ) +WORKLOAD_OVERRIDES=() DRIVE_SYNC_MATRIX=("sync-on|true" "sync-off|false") +DRIVE_SYNC_OVERRIDES=() usage() { cat <<'USAGE' @@ -94,6 +104,22 @@ Production / cluster mode: mode, then write a non-empty evidence file. --allow-unmanaged-external Preserve legacy external mode without a deploy hook. Its output is not formal ABBA evidence. + --after-probe Enable per-round PUT HEAD/GET/hash verification. + --node-ssh-target + Capture node CPU/RSS/IOPS/await/PSI/scheduler telemetry. + --node-ssh-identity-file + SSH identity file for node telemetry. + --node-ssh-timeout-secs + Bound each remote telemetry command (default 30). + --require-node-telemetry Fail a round if node telemetry is incomplete. + --service-prometheus-query-url + Prometheus query endpoint for PUT stage snapshots. + --service-prometheus-query + PromQL selector; include internode RPC metrics when needed. + --service-metrics-service-name + Optional service.name filter for stage snapshots. + --workload Restrict the matrix; repeat for multiple workloads. + --drive-sync Restrict durability cells; repeat if needed. --health-path Readiness path (default /health). Benchmark: @@ -150,6 +176,23 @@ validate_positive_int() { [[ "$value" =~ ^[0-9]+$ && "$value" -gt 0 ]] || die "$name must be a positive integer" } +validate_workload_spec() { + local spec="$1" name mode size extra + IFS='|' read -r name mode size extra <<<"$spec" + [[ -n "$name" && -n "$mode" && -n "$size" && -z "${extra:-}" ]] || die "--workload must be name|mode|size" + case "$mode" in + put|get|mixed) ;; + *) die "--workload mode must be put, get, or mixed" ;; + esac +} + +validate_drive_sync_spec() { + local spec="$1" label value extra + IFS='|' read -r label value extra <<<"$spec" + [[ -n "$label" && -n "$value" && -z "${extra:-}" ]] || die "--drive-sync must be label|true|false" + [[ "$value" == "true" || "$value" == "false" ]] || die "--drive-sync value must be true or false" +} + while [[ $# -gt 0 ]]; do case "$1" in --baseline-bin) BASELINE_BIN="$2"; shift 2 ;; @@ -178,11 +221,34 @@ while [[ $# -gt 0 ]]; do --out-dir) OUT_DIR="$2"; shift 2 ;; --dry-run) DRY_RUN=true; shift ;; --allow-unmanaged-external) ALLOW_UNMANAGED_EXTERNAL=true; shift ;; + --after-probe) AFTER_PROBE=true; shift ;; + --node-ssh-target) NODE_SSH_TARGETS+=("$2"); shift 2 ;; + --node-ssh-identity-file) NODE_SSH_IDENTITY_FILE="$2"; shift 2 ;; + --node-ssh-timeout-secs) NODE_SSH_TIMEOUT_SECS="$2"; shift 2 ;; + --require-node-telemetry) REQUIRE_NODE_TELEMETRY=true; shift ;; + --service-prometheus-query-url) SERVICE_PROMETHEUS_QUERY_URL="$2"; shift 2 ;; + --service-prometheus-query) SERVICE_PROMETHEUS_QUERY="$2"; shift 2 ;; + --service-metrics-service-name) SERVICE_METRICS_SERVICE_NAME="$2"; shift 2 ;; + --workload) WORKLOAD_OVERRIDES+=("$2"); shift 2 ;; + --drive-sync) DRIVE_SYNC_OVERRIDES+=("$2"); shift 2 ;; -h|--help) usage; exit 0 ;; *) die "unknown argument: $1" ;; esac done +if ((${#WORKLOAD_OVERRIDES[@]} > 0)); then + for workload in "${WORKLOAD_OVERRIDES[@]}"; do + validate_workload_spec "$workload" + done + WORKLOADS=("${WORKLOAD_OVERRIDES[@]}") +fi +if ((${#DRIVE_SYNC_OVERRIDES[@]} > 0)); then + for drive_sync in "${DRIVE_SYNC_OVERRIDES[@]}"; do + validate_drive_sync_spec "$drive_sync" + done + DRIVE_SYNC_MATRIX=("${DRIVE_SYNC_OVERRIDES[@]}") +fi + validate_positive_int "$DISKS" "--disks" validate_positive_int "$CONCURRENCY" "--concurrency" validate_positive_int "$ROUNDS" "--rounds" @@ -354,6 +420,7 @@ EOF measure() { local leg="$1" workload="$2" mode="$3" size="$4" sync_label="$5" bucket="$6" baseline_csv="${7:-}" + local node_target local cell="$OUT_DIR/$workload/$sync_label/$leg" local args=( --tool warp --warp-bin "$WARP_BIN" --warp-mode "$mode" @@ -362,6 +429,24 @@ measure() { --duration "$DURATION" --rounds "$ROUNDS" --cooldown-secs "$COOLDOWN_SECS" --out-dir "$cell" ) + if [[ "$AFTER_PROBE" == "true" && "$mode" == "put" ]]; then + args+=(--after-probe) + fi + if ((${#NODE_SSH_TARGETS[@]} > 0)); then + for node_target in "${NODE_SSH_TARGETS[@]}"; do + args+=(--node-ssh-target "$node_target") + done + fi + [[ -n "$NODE_SSH_IDENTITY_FILE" ]] && args+=(--node-ssh-identity-file "$NODE_SSH_IDENTITY_FILE") + if [[ "$REQUIRE_NODE_TELEMETRY" == "true" ]]; then + args+=(--require-node-telemetry) + fi + args+=(--node-ssh-timeout-secs "$NODE_SSH_TIMEOUT_SECS") + if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then + args+=(--service-prometheus-query-url "$SERVICE_PROMETHEUS_QUERY_URL" --service-metrics-dir "$cell/service_metrics") + [[ -n "$SERVICE_PROMETHEUS_QUERY" ]] && args+=(--service-prometheus-query "$SERVICE_PROMETHEUS_QUERY") + [[ -n "$SERVICE_METRICS_SERVICE_NAME" ]] && args+=(--service-metrics-service-name "$SERVICE_METRICS_SERVICE_NAME") + fi if [[ "$mode" != "put" ]]; then # Warp defaults to 2,500 setup objects per round. At 10 MiB that writes # 25 GiB before every 12-second measurement, so the matrix cannot finish @@ -462,6 +547,14 @@ external_isolation=$(isolation_mode) evidence_mode=$(evidence_mode) formal_evidence=$(formal_evidence) performance_conclusion=$(performance_conclusion) +after_probe=$AFTER_PROBE +node_ssh_target_count=${#NODE_SSH_TARGETS[@]} +node_ssh_identity_file=${NODE_SSH_IDENTITY_FILE:-N/A} +node_ssh_timeout_secs=$NODE_SSH_TIMEOUT_SECS +require_node_telemetry=$REQUIRE_NODE_TELEMETRY +service_prometheus_query_url=${SERVICE_PROMETHEUS_QUERY_URL:-N/A} +service_prometheus_query=${SERVICE_PROMETHEUS_QUERY:-default} +service_metrics_service_name=${SERVICE_METRICS_SERVICE_NAME:-N/A} dataset_namespace=$DATASET_NAMESPACE local_run_data_root=$RUN_DATA_ROOT bucket_isolation=per-leg diff --git a/scripts/run_object_batch_bench_enhanced.sh b/scripts/run_object_batch_bench_enhanced.sh index 56f368c23..939645133 100755 --- a/scripts/run_object_batch_bench_enhanced.sh +++ b/scripts/run_object_batch_bench_enhanced.sh @@ -61,6 +61,13 @@ REQUIRE_SERVER_PROVENANCE=false RUN_LABELS=() NODE_METRICS_URLS=() NODE_DOCKER_CONTAINERS=() +NODE_SSH_TARGETS=() +NODE_SSH_IDENTITY_FILE="" +NODE_SSH_TIMEOUT_SECS=30 +REQUIRE_NODE_TELEMETRY=false +NODE_TELEMETRY_FAILED=false +AFTER_PROBE=false +AFTER_PROBE_DIR="" usage() { cat <<'USAGE' @@ -135,6 +142,14 @@ Enhanced options: --node-docker-container Repeatable Docker container mapped to a node; captures CPU, memory, network, and block I/O before/after each round + --node-ssh-target Repeatable SSH target; captures process CPU/RSS, + iostat IOPS/await, and /proc/pressure/io before/after + --node-ssh-identity-file + Optional SSH private key used for node telemetry + --node-ssh-timeout-secs Bound each remote telemetry command (default 30) + --require-node-telemetry Mark a round failed when any node snapshot fails + --after-probe After successful PUT, HEAD and GET one sampled object, + verify length, ETag presence, and repeatable SHA-256 Output files: round_results.csv One row per round attempt (with retry trace) @@ -144,6 +159,7 @@ Output files: node_inventory.csv Per-node container and immutable image identity node_metrics_captures.csv Per-node metric snapshot inventory node_resource_captures.csv Per-node Docker resource and block I/O snapshots + after_probe.csv Sampled PUT object HEAD/GET length/hash verification Example: scripts/run_object_batch_bench_enhanced.sh \ @@ -171,6 +187,35 @@ normalize_warp_host() { echo "$raw" } +normalize_s3_endpoint() { + local raw="$1" + if [[ "$raw" == http://* || "$raw" == https://* ]]; then + echo "$raw" + else + echo "http://$raw" + fi +} + +size_to_bytes() { + local value="$1" number unit factor + if [[ "$value" =~ ^([0-9]+)(B|KB|MB|GB|KiB|MiB|GiB)$ ]]; then + number="${BASH_REMATCH[1]}" + unit="${BASH_REMATCH[2]}" + case "$unit" in + B) factor=1 ;; + KB) factor=1000 ;; + MB) factor=1000000 ;; + GB) factor=1000000000 ;; + KiB) factor=1024 ;; + MiB) factor=1048576 ;; + GiB) factor=1073741824 ;; + esac + echo $((number * factor)) + else + echo N/A + fi +} + parse_args() { while [[ $# -gt 0 ]]; do case "$1" in @@ -217,6 +262,11 @@ parse_args() { --label) RUN_LABELS+=("$2"); shift 2 ;; --node-metrics-url) NODE_METRICS_URLS+=("$2"); shift 2 ;; --node-docker-container) NODE_DOCKER_CONTAINERS+=("$2"); shift 2 ;; + --node-ssh-target) NODE_SSH_TARGETS+=("$2"); shift 2 ;; + --node-ssh-identity-file) NODE_SSH_IDENTITY_FILE="$2"; shift 2 ;; + --node-ssh-timeout-secs) NODE_SSH_TIMEOUT_SECS="$2"; shift 2 ;; + --require-node-telemetry) REQUIRE_NODE_TELEMETRY=true; shift ;; + --after-probe) AFTER_PROBE=true; shift ;; --extra-args) # shellcheck disable=SC2206 EXTRA_ARGS=($2) @@ -317,17 +367,17 @@ default_service_metrics_filter_regex() { return ;; put) - echo "rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_zero_copy_write|rustfs_buffer_|rustfs_ec_|rustfs_io_bytespool_" + echo "rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_zero_copy_write|rustfs_buffer_|rustfs_ec_|rustfs_io_bytespool_|rustfs_system_network_internode_" return ;; mixed) - echo "rustfs_s3_get_object_|rustfs_io_get_object_|rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_zero_copy_|rustfs_buffer_|rustfs_ec_" + echo "rustfs_s3_get_object_|rustfs_io_get_object_|rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_zero_copy_|rustfs_buffer_|rustfs_ec_|rustfs_system_network_internode_" return ;; esac fi - echo "rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_s3_get_object_|rustfs_io_get_object_" + echo "rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_s3_get_object_|rustfs_io_get_object_|rustfs_system_network_internode_" } default_service_prometheus_query() { @@ -338,17 +388,17 @@ default_service_prometheus_query() { return ;; put) - echo '{__name__=~"rustfs_(s3_put_object|io_put_object|zero_copy_write|buffer|ec|io_bytespool)_.*"}' + echo '{__name__=~"rustfs_(s3_put_object|io_put_object|zero_copy_write|buffer|ec|io_bytespool|system_network_internode)_.*"}' return ;; mixed) - echo '{__name__=~"rustfs_(s3_get_object|io_get_object|s3_put_object|io_put_object|zero_copy|buffer|ec)_.*"}' + echo '{__name__=~"rustfs_(s3_get_object|io_get_object|s3_put_object|io_put_object|zero_copy|buffer|ec|system_network_internode)_.*"}' return ;; esac fi - echo '{__name__=~"rustfs_(s3_put_object|io_put_object|s3_get_object|io_get_object)_.*"}' + echo '{__name__=~"rustfs_(s3_put_object|io_put_object|s3_get_object|io_get_object|system_network_internode)_.*"}' } validate_args() { @@ -370,6 +420,7 @@ validate_args() { validate_positive_int "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS" "--service-metrics-connect-timeout-secs" validate_positive_int "$SERVICE_METRICS_MAX_TIME_SECS" "--service-metrics-max-time-secs" validate_nonnegative_int "$SERVICE_METRICS_SETTLE_SECS" "--service-metrics-settle-secs" + validate_positive_int "$NODE_SSH_TIMEOUT_SECS" "--node-ssh-timeout-secs" if [[ -n "$SERVICE_METRICS_URL" && -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then echo "ERROR: --service-metrics-url and --service-prometheus-query-url are mutually exclusive" >&2 exit 1 @@ -387,6 +438,11 @@ validate_args() { if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)) && [[ "$DRY_RUN" != "true" ]]; then require_cmd docker fi + if [[ "$AFTER_PROBE" == "true" ]] && [[ "$DRY_RUN" != "true" ]]; then + require_cmd mc + require_cmd jq + require_cmd sha256sum + fi if [[ "$TOOL" == "s3bench" ]]; then validate_positive_int "$SAMPLES" "--samples" fi @@ -404,6 +460,15 @@ validate_args() { if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)); then validate_named_values "--node-docker-container" "${NODE_DOCKER_CONTAINERS[@]}" fi + if ((${#NODE_SSH_TARGETS[@]} > 0)); then + validate_named_values "--node-ssh-target" "${NODE_SSH_TARGETS[@]}" + require_cmd ssh + [[ "$DRY_RUN" == "true" ]] || require_cmd timeout + if [[ -n "$NODE_SSH_IDENTITY_FILE" && "$DRY_RUN" != "true" && ! -r "$NODE_SSH_IDENTITY_FILE" ]]; then + echo "ERROR: --node-ssh-identity-file is not readable: $NODE_SSH_IDENTITY_FILE" >&2 + exit 1 + fi + fi validate_manifest_value "$SERVER_IMAGE_REF" "--server-image-ref" validate_manifest_value "$SERVER_IMAGE_DIGEST" "--server-image-digest" validate_manifest_value "$SERVER_REVISION" "--server-revision" @@ -533,6 +598,11 @@ EOF fi echo "node_metrics_count=${#NODE_METRICS_URLS[@]}" echo "node_docker_container_count=${#NODE_DOCKER_CONTAINERS[@]}" + echo "node_ssh_target_count=${#NODE_SSH_TARGETS[@]}" + echo "node_ssh_identity_file=${NODE_SSH_IDENTITY_FILE:-N/A}" + echo "node_ssh_timeout_secs=${NODE_SSH_TIMEOUT_SECS}" + echo "require_node_telemetry=${REQUIRE_NODE_TELEMETRY}" + echo "after_probe=${AFTER_PROBE}" } >>"$manifest_file" } @@ -573,6 +643,9 @@ setup_output() { if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)); then mkdir -p "$OUT_DIR/node_resources" fi + if ((${#NODE_SSH_TARGETS[@]} > 0)); then + mkdir -p "$OUT_DIR/node_telemetry" + fi ROUND_CSV="$OUT_DIR/round_results.csv" MEDIAN_CSV="$OUT_DIR/median_summary.csv" @@ -580,6 +653,9 @@ setup_output() { SERVICE_METRICS_CSV="$OUT_DIR/service_metrics_captures.csv" NODE_METRICS_CSV="$OUT_DIR/node_metrics_captures.csv" NODE_RESOURCE_CSV="$OUT_DIR/node_resource_captures.csv" + NODE_SSH_CSV="$OUT_DIR/node_telemetry_captures.csv" + AFTER_PROBE_CSV="$OUT_DIR/after_probe.csv" + AFTER_PROBE_DIR="${OUT_DIR}/after_probe" echo "size,tool,round,attempt,concurrency,status,exit_code,round_started_at_utc,round_finished_at_utc,throughput_human,throughput_bps,reqps,latency_human,latency_ms,log_file,req_p90_human,req_p90_ms,req_p99_human,req_p99_ms" > "$ROUND_CSV" echo "size,tool,concurrency,successful_rounds,failed_rounds,median_throughput_bps,median_reqps,median_latency_ms,median_req_p90_ms,median_req_p99_ms" > "$MEDIAN_CSV" @@ -592,6 +668,14 @@ setup_output() { if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)); then echo "size,tool,round,attempt,phase,node,container,status,snapshot_file" > "$NODE_RESOURCE_CSV" fi + if ((${#NODE_SSH_TARGETS[@]} > 0)); then + mkdir -p "$OUT_DIR/node_telemetry" + echo "size,tool,round,attempt,phase,node,host,status,snapshot_file" > "$NODE_SSH_CSV" + fi + if [[ "$AFTER_PROBE" == "true" ]]; then + mkdir -p "$AFTER_PROBE_DIR" + echo "size,tool,round,attempt,status,bucket,key,expected_size,head_size,get_size,hash1,hash2,etag,error" > "$AFTER_PROBE_CSV" + fi write_run_manifest write_node_inventory } @@ -1088,6 +1172,138 @@ capture_round_node_resources() { done } +capture_round_node_telemetry() { + local size="$1" round="$2" attempt="$3" phase="$4" entry node host token snapshot_file + if ((${#NODE_SSH_TARGETS[@]} == 0)); then + return + fi + for entry in "${NODE_SSH_TARGETS[@]}"; do + node="${entry%%=*}" + host="${entry#*=}" + local -a ssh_args=(-o BatchMode=yes -o ConnectTimeout=5) + if [[ -n "$NODE_SSH_IDENTITY_FILE" ]]; then + ssh_args+=(-i "$NODE_SSH_IDENTITY_FILE") + fi + token="$(metric_snapshot_token "$size" "$round" "$attempt")" + snapshot_file="$OUT_DIR/node_telemetry/${token}_${node}_${phase}.txt" + if [[ "$DRY_RUN" == "true" ]]; then + : >"$snapshot_file" + echo "$size,$TOOL,$round,$attempt,$phase,$node,$host,not_run_dry_run,$snapshot_file" >>"$NODE_SSH_CSV" + continue + fi + if timeout "$NODE_SSH_TIMEOUT_SECS" ssh "${ssh_args[@]}" "$host" 'set -u + pid="$(pgrep -o -f /usr/local/bin/rustfs || true)" + echo "captured_at_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "pid=${pid:-N/A}" + if [[ -n "$pid" ]]; then + process_metrics="$(ps -p "$pid" -o pcpu=,pmem=,rss=,vsz=,nlwp= | awk '\''NR == 1 {print $1, $2, $3, $4, $5}'\'')" + read -r process_cpu process_mem process_rss process_vsz process_threads <<<"$process_metrics" + echo "process_cpu_percent=${process_cpu:-N/A}" + echo "process_mem_percent=${process_mem:-N/A}" + echo "process_rss_kb=${process_rss:-N/A}" + echo "process_vsz_kb=${process_vsz:-N/A}" + echo "process_threads=${process_threads:-N/A}" + else + echo "process_status=missing" + fi + if command -v iostat >/dev/null 2>&1; then + echo "iostat_begin" + iostat -c -dx 1 1 2>/dev/null || true + echo "iostat_end" + else + echo "iostat_status=unavailable" + fi + echo "scheduler_begin" + echo "loadavg=$(cat /proc/loadavg 2>/dev/null || echo N/A)" + awk "/^ctxt / || /^procs_running / || /^procs_blocked / {print}" /proc/stat 2>/dev/null || true + if command -v vmstat >/dev/null 2>&1; then + vmstat_sample="$(vmstat 1 2 2>/dev/null | tail -n 1 || true)" + echo "vmstat_sample=${vmstat_sample:-N/A}" + if [[ -n "$vmstat_sample" ]]; then + read -r vm_r vm_b vm_swpd vm_free vm_buff vm_cache vm_si vm_so vm_bi vm_bo vm_in vm_cs vm_us vm_sy vm_id vm_wa vm_st <<<"$vmstat_sample" + echo "vmstat_r=${vm_r:-N/A}" + echo "vmstat_b=${vm_b:-N/A}" + echo "vmstat_iowait_pct=${vm_wa:-N/A}" + echo "vmstat_steal_pct=${vm_st:-N/A}" + fi + else + echo "vmstat_status=unavailable" + fi + echo "scheduler_end" + echo "io_psi_begin" + cat /proc/pressure/io 2>/dev/null || echo "io_psi_status=unavailable" + echo "io_psi_end"' >"$snapshot_file" && [[ -s "$snapshot_file" ]]; then + echo "$size,$TOOL,$round,$attempt,$phase,$node,$host,ok,$snapshot_file" >>"$NODE_SSH_CSV" + else + NODE_TELEMETRY_FAILED=true + : >"$snapshot_file" + echo "$size,$TOOL,$round,$attempt,$phase,$node,$host,capture_failed,$snapshot_file" >>"$NODE_SSH_CSV" + echo "WARN: failed to capture node telemetry node=${node} host=${host}" >&2 + fi + done +} + +capture_after_probe() { + local size="$1" round="$2" attempt="$3" bucket alias probe_root key stat_json expected_size head_size get_size hash1 hash2 etag error + if [[ "$AFTER_PROBE" != "true" || "$TOOL" != "warp" || "$WARP_MODE" != "put" ]]; then + return 0 + fi + probe_root="$AFTER_PROBE_DIR/probe-${size}-${round}-${attempt}" + mkdir -p "$probe_root" + bucket="$(bucket_for_size "$size")" + alias="probe" + key="" + stat_json="" + expected_size="$(size_to_bytes "$size")" + head_size="N/A" + get_size="N/A" + hash1="N/A" + hash2="N/A" + etag="N/A" + error="" + + if [[ "$DRY_RUN" == "true" ]]; then + echo "$size,$TOOL,$round,$attempt,not_run_dry_run,$bucket,N/A,$expected_size,N/A,N/A,N/A,N/A,N/A,dry_run" >>"$AFTER_PROBE_CSV" + return 0 + fi + + export MC_CONFIG_DIR="$probe_root/mc-config" + if ! mc alias set "$alias" "$(normalize_s3_endpoint "$ENDPOINT")" "$ACCESS_KEY" "$SECRET_KEY" --api S3v4 >/dev/null 2>"$probe_root/alias.err"; then + error="alias_setup_failed" + else + key="$(mc ls --json --recursive "$alias/$bucket" 2>"$probe_root/ls.err" | jq -r 'select(.type == "file") | [(.lastModified // ""), .key] | @tsv' | sort -r | head -n1 | cut -f2- || true)" + if [[ -z "$key" ]]; then + error="no_object_found" + else + stat_json="$(mc stat --json "$alias/$bucket/$key" 2>"$probe_root/stat.err" || true)" + head_size="$(printf '%s' "$stat_json" | jq -r '.size // empty' 2>/dev/null || true)" + etag="$(printf '%s' "$stat_json" | jq -r '.etag // empty' 2>/dev/null || true)" + head_size="${head_size:-N/A}" + etag="${etag:-N/A}" + mc cat "$alias/$bucket/$key" >"$probe_root/get1.bin" 2>"$probe_root/get1.err" || true + mc cat "$alias/$bucket/$key" >"$probe_root/get2.bin" 2>"$probe_root/get2.err" || true + if [[ -f "$probe_root/get1.bin" ]]; then + get_size="$(wc -c <"$probe_root/get1.bin" | tr -d '[:space:]')" + hash1="$(sha256sum "$probe_root/get1.bin" | awk '{print $1}')" + fi + if [[ -f "$probe_root/get2.bin" ]]; then + hash2="$(sha256sum "$probe_root/get2.bin" | awk '{print $1}')" + fi + if [[ "$head_size" != "$expected_size" ]]; then error="head_size_mismatch"; fi + if [[ "$get_size" != "$expected_size" ]]; then error="${error:+$error; }get_size_mismatch"; fi + if [[ "$hash1" == "N/A" || "$hash1" != "$hash2" ]]; then error="${error:+$error; }hash_mismatch_or_missing"; fi + fi + fi + rm -rf "$MC_CONFIG_DIR" + if [[ -z "$error" ]]; then + echo "$size,$TOOL,$round,$attempt,ok,$bucket,$key,$expected_size,$head_size,$get_size,$hash1,$hash2,$etag,N/A" >>"$AFTER_PROBE_CSV" + return 0 + fi + echo "$size,$TOOL,$round,$attempt,failed,$bucket,$key,$expected_size,$head_size,$get_size,$hash1,$hash2,$etag,$error" >>"$AFTER_PROBE_CSV" + echo "WARN: after-probe failed size=${size} round=${round} attempt=${attempt}: ${error}" >&2 + return 1 +} + median_from_numbers() { local values="$1" local count @@ -1118,9 +1334,11 @@ run_one_attempt() { local status="ok" local exit_code=0 local started_at_utc finished_at_utc + NODE_TELEMETRY_FAILED=false capture_round_service_metrics "$size" "$round" "$attempt" before capture_round_node_metrics "$size" "$round" "$attempt" before capture_round_node_resources "$size" "$round" "$attempt" before + capture_round_node_telemetry "$size" "$round" "$attempt" before started_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)" if [[ "$TOOL" == "warp" ]]; then @@ -1140,6 +1358,9 @@ run_one_attempt() { "--no-color" "--analyze.v" ) + if [[ "$AFTER_PROBE" == "true" && "$WARP_MODE" == "put" ]]; then + cmd+=("--noclear") + fi if [[ "$INSECURE" == "true" ]]; then cmd+=("--insecure") fi @@ -1198,6 +1419,15 @@ run_one_attempt() { capture_round_service_metrics "$size" "$round" "$attempt" after capture_round_node_metrics "$size" "$round" "$attempt" after capture_round_node_resources "$size" "$round" "$attempt" after + capture_round_node_telemetry "$size" "$round" "$attempt" after + if [[ "$REQUIRE_NODE_TELEMETRY" == "true" && "$NODE_TELEMETRY_FAILED" == "true" ]]; then + status="failed" + exit_code=1 + fi + if ! capture_after_probe "$size" "$round" "$attempt"; then + status="failed" + exit_code=1 + fi local metrics throughput_human reqps latency_human throughput_bps latency_ms req_p90_human req_p90_ms req_p99_human req_p99_ms if [[ "$DRY_RUN" == "true" ]]; then diff --git a/scripts/test_hotpath_warp_abba.sh b/scripts/test_hotpath_warp_abba.sh index 8184e0be4..4f2d54f79 100755 --- a/scripts/test_hotpath_warp_abba.sh +++ b/scripts/test_hotpath_warp_abba.sh @@ -69,6 +69,32 @@ if rg -q -- 'rustfs-bench' "$TRACE_FILE"; then echo "unexpected rustfs-bench command" >&2 exit 1 fi + +FOCUSED_OUT_DIR="${TMP_DIR}/focused" +"$RUNNER" \ + --baseline-bin /usr/bin/true \ + --candidate-bin /usr/bin/true \ + --baseline-revision baseline-test \ + --candidate-revision candidate-test \ + --warp-bin /usr/bin/true \ + --rounds 3 \ + --workload 'put-128kib|put|128KiB' \ + --workload 'put-512kib|put|512KiB' \ + --drive-sync 'sync-on|true' \ + --out-dir "$FOCUSED_OUT_DIR" \ + --dry-run >/dev/null 2>&1 +awk -F',' 'NR == 1 {next} {seen[$3]++; if ($1 != "sync-on" || $3 !~ /^put-(128|512)kib$/) exit 1} END {exit (seen["put-128kib"] == 4 && seen["put-512kib"] == 4) ? 0 : 1}' "$FOCUSED_OUT_DIR/abba_schedule.csv" +rg -Fxq 'workloads=put-128kib|put|128KiB;put-512kib|put|512KiB' "$FOCUSED_OUT_DIR/manifest.env" +rg -Fxq 'drive_sync_matrix=sync-on|true' "$FOCUSED_OUT_DIR/manifest.env" + +if "$RUNNER" --baseline-bin /usr/bin/true --candidate-bin /usr/bin/true --baseline-revision baseline-test --candidate-revision candidate-test --warp-bin /usr/bin/true --rounds 3 --workload 'bad|delete|1KiB' --out-dir "${TMP_DIR}/bad-workload" --dry-run >/dev/null 2>&1; then + echo "expected invalid --workload mode to fail" >&2 + exit 1 +fi +if "$RUNNER" --baseline-bin /usr/bin/true --candidate-bin /usr/bin/true --baseline-revision baseline-test --candidate-revision candidate-test --warp-bin /usr/bin/true --rounds 3 --drive-sync 'sync-on|maybe' --out-dir "${TMP_DIR}/bad-drive-sync" --dry-run >/dev/null 2>&1; then + echo "expected invalid --drive-sync value to fail" >&2 + exit 1 +fi rg -qF -- '--labeled-compare-csv sync-on/put-4kib/B1-vs-A1' "$TRACE_FILE" rg -qF -- '--labeled-compare-csv sync-off/put-4kib/B1-vs-A1' "$TRACE_FILE" rg -qF -- '--labeled-compare-csv sync-on/put-4kib/A2-vs-A1' "$TRACE_FILE" diff --git a/scripts/test_object_batch_bench_enhanced.sh b/scripts/test_object_batch_bench_enhanced.sh index a1cf66190..704a7c1ea 100755 --- a/scripts/test_object_batch_bench_enhanced.sh +++ b/scripts/test_object_batch_bench_enhanced.sh @@ -23,7 +23,9 @@ trap cleanup EXIT --duration 1s \ --out-dir "$OUT_DIR" \ --warp-bin true \ + --warp-mode put \ --dry-run \ + --after-probe \ --service-metrics-dir "${OUT_DIR}/metrics" \ --server-image-ref rustfs/rustfs:bench \ --server-image-digest sha256:0123456789abcdef \ @@ -32,6 +34,10 @@ trap cleanup EXIT --label topology=4x2 \ --label workload=get \ --node-metrics-url node1=http://127.0.0.1:9001/metrics \ + --node-ssh-target node1=localhost \ + --node-ssh-identity-file /tmp/nonexistent-key \ + --node-ssh-timeout-secs 7 \ + --require-node-telemetry \ --node-docker-container node1=rustfs-bench-1 >/dev/null rg -qx 'server_image_ref=rustfs/rustfs:bench' "${OUT_DIR}/run_manifest.env" @@ -43,6 +49,29 @@ rg -qx 'run_label_workload=get' "${OUT_DIR}/run_manifest.env" rg -q '^node1,rustfs-bench-1,not_run_dry_run,N/A,N/A,N/A,N/A$' "${OUT_DIR}/node_inventory.csv" rg -q '^1MiB,warp,1,1,before,node1,not_run_dry_run,' "${OUT_DIR}/node_metrics_captures.csv" rg -q '^1MiB,warp,1,1,after,node1,rustfs-bench-1,not_run_dry_run,' "${OUT_DIR}/node_resource_captures.csv" +rg -qx 'after_probe=true' "${OUT_DIR}/run_manifest.env" +rg -qx 'node_ssh_identity_file=/tmp/nonexistent-key' "${OUT_DIR}/run_manifest.env" +rg -qx 'node_ssh_timeout_secs=7' "${OUT_DIR}/run_manifest.env" +rg -qx 'require_node_telemetry=true' "${OUT_DIR}/run_manifest.env" +rg -q '^1MiB,warp,1,1,before,node1,localhost,not_run_dry_run,' "${OUT_DIR}/node_telemetry_captures.csv" +rg -q '^1MiB,warp,1,1,not_run_dry_run,rustfs-bench,N/A,1048576,N/A,N/A,N/A,N/A,N/A,dry_run$' "${OUT_DIR}/after_probe.csv" + +"$RUNNER" \ + --tool warp \ + --endpoint http://127.0.0.1:9000 \ + --access-key test-access \ + --secret-key test-secret \ + --sizes 1MB \ + --rounds 1 \ + --retry-per-round 1 \ + --cooldown-secs 0 \ + --duration 1s \ + --out-dir "${TMP_DIR}/decimal-size" \ + --warp-bin true \ + --warp-mode put \ + --dry-run \ + --after-probe >/dev/null +rg -q '^1MB,warp,1,1,not_run_dry_run,rustfs-bench,N/A,1000000,N/A,N/A,N/A,N/A,N/A,dry_run$' "${TMP_DIR}/decimal-size/after_probe.csv" if "$RUNNER" \ --tool warp \