diff --git a/scripts/run_object_batch_bench_enhanced.sh b/scripts/run_object_batch_bench_enhanced.sh index d0f22853d..08163070e 100755 --- a/scripts/run_object_batch_bench_enhanced.sh +++ b/scripts/run_object_batch_bench_enhanced.sh @@ -672,7 +672,7 @@ extract_report_line() { local regex="$1" local file="$2" awk -v regex="$regex" ' - /^Report:/ { + /^(Report|Operation):/ { in_report = 1 next } @@ -703,9 +703,9 @@ normalize_duration_metric() { extract_metrics() { local log_file="$1" - local average_line reqs_line throughput reqps latency req_p90 req_p99 reqps_num + local average_line request_line throughput reqps latency req_p90 req_p99 reqps_num average_line="$(extract_report_line '^[[:space:]]*[*][[:space:]]+Average:' "$log_file")" - reqs_line="$(extract_report_line '^[[:space:]]*[*][[:space:]]+Reqs:' "$log_file")" + request_line="$(extract_report_line '^[[:space:]]*[*][[:space:]]+(Reqs:[[:space:]]+)?Avg:' "$log_file")" if [[ -n "$average_line" ]]; then throughput="$(echo "$average_line" | sed -E 's/^.*Average:[[:space:]]*//; s/,[[:space:]]*.*$//')" @@ -715,20 +715,16 @@ extract_metrics() { reqps="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(obj/s|req/s|ops/s|requests/s)' "$log_file")" fi - if [[ -n "$reqs_line" ]]; then - latency="$(echo "$reqs_line" | rg -o 'Avg:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | sed -E 's/^Avg:[[:space:]]+//')" - req_p90="$(echo "$reqs_line" | rg -o '90%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | sed -E 's/^90%:[[:space:]]+//')" - req_p99="$(echo "$reqs_line" | rg -o '99%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | sed -E 's/^99%:[[:space:]]+//')" + if [[ -n "$request_line" ]]; then + latency="$(echo "$request_line" | rg -o 'Avg:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | sed -E 's/^Avg:[[:space:]]+//')" + req_p90="$(echo "$request_line" | rg -o '90%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | sed -E 's/^90%:[[:space:]]+//')" + req_p99="$(echo "$request_line" | rg -o '99%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | sed -E 's/^99%:[[:space:]]+//')" else latency="$(rg -o 'Reqs:[[:space:]]+Avg:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^Reqs:[[:space:]]+Avg:[[:space:]]+//')" req_p90="$(rg -o '90%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^90%:[[:space:]]+//')" req_p99="$(rg -o '99%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^99%:[[:space:]]+//')" fi - if [[ -z "$latency" ]]; then - latency="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(ms|us|µs|s)' "$log_file")" - fi - throughput="$(trim "${throughput:-N/A}")" reqps="$(trim "${reqps:-N/A}")" latency="$(trim "${latency:-N/A}")" @@ -1128,6 +1124,8 @@ run_one_attempt() { "--concurrent" "$CONCURRENCY" "--duration" "$DURATION" "--region" "$REGION" + "--no-color" + "--analyze.v" ) if [[ "$INSECURE" == "true" ]]; then cmd+=("--insecure") @@ -1212,6 +1210,12 @@ run_one_attempt() { req_p99_ms="$(to_ms "$req_p99_human")" fi + if [[ "$DRY_RUN" != "true" && "$TOOL" == "warp" && "$status" == "ok" ]] \ + && rg -q '^[[:space:]]*(Total[[:space:]]+)?Errors:[[:space:]]+[1-9][0-9]*[.]?([[:space:]]|$)' "$log_file"; then + status="failed" + exit_code=1 + fi + if [[ "$DRY_RUN" != "true" && "$status" == "ok" ]]; then if [[ "$throughput_bps" == "N/A" && "$reqps" == "N/A" ]]; then status="failed" diff --git a/scripts/test_object_batch_bench_enhanced.sh b/scripts/test_object_batch_bench_enhanced.sh index 6722f723f..be8f90dca 100755 --- a/scripts/test_object_batch_bench_enhanced.sh +++ b/scripts/test_object_batch_bench_enhanced.sh @@ -74,13 +74,19 @@ FAKE_WARP="${TMP_DIR}/fake-warp" cat >"$FAKE_WARP" <<'EOF' #!/usr/bin/env bash set -euo pipefail +[[ " $* " == *" --analyze.v "* ]] +[[ " $* " == *" --no-color "* ]] cat <<'LOG' - PUT Average: 161 Obj/s, 5.0MiB/s; Current 161 Obj/s, 5.0MiB/s. -Report: GET. Concurrency: 64. Ran: 7s +Operation: GET. Concurrency: 64. Ran: 7s +Requests considered: 1000: * Average: 653.90 MiB/s, 20925.58 obj/s - * Reqs: Avg: 3.5ms, 50%: 2.0ms, 90%: 3.6ms, 99%: 24.1ms, Fastest: 0.2ms, Slowest: 607.7ms, StdDev: 20.6ms + * Avg: 3.5ms, 50%: 2.0ms, 90%: 3.6ms, 99%: 24.1ms, Fastest: 0.2ms, Slowest: 607.7ms, StdDev: 20.6ms Throughput, split into 7 x 1s: LOG +if [[ "${FAKE_WARP_ERRORS:-0}" == "1" ]]; then + echo 'Total Errors: 1.' +fi EOF chmod +x "$FAKE_WARP" @@ -103,6 +109,32 @@ chmod +x "$FAKE_WARP" rg -q '^32767B,warp,1,1,128,ok,0,[^,]+,[^,]+,653.90 MiB/s,685663846.400000,20925.58,3.5 ms,3.500000,[^,]+,3.6 ms,3.600000,24.1 ms,24.100000$' "${TMP_DIR}/fake-warp-run/round_results.csv" +cat >"${TMP_DIR}/warp-no-details.log" <<'EOF' +warp: Starting benchmark in 3s... +Operation: PUT. Concurrency: 8 + * Average: 2.76 MiB/s, 707.03 obj/s +EOF +"$RUNNER" --extract-metrics-from-log "${TMP_DIR}/warp-no-details.log" >"${TMP_DIR}/warp-no-details.csv" +rg -qx '2.76 MiB/s,2894069.760000,707.03,N/A,N/A,N/A,N/A,N/A,N/A' "${TMP_DIR}/warp-no-details.csv" + +if FAKE_WARP_ERRORS=1 "$RUNNER" \ + --tool warp \ + --endpoint http://127.0.0.1:9000 \ + --access-key test-access \ + --secret-key test-secret \ + --sizes 32767B \ + --rounds 1 \ + --retry-per-round 1 \ + --retry-sleep-secs 1 \ + --cooldown-secs 0 \ + --duration 1s \ + --out-dir "${TMP_DIR}/fake-warp-errors" \ + --warp-bin "$FAKE_WARP" >/dev/null 2>&1; then + echo "expected Warp request errors to fail the benchmark" >&2 + exit 1 +fi +rg -q ',failed,1,' "${TMP_DIR}/fake-warp-errors/round_results.csv" + "$RUNNER" \ --tool warp \ --endpoint http://127.0.0.1:9000 \