From d5409845e2a0ec84439cf339a48485addf318471 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 26 Jul 2026 13:41:00 +0800 Subject: [PATCH] test(perf): add exact 1MiB ABBA benchmark drivers (#5260) * test(perf): add exact 1mib get abba harness Add a backlog#1434 exact-1MiB GET attribution wrapper around the codec streaming smoke runner. The harness fixes the legacy versus codec-legacy ABBA profile order, the outer ReaderStream buffer axis, path-proof artifacts, and GET stage-metrics capture so reviewers can reproduce isolated-host attribution without drifting the experiment matrix. Verification: - scripts/test_get_1mib_abba_stage_metrics.sh - bash -n scripts/run_get_1mib_abba_stage_metrics.sh scripts/test_get_1mib_abba_stage_metrics.sh - git diff --check Co-Authored-By: heihutu * test(perf): add exact 1MiB handoff ABBA driver Add a backlog#1434 benchmark orchestrator for isolated-host exact-1MiB handoff evidence. The runner records A/B/B/A and C/D/D/C legs, server provenance, expected reader path labels, inner and outer buffer capacity labels, and stage metrics capture arguments while delegating workload execution to the existing enhanced batch benchmark driver. Verification: - scripts/test_exact_1mib_handoff_abba.sh - bash -n scripts/run_exact_1mib_handoff_abba.sh scripts/test_exact_1mib_handoff_abba.sh - make script-tests - git diff --check Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- .config/make/tests.mak | 1 + scripts/README.md | 1 + scripts/run_exact_1mib_handoff_abba.sh | 277 ++++++++++++++ scripts/run_get_1mib_abba_stage_metrics.sh | 398 ++++++++++++++++++++ scripts/test_exact_1mib_handoff_abba.sh | 88 +++++ scripts/test_get_1mib_abba_stage_metrics.sh | 87 +++++ 6 files changed, 852 insertions(+) create mode 100755 scripts/run_exact_1mib_handoff_abba.sh create mode 100755 scripts/run_get_1mib_abba_stage_metrics.sh create mode 100755 scripts/test_exact_1mib_handoff_abba.sh create mode 100755 scripts/test_get_1mib_abba_stage_metrics.sh diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 0eadd0f2c..d1fe55cc7 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -28,6 +28,7 @@ script-tests: ## Run shell script tests ./scripts/test_entrypoint_credentials.sh ./scripts/test_internode_grpc_ab_bench.sh ./scripts/test_object_batch_bench_enhanced.sh + ./scripts/test_exact_1mib_handoff_abba.sh ./scripts/test_pinned_paired_abba_bench.sh bash -n ./scripts/validate_object_data_cache_cold_stampede.sh python3 ./scripts/check_object_data_cache_follower_samples.py --self-test diff --git a/scripts/README.md b/scripts/README.md index 262b40aba..4fc034bbb 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -70,6 +70,7 @@ their issue closes. | `run_object_batch_bench_enhanced.sh` | dev-tool | Enhanced batch benchmark runner; hub used by the smoke rigs | hotpath runbook | | `run_pinned_paired_abba_bench.sh` | dev-tool | Pinned RustFS/MinIO paired ABBA benchmark orchestrator for backlog#1432 | `test_pinned_paired_abba_bench.sh` | | `run_get_codec_streaming_smoke.sh` | dev-tool | Local GET benchmark harness for the codec streaming read path | `docs/testing/ecstore-validation-suite-design.md` | +| `run_get_1mib_abba_stage_metrics.sh` | dev-tool | Exact-1MiB isolated-host GET ABBA/stage-metrics harness for backlog#1434 | `test_get_1mib_abba_stage_metrics.sh` | | `run_gt1g_get_http_matrix.sh` | dev-tool | >1 GiB GET HTTP matrix | `docs/testing/ecstore-validation-suite-design.md` | | `run_gt1g_multipart_put_matrix.sh` | dev-tool | >1 GiB multipart PUT matrix | `docs/testing/ecstore-validation-suite-design.md` | | `run_scanner_benchmarks.sh` | dev-tool (disposition pending) | Scanner performance benchmark runner. Contains a hardcoded stale path; **disposition owned by backlog perf-10 — do not fix, move, or delete it here** | — | diff --git a/scripts/run_exact_1mib_handoff_abba.sh b/scripts/run_exact_1mib_handoff_abba.sh new file mode 100755 index 000000000..2f5d3711f --- /dev/null +++ b/scripts/run_exact_1mib_handoff_abba.sh @@ -0,0 +1,277 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +ENHANCED_BENCH="${PROJECT_ROOT}/scripts/run_object_batch_bench_enhanced.sh" + +OUT_DIR="" +ACCESS_KEY="" +SECRET_KEY="" +WARP_BIN="warp" +DURATION="60s" +CONCURRENCIES="64,128" +SIZES="524288B,1048575B,1048576B,1048577B,4194304B" +ROUNDS_PER_LEG=5 +COOLDOWN_SECS=20 +BUCKET_PREFIX="rustfs-1434-handoff" +DRY_RUN=false + +ARM_A="" +ARM_B="" +ARM_C="" +ARM_D="" + +usage() { + cat <<'USAGE' +Usage: + scripts/run_exact_1mib_handoff_abba.sh --access-key --secret-key \ + --arm-a --arm-b --arm-c --arm-d [options] + +Arm spec: + name|endpoint|server_image_ref|server_image_digest|server_revision|inner_capacity|outer_capacity|outer_source + +Required arms: + A = current adaptive inner, outer auto + B = exact-1MiB inner candidate, outer auto + C = current adaptive inner, outer 1MiB override + D = exact-1MiB inner candidate, outer 1MiB override + +Options: + --out-dir Default: target/bench/exact-1mib-handoff-abba- + --concurrencies Default: 64,128 + --sizes Default: 512KiB, 1MiB-1, 1MiB, 1MiB+1, 4MiB + --rounds-per-leg Default: 5 + --cooldown-secs Default: 20 + --bucket-prefix Default: rustfs-1434-handoff + --warp-bin Default: warp + --duration Default: 60s + --dry-run Materialize manifest and commands only + -h, --help + +This runner records isolated-host ABBA evidence for rustfs/backlog#1434 only. It does not change production defaults and does not claim an optimization unless the raw artifact gate is analyzed separately. +USAGE +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +validate_nonempty() { + local value="$1" + local name="$2" + [[ -n "$value" ]] || die "$name is required" +} + +validate_positive_int() { + local value="$1" + local name="$2" + [[ "$value" =~ ^[0-9]+$ && "$value" -gt 0 ]] || die "$name must be a positive integer, got: $value" +} + +validate_nonnegative_int() { + local value="$1" + local name="$2" + [[ "$value" =~ ^[0-9]+$ ]] || die "$name must be a nonnegative integer, got: $value" +} + +validate_csv_nonempty() { + local value="$1" + local name="$2" + [[ -n "$value" && "$value" != *",,"* ]] || die "$name must be a non-empty CSV" +} + +validate_digest() { + local value="$1" + local name="$2" + [[ "$value" =~ ^sha256:[0-9a-fA-F]{64}$ ]] || die "$name must be an immutable sha256 digest" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --out-dir) OUT_DIR="$2"; shift 2 ;; + --access-key) ACCESS_KEY="$2"; shift 2 ;; + --secret-key) SECRET_KEY="$2"; shift 2 ;; + --warp-bin) WARP_BIN="$2"; shift 2 ;; + --duration) DURATION="$2"; shift 2 ;; + --concurrencies) CONCURRENCIES="$2"; shift 2 ;; + --sizes) SIZES="$2"; shift 2 ;; + --rounds-per-leg) ROUNDS_PER_LEG="$2"; shift 2 ;; + --cooldown-secs) COOLDOWN_SECS="$2"; shift 2 ;; + --bucket-prefix) BUCKET_PREFIX="$2"; shift 2 ;; + --arm-a) ARM_A="$2"; shift 2 ;; + --arm-b) ARM_B="$2"; shift 2 ;; + --arm-c) ARM_C="$2"; shift 2 ;; + --arm-d) ARM_D="$2"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage; exit 0 ;; + *) die "unknown arg: $1" ;; + esac + done +} + +split_arm() { + local spec="$1" + local prefix="$2" + local -a fields + IFS='|' read -r -a fields <<< "$spec" + [[ "${#fields[@]}" -eq 8 ]] || die "$prefix must have 8 pipe-separated fields" + printf -v "${prefix}_name" '%s' "${fields[0]}" + printf -v "${prefix}_endpoint" '%s' "${fields[1]}" + printf -v "${prefix}_image_ref" '%s' "${fields[2]}" + printf -v "${prefix}_image_digest" '%s' "${fields[3]}" + printf -v "${prefix}_revision" '%s' "${fields[4]}" + printf -v "${prefix}_inner" '%s' "${fields[5]}" + printf -v "${prefix}_outer" '%s' "${fields[6]}" + printf -v "${prefix}_outer_source" '%s' "${fields[7]}" +} + +validate_arm() { + local prefix="$1" + local expected_name="$2" + local name_var="${prefix}_name" + local endpoint_var="${prefix}_endpoint" + local digest_var="${prefix}_image_digest" + local inner_var="${prefix}_inner" + local outer_var="${prefix}_outer" + local source_var="${prefix}_outer_source" + [[ "${!name_var}" == "$expected_name" ]] || die "$prefix name must be $expected_name" + validate_nonempty "${!endpoint_var}" "$prefix endpoint" + validate_digest "${!digest_var}" "$prefix image digest" + [[ "${!inner_var}" =~ ^[0-9]+$ ]] || die "$prefix inner capacity must be numeric bytes" + [[ "${!outer_var}" == "auto" || "${!outer_var}" =~ ^[0-9]+$ ]] || die "$prefix outer capacity must be auto or numeric bytes" + case "${!source_var}" in + selected|env_override) ;; + *) die "$prefix outer source must be selected or env_override" ;; + esac +} + +validate_args() { + validate_nonempty "$ACCESS_KEY" "--access-key" + validate_nonempty "$SECRET_KEY" "--secret-key" + validate_nonempty "$ARM_A" "--arm-a" + validate_nonempty "$ARM_B" "--arm-b" + validate_nonempty "$ARM_C" "--arm-c" + validate_nonempty "$ARM_D" "--arm-d" + validate_csv_nonempty "$CONCURRENCIES" "--concurrencies" + validate_csv_nonempty "$SIZES" "--sizes" + validate_positive_int "$ROUNDS_PER_LEG" "--rounds-per-leg" + validate_nonnegative_int "$COOLDOWN_SECS" "--cooldown-secs" + split_arm "$ARM_A" "A" + split_arm "$ARM_B" "B" + split_arm "$ARM_C" "C" + split_arm "$ARM_D" "D" + validate_arm "A" "A" + validate_arm "B" "B" + validate_arm "C" "C" + validate_arm "D" "D" +} + +setup_output() { + if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="${PROJECT_ROOT}/target/bench/exact-1mib-handoff-abba-$(date -u +%Y%m%dT%H%M%SZ)" + fi + mkdir -p "$OUT_DIR" +} + +arm_value() { + local arm="$1" + local field="$2" + local var="${arm}_${field}" + printf '%s' "${!var}" +} + +write_manifest() { + local manifest="$OUT_DIR/handoff_manifest.env" + { + printf 'created_utc=%q\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'benchmark_issue=%q\n' "rustfs/backlog#1434" + printf 'schedule=%q\n' "ABBA_then_CDDC" + printf 'sizes=%q\n' "$SIZES" + printf 'concurrencies=%q\n' "$CONCURRENCIES" + printf 'rounds_per_leg=%q\n' "$ROUNDS_PER_LEG" + printf 'cooldown_secs=%q\n' "$COOLDOWN_SECS" + printf 'access_key=%q\n' "REDACTED" + printf 'secret_key=%q\n' "REDACTED" + } > "$manifest" +} + +write_schedule() { + local schedule="$OUT_DIR/handoff_abba_schedule.csv" + echo "leg,arm,endpoint,server_image_ref,server_image_digest,server_revision,inner_capacity,outer_capacity,outer_source,leg_out_dir" > "$schedule" + for leg in A1 B1 B2 A2 C1 D1 D2 C2; do + local arm="${leg:0:1}" + echo "$leg,$arm,$(arm_value "$arm" endpoint),$(arm_value "$arm" image_ref),$(arm_value "$arm" image_digest),$(arm_value "$arm" revision),$(arm_value "$arm" inner),$(arm_value "$arm" outer),$(arm_value "$arm" outer_source),$OUT_DIR/$leg-arm-$arm" >> "$schedule" + done +} + +quote_cmd() { + printf '%q ' "$@" +} + +run_leg() { + local leg="$1" + local arm="${leg:0:1}" + local conc="$2" + local leg_lower + leg_lower="$(printf '%s' "$leg" | tr '[:upper:]' '[:lower:]')" + local bucket="${BUCKET_PREFIX}-${leg_lower}-c${conc}" + local leg_out_dir="$OUT_DIR/$leg-arm-$arm/c${conc}" + local -a cmd=( + "$ENHANCED_BENCH" + --tool warp + --endpoint "$(arm_value "$arm" endpoint)" + --access-key "$ACCESS_KEY" + --secret-key "$SECRET_KEY" + --bucket "$bucket" + --bucket-size-suffix + --concurrency "$conc" + --sizes "$SIZES" + --rounds "$ROUNDS_PER_LEG" + --cooldown-secs "$COOLDOWN_SECS" + --warp-bin "$WARP_BIN" + --warp-mode get + --duration "$DURATION" + --service-metrics-dir "$leg_out_dir/service-metrics" + --service-metrics-filter-regex 'rustfs_io_get_object_|rustfs_s3_get_object_' + --server-image-ref "$(arm_value "$arm" image_ref)" + --server-image-digest "$(arm_value "$arm" image_digest)" + --server-revision "$(arm_value "$arm" revision)" + --require-server-provenance + --label bench_issue=1434 + --label handoff_arm="$arm" + --label expected_reader_path=legacy_duplex + --label expected_inner_capacity="$(arm_value "$arm" inner)" + --label expected_outer_capacity="$(arm_value "$arm" outer)" + --label expected_outer_source="$(arm_value "$arm" outer_source)" + --out-dir "$leg_out_dir" + ) + if [[ "$DRY_RUN" == "true" ]]; then + echo "-- dry-run leg=$leg arm=$arm concurrency=$conc" + echo " $(quote_cmd "${cmd[@]}" --dry-run)" + else + "${cmd[@]}" + fi +} + +run_schedule() { + local conc leg + IFS=',' read -r -a conc_values <<< "$CONCURRENCIES" + for conc in "${conc_values[@]}"; do + validate_positive_int "$conc" "concurrency cell" + for leg in A1 B1 B2 A2 C1 D1 D2 C2; do + run_leg "$leg" "$conc" + done + done +} + +parse_args "$@" +validate_args +setup_output +write_manifest +write_schedule +run_schedule + +echo "exact 1MiB handoff ABBA artifact root: $OUT_DIR" diff --git a/scripts/run_get_1mib_abba_stage_metrics.sh b/scripts/run_get_1mib_abba_stage_metrics.sh new file mode 100755 index 000000000..d615d75d2 --- /dev/null +++ b/scripts/run_get_1mib_abba_stage_metrics.sh @@ -0,0 +1,398 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Dedicated exact-1MiB GET attribution harness for rustfs/backlog#1434. +# +# The heavy lifting stays in run_get_codec_streaming_smoke.sh. This wrapper only +# fixes the experiment matrix so a reviewer can reproduce the isolated-host +# ABBA evidence without accidentally drifting object size, order, path proof, or +# stage-metrics capture. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +SMOKE_RUNNER="${PROJECT_ROOT}/scripts/run_get_codec_streaming_smoke.sh" + +ADDRESS="127.0.0.1:19030" +ACCESS_KEY="rustfsadmin" +SECRET_KEY="rustfsadmin" +BUCKET="rustfs-get-1mib-abba" +REGION="us-east-1" +CONCURRENCY=16 +DURATION="30s" +ROUNDS=3 +RETRY_PER_ROUND=1 +ROUND_COOLDOWN_SECS=20 +WARP_OBJECTS="" +WARP_OBJECT_LIFECYCLE="per-round" +WARP_PREPARE_DURATION="1s" +WARP_EXTRA_ARGS="" +WARP_WARMUP_GET_BEFORE_BENCH=false +OUTER_READER_STREAM_BUFFER_SIZES="65536,1048576" +PROFILE_ORDERS="normal,reverse" +CODEC_MIN_SIZE=1048576 +RUSTFS_BIN="${PROJECT_ROOT}/target/release/rustfs" +WARP_BIN="warp" +PYTHON_BIN="python3" +OUT_DIR="" +DIAGNOSTIC_METRICS_URL="http://127.0.0.1:8889/metrics" +DIAGNOSTIC_METRICS_SETTLE_SECS="${RUSTFS_DIAGNOSTIC_METRICS_SETTLE_SECS:-2}" +DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS="${RUSTFS_DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS:-5}" +DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS="${RUSTFS_DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS:-1}" +DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS="${RUSTFS_DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS:-2}" +DIAGNOSTIC_METRICS_MAX_TIME_SECS="${RUSTFS_DIAGNOSTIC_METRICS_MAX_TIME_SECS:-15}" +DIAGNOSTIC_METRICS_FILTER_REGEX="${RUSTFS_DIAGNOSTIC_METRICS_FILTER_REGEX:-rustfs_io_get_object_}" +DIAGNOSTIC_OBS_ENDPOINT="${RUSTFS_OBS_ENDPOINT:-}" +DIAGNOSTIC_OBS_METRIC_ENDPOINT="${RUSTFS_OBS_METRIC_ENDPOINT:-}" +DIAGNOSTIC_OBS_METER_INTERVAL="${RUSTFS_OBS_METER_INTERVAL:-1}" +DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX="${RUSTFS_OBS_SERVICE_NAME:-RustFS-get-1mib-abba}" +RESOURCE_SAMPLE_INTERVAL_SECS="${RUSTFS_GET_BENCH_RESOURCE_SAMPLE_INTERVAL_SECS:-5}" +HEALTH_TIMEOUT_SECS=60 +SKIP_BUILD=false +DRY_RUN=false +ORIGINAL_ARGS=("$@") + +usage() { + cat <<'USAGE' +Usage: + scripts/run_get_1mib_abba_stage_metrics.sh [options] + +Purpose: + Run the rustfs/backlog#1434 exact-1MiB isolated-host GET attribution matrix: + - object size fixed to 1MiB / 1048576 bytes + - legacy and codec-legacy read-path profiles + - normal and reverse profile ordering for ABBA order-bias checks + - two outer ReaderStream capacities via RUSTFS_GET_READER_STREAM_BUFFER_SIZE + - output handoff attribution and GET stage metrics enabled + +Core options: + --outer-reader-stream-buffer-sizes + RUSTFS_GET_READER_STREAM_BUFFER_SIZE values + used for the outer response ReaderStream axis + (default: 65536,1048576) + --profile-orders Profile orders to run (default: normal,reverse) + --address RustFS listen address (default: 127.0.0.1:19030) + --bucket Bucket prefix (default: rustfs-get-1mib-abba) + --concurrency warp concurrency (default: 16) + --duration warp duration per round (default: 30s) + --rounds rounds per matrix cell (default: 3) + --retry-per-round failed-attempt retries per round (default: 1) + --round-cooldown-secs cooldown seconds after each completed round + (default: 20) + --warp-objects Number of objects prepared by warp + --warp-object-lifecycle per-round|prepare-once|existing-only + (default: per-round) + --warp-prepare-duration Duration used by prepare-once warmup + (default: 1s) + --warp-extra-args Extra args forwarded to warp through the smoke + runner + --warp-warmup-get-before-bench Run a GET warmup before measured rounds + --out-dir Output directory (default target/bench/get-1mib-abba-stage-metrics-) + +Diagnostics: + --diagnostic-metrics-url GET metrics scrape URL (default: http://127.0.0.1:8889/metrics) + --diagnostic-metrics-settle-secs + --diagnostic-metrics-capture-attempts + --diagnostic-metrics-capture-retry-secs + --diagnostic-metrics-connect-timeout-secs + --diagnostic-metrics-max-time-secs + --diagnostic-metrics-filter-regex + --diagnostic-obs-endpoint + --diagnostic-obs-metric-endpoint + --diagnostic-obs-meter-interval + --diagnostic-obs-service-name-prefix + +Binary/options: + --rustfs-bin RustFS binary (default: target/release/rustfs) + --warp-bin warp binary (default: warp) + --python-bin Python binary (default: python3) + --skip-build Do not build RustFS in the smoke runner + --dry-run Validate wiring without starting RustFS + -h, --help Show this help + +Output: + /manifest.env + /abba_matrix.csv + /-outer-/manifest.env + /-outer-/metrics_summary.csv + /-outer-/baseline_compare.csv + /-outer-/service_metrics_summary.csv + /-outer-/service_metrics_round_summary.csv + /-outer-/service_metrics_stage_distribution.csv + /-outer-/service_metrics_round_percentiles.csv + /-outer-/service_metrics_acceptance.csv +USAGE +} + +die() { + echo "error: $*" >&2 + exit 2 +} + +log() { + printf '[get-1mib-abba] %s\n' "$*" >&2 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --outer-reader-stream-buffer-sizes) OUTER_READER_STREAM_BUFFER_SIZES="$2"; shift 2 ;; + --profile-orders) PROFILE_ORDERS="$2"; shift 2 ;; + --address) ADDRESS="$2"; shift 2 ;; + --bucket) BUCKET="$2"; shift 2 ;; + --access-key) ACCESS_KEY="$2"; shift 2 ;; + --secret-key) SECRET_KEY="$2"; shift 2 ;; + --region) REGION="$2"; shift 2 ;; + --concurrency) CONCURRENCY="$2"; shift 2 ;; + --duration) DURATION="$2"; shift 2 ;; + --rounds) ROUNDS="$2"; shift 2 ;; + --retry-per-round) RETRY_PER_ROUND="$2"; shift 2 ;; + --round-cooldown-secs) ROUND_COOLDOWN_SECS="$2"; shift 2 ;; + --warp-objects) WARP_OBJECTS="$2"; shift 2 ;; + --warp-object-lifecycle) WARP_OBJECT_LIFECYCLE="$2"; shift 2 ;; + --warp-prepare-duration) WARP_PREPARE_DURATION="$2"; shift 2 ;; + --warp-extra-args) WARP_EXTRA_ARGS="$2"; shift 2 ;; + --warp-warmup-get-before-bench) WARP_WARMUP_GET_BEFORE_BENCH=true; shift ;; + --out-dir) OUT_DIR="$2"; shift 2 ;; + --diagnostic-metrics-url) DIAGNOSTIC_METRICS_URL="$2"; shift 2 ;; + --diagnostic-metrics-settle-secs) DIAGNOSTIC_METRICS_SETTLE_SECS="$2"; shift 2 ;; + --diagnostic-metrics-capture-attempts) DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS="$2"; shift 2 ;; + --diagnostic-metrics-capture-retry-secs) DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS="$2"; shift 2 ;; + --diagnostic-metrics-connect-timeout-secs) DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS="$2"; shift 2 ;; + --diagnostic-metrics-max-time-secs) DIAGNOSTIC_METRICS_MAX_TIME_SECS="$2"; shift 2 ;; + --diagnostic-metrics-filter-regex) DIAGNOSTIC_METRICS_FILTER_REGEX="$2"; shift 2 ;; + --diagnostic-obs-endpoint) DIAGNOSTIC_OBS_ENDPOINT="$2"; shift 2 ;; + --diagnostic-obs-metric-endpoint) DIAGNOSTIC_OBS_METRIC_ENDPOINT="$2"; shift 2 ;; + --diagnostic-obs-meter-interval) DIAGNOSTIC_OBS_METER_INTERVAL="$2"; shift 2 ;; + --diagnostic-obs-service-name-prefix) DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX="$2"; shift 2 ;; + --rustfs-bin) RUSTFS_BIN="$2"; shift 2 ;; + --warp-bin) WARP_BIN="$2"; shift 2 ;; + --python-bin) PYTHON_BIN="$2"; shift 2 ;; + --skip-build) SKIP_BUILD=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage; exit 0 ;; + *) die "unknown argument: $1" ;; + esac +done + +[[ -x "$SMOKE_RUNNER" ]] || die "missing smoke runner: $SMOKE_RUNNER" +[[ -n "$OUTER_READER_STREAM_BUFFER_SIZES" ]] || die "--outer-reader-stream-buffer-sizes must not be empty" +[[ -n "$PROFILE_ORDERS" ]] || die "--profile-orders must not be empty" + +validate_positive_int() { + local value="$1" label="$2" + [[ "$value" =~ ^[0-9]+$ && "$value" -gt 0 ]] || die "${label} must be a positive integer" +} + +validate_non_negative_int() { + local value="$1" label="$2" + [[ "$value" =~ ^[0-9]+$ ]] || die "${label} must be a non-negative integer" +} + +validate_positive_int "$CONCURRENCY" "--concurrency" +validate_positive_int "$ROUNDS" "--rounds" +validate_positive_int "$RETRY_PER_ROUND" "--retry-per-round" +validate_non_negative_int "$ROUND_COOLDOWN_SECS" "--round-cooldown-secs" +validate_non_negative_int "$DIAGNOSTIC_METRICS_SETTLE_SECS" "--diagnostic-metrics-settle-secs" +validate_positive_int "$DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS" "--diagnostic-metrics-capture-attempts" +validate_non_negative_int "$DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS" "--diagnostic-metrics-capture-retry-secs" +validate_positive_int "$DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS" "--diagnostic-metrics-connect-timeout-secs" +validate_positive_int "$DIAGNOSTIC_METRICS_MAX_TIME_SECS" "--diagnostic-metrics-max-time-secs" +validate_positive_int "$DIAGNOSTIC_OBS_METER_INTERVAL" "--diagnostic-obs-meter-interval" + +IFS=',' read -r -a outer_sizes <<< "$OUTER_READER_STREAM_BUFFER_SIZES" +IFS=',' read -r -a profile_orders <<< "$PROFILE_ORDERS" + +cleaned_outer_sizes=() +for raw_outer_size in "${outer_sizes[@]}"; do + clean_outer_size="${raw_outer_size//[[:space:]]/}" + [[ -n "$clean_outer_size" ]] && cleaned_outer_sizes+=("$clean_outer_size") +done +outer_sizes=("${cleaned_outer_sizes[@]}") + +cleaned_profile_orders=() +for raw_profile_order in "${profile_orders[@]}"; do + clean_profile_order="${raw_profile_order//[[:space:]]/}" + [[ -n "$clean_profile_order" ]] && cleaned_profile_orders+=("$clean_profile_order") +done +profile_orders=("${cleaned_profile_orders[@]}") +[[ "${#outer_sizes[@]}" -gt 0 ]] || die "--outer-reader-stream-buffer-sizes must include at least one value" +[[ "${#profile_orders[@]}" -gt 0 ]] || die "--profile-orders must include at least one value" + +for outer_size in "${outer_sizes[@]}"; do + validate_positive_int "$outer_size" "--outer-reader-stream-buffer-sizes entry" +done +for profile_order in "${profile_orders[@]}"; do + case "$profile_order" in + normal|reverse) ;; + *) die "--profile-orders entries must be normal or reverse" ;; + esac +done + +if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="${PROJECT_ROOT}/target/bench/get-1mib-abba-stage-metrics-$(date -u +%Y%m%dT%H%M%SZ 2>/dev/null || echo run)" +fi +mkdir -p "$OUT_DIR" + +command_line() { + printf '%q ' "$0" "$@" +} + +branch="$(git -C "$PROJECT_ROOT" rev-parse --abbrev-ref HEAD)" +git_head="$(git -C "$PROJECT_ROOT" rev-parse HEAD)" +git_dirty_count="$(git -C "$PROJECT_ROOT" status --porcelain | awk 'END { print NR + 0 }')" +rustc_version="$(rustc --version 2>/dev/null || echo unavailable)" +cargo_version="$(cargo --version 2>/dev/null || echo unavailable)" + +cat >"${OUT_DIR}/manifest.env" <"$matrix_csv" <<'EOF' +order,outer_reader_stream_buffer_size,run_dir,read_path_profiles,expected_profile_sequence,stage_metrics_required,body_header_parity_required,status +EOF + +run_cell() { + local profile_order="$1" + local outer_size="$2" + local run_name="${profile_order}-outer-${outer_size}" + local run_dir="${OUT_DIR}/${run_name}" + local bucket="${BUCKET}-${profile_order}-${outer_size}" + local profile_sequence + case "$profile_order" in + normal) profile_sequence="legacy>codec-legacy" ;; + reverse) profile_sequence="codec-legacy>legacy" ;; + *) die "unexpected profile order: $profile_order" ;; + esac + + log "running cell order=${profile_order} outer_reader_stream_buffer_size=${outer_size} out=${run_dir}" + local cmd=( + "$SMOKE_RUNNER" + --mode both + --profile-order "$profile_order" + --codec-engine legacy + --codec-min-size "$CODEC_MIN_SIZE" + --handoff-attribution + --diagnostic-metrics + --diagnostic-metrics-url "$DIAGNOSTIC_METRICS_URL" + --diagnostic-metrics-settle-secs "$DIAGNOSTIC_METRICS_SETTLE_SECS" + --diagnostic-metrics-capture-attempts "$DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS" + --diagnostic-metrics-capture-retry-secs "$DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS" + --diagnostic-metrics-connect-timeout-secs "$DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS" + --diagnostic-metrics-max-time-secs "$DIAGNOSTIC_METRICS_MAX_TIME_SECS" + --diagnostic-metrics-filter-regex "$DIAGNOSTIC_METRICS_FILTER_REGEX" + --diagnostic-obs-meter-interval "$DIAGNOSTIC_OBS_METER_INTERVAL" + --diagnostic-obs-service-name-prefix "${DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX}-${profile_order}-${outer_size}" + --address "$ADDRESS" + --bucket "$bucket" + --access-key "$ACCESS_KEY" + --secret-key "$SECRET_KEY" + --region "$REGION" + --sizes 1MiB + --concurrency "$CONCURRENCY" + --duration "$DURATION" + --rounds "$ROUNDS" + --retry-per-round "$RETRY_PER_ROUND" + --round-cooldown-secs "$ROUND_COOLDOWN_SECS" + --warp-object-lifecycle "$WARP_OBJECT_LIFECYCLE" + --warp-prepare-duration "$WARP_PREPARE_DURATION" + --out-dir "$run_dir" + --rustfs-bin "$RUSTFS_BIN" + --warp-bin "$WARP_BIN" + --python-bin "$PYTHON_BIN" + --resource-sample-interval-secs "$RESOURCE_SAMPLE_INTERVAL_SECS" + ) + if [[ -n "$DIAGNOSTIC_OBS_ENDPOINT" ]]; then + cmd+=(--diagnostic-obs-endpoint "$DIAGNOSTIC_OBS_ENDPOINT") + fi + if [[ -n "$DIAGNOSTIC_OBS_METRIC_ENDPOINT" ]]; then + cmd+=(--diagnostic-obs-metric-endpoint "$DIAGNOSTIC_OBS_METRIC_ENDPOINT") + fi + if [[ -n "$WARP_OBJECTS" ]]; then + cmd+=(--warp-objects "$WARP_OBJECTS") + fi + if [[ -n "$WARP_EXTRA_ARGS" ]]; then + cmd+=(--warp-extra-args "$WARP_EXTRA_ARGS") + fi + if [[ "$WARP_WARMUP_GET_BEFORE_BENCH" == "true" ]]; then + cmd+=(--warp-warmup-get-before-bench) + fi + if [[ "$SKIP_BUILD" == "true" ]]; then + cmd+=(--skip-build) + fi + if [[ "$DRY_RUN" == "true" ]]; then + cmd+=(--dry-run) + fi + + if RUSTFS_GET_READER_STREAM_BUFFER_SIZE="$outer_size" "${cmd[@]}"; then + printf '%s,%s,%s,%s,%s,%s,%s,%s\n' \ + "$profile_order" "$outer_size" "$run_dir" "legacy;codec-legacy" "$profile_sequence" \ + "service_metrics_round_summary.csv;service_metrics_stage_distribution.csv;service_metrics_round_percentiles.csv" \ + "compat_summary.csv;response_headers_legacy.json;response_headers_codec_legacy.json;body_sha256_legacy.txt;body_sha256_codec_legacy.txt" \ + "ok" >>"$matrix_csv" + else + printf '%s,%s,%s,%s,%s,%s,%s,%s\n' \ + "$profile_order" "$outer_size" "$run_dir" "legacy;codec-legacy" "$profile_sequence" \ + "service_metrics_round_summary.csv;service_metrics_stage_distribution.csv;service_metrics_round_percentiles.csv" \ + "compat_summary.csv;response_headers_legacy.json;response_headers_codec_legacy.json;body_sha256_legacy.txt;body_sha256_codec_legacy.txt" \ + "failed" >>"$matrix_csv" + return 1 + fi +} + +failures=0 +for outer_size in "${outer_sizes[@]}"; do + for profile_order in "${profile_orders[@]}"; do + if ! run_cell "$profile_order" "$outer_size"; then + failures=$((failures + 1)) + fi + done +done + +log "manifest: ${OUT_DIR}/manifest.env" +log "matrix: ${OUT_DIR}/abba_matrix.csv" +if [[ "$failures" -gt 0 ]]; then + die "${failures} matrix cell(s) failed" +fi +log "exact-1MiB ABBA stage metrics harness finished." diff --git a/scripts/test_exact_1mib_handoff_abba.sh b/scripts/test_exact_1mib_handoff_abba.sh new file mode 100755 index 000000000..6c347d716 --- /dev/null +++ b/scripts/test_exact_1mib_handoff_abba.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="${SCRIPT_DIR}/run_exact_1mib_handoff_abba.sh" +TMP_DIR="$(mktemp -d)" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +if "$RUNNER" --dry-run >/dev/null 2>&1; then + echo "expected missing arm specifications to fail" >&2 + exit 1 +fi + +if "$RUNNER" \ + --access-key minioadmin \ + --secret-key minioadmin \ + --arm-a "A|http://127.0.0.1:9000|rustfs/rustfs:current|latest|42fc84063|65536|auto|selected" \ + --arm-b "B|http://127.0.0.1:9001|rustfs/rustfs:candidate|sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb|42fc84063-inner|1048576|auto|selected" \ + --arm-c "C|http://127.0.0.1:9002|rustfs/rustfs:current-outer|sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc|42fc84063|65536|1048576|env_override" \ + --arm-d "D|http://127.0.0.1:9003|rustfs/rustfs:candidate-outer|sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd|42fc84063-inner|1048576|1048576|env_override" \ + --out-dir "$TMP_DIR/bad-digest" \ + --dry-run >/dev/null 2>&1; then + echo "expected mutable arm digest to fail" >&2 + exit 1 +fi + +if "$RUNNER" \ + --access-key minioadmin \ + --secret-key minioadmin \ + --arm-a "A|http://127.0.0.1:9000|rustfs/rustfs:current|sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa|42fc84063|65536|auto|manual" \ + --arm-b "B|http://127.0.0.1:9001|rustfs/rustfs:candidate|sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb|42fc84063-inner|1048576|auto|selected" \ + --arm-c "C|http://127.0.0.1:9002|rustfs/rustfs:current-outer|sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc|42fc84063|65536|1048576|env_override" \ + --arm-d "D|http://127.0.0.1:9003|rustfs/rustfs:candidate-outer|sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd|42fc84063-inner|1048576|1048576|env_override" \ + --out-dir "$TMP_DIR/bad-source" \ + --dry-run >/dev/null 2>&1; then + echo "expected unsupported outer source to fail" >&2 + exit 1 +fi + +"$RUNNER" \ + --access-key minioadmin \ + --secret-key minioadmin \ + --arm-a "A|http://127.0.0.1:9000|rustfs/rustfs:current|sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa|42fc84063|65536|auto|selected" \ + --arm-b "B|http://127.0.0.1:9001|rustfs/rustfs:candidate|sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb|42fc84063-inner|1048576|auto|selected" \ + --arm-c "C|http://127.0.0.1:9002|rustfs/rustfs:current-outer|sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc|42fc84063|65536|1048576|env_override" \ + --arm-d "D|http://127.0.0.1:9003|rustfs/rustfs:candidate-outer|sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd|42fc84063-inner|1048576|1048576|env_override" \ + --concurrencies 64 \ + --sizes 1048575B,1048576B,1048577B \ + --rounds-per-leg 5 \ + --cooldown-secs 0 \ + --out-dir "$TMP_DIR/good" \ + --dry-run >"$TMP_DIR/dry-run.log" + +MANIFEST="$TMP_DIR/good/handoff_manifest.env" +SCHEDULE="$TMP_DIR/good/handoff_abba_schedule.csv" + +rg -qx "benchmark_issue=rustfs/backlog#1434" "$MANIFEST" +rg -qx "schedule=ABBA_then_CDDC" "$MANIFEST" +rg -F -q -x 'sizes=1048575B\,1048576B\,1048577B' "$MANIFEST" +rg -qx "concurrencies=64" "$MANIFEST" +rg -qx "access_key=REDACTED" "$MANIFEST" +rg -qx "secret_key=REDACTED" "$MANIFEST" + +rg -qx "leg,arm,endpoint,server_image_ref,server_image_digest,server_revision,inner_capacity,outer_capacity,outer_source,leg_out_dir" "$SCHEDULE" +rg -qx "A1,A,http://127.0.0.1:9000,rustfs/rustfs:current,sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,42fc84063,65536,auto,selected,$TMP_DIR/good/A1-arm-A" "$SCHEDULE" +rg -qx "B1,B,http://127.0.0.1:9001,rustfs/rustfs:candidate,sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb,42fc84063-inner,1048576,auto,selected,$TMP_DIR/good/B1-arm-B" "$SCHEDULE" +rg -qx "B2,B,http://127.0.0.1:9001,rustfs/rustfs:candidate,sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb,42fc84063-inner,1048576,auto,selected,$TMP_DIR/good/B2-arm-B" "$SCHEDULE" +rg -qx "A2,A,http://127.0.0.1:9000,rustfs/rustfs:current,sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,42fc84063,65536,auto,selected,$TMP_DIR/good/A2-arm-A" "$SCHEDULE" +rg -qx "C1,C,http://127.0.0.1:9002,rustfs/rustfs:current-outer,sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc,42fc84063,65536,1048576,env_override,$TMP_DIR/good/C1-arm-C" "$SCHEDULE" +rg -qx "D1,D,http://127.0.0.1:9003,rustfs/rustfs:candidate-outer,sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd,42fc84063-inner,1048576,1048576,env_override,$TMP_DIR/good/D1-arm-D" "$SCHEDULE" +rg -qx "D2,D,http://127.0.0.1:9003,rustfs/rustfs:candidate-outer,sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd,42fc84063-inner,1048576,1048576,env_override,$TMP_DIR/good/D2-arm-D" "$SCHEDULE" +rg -qx "C2,C,http://127.0.0.1:9002,rustfs/rustfs:current-outer,sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc,42fc84063,65536,1048576,env_override,$TMP_DIR/good/C2-arm-C" "$SCHEDULE" + +rg -q -- "--service-metrics-dir" "$TMP_DIR/dry-run.log" +rg -F -q -- "--service-metrics-filter-regex rustfs_io_get_object_\\|rustfs_s3_get_object_" "$TMP_DIR/dry-run.log" +rg -q -- "--require-server-provenance" "$TMP_DIR/dry-run.log" +rg -q -- "--label bench_issue=1434" "$TMP_DIR/dry-run.log" +rg -q -- "--label handoff_arm=D" "$TMP_DIR/dry-run.log" +rg -q -- "--label expected_reader_path=legacy_duplex" "$TMP_DIR/dry-run.log" +rg -q -- "--label expected_inner_capacity=1048576" "$TMP_DIR/dry-run.log" +rg -q -- "--label expected_outer_capacity=1048576" "$TMP_DIR/dry-run.log" +rg -q -- "--label expected_outer_source=env_override" "$TMP_DIR/dry-run.log" + +echo "exact 1MiB handoff ABBA tests passed" diff --git a/scripts/test_get_1mib_abba_stage_metrics.sh b/scripts/test_get_1mib_abba_stage_metrics.sh new file mode 100755 index 000000000..0f6cb5b1b --- /dev/null +++ b/scripts/test_get_1mib_abba_stage_metrics.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="${SCRIPT_DIR}/run_get_1mib_abba_stage_metrics.sh" +TMP_DIR="$(mktemp -d)" +OUT_DIR="${TMP_DIR}/run" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +"$RUNNER" \ + --outer-reader-stream-buffer-sizes 65536,1048576 \ + --profile-orders normal,reverse \ + --concurrency 1 \ + --duration 1s \ + --rounds 1 \ + --retry-per-round 1 \ + --round-cooldown-secs 0 \ + --out-dir "$OUT_DIR" \ + --warp-bin true \ + --skip-build \ + --dry-run >/dev/null + +rg -qx 'issue=rustfs/backlog#1434' "${OUT_DIR}/manifest.env" +rg -qx 'exact_size=1MiB' "${OUT_DIR}/manifest.env" +rg -qx 'exact_size_bytes=1048576' "${OUT_DIR}/manifest.env" +rg -qx 'read_path_profiles=legacy,codec-legacy' "${OUT_DIR}/manifest.env" +rg -qx 'abba_profile_orders=normal,reverse' "${OUT_DIR}/manifest.env" +rg -qx 'outer_reader_stream_buffer_sizes=65536,1048576' "${OUT_DIR}/manifest.env" +rg -qx 'outer_reader_stream_env=RUSTFS_GET_READER_STREAM_BUFFER_SIZE' "${OUT_DIR}/manifest.env" +rg -qx 'inner_legacy_duplex_buffer_policy=adaptive_duplex_buffer_size' "${OUT_DIR}/manifest.env" +rg -qx 'exact_1mib_legacy_duplex_buffer_bytes=65536' "${OUT_DIR}/manifest.env" +rg -qx 'handoff_attribution=true' "${OUT_DIR}/manifest.env" +rg -qx 'diagnostic_metrics=true' "${OUT_DIR}/manifest.env" +rg -qx 'performance_conclusion=not_encoded_by_harness_collect_raw_abba_stage_metrics_first' "${OUT_DIR}/manifest.env" + +matrix_rows="$(awk -F',' 'NR > 1 { count++ } END { print count + 0 }' "${OUT_DIR}/abba_matrix.csv")" +if [[ "$matrix_rows" != "4" ]]; then + echo "expected 4 ABBA matrix rows, got ${matrix_rows}" >&2 + exit 1 +fi + +for outer_size in 65536 1048576; do + for profile_order in normal reverse; do + cell_dir="${OUT_DIR}/${profile_order}-outer-${outer_size}" + rg -qx "profile_order=${profile_order}" "${cell_dir}/manifest.env" + rg -qx 'sizes=1MiB' "${cell_dir}/manifest.env" + rg -qx 'mode=both' "${cell_dir}/manifest.env" + rg -qx 'codec_engines=legacy' "${cell_dir}/manifest.env" + rg -qx 'codec_min_size=1048576' "${cell_dir}/manifest.env" + rg -qx 'output_handoff_attribution=true' "${cell_dir}/manifest.env" + rg -qx 'diagnostic_metrics_enabled=true' "${cell_dir}/manifest.env" + rg -qx 'dry_run=true' "${cell_dir}/manifest.env" + + if [[ "$profile_order" == "normal" ]]; then + rg -qx 'profiles=legacy,codec-legacy' "${cell_dir}/manifest.env" + rg -q "^normal,${outer_size},.*,legacy;codec-legacy,legacy>codec-legacy,.*,ok$" "${OUT_DIR}/abba_matrix.csv" + else + rg -qx 'profiles=codec-legacy,legacy' "${cell_dir}/manifest.env" + rg -q "^reverse,${outer_size},.*,legacy;codec-legacy,codec-legacy>legacy,.*,ok$" "${OUT_DIR}/abba_matrix.csv" + fi + + for profile in legacy codec-legacy; do + profile_manifest="${cell_dir}/${profile}/manifest.env" + rg -qx 'sizes=1MiB' "$profile_manifest" + rg -qx 'RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE=true' "$profile_manifest" + rg -qx 'RUSTFS_OBS_METRICS_EXPORT_ENABLED=true' "$profile_manifest" + rg -qx 'RUSTFS_GET_CODEC_STREAMING_MIN_SIZE=1048576' "$profile_manifest" + test -f "${cell_dir}/${profile}/service_metrics_round_summary.csv" + test -f "${cell_dir}/${profile}/service_metrics_stage_distribution.csv" + test -f "${cell_dir}/${profile}/service_metrics_round_percentiles.csv" + done + done +done + +if "$RUNNER" --outer-reader-stream-buffer-sizes 0 --dry-run >/dev/null 2>&1; then + echo "expected zero outer ReaderStream buffer to be rejected" >&2 + exit 1 +fi + +if "$RUNNER" --profile-orders normal,sideways --dry-run >/dev/null 2>&1; then + echo "expected invalid profile order to be rejected" >&2 + exit 1 +fi