mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
test(perf): add exact 1MiB ABBA benchmark drivers (#5260)
* test(perf): add exact 1mib get abba harness Add a backlog#1434 exact-1MiB GET attribution wrapper around the codec streaming smoke runner. The harness fixes the legacy versus codec-legacy ABBA profile order, the outer ReaderStream buffer axis, path-proof artifacts, and GET stage-metrics capture so reviewers can reproduce isolated-host attribution without drifting the experiment matrix. Verification: - scripts/test_get_1mib_abba_stage_metrics.sh - bash -n scripts/run_get_1mib_abba_stage_metrics.sh scripts/test_get_1mib_abba_stage_metrics.sh - git diff --check Co-Authored-By: heihutu <heihutu@gmail.com> * test(perf): add exact 1MiB handoff ABBA driver Add a backlog#1434 benchmark orchestrator for isolated-host exact-1MiB handoff evidence. The runner records A/B/B/A and C/D/D/C legs, server provenance, expected reader path labels, inner and outer buffer capacity labels, and stage metrics capture arguments while delegating workload execution to the existing enhanced batch benchmark driver. Verification: - scripts/test_exact_1mib_handoff_abba.sh - bash -n scripts/run_exact_1mib_handoff_abba.sh scripts/test_exact_1mib_handoff_abba.sh - make script-tests - git diff --check Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
@@ -28,6 +28,7 @@ script-tests: ## Run shell script tests
|
|||||||
./scripts/test_entrypoint_credentials.sh
|
./scripts/test_entrypoint_credentials.sh
|
||||||
./scripts/test_internode_grpc_ab_bench.sh
|
./scripts/test_internode_grpc_ab_bench.sh
|
||||||
./scripts/test_object_batch_bench_enhanced.sh
|
./scripts/test_object_batch_bench_enhanced.sh
|
||||||
|
./scripts/test_exact_1mib_handoff_abba.sh
|
||||||
./scripts/test_pinned_paired_abba_bench.sh
|
./scripts/test_pinned_paired_abba_bench.sh
|
||||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||||
python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
|
python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||||
|
|||||||
@@ -70,6 +70,7 @@ their issue closes.
|
|||||||
| `run_object_batch_bench_enhanced.sh` | dev-tool | Enhanced batch benchmark runner; hub used by the smoke rigs | hotpath runbook |
|
| `run_object_batch_bench_enhanced.sh` | dev-tool | Enhanced batch benchmark runner; hub used by the smoke rigs | hotpath runbook |
|
||||||
| `run_pinned_paired_abba_bench.sh` | dev-tool | Pinned RustFS/MinIO paired ABBA benchmark orchestrator for backlog#1432 | `test_pinned_paired_abba_bench.sh` |
|
| `run_pinned_paired_abba_bench.sh` | dev-tool | Pinned RustFS/MinIO paired ABBA benchmark orchestrator for backlog#1432 | `test_pinned_paired_abba_bench.sh` |
|
||||||
| `run_get_codec_streaming_smoke.sh` | dev-tool | Local GET benchmark harness for the codec streaming read path | `docs/testing/ecstore-validation-suite-design.md` |
|
| `run_get_codec_streaming_smoke.sh` | dev-tool | Local GET benchmark harness for the codec streaming read path | `docs/testing/ecstore-validation-suite-design.md` |
|
||||||
|
| `run_get_1mib_abba_stage_metrics.sh` | dev-tool | Exact-1MiB isolated-host GET ABBA/stage-metrics harness for backlog#1434 | `test_get_1mib_abba_stage_metrics.sh` |
|
||||||
| `run_gt1g_get_http_matrix.sh` | dev-tool | >1 GiB GET HTTP matrix | `docs/testing/ecstore-validation-suite-design.md` |
|
| `run_gt1g_get_http_matrix.sh` | dev-tool | >1 GiB GET HTTP matrix | `docs/testing/ecstore-validation-suite-design.md` |
|
||||||
| `run_gt1g_multipart_put_matrix.sh` | dev-tool | >1 GiB multipart PUT matrix | `docs/testing/ecstore-validation-suite-design.md` |
|
| `run_gt1g_multipart_put_matrix.sh` | dev-tool | >1 GiB multipart PUT matrix | `docs/testing/ecstore-validation-suite-design.md` |
|
||||||
| `run_scanner_benchmarks.sh` | dev-tool (disposition pending) | Scanner performance benchmark runner. Contains a hardcoded stale path; **disposition owned by backlog perf-10 — do not fix, move, or delete it here** | — |
|
| `run_scanner_benchmarks.sh` | dev-tool (disposition pending) | Scanner performance benchmark runner. Contains a hardcoded stale path; **disposition owned by backlog perf-10 — do not fix, move, or delete it here** | — |
|
||||||
|
|||||||
Executable
+277
@@ -0,0 +1,277 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||||
|
ENHANCED_BENCH="${PROJECT_ROOT}/scripts/run_object_batch_bench_enhanced.sh"
|
||||||
|
|
||||||
|
OUT_DIR=""
|
||||||
|
ACCESS_KEY=""
|
||||||
|
SECRET_KEY=""
|
||||||
|
WARP_BIN="warp"
|
||||||
|
DURATION="60s"
|
||||||
|
CONCURRENCIES="64,128"
|
||||||
|
SIZES="524288B,1048575B,1048576B,1048577B,4194304B"
|
||||||
|
ROUNDS_PER_LEG=5
|
||||||
|
COOLDOWN_SECS=20
|
||||||
|
BUCKET_PREFIX="rustfs-1434-handoff"
|
||||||
|
DRY_RUN=false
|
||||||
|
|
||||||
|
ARM_A=""
|
||||||
|
ARM_B=""
|
||||||
|
ARM_C=""
|
||||||
|
ARM_D=""
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Usage:
|
||||||
|
scripts/run_exact_1mib_handoff_abba.sh --access-key <ak> --secret-key <sk> \
|
||||||
|
--arm-a <spec> --arm-b <spec> --arm-c <spec> --arm-d <spec> [options]
|
||||||
|
|
||||||
|
Arm spec:
|
||||||
|
name|endpoint|server_image_ref|server_image_digest|server_revision|inner_capacity|outer_capacity|outer_source
|
||||||
|
|
||||||
|
Required arms:
|
||||||
|
A = current adaptive inner, outer auto
|
||||||
|
B = exact-1MiB inner candidate, outer auto
|
||||||
|
C = current adaptive inner, outer 1MiB override
|
||||||
|
D = exact-1MiB inner candidate, outer 1MiB override
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--out-dir <dir> Default: target/bench/exact-1mib-handoff-abba-<timestamp>
|
||||||
|
--concurrencies <csv> Default: 64,128
|
||||||
|
--sizes <csv> Default: 512KiB, 1MiB-1, 1MiB, 1MiB+1, 4MiB
|
||||||
|
--rounds-per-leg <n> Default: 5
|
||||||
|
--cooldown-secs <n> Default: 20
|
||||||
|
--bucket-prefix <prefix> Default: rustfs-1434-handoff
|
||||||
|
--warp-bin <path> Default: warp
|
||||||
|
--duration <dur> Default: 60s
|
||||||
|
--dry-run Materialize manifest and commands only
|
||||||
|
-h, --help
|
||||||
|
|
||||||
|
This runner records isolated-host ABBA evidence for rustfs/backlog#1434 only. It does not change production defaults and does not claim an optimization unless the raw artifact gate is analyzed separately.
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
die() {
|
||||||
|
echo "ERROR: $*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_nonempty() {
|
||||||
|
local value="$1"
|
||||||
|
local name="$2"
|
||||||
|
[[ -n "$value" ]] || die "$name is required"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_positive_int() {
|
||||||
|
local value="$1"
|
||||||
|
local name="$2"
|
||||||
|
[[ "$value" =~ ^[0-9]+$ && "$value" -gt 0 ]] || die "$name must be a positive integer, got: $value"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_nonnegative_int() {
|
||||||
|
local value="$1"
|
||||||
|
local name="$2"
|
||||||
|
[[ "$value" =~ ^[0-9]+$ ]] || die "$name must be a nonnegative integer, got: $value"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_csv_nonempty() {
|
||||||
|
local value="$1"
|
||||||
|
local name="$2"
|
||||||
|
[[ -n "$value" && "$value" != *",,"* ]] || die "$name must be a non-empty CSV"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_digest() {
|
||||||
|
local value="$1"
|
||||||
|
local name="$2"
|
||||||
|
[[ "$value" =~ ^sha256:[0-9a-fA-F]{64}$ ]] || die "$name must be an immutable sha256 digest"
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_args() {
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--out-dir) OUT_DIR="$2"; shift 2 ;;
|
||||||
|
--access-key) ACCESS_KEY="$2"; shift 2 ;;
|
||||||
|
--secret-key) SECRET_KEY="$2"; shift 2 ;;
|
||||||
|
--warp-bin) WARP_BIN="$2"; shift 2 ;;
|
||||||
|
--duration) DURATION="$2"; shift 2 ;;
|
||||||
|
--concurrencies) CONCURRENCIES="$2"; shift 2 ;;
|
||||||
|
--sizes) SIZES="$2"; shift 2 ;;
|
||||||
|
--rounds-per-leg) ROUNDS_PER_LEG="$2"; shift 2 ;;
|
||||||
|
--cooldown-secs) COOLDOWN_SECS="$2"; shift 2 ;;
|
||||||
|
--bucket-prefix) BUCKET_PREFIX="$2"; shift 2 ;;
|
||||||
|
--arm-a) ARM_A="$2"; shift 2 ;;
|
||||||
|
--arm-b) ARM_B="$2"; shift 2 ;;
|
||||||
|
--arm-c) ARM_C="$2"; shift 2 ;;
|
||||||
|
--arm-d) ARM_D="$2"; shift 2 ;;
|
||||||
|
--dry-run) DRY_RUN=true; shift ;;
|
||||||
|
-h|--help) usage; exit 0 ;;
|
||||||
|
*) die "unknown arg: $1" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
split_arm() {
|
||||||
|
local spec="$1"
|
||||||
|
local prefix="$2"
|
||||||
|
local -a fields
|
||||||
|
IFS='|' read -r -a fields <<< "$spec"
|
||||||
|
[[ "${#fields[@]}" -eq 8 ]] || die "$prefix must have 8 pipe-separated fields"
|
||||||
|
printf -v "${prefix}_name" '%s' "${fields[0]}"
|
||||||
|
printf -v "${prefix}_endpoint" '%s' "${fields[1]}"
|
||||||
|
printf -v "${prefix}_image_ref" '%s' "${fields[2]}"
|
||||||
|
printf -v "${prefix}_image_digest" '%s' "${fields[3]}"
|
||||||
|
printf -v "${prefix}_revision" '%s' "${fields[4]}"
|
||||||
|
printf -v "${prefix}_inner" '%s' "${fields[5]}"
|
||||||
|
printf -v "${prefix}_outer" '%s' "${fields[6]}"
|
||||||
|
printf -v "${prefix}_outer_source" '%s' "${fields[7]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_arm() {
|
||||||
|
local prefix="$1"
|
||||||
|
local expected_name="$2"
|
||||||
|
local name_var="${prefix}_name"
|
||||||
|
local endpoint_var="${prefix}_endpoint"
|
||||||
|
local digest_var="${prefix}_image_digest"
|
||||||
|
local inner_var="${prefix}_inner"
|
||||||
|
local outer_var="${prefix}_outer"
|
||||||
|
local source_var="${prefix}_outer_source"
|
||||||
|
[[ "${!name_var}" == "$expected_name" ]] || die "$prefix name must be $expected_name"
|
||||||
|
validate_nonempty "${!endpoint_var}" "$prefix endpoint"
|
||||||
|
validate_digest "${!digest_var}" "$prefix image digest"
|
||||||
|
[[ "${!inner_var}" =~ ^[0-9]+$ ]] || die "$prefix inner capacity must be numeric bytes"
|
||||||
|
[[ "${!outer_var}" == "auto" || "${!outer_var}" =~ ^[0-9]+$ ]] || die "$prefix outer capacity must be auto or numeric bytes"
|
||||||
|
case "${!source_var}" in
|
||||||
|
selected|env_override) ;;
|
||||||
|
*) die "$prefix outer source must be selected or env_override" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_args() {
|
||||||
|
validate_nonempty "$ACCESS_KEY" "--access-key"
|
||||||
|
validate_nonempty "$SECRET_KEY" "--secret-key"
|
||||||
|
validate_nonempty "$ARM_A" "--arm-a"
|
||||||
|
validate_nonempty "$ARM_B" "--arm-b"
|
||||||
|
validate_nonempty "$ARM_C" "--arm-c"
|
||||||
|
validate_nonempty "$ARM_D" "--arm-d"
|
||||||
|
validate_csv_nonempty "$CONCURRENCIES" "--concurrencies"
|
||||||
|
validate_csv_nonempty "$SIZES" "--sizes"
|
||||||
|
validate_positive_int "$ROUNDS_PER_LEG" "--rounds-per-leg"
|
||||||
|
validate_nonnegative_int "$COOLDOWN_SECS" "--cooldown-secs"
|
||||||
|
split_arm "$ARM_A" "A"
|
||||||
|
split_arm "$ARM_B" "B"
|
||||||
|
split_arm "$ARM_C" "C"
|
||||||
|
split_arm "$ARM_D" "D"
|
||||||
|
validate_arm "A" "A"
|
||||||
|
validate_arm "B" "B"
|
||||||
|
validate_arm "C" "C"
|
||||||
|
validate_arm "D" "D"
|
||||||
|
}
|
||||||
|
|
||||||
|
setup_output() {
|
||||||
|
if [[ -z "$OUT_DIR" ]]; then
|
||||||
|
OUT_DIR="${PROJECT_ROOT}/target/bench/exact-1mib-handoff-abba-$(date -u +%Y%m%dT%H%M%SZ)"
|
||||||
|
fi
|
||||||
|
mkdir -p "$OUT_DIR"
|
||||||
|
}
|
||||||
|
|
||||||
|
arm_value() {
|
||||||
|
local arm="$1"
|
||||||
|
local field="$2"
|
||||||
|
local var="${arm}_${field}"
|
||||||
|
printf '%s' "${!var}"
|
||||||
|
}
|
||||||
|
|
||||||
|
write_manifest() {
|
||||||
|
local manifest="$OUT_DIR/handoff_manifest.env"
|
||||||
|
{
|
||||||
|
printf 'created_utc=%q\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||||
|
printf 'benchmark_issue=%q\n' "rustfs/backlog#1434"
|
||||||
|
printf 'schedule=%q\n' "ABBA_then_CDDC"
|
||||||
|
printf 'sizes=%q\n' "$SIZES"
|
||||||
|
printf 'concurrencies=%q\n' "$CONCURRENCIES"
|
||||||
|
printf 'rounds_per_leg=%q\n' "$ROUNDS_PER_LEG"
|
||||||
|
printf 'cooldown_secs=%q\n' "$COOLDOWN_SECS"
|
||||||
|
printf 'access_key=%q\n' "REDACTED"
|
||||||
|
printf 'secret_key=%q\n' "REDACTED"
|
||||||
|
} > "$manifest"
|
||||||
|
}
|
||||||
|
|
||||||
|
write_schedule() {
|
||||||
|
local schedule="$OUT_DIR/handoff_abba_schedule.csv"
|
||||||
|
echo "leg,arm,endpoint,server_image_ref,server_image_digest,server_revision,inner_capacity,outer_capacity,outer_source,leg_out_dir" > "$schedule"
|
||||||
|
for leg in A1 B1 B2 A2 C1 D1 D2 C2; do
|
||||||
|
local arm="${leg:0:1}"
|
||||||
|
echo "$leg,$arm,$(arm_value "$arm" endpoint),$(arm_value "$arm" image_ref),$(arm_value "$arm" image_digest),$(arm_value "$arm" revision),$(arm_value "$arm" inner),$(arm_value "$arm" outer),$(arm_value "$arm" outer_source),$OUT_DIR/$leg-arm-$arm" >> "$schedule"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
quote_cmd() {
|
||||||
|
printf '%q ' "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_leg() {
|
||||||
|
local leg="$1"
|
||||||
|
local arm="${leg:0:1}"
|
||||||
|
local conc="$2"
|
||||||
|
local leg_lower
|
||||||
|
leg_lower="$(printf '%s' "$leg" | tr '[:upper:]' '[:lower:]')"
|
||||||
|
local bucket="${BUCKET_PREFIX}-${leg_lower}-c${conc}"
|
||||||
|
local leg_out_dir="$OUT_DIR/$leg-arm-$arm/c${conc}"
|
||||||
|
local -a cmd=(
|
||||||
|
"$ENHANCED_BENCH"
|
||||||
|
--tool warp
|
||||||
|
--endpoint "$(arm_value "$arm" endpoint)"
|
||||||
|
--access-key "$ACCESS_KEY"
|
||||||
|
--secret-key "$SECRET_KEY"
|
||||||
|
--bucket "$bucket"
|
||||||
|
--bucket-size-suffix
|
||||||
|
--concurrency "$conc"
|
||||||
|
--sizes "$SIZES"
|
||||||
|
--rounds "$ROUNDS_PER_LEG"
|
||||||
|
--cooldown-secs "$COOLDOWN_SECS"
|
||||||
|
--warp-bin "$WARP_BIN"
|
||||||
|
--warp-mode get
|
||||||
|
--duration "$DURATION"
|
||||||
|
--service-metrics-dir "$leg_out_dir/service-metrics"
|
||||||
|
--service-metrics-filter-regex 'rustfs_io_get_object_|rustfs_s3_get_object_'
|
||||||
|
--server-image-ref "$(arm_value "$arm" image_ref)"
|
||||||
|
--server-image-digest "$(arm_value "$arm" image_digest)"
|
||||||
|
--server-revision "$(arm_value "$arm" revision)"
|
||||||
|
--require-server-provenance
|
||||||
|
--label bench_issue=1434
|
||||||
|
--label handoff_arm="$arm"
|
||||||
|
--label expected_reader_path=legacy_duplex
|
||||||
|
--label expected_inner_capacity="$(arm_value "$arm" inner)"
|
||||||
|
--label expected_outer_capacity="$(arm_value "$arm" outer)"
|
||||||
|
--label expected_outer_source="$(arm_value "$arm" outer_source)"
|
||||||
|
--out-dir "$leg_out_dir"
|
||||||
|
)
|
||||||
|
if [[ "$DRY_RUN" == "true" ]]; then
|
||||||
|
echo "-- dry-run leg=$leg arm=$arm concurrency=$conc"
|
||||||
|
echo " $(quote_cmd "${cmd[@]}" --dry-run)"
|
||||||
|
else
|
||||||
|
"${cmd[@]}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
run_schedule() {
|
||||||
|
local conc leg
|
||||||
|
IFS=',' read -r -a conc_values <<< "$CONCURRENCIES"
|
||||||
|
for conc in "${conc_values[@]}"; do
|
||||||
|
validate_positive_int "$conc" "concurrency cell"
|
||||||
|
for leg in A1 B1 B2 A2 C1 D1 D2 C2; do
|
||||||
|
run_leg "$leg" "$conc"
|
||||||
|
done
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_args "$@"
|
||||||
|
validate_args
|
||||||
|
setup_output
|
||||||
|
write_manifest
|
||||||
|
write_schedule
|
||||||
|
run_schedule
|
||||||
|
|
||||||
|
echo "exact 1MiB handoff ABBA artifact root: $OUT_DIR"
|
||||||
Executable
+398
@@ -0,0 +1,398 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Dedicated exact-1MiB GET attribution harness for rustfs/backlog#1434.
|
||||||
|
#
|
||||||
|
# The heavy lifting stays in run_get_codec_streaming_smoke.sh. This wrapper only
|
||||||
|
# fixes the experiment matrix so a reviewer can reproduce the isolated-host
|
||||||
|
# ABBA evidence without accidentally drifting object size, order, path proof, or
|
||||||
|
# stage-metrics capture.
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||||
|
SMOKE_RUNNER="${PROJECT_ROOT}/scripts/run_get_codec_streaming_smoke.sh"
|
||||||
|
|
||||||
|
ADDRESS="127.0.0.1:19030"
|
||||||
|
ACCESS_KEY="rustfsadmin"
|
||||||
|
SECRET_KEY="rustfsadmin"
|
||||||
|
BUCKET="rustfs-get-1mib-abba"
|
||||||
|
REGION="us-east-1"
|
||||||
|
CONCURRENCY=16
|
||||||
|
DURATION="30s"
|
||||||
|
ROUNDS=3
|
||||||
|
RETRY_PER_ROUND=1
|
||||||
|
ROUND_COOLDOWN_SECS=20
|
||||||
|
WARP_OBJECTS=""
|
||||||
|
WARP_OBJECT_LIFECYCLE="per-round"
|
||||||
|
WARP_PREPARE_DURATION="1s"
|
||||||
|
WARP_EXTRA_ARGS=""
|
||||||
|
WARP_WARMUP_GET_BEFORE_BENCH=false
|
||||||
|
OUTER_READER_STREAM_BUFFER_SIZES="65536,1048576"
|
||||||
|
PROFILE_ORDERS="normal,reverse"
|
||||||
|
CODEC_MIN_SIZE=1048576
|
||||||
|
RUSTFS_BIN="${PROJECT_ROOT}/target/release/rustfs"
|
||||||
|
WARP_BIN="warp"
|
||||||
|
PYTHON_BIN="python3"
|
||||||
|
OUT_DIR=""
|
||||||
|
DIAGNOSTIC_METRICS_URL="http://127.0.0.1:8889/metrics"
|
||||||
|
DIAGNOSTIC_METRICS_SETTLE_SECS="${RUSTFS_DIAGNOSTIC_METRICS_SETTLE_SECS:-2}"
|
||||||
|
DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS="${RUSTFS_DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS:-5}"
|
||||||
|
DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS="${RUSTFS_DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS:-1}"
|
||||||
|
DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS="${RUSTFS_DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS:-2}"
|
||||||
|
DIAGNOSTIC_METRICS_MAX_TIME_SECS="${RUSTFS_DIAGNOSTIC_METRICS_MAX_TIME_SECS:-15}"
|
||||||
|
DIAGNOSTIC_METRICS_FILTER_REGEX="${RUSTFS_DIAGNOSTIC_METRICS_FILTER_REGEX:-rustfs_io_get_object_}"
|
||||||
|
DIAGNOSTIC_OBS_ENDPOINT="${RUSTFS_OBS_ENDPOINT:-}"
|
||||||
|
DIAGNOSTIC_OBS_METRIC_ENDPOINT="${RUSTFS_OBS_METRIC_ENDPOINT:-}"
|
||||||
|
DIAGNOSTIC_OBS_METER_INTERVAL="${RUSTFS_OBS_METER_INTERVAL:-1}"
|
||||||
|
DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX="${RUSTFS_OBS_SERVICE_NAME:-RustFS-get-1mib-abba}"
|
||||||
|
RESOURCE_SAMPLE_INTERVAL_SECS="${RUSTFS_GET_BENCH_RESOURCE_SAMPLE_INTERVAL_SECS:-5}"
|
||||||
|
HEALTH_TIMEOUT_SECS=60
|
||||||
|
SKIP_BUILD=false
|
||||||
|
DRY_RUN=false
|
||||||
|
ORIGINAL_ARGS=("$@")
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Usage:
|
||||||
|
scripts/run_get_1mib_abba_stage_metrics.sh [options]
|
||||||
|
|
||||||
|
Purpose:
|
||||||
|
Run the rustfs/backlog#1434 exact-1MiB isolated-host GET attribution matrix:
|
||||||
|
- object size fixed to 1MiB / 1048576 bytes
|
||||||
|
- legacy and codec-legacy read-path profiles
|
||||||
|
- normal and reverse profile ordering for ABBA order-bias checks
|
||||||
|
- two outer ReaderStream capacities via RUSTFS_GET_READER_STREAM_BUFFER_SIZE
|
||||||
|
- output handoff attribution and GET stage metrics enabled
|
||||||
|
|
||||||
|
Core options:
|
||||||
|
--outer-reader-stream-buffer-sizes <csv>
|
||||||
|
RUSTFS_GET_READER_STREAM_BUFFER_SIZE values
|
||||||
|
used for the outer response ReaderStream axis
|
||||||
|
(default: 65536,1048576)
|
||||||
|
--profile-orders <csv> Profile orders to run (default: normal,reverse)
|
||||||
|
--address <host:port> RustFS listen address (default: 127.0.0.1:19030)
|
||||||
|
--bucket <name> Bucket prefix (default: rustfs-get-1mib-abba)
|
||||||
|
--concurrency <n> warp concurrency (default: 16)
|
||||||
|
--duration <duration> warp duration per round (default: 30s)
|
||||||
|
--rounds <n> rounds per matrix cell (default: 3)
|
||||||
|
--retry-per-round <n> failed-attempt retries per round (default: 1)
|
||||||
|
--round-cooldown-secs <n> cooldown seconds after each completed round
|
||||||
|
(default: 20)
|
||||||
|
--warp-objects <n> Number of objects prepared by warp
|
||||||
|
--warp-object-lifecycle <mode> per-round|prepare-once|existing-only
|
||||||
|
(default: per-round)
|
||||||
|
--warp-prepare-duration <dur> Duration used by prepare-once warmup
|
||||||
|
(default: 1s)
|
||||||
|
--warp-extra-args <args> Extra args forwarded to warp through the smoke
|
||||||
|
runner
|
||||||
|
--warp-warmup-get-before-bench Run a GET warmup before measured rounds
|
||||||
|
--out-dir <path> Output directory (default target/bench/get-1mib-abba-stage-metrics-<timestamp>)
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
--diagnostic-metrics-url <url> GET metrics scrape URL (default: http://127.0.0.1:8889/metrics)
|
||||||
|
--diagnostic-metrics-settle-secs <n>
|
||||||
|
--diagnostic-metrics-capture-attempts <n>
|
||||||
|
--diagnostic-metrics-capture-retry-secs <n>
|
||||||
|
--diagnostic-metrics-connect-timeout-secs <n>
|
||||||
|
--diagnostic-metrics-max-time-secs <n>
|
||||||
|
--diagnostic-metrics-filter-regex <regex>
|
||||||
|
--diagnostic-obs-endpoint <url>
|
||||||
|
--diagnostic-obs-metric-endpoint <url>
|
||||||
|
--diagnostic-obs-meter-interval <secs>
|
||||||
|
--diagnostic-obs-service-name-prefix <name>
|
||||||
|
|
||||||
|
Binary/options:
|
||||||
|
--rustfs-bin <path> RustFS binary (default: target/release/rustfs)
|
||||||
|
--warp-bin <path> warp binary (default: warp)
|
||||||
|
--python-bin <path> Python binary (default: python3)
|
||||||
|
--skip-build Do not build RustFS in the smoke runner
|
||||||
|
--dry-run Validate wiring without starting RustFS
|
||||||
|
-h, --help Show this help
|
||||||
|
|
||||||
|
Output:
|
||||||
|
<out-dir>/manifest.env
|
||||||
|
<out-dir>/abba_matrix.csv
|
||||||
|
<out-dir>/<order>-outer-<bytes>/manifest.env
|
||||||
|
<out-dir>/<order>-outer-<bytes>/metrics_summary.csv
|
||||||
|
<out-dir>/<order>-outer-<bytes>/baseline_compare.csv
|
||||||
|
<out-dir>/<order>-outer-<bytes>/service_metrics_summary.csv
|
||||||
|
<out-dir>/<order>-outer-<bytes>/service_metrics_round_summary.csv
|
||||||
|
<out-dir>/<order>-outer-<bytes>/service_metrics_stage_distribution.csv
|
||||||
|
<out-dir>/<order>-outer-<bytes>/service_metrics_round_percentiles.csv
|
||||||
|
<out-dir>/<order>-outer-<bytes>/service_metrics_acceptance.csv
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
die() {
|
||||||
|
echo "error: $*" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
log() {
|
||||||
|
printf '[get-1mib-abba] %s\n' "$*" >&2
|
||||||
|
}
|
||||||
|
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--outer-reader-stream-buffer-sizes) OUTER_READER_STREAM_BUFFER_SIZES="$2"; shift 2 ;;
|
||||||
|
--profile-orders) PROFILE_ORDERS="$2"; shift 2 ;;
|
||||||
|
--address) ADDRESS="$2"; shift 2 ;;
|
||||||
|
--bucket) BUCKET="$2"; shift 2 ;;
|
||||||
|
--access-key) ACCESS_KEY="$2"; shift 2 ;;
|
||||||
|
--secret-key) SECRET_KEY="$2"; shift 2 ;;
|
||||||
|
--region) REGION="$2"; shift 2 ;;
|
||||||
|
--concurrency) CONCURRENCY="$2"; shift 2 ;;
|
||||||
|
--duration) DURATION="$2"; shift 2 ;;
|
||||||
|
--rounds) ROUNDS="$2"; shift 2 ;;
|
||||||
|
--retry-per-round) RETRY_PER_ROUND="$2"; shift 2 ;;
|
||||||
|
--round-cooldown-secs) ROUND_COOLDOWN_SECS="$2"; shift 2 ;;
|
||||||
|
--warp-objects) WARP_OBJECTS="$2"; shift 2 ;;
|
||||||
|
--warp-object-lifecycle) WARP_OBJECT_LIFECYCLE="$2"; shift 2 ;;
|
||||||
|
--warp-prepare-duration) WARP_PREPARE_DURATION="$2"; shift 2 ;;
|
||||||
|
--warp-extra-args) WARP_EXTRA_ARGS="$2"; shift 2 ;;
|
||||||
|
--warp-warmup-get-before-bench) WARP_WARMUP_GET_BEFORE_BENCH=true; shift ;;
|
||||||
|
--out-dir) OUT_DIR="$2"; shift 2 ;;
|
||||||
|
--diagnostic-metrics-url) DIAGNOSTIC_METRICS_URL="$2"; shift 2 ;;
|
||||||
|
--diagnostic-metrics-settle-secs) DIAGNOSTIC_METRICS_SETTLE_SECS="$2"; shift 2 ;;
|
||||||
|
--diagnostic-metrics-capture-attempts) DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS="$2"; shift 2 ;;
|
||||||
|
--diagnostic-metrics-capture-retry-secs) DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS="$2"; shift 2 ;;
|
||||||
|
--diagnostic-metrics-connect-timeout-secs) DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS="$2"; shift 2 ;;
|
||||||
|
--diagnostic-metrics-max-time-secs) DIAGNOSTIC_METRICS_MAX_TIME_SECS="$2"; shift 2 ;;
|
||||||
|
--diagnostic-metrics-filter-regex) DIAGNOSTIC_METRICS_FILTER_REGEX="$2"; shift 2 ;;
|
||||||
|
--diagnostic-obs-endpoint) DIAGNOSTIC_OBS_ENDPOINT="$2"; shift 2 ;;
|
||||||
|
--diagnostic-obs-metric-endpoint) DIAGNOSTIC_OBS_METRIC_ENDPOINT="$2"; shift 2 ;;
|
||||||
|
--diagnostic-obs-meter-interval) DIAGNOSTIC_OBS_METER_INTERVAL="$2"; shift 2 ;;
|
||||||
|
--diagnostic-obs-service-name-prefix) DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX="$2"; shift 2 ;;
|
||||||
|
--rustfs-bin) RUSTFS_BIN="$2"; shift 2 ;;
|
||||||
|
--warp-bin) WARP_BIN="$2"; shift 2 ;;
|
||||||
|
--python-bin) PYTHON_BIN="$2"; shift 2 ;;
|
||||||
|
--skip-build) SKIP_BUILD=true; shift ;;
|
||||||
|
--dry-run) DRY_RUN=true; shift ;;
|
||||||
|
-h|--help) usage; exit 0 ;;
|
||||||
|
*) die "unknown argument: $1" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
[[ -x "$SMOKE_RUNNER" ]] || die "missing smoke runner: $SMOKE_RUNNER"
|
||||||
|
[[ -n "$OUTER_READER_STREAM_BUFFER_SIZES" ]] || die "--outer-reader-stream-buffer-sizes must not be empty"
|
||||||
|
[[ -n "$PROFILE_ORDERS" ]] || die "--profile-orders must not be empty"
|
||||||
|
|
||||||
|
validate_positive_int() {
|
||||||
|
local value="$1" label="$2"
|
||||||
|
[[ "$value" =~ ^[0-9]+$ && "$value" -gt 0 ]] || die "${label} must be a positive integer"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_non_negative_int() {
|
||||||
|
local value="$1" label="$2"
|
||||||
|
[[ "$value" =~ ^[0-9]+$ ]] || die "${label} must be a non-negative integer"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_positive_int "$CONCURRENCY" "--concurrency"
|
||||||
|
validate_positive_int "$ROUNDS" "--rounds"
|
||||||
|
validate_positive_int "$RETRY_PER_ROUND" "--retry-per-round"
|
||||||
|
validate_non_negative_int "$ROUND_COOLDOWN_SECS" "--round-cooldown-secs"
|
||||||
|
validate_non_negative_int "$DIAGNOSTIC_METRICS_SETTLE_SECS" "--diagnostic-metrics-settle-secs"
|
||||||
|
validate_positive_int "$DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS" "--diagnostic-metrics-capture-attempts"
|
||||||
|
validate_non_negative_int "$DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS" "--diagnostic-metrics-capture-retry-secs"
|
||||||
|
validate_positive_int "$DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS" "--diagnostic-metrics-connect-timeout-secs"
|
||||||
|
validate_positive_int "$DIAGNOSTIC_METRICS_MAX_TIME_SECS" "--diagnostic-metrics-max-time-secs"
|
||||||
|
validate_positive_int "$DIAGNOSTIC_OBS_METER_INTERVAL" "--diagnostic-obs-meter-interval"
|
||||||
|
|
||||||
|
IFS=',' read -r -a outer_sizes <<< "$OUTER_READER_STREAM_BUFFER_SIZES"
|
||||||
|
IFS=',' read -r -a profile_orders <<< "$PROFILE_ORDERS"
|
||||||
|
|
||||||
|
cleaned_outer_sizes=()
|
||||||
|
for raw_outer_size in "${outer_sizes[@]}"; do
|
||||||
|
clean_outer_size="${raw_outer_size//[[:space:]]/}"
|
||||||
|
[[ -n "$clean_outer_size" ]] && cleaned_outer_sizes+=("$clean_outer_size")
|
||||||
|
done
|
||||||
|
outer_sizes=("${cleaned_outer_sizes[@]}")
|
||||||
|
|
||||||
|
cleaned_profile_orders=()
|
||||||
|
for raw_profile_order in "${profile_orders[@]}"; do
|
||||||
|
clean_profile_order="${raw_profile_order//[[:space:]]/}"
|
||||||
|
[[ -n "$clean_profile_order" ]] && cleaned_profile_orders+=("$clean_profile_order")
|
||||||
|
done
|
||||||
|
profile_orders=("${cleaned_profile_orders[@]}")
|
||||||
|
[[ "${#outer_sizes[@]}" -gt 0 ]] || die "--outer-reader-stream-buffer-sizes must include at least one value"
|
||||||
|
[[ "${#profile_orders[@]}" -gt 0 ]] || die "--profile-orders must include at least one value"
|
||||||
|
|
||||||
|
for outer_size in "${outer_sizes[@]}"; do
|
||||||
|
validate_positive_int "$outer_size" "--outer-reader-stream-buffer-sizes entry"
|
||||||
|
done
|
||||||
|
for profile_order in "${profile_orders[@]}"; do
|
||||||
|
case "$profile_order" in
|
||||||
|
normal|reverse) ;;
|
||||||
|
*) die "--profile-orders entries must be normal or reverse" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -z "$OUT_DIR" ]]; then
|
||||||
|
OUT_DIR="${PROJECT_ROOT}/target/bench/get-1mib-abba-stage-metrics-$(date -u +%Y%m%dT%H%M%SZ 2>/dev/null || echo run)"
|
||||||
|
fi
|
||||||
|
mkdir -p "$OUT_DIR"
|
||||||
|
|
||||||
|
command_line() {
|
||||||
|
printf '%q ' "$0" "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
branch="$(git -C "$PROJECT_ROOT" rev-parse --abbrev-ref HEAD)"
|
||||||
|
git_head="$(git -C "$PROJECT_ROOT" rev-parse HEAD)"
|
||||||
|
git_dirty_count="$(git -C "$PROJECT_ROOT" status --porcelain | awk 'END { print NR + 0 }')"
|
||||||
|
rustc_version="$(rustc --version 2>/dev/null || echo unavailable)"
|
||||||
|
cargo_version="$(cargo --version 2>/dev/null || echo unavailable)"
|
||||||
|
|
||||||
|
cat >"${OUT_DIR}/manifest.env" <<EOF
|
||||||
|
issue=rustfs/backlog#1434
|
||||||
|
generated_at_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||||
|
branch=${branch}
|
||||||
|
git_head=${git_head}
|
||||||
|
git_dirty_count=${git_dirty_count}
|
||||||
|
rustc_version=${rustc_version}
|
||||||
|
cargo_version=${cargo_version}
|
||||||
|
host_scope=isolated_local_single_node_four_disks
|
||||||
|
exact_size=1MiB
|
||||||
|
exact_size_bytes=1048576
|
||||||
|
read_path_profiles=legacy,codec-legacy
|
||||||
|
abba_profile_orders=${PROFILE_ORDERS}
|
||||||
|
outer_reader_stream_buffer_sizes=${OUTER_READER_STREAM_BUFFER_SIZES}
|
||||||
|
outer_reader_stream_env=RUSTFS_GET_READER_STREAM_BUFFER_SIZE
|
||||||
|
inner_legacy_duplex_buffer_policy=adaptive_duplex_buffer_size
|
||||||
|
exact_1mib_legacy_duplex_buffer_bytes=65536
|
||||||
|
codec_engine=legacy
|
||||||
|
codec_min_size=${CODEC_MIN_SIZE}
|
||||||
|
handoff_attribution=true
|
||||||
|
diagnostic_metrics=true
|
||||||
|
diagnostic_metrics_url=${DIAGNOSTIC_METRICS_URL}
|
||||||
|
stage_metrics_artifacts=service_metrics_summary.csv,service_metrics_round_summary.csv,service_metrics_stage_distribution.csv,service_metrics_round_percentiles.csv
|
||||||
|
body_header_parity_artifacts=compat_summary.csv,response_headers_legacy.json,response_headers_codec_legacy.json,body_sha256_legacy.txt,body_sha256_codec_legacy.txt
|
||||||
|
performance_conclusion=not_encoded_by_harness_collect_raw_abba_stage_metrics_first
|
||||||
|
address=${ADDRESS}
|
||||||
|
bucket=${BUCKET}
|
||||||
|
region=${REGION}
|
||||||
|
concurrency=${CONCURRENCY}
|
||||||
|
duration=${DURATION}
|
||||||
|
rounds=${ROUNDS}
|
||||||
|
retry_per_round=${RETRY_PER_ROUND}
|
||||||
|
round_cooldown_secs=${ROUND_COOLDOWN_SECS}
|
||||||
|
warp_objects=${WARP_OBJECTS}
|
||||||
|
warp_object_lifecycle=${WARP_OBJECT_LIFECYCLE}
|
||||||
|
warp_prepare_duration=${WARP_PREPARE_DURATION}
|
||||||
|
warp_extra_args=${WARP_EXTRA_ARGS}
|
||||||
|
warp_warmup_get_before_bench=${WARP_WARMUP_GET_BEFORE_BENCH}
|
||||||
|
skip_build=${SKIP_BUILD}
|
||||||
|
dry_run=${DRY_RUN}
|
||||||
|
rustfs_bin=${RUSTFS_BIN}
|
||||||
|
warp_bin=${WARP_BIN}
|
||||||
|
python_bin=${PYTHON_BIN}
|
||||||
|
command_line=$(command_line "${ORIGINAL_ARGS[@]}")
|
||||||
|
EOF
|
||||||
|
|
||||||
|
matrix_csv="${OUT_DIR}/abba_matrix.csv"
|
||||||
|
cat >"$matrix_csv" <<'EOF'
|
||||||
|
order,outer_reader_stream_buffer_size,run_dir,read_path_profiles,expected_profile_sequence,stage_metrics_required,body_header_parity_required,status
|
||||||
|
EOF
|
||||||
|
|
||||||
|
run_cell() {
|
||||||
|
local profile_order="$1"
|
||||||
|
local outer_size="$2"
|
||||||
|
local run_name="${profile_order}-outer-${outer_size}"
|
||||||
|
local run_dir="${OUT_DIR}/${run_name}"
|
||||||
|
local bucket="${BUCKET}-${profile_order}-${outer_size}"
|
||||||
|
local profile_sequence
|
||||||
|
case "$profile_order" in
|
||||||
|
normal) profile_sequence="legacy>codec-legacy" ;;
|
||||||
|
reverse) profile_sequence="codec-legacy>legacy" ;;
|
||||||
|
*) die "unexpected profile order: $profile_order" ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
log "running cell order=${profile_order} outer_reader_stream_buffer_size=${outer_size} out=${run_dir}"
|
||||||
|
local cmd=(
|
||||||
|
"$SMOKE_RUNNER"
|
||||||
|
--mode both
|
||||||
|
--profile-order "$profile_order"
|
||||||
|
--codec-engine legacy
|
||||||
|
--codec-min-size "$CODEC_MIN_SIZE"
|
||||||
|
--handoff-attribution
|
||||||
|
--diagnostic-metrics
|
||||||
|
--diagnostic-metrics-url "$DIAGNOSTIC_METRICS_URL"
|
||||||
|
--diagnostic-metrics-settle-secs "$DIAGNOSTIC_METRICS_SETTLE_SECS"
|
||||||
|
--diagnostic-metrics-capture-attempts "$DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS"
|
||||||
|
--diagnostic-metrics-capture-retry-secs "$DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS"
|
||||||
|
--diagnostic-metrics-connect-timeout-secs "$DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS"
|
||||||
|
--diagnostic-metrics-max-time-secs "$DIAGNOSTIC_METRICS_MAX_TIME_SECS"
|
||||||
|
--diagnostic-metrics-filter-regex "$DIAGNOSTIC_METRICS_FILTER_REGEX"
|
||||||
|
--diagnostic-obs-meter-interval "$DIAGNOSTIC_OBS_METER_INTERVAL"
|
||||||
|
--diagnostic-obs-service-name-prefix "${DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX}-${profile_order}-${outer_size}"
|
||||||
|
--address "$ADDRESS"
|
||||||
|
--bucket "$bucket"
|
||||||
|
--access-key "$ACCESS_KEY"
|
||||||
|
--secret-key "$SECRET_KEY"
|
||||||
|
--region "$REGION"
|
||||||
|
--sizes 1MiB
|
||||||
|
--concurrency "$CONCURRENCY"
|
||||||
|
--duration "$DURATION"
|
||||||
|
--rounds "$ROUNDS"
|
||||||
|
--retry-per-round "$RETRY_PER_ROUND"
|
||||||
|
--round-cooldown-secs "$ROUND_COOLDOWN_SECS"
|
||||||
|
--warp-object-lifecycle "$WARP_OBJECT_LIFECYCLE"
|
||||||
|
--warp-prepare-duration "$WARP_PREPARE_DURATION"
|
||||||
|
--out-dir "$run_dir"
|
||||||
|
--rustfs-bin "$RUSTFS_BIN"
|
||||||
|
--warp-bin "$WARP_BIN"
|
||||||
|
--python-bin "$PYTHON_BIN"
|
||||||
|
--resource-sample-interval-secs "$RESOURCE_SAMPLE_INTERVAL_SECS"
|
||||||
|
)
|
||||||
|
if [[ -n "$DIAGNOSTIC_OBS_ENDPOINT" ]]; then
|
||||||
|
cmd+=(--diagnostic-obs-endpoint "$DIAGNOSTIC_OBS_ENDPOINT")
|
||||||
|
fi
|
||||||
|
if [[ -n "$DIAGNOSTIC_OBS_METRIC_ENDPOINT" ]]; then
|
||||||
|
cmd+=(--diagnostic-obs-metric-endpoint "$DIAGNOSTIC_OBS_METRIC_ENDPOINT")
|
||||||
|
fi
|
||||||
|
if [[ -n "$WARP_OBJECTS" ]]; then
|
||||||
|
cmd+=(--warp-objects "$WARP_OBJECTS")
|
||||||
|
fi
|
||||||
|
if [[ -n "$WARP_EXTRA_ARGS" ]]; then
|
||||||
|
cmd+=(--warp-extra-args "$WARP_EXTRA_ARGS")
|
||||||
|
fi
|
||||||
|
if [[ "$WARP_WARMUP_GET_BEFORE_BENCH" == "true" ]]; then
|
||||||
|
cmd+=(--warp-warmup-get-before-bench)
|
||||||
|
fi
|
||||||
|
if [[ "$SKIP_BUILD" == "true" ]]; then
|
||||||
|
cmd+=(--skip-build)
|
||||||
|
fi
|
||||||
|
if [[ "$DRY_RUN" == "true" ]]; then
|
||||||
|
cmd+=(--dry-run)
|
||||||
|
fi
|
||||||
|
|
||||||
|
if RUSTFS_GET_READER_STREAM_BUFFER_SIZE="$outer_size" "${cmd[@]}"; then
|
||||||
|
printf '%s,%s,%s,%s,%s,%s,%s,%s\n' \
|
||||||
|
"$profile_order" "$outer_size" "$run_dir" "legacy;codec-legacy" "$profile_sequence" \
|
||||||
|
"service_metrics_round_summary.csv;service_metrics_stage_distribution.csv;service_metrics_round_percentiles.csv" \
|
||||||
|
"compat_summary.csv;response_headers_legacy.json;response_headers_codec_legacy.json;body_sha256_legacy.txt;body_sha256_codec_legacy.txt" \
|
||||||
|
"ok" >>"$matrix_csv"
|
||||||
|
else
|
||||||
|
printf '%s,%s,%s,%s,%s,%s,%s,%s\n' \
|
||||||
|
"$profile_order" "$outer_size" "$run_dir" "legacy;codec-legacy" "$profile_sequence" \
|
||||||
|
"service_metrics_round_summary.csv;service_metrics_stage_distribution.csv;service_metrics_round_percentiles.csv" \
|
||||||
|
"compat_summary.csv;response_headers_legacy.json;response_headers_codec_legacy.json;body_sha256_legacy.txt;body_sha256_codec_legacy.txt" \
|
||||||
|
"failed" >>"$matrix_csv"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
failures=0
|
||||||
|
for outer_size in "${outer_sizes[@]}"; do
|
||||||
|
for profile_order in "${profile_orders[@]}"; do
|
||||||
|
if ! run_cell "$profile_order" "$outer_size"; then
|
||||||
|
failures=$((failures + 1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
done
|
||||||
|
|
||||||
|
log "manifest: ${OUT_DIR}/manifest.env"
|
||||||
|
log "matrix: ${OUT_DIR}/abba_matrix.csv"
|
||||||
|
if [[ "$failures" -gt 0 ]]; then
|
||||||
|
die "${failures} matrix cell(s) failed"
|
||||||
|
fi
|
||||||
|
log "exact-1MiB ABBA stage metrics harness finished."
|
||||||
Executable
+88
@@ -0,0 +1,88 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
RUNNER="${SCRIPT_DIR}/run_exact_1mib_handoff_abba.sh"
|
||||||
|
TMP_DIR="$(mktemp -d)"
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
rm -rf "$TMP_DIR"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
if "$RUNNER" --dry-run >/dev/null 2>&1; then
|
||||||
|
echo "expected missing arm specifications to fail" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if "$RUNNER" \
|
||||||
|
--access-key minioadmin \
|
||||||
|
--secret-key minioadmin \
|
||||||
|
--arm-a "A|http://127.0.0.1:9000|rustfs/rustfs:current|latest|42fc84063|65536|auto|selected" \
|
||||||
|
--arm-b "B|http://127.0.0.1:9001|rustfs/rustfs:candidate|sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb|42fc84063-inner|1048576|auto|selected" \
|
||||||
|
--arm-c "C|http://127.0.0.1:9002|rustfs/rustfs:current-outer|sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc|42fc84063|65536|1048576|env_override" \
|
||||||
|
--arm-d "D|http://127.0.0.1:9003|rustfs/rustfs:candidate-outer|sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd|42fc84063-inner|1048576|1048576|env_override" \
|
||||||
|
--out-dir "$TMP_DIR/bad-digest" \
|
||||||
|
--dry-run >/dev/null 2>&1; then
|
||||||
|
echo "expected mutable arm digest to fail" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if "$RUNNER" \
|
||||||
|
--access-key minioadmin \
|
||||||
|
--secret-key minioadmin \
|
||||||
|
--arm-a "A|http://127.0.0.1:9000|rustfs/rustfs:current|sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa|42fc84063|65536|auto|manual" \
|
||||||
|
--arm-b "B|http://127.0.0.1:9001|rustfs/rustfs:candidate|sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb|42fc84063-inner|1048576|auto|selected" \
|
||||||
|
--arm-c "C|http://127.0.0.1:9002|rustfs/rustfs:current-outer|sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc|42fc84063|65536|1048576|env_override" \
|
||||||
|
--arm-d "D|http://127.0.0.1:9003|rustfs/rustfs:candidate-outer|sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd|42fc84063-inner|1048576|1048576|env_override" \
|
||||||
|
--out-dir "$TMP_DIR/bad-source" \
|
||||||
|
--dry-run >/dev/null 2>&1; then
|
||||||
|
echo "expected unsupported outer source to fail" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
"$RUNNER" \
|
||||||
|
--access-key minioadmin \
|
||||||
|
--secret-key minioadmin \
|
||||||
|
--arm-a "A|http://127.0.0.1:9000|rustfs/rustfs:current|sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa|42fc84063|65536|auto|selected" \
|
||||||
|
--arm-b "B|http://127.0.0.1:9001|rustfs/rustfs:candidate|sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb|42fc84063-inner|1048576|auto|selected" \
|
||||||
|
--arm-c "C|http://127.0.0.1:9002|rustfs/rustfs:current-outer|sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc|42fc84063|65536|1048576|env_override" \
|
||||||
|
--arm-d "D|http://127.0.0.1:9003|rustfs/rustfs:candidate-outer|sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd|42fc84063-inner|1048576|1048576|env_override" \
|
||||||
|
--concurrencies 64 \
|
||||||
|
--sizes 1048575B,1048576B,1048577B \
|
||||||
|
--rounds-per-leg 5 \
|
||||||
|
--cooldown-secs 0 \
|
||||||
|
--out-dir "$TMP_DIR/good" \
|
||||||
|
--dry-run >"$TMP_DIR/dry-run.log"
|
||||||
|
|
||||||
|
MANIFEST="$TMP_DIR/good/handoff_manifest.env"
|
||||||
|
SCHEDULE="$TMP_DIR/good/handoff_abba_schedule.csv"
|
||||||
|
|
||||||
|
rg -qx "benchmark_issue=rustfs/backlog#1434" "$MANIFEST"
|
||||||
|
rg -qx "schedule=ABBA_then_CDDC" "$MANIFEST"
|
||||||
|
rg -F -q -x 'sizes=1048575B\,1048576B\,1048577B' "$MANIFEST"
|
||||||
|
rg -qx "concurrencies=64" "$MANIFEST"
|
||||||
|
rg -qx "access_key=REDACTED" "$MANIFEST"
|
||||||
|
rg -qx "secret_key=REDACTED" "$MANIFEST"
|
||||||
|
|
||||||
|
rg -qx "leg,arm,endpoint,server_image_ref,server_image_digest,server_revision,inner_capacity,outer_capacity,outer_source,leg_out_dir" "$SCHEDULE"
|
||||||
|
rg -qx "A1,A,http://127.0.0.1:9000,rustfs/rustfs:current,sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,42fc84063,65536,auto,selected,$TMP_DIR/good/A1-arm-A" "$SCHEDULE"
|
||||||
|
rg -qx "B1,B,http://127.0.0.1:9001,rustfs/rustfs:candidate,sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb,42fc84063-inner,1048576,auto,selected,$TMP_DIR/good/B1-arm-B" "$SCHEDULE"
|
||||||
|
rg -qx "B2,B,http://127.0.0.1:9001,rustfs/rustfs:candidate,sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb,42fc84063-inner,1048576,auto,selected,$TMP_DIR/good/B2-arm-B" "$SCHEDULE"
|
||||||
|
rg -qx "A2,A,http://127.0.0.1:9000,rustfs/rustfs:current,sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,42fc84063,65536,auto,selected,$TMP_DIR/good/A2-arm-A" "$SCHEDULE"
|
||||||
|
rg -qx "C1,C,http://127.0.0.1:9002,rustfs/rustfs:current-outer,sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc,42fc84063,65536,1048576,env_override,$TMP_DIR/good/C1-arm-C" "$SCHEDULE"
|
||||||
|
rg -qx "D1,D,http://127.0.0.1:9003,rustfs/rustfs:candidate-outer,sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd,42fc84063-inner,1048576,1048576,env_override,$TMP_DIR/good/D1-arm-D" "$SCHEDULE"
|
||||||
|
rg -qx "D2,D,http://127.0.0.1:9003,rustfs/rustfs:candidate-outer,sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd,42fc84063-inner,1048576,1048576,env_override,$TMP_DIR/good/D2-arm-D" "$SCHEDULE"
|
||||||
|
rg -qx "C2,C,http://127.0.0.1:9002,rustfs/rustfs:current-outer,sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc,42fc84063,65536,1048576,env_override,$TMP_DIR/good/C2-arm-C" "$SCHEDULE"
|
||||||
|
|
||||||
|
rg -q -- "--service-metrics-dir" "$TMP_DIR/dry-run.log"
|
||||||
|
rg -F -q -- "--service-metrics-filter-regex rustfs_io_get_object_\\|rustfs_s3_get_object_" "$TMP_DIR/dry-run.log"
|
||||||
|
rg -q -- "--require-server-provenance" "$TMP_DIR/dry-run.log"
|
||||||
|
rg -q -- "--label bench_issue=1434" "$TMP_DIR/dry-run.log"
|
||||||
|
rg -q -- "--label handoff_arm=D" "$TMP_DIR/dry-run.log"
|
||||||
|
rg -q -- "--label expected_reader_path=legacy_duplex" "$TMP_DIR/dry-run.log"
|
||||||
|
rg -q -- "--label expected_inner_capacity=1048576" "$TMP_DIR/dry-run.log"
|
||||||
|
rg -q -- "--label expected_outer_capacity=1048576" "$TMP_DIR/dry-run.log"
|
||||||
|
rg -q -- "--label expected_outer_source=env_override" "$TMP_DIR/dry-run.log"
|
||||||
|
|
||||||
|
echo "exact 1MiB handoff ABBA tests passed"
|
||||||
Executable
+87
@@ -0,0 +1,87 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
RUNNER="${SCRIPT_DIR}/run_get_1mib_abba_stage_metrics.sh"
|
||||||
|
TMP_DIR="$(mktemp -d)"
|
||||||
|
OUT_DIR="${TMP_DIR}/run"
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
rm -rf "$TMP_DIR"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
"$RUNNER" \
|
||||||
|
--outer-reader-stream-buffer-sizes 65536,1048576 \
|
||||||
|
--profile-orders normal,reverse \
|
||||||
|
--concurrency 1 \
|
||||||
|
--duration 1s \
|
||||||
|
--rounds 1 \
|
||||||
|
--retry-per-round 1 \
|
||||||
|
--round-cooldown-secs 0 \
|
||||||
|
--out-dir "$OUT_DIR" \
|
||||||
|
--warp-bin true \
|
||||||
|
--skip-build \
|
||||||
|
--dry-run >/dev/null
|
||||||
|
|
||||||
|
rg -qx 'issue=rustfs/backlog#1434' "${OUT_DIR}/manifest.env"
|
||||||
|
rg -qx 'exact_size=1MiB' "${OUT_DIR}/manifest.env"
|
||||||
|
rg -qx 'exact_size_bytes=1048576' "${OUT_DIR}/manifest.env"
|
||||||
|
rg -qx 'read_path_profiles=legacy,codec-legacy' "${OUT_DIR}/manifest.env"
|
||||||
|
rg -qx 'abba_profile_orders=normal,reverse' "${OUT_DIR}/manifest.env"
|
||||||
|
rg -qx 'outer_reader_stream_buffer_sizes=65536,1048576' "${OUT_DIR}/manifest.env"
|
||||||
|
rg -qx 'outer_reader_stream_env=RUSTFS_GET_READER_STREAM_BUFFER_SIZE' "${OUT_DIR}/manifest.env"
|
||||||
|
rg -qx 'inner_legacy_duplex_buffer_policy=adaptive_duplex_buffer_size' "${OUT_DIR}/manifest.env"
|
||||||
|
rg -qx 'exact_1mib_legacy_duplex_buffer_bytes=65536' "${OUT_DIR}/manifest.env"
|
||||||
|
rg -qx 'handoff_attribution=true' "${OUT_DIR}/manifest.env"
|
||||||
|
rg -qx 'diagnostic_metrics=true' "${OUT_DIR}/manifest.env"
|
||||||
|
rg -qx 'performance_conclusion=not_encoded_by_harness_collect_raw_abba_stage_metrics_first' "${OUT_DIR}/manifest.env"
|
||||||
|
|
||||||
|
matrix_rows="$(awk -F',' 'NR > 1 { count++ } END { print count + 0 }' "${OUT_DIR}/abba_matrix.csv")"
|
||||||
|
if [[ "$matrix_rows" != "4" ]]; then
|
||||||
|
echo "expected 4 ABBA matrix rows, got ${matrix_rows}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
for outer_size in 65536 1048576; do
|
||||||
|
for profile_order in normal reverse; do
|
||||||
|
cell_dir="${OUT_DIR}/${profile_order}-outer-${outer_size}"
|
||||||
|
rg -qx "profile_order=${profile_order}" "${cell_dir}/manifest.env"
|
||||||
|
rg -qx 'sizes=1MiB' "${cell_dir}/manifest.env"
|
||||||
|
rg -qx 'mode=both' "${cell_dir}/manifest.env"
|
||||||
|
rg -qx 'codec_engines=legacy' "${cell_dir}/manifest.env"
|
||||||
|
rg -qx 'codec_min_size=1048576' "${cell_dir}/manifest.env"
|
||||||
|
rg -qx 'output_handoff_attribution=true' "${cell_dir}/manifest.env"
|
||||||
|
rg -qx 'diagnostic_metrics_enabled=true' "${cell_dir}/manifest.env"
|
||||||
|
rg -qx 'dry_run=true' "${cell_dir}/manifest.env"
|
||||||
|
|
||||||
|
if [[ "$profile_order" == "normal" ]]; then
|
||||||
|
rg -qx 'profiles=legacy,codec-legacy' "${cell_dir}/manifest.env"
|
||||||
|
rg -q "^normal,${outer_size},.*,legacy;codec-legacy,legacy>codec-legacy,.*,ok$" "${OUT_DIR}/abba_matrix.csv"
|
||||||
|
else
|
||||||
|
rg -qx 'profiles=codec-legacy,legacy' "${cell_dir}/manifest.env"
|
||||||
|
rg -q "^reverse,${outer_size},.*,legacy;codec-legacy,codec-legacy>legacy,.*,ok$" "${OUT_DIR}/abba_matrix.csv"
|
||||||
|
fi
|
||||||
|
|
||||||
|
for profile in legacy codec-legacy; do
|
||||||
|
profile_manifest="${cell_dir}/${profile}/manifest.env"
|
||||||
|
rg -qx 'sizes=1MiB' "$profile_manifest"
|
||||||
|
rg -qx 'RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE=true' "$profile_manifest"
|
||||||
|
rg -qx 'RUSTFS_OBS_METRICS_EXPORT_ENABLED=true' "$profile_manifest"
|
||||||
|
rg -qx 'RUSTFS_GET_CODEC_STREAMING_MIN_SIZE=1048576' "$profile_manifest"
|
||||||
|
test -f "${cell_dir}/${profile}/service_metrics_round_summary.csv"
|
||||||
|
test -f "${cell_dir}/${profile}/service_metrics_stage_distribution.csv"
|
||||||
|
test -f "${cell_dir}/${profile}/service_metrics_round_percentiles.csv"
|
||||||
|
done
|
||||||
|
done
|
||||||
|
done
|
||||||
|
|
||||||
|
if "$RUNNER" --outer-reader-stream-buffer-sizes 0 --dry-run >/dev/null 2>&1; then
|
||||||
|
echo "expected zero outer ReaderStream buffer to be rejected" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if "$RUNNER" --profile-orders normal,sideways --dry-run >/dev/null 2>&1; then
|
||||||
|
echo "expected invalid profile order to be rejected" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
Reference in New Issue
Block a user