From 0eb9dd5bdcb39c230c5019fdbe496c9180ae7647 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 26 Jul 2026 11:51:35 +0800 Subject: [PATCH] test(perf): add pinned paired ABBA benchmark (#5259) test(perf): add pinned paired abba bench Add a backlog#1432 paired benchmark orchestrator that runs MinIO/RustFS in an A1/B1/B2/A2 schedule while requiring immutable server identity, source revision or release, and ACK contract labels. The runner delegates actual workload execution to run_object_batch_bench_enhanced.sh so node metrics, resource captures, and provenance stay in the shared artifact format. Verification: - scripts/test_pinned_paired_abba_bench.sh - bash -n scripts/run_pinned_paired_abba_bench.sh scripts/test_pinned_paired_abba_bench.sh - git diff --check Co-authored-by: heihutu --- .config/make/tests.mak | 1 + scripts/README.md | 1 + scripts/run_pinned_paired_abba_bench.sh | 336 +++++++++++++++++++++++ scripts/test_pinned_paired_abba_bench.sh | 82 ++++++ 4 files changed, 420 insertions(+) create mode 100755 scripts/run_pinned_paired_abba_bench.sh create mode 100755 scripts/test_pinned_paired_abba_bench.sh diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 9825d505b..0eadd0f2c 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -28,6 +28,7 @@ script-tests: ## Run shell script tests ./scripts/test_entrypoint_credentials.sh ./scripts/test_internode_grpc_ab_bench.sh ./scripts/test_object_batch_bench_enhanced.sh + ./scripts/test_pinned_paired_abba_bench.sh bash -n ./scripts/validate_object_data_cache_cold_stampede.sh python3 ./scripts/check_object_data_cache_follower_samples.py --self-test ./scripts/validate_object_data_cache_cold_stampede.sh --self-test diff --git a/scripts/README.md b/scripts/README.md index e59ebd4e6..262b40aba 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -68,6 +68,7 @@ their issue closes. | `run_four_node_cluster_failover_bench.sh` | dev-tool | Four-node cluster failover benchmark | `docker/compose/README.md`; internode runbook | | `run_object_batch_bench.sh` | dev-tool | Batch object benchmark runner (warp/s3bench) | internode + scanner runbooks | | `run_object_batch_bench_enhanced.sh` | dev-tool | Enhanced batch benchmark runner; hub used by the smoke rigs | hotpath runbook | +| `run_pinned_paired_abba_bench.sh` | dev-tool | Pinned RustFS/MinIO paired ABBA benchmark orchestrator for backlog#1432 | `test_pinned_paired_abba_bench.sh` | | `run_get_codec_streaming_smoke.sh` | dev-tool | Local GET benchmark harness for the codec streaming read path | `docs/testing/ecstore-validation-suite-design.md` | | `run_gt1g_get_http_matrix.sh` | dev-tool | >1 GiB GET HTTP matrix | `docs/testing/ecstore-validation-suite-design.md` | | `run_gt1g_multipart_put_matrix.sh` | dev-tool | >1 GiB multipart PUT matrix | `docs/testing/ecstore-validation-suite-design.md` | diff --git a/scripts/run_pinned_paired_abba_bench.sh b/scripts/run_pinned_paired_abba_bench.sh new file mode 100755 index 000000000..72181a7f5 --- /dev/null +++ b/scripts/run_pinned_paired_abba_bench.sh @@ -0,0 +1,336 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +ENHANCED_BENCH="${PROJECT_ROOT}/scripts/run_object_batch_bench_enhanced.sh" + +OUT_DIR="" +ACCESS_KEY="" +SECRET_KEY="" +REGION="us-east-1" +TOOL="warp" +WARP_BIN="warp" +WARP_MODE="get" +DURATION="60s" +CONCURRENCIES="1,64" +SIZES="0B,16383B,16384B,16385B,32767B,32768B,32769B,131071B,131072B,131073B,262143B,262144B,262145B,1048575B,1048576B,1048577B,4194304B" +ROUNDS_PER_LEG=5 +COOLDOWN_SECS=20 +BUCKET_PREFIX="rustfs-1432-paired" +DRY_RUN=false + +RUSTFS_ENDPOINT="" +RUSTFS_IMAGE_REF="" +RUSTFS_IMAGE_DIGEST="" +RUSTFS_REVISION="" +RUSTFS_ACK_CONTRACT="" + +MINIO_ENDPOINT="" +MINIO_IMAGE_REF="" +MINIO_IMAGE_DIGEST="" +MINIO_RELEASE="" +MINIO_ACK_CONTRACT="" + +COMMON_LABELS=() +RUSTFS_NODE_ARGS=() +MINIO_NODE_ARGS=() + +usage() { + cat <<'USAGE' +Usage: + scripts/run_pinned_paired_abba_bench.sh --access-key --secret-key \ + --rustfs-endpoint --rustfs-image-ref --rustfs-image-digest --rustfs-revision --rustfs-ack-contract \ + --minio-endpoint --minio-image-ref --minio-image-digest --minio-release --minio-ack-contract [options] + +Options: + --out-dir Default: target/bench/pinned-paired-abba- + --tool Default: warp + --warp-bin Default: warp + --warp-mode Default: get + --duration Default: 60s + --concurrencies Default: 1,64 + --sizes Default: #1432 boundary matrix + --rounds-per-leg Default: 5 + --cooldown-secs Default: 20 + --bucket-prefix Default: rustfs-1432-paired + --label Repeatable common label passed to every leg + --rustfs-node-metrics-url Repeatable RustFS node metrics URL + --rustfs-node-docker-container + --minio-node-metrics-url Repeatable MinIO node metrics URL + --minio-node-docker-container + --dry-run Materialize manifest and commands only + -h, --help + +This runner only orchestrates comparable paired evidence. It does not claim performance causality by itself; every emitted leg delegates to run_object_batch_bench_enhanced.sh with --require-server-provenance. +USAGE +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +validate_nonempty() { + local value="$1" + local name="$2" + [[ -n "$value" ]] || die "$name is required" +} + +validate_csv_nonempty() { + local value="$1" + local name="$2" + [[ -n "$value" && "$value" != *",,"* ]] || die "$name must be a non-empty CSV" +} + +validate_positive_int() { + local value="$1" + local name="$2" + [[ "$value" =~ ^[0-9]+$ && "$value" -gt 0 ]] || die "$name must be a positive integer, got: $value" +} + +validate_nonnegative_int() { + local value="$1" + local name="$2" + [[ "$value" =~ ^[0-9]+$ ]] || die "$name must be a nonnegative integer, got: $value" +} + +validate_digest() { + local value="$1" + local name="$2" + [[ "$value" =~ ^sha256:[0-9a-fA-F]{64}$ ]] || die "$name must be an immutable sha256 digest" +} + +validate_ack_contract() { + local value="$1" + local name="$2" + case "$value" in + strict|relaxed) ;; + *) die "$name must be strict or relaxed" ;; + esac +} + +validate_label() { + local value="$1" + local option="$2" + [[ "$value" =~ ^[A-Za-z_][A-Za-z0-9_]*=.+$ && "$value" != *$'\n'* && "$value" != *,* ]] || die "$option must be key=value without commas/newlines and with a shell-safe key" +} + +append_node_arg() { + local target="$1" + local option="$2" + local value="$3" + validate_label "$value" "$option" + if [[ "$target" == "rustfs" ]]; then + RUSTFS_NODE_ARGS+=("$option" "$value") + else + MINIO_NODE_ARGS+=("$option" "$value") + fi +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --out-dir) OUT_DIR="$2"; shift 2 ;; + --access-key) ACCESS_KEY="$2"; shift 2 ;; + --secret-key) SECRET_KEY="$2"; shift 2 ;; + --region) REGION="$2"; shift 2 ;; + --tool) TOOL="$2"; shift 2 ;; + --warp-bin) WARP_BIN="$2"; shift 2 ;; + --warp-mode) WARP_MODE="$2"; shift 2 ;; + --duration) DURATION="$2"; shift 2 ;; + --concurrencies) CONCURRENCIES="$2"; shift 2 ;; + --sizes) SIZES="$2"; shift 2 ;; + --rounds-per-leg) ROUNDS_PER_LEG="$2"; shift 2 ;; + --cooldown-secs) COOLDOWN_SECS="$2"; shift 2 ;; + --bucket-prefix) BUCKET_PREFIX="$2"; shift 2 ;; + --label) validate_label "$2" "--label"; COMMON_LABELS+=("--label" "$2"); shift 2 ;; + --rustfs-endpoint) RUSTFS_ENDPOINT="$2"; shift 2 ;; + --rustfs-image-ref) RUSTFS_IMAGE_REF="$2"; shift 2 ;; + --rustfs-image-digest) RUSTFS_IMAGE_DIGEST="$2"; shift 2 ;; + --rustfs-revision) RUSTFS_REVISION="$2"; shift 2 ;; + --rustfs-ack-contract) RUSTFS_ACK_CONTRACT="$2"; shift 2 ;; + --rustfs-node-metrics-url) append_node_arg rustfs "--node-metrics-url" "$2"; shift 2 ;; + --rustfs-node-docker-container) append_node_arg rustfs "--node-docker-container" "$2"; shift 2 ;; + --minio-endpoint) MINIO_ENDPOINT="$2"; shift 2 ;; + --minio-image-ref) MINIO_IMAGE_REF="$2"; shift 2 ;; + --minio-image-digest) MINIO_IMAGE_DIGEST="$2"; shift 2 ;; + --minio-release) MINIO_RELEASE="$2"; shift 2 ;; + --minio-ack-contract) MINIO_ACK_CONTRACT="$2"; shift 2 ;; + --minio-node-metrics-url) append_node_arg minio "--node-metrics-url" "$2"; shift 2 ;; + --minio-node-docker-container) append_node_arg minio "--node-docker-container" "$2"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage; exit 0 ;; + *) die "unknown arg: $1" ;; + esac + done +} + +validate_args() { + validate_nonempty "$ACCESS_KEY" "--access-key" + validate_nonempty "$SECRET_KEY" "--secret-key" + validate_nonempty "$RUSTFS_ENDPOINT" "--rustfs-endpoint" + validate_nonempty "$RUSTFS_IMAGE_REF" "--rustfs-image-ref" + validate_nonempty "$RUSTFS_IMAGE_DIGEST" "--rustfs-image-digest" + validate_nonempty "$RUSTFS_REVISION" "--rustfs-revision" + validate_nonempty "$RUSTFS_ACK_CONTRACT" "--rustfs-ack-contract" + validate_nonempty "$MINIO_ENDPOINT" "--minio-endpoint" + validate_nonempty "$MINIO_IMAGE_REF" "--minio-image-ref" + validate_nonempty "$MINIO_IMAGE_DIGEST" "--minio-image-digest" + validate_nonempty "$MINIO_RELEASE" "--minio-release" + validate_nonempty "$MINIO_ACK_CONTRACT" "--minio-ack-contract" + validate_digest "$RUSTFS_IMAGE_DIGEST" "--rustfs-image-digest" + validate_digest "$MINIO_IMAGE_DIGEST" "--minio-image-digest" + validate_ack_contract "$RUSTFS_ACK_CONTRACT" "--rustfs-ack-contract" + validate_ack_contract "$MINIO_ACK_CONTRACT" "--minio-ack-contract" + validate_csv_nonempty "$CONCURRENCIES" "--concurrencies" + validate_csv_nonempty "$SIZES" "--sizes" + validate_positive_int "$ROUNDS_PER_LEG" "--rounds-per-leg" + validate_nonnegative_int "$COOLDOWN_SECS" "--cooldown-secs" + [[ "$TOOL" == "warp" || "$TOOL" == "s3bench" ]] || die "--tool must be warp or s3bench" + [[ "$WARP_MODE" == "get" || "$WARP_MODE" == "put" || "$WARP_MODE" == "mixed" ]] || die "--warp-mode must be get, put, or mixed" +} + +setup_output() { + if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="${PROJECT_ROOT}/target/bench/pinned-paired-abba-$(date -u +%Y%m%dT%H%M%SZ)" + fi + mkdir -p "$OUT_DIR" +} + +write_manifest() { + local manifest="$OUT_DIR/paired_manifest.env" + { + printf 'created_utc=%q\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'benchmark_issue=%q\n' "rustfs/backlog#1432" + printf 'schedule=%q\n' "ABBA" + printf 'tool=%q\n' "$TOOL" + printf 'warp_mode=%q\n' "$WARP_MODE" + printf 'duration=%q\n' "$DURATION" + printf 'concurrencies=%q\n' "$CONCURRENCIES" + printf 'sizes=%q\n' "$SIZES" + printf 'rounds_per_leg=%q\n' "$ROUNDS_PER_LEG" + printf 'cooldown_secs=%q\n' "$COOLDOWN_SECS" + printf 'rustfs_endpoint=%q\n' "$RUSTFS_ENDPOINT" + printf 'rustfs_image_ref=%q\n' "$RUSTFS_IMAGE_REF" + printf 'rustfs_image_digest=%q\n' "$RUSTFS_IMAGE_DIGEST" + printf 'rustfs_revision=%q\n' "$RUSTFS_REVISION" + printf 'rustfs_ack_contract=%q\n' "$RUSTFS_ACK_CONTRACT" + printf 'minio_endpoint=%q\n' "$MINIO_ENDPOINT" + printf 'minio_image_ref=%q\n' "$MINIO_IMAGE_REF" + printf 'minio_image_digest=%q\n' "$MINIO_IMAGE_DIGEST" + printf 'minio_release=%q\n' "$MINIO_RELEASE" + printf 'minio_ack_contract=%q\n' "$MINIO_ACK_CONTRACT" + printf 'access_key=%q\n' "REDACTED" + printf 'secret_key=%q\n' "REDACTED" + } > "$manifest" +} + +write_schedule() { + local schedule="$OUT_DIR/abba_schedule.csv" + echo "leg,product,endpoint,server_image_ref,server_image_digest,server_revision_or_release,ack_contract,leg_out_dir" > "$schedule" + echo "A1,minio,$MINIO_ENDPOINT,$MINIO_IMAGE_REF,$MINIO_IMAGE_DIGEST,$MINIO_RELEASE,$MINIO_ACK_CONTRACT,$OUT_DIR/A1-minio" >> "$schedule" + echo "B1,rustfs,$RUSTFS_ENDPOINT,$RUSTFS_IMAGE_REF,$RUSTFS_IMAGE_DIGEST,$RUSTFS_REVISION,$RUSTFS_ACK_CONTRACT,$OUT_DIR/B1-rustfs" >> "$schedule" + echo "B2,rustfs,$RUSTFS_ENDPOINT,$RUSTFS_IMAGE_REF,$RUSTFS_IMAGE_DIGEST,$RUSTFS_REVISION,$RUSTFS_ACK_CONTRACT,$OUT_DIR/B2-rustfs" >> "$schedule" + echo "A2,minio,$MINIO_ENDPOINT,$MINIO_IMAGE_REF,$MINIO_IMAGE_DIGEST,$MINIO_RELEASE,$MINIO_ACK_CONTRACT,$OUT_DIR/A2-minio" >> "$schedule" +} + +RUNNER_CMD=() + +build_runner_cmd_for_product() { + local product="$1" + local concurrency="$2" + local bucket="$3" + local leg_out_dir="$4" + local endpoint image_ref image_digest revision_or_release ack_contract + local -a node_args + if [[ "$product" == "rustfs" ]]; then + endpoint="$RUSTFS_ENDPOINT" + image_ref="$RUSTFS_IMAGE_REF" + image_digest="$RUSTFS_IMAGE_DIGEST" + revision_or_release="$RUSTFS_REVISION" + ack_contract="$RUSTFS_ACK_CONTRACT" + node_args=("${RUSTFS_NODE_ARGS[@]}") + else + endpoint="$MINIO_ENDPOINT" + image_ref="$MINIO_IMAGE_REF" + image_digest="$MINIO_IMAGE_DIGEST" + revision_or_release="$MINIO_RELEASE" + ack_contract="$MINIO_ACK_CONTRACT" + node_args=("${MINIO_NODE_ARGS[@]}") + fi + + RUNNER_CMD=( + "$ENHANCED_BENCH" \ + --tool "$TOOL" \ + --endpoint "$endpoint" \ + --access-key "$ACCESS_KEY" \ + --secret-key "$SECRET_KEY" \ + --region "$REGION" \ + --bucket-size-suffix \ + --concurrency "$concurrency" \ + --sizes "$SIZES" \ + --rounds "$ROUNDS_PER_LEG" \ + --cooldown-secs "$COOLDOWN_SECS" \ + --server-image-ref "$image_ref" \ + --server-image-digest "$image_digest" \ + --server-revision "$revision_or_release" \ + --require-server-provenance \ + --label "bench_issue=1432" \ + --label "product=$product" \ + --label "ack_contract=$ack_contract" \ + --label "schedule=ABBA" \ + --warp-bin "$WARP_BIN" \ + --warp-mode "$WARP_MODE" \ + --duration "$DURATION" \ + --bucket "$bucket" \ + --out-dir "$leg_out_dir" + ) + RUNNER_CMD+=("${COMMON_LABELS[@]}" "${node_args[@]}") + if [[ "$DRY_RUN" == "true" ]]; then + RUNNER_CMD+=(--dry-run) + fi +} + +quote_cmd() { + printf '%q ' "$@" +} + +run_leg() { + local leg="$1" + local product="$2" + local leg_out_dir="$3" + local conc="$4" + local leg_lower + leg_lower="$(printf '%s' "$leg" | tr '[:upper:]' '[:lower:]')" + local bucket="${BUCKET_PREFIX}-${leg_lower}-c${conc}" + build_runner_cmd_for_product "$product" "$conc" "$bucket" "$leg_out_dir/c${conc}" + + if [[ "$DRY_RUN" == "true" ]]; then + echo "-- dry-run leg=$leg product=$product concurrency=$conc" + echo " $(quote_cmd "${RUNNER_CMD[@]}")" + else + "${RUNNER_CMD[@]}" + fi +} + +run_schedule() { + local conc leg product leg_out_dir + IFS=',' read -r -a conc_values <<< "$CONCURRENCIES" + for conc in "${conc_values[@]}"; do + validate_positive_int "$conc" "concurrency cell" + while IFS=',' read -r leg product _ _ _ _ _ leg_out_dir; do + [[ "$leg" != "leg" ]] || continue + run_leg "$leg" "$product" "$leg_out_dir" "$conc" + done < "$OUT_DIR/abba_schedule.csv" + done +} + +parse_args "$@" +validate_args +setup_output +write_manifest +write_schedule +run_schedule + +echo "paired ABBA artifact root: $OUT_DIR" diff --git a/scripts/test_pinned_paired_abba_bench.sh b/scripts/test_pinned_paired_abba_bench.sh new file mode 100755 index 000000000..f462afeb0 --- /dev/null +++ b/scripts/test_pinned_paired_abba_bench.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="${SCRIPT_DIR}/run_pinned_paired_abba_bench.sh" +TMP_DIR="$(mktemp -d)" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +if "$RUNNER" --dry-run >/dev/null 2>&1; then + echo "expected missing pinned identities to fail" >&2 + exit 1 +fi + +if "$RUNNER" \ + --access-key minioadmin \ + --secret-key minioadmin \ + --rustfs-endpoint http://127.0.0.1:9000 \ + --rustfs-image-ref rustfs/rustfs:bench \ + --rustfs-image-digest latest \ + --rustfs-revision 42fc84063 \ + --rustfs-ack-contract strict \ + --minio-endpoint http://127.0.0.1:9100 \ + --minio-image-ref quay.io/minio/minio:RELEASE.2025-07-18T21-56-31Z \ + --minio-image-digest sha256:1111222233334444 \ + --minio-release RELEASE.2025-07-18T21-56-31Z \ + --minio-ack-contract strict \ + --out-dir "$TMP_DIR/bad-digest" \ + --dry-run >/dev/null 2>&1; then + echo "expected mutable rustfs digest to fail" >&2 + exit 1 +fi + +"$RUNNER" \ + --access-key minioadmin \ + --secret-key minioadmin \ + --rustfs-endpoint http://127.0.0.1:9000 \ + --rustfs-image-ref rustfs/rustfs:bench \ + --rustfs-image-digest sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef \ + --rustfs-revision 42fc84063 \ + --rustfs-ack-contract relaxed \ + --minio-endpoint http://127.0.0.1:9100 \ + --minio-image-ref quay.io/minio/minio:RELEASE.2025-07-18T21-56-31Z \ + --minio-image-digest sha256:1111222233334444111122223333444411112222333344441111222233334444 \ + --minio-release RELEASE.2025-07-18T21-56-31Z \ + --minio-ack-contract strict \ + --concurrencies 1,64 \ + --sizes 1048575B,1048576B,1048577B \ + --rounds-per-leg 5 \ + --cooldown-secs 0 \ + --label topology=four_node_16_disk \ + --rustfs-node-metrics-url node1=http://127.0.0.1:9001/metrics \ + --minio-node-metrics-url node1=http://127.0.0.1:9101/minio/v2/metrics/cluster \ + --out-dir "$TMP_DIR/good" \ + --dry-run >"$TMP_DIR/dry-run.log" + +MANIFEST="$TMP_DIR/good/paired_manifest.env" +SCHEDULE="$TMP_DIR/good/abba_schedule.csv" + +rg -qx "schedule=ABBA" "$MANIFEST" +rg -qx "rustfs_ack_contract=relaxed" "$MANIFEST" +rg -qx "minio_ack_contract=strict" "$MANIFEST" +rg -qx "access_key=REDACTED" "$MANIFEST" +rg -qx "secret_key=REDACTED" "$MANIFEST" +rg -qx "leg,product,endpoint,server_image_ref,server_image_digest,server_revision_or_release,ack_contract,leg_out_dir" "$SCHEDULE" +rg -qx "A1,minio,http://127.0.0.1:9100,quay.io/minio/minio:RELEASE.2025-07-18T21-56-31Z,sha256:1111222233334444111122223333444411112222333344441111222233334444,RELEASE.2025-07-18T21-56-31Z,strict,$TMP_DIR/good/A1-minio" "$SCHEDULE" +rg -qx "B1,rustfs,http://127.0.0.1:9000,rustfs/rustfs:bench,sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef,42fc84063,relaxed,$TMP_DIR/good/B1-rustfs" "$SCHEDULE" +rg -qx "B2,rustfs,http://127.0.0.1:9000,rustfs/rustfs:bench,sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef,42fc84063,relaxed,$TMP_DIR/good/B2-rustfs" "$SCHEDULE" +rg -qx "A2,minio,http://127.0.0.1:9100,quay.io/minio/minio:RELEASE.2025-07-18T21-56-31Z,sha256:1111222233334444111122223333444411112222333344441111222233334444,RELEASE.2025-07-18T21-56-31Z,strict,$TMP_DIR/good/A2-minio" "$SCHEDULE" + +rg -q -- "--server-image-ref rustfs/rustfs:bench" "$TMP_DIR/dry-run.log" +rg -q -- "--server-image-ref quay.io/minio/minio:RELEASE.2025-07-18T21-56-31Z" "$TMP_DIR/dry-run.log" +rg -q -- "--require-server-provenance" "$TMP_DIR/dry-run.log" +rg -q -- "--label bench_issue=1432" "$TMP_DIR/dry-run.log" +rg -q -- "--label topology=four_node_16_disk" "$TMP_DIR/dry-run.log" +rg -q -- "--node-metrics-url node1=http://127.0.0.1:9001/metrics" "$TMP_DIR/dry-run.log" +rg -q -- "--node-metrics-url node1=http://127.0.0.1:9101/minio/v2/metrics/cluster" "$TMP_DIR/dry-run.log" + +echo "pinned paired ABBA bench tests passed"