From 14f31b797af9fe43724defecfb48a0558afd0123 Mon Sep 17 00:00:00 2001 From: houseme Date: Thu, 23 Jul 2026 23:56:48 +0800 Subject: [PATCH] feat(bench): capture node attribution metadata (#5158) Co-authored-by: heihutu --- .config/make/tests.mak | 1 + scripts/run_object_batch_bench_enhanced.sh | 289 ++++++++++++++++++-- scripts/test_object_batch_bench_enhanced.sh | 71 +++++ 3 files changed, 340 insertions(+), 21 deletions(-) create mode 100755 scripts/test_object_batch_bench_enhanced.sh diff --git a/.config/make/tests.mak b/.config/make/tests.mak index c83dcd653..73ff9aaeb 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -26,6 +26,7 @@ script-tests: ## Run shell script tests @echo "Running script tests..." ./scripts/test_build_rustfs_options.sh ./scripts/test_entrypoint_credentials.sh + ./scripts/test_object_batch_bench_enhanced.sh bash -n ./scripts/validate_object_data_cache_cold_stampede.sh python3 ./scripts/check_object_data_cache_follower_samples.py --self-test ./scripts/validate_object_data_cache_cold_stampede.sh --self-test diff --git a/scripts/run_object_batch_bench_enhanced.sh b/scripts/run_object_batch_bench_enhanced.sh index e8a774fc1..a5404b52e 100755 --- a/scripts/run_object_batch_bench_enhanced.sh +++ b/scripts/run_object_batch_bench_enhanced.sh @@ -53,6 +53,13 @@ SERVICE_PROMETHEUS_QUERY_URL="" SERVICE_PROMETHEUS_QUERY="" SERVICE_PROMETHEUS_QUERY_EXPLICIT=false SERVICE_METRICS_SERVICE_NAME="" +SERVER_IMAGE_REF="" +SERVER_IMAGE_DIGEST="" +SERVER_REVISION="" +REQUIRE_SERVER_PROVENANCE=false +RUN_LABELS=() +NODE_METRICS_URLS=() +NODE_DOCKER_CONTAINERS=() usage() { cat <<'USAGE' @@ -114,12 +121,26 @@ Enhanced options: --service-metrics-filter-regex Regex for retained metrics lines after scrape (default: auto by tool/mode) + --server-image-ref Image reference of the benchmarked server + --server-image-digest Immutable image digest of the benchmarked server + --server-revision Source revision built into the benchmarked server + --require-server-provenance Fail unless the three server identity fields are set + --label Repeatable run label written to run_manifest.env + --node-metrics-url + Repeatable node-specific Prometheus scrape URL captured + before/after each round attempt + --node-docker-container + Repeatable Docker container mapped to a node; captures + CPU, memory, network, and block I/O before/after each round Output files: round_results.csv One row per round attempt (with retry trace) median_summary.csv Median metrics per object size baseline_compare.csv Delta vs baseline (if --baseline-csv is set) /*.prom Optional per-round service metric snapshots + node_inventory.csv Per-node container and immutable image identity + node_metrics_captures.csv Per-node metric snapshot inventory + node_resource_captures.csv Per-node Docker resource and block I/O snapshots Example: scripts/run_object_batch_bench_enhanced.sh \ @@ -185,6 +206,13 @@ parse_args() { --service-metrics-max-time-secs) SERVICE_METRICS_MAX_TIME_SECS="$2"; shift 2 ;; --service-metrics-settle-secs) SERVICE_METRICS_SETTLE_SECS="$2"; shift 2 ;; --service-metrics-filter-regex) SERVICE_METRICS_FILTER_REGEX="$2"; SERVICE_METRICS_FILTER_REGEX_EXPLICIT=true; shift 2 ;; + --server-image-ref) SERVER_IMAGE_REF="$2"; shift 2 ;; + --server-image-digest) SERVER_IMAGE_DIGEST="$2"; shift 2 ;; + --server-revision) SERVER_REVISION="$2"; shift 2 ;; + --require-server-provenance) REQUIRE_SERVER_PROVENANCE=true; shift ;; + --label) RUN_LABELS+=("$2"); shift 2 ;; + --node-metrics-url) NODE_METRICS_URLS+=("$2"); shift 2 ;; + --node-docker-container) NODE_DOCKER_CONTAINERS+=("$2"); shift 2 ;; --extra-args) # shellcheck disable=SC2206 EXTRA_ARGS=($2) @@ -218,6 +246,53 @@ validate_nonnegative_int() { fi } +validate_named_value() { + local value="$1" + local option_name="$2" + if ! [[ "$value" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*=.+$ ]] || [[ "$value" == *$'\n'* || "$value" == *,* ]]; then + echo "ERROR: $option_name must be name=value without commas or newlines, got: $value" >&2 + exit 1 + fi +} + +validate_named_values() { + local option_name="$1" + shift + local entry name seen_names="" + for entry in "$@"; do + validate_named_value "$entry" "$option_name" + name="${entry%%=*}" + if [[ " $seen_names " == *" $name "* ]]; then + echo "ERROR: $option_name repeats name: $name" >&2 + exit 1 + fi + seen_names="${seen_names} ${name}" + done +} + +validate_manifest_labels() { + local entry key + if ((${#RUN_LABELS[@]} == 0)); then + return + fi + for entry in "${RUN_LABELS[@]}"; do + key="${entry%%=*}" + if ! [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + echo "ERROR: --label name must be a shell-compatible manifest key, got: $key" >&2 + exit 1 + fi + done +} + +validate_manifest_value() { + local value="$1" + local option_name="$2" + if [[ "$value" == *$'\n'* || "$value" == *,* ]]; then + echo "ERROR: $option_name must not contain commas or newlines" >&2 + exit 1 + fi +} + cooldown_sleep() { local secs="$1" if (( secs <= 0 )); then @@ -295,16 +370,19 @@ validate_args() { echo "ERROR: --service-metrics-url and --service-prometheus-query-url are mutually exclusive" >&2 exit 1 fi - if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]] && [[ -z "$SERVICE_METRICS_DIR" ]]; then + if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" || ${#NODE_METRICS_URLS[@]} -gt 0 ]] && [[ -z "$SERVICE_METRICS_DIR" ]]; then echo "ERROR: --service-metrics-dir is required when service metrics capture is enabled" >&2 exit 1 fi - if [[ -n "$SERVICE_METRICS_URL" ]]; then + if [[ -n "$SERVICE_METRICS_URL" || ${#NODE_METRICS_URLS[@]} -gt 0 ]]; then require_cmd curl fi if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then require_cmd "$PYTHON_BIN" fi + if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)) && [[ "$DRY_RUN" != "true" ]]; then + require_cmd docker + fi if [[ "$TOOL" == "s3bench" ]]; then validate_positive_int "$SAMPLES" "--samples" fi @@ -312,6 +390,23 @@ validate_args() { echo "ERROR: --baseline-csv does not exist: $BASELINE_CSV" >&2 exit 1 fi + if ((${#RUN_LABELS[@]} > 0)); then + validate_named_values "--label" "${RUN_LABELS[@]}" + fi + validate_manifest_labels + if ((${#NODE_METRICS_URLS[@]} > 0)); then + validate_named_values "--node-metrics-url" "${NODE_METRICS_URLS[@]}" + fi + if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)); then + validate_named_values "--node-docker-container" "${NODE_DOCKER_CONTAINERS[@]}" + fi + validate_manifest_value "$SERVER_IMAGE_REF" "--server-image-ref" + validate_manifest_value "$SERVER_IMAGE_DIGEST" "--server-image-digest" + validate_manifest_value "$SERVER_REVISION" "--server-revision" + if [[ "$REQUIRE_SERVER_PROVENANCE" == "true" ]] && { [[ -z "$SERVER_IMAGE_REF" ]] || [[ -z "$SERVER_IMAGE_DIGEST" ]] || [[ -z "$SERVER_REVISION" ]]; }; then + echo "ERROR: --require-server-provenance needs --server-image-ref, --server-image-digest, and --server-revision" >&2 + exit 1 + fi if [[ "$TOOL" == "warp" ]]; then if [[ "$WARP_MODE" != "get" && "$WARP_MODE" != "put" && "$WARP_MODE" != "mixed" ]]; then echo "ERROR: --warp-mode must be get, put, or mixed" >&2 @@ -354,6 +449,10 @@ git_dirty_state() { echo "clean" } +write_manifest_entry() { + printf '%s=%q\n' "$1" "$2" +} + write_run_manifest() { local manifest_file="$OUT_DIR/run_manifest.env" local started_at_utc git_commit git_branch git_dirty rustc_version uname_s service_metrics_csv @@ -367,9 +466,13 @@ write_run_manifest() { cat >"$manifest_file" < 0)); then + local label + for label in "${RUN_LABELS[@]}"; do + write_manifest_entry "run_label_${label%%=*}" "${label#*=}" + done + fi + echo "node_metrics_count=${#NODE_METRICS_URLS[@]}" + echo "node_docker_container_count=${#NODE_DOCKER_CONTAINERS[@]}" + } >>"$manifest_file" +} + +write_node_inventory() { + local inventory_csv="$OUT_DIR/node_inventory.csv" + local entry node container container_id image_id image_ref image_digest status + if ((${#NODE_DOCKER_CONTAINERS[@]} == 0)); then + return + fi + + echo "node,container,status,container_id,image_id,image_ref,image_digest" > "$inventory_csv" + for entry in "${NODE_DOCKER_CONTAINERS[@]}"; do + node="${entry%%=*}" + container="${entry#*=}" + if [[ "$DRY_RUN" == "true" ]]; then + echo "$node,$container,not_run_dry_run,N/A,N/A,N/A,N/A" >> "$inventory_csv" + continue + fi + if ! container_id="$(docker inspect --format '{{.Id}}' "$container" 2>/dev/null)" || ! image_id="$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null)" || ! image_ref="$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null)"; then + echo "$node,$container,capture_failed,N/A,N/A,N/A,N/A" >> "$inventory_csv" + echo "WARN: failed to inspect Docker node=${node} container=${container}" >&2 + continue + fi + image_digest="$(docker image inspect --format '{{join .RepoDigests ";"}}' "$image_id" 2>/dev/null || echo N/A)" + status=ok + echo "$node,$container,$status,$container_id,$image_id,$image_ref,${image_digest:-N/A}" >> "$inventory_csv" + done } setup_output() { @@ -414,21 +558,33 @@ setup_output() { OUT_DIR="target/bench/object-batch-enhanced-$(date +%Y%m%d-%H%M%S)" fi mkdir -p "$OUT_DIR/logs" - if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then + if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" || ${#NODE_METRICS_URLS[@]} -gt 0 ]]; then mkdir -p "$SERVICE_METRICS_DIR" fi + if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)); then + mkdir -p "$OUT_DIR/node_resources" + fi ROUND_CSV="$OUT_DIR/round_results.csv" MEDIAN_CSV="$OUT_DIR/median_summary.csv" COMPARE_CSV="$OUT_DIR/baseline_compare.csv" SERVICE_METRICS_CSV="$OUT_DIR/service_metrics_captures.csv" + NODE_METRICS_CSV="$OUT_DIR/node_metrics_captures.csv" + NODE_RESOURCE_CSV="$OUT_DIR/node_resource_captures.csv" echo "size,tool,round,attempt,concurrency,status,exit_code,round_started_at_utc,round_finished_at_utc,throughput_human,throughput_bps,reqps,latency_human,latency_ms,log_file,req_p90_human,req_p90_ms,req_p99_human,req_p99_ms" > "$ROUND_CSV" echo "size,tool,concurrency,successful_rounds,failed_rounds,median_throughput_bps,median_reqps,median_latency_ms,median_req_p90_ms,median_req_p99_ms" > "$MEDIAN_CSV" if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then echo "size,tool,round,attempt,phase,source,status,capture_attempt,raw_bytes,snapshot_bytes,status_file,snapshot_file,filter_regex,prometheus_query" > "$SERVICE_METRICS_CSV" fi + if ((${#NODE_METRICS_URLS[@]} > 0)); then + echo "size,tool,round,attempt,phase,node,status,capture_attempt,raw_bytes,snapshot_bytes,status_file,snapshot_file,filter_regex" > "$NODE_METRICS_CSV" + fi + if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)); then + echo "size,tool,round,attempt,phase,node,container,status,snapshot_file" > "$NODE_RESOURCE_CSV" + fi write_run_manifest + write_node_inventory } trim() { @@ -759,24 +915,65 @@ EOF return 0 fi - for ((capture_attempt=1; capture_attempt<=SERVICE_METRICS_CAPTURE_ATTEMPTS; capture_attempt++)); do - if curl -fsS --noproxy '*' \ - --connect-timeout "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS" \ - --max-time "$SERVICE_METRICS_MAX_TIME_SECS" \ - "$SERVICE_METRICS_URL" >"$tmp_file"; then - if [[ -s "$tmp_file" ]]; then - raw_bytes="$(wc -c <"$tmp_file" | tr -d '[:space:]')" - filter_service_metrics_snapshot "$tmp_file" "$filtered_file" - mv "$filtered_file" "$snapshot_file" - snapshot_bytes="$(wc -c <"$snapshot_file" | tr -d '[:space:]')" - cat >"$status_file" <> "$SERVICE_METRICS_CSV" + return + fi + echo "$size,$TOOL,$round,$attempt,$phase,$node,$status,$capture_attempt,$raw_bytes,$snapshot_bytes,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX" >> "$NODE_METRICS_CSV" +} + +capture_direct_metrics_snapshot() { + local size="$1" round="$2" attempt="$3" phase="$4" node="$5" metrics_url="$6" + local token snapshot_file status_file tmp_file filtered_file capture_attempt raw_bytes snapshot_bytes + token="$(metric_snapshot_token "$size" "$round" "$attempt")" + if [[ "$node" == "service" ]]; then + snapshot_file="${SERVICE_METRICS_DIR}/${token}_${phase}.prom" + else + snapshot_file="${SERVICE_METRICS_DIR}/${token}_${node}_${phase}.prom" + fi + status_file="${SERVICE_METRICS_DIR}/${token}_${node}_${phase}.status" + tmp_file="${snapshot_file}.tmp" + filtered_file="${snapshot_file}.filtered.tmp" + + if [[ "$DRY_RUN" == "true" ]]; then + : >"$snapshot_file" + cat >"$status_file" <"$tmp_file" && [[ -s "$tmp_file" ]]; then + raw_bytes="$(wc -c <"$tmp_file" | tr -d '[:space:]')" + filter_service_metrics_snapshot "$tmp_file" "$filtered_file" + mv "$filtered_file" "$snapshot_file" + snapshot_bytes="$(wc -c <"$snapshot_file" | tr -d '[:space:]')" + cat >"$status_file" <> "$SERVICE_METRICS_CSV" - return 0 - fi + append_direct_metrics_capture "$size" "$round" "$attempt" "$phase" "$node" ok "$capture_attempt" "$raw_bytes" "$snapshot_bytes" "$status_file" "$snapshot_file" + return fi rm -f "$tmp_file" "$filtered_file" @@ -801,16 +997,52 @@ size=${size} round=${round} attempt=${attempt} phase=${phase} +node=${node} status=capture_failed capture_attempts=${SERVICE_METRICS_CAPTURE_ATTEMPTS} -url=${SERVICE_METRICS_URL} +url=${metrics_url} connect_timeout_secs=${SERVICE_METRICS_CONNECT_TIMEOUT_SECS} max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS} filter_regex=${SERVICE_METRICS_FILTER_REGEX} settle_secs=${SERVICE_METRICS_SETTLE_SECS} EOF - echo "$size,$TOOL,$round,$attempt,$phase,prometheus_text,capture_failed,$SERVICE_METRICS_CAPTURE_ATTEMPTS,N/A,N/A,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX,$SERVICE_PROMETHEUS_QUERY" >> "$SERVICE_METRICS_CSV" - echo "WARN: failed to capture service metrics size=${size} round=${round} attempt=${attempt} phase=${phase}" >&2 + append_direct_metrics_capture "$size" "$round" "$attempt" "$phase" "$node" capture_failed "$SERVICE_METRICS_CAPTURE_ATTEMPTS" N/A N/A "$status_file" "$snapshot_file" + echo "WARN: failed to capture metrics size=${size} round=${round} attempt=${attempt} phase=${phase} node=${node}" >&2 +} + +capture_round_node_metrics() { + local size="$1" round="$2" attempt="$3" phase="$4" entry node metrics_url + if ((${#NODE_METRICS_URLS[@]} == 0)); then + return + fi + for entry in "${NODE_METRICS_URLS[@]}"; do + node="${entry%%=*}" + metrics_url="${entry#*=}" + capture_direct_metrics_snapshot "$size" "$round" "$attempt" "$phase" "$node" "$metrics_url" + done +} + +capture_round_node_resources() { + local size="$1" round="$2" attempt="$3" phase="$4" entry node container token snapshot_file + if ((${#NODE_DOCKER_CONTAINERS[@]} == 0)); then + return + fi + for entry in "${NODE_DOCKER_CONTAINERS[@]}"; do + node="${entry%%=*}" + container="${entry#*=}" + token="$(metric_snapshot_token "$size" "$round" "$attempt")" + snapshot_file="$OUT_DIR/node_resources/${token}_${node}_${phase}.json" + if [[ "$DRY_RUN" == "true" ]]; then + : >"$snapshot_file" + echo "$size,$TOOL,$round,$attempt,$phase,$node,$container,not_run_dry_run,$snapshot_file" >> "$NODE_RESOURCE_CSV" + elif docker stats --no-stream --format '{{json .}}' "$container" >"$snapshot_file" && [[ -s "$snapshot_file" ]]; then + echo "$size,$TOOL,$round,$attempt,$phase,$node,$container,ok,$snapshot_file" >> "$NODE_RESOURCE_CSV" + else + : >"$snapshot_file" + echo "$size,$TOOL,$round,$attempt,$phase,$node,$container,capture_failed,$snapshot_file" >> "$NODE_RESOURCE_CSV" + echo "WARN: failed to capture Docker resources node=${node} container=${container}" >&2 + fi + done } median_from_numbers() { @@ -844,6 +1076,8 @@ run_one_attempt() { local exit_code=0 local started_at_utc finished_at_utc capture_round_service_metrics "$size" "$round" "$attempt" before + capture_round_node_metrics "$size" "$round" "$attempt" before + capture_round_node_resources "$size" "$round" "$attempt" before started_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)" if [[ "$TOOL" == "warp" ]]; then @@ -917,6 +1151,8 @@ run_one_attempt() { finished_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)" settle_before_after_metrics_capture capture_round_service_metrics "$size" "$round" "$attempt" after + capture_round_node_metrics "$size" "$round" "$attempt" after + capture_round_node_resources "$size" "$round" "$attempt" after local metrics throughput_human reqps latency_human throughput_bps latency_ms req_p90_human req_p90_ms req_p99_human req_p99_ms if [[ "$DRY_RUN" == "true" ]]; then @@ -1069,6 +1305,17 @@ main() { echo "Rounds: $ROUNDS" echo "Retry per round: $RETRY_PER_ROUND" echo "Cooldown secs: $COOLDOWN_SECS" + echo "Server image digest: ${SERVER_IMAGE_DIGEST:-unknown}" + echo "Server revision: ${SERVER_REVISION:-unknown}" + if ((${#RUN_LABELS[@]} > 0)); then + echo "Run labels: ${RUN_LABELS[*]}" + fi + if ((${#NODE_METRICS_URLS[@]} > 0)); then + echo "Node metric targets: ${#NODE_METRICS_URLS[@]}" + fi + if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)); then + echo "Node Docker targets: ${#NODE_DOCKER_CONTAINERS[@]}" + fi if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then echo "Service metrics filter: $SERVICE_METRICS_FILTER_REGEX" if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then diff --git a/scripts/test_object_batch_bench_enhanced.sh b/scripts/test_object_batch_bench_enhanced.sh new file mode 100755 index 000000000..c9cdce0a9 --- /dev/null +++ b/scripts/test_object_batch_bench_enhanced.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="${SCRIPT_DIR}/run_object_batch_bench_enhanced.sh" +TMP_DIR="$(mktemp -d)" +OUT_DIR="${TMP_DIR}/run" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +"$RUNNER" \ + --tool warp \ + --endpoint http://127.0.0.1:9000 \ + --access-key test-access \ + --secret-key test-secret \ + --sizes 1MiB \ + --rounds 1 \ + --retry-per-round 1 \ + --cooldown-secs 0 \ + --duration 1s \ + --out-dir "$OUT_DIR" \ + --warp-bin true \ + --dry-run \ + --service-metrics-dir "${OUT_DIR}/metrics" \ + --server-image-ref rustfs/rustfs:bench \ + --server-image-digest sha256:0123456789abcdef \ + --server-revision 9f61bad94 \ + --require-server-provenance \ + --label topology=4x2 \ + --label workload=get \ + --node-metrics-url node1=http://127.0.0.1:9001/metrics \ + --node-docker-container node1=rustfs-bench-1 >/dev/null + +rg -qx 'server_image_ref=rustfs/rustfs:bench' "${OUT_DIR}/run_manifest.env" +rg -qx 'server_image_digest=sha256:0123456789abcdef' "${OUT_DIR}/run_manifest.env" +rg -qx 'server_revision=9f61bad94' "${OUT_DIR}/run_manifest.env" +rg -qx 'run_label_topology=4x2' "${OUT_DIR}/run_manifest.env" +rg -qx 'run_label_workload=get' "${OUT_DIR}/run_manifest.env" +rg -q '^node1,rustfs-bench-1,not_run_dry_run,N/A,N/A,N/A,N/A$' "${OUT_DIR}/node_inventory.csv" +rg -q '^1MiB,warp,1,1,before,node1,not_run_dry_run,' "${OUT_DIR}/node_metrics_captures.csv" +rg -q '^1MiB,warp,1,1,after,node1,rustfs-bench-1,not_run_dry_run,' "${OUT_DIR}/node_resource_captures.csv" + +if "$RUNNER" \ + --tool warp \ + --endpoint http://127.0.0.1:9000 \ + --access-key test-access \ + --secret-key test-secret \ + --warp-bin true \ + --dry-run \ + --require-server-provenance >/dev/null 2>&1; then + echo "expected --require-server-provenance to reject missing identity" >&2 + exit 1 +fi + +"$RUNNER" \ + --tool warp \ + --endpoint http://127.0.0.1:9000 \ + --access-key test-access \ + --secret-key test-secret \ + --sizes 1MiB \ + --rounds 1 \ + --retry-per-round 1 \ + --cooldown-secs 0 \ + --out-dir "${TMP_DIR}/default" \ + --warp-bin true \ + --dry-run >/dev/null + +rg -qx 'run_label_count=0' "${TMP_DIR}/default/run_manifest.env"