From f460b6f8043930b8ae33132fa02edf8f2ecf2c9d Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 26 Jul 2026 15:35:52 +0800 Subject: [PATCH] test(perf): fix GET stage metrics capture Co-Authored-By: heihutu --- scripts/run_get_1mib_abba_stage_metrics.sh | 24 ++++++--- scripts/run_get_codec_streaming_smoke.sh | 54 ++++++++++++++++++--- scripts/test_get_1mib_abba_stage_metrics.sh | 7 +++ 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/scripts/run_get_1mib_abba_stage_metrics.sh b/scripts/run_get_1mib_abba_stage_metrics.sh index d615d75d2..eee1c16a2 100755 --- a/scripts/run_get_1mib_abba_stage_metrics.sh +++ b/scripts/run_get_1mib_abba_stage_metrics.sh @@ -35,6 +35,7 @@ WARP_BIN="warp" PYTHON_BIN="python3" OUT_DIR="" DIAGNOSTIC_METRICS_URL="http://127.0.0.1:8889/metrics" +DEFAULT_DIAGNOSTIC_OBS_ENDPOINT="http://127.0.0.1:4318" DIAGNOSTIC_METRICS_SETTLE_SECS="${RUSTFS_DIAGNOSTIC_METRICS_SETTLE_SECS:-2}" DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS="${RUSTFS_DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS:-5}" DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS="${RUSTFS_DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS:-1}" @@ -96,8 +97,10 @@ Diagnostics: --diagnostic-metrics-connect-timeout-secs --diagnostic-metrics-max-time-secs --diagnostic-metrics-filter-regex - --diagnostic-obs-endpoint - --diagnostic-obs-metric-endpoint + --diagnostic-obs-endpoint RUSTFS_OBS_ENDPOINT passed to RustFS + (default: http://127.0.0.1:4318) + --diagnostic-obs-metric-endpoint RUSTFS_OBS_METRIC_ENDPOINT passed to RustFS + (default: /v1/metrics) --diagnostic-obs-meter-interval --diagnostic-obs-service-name-prefix @@ -197,6 +200,12 @@ validate_non_negative_int "$DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS" "--diagnostic validate_positive_int "$DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS" "--diagnostic-metrics-connect-timeout-secs" validate_positive_int "$DIAGNOSTIC_METRICS_MAX_TIME_SECS" "--diagnostic-metrics-max-time-secs" validate_positive_int "$DIAGNOSTIC_OBS_METER_INTERVAL" "--diagnostic-obs-meter-interval" +if [[ -z "$DIAGNOSTIC_OBS_ENDPOINT" ]]; then + DIAGNOSTIC_OBS_ENDPOINT="$DEFAULT_DIAGNOSTIC_OBS_ENDPOINT" +fi +if [[ -z "$DIAGNOSTIC_OBS_METRIC_ENDPOINT" ]]; then + DIAGNOSTIC_OBS_METRIC_ENDPOINT="${DIAGNOSTIC_OBS_ENDPOINT%/}/v1/metrics" +fi IFS=',' read -r -a outer_sizes <<< "$OUTER_READER_STREAM_BUFFER_SIZES" IFS=',' read -r -a profile_orders <<< "$PROFILE_ORDERS" @@ -264,6 +273,9 @@ codec_min_size=${CODEC_MIN_SIZE} handoff_attribution=true diagnostic_metrics=true diagnostic_metrics_url=${DIAGNOSTIC_METRICS_URL} +diagnostic_obs_endpoint=${DIAGNOSTIC_OBS_ENDPOINT} +diagnostic_obs_metric_endpoint=${DIAGNOSTIC_OBS_METRIC_ENDPOINT} +diagnostic_obs_meter_interval=${DIAGNOSTIC_OBS_METER_INTERVAL} stage_metrics_artifacts=service_metrics_summary.csv,service_metrics_round_summary.csv,service_metrics_stage_distribution.csv,service_metrics_round_percentiles.csv body_header_parity_artifacts=compat_summary.csv,response_headers_legacy.json,response_headers_codec_legacy.json,body_sha256_legacy.txt,body_sha256_codec_legacy.txt performance_conclusion=not_encoded_by_harness_collect_raw_abba_stage_metrics_first @@ -322,6 +334,8 @@ run_cell() { --diagnostic-metrics-connect-timeout-secs "$DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS" --diagnostic-metrics-max-time-secs "$DIAGNOSTIC_METRICS_MAX_TIME_SECS" --diagnostic-metrics-filter-regex "$DIAGNOSTIC_METRICS_FILTER_REGEX" + --diagnostic-obs-endpoint "$DIAGNOSTIC_OBS_ENDPOINT" + --diagnostic-obs-metric-endpoint "$DIAGNOSTIC_OBS_METRIC_ENDPOINT" --diagnostic-obs-meter-interval "$DIAGNOSTIC_OBS_METER_INTERVAL" --diagnostic-obs-service-name-prefix "${DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX}-${profile_order}-${outer_size}" --address "$ADDRESS" @@ -343,12 +357,6 @@ run_cell() { --python-bin "$PYTHON_BIN" --resource-sample-interval-secs "$RESOURCE_SAMPLE_INTERVAL_SECS" ) - if [[ -n "$DIAGNOSTIC_OBS_ENDPOINT" ]]; then - cmd+=(--diagnostic-obs-endpoint "$DIAGNOSTIC_OBS_ENDPOINT") - fi - if [[ -n "$DIAGNOSTIC_OBS_METRIC_ENDPOINT" ]]; then - cmd+=(--diagnostic-obs-metric-endpoint "$DIAGNOSTIC_OBS_METRIC_ENDPOINT") - fi if [[ -n "$WARP_OBJECTS" ]]; then cmd+=(--warp-objects "$WARP_OBJECTS") fi diff --git a/scripts/run_get_codec_streaming_smoke.sh b/scripts/run_get_codec_streaming_smoke.sh index dfbcd68a1..f325ddc48 100755 --- a/scripts/run_get_codec_streaming_smoke.sh +++ b/scripts/run_get_codec_streaming_smoke.sh @@ -49,6 +49,7 @@ DIAGNOSTIC_METRICS=false DIAGNOSTIC_METRICS_URL="http://127.0.0.1:8889/metrics" DIAGNOSTIC_PROMETHEUS_QUERY_URL="" DIAGNOSTIC_PROMETHEUS_QUERY='{__name__=~"rustfs_io_get_object_.*"}' +DEFAULT_DIAGNOSTIC_OBS_ENDPOINT="http://127.0.0.1:4318" DIAGNOSTIC_METRICS_SETTLE_SECS="${RUSTFS_DIAGNOSTIC_METRICS_SETTLE_SECS:-2}" DIAGNOSTIC_OBS_ENDPOINT="${RUSTFS_OBS_ENDPOINT:-}" DIAGNOSTIC_OBS_METRIC_ENDPOINT="${RUSTFS_OBS_METRIC_ENDPOINT:-}" @@ -149,8 +150,10 @@ Core options: (default: rustfs_io_get_object_) --diagnostic-obs-endpoint RUSTFS_OBS_ENDPOINT passed to RustFS during diagnostic runs + (default when --diagnostic-metrics is enabled: http://127.0.0.1:4318) --diagnostic-obs-metric-endpoint RUSTFS_OBS_METRIC_ENDPOINT passed to RustFS during diagnostic runs + (default when --diagnostic-metrics is enabled: /v1/metrics) --diagnostic-obs-meter-interval RUSTFS_OBS_METER_INTERVAL passed during diagnostic runs (default: 1) --diagnostic-obs-service-name-prefix @@ -483,6 +486,18 @@ validate_args() { fi } +resolve_diagnostic_observability_endpoints() { + if [[ "$DIAGNOSTIC_METRICS" != "true" ]]; then + return 0 + fi + if [[ -z "$DIAGNOSTIC_OBS_ENDPOINT" ]]; then + DIAGNOSTIC_OBS_ENDPOINT="$DEFAULT_DIAGNOSTIC_OBS_ENDPOINT" + fi + if [[ -z "$DIAGNOSTIC_OBS_METRIC_ENDPOINT" ]]; then + DIAGNOSTIC_OBS_METRIC_ENDPOINT="${DIAGNOSTIC_OBS_ENDPOINT%/}/v1/metrics" + fi +} + setup_output() { if [[ -z "$OUT_DIR" ]]; then OUT_DIR="${PROJECT_ROOT}/target/bench/get-codec-streaming-$(date +%Y%m%d-%H%M%S)" @@ -1501,6 +1516,16 @@ def sample_to_line(sample): return f"{name} {value}" +def sample_matches_service(sample, service_name): + if not service_name: + return True + metric = sample.get("metric", {}) + return any( + metric.get(key) == service_name + for key in ("service.name", "service_name", "job", "otel_scope_name") + ) + + try: separator = "&" if "?" in query_url else "?" url = f"{query_url}{separator}{urllib.parse.urlencode({'query': query})}" @@ -1520,9 +1545,7 @@ if payload.get("status") != "success": samples = [ sample for sample in payload.get("data", {}).get("result", []) - if not service_name - or sample.get("metric", {}).get("service.name") == service_name - or sample.get("metric", {}).get("service_name") == service_name + if sample_matches_service(sample, service_name) ] lines = [line for sample in samples if (line := sample_to_line(sample))] if not lines: @@ -1702,6 +1725,15 @@ def classify(metric): return "get_object" +def has_service_label(labels, service_name): + if not service_name: + return True + return any( + f'{key}="{service_name}"' in labels + for key in ("service_name", "service.name", "job", "otel_scope_name") + ) + + def read_metrics(path): rows = {} for raw in path.read_text(encoding="utf-8", errors="replace").splitlines(): @@ -1714,8 +1746,7 @@ def read_metrics(path): if not metric.startswith(metric_prefix): continue labels = labels or "" - if service_name and f'service_name="{service_name}"' not in labels \ - and f'service.name="{service_name}"' not in labels: + if not has_service_label(labels, service_name): continue rows[(metric, labels)] = float(value) return rows @@ -1892,6 +1923,15 @@ def label_value(labels, key): return match.group(1).replace(r'\"', '"').replace(r"\\", "\\") +def has_service_label(labels, service_name): + if not service_name: + return True + return any( + f'{key}="{service_name}"' in labels + for key in ("service_name", "service.name", "job", "otel_scope_name") + ) + + def read_metrics(path): rows = {} if not path.exists() or path.stat().st_size == 0: @@ -1906,8 +1946,7 @@ def read_metrics(path): if not metric.startswith(metric_prefix): continue labels = labels or "" - if service_name and f'service_name="{service_name}"' not in labels \ - and f'service.name="{service_name}"' not in labels: + if not has_service_label(labels, service_name): continue rows[(metric, labels)] = float(value) return rows @@ -4135,6 +4174,7 @@ main() { ORIGINAL_ARGS=("$@") parse_args "$@" validate_args + resolve_diagnostic_observability_endpoints setup_output run_detached_if_requested local codec_profiles=() diff --git a/scripts/test_get_1mib_abba_stage_metrics.sh b/scripts/test_get_1mib_abba_stage_metrics.sh index 0f6cb5b1b..805ab38e9 100755 --- a/scripts/test_get_1mib_abba_stage_metrics.sh +++ b/scripts/test_get_1mib_abba_stage_metrics.sh @@ -35,7 +35,12 @@ rg -qx 'inner_legacy_duplex_buffer_policy=adaptive_duplex_buffer_size' "${OUT_DI rg -qx 'exact_1mib_legacy_duplex_buffer_bytes=65536' "${OUT_DIR}/manifest.env" rg -qx 'handoff_attribution=true' "${OUT_DIR}/manifest.env" rg -qx 'diagnostic_metrics=true' "${OUT_DIR}/manifest.env" +rg -qx 'diagnostic_obs_endpoint=http://127.0.0.1:4318' "${OUT_DIR}/manifest.env" +rg -qx 'diagnostic_obs_metric_endpoint=http://127.0.0.1:4318/v1/metrics' "${OUT_DIR}/manifest.env" +rg -qx 'diagnostic_obs_meter_interval=1' "${OUT_DIR}/manifest.env" rg -qx 'performance_conclusion=not_encoded_by_harness_collect_raw_abba_stage_metrics_first' "${OUT_DIR}/manifest.env" +rg -Fq '("service.name", "service_name", "job", "otel_scope_name")' "${SCRIPT_DIR}/run_get_codec_streaming_smoke.sh" +rg -Fq '("service_name", "service.name", "job", "otel_scope_name")' "${SCRIPT_DIR}/run_get_codec_streaming_smoke.sh" matrix_rows="$(awk -F',' 'NR > 1 { count++ } END { print count + 0 }' "${OUT_DIR}/abba_matrix.csv")" if [[ "$matrix_rows" != "4" ]]; then @@ -68,6 +73,8 @@ for outer_size in 65536 1048576; do rg -qx 'sizes=1MiB' "$profile_manifest" rg -qx 'RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE=true' "$profile_manifest" rg -qx 'RUSTFS_OBS_METRICS_EXPORT_ENABLED=true' "$profile_manifest" + rg -qx 'RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318' "$profile_manifest" + rg -qx 'RUSTFS_OBS_METRIC_ENDPOINT=http://127.0.0.1:4318/v1/metrics' "$profile_manifest" rg -qx 'RUSTFS_GET_CODEC_STREAMING_MIN_SIZE=1048576' "$profile_manifest" test -f "${cell_dir}/${profile}/service_metrics_round_summary.csv" test -f "${cell_dir}/${profile}/service_metrics_stage_distribution.csv"