gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(perf): fix GET stage metrics capture

Browse Source 
Co-Authored-By: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-07-26 15:35:52 +08:00
parent 23f4683f20
commit f460b6f804
3 changed files with 70 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/run_get_1mib_abba_stage_metrics.sh
+16 -8
View File
@@ -35,6 +35,7 @@ WARP_BIN="warp"
PYTHON_BIN="python3"
OUT_DIR=""
DIAGNOSTIC_METRICS_URL="http://127.0.0.1:8889/metrics"
DEFAULT_DIAGNOSTIC_OBS_ENDPOINT="http://127.0.0.1:4318"
DIAGNOSTIC_METRICS_SETTLE_SECS="${RUSTFS_DIAGNOSTIC_METRICS_SETTLE_SECS:-2}"
DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS="${RUSTFS_DIAGNOSTIC_METRICS_CAPTURE_ATTEMPTS:-5}"
DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS="${RUSTFS_DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS:-1}"
@@ -96,8 +97,10 @@ Diagnostics:
--diagnostic-metrics-connect-timeout-secs <n>
--diagnostic-metrics-max-time-secs <n>
--diagnostic-metrics-filter-regex <regex>
--diagnostic-obs-endpoint <url>
--diagnostic-obs-metric-endpoint <url>
--diagnostic-obs-endpoint <url> RUSTFS_OBS_ENDPOINT passed to RustFS
(default: http://127.0.0.1:4318)
--diagnostic-obs-metric-endpoint <url> RUSTFS_OBS_METRIC_ENDPOINT passed to RustFS
(default: <obs-endpoint>/v1/metrics)
--diagnostic-obs-meter-interval <secs>
--diagnostic-obs-service-name-prefix <name>
@@ -197,6 +200,12 @@ validate_non_negative_int "$DIAGNOSTIC_METRICS_CAPTURE_RETRY_SECS" "--diagnostic
validate_positive_int "$DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS" "--diagnostic-metrics-connect-timeout-secs"
validate_positive_int "$DIAGNOSTIC_METRICS_MAX_TIME_SECS" "--diagnostic-metrics-max-time-secs"
validate_positive_int "$DIAGNOSTIC_OBS_METER_INTERVAL" "--diagnostic-obs-meter-interval"
if [[ -z "$DIAGNOSTIC_OBS_ENDPOINT" ]]; then
DIAGNOSTIC_OBS_ENDPOINT="$DEFAULT_DIAGNOSTIC_OBS_ENDPOINT"
fi
if [[ -z "$DIAGNOSTIC_OBS_METRIC_ENDPOINT" ]]; then
DIAGNOSTIC_OBS_METRIC_ENDPOINT="${DIAGNOSTIC_OBS_ENDPOINT%/}/v1/metrics"
fi
IFS=',' read -r -a outer_sizes <<< "$OUTER_READER_STREAM_BUFFER_SIZES"
IFS=',' read -r -a profile_orders <<< "$PROFILE_ORDERS"
@@ -264,6 +273,9 @@ codec_min_size=${CODEC_MIN_SIZE}
handoff_attribution=true
diagnostic_metrics=true
diagnostic_metrics_url=${DIAGNOSTIC_METRICS_URL}
diagnostic_obs_endpoint=${DIAGNOSTIC_OBS_ENDPOINT}
diagnostic_obs_metric_endpoint=${DIAGNOSTIC_OBS_METRIC_ENDPOINT}
diagnostic_obs_meter_interval=${DIAGNOSTIC_OBS_METER_INTERVAL}
stage_metrics_artifacts=service_metrics_summary.csv,service_metrics_round_summary.csv,service_metrics_stage_distribution.csv,service_metrics_round_percentiles.csv
body_header_parity_artifacts=compat_summary.csv,response_headers_legacy.json,response_headers_codec_legacy.json,body_sha256_legacy.txt,body_sha256_codec_legacy.txt
performance_conclusion=not_encoded_by_harness_collect_raw_abba_stage_metrics_first
@@ -322,6 +334,8 @@ run_cell() {
--diagnostic-metrics-connect-timeout-secs "$DIAGNOSTIC_METRICS_CONNECT_TIMEOUT_SECS"
--diagnostic-metrics-max-time-secs "$DIAGNOSTIC_METRICS_MAX_TIME_SECS"
--diagnostic-metrics-filter-regex "$DIAGNOSTIC_METRICS_FILTER_REGEX"
--diagnostic-obs-endpoint "$DIAGNOSTIC_OBS_ENDPOINT"
--diagnostic-obs-metric-endpoint "$DIAGNOSTIC_OBS_METRIC_ENDPOINT"
--diagnostic-obs-meter-interval "$DIAGNOSTIC_OBS_METER_INTERVAL"
--diagnostic-obs-service-name-prefix "${DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX}-${profile_order}-${outer_size}"
--address "$ADDRESS"
@@ -343,12 +357,6 @@ run_cell() {
--python-bin "$PYTHON_BIN"
--resource-sample-interval-secs "$RESOURCE_SAMPLE_INTERVAL_SECS"
)
if [[ -n "$DIAGNOSTIC_OBS_ENDPOINT" ]]; then
cmd+=(--diagnostic-obs-endpoint "$DIAGNOSTIC_OBS_ENDPOINT")
fi
if [[ -n "$DIAGNOSTIC_OBS_METRIC_ENDPOINT" ]]; then
cmd+=(--diagnostic-obs-metric-endpoint "$DIAGNOSTIC_OBS_METRIC_ENDPOINT")
fi
if [[ -n "$WARP_OBJECTS" ]]; then
cmd+=(--warp-objects "$WARP_OBJECTS")
fi
scripts/run_get_codec_streaming_smoke.sh
+47 -7
View File
@@ -49,6 +49,7 @@ DIAGNOSTIC_METRICS=false
DIAGNOSTIC_METRICS_URL="http://127.0.0.1:8889/metrics"
DIAGNOSTIC_PROMETHEUS_QUERY_URL=""
DIAGNOSTIC_PROMETHEUS_QUERY='{__name__=~"rustfs_io_get_object_.*"}'
DEFAULT_DIAGNOSTIC_OBS_ENDPOINT="http://127.0.0.1:4318"
DIAGNOSTIC_METRICS_SETTLE_SECS="${RUSTFS_DIAGNOSTIC_METRICS_SETTLE_SECS:-2}"
DIAGNOSTIC_OBS_ENDPOINT="${RUSTFS_OBS_ENDPOINT:-}"
DIAGNOSTIC_OBS_METRIC_ENDPOINT="${RUSTFS_OBS_METRIC_ENDPOINT:-}"
@@ -149,8 +150,10 @@ Core options:
(default: rustfs_io_get_object_)
--diagnostic-obs-endpoint <url>
RUSTFS_OBS_ENDPOINT passed to RustFS during diagnostic runs
(default when --diagnostic-metrics is enabled: http://127.0.0.1:4318)
--diagnostic-obs-metric-endpoint <url>
RUSTFS_OBS_METRIC_ENDPOINT passed to RustFS during diagnostic runs
(default when --diagnostic-metrics is enabled: <obs-endpoint>/v1/metrics)
--diagnostic-obs-meter-interval <secs>
RUSTFS_OBS_METER_INTERVAL passed during diagnostic runs (default: 1)
--diagnostic-obs-service-name-prefix <name>
@@ -483,6 +486,18 @@ validate_args() {
fi
}
resolve_diagnostic_observability_endpoints() {
if [[ "$DIAGNOSTIC_METRICS" != "true" ]]; then
return 0
fi
if [[ -z "$DIAGNOSTIC_OBS_ENDPOINT" ]]; then
DIAGNOSTIC_OBS_ENDPOINT="$DEFAULT_DIAGNOSTIC_OBS_ENDPOINT"
fi
if [[ -z "$DIAGNOSTIC_OBS_METRIC_ENDPOINT" ]]; then
DIAGNOSTIC_OBS_METRIC_ENDPOINT="${DIAGNOSTIC_OBS_ENDPOINT%/}/v1/metrics"
fi
}
setup_output() {
if [[ -z "$OUT_DIR" ]]; then
OUT_DIR="${PROJECT_ROOT}/target/bench/get-codec-streaming-$(date +%Y%m%d-%H%M%S)"
@@ -1501,6 +1516,16 @@ def sample_to_line(sample):
return f"{name} {value}"
def sample_matches_service(sample, service_name):
if not service_name:
return True
metric = sample.get("metric", {})
return any(
metric.get(key) == service_name
for key in ("service.name", "service_name", "job", "otel_scope_name")
)
try:
separator = "&" if "?" in query_url else "?"
url = f"{query_url}{separator}{urllib.parse.urlencode({'query': query})}"
@@ -1520,9 +1545,7 @@ if payload.get("status") != "success":
samples = [
sample
for sample in payload.get("data", {}).get("result", [])
if not service_name
or sample.get("metric", {}).get("service.name") == service_name
or sample.get("metric", {}).get("service_name") == service_name
if sample_matches_service(sample, service_name)
]
lines = [line for sample in samples if (line := sample_to_line(sample))]
if not lines:
@@ -1702,6 +1725,15 @@ def classify(metric):
return "get_object"
def has_service_label(labels, service_name):
if not service_name:
return True
return any(
f'{key}="{service_name}"' in labels
for key in ("service_name", "service.name", "job", "otel_scope_name")
)
def read_metrics(path):
rows = {}
for raw in path.read_text(encoding="utf-8", errors="replace").splitlines():
@@ -1714,8 +1746,7 @@ def read_metrics(path):
if not metric.startswith(metric_prefix):
continue
labels = labels or ""
if service_name and f'service_name="{service_name}"' not in labels \
and f'service.name="{service_name}"' not in labels:
if not has_service_label(labels, service_name):
continue
rows[(metric, labels)] = float(value)
return rows
@@ -1892,6 +1923,15 @@ def label_value(labels, key):
return match.group(1).replace(r'\"', '"').replace(r"\\", "\\")
def has_service_label(labels, service_name):
if not service_name:
return True
return any(
f'{key}="{service_name}"' in labels
for key in ("service_name", "service.name", "job", "otel_scope_name")
)
def read_metrics(path):
rows = {}
if not path.exists() or path.stat().st_size == 0:
@@ -1906,8 +1946,7 @@ def read_metrics(path):
if not metric.startswith(metric_prefix):
continue
labels = labels or ""
if service_name and f'service_name="{service_name}"' not in labels \
and f'service.name="{service_name}"' not in labels:
if not has_service_label(labels, service_name):
continue
rows[(metric, labels)] = float(value)
return rows
@@ -4135,6 +4174,7 @@ main() {
ORIGINAL_ARGS=("$@")
parse_args "$@"
validate_args
resolve_diagnostic_observability_endpoints
setup_output
run_detached_if_requested
local codec_profiles=()
scripts/test_get_1mib_abba_stage_metrics.sh
+7
View File
@@ -35,7 +35,12 @@ rg -qx 'inner_legacy_duplex_buffer_policy=adaptive_duplex_buffer_size' "${OUT_DI
rg -qx 'exact_1mib_legacy_duplex_buffer_bytes=65536' "${OUT_DIR}/manifest.env"
rg -qx 'handoff_attribution=true' "${OUT_DIR}/manifest.env"
rg -qx 'diagnostic_metrics=true' "${OUT_DIR}/manifest.env"
rg -qx 'diagnostic_obs_endpoint=http://127.0.0.1:4318' "${OUT_DIR}/manifest.env"
rg -qx 'diagnostic_obs_metric_endpoint=http://127.0.0.1:4318/v1/metrics' "${OUT_DIR}/manifest.env"
rg -qx 'diagnostic_obs_meter_interval=1' "${OUT_DIR}/manifest.env"
rg -qx 'performance_conclusion=not_encoded_by_harness_collect_raw_abba_stage_metrics_first' "${OUT_DIR}/manifest.env"
rg -Fq '("service.name", "service_name", "job", "otel_scope_name")' "${SCRIPT_DIR}/run_get_codec_streaming_smoke.sh"
rg -Fq '("service_name", "service.name", "job", "otel_scope_name")' "${SCRIPT_DIR}/run_get_codec_streaming_smoke.sh"
matrix_rows="$(awk -F',' 'NR > 1 { count++ } END { print count + 0 }' "${OUT_DIR}/abba_matrix.csv")"
if [[ "$matrix_rows" != "4" ]]; then
@@ -68,6 +73,8 @@ for outer_size in 65536 1048576; do
rg -qx 'sizes=1MiB' "$profile_manifest"
rg -qx 'RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE=true' "$profile_manifest"
rg -qx 'RUSTFS_OBS_METRICS_EXPORT_ENABLED=true' "$profile_manifest"
rg -qx 'RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318' "$profile_manifest"
rg -qx 'RUSTFS_OBS_METRIC_ENDPOINT=http://127.0.0.1:4318/v1/metrics' "$profile_manifest"
rg -qx 'RUSTFS_GET_CODEC_STREAMING_MIN_SIZE=1048576' "$profile_manifest"
test -f "${cell_dir}/${profile}/service_metrics_round_summary.csv"
test -f "${cell_dir}/${profile}/service_metrics_stage_distribution.csv"