test(scanner): add validation harness (#3428)

* test(scanner): add validation harness * fix(scanner): harden validation harness --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>
2026-07-27 00:38:16 +00:00 · 2026-06-14 12:40:05 +08:00
parent 61a5500232
commit dc82efbab4
3 changed files with 521 additions and 0 deletions
@@ -137,6 +137,25 @@ same command with new buckets or prefixes and record the final object count.
 Capture scanner status before the workload, after the workload finishes, and
 throughout the idle observation window.

+The repository includes a scanner validation harness for repeatable collection:
+
+```bash
+export RUSTFS_ACCESS_KEY="<admin-access-key>"
+export RUSTFS_SECRET_KEY="<admin-secret-key>"
+
+scripts/run_scanner_validation_harness.sh \
+  --alias ALIAS \
+  --endpoint http://127.0.0.1:9000 \
+  --deployment single-disk \
+  --workload-label small-object-idle \
+  --samples 30 \
+  --interval-secs 60 \
+  --out-dir artifacts/scanner-validation
+```
+
+The harness writes scanner/heal config snapshots, scanner status samples, host
+telemetry when available, run metadata, and `scanner-summary.csv`.
+
 Example status request:

 ```bash
@@ -0,0 +1,298 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ALIAS=""
+ENDPOINT=""
+ACCESS_KEY="${RUSTFS_ACCESS_KEY:-}"
+SECRET_KEY=""
+SECRET_KEY_ENV="RUSTFS_SECRET_KEY"
+REGION="us-east-1"
+DEPLOYMENT="single-disk"
+WORKLOAD_LABEL="unspecified"
+SAMPLES=30
+INTERVAL_SECS=60
+OUT_DIR=""
+MC_BIN="mc"
+AWSCURL_BIN="awscurl"
+JQ_BIN="jq"
+SKIP_HOST_TELEMETRY=false
+RUSTFS_PID=""
+TELEMETRY_PIDS=()
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/run_scanner_validation_harness.sh --alias <admin-alias> \
+    --endpoint <url> [options]
+
+Required:
+  --alias                 Admin client alias used for config snapshots.
+  --endpoint              RustFS endpoint, for example http://127.0.0.1:9000.
+  RUSTFS_ACCESS_KEY       Admin access key for scanner status requests.
+  RUSTFS_SECRET_KEY       Admin secret key for scanner status requests.
+
+Optional:
+  --access-key            Override RUSTFS_ACCESS_KEY for scanner status requests.
+  --secret-key-env        Environment variable that stores the admin secret key
+                          (default: RUSTFS_SECRET_KEY).
+  --region                SigV4 region (default: us-east-1).
+  --deployment            single-disk | multi-disk | distributed (default: single-disk).
+  --workload-label        Free-form workload label written to metadata.
+  --samples               Number of scanner status samples (default: 30).
+  --interval-secs         Seconds between samples (default: 60).
+  --out-dir               Output directory (default: target/bench/scanner-validation-<timestamp>).
+  --mc-bin                mc-compatible admin client (default: mc).
+  --awscurl-bin           SigV4 HTTP client (default: awscurl).
+  --jq-bin                jq-compatible JSON processor (default: jq).
+  --rustfs-pid            RustFS process id for pidstat. If omitted, pidof rustfs is used.
+  --skip-host-telemetry   Do not run pidstat/iostat/mpstat.
+  -h, --help              Show this help.
+
+The harness collects scanner/heal config snapshots, scanner status samples,
+host telemetry when available, and a compact scanner-summary.csv file. It does
+not generate object workload or modify scanner configuration.
+USAGE
+}
+
+require_cmd() {
+  if ! command -v "$1" >/dev/null 2>&1; then
+    echo "ERROR: command not found: $1" >&2
+    exit 1
+  fi
+}
+
+is_nonnegative_integer() {
+  [[ "$1" =~ ^[0-9]+$ ]]
+}
+
+is_positive_integer() {
+  [[ "$1" =~ ^[1-9][0-9]*$ ]]
+}
+
+arg_value() {
+  local flag="$1"
+  local value="${2:-}"
+
+  if [[ -z "$value" || "$value" == --* ]]; then
+    echo "ERROR: missing value for $flag" >&2
+    exit 1
+  fi
+
+  printf '%s\n' "$value"
+}
+
+parse_args() {
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --alias) ALIAS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --access-key) ACCESS_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --secret-key-env) SECRET_KEY_ENV="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --region) REGION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --deployment) DEPLOYMENT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --workload-label) WORKLOAD_LABEL="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --samples) SAMPLES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --interval-secs) INTERVAL_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --mc-bin) MC_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --awscurl-bin) AWSCURL_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --jq-bin) JQ_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --rustfs-pid) RUSTFS_PID="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --skip-host-telemetry) SKIP_HOST_TELEMETRY=true; shift ;;
+      -h|--help) usage; exit 0 ;;
+      *)
+        echo "ERROR: unknown arg: $1" >&2
+        usage
+        exit 1
+        ;;
+    esac
+  done
+}
+
+validate_args() {
+  if [[ -z "$ALIAS" || -z "$ENDPOINT" || -z "$ACCESS_KEY" ]]; then
+    echo "ERROR: --alias, --endpoint, and RUSTFS_ACCESS_KEY (or --access-key) are required" >&2
+    exit 1
+  fi
+
+  if ! [[ "$SECRET_KEY_ENV" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
+    echo "ERROR: --secret-key-env must be a valid environment variable name" >&2
+    exit 1
+  fi
+
+  SECRET_KEY="${!SECRET_KEY_ENV:-}"
+  if [[ -z "$SECRET_KEY" ]]; then
+    echo "ERROR: $SECRET_KEY_ENV is required for scanner status requests" >&2
+    exit 1
+  fi
+
+  case "$DEPLOYMENT" in
+    single-disk|multi-disk|distributed) ;;
+    *)
+      echo "ERROR: --deployment must be single-disk, multi-disk, or distributed" >&2
+      exit 1
+      ;;
+  esac
+
+  if ! is_positive_integer "$SAMPLES"; then
+    echo "ERROR: --samples must be a positive integer" >&2
+    exit 1
+  fi
+
+  if ! is_nonnegative_integer "$INTERVAL_SECS"; then
+    echo "ERROR: --interval-secs must be a nonnegative integer" >&2
+    exit 1
+  fi
+}
+
+setup_output() {
+  if [[ -z "$OUT_DIR" ]]; then
+    OUT_DIR="target/bench/scanner-validation-$(date -u +%Y%m%dT%H%M%SZ)"
+  fi
+
+  mkdir -p "$OUT_DIR/status"
+  SUMMARY_CSV="$OUT_DIR/scanner-summary.csv"
+  echo "timestamp,primary_pressure,current_cycle_objects_scanned,current_cycle_directories_scanned,last_cycle_result,last_cycle_partial_reason,last_cycle_partial_source,lifecycle_transition_scanner_missed,source_work_missed_total" >"$SUMMARY_CSV"
+}
+
+git_value() {
+  local args=("$@")
+  git "${args[@]}" 2>/dev/null || true
+}
+
+write_metadata() {
+  local started_at="$1"
+
+  {
+    printf 'started_at=%s\n' "$started_at"
+    printf 'deployment=%s\n' "$DEPLOYMENT"
+    printf 'workload_label=%s\n' "$WORKLOAD_LABEL"
+    printf 'endpoint=%s\n' "$ENDPOINT"
+    printf 'region=%s\n' "$REGION"
+    printf 'samples=%s\n' "$SAMPLES"
+    printf 'interval_secs=%s\n' "$INTERVAL_SECS"
+    printf 'git_commit=%s\n' "$(git_value rev-parse HEAD)"
+    printf 'git_branch=%s\n' "$(git_value branch --show-current)"
+  } >"$OUT_DIR/run-metadata.env"
+}
+
+capture_config_snapshots() {
+  "$MC_BIN" admin config get "$ALIAS" scanner >"$OUT_DIR/scanner-config.before.txt"
+  "$MC_BIN" admin config get "$ALIAS" heal >"$OUT_DIR/heal-config.before.txt"
+}
+
+first_rustfs_pid() {
+  if [[ -n "$RUSTFS_PID" ]]; then
+    echo "$RUSTFS_PID"
+    return
+  fi
+
+  pidof rustfs 2>/dev/null | awk '{ print $1 }' || true
+}
+
+start_host_telemetry() {
+  TELEMETRY_PIDS=()
+
+  if [[ "$SKIP_HOST_TELEMETRY" == "true" || "$INTERVAL_SECS" == "0" ]]; then
+    return
+  fi
+
+  local pid
+  pid="$(first_rustfs_pid)"
+
+  if [[ -n "$pid" ]] && command -v pidstat >/dev/null 2>&1; then
+    pidstat -p "$pid" "$INTERVAL_SECS" "$SAMPLES" >"$OUT_DIR/pidstat.txt" 2>&1 &
+    TELEMETRY_PIDS+=("$!")
+  fi
+
+  if command -v iostat >/dev/null 2>&1; then
+    iostat -xz "$INTERVAL_SECS" "$SAMPLES" >"$OUT_DIR/iostat.txt" 2>&1 &
+    TELEMETRY_PIDS+=("$!")
+  fi
+
+  if command -v mpstat >/dev/null 2>&1; then
+    mpstat "$INTERVAL_SECS" "$SAMPLES" >"$OUT_DIR/mpstat.txt" 2>&1 &
+    TELEMETRY_PIDS+=("$!")
+  fi
+}
+
+wait_host_telemetry() {
+  local pid
+
+  if [[ ${#TELEMETRY_PIDS[@]} -eq 0 ]]; then
+    return
+  fi
+
+  for pid in "${TELEMETRY_PIDS[@]}"; do
+    wait "$pid" || true
+  done
+}
+
+scanner_status_url() {
+  printf '%s/rustfs/admin/v3/scanner/status\n' "${ENDPOINT%/}"
+}
+
+capture_status_sample() {
+  local index="$1"
+  local ts="$2"
+  local status_file="$OUT_DIR/status/scanner-status.${index}.${ts}.json"
+
+  AWS_ACCESS_KEY_ID="$ACCESS_KEY" \
+  AWS_SECRET_ACCESS_KEY="$SECRET_KEY" \
+  AWS_DEFAULT_REGION="$REGION" \
+  "$AWSCURL_BIN" \
+    --service s3 \
+    --region "$REGION" \
+    --request GET \
+    "$(scanner_status_url)" \
+    | "$JQ_BIN" . >"$status_file"
+
+  "$JQ_BIN" -r --arg ts "$ts" '
+    [
+      $ts,
+      (.metrics.pacing_pressure.primary_pressure // ""),
+      (.metrics.current_cycle_objects_scanned // 0),
+      (.metrics.current_cycle_directories_scanned // 0),
+      (.metrics.last_cycle_result // ""),
+      (.metrics.last_cycle_partial_reason // ""),
+      (.metrics.last_cycle_partial_source // ""),
+      (.metrics.lifecycle_transition.scanner_missed // 0),
+      ((.metrics.source_work // []) | map(.missed // 0) | add // 0)
+    ] | @csv
+  ' "$status_file" >>"$SUMMARY_CSV"
+}
+
+capture_status_series() {
+  local index ts
+
+  for ((index = 1; index <= SAMPLES; index++)); do
+    ts="$(date -u +%Y%m%dT%H%M%SZ)"
+    capture_status_sample "$index" "$ts"
+
+    if [[ "$index" -lt "$SAMPLES" && "$INTERVAL_SECS" != "0" ]]; then
+      sleep "$INTERVAL_SECS"
+    fi
+  done
+}
+
+main() {
+  parse_args "$@"
+  validate_args
+  require_cmd "$MC_BIN"
+  require_cmd "$AWSCURL_BIN"
+  require_cmd "$JQ_BIN"
+  setup_output
+
+  local started_at
+  started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+  write_metadata "$started_at"
+  capture_config_snapshots
+  start_host_telemetry
+  capture_status_series
+  wait_host_telemetry
+
+  echo "Scanner validation artifacts written to $OUT_DIR"
+}
+
+main "$@"
@@ -0,0 +1,204 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+SCRIPT="$ROOT_DIR/scripts/run_scanner_validation_harness.sh"
+TMP_DIR=$(mktemp -d)
+trap 'rm -rf "$TMP_DIR"' EXIT
+
+BIN_DIR="$TMP_DIR/bin"
+OUT_DIR="$TMP_DIR/out"
+mkdir -p "$BIN_DIR"
+
+cat >"$BIN_DIR/mc" <<'STUB'
+#!/usr/bin/env bash
+set -euo pipefail
+
+printf '%s\n' "$*" >>"${MC_LOG:?}"
+if [[ "$*" == "admin config get rustfs-local scanner" ]]; then
+  printf 'scanner delay="30" max_wait="15"\n'
+elif [[ "$*" == "admin config get rustfs-local heal" ]]; then
+  printf 'heal bitrot_cycle="2592000"\n'
+else
+  printf 'unexpected mc args: %s\n' "$*" >&2
+  exit 1
+fi
+STUB
+
+cat >"$BIN_DIR/awscurl" <<'STUB'
+#!/usr/bin/env bash
+set -euo pipefail
+
+printf '%s\n' "$*" >>"${AWSCURL_LOG:?}"
+printf 'access-env-present=%s\n' "${AWS_ACCESS_KEY_ID:+yes}" >>"${AWSCURL_LOG:?}"
+printf 'secret-env-present=%s\n' "${AWS_SECRET_ACCESS_KEY:+yes}" >>"${AWSCURL_LOG:?}"
+cat <<'JSON'
+{
+  "runtime_config": {
+    "delay": { "value": 30, "source": "config" }
+  },
+  "metrics": {
+    "pacing_pressure": {
+      "primary_pressure": "active_scans"
+    },
+    "current_cycle_objects_scanned": 5,
+    "current_cycle_directories_scanned": 2,
+    "last_cycle_result": "success",
+    "last_cycle_partial_reason": "",
+    "last_cycle_partial_source": "",
+    "lifecycle_transition": {
+      "scanner_missed": 0
+    },
+    "source_work": [
+      { "source": "usage", "missed": 0 },
+      { "source": "lifecycle", "missed": 0 }
+    ]
+  }
+}
+JSON
+STUB
+
+cat >"$BIN_DIR/pidof" <<'STUB'
+#!/usr/bin/env bash
+exit 1
+STUB
+
+cat >"$BIN_DIR/iostat" <<'STUB'
+#!/usr/bin/env bash
+printf 'iostat sample\n'
+STUB
+
+cat >"$BIN_DIR/mpstat" <<'STUB'
+#!/usr/bin/env bash
+printf 'mpstat sample\n'
+STUB
+
+cat >"$BIN_DIR/jq" <<'STUB'
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "$1" == "." ]]; then
+  cat
+  exit 0
+fi
+
+if [[ "$1" == "-r" ]]; then
+  while [[ $# -gt 0 ]]; do
+    if [[ "$1" == "--arg" && "${2:-}" == "ts" ]]; then
+      printf '"%s","active_scans",5,2,"success","","",0,0\n' "$3"
+      exit 0
+    fi
+    shift
+  done
+fi
+
+printf 'unexpected jq args: %s\n' "$*" >&2
+exit 1
+STUB
+
+chmod +x "$BIN_DIR/mc" "$BIN_DIR/awscurl" "$BIN_DIR/pidof" "$BIN_DIR/iostat" "$BIN_DIR/mpstat" "$BIN_DIR/jq"
+
+mc_log="$TMP_DIR/mc.log"
+awscurl_log="$TMP_DIR/awscurl.log"
+RUSTFS_SECRET_KEY=rustfsadmin MC_LOG="$mc_log" AWSCURL_LOG="$awscurl_log" PATH="$BIN_DIR:$PATH" "$SCRIPT" \
+  --alias rustfs-local \
+  --endpoint http://127.0.0.1:9000 \
+  --access-key rustfsadmin \
+  --deployment single-disk \
+  --workload-label small-object-idle \
+  --samples 2 \
+  --interval-secs 0 \
+  --out-dir "$OUT_DIR" \
+  --skip-host-telemetry
+
+test -s "$OUT_DIR/scanner-config.before.txt"
+test -s "$OUT_DIR/heal-config.before.txt"
+grep -q 'scanner delay="30" max_wait="15"' "$OUT_DIR/scanner-config.before.txt"
+grep -q 'heal bitrot_cycle="2592000"' "$OUT_DIR/heal-config.before.txt"
+
+status_count=$(find "$OUT_DIR/status" -type f -name 'scanner-status.*.json' | wc -l | tr -d ' ')
+if [[ "$status_count" != "2" ]]; then
+  echo "Expected 2 scanner status snapshots, got $status_count" >&2
+  exit 1
+fi
+
+test -s "$OUT_DIR/scanner-summary.csv"
+if [[ "$(wc -l <"$OUT_DIR/scanner-summary.csv" | tr -d ' ')" != "3" ]]; then
+  echo "Expected scanner summary header plus 2 rows" >&2
+  exit 1
+fi
+
+grep -q '^deployment=single-disk$' "$OUT_DIR/run-metadata.env"
+grep -q '^workload_label=small-object-idle$' "$OUT_DIR/run-metadata.env"
+grep -q 'admin config get rustfs-local scanner' "$mc_log"
+grep -q 'admin config get rustfs-local heal' "$mc_log"
+grep -q -- '--request GET' "$awscurl_log"
+grep -q -- 'access-env-present=yes' "$awscurl_log"
+grep -q -- 'secret-env-present=yes' "$awscurl_log"
+if grep -q -- '--secret_key' "$awscurl_log"; then
+  echo "Expected awscurl to receive the secret through the environment, not argv" >&2
+  exit 1
+fi
+grep -q -- 'http://127.0.0.1:9000/rustfs/admin/v3/scanner/status' "$awscurl_log"
+
+missing_pid_out="$TMP_DIR/out-missing-pid"
+RUSTFS_SECRET_KEY=rustfsadmin MC_LOG="$mc_log" AWSCURL_LOG="$awscurl_log" PATH="$BIN_DIR:$PATH" "$SCRIPT" \
+  --alias rustfs-local \
+  --endpoint http://127.0.0.1:9000 \
+  --access-key rustfsadmin \
+  --deployment single-disk \
+  --workload-label missing-pid \
+  --samples 1 \
+  --interval-secs 1 \
+  --out-dir "$missing_pid_out"
+
+test -s "$missing_pid_out/status/scanner-status.1."*.json
+test -s "$missing_pid_out/iostat.txt"
+test -s "$missing_pid_out/mpstat.txt"
+
+missing_args_log="$TMP_DIR/missing-args.log"
+if PATH="$BIN_DIR:$PATH" "$SCRIPT" --alias rustfs-local >"$missing_args_log" 2>&1; then
+  echo "Expected missing required arguments to fail" >&2
+  exit 1
+fi
+
+grep -q -- '--alias, --endpoint, and RUSTFS_ACCESS_KEY (or --access-key) are required' "$missing_args_log"
+
+missing_secret_log="$TMP_DIR/missing-secret.log"
+if PATH="$BIN_DIR:$PATH" "$SCRIPT" \
+  --alias rustfs-local \
+  --endpoint http://127.0.0.1:9000 \
+  --access-key rustfsadmin >"$missing_secret_log" 2>&1; then
+  echo "Expected missing RUSTFS_SECRET_KEY to fail" >&2
+  exit 1
+fi
+
+grep -q -- 'RUSTFS_SECRET_KEY is required for scanner status requests' "$missing_secret_log"
+
+invalid_secret_env_log="$TMP_DIR/invalid-secret-env.log"
+if PATH="$BIN_DIR:$PATH" "$SCRIPT" \
+  --alias rustfs-local \
+  --endpoint http://127.0.0.1:9000 \
+  --access-key rustfsadmin \
+  --secret-key-env 'not-valid!' >"$invalid_secret_env_log" 2>&1; then
+  echo "Expected invalid --secret-key-env to fail" >&2
+  exit 1
+fi
+
+grep -q -- '--secret-key-env must be a valid environment variable name' "$invalid_secret_env_log"
+
+missing_value_log="$TMP_DIR/missing-value.log"
+if PATH="$BIN_DIR:$PATH" "$SCRIPT" --alias >"$missing_value_log" 2>&1; then
+  echo "Expected --alias without a value to fail" >&2
+  exit 1
+fi
+
+grep -q -- 'missing value for --alias' "$missing_value_log"
+
+secret_arg_log="$TMP_DIR/secret-arg.log"
+if PATH="$BIN_DIR:$PATH" "$SCRIPT" --secret-key rustfsadmin >"$secret_arg_log" 2>&1; then
+  echo "Expected --secret-key argv to be rejected" >&2
+  exit 1
+fi
+
+grep -q -- 'unknown arg: --secret-key' "$secret_arg_log"