feat(ecstore): add deeper zero-copy ingest experiment (#3847)

* feat(storage): add multipart put stage metrics

* feat(scripts): add multipart put focus runner

* docs(operations): add multipart put server-path guides

* chore(scripts): add local rustfs restart helper

* docs(observability): add local metrics backend guide

* docs(observability): add localized multipart guides

* fix(ecstore): validate multipart batching path

* feat(obs): add erasure encode overlap metrics

* docs(ops): update overlap retest summary

* docs(ops): add batchblocks retest matrix

* docs(ops): extend overlap candidate summary

* docs(ops): capture 8-run overlap summary

* feat(storage): switch rename_data to msgpack map

* test(storage): add rename_data payload checks

* feat(object): add zero_copy_eager put path

* docs(ops): add zero_copy_eager put guide

* docs(ops): add deeper zero-copy next steps

* feat(ecstore): add bytesmut erasure ingest gate

* docs(ops): add bytesmut ingest summary

* docs(ops): extend bytesmut ingest matrix summary

* docs(ops): extend bytesmut larger-object summary

* docs(ops): capture bytesmut variability summary

* chore(scripts): add deeper zero-copy capture flow

* chore(scripts): add deeper zero-copy capture support

* docs(ops): add capture-backed bytesmut retest

* docs(ops): update deeper zero-copy retests

* chore(docs): keep issue-712 notes local only

Co-Authored-By: heihutu <heihutu@gmail.com>

scripts/collect_put_large_stage_breakdown_artifacts.sh

@@ -30,6 +30,7 @@ AWSCURL_BIN="awscurl"
 CURL_BIN="curl"
 JQ_BIN="jq"
 DRY_RUN=false
+AWSCURL_AVAILABLE=true
 
 usage() {
   cat <<'USAGE'
@@ -183,7 +184,19 @@ resolve_pid() {
     echo "$RUSTFS_PID"
     return
   fi
-  pidof rustfs 2>/dev/null | awk '{print $1}' || true
+  if command -v pidof >/dev/null 2>&1; then
+    pidof rustfs 2>/dev/null | awk '{print $1}' && return 0
+  fi
+
+  if command -v pgrep >/dev/null 2>&1; then
+    pgrep -f 'rustfs server' 2>/dev/null | head -n 1 && return 0
+  fi
+
+  if command -v ps >/dev/null 2>&1; then
+    ps -ef 2>/dev/null | awk '/[r]ustfs server/ {print $2; exit}' && return 0
+  fi
+
+  true
 }
 
 health_url() {
@@ -284,6 +297,11 @@ capture_admin_metrics_sample() {
     return
   fi
 
+  if [[ "$AWSCURL_AVAILABLE" != "true" ]]; then
+    echo "awscurl unavailable; signed admin metrics skipped" > "$file"
+    return
+  fi
+
   if [[ -z "$ACCESS_KEY" || -z "$SECRET_KEY" ]]; then
     echo "missing access/secret for signed metrics capture" > "$file"
     return
@@ -450,7 +468,12 @@ main() {
     require_cmd date
   fi
   if [[ "$DRY_RUN" != "true" && -n "$ACCESS_KEY" && -n "$SECRET_KEY" ]]; then
-    require_cmd "$AWSCURL_BIN"
+    if command -v "$AWSCURL_BIN" >/dev/null 2>&1; then
+      AWSCURL_AVAILABLE=true
+    else
+      AWSCURL_AVAILABLE=false
+      echo "WARN: ${AWSCURL_BIN} not found; signed admin metrics snapshots will be skipped" >&2
+    fi
   fi
 
   setup_output

scripts/run_issue712_deeper_zero_copy_put_with_capture.sh

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+RUNNER_SCRIPT="${PROJECT_ROOT}/scripts/run_put_large_stage_breakdown_with_capture.sh"
+
+ENDPOINT="${ENDPOINT:-http://127.0.0.1:9000}"
+ACCESS_KEY="${ACCESS_KEY:-}"
+SECRET_KEY="${SECRET_KEY:-}"
+REGION="${REGION:-us-east-1}"
+SIZES="${SIZES:-64MiB,128MiB,256MiB}"
+CONCURRENCIES="${CONCURRENCIES:-16}"
+DURATION="${DURATION:-60s}"
+ROUNDS="${ROUNDS:-1}"
+COOLDOWN_SECS="${COOLDOWN_SECS:-15}"
+OUT_DIR="${OUT_DIR:-target/bench/issue712-deeper-zero-copy-capture-$(date -u +%Y%m%dT%H%M%SZ)}"
+CAPTURE_INTERVAL_SECS="${CAPTURE_INTERVAL_SECS:-15}"
+CAPTURE_PROM_METRICS_URLS="${CAPTURE_PROM_METRICS_URLS:-http://127.0.0.1:8889/metrics}"
+CAPTURE_RUSTFS_PID="${CAPTURE_RUSTFS_PID:-}"
+WORKLOAD_LABEL="${WORKLOAD_LABEL:-issue-712-deeper-zero-copy}"
+DRY_RUN=false
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/run_issue712_deeper_zero_copy_put_with_capture.sh \
+    --access-key <ak> --secret-key <sk> [options]
+
+Required:
+  --access-key <ak>
+  --secret-key <sk>
+
+Options:
+  --endpoint <url>                 Default: http://127.0.0.1:9000
+  --region <name>                  Default: us-east-1
+  --sizes <csv>                    Default: 64MiB,128MiB,256MiB
+  --concurrencies <csv>            Default: 16
+  --duration <dur>                 Default: 60s
+  --rounds <n>                     Default: 1
+  --cooldown-secs <n>              Default: 15
+  --out-dir <dir>                  Default: target/bench/issue712-deeper-zero-copy-capture-<timestamp>
+  --capture-interval-secs <n>      Default: 15
+  --capture-prom-metrics-urls <csv>
+                                   Default: http://127.0.0.1:8889/metrics
+  --capture-rustfs-pid <pid>       Optional explicit rustfs pid
+  --dry-run
+  -h, --help
+
+Notes:
+  - This wrapper assumes the local RustFS server is already running with:
+      RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST=true
+  - It reuses scripts/run_put_large_stage_breakdown_with_capture.sh
+    and only narrows the matrix to the deeper-zero-copy focus area.
+USAGE
+}
+
+arg_value() {
+  local flag="$1"
+  local value="${2:-}"
+  if [[ -z "$value" || "$value" == --* ]]; then
+    echo "ERROR: missing value for $flag" >&2
+    exit 1
+  fi
+  printf '%s\n' "$value"
+}
+
+parse_args() {
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --access-key) ACCESS_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --secret-key) SECRET_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --region) REGION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --sizes) SIZES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --concurrencies) CONCURRENCIES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --duration) DURATION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --rounds) ROUNDS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --cooldown-secs) COOLDOWN_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --capture-interval-secs) CAPTURE_INTERVAL_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --capture-prom-metrics-urls) CAPTURE_PROM_METRICS_URLS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --capture-rustfs-pid) CAPTURE_RUSTFS_PID="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --dry-run) DRY_RUN=true; shift ;;
+      -h|--help) usage; exit 0 ;;
+      *)
+        echo "ERROR: unknown arg: $1" >&2
+        usage
+        exit 1
+        ;;
+    esac
+  done
+}
+
+validate_args() {
+  if [[ -z "$ACCESS_KEY" || -z "$SECRET_KEY" ]]; then
+    echo "ERROR: --access-key and --secret-key are required" >&2
+    exit 1
+  fi
+}
+
+main() {
+  parse_args "$@"
+  validate_args
+
+  local -a cmd=(
+    bash "$RUNNER_SCRIPT"
+    --endpoint "$ENDPOINT"
+    --access-key "$ACCESS_KEY"
+    --secret-key "$SECRET_KEY"
+    --region "$REGION"
+    --sizes "$SIZES"
+    --concurrencies "$CONCURRENCIES"
+    --duration "$DURATION"
+    --rounds "$ROUNDS"
+    --retry-per-round 1
+    --retry-sleep-secs 2
+    --cooldown-secs "$COOLDOWN_SECS"
+    --out-dir "$OUT_DIR"
+    --workload-label "$WORKLOAD_LABEL"
+    --capture-label deeper-zero-copy-window
+    --capture-interval-secs "$CAPTURE_INTERVAL_SECS"
+    --capture-prom-metrics-urls "$CAPTURE_PROM_METRICS_URLS"
+  )
+
+  if [[ -n "$CAPTURE_RUSTFS_PID" ]]; then
+    cmd+=(--capture-rustfs-pid "$CAPTURE_RUSTFS_PID")
+  fi
+
+  if [[ "$DRY_RUN" == "true" ]]; then
+    cmd+=(--dry-run)
+  fi
+
+  printf 'Command:'
+  printf ' %q' "${cmd[@]}"
+  printf '\n'
+
+  "${cmd[@]}"
+}
+
+main "$@"