From d887e7e31ded13c0f7a452023d59a9f90f710388 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 26 Jul 2026 11:58:47 +0800 Subject: [PATCH] test(internode): pin msgpack rollout gates (#5261) Extend the internode gRPC benchmark driver with P2 request-only, canary, after, and rollback phases. The request-only phase proves that requesting msgpack-only without fleet confirmation keeps compatibility JSON fields, while canary and rollback materialize the operator states needed for mixed-version convergence without claiming real fleet soak evidence. Verification: - scripts/test_internode_grpc_ab_bench.sh - bash -n scripts/run_internode_grpc_ab_bench.sh scripts/test_internode_grpc_ab_bench.sh - git diff --check Co-authored-by: heihutu --- .../internode-grpc-benchmark-runbook.md | 11 +++++--- ...ernode-msgpack-json-convergence-runbook.md | 15 +++++++++-- scripts/run_internode_grpc_ab_bench.sh | 25 +++++++++++++++---- scripts/test_internode_grpc_ab_bench.sh | 23 +++++++++++++++++ 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/docs/operations/internode-grpc-benchmark-runbook.md b/docs/operations/internode-grpc-benchmark-runbook.md index 298863342..d73ea1230 100644 --- a/docs/operations/internode-grpc-benchmark-runbook.md +++ b/docs/operations/internode-grpc-benchmark-runbook.md @@ -10,7 +10,7 @@ optimization stage (grpc-optimization P0–P3). Every stage is env-gated, so "be ## One-click driver -`scripts/run_internode_grpc_ab_bench.sh --stage --phase [-- ]` +`scripts/run_internode_grpc_ab_bench.sh --stage --phase [-- ]` wraps the env matrix below: it writes the stage/phase **server** env to `/server-env.sh`, then runs the right underlying bench into `target/bench/internode-transport/-/`. @@ -21,11 +21,13 @@ scripts/run_internode_grpc_ab_bench.sh --stage p1 --phase before -- --access-key scripts/run_internode_grpc_ab_bench.sh --stage p1 --phase after -- --access-key AK --secret-key SK --metrics-url http://node1:9000/metrics # P3 failover A/B (docker four-node): scripts/run_internode_grpc_ab_bench.sh --stage p3 --phase after +# P2 rollout gates (env preview for request-only rehearsal, canary, and rollback): +scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase request-only --dry-run +scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase canary --dry-run +scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase rollback --dry-run ``` -`RUSTFS_INTERNODE_*` are **server** env: for the load-driven stages (p0/p1/p2) source the emitted -`server-env.sh` on every node and restart rustfs *before* the run — the driver cannot mutate an -already-running server. Use `--dry-run` to preview the env and command. +`RUSTFS_INTERNODE_*` are **server** env: for the load-driven stages (p0/p1/p2) source the emitted `server-env.sh` on every node and restart rustfs *before* the run — the driver cannot mutate an already-running server. The P2 `canary` phase is the exception: source it only on the selected canary node after the release-window counters and fleet-support checks pass, and keep the rest of the fleet on `before` or `request-only` while observing fallback/decode-error counters. Use `--dry-run` to preview the env and command. ## Harness @@ -115,6 +117,7 @@ Artifact layout per stage (attach both halves + the diff): target/bench/internode-transport/ p0-before/ p0-after/ # server-env.sh + object-bench summaries + metric deltas p1-before/ p1-after/ # + lock p99 delta (the ≥20% gate) + p2-before/ p2-request-only/ p2-canary/ p2-after/ p2-rollback/ # env gate artifacts + fallback/decode-error observations p3-before/ p3-after/ # cold-start + sustained-offline experiment + offline gauge trace ``` diff --git a/docs/operations/internode-msgpack-json-convergence-runbook.md b/docs/operations/internode-msgpack-json-convergence-runbook.md index 67e5dbca2..461b10ef5 100644 --- a/docs/operations/internode-msgpack-json-convergence-runbook.md +++ b/docs/operations/internode-msgpack-json-convergence-runbook.md @@ -135,10 +135,21 @@ The first flag only requests msgpack-only. The second flag is the explicit proof Only enable it **after** Stage 0 has read zero for a full window across the fleet: 1. Ship with the flag **off** (no behavior change). -2. Enable it on one node with both `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` and `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=true`, then restart and watch the fallback counter for a soak period. If it stays zero, enable fleet-wide. -3. **Rollback:** set either `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=false` or `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=false` (or unset either flag) and restart. No +2. Rehearse the `request-only` gate with `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` and `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=false`. This must keep writing JSON compatibility fields and is safe with unsupported/old peers; any fallback or decode-error increment still blocks convergence. +3. Enable it on one canary node with both `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` and `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=true`, then restart that node only and watch the fallback and decode-error counters for a soak period while real internode traffic is present. If they stay zero, enable fleet-wide. +4. **Rollback:** set either `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=false` or `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=false` (or unset either flag) and restart. No wire-format was broken in this stage, so rollback is immediate and safe. +The benchmark helper pins these operator states as dry-run phases: + +```bash +scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase before --dry-run +scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase request-only --dry-run +scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase canary --dry-run +scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase after --dry-run +scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase rollback --dry-run +``` + ## Stage 2 — Remove the proto JSON fields (next release, N+1) Only after Stage 1 has been stable with the flag on for a full window and the counter is diff --git a/scripts/run_internode_grpc_ab_bench.sh b/scripts/run_internode_grpc_ab_bench.sh index f3acce85f..3ded27b15 100755 --- a/scripts/run_internode_grpc_ab_bench.sh +++ b/scripts/run_internode_grpc_ab_bench.sh @@ -29,11 +29,11 @@ PASSTHROUGH=() usage() { cat <<'USAGE' Usage: - scripts/run_internode_grpc_ab_bench.sh --stage --phase [options] [-- ] + scripts/run_internode_grpc_ab_bench.sh --stage --phase [options] [-- ] Required: --stage Which optimization stage to A/B. - --phase baseline (feature off) or enabled (feature on). + --phase baseline or enabled phase. p0/p1/p3 accept before|after; p2 also accepts request-only|canary|rollback. Options: --out-root Artifact root. Default: target/bench/internode-transport @@ -67,7 +67,13 @@ while [[ $# -gt 0 ]]; do done [[ -n "${STAGE}" && -n "${PHASE}" ]] || { echo "ERROR: --stage and --phase are required" >&2; usage; exit 2; } -case "${PHASE}" in before|after) ;; *) echo "ERROR: --phase must be before|after" >&2; exit 2 ;; esac +case "${PHASE}" in before|after|request-only|canary|rollback) ;; *) echo "ERROR: --phase must be before, after, request-only, canary, or rollback" >&2; exit 2 ;; esac +if [[ "${STAGE}" != "p2" ]]; then + case "${PHASE}" in + before|after) ;; + *) echo "ERROR: --phase ${PHASE} is only valid for --stage p2" >&2; exit 2 ;; + esac +fi # Emit the RUSTFS_INTERNODE_* env for the stage/phase as `KEY=VALUE` lines. Empty output means # "cluster defaults" (nothing to set). "before" = feature disabled, "after" = feature enabled. @@ -94,12 +100,17 @@ stage_env() { fi ;; p2) - if [[ "${phase}" == "after" ]]; then + if [[ "${phase}" == "after" || "${phase}" == "canary" ]]; then # Only enable after the msgpack json-fallback counter has read zero across a window and # the fleet confirmation gate has passed. printf '%s\n' \ 'RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true' \ 'RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=true' + elif [[ "${phase}" == "request-only" ]]; then + # Requesting msgpack-only without fleet confirmation must keep the compatibility JSON fields. + printf '%s\n' \ + 'RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true' \ + 'RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=false' else printf '%s\n' \ 'RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=false' \ @@ -130,7 +141,11 @@ ENV_FILE="${OUT_DIR}/server-env.sh" { echo "# RustFS server env for internode stage ${STAGE} / phase ${PHASE}." - echo "# Source this on EVERY rustfs node before starting the server, then run the bench." + if [[ "${STAGE}" == "p2" && "${PHASE}" == "canary" ]]; then + echo "# Source this only on the selected canary RustFS node before starting that node; keep the rest on p2/before or p2/request-only env." + else + echo "# Source this on EVERY rustfs node before starting the server, then run the bench." + fi if [[ -n "${ENV_LINES}" ]]; then while IFS= read -r line; do echo "export ${line}"; done <<<"${ENV_LINES}" else diff --git a/scripts/test_internode_grpc_ab_bench.sh b/scripts/test_internode_grpc_ab_bench.sh index ce25673f7..4e8fc0238 100755 --- a/scripts/test_internode_grpc_ab_bench.sh +++ b/scripts/test_internode_grpc_ab_bench.sh @@ -11,10 +11,16 @@ cleanup() { trap cleanup EXIT "$RUNNER" --stage p2 --phase before --out-root "$TMP_DIR" --dry-run >/dev/null +"$RUNNER" --stage p2 --phase request-only --out-root "$TMP_DIR" --dry-run >/dev/null +"$RUNNER" --stage p2 --phase canary --out-root "$TMP_DIR" --dry-run >/dev/null "$RUNNER" --stage p2 --phase after --out-root "$TMP_DIR" --dry-run >/dev/null +"$RUNNER" --stage p2 --phase rollback --out-root "$TMP_DIR" --dry-run >/dev/null BEFORE_ENV="${TMP_DIR}/p2-before/server-env.sh" +REQUEST_ONLY_ENV="${TMP_DIR}/p2-request-only/server-env.sh" +CANARY_ENV="${TMP_DIR}/p2-canary/server-env.sh" AFTER_ENV="${TMP_DIR}/p2-after/server-env.sh" +ROLLBACK_ENV="${TMP_DIR}/p2-rollback/server-env.sh" P0_BEFORE_ENV="${TMP_DIR}/p0-before/server-env.sh" P0_AFTER_ENV="${TMP_DIR}/p0-after/server-env.sh" P1_BEFORE_ENV="${TMP_DIR}/p1-before/server-env.sh" @@ -24,8 +30,15 @@ P3_AFTER_ENV="${TMP_DIR}/p3-after/server-env.sh" rg -qx 'export RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=false' "$BEFORE_ENV" rg -qx 'export RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=false' "$BEFORE_ENV" +rg -qx 'export RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true' "$REQUEST_ONLY_ENV" +rg -qx 'export RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=false' "$REQUEST_ONLY_ENV" +rg -qx 'export RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true' "$CANARY_ENV" +rg -qx 'export RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=true' "$CANARY_ENV" +rg -q 'selected canary RustFS node' "$CANARY_ENV" rg -qx 'export RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true' "$AFTER_ENV" rg -qx 'export RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=true' "$AFTER_ENV" +rg -qx 'export RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=false' "$ROLLBACK_ENV" +rg -qx 'export RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=false' "$ROLLBACK_ENV" "$RUNNER" --stage p0 --phase before --out-root "$TMP_DIR" --dry-run >/dev/null "$RUNNER" --stage p0 --phase after --out-root "$TMP_DIR" --dry-run >/dev/null "$RUNNER" --stage p1 --phase before --out-root "$TMP_DIR" --dry-run >/dev/null @@ -43,6 +56,16 @@ if grep -q 'RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=true' "$BEFORE_ENV exit 1 fi +if grep -q 'RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=true' "$REQUEST_ONLY_ENV"; then + echo "p2 request-only must not confirm fleet msgpack-only" >&2 + exit 1 +fi + +if "$RUNNER" --stage p1 --phase canary --out-root "$TMP_DIR" --dry-run >/dev/null 2>&1; then + echo "expected canary phase to be p2-only" >&2 + exit 1 +fi + rg -qx 'export RUSTFS_INTERNODE_RPC_TCP_NODELAY=false' "$P0_BEFORE_ENV" rg -qx 'export RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE=0' "$P0_BEFORE_ENV" rg -qx 'export RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE=0' "$P0_BEFORE_ENV"