chore(scripts): add manual transition follow-up monitor (#5328)

* test(scripts): cover manual transition runbooks

Add a script-test entry for the manual transition rollout and soak runbook generators. The test covers ratio fail-fast behavior, generated status/metrics/journal checks, and runnable dry-run artifacts.

Verification:

- ./scripts/test_manual_transition_runbooks.sh

- make script-tests

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(scripts): assert nightly stress snapshot hooks

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(scripts): add transition CI follow-up monitor

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>

scripts/README.md

@@ -90,6 +90,7 @@ their issue closes.
 | `manual_transition_journal_audit.sh` | dev-tool | Journal + metrics + log audit for manual transition jobs | — |
 | `manual_transition_mixed_rollout_matrix.sh` | dev-tool | Matrix generator for mixed-version rollout phases | — |
 | `manual_transition_mixed_rollout_runbook.sh` | dev-tool | Reusable mixed-version rollout runbook generator (external run) | — |
+| `monitor_manual_transition_ci.sh` | dev-tool | CI workflow/status watcher for manual transition follow-up monitoring | — |
 | `manual_transition_soak_matrix.sh` | dev-tool | Matrix generator for nightly stress windows | — |
 | `manual_transition_nightly_stress_runbook.sh` | dev-tool | Nightly stress entrypoint with failure snapshot templates | — |
 | `install-flatc.sh` | dev-tool | Local flatc installer (macOS) | — |

scripts/manual_transition_nightly_stress_runbook.sh

@@ -297,6 +297,8 @@ run_entry() {
   local response
   local job_id
   local status
+  local status_json
+  local status_info
   local headers=()
 
   if [[ "$budget_status" == "over-budget" ]]; then
@@ -304,10 +306,6 @@ run_entry() {
     return 0
   fi
 
-  if [[ "$UNKNOWN_FAILURE_COUNT_THRESHOLD" -gt 0 && "$expected_ops" -gt "$UNKNOWN_FAILURE_COUNT_THRESHOLD" ]]; then
-    :
-  fi
-
   prefix="${JOB_PREFIX}/${tag}/${size}/${read_pct}r${write_pct}w"
   query="bucket=$(url_encode "$JOB_BUCKET")"
   query="${query}&prefix=$(url_encode "$prefix")"
@@ -335,10 +333,77 @@ run_entry() {
     return 1
   fi
 
-  status="$(curl -sS "${headers[@]}" -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" | jq -r '.failure_reason // empty')"
-  if [[ -n "$status" && "$status" != "null" ]]; then
-    snapshot_failure "$tag" "failure_reason=${status}" "$job_id"
+  if ! status_json="$(curl -sS "${headers[@]}" -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}")"; then
+    snapshot_failure "$tag" "job_status_fetch_failed" "$job_id"
+    return 1
   fi
+  if ! status_info="$(printf '%s' "$status_json" | jq -r '[.status // "", .failure_reason // "", (.report.enqueued // 0), (.report.transition_completed // 0), (.report.transition_failed // 0), (.report.tier_failure_by_reason["unknown"] // 0), (.queue_snapshot.queued // 0), (.queue_snapshot.active // 0), (.queue_snapshot.compensation_pending // 0), (.queue_snapshot.compensation_running // 0), (.queue_snapshot.queue_full // 0), (.queue_snapshot.queue_send_timeout // 0)] | map(tostring) | join(\"\\u0000\")')"; then
+    snapshot_failure "$tag" "invalid_job_status_json" "$job_id"
+    return 1
+  fi
+  IFS=$'\0' read -r status failure_reason report_enqueued report_completed report_failed report_unknown_failure queue_snapshot_queued queue_snapshot_active compensation_pending compensation_running queue_full queue_send_timeout <<< "$status_info"
+
+  if [[ -n "$failure_reason" && "$failure_reason" != "null" ]]; then
+    snapshot_failure "$tag" "failure_reason=${failure_reason}" "$job_id"
+  fi
+
+  # Threshold checks are applied to terminal job statuses only to avoid transient noise
+  # from active windows that are still processing.
+  if is_terminal_status "$status"; then
+    local mismatch_count
+    local mismatch_ratio
+    local unknown_ratio
+    local ratio_denominator
+    mismatch_count="$((report_enqueued - report_completed - report_failed))"
+    if (( mismatch_count < 0 )); then
+      mismatch_count=0
+    fi
+    mismatch_count="$((queue_snapshot_queued + queue_snapshot_active + compensation_pending + compensation_running + queue_full + queue_send_timeout + mismatch_count))"
+
+    if (( expected_ops > 0 )); then
+      ratio_denominator="$expected_ops"
+    else
+      ratio_denominator="$report_enqueued"
+    fi
+    if (( ratio_denominator <= 0 )); then
+      ratio_denominator=0
+    fi
+
+    if (( ratio_denominator > 0 )); then
+      unknown_ratio="$(awk -v unknown="$report_unknown_failure" -v denom="$ratio_denominator" 'BEGIN{printf "%.6f", (unknown / denom)}')"
+      mismatch_ratio="$(awk -v mismatch="$mismatch_count" -v denom="$ratio_denominator" 'BEGIN{printf "%.6f", (mismatch / denom)}')"
+
+      if (( UNKNOWN_FAILURE_RATIO_THRESHOLD > 0 )) && \
+         awk "BEGIN{exit !($unknown_ratio > ${UNKNOWN_FAILURE_RATIO_THRESHOLD})}"; then
+        snapshot_failure "$tag" "unknown_failure_ratio=${unknown_ratio}/expected=${ratio_denominator}/threshold=${UNKNOWN_FAILURE_RATIO_THRESHOLD}" "$job_id"
+      fi
+
+      if (( UNKNOWN_FAILURE_COUNT_THRESHOLD > 0 )) && (( report_unknown_failure > UNKNOWN_FAILURE_COUNT_THRESHOLD )); then
+        snapshot_failure "$tag" "unknown_failure_count=${report_unknown_failure}/threshold=${UNKNOWN_FAILURE_COUNT_THRESHOLD}" "$job_id"
+      fi
+
+      if (( QUEUE_MISMATCH_RATIO_THRESHOLD > 0 )) && \
+         awk "BEGIN{exit !($mismatch_ratio > ${QUEUE_MISMATCH_RATIO_THRESHOLD})}"; then
+        snapshot_failure "$tag" "queue_mismatch_ratio=${mismatch_ratio}/expected=${ratio_denominator}/threshold=${QUEUE_MISMATCH_RATIO_THRESHOLD}" "$job_id"
+      fi
+    fi
+  fi
+
+  # Keep compatibility with older callers that monitor queue drift via full terminal status.
+  curl -sS "${headers[@]}" -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" | jq '{status, report, queue_snapshot, failure_reason}'
+}
+
+is_terminal_status() {
+  local current_status="$1"
+
+  case "$current_status" in
+    completed|partial|failed|cancelled|unknown)
+      return 0
+      ;;
+    *)
+      return 1
+      ;;
+  esac
 }
 
 main() {

scripts/monitor_manual_transition_ci.sh

@@ -0,0 +1,294 @@
+#!/usr/bin/env bash
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/monitor_manual_transition_ci.sh [options]
+
+Optional:
+  --repo                 Target repo for gh run listing (default: rustfs/rustfs)
+  --workflows            Comma-separated workflow file names (default: ci.yml,e2e-s3tests.yml,e2e-replication-nightly.yml,mint.yml,fuzz.yml)
+  --branch               Branch to check (default: main)
+  --runs                 Number of latest runs to sample per workflow (default: 5)
+  --require-complete      Fail if any sampled run is still queued/in_progress/running
+  --pr                    Check PR checks for a specific PR number in --repo
+  --issues                Comma-separated issue numbers for markdown follow-up snapshot
+  --help
+
+This helper is read-only and does not mutate repository state.
+USAGE
+}
+
+require_cmd() {
+  local cmd="$1"
+  if ! command -v "$cmd" >/dev/null 2>&1; then
+    echo "ERROR: required command not found: $cmd" >&2
+    exit 1
+  fi
+}
+
+arg_value() {
+  local flag="$1"
+  local value="${2:-}"
+  if [[ -z "$value" || "$value" == --* ]]; then
+    echo "ERROR: missing value for $flag" >&2
+    exit 1
+  fi
+  printf '%s' "$value"
+}
+
+normalize_issue_ref() {
+  local ref="$1"
+  local issue_repo="$REPO"
+  local issue_num
+
+  ref="${ref//$'\r'/}"
+  ref="${ref//[[:space:]]/}"
+  if [[ -z "$ref" ]]; then
+    return 1
+  fi
+
+  if [[ "$ref" == */*#* ]]; then
+    issue_repo="${ref%#*}"
+    issue_num="${ref##*#}"
+  elif [[ "$ref" == [#]* ]]; then
+    issue_num="${ref#\#}"
+  else
+    issue_num="$ref"
+  fi
+
+  if ! [[ "$issue_num" =~ ^[0-9]+$ ]]; then
+    return 2
+  fi
+
+  printf '%s;%s' "$issue_repo" "$issue_num"
+}
+
+REPO="rustfs/rustfs"
+WORKFLOWS="ci.yml,e2e-s3tests.yml,e2e-replication-nightly.yml,mint.yml,fuzz.yml"
+BRANCH="main"
+RUNS=5
+REQUIRE_COMPLETE="false"
+PR_NUMBER=""
+ISSUES=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --repo)
+      REPO="$(arg_value "$1" "${2:-}")"
+      shift 2
+      ;;
+    --workflows)
+      WORKFLOWS="$(arg_value "$1" "${2:-}")"
+      shift 2
+      ;;
+    --branch)
+      BRANCH="$(arg_value "$1" "${2:-}")"
+      shift 2
+      ;;
+    --runs)
+      RUNS="$(arg_value "$1" "${2:-}")"
+      shift 2
+      ;;
+    --require-complete)
+      REQUIRE_COMPLETE="true"
+      shift
+      ;;
+    --pr)
+      PR_NUMBER="$(arg_value "$1" "${2:-}")"
+      shift 2
+      ;;
+    --issues)
+      ISSUES="$(arg_value "$1" "${2:-}")"
+      shift 2
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "ERROR: unknown arg: $1" >&2
+      usage
+      exit 1
+      ;;
+  esac
+done
+
+require_cmd gh
+require_cmd jq
+require_cmd date
+
+if ! [[ "${RUNS}" =~ ^[0-9]+$ ]] || (( RUNS == 0 )); then
+  echo "ERROR: --runs must be a positive integer" >&2
+  exit 1
+fi
+
+run_failed=0
+active_found=0
+
+check_workflow() {
+  local workflow="$1"
+  local records
+  local line_count
+
+  if ! records="$(gh run list --repo "$REPO" --workflow "$workflow" --branch "$BRANCH" --limit "$RUNS" --json databaseId,name,status,conclusion,headBranch,headSha,startedAt,updatedAt,url | jq -r '.[] | "\(.name)\t\(.status)\t\(.conclusion // "none")\t\(.headBranch // "")\t\(.headSha // "")\t\(.url)"')"; then
+    echo "ERROR: cannot query workflow=${workflow} in ${REPO}" >&2
+    run_failed=1
+    return
+  fi
+
+  if [[ -z "$records" ]]; then
+    echo "[warn] no runs for workflow=${workflow}" >&2
+    return
+  fi
+
+  line_count=$(printf '%s\n' "$records" | wc -l | tr -d ' ')
+  echo "workflow=${workflow} samples=${line_count}/${RUNS}"
+  while IFS=$'\t' read -r wf status conclusion head_branch head_sha url; do
+    if [[ "$status" != completed ]]; then
+      echo "  - ${status} | ${conclusion} | branch=${head_branch} | sha=${head_sha:0:9}... | ${url}"
+      active_found=1
+    else
+      echo "  - ${status} | ${conclusion} | branch=${head_branch} | sha=${head_sha:0:9}... | ${url}"
+    fi
+
+    if [[ "$status" == "completed" && "$conclusion" != "success" && "$conclusion" != "skipped" && "$conclusion" != "neutral" ]]; then
+      run_failed=1
+    fi
+  done < <(printf '%s\n' "$records")
+
+  if [[ "$active_found" == "1" && "$REQUIRE_COMPLETE" == "true" ]]; then
+    run_failed=1
+  fi
+}
+
+check_pr_checks() {
+  if [[ -z "$PR_NUMBER" ]]; then
+    return
+  fi
+
+  if ! records="$(gh pr checks "$PR_NUMBER" --repo "$REPO" --json name,state,conclusion,detailsUrl 2>/dev/null)"; then
+    echo "ERROR: cannot query PR checks for #${PR_NUMBER} in ${REPO}" >&2
+    run_failed=1
+    return
+  fi
+
+  if [[ "$records" == "[]" ]]; then
+    echo "WARN: PR #${PR_NUMBER} has no recorded checks in ${REPO}"
+    return
+  fi
+
+  local failed_checks
+  failed_checks="$(echo "$records" | jq -r '.[] | select(.state != "completed" or (.conclusion != null and (.conclusion != "success" and .conclusion != "neutral" and .conclusion != "skipped"))) | "\(.name)\t\(.state)\t\(.conclusion // \"none\")\t\(.detailsUrl // \"\")"')"
+
+  echo "pr=${PR_NUMBER} check-runs"
+  if [[ -n "$failed_checks" ]]; then
+    echo "$failed_checks" | while IFS=$'\t' read -r name state conclusion url; do
+      echo "  - ${name}: ${state}/${conclusion} -> ${url}"
+    done
+    run_failed=1
+  else
+    echo "  - all checks passed or pending"
+  fi
+}
+
+check_issues() {
+  local issues_csv="$1"
+  local issue_num
+  local issue_repo
+  local issue_id
+  local resolved
+  local issue_rows
+  local updated
+  local issue_data
+  local state
+  local title
+  local url
+
+  if [[ -z "$issues_csv" ]]; then
+    return
+  fi
+
+  IFS=',' read -r -a issue_arr <<< "$issues_csv"
+  issue_rows=0
+
+  echo "## Manual transition follow-up snapshot"
+  echo "issue_followup_snapshot:"
+  echo "| issue | title | state | updated_at | link |"
+  echo "| --- | --- | --- | --- | --- |"
+
+  for issue_num in "${issue_arr[@]}"; do
+    [[ -z "${issue_num}" ]] && continue
+
+    if ! resolved="$(normalize_issue_ref "${issue_num}")"; then
+      echo "| ${issue_num} | invalid format | unknown | unknown | https://github.com/${REPO}/issues/${issue_num} |"
+      continue
+    fi
+
+    issue_repo="${resolved%;*}"
+    issue_id="${resolved#*;}"
+
+    if ! issue_data="$(gh issue view "$issue_id" --repo "$issue_repo" --json number,title,state,updatedAt,url 2>/dev/null)"; then
+      echo "| #${issue_id} (${issue_repo}) | unavailable | unknown | unavailable | https://github.com/${issue_repo}/issues/${issue_id} |"
+      continue
+    fi
+
+    state="$(echo "$issue_data" | jq -r '.state // "unknown"')"
+    title="$(echo "$issue_data" | jq -r '.title // "unknown"')"
+    updated="$(echo "$issue_data" | jq -r '.updatedAt // "unknown"')"
+    url="$(echo "$issue_data" | jq -r '.url // ""')"
+    if [[ -z "$url" ]]; then
+      url="https://github.com/${issue_repo}/issues/${issue_id}"
+    fi
+    echo "| #${issue_id} (${issue_repo}) | ${title} | ${state} | ${updated} | ${url} |"
+    issue_rows=$((issue_rows + 1))
+  done
+
+  if (( issue_rows == 0 )); then
+    echo "| (none) | — | — | — | — |"
+  fi
+}
+
+IFS=',' read -r -a workflow_arr <<< "$WORKFLOWS"
+for workflow in "${workflow_arr[@]}"; do
+  workflow="${workflow//$'\r'/}"
+  workflow="${workflow//[[:space:]]/}"
+  [[ -z "$workflow" ]] && continue
+  check_workflow "$workflow"
+done
+
+check_pr_checks
+check_issues "$ISSUES"
+
+echo "summary"
+echo "  - failed_runs: $run_failed"
+echo "  - active_runs: $active_found"
+
+echo "  - repo: $REPO"
+echo "  - branch: $BRANCH"
+echo "  - workflows: $WORKFLOWS"
+echo "  - sampled: $RUNS"
+
+timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+echo "  - scanned_at: $timestamp"
+
+if (( run_failed == 1 )); then
+  exit 1
+fi
+
+exit 0

scripts/test_manual_transition_runbooks.sh

@@ -88,5 +88,29 @@ rg -q "UNKNOWN_FAILURE_RATIO_THRESHOLD" "$TMP_DIR/stress-runbook/run_nightly_str
 rg -q "QUEUE_MISMATCH_RATIO_THRESHOLD" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
 rg -q "UNKNOWN_FAILURE_COUNT_THRESHOLD" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
 rg -q "snapshot_failure" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
+rg -q "is_terminal_status" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
+rg -q "status_json" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
+rg -q "unknown_failure_ratio" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
+rg -q "queue_mismatch_ratio" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
+rg -q "report_unknown_failure" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
 rg -q "Manual transition nightly/stress stress-runbook" "$TMP_DIR/stress-runbook/manual_transition_nightly_stress_runbook.md"
 rg -q "SOAK_MATRIX_CSV='$TMP_DIR/stress-runbook/nightly_soak_matrix.csv'" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
+
+bash scripts/monitor_manual_transition_ci.sh --help | rg -q "Usage:"
+if bash scripts/monitor_manual_transition_ci.sh --issues >/tmp/monitor_manual_transition_ci.err 2>&1; then
+  echo "monitor script should fail when --issues has no value" >&2
+  exit 1
+fi
+if ! rg -q "ERROR: missing value for --issues" /tmp/monitor_manual_transition_ci.err; then
+  echo "monitor script missing missing-value guard for --issues" >&2
+  exit 1
+fi
+
+if bash scripts/monitor_manual_transition_ci.sh --runs 0 >/tmp/monitor_manual_transition_ci.err 2>&1; then
+  echo "monitor script should fail on invalid --runs" >&2
+  exit 1
+fi
+if ! rg -q "ERROR: --runs must be a positive integer" /tmp/monitor_manual_transition_ci.err; then
+  echo "monitor script missing invalid --runs guard output" >&2
+  exit 1
+fi