From 464bf45e1507fdae4140a17be9a4014b57f08990 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 27 Jul 2026 16:36:41 +0800 Subject: [PATCH] chore(scripts): add manual transition follow-up monitor (#5328) * test(scripts): cover manual transition runbooks Add a script-test entry for the manual transition rollout and soak runbook generators. The test covers ratio fail-fast behavior, generated status/metrics/journal checks, and runnable dry-run artifacts. Verification: - ./scripts/test_manual_transition_runbooks.sh - make script-tests Co-Authored-By: heihutu * test(scripts): assert nightly stress snapshot hooks Co-Authored-By: heihutu * feat(scripts): add transition CI follow-up monitor Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- scripts/README.md | 1 + ...anual_transition_nightly_stress_runbook.sh | 79 ++++- scripts/monitor_manual_transition_ci.sh | 294 ++++++++++++++++++ scripts/test_manual_transition_runbooks.sh | 24 ++ 4 files changed, 391 insertions(+), 7 deletions(-) create mode 100755 scripts/monitor_manual_transition_ci.sh diff --git a/scripts/README.md b/scripts/README.md index 4fe6207fd..61a6475c9 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -90,6 +90,7 @@ their issue closes. | `manual_transition_journal_audit.sh` | dev-tool | Journal + metrics + log audit for manual transition jobs | — | | `manual_transition_mixed_rollout_matrix.sh` | dev-tool | Matrix generator for mixed-version rollout phases | — | | `manual_transition_mixed_rollout_runbook.sh` | dev-tool | Reusable mixed-version rollout runbook generator (external run) | — | +| `monitor_manual_transition_ci.sh` | dev-tool | CI workflow/status watcher for manual transition follow-up monitoring | — | | `manual_transition_soak_matrix.sh` | dev-tool | Matrix generator for nightly stress windows | — | | `manual_transition_nightly_stress_runbook.sh` | dev-tool | Nightly stress entrypoint with failure snapshot templates | — | | `install-flatc.sh` | dev-tool | Local flatc installer (macOS) | — | diff --git a/scripts/manual_transition_nightly_stress_runbook.sh b/scripts/manual_transition_nightly_stress_runbook.sh index 13f79a5ac..916a0f297 100755 --- a/scripts/manual_transition_nightly_stress_runbook.sh +++ b/scripts/manual_transition_nightly_stress_runbook.sh @@ -297,6 +297,8 @@ run_entry() { local response local job_id local status + local status_json + local status_info local headers=() if [[ "$budget_status" == "over-budget" ]]; then @@ -304,10 +306,6 @@ run_entry() { return 0 fi - if [[ "$UNKNOWN_FAILURE_COUNT_THRESHOLD" -gt 0 && "$expected_ops" -gt "$UNKNOWN_FAILURE_COUNT_THRESHOLD" ]]; then - : - fi - prefix="${JOB_PREFIX}/${tag}/${size}/${read_pct}r${write_pct}w" query="bucket=$(url_encode "$JOB_BUCKET")" query="${query}&prefix=$(url_encode "$prefix")" @@ -335,10 +333,77 @@ run_entry() { return 1 fi - status="$(curl -sS "${headers[@]}" -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" | jq -r '.failure_reason // empty')" - if [[ -n "$status" && "$status" != "null" ]]; then - snapshot_failure "$tag" "failure_reason=${status}" "$job_id" + if ! status_json="$(curl -sS "${headers[@]}" -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}")"; then + snapshot_failure "$tag" "job_status_fetch_failed" "$job_id" + return 1 fi + if ! status_info="$(printf '%s' "$status_json" | jq -r '[.status // "", .failure_reason // "", (.report.enqueued // 0), (.report.transition_completed // 0), (.report.transition_failed // 0), (.report.tier_failure_by_reason["unknown"] // 0), (.queue_snapshot.queued // 0), (.queue_snapshot.active // 0), (.queue_snapshot.compensation_pending // 0), (.queue_snapshot.compensation_running // 0), (.queue_snapshot.queue_full // 0), (.queue_snapshot.queue_send_timeout // 0)] | map(tostring) | join(\"\\u0000\")')"; then + snapshot_failure "$tag" "invalid_job_status_json" "$job_id" + return 1 + fi + IFS=$'\0' read -r status failure_reason report_enqueued report_completed report_failed report_unknown_failure queue_snapshot_queued queue_snapshot_active compensation_pending compensation_running queue_full queue_send_timeout <<< "$status_info" + + if [[ -n "$failure_reason" && "$failure_reason" != "null" ]]; then + snapshot_failure "$tag" "failure_reason=${failure_reason}" "$job_id" + fi + + # Threshold checks are applied to terminal job statuses only to avoid transient noise + # from active windows that are still processing. + if is_terminal_status "$status"; then + local mismatch_count + local mismatch_ratio + local unknown_ratio + local ratio_denominator + mismatch_count="$((report_enqueued - report_completed - report_failed))" + if (( mismatch_count < 0 )); then + mismatch_count=0 + fi + mismatch_count="$((queue_snapshot_queued + queue_snapshot_active + compensation_pending + compensation_running + queue_full + queue_send_timeout + mismatch_count))" + + if (( expected_ops > 0 )); then + ratio_denominator="$expected_ops" + else + ratio_denominator="$report_enqueued" + fi + if (( ratio_denominator <= 0 )); then + ratio_denominator=0 + fi + + if (( ratio_denominator > 0 )); then + unknown_ratio="$(awk -v unknown="$report_unknown_failure" -v denom="$ratio_denominator" 'BEGIN{printf "%.6f", (unknown / denom)}')" + mismatch_ratio="$(awk -v mismatch="$mismatch_count" -v denom="$ratio_denominator" 'BEGIN{printf "%.6f", (mismatch / denom)}')" + + if (( UNKNOWN_FAILURE_RATIO_THRESHOLD > 0 )) && \ + awk "BEGIN{exit !($unknown_ratio > ${UNKNOWN_FAILURE_RATIO_THRESHOLD})}"; then + snapshot_failure "$tag" "unknown_failure_ratio=${unknown_ratio}/expected=${ratio_denominator}/threshold=${UNKNOWN_FAILURE_RATIO_THRESHOLD}" "$job_id" + fi + + if (( UNKNOWN_FAILURE_COUNT_THRESHOLD > 0 )) && (( report_unknown_failure > UNKNOWN_FAILURE_COUNT_THRESHOLD )); then + snapshot_failure "$tag" "unknown_failure_count=${report_unknown_failure}/threshold=${UNKNOWN_FAILURE_COUNT_THRESHOLD}" "$job_id" + fi + + if (( QUEUE_MISMATCH_RATIO_THRESHOLD > 0 )) && \ + awk "BEGIN{exit !($mismatch_ratio > ${QUEUE_MISMATCH_RATIO_THRESHOLD})}"; then + snapshot_failure "$tag" "queue_mismatch_ratio=${mismatch_ratio}/expected=${ratio_denominator}/threshold=${QUEUE_MISMATCH_RATIO_THRESHOLD}" "$job_id" + fi + fi + fi + + # Keep compatibility with older callers that monitor queue drift via full terminal status. + curl -sS "${headers[@]}" -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" | jq '{status, report, queue_snapshot, failure_reason}' +} + +is_terminal_status() { + local current_status="$1" + + case "$current_status" in + completed|partial|failed|cancelled|unknown) + return 0 + ;; + *) + return 1 + ;; + esac } main() { diff --git a/scripts/monitor_manual_transition_ci.sh b/scripts/monitor_manual_transition_ci.sh new file mode 100755 index 000000000..86d0d2ae7 --- /dev/null +++ b/scripts/monitor_manual_transition_ci.sh @@ -0,0 +1,294 @@ +#!/usr/bin/env bash +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: + scripts/monitor_manual_transition_ci.sh [options] + +Optional: + --repo Target repo for gh run listing (default: rustfs/rustfs) + --workflows Comma-separated workflow file names (default: ci.yml,e2e-s3tests.yml,e2e-replication-nightly.yml,mint.yml,fuzz.yml) + --branch Branch to check (default: main) + --runs Number of latest runs to sample per workflow (default: 5) + --require-complete Fail if any sampled run is still queued/in_progress/running + --pr Check PR checks for a specific PR number in --repo + --issues Comma-separated issue numbers for markdown follow-up snapshot + --help + +This helper is read-only and does not mutate repository state. +USAGE +} + +require_cmd() { + local cmd="$1" + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "ERROR: required command not found: $cmd" >&2 + exit 1 + fi +} + +arg_value() { + local flag="$1" + local value="${2:-}" + if [[ -z "$value" || "$value" == --* ]]; then + echo "ERROR: missing value for $flag" >&2 + exit 1 + fi + printf '%s' "$value" +} + +normalize_issue_ref() { + local ref="$1" + local issue_repo="$REPO" + local issue_num + + ref="${ref//$'\r'/}" + ref="${ref//[[:space:]]/}" + if [[ -z "$ref" ]]; then + return 1 + fi + + if [[ "$ref" == */*#* ]]; then + issue_repo="${ref%#*}" + issue_num="${ref##*#}" + elif [[ "$ref" == [#]* ]]; then + issue_num="${ref#\#}" + else + issue_num="$ref" + fi + + if ! [[ "$issue_num" =~ ^[0-9]+$ ]]; then + return 2 + fi + + printf '%s;%s' "$issue_repo" "$issue_num" +} + +REPO="rustfs/rustfs" +WORKFLOWS="ci.yml,e2e-s3tests.yml,e2e-replication-nightly.yml,mint.yml,fuzz.yml" +BRANCH="main" +RUNS=5 +REQUIRE_COMPLETE="false" +PR_NUMBER="" +ISSUES="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --repo) + REPO="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --workflows) + WORKFLOWS="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --branch) + BRANCH="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --runs) + RUNS="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --require-complete) + REQUIRE_COMPLETE="true" + shift + ;; + --pr) + PR_NUMBER="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --issues) + ISSUES="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "ERROR: unknown arg: $1" >&2 + usage + exit 1 + ;; + esac +done + +require_cmd gh +require_cmd jq +require_cmd date + +if ! [[ "${RUNS}" =~ ^[0-9]+$ ]] || (( RUNS == 0 )); then + echo "ERROR: --runs must be a positive integer" >&2 + exit 1 +fi + +run_failed=0 +active_found=0 + +check_workflow() { + local workflow="$1" + local records + local line_count + + if ! records="$(gh run list --repo "$REPO" --workflow "$workflow" --branch "$BRANCH" --limit "$RUNS" --json databaseId,name,status,conclusion,headBranch,headSha,startedAt,updatedAt,url | jq -r '.[] | "\(.name)\t\(.status)\t\(.conclusion // "none")\t\(.headBranch // "")\t\(.headSha // "")\t\(.url)"')"; then + echo "ERROR: cannot query workflow=${workflow} in ${REPO}" >&2 + run_failed=1 + return + fi + + if [[ -z "$records" ]]; then + echo "[warn] no runs for workflow=${workflow}" >&2 + return + fi + + line_count=$(printf '%s\n' "$records" | wc -l | tr -d ' ') + echo "workflow=${workflow} samples=${line_count}/${RUNS}" + while IFS=$'\t' read -r wf status conclusion head_branch head_sha url; do + if [[ "$status" != completed ]]; then + echo " - ${status} | ${conclusion} | branch=${head_branch} | sha=${head_sha:0:9}... | ${url}" + active_found=1 + else + echo " - ${status} | ${conclusion} | branch=${head_branch} | sha=${head_sha:0:9}... | ${url}" + fi + + if [[ "$status" == "completed" && "$conclusion" != "success" && "$conclusion" != "skipped" && "$conclusion" != "neutral" ]]; then + run_failed=1 + fi + done < <(printf '%s\n' "$records") + + if [[ "$active_found" == "1" && "$REQUIRE_COMPLETE" == "true" ]]; then + run_failed=1 + fi +} + +check_pr_checks() { + if [[ -z "$PR_NUMBER" ]]; then + return + fi + + if ! records="$(gh pr checks "$PR_NUMBER" --repo "$REPO" --json name,state,conclusion,detailsUrl 2>/dev/null)"; then + echo "ERROR: cannot query PR checks for #${PR_NUMBER} in ${REPO}" >&2 + run_failed=1 + return + fi + + if [[ "$records" == "[]" ]]; then + echo "WARN: PR #${PR_NUMBER} has no recorded checks in ${REPO}" + return + fi + + local failed_checks + failed_checks="$(echo "$records" | jq -r '.[] | select(.state != "completed" or (.conclusion != null and (.conclusion != "success" and .conclusion != "neutral" and .conclusion != "skipped"))) | "\(.name)\t\(.state)\t\(.conclusion // \"none\")\t\(.detailsUrl // \"\")"')" + + echo "pr=${PR_NUMBER} check-runs" + if [[ -n "$failed_checks" ]]; then + echo "$failed_checks" | while IFS=$'\t' read -r name state conclusion url; do + echo " - ${name}: ${state}/${conclusion} -> ${url}" + done + run_failed=1 + else + echo " - all checks passed or pending" + fi +} + +check_issues() { + local issues_csv="$1" + local issue_num + local issue_repo + local issue_id + local resolved + local issue_rows + local updated + local issue_data + local state + local title + local url + + if [[ -z "$issues_csv" ]]; then + return + fi + + IFS=',' read -r -a issue_arr <<< "$issues_csv" + issue_rows=0 + + echo "## Manual transition follow-up snapshot" + echo "issue_followup_snapshot:" + echo "| issue | title | state | updated_at | link |" + echo "| --- | --- | --- | --- | --- |" + + for issue_num in "${issue_arr[@]}"; do + [[ -z "${issue_num}" ]] && continue + + if ! resolved="$(normalize_issue_ref "${issue_num}")"; then + echo "| ${issue_num} | invalid format | unknown | unknown | https://github.com/${REPO}/issues/${issue_num} |" + continue + fi + + issue_repo="${resolved%;*}" + issue_id="${resolved#*;}" + + if ! issue_data="$(gh issue view "$issue_id" --repo "$issue_repo" --json number,title,state,updatedAt,url 2>/dev/null)"; then + echo "| #${issue_id} (${issue_repo}) | unavailable | unknown | unavailable | https://github.com/${issue_repo}/issues/${issue_id} |" + continue + fi + + state="$(echo "$issue_data" | jq -r '.state // "unknown"')" + title="$(echo "$issue_data" | jq -r '.title // "unknown"')" + updated="$(echo "$issue_data" | jq -r '.updatedAt // "unknown"')" + url="$(echo "$issue_data" | jq -r '.url // ""')" + if [[ -z "$url" ]]; then + url="https://github.com/${issue_repo}/issues/${issue_id}" + fi + echo "| #${issue_id} (${issue_repo}) | ${title} | ${state} | ${updated} | ${url} |" + issue_rows=$((issue_rows + 1)) + done + + if (( issue_rows == 0 )); then + echo "| (none) | — | — | — | — |" + fi +} + +IFS=',' read -r -a workflow_arr <<< "$WORKFLOWS" +for workflow in "${workflow_arr[@]}"; do + workflow="${workflow//$'\r'/}" + workflow="${workflow//[[:space:]]/}" + [[ -z "$workflow" ]] && continue + check_workflow "$workflow" +done + +check_pr_checks +check_issues "$ISSUES" + +echo "summary" +echo " - failed_runs: $run_failed" +echo " - active_runs: $active_found" + +echo " - repo: $REPO" +echo " - branch: $BRANCH" +echo " - workflows: $WORKFLOWS" +echo " - sampled: $RUNS" + +timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ) +echo " - scanned_at: $timestamp" + +if (( run_failed == 1 )); then + exit 1 +fi + +exit 0 diff --git a/scripts/test_manual_transition_runbooks.sh b/scripts/test_manual_transition_runbooks.sh index 766be4687..67806a706 100755 --- a/scripts/test_manual_transition_runbooks.sh +++ b/scripts/test_manual_transition_runbooks.sh @@ -88,5 +88,29 @@ rg -q "UNKNOWN_FAILURE_RATIO_THRESHOLD" "$TMP_DIR/stress-runbook/run_nightly_str rg -q "QUEUE_MISMATCH_RATIO_THRESHOLD" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "UNKNOWN_FAILURE_COUNT_THRESHOLD" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "snapshot_failure" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q "is_terminal_status" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q "status_json" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q "unknown_failure_ratio" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q "queue_mismatch_ratio" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q "report_unknown_failure" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "Manual transition nightly/stress stress-runbook" "$TMP_DIR/stress-runbook/manual_transition_nightly_stress_runbook.md" rg -q "SOAK_MATRIX_CSV='$TMP_DIR/stress-runbook/nightly_soak_matrix.csv'" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" + +bash scripts/monitor_manual_transition_ci.sh --help | rg -q "Usage:" +if bash scripts/monitor_manual_transition_ci.sh --issues >/tmp/monitor_manual_transition_ci.err 2>&1; then + echo "monitor script should fail when --issues has no value" >&2 + exit 1 +fi +if ! rg -q "ERROR: missing value for --issues" /tmp/monitor_manual_transition_ci.err; then + echo "monitor script missing missing-value guard for --issues" >&2 + exit 1 +fi + +if bash scripts/monitor_manual_transition_ci.sh --runs 0 >/tmp/monitor_manual_transition_ci.err 2>&1; then + echo "monitor script should fail on invalid --runs" >&2 + exit 1 +fi +if ! rg -q "ERROR: --runs must be a positive integer" /tmp/monitor_manual_transition_ci.err; then + echo "monitor script missing invalid --runs guard output" >&2 + exit 1 +fi