From e076e8cc6e873ac52650a1f6e57d63a12c867614 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 27 Jul 2026 14:47:28 +0800 Subject: [PATCH] feat(scripts): add runbook entrypoints for #1508 #1510 (#5326) * feat(ecstore): attribute manual transition worker failures Add manual transition worker failure reason tracking and persistence recovery compatibility for checksum validation. Co-Authored-By: heihutu * chore(ilm): add manual transition diagnostics scripts Co-Authored-By: heihutu * style(ecstore): format manual transition attribution Co-Authored-By: heihutu * fix(ecstore): avoid copying failure reasons via clone Co-Authored-By: heihutu * fix(manual-transition): improve matrix verification scripts - Add strict mixed rollout phase ratio validation. - Add strict read/write ratio validation for soak mix. - Inline generated admin-check command blocks into mixed-rollout run script. Co-Authored-By: heihutu * feat(scripts): add runbook entrypoints for #1508 #1510 --------- Co-authored-by: heihutu --- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 45 ++ .../bucket/lifecycle/manual_transition_job.rs | 4 +- scripts/README.md | 6 + .../manual_transition_mixed_rollout_matrix.sh | 5 + ...manual_transition_mixed_rollout_runbook.sh | 402 ++++++++++++++++++ ...anual_transition_nightly_stress_runbook.sh | 398 +++++++++++++++++ scripts/manual_transition_soak_matrix.sh | 4 + 7 files changed, 863 insertions(+), 1 deletion(-) create mode 100755 scripts/manual_transition_mixed_rollout_runbook.sh create mode 100755 scripts/manual_transition_nightly_stress_runbook.sh diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 220052584..144336b40 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -9376,6 +9376,9 @@ mod tests { ..Default::default() }; let mut stale = ManualTransitionJobRecord::new(Uuid::new_v4(), &bucket, &options, "old-owner"); + if stale_case == "missing-job" { + stale.lease_expires_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos() - 1; + } if stale_case == "terminal-job" { stale.complete( ManualTransitionRunReport { @@ -9412,6 +9415,48 @@ mod tests { } } + #[tokio::test] + #[serial] + async fn manual_transition_admission_concurrent_same_scope_writes_is_singleton() { + let (_paths, ecstore) = setup_test_env().await; + let options = ManualTransitionRunOptions { + prefix: "logs/".to_string(), + tier: Some("warm".to_string()), + ..Default::default() + }; + let first = ManualTransitionJobRecord::new(Uuid::new_v4(), "manual-concurrent-scope-bucket", &options, "owner-a"); + let first_admission = ManualTransitionScopeAdmission::from_job(&first); + let second = ManualTransitionJobRecord::new(Uuid::new_v4(), "manual-concurrent-scope-bucket", &options, "owner-b"); + let second_admission = ManualTransitionScopeAdmission::from_job(&second); + + let first_claim = claim_manual_transition_scope_admission(ecstore.clone(), &first_admission); + let second_claim = claim_manual_transition_scope_admission(ecstore.clone(), &second_admission); + let (first_result, second_result) = tokio::join!(first_claim, second_claim); + let first_claim = first_result.expect("first concurrent claim should resolve"); + let second_claim = second_result.expect("second concurrent claim should resolve"); + + let mut claimed = 0; + let mut conflicted = 0; + let mut active_job_id = None; + for item in [first_claim, second_claim] { + match item { + ManualTransitionScopeAdmissionClaim::Claimed => claimed += 1, + ManualTransitionScopeAdmissionClaim::Conflict(active) => { + conflicted += 1; + active_job_id = Some(active.job_id); + } + } + } + + assert_eq!(claimed, 1, "only one concurrent same-scope claim should be accepted"); + assert_eq!(conflicted, 1, "only one concurrent same-scope claim should report conflict"); + let active = load_manual_transition_scope_admission(ecstore.clone(), &first.scope_key) + .await + .expect("scope admission should remain"); + assert!(active.job_id == first.job_id || active.job_id == second.job_id); + assert_eq!(active_job_id, Some(active.job_id), "conflict response must carry active owner"); + } + #[tokio::test] async fn existing_object_lifecycle_allows_expired_marker_after_replication_completed() { let lc = expired_delete_marker_lifecycle(); diff --git a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs index 7042d2224..006ddd7c2 100644 --- a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs +++ b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs @@ -1405,7 +1405,9 @@ pub async fn claim_manual_transition_scope_admission( Ok(active_job) => { active_job.is_terminal() || (scope_lease_expired && manual_transition_job_lease_expired(&active_job)) } - Err(Error::ConfigNotFound) => true, + // Missing active job metadata can be transient (for example, immediately after admission creation); + // require an expired scope lease before treating it as reclaimable. + Err(Error::ConfigNotFound) => scope_lease_expired, Err(err) => return Err(err), } }; diff --git a/scripts/README.md b/scripts/README.md index 4fc034bbb..4fe6207fd 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -86,6 +86,12 @@ their issue closes. | `restart_local_single_node_multidisk_rustfs.sh` | dev-tool | Restart a local single-node multi-disk instance | — | | `inspect_dashboard.sh` | dev-tool | Sanity-checks the Grafana dashboard JSON | `.docker/observability` | | `notify.sh` | dev-tool | Starts a local webhook receiver for notify-target development | — | +| `manual_transition_debug.sh` | dev-tool | Log/metrics helper for manual transition troubleshooting | — | +| `manual_transition_journal_audit.sh` | dev-tool | Journal + metrics + log audit for manual transition jobs | — | +| `manual_transition_mixed_rollout_matrix.sh` | dev-tool | Matrix generator for mixed-version rollout phases | — | +| `manual_transition_mixed_rollout_runbook.sh` | dev-tool | Reusable mixed-version rollout runbook generator (external run) | — | +| `manual_transition_soak_matrix.sh` | dev-tool | Matrix generator for nightly stress windows | — | +| `manual_transition_nightly_stress_runbook.sh` | dev-tool | Nightly stress entrypoint with failure snapshot templates | — | | `install-flatc.sh` | dev-tool | Local flatc installer (macOS) | — | | `install-protoc.sh` | dev-tool | Local protoc installer (macOS/Linux) | — | | `makefile-header.sh` | dev-tool | Generates the `## —— section ——` header lines used in `.config/make/*.mak` | — | diff --git a/scripts/manual_transition_mixed_rollout_matrix.sh b/scripts/manual_transition_mixed_rollout_matrix.sh index ede17fcf7..f5a345a9c 100755 --- a/scripts/manual_transition_mixed_rollout_matrix.sh +++ b/scripts/manual_transition_mixed_rollout_matrix.sh @@ -127,6 +127,10 @@ parse_phase_row() { echo "ERROR: phase ratio/duration must be integers: $spec" >&2 exit 1 fi + if (( old_ratio + new_ratio != 100 )); then + echo "ERROR: phase ratios must sum to 100: $spec" >&2 + exit 1 + fi echo "$name|$old_ratio|$new_ratio|$duration_min" } @@ -249,6 +253,7 @@ run_rows() { printf " --admin-token \"\${ADMIN_TOKEN}\"" fi printf '\n' + admin_check_cmd "$phase_name" "$job_id_ref" 1 1 echo "" } >> "$run_script" done diff --git a/scripts/manual_transition_mixed_rollout_runbook.sh b/scripts/manual_transition_mixed_rollout_runbook.sh new file mode 100755 index 000000000..eb1a8fbf8 --- /dev/null +++ b/scripts/manual_transition_mixed_rollout_runbook.sh @@ -0,0 +1,402 @@ +#!/usr/bin/env bash +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +ENDPOINT="" +ADMIN_TOKEN="" +OUT_DIR="" +PHASE_MATRIX="request-only:100:0:30,canary:90:10:120,mixed:50:50:240,full-rollout:0:100:360,rollback:100:0:30" +CONCURRENCIES="8,16,32" +OBJECT_COUNTS="5k,20k,50k" +JOB_BUCKET="manual-transition" +JOB_PREFIX="journal-mixed-rollout" +TIER="" +READ_RATIO="90" +RUN_ADMIN_CHECKS=true +DRY_RUN=false + +MIXED_MATRIX_SCRIPT="${PROJECT_ROOT}/scripts/manual_transition_mixed_rollout_matrix.sh" +RUNBOOK_FILE="" +COMMAND_FILE="" + +usage() { + cat <<'USAGE' +Usage: + scripts/manual_transition_mixed_rollout_runbook.sh --endpoint [options] + +Required: + --endpoint Admin API base, e.g. https://127.0.0.1:9000 + +Optional: + --admin-token Bearer token for admin endpoints + --job-bucket Job bucket for transition scope (default: manual-transition) + --job-prefix Prefix for generated runbook phase names (default: journal-mixed-rollout) + --tier Transition tier (default: empty) + --phase-matrix Comma-separated phase spec: name:old_pct:new_pct:duration_min + --concurrencies Comma-separated concurrency list + --object-counts Comma-separated object-count workload list + --read-ratio Read ratio percent used in mixed workload commands + --out-dir Output directory for generated runbook artifacts + --no-admin-checks Omit generated admin check commands in the runbook + --dry-run Generate artifacts only; skip executable plan file creation + --help + +Artifacts: + - mixed-rollout runbook markdown + - matrix/run scripts from manual_transition_mixed_rollout_matrix.sh + - reusable command template script +USAGE +} + +arg_value() { + local flag="$1" + local value="${2:-}" + if [[ -z "$value" || "$value" == --* ]]; then + echo "ERROR: missing value for $flag" >&2 + exit 1 + fi + printf '%s' "$value" +} + +require_cmd() { + local cmd="$1" + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "ERROR: required command not found: $cmd" >&2 + exit 1 + fi +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --endpoint) + ENDPOINT="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --admin-token) + ADMIN_TOKEN="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --job-bucket) + JOB_BUCKET="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --job-prefix) + JOB_PREFIX="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --tier) + TIER="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --phase-matrix) + PHASE_MATRIX="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --concurrencies) + CONCURRENCIES="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --object-counts) + OBJECT_COUNTS="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --read-ratio) + READ_RATIO="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --out-dir) + OUT_DIR="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --no-admin-checks) + RUN_ADMIN_CHECKS=false + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "ERROR: unknown arg: $1" >&2 + usage + exit 1 + ;; + esac + done +} + +main_matrix() { + local args=("${MIXED_MATRIX_SCRIPT}" --endpoint "$ENDPOINT") + + args+=(--phase-matrix "$PHASE_MATRIX") + args+=(--concurrencies "$CONCURRENCIES") + args+=(--object-counts "$OBJECT_COUNTS") + args+=(--job-bucket "$JOB_BUCKET") + args+=(--job-prefix "$JOB_PREFIX") + args+=(--read-ratio "$READ_RATIO") + [[ -n "$ADMIN_TOKEN" ]] && args+=(--admin-token "$ADMIN_TOKEN") + [[ "$RUN_ADMIN_CHECKS" == true ]] || args+=(--no-admin-checks) + [[ -n "$OUT_DIR" ]] && args+=(--out-dir "$OUT_DIR") + [[ "$DRY_RUN" == true ]] && args+=(--dry-run) + + "${args[@]}" +} + +command_template() { + local matrix_csv="${OUT_DIR}/mixed_rollout_matrix.csv" + local matrix_cmd="${OUT_DIR}/run_phase_commands.sh" + local manifest="${OUT_DIR}/mixed_rollout_checklist.md" + local out_cmd + local token_note + local tier_note + + RUNBOOK_FILE="${OUT_DIR}/manual_transition_mixed_rollout_runbook.md" + COMMAND_FILE="${OUT_DIR}/run_mixed_rollout_plan.sh" + + if [[ -z "$TIER" ]]; then + tier_note="No tier value is set; default transition scope is all tiers under scope settings." + else + tier_note="tier=${TIER}" + fi + if [[ -n "$ADMIN_TOKEN" ]]; then + token_note='export ADMIN_TOKEN=' + else + token_note='export ADMIN_TOKEN="" (set this value)' + fi + + cat >"$RUNBOOK_FILE" <<'RUNBOOK' +# Manual transition mixed-version rollout runbook + +## Minimum runtime checklist + +- endpoint: __ENDPOINT__ +- admin token: __TOKEN_NOTE__ +- matrix input: __MATRIX_CSV__ +- read ratio target: __READ_RATIO__% +- scope: bucket=__JOB_BUCKET__, prefix template starts at __JOB_PREFIX__, __TIER_NOTE__ +- required tools: bash, curl, jq, awk, sed, date + +## Commands + +Use the commands below directly: + +- chmod +x __COMMAND_FILE__ +- bash __COMMAND_FILE__ + +## Notes + +- generated files: + - __MATRIX_CSV__ + - __MATRIX_CMD__ + - __MANIFEST__ + - __RUNBOOK_FILE__ + - __COMMAND_FILE__ +- generated 'run_phase_commands.sh' is a per-phase audit starter; 'run_mixed_rollout_plan.sh' is the full runnable template. +- recommended runbook order: + 1. Review 'mixed_rollout_checklist.md'. + 2. Inspect __MATRIX_CMD__ for each phase note. + 3. Run '__COMMAND_FILE__' to trigger the rollout template execution. +RUNBOOK + + perl -0pi -e "s#__ENDPOINT__#${ENDPOINT//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__TOKEN_NOTE__#${token_note//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__MATRIX_CSV__#${matrix_csv//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__READ_RATIO__#${READ_RATIO//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__JOB_BUCKET__#${JOB_BUCKET//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__JOB_PREFIX__#${JOB_PREFIX//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__TIER_NOTE__#${tier_note//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__MATRIX_CMD__#${matrix_cmd//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__MANIFEST__#${manifest//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__RUNBOOK_FILE__#${RUNBOOK_FILE//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__COMMAND_FILE__#${COMMAND_FILE//#/#}#g" "$RUNBOOK_FILE" + + cat >"$COMMAND_FILE" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +MIXED_MATRIX_CSV='__MIXED_MATRIX_CSV__' +ENDPOINT='__ENDPOINT__' +ADMIN_TOKEN='__ADMIN_TOKEN__' +JOB_BUCKET='__JOB_BUCKET__' +JOB_PREFIX='__JOB_PREFIX__' +TIER='__TIER__' +OUT_DIR='__OUT_DIR__' + +require_cmd() { + local cmd="$1" + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "ERROR: required command not found: $cmd" >&2 + exit 1 + fi +} + +url_encode() { + local value="$1" + jq -rn --arg v "$value" '$v|@uri' +} + +normalize_object_count() { + local raw + local lower + raw="$1" + raw="${raw// /}" + lower="${raw,,}" + + if [[ "$lower" =~ ^([0-9]+)$ ]]; then + echo "$lower" + return 0 + fi + + if [[ "$lower" =~ ^([0-9]+)k$ ]]; then + echo $((BASH_REMATCH[1] * 1000)) + return 0 + fi + + if [[ "$lower" =~ ^([0-9]+)m$ ]]; then + echo $((BASH_REMATCH[1] * 1000000)) + return 0 + fi + + echo "" + return 1 +} + +run_transition() { + local phase="$1" + local concurrency="$2" + local object_count_label="$3" + local duration_min="$4" + local duration_sec + local object_count + local prefix + local query + local url + local response + local job_id + local status_url + local headers=() + + duration_sec=$((duration_min * 60)) + if ! object_count="$(normalize_object_count "$object_count_label")"; then + echo "[warn] skip ${phase}: invalid object_count=${object_count_label}" >&2 + return 0 + fi + if (( object_count <= 0 )); then + echo "[warn] skip ${phase}: non-positive object_count=${object_count}" >&2 + return 0 + fi + + prefix="${JOB_PREFIX}/${phase}/${concurrency}c_${object_count_label}o" + + query="bucket=$(url_encode "$JOB_BUCKET")" + query="${query}&prefix=$(url_encode "$prefix")" + query="${query}&maxObjects=${object_count}" + query="${query}&maxDurationSeconds=${duration_sec}" + query="${query}&mode=async" + query="${query}&dryRun=false" + if [[ -n "$TIER" ]]; then + query="${query}&tier=$(url_encode "$TIER")" + fi + + url="${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/run?${query}" + + if [[ -n "$ADMIN_TOKEN" ]]; then + headers+=("-H" "Authorization: Bearer ${ADMIN_TOKEN}") + fi + + echo "==> phase=${phase} concurrency=${concurrency} object_count=${object_count} duration_min=${duration_min}" + response="$(curl -sS "${headers[@]}" -X POST "$url")" + job_id="$(printf '%s' "$response" | jq -r '.job_id // empty')" + if [[ -z "$job_id" || "$job_id" == "null" ]]; then + echo "ERROR: transition run did not return job_id for ${phase}" >&2 + echo "$response" + return 1 + fi + + echo "job_id=${job_id}" + status_url="${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" + ./scripts/manual_transition_journal_audit.sh --endpoint "$ENDPOINT" --job-id "$job_id" ${ADMIN_TOKEN:+--admin-token "$ADMIN_TOKEN"} --out-dir "${OUT_DIR}/journal-audit-${phase}-${concurrency}-${object_count_label}" || true + printf '%s\n' "$response" > "${OUT_DIR}/run-response-${phase}-${concurrency}-${object_count_label}.json" + echo "status_url=${status_url}" + curl -sS "${headers[@]}" -X GET "$status_url" | jq '{status, report, queue_snapshot, failure_reason}' +} + +main() { + require_cmd bash + require_cmd curl + require_cmd jq + require_cmd awk + require_cmd sed + + if [[ ! -f "$MIXED_MATRIX_CSV" ]]; then + echo "ERROR: expected matrix file missing: $MIXED_MATRIX_CSV" >&2 + echo "Run manual_transition_mixed_rollout_matrix.sh first." >&2 + exit 1 + fi + + while IFS=',' read -r phase old_pct new_pct duration_min concurrency object_count read_ratio gate admin_check; do + if [[ -z "$phase" || "$phase" == "phase" ]]; then + continue + fi + run_transition "$phase" "$concurrency" "$object_count" "$duration_min" + echo "" + done < <(tail -n +2 "$MIXED_MATRIX_CSV") +} + +main "$@" +EOF + + perl -0pi -e "s#__MIXED_MATRIX_CSV__#${matrix_csv//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__ENDPOINT__#${ENDPOINT//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__ADMIN_TOKEN__#${ADMIN_TOKEN//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__JOB_BUCKET__#${JOB_BUCKET//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__JOB_PREFIX__#${JOB_PREFIX//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__TIER__#${TIER//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__OUT_DIR__#${OUT_DIR//#/#}#g" "$COMMAND_FILE" + chmod +x "$COMMAND_FILE" +} + +main() { + parse_args "$@" + if [[ -z "$ENDPOINT" ]]; then + echo "ERROR: --endpoint is required" >&2 + usage + exit 1 + fi + + if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="${PROJECT_ROOT}/target/manual-transition-mixed-rollout-runbook/$(date +%Y%m%dT%H%M%S)" + fi + + mkdir -p "$OUT_DIR" + require_cmd awk + require_cmd jq + + main_matrix + command_template + + echo "Generated runbook: ${RUNBOOK_FILE}" +} + +main "$@" diff --git a/scripts/manual_transition_nightly_stress_runbook.sh b/scripts/manual_transition_nightly_stress_runbook.sh new file mode 100755 index 000000000..13f79a5ac --- /dev/null +++ b/scripts/manual_transition_nightly_stress_runbook.sh @@ -0,0 +1,398 @@ +#!/usr/bin/env bash +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +ENDPOINT="" +ADMIN_TOKEN="" +OUT_DIR="" +WINDOW_SPEC="nightly-2h:120:16:5000:balanced,nightly-12h:720:24:8000:write-heavy,nightly-24h:1440:32:12000:read-heavy" +SOAK_RATIOS="read-heavy:90:10,balanced:70:30,write-heavy:40:60" +WORKLOAD_SIZES="4KiB,1MiB,16MiB" +JOB_BUCKET="manual-transition" +JOB_PREFIX="journal-nightly-stress" +TIER="" +RUN_ADMIN_CHECKS=true +DRY_RUN=false +UNKNOWN_FAILURE_RATIO_THRESHOLD="0.00" +QUEUE_MISMATCH_RATIO_THRESHOLD="0.00" +UNKNOWN_FAILURE_COUNT_THRESHOLD="0" + +STRESS_MATRIX_SCRIPT="${PROJECT_ROOT}/scripts/manual_transition_soak_matrix.sh" +RUNBOOK_FILE="" +COMMAND_FILE="" + +usage() { + cat <<'USAGE' +Usage: + scripts/manual_transition_nightly_stress_runbook.sh --endpoint [options] + +Required: + --endpoint Admin API base, e.g. https://127.0.0.1:9000 + +Optional: + --admin-token Bearer token for admin endpoints + --window-spec Comma-separated run spec: window:duration_min:concurrency:ops_per_min:mix_name + --soak-ratios Comma-separated mix ratios: label:read_pct:write_pct + --workload-sizes Object-size workload set + --job-bucket Job bucket scope for transition commands + --job-prefix Prefix for generated commands (default: journal-nightly-stress) + --tier Transition tier (default: empty) + --out-dir Output directory for generated artifacts + --no-admin-checks Omit generated admin check commands in the runbook + --dry-run Generate artifacts only; do not create executable runner + --help + +Artifacts: + - nightly stress runbook markdown + - matrix/run scripts from manual_transition_soak_matrix.sh + - runnable command entry template (includes failure snapshot hook) +USAGE +} + +arg_value() { + local flag="$1" + local value="${2:-}" + if [[ -z "$value" || "$value" == --* ]]; then + echo "ERROR: missing value for $flag" >&2 + exit 1 + fi + printf '%s' "$value" +} + +require_cmd() { + local cmd="$1" + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "ERROR: required command not found: $cmd" >&2 + exit 1 + fi +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --endpoint) + ENDPOINT="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --admin-token) + ADMIN_TOKEN="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --window-spec) + WINDOW_SPEC="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --soak-ratios) + SOAK_RATIOS="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --workload-sizes) + WORKLOAD_SIZES="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --job-bucket) + JOB_BUCKET="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --job-prefix) + JOB_PREFIX="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --tier) + TIER="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --out-dir) + OUT_DIR="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --no-admin-checks) + RUN_ADMIN_CHECKS=false + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "ERROR: unknown arg: $1" >&2 + usage + exit 1 + ;; + esac + done +} + +main_matrix() { + local args=("${STRESS_MATRIX_SCRIPT}" --endpoint "$ENDPOINT") + + args+=(--window-spec "$WINDOW_SPEC") + args+=(--soak-ratios "$SOAK_RATIOS") + args+=(--workload-sizes "$WORKLOAD_SIZES") + [[ -n "$ADMIN_TOKEN" ]] && args+=(--admin-token "$ADMIN_TOKEN") + [[ "$RUN_ADMIN_CHECKS" == true ]] || args+=(--no-admin-checks) + [[ -n "$OUT_DIR" ]] && args+=(--out-dir "$OUT_DIR") + [[ "$DRY_RUN" == true ]] && args+=(--dry-run) + + "${args[@]}" +} + +command_template() { + local matrix_csv="${OUT_DIR}/nightly_soak_matrix.csv" + local matrix_cmd="${OUT_DIR}/run_soak_matrix.sh" + local notes="${OUT_DIR}/soak_notes.md" + RUNBOOK_FILE="${OUT_DIR}/manual_transition_nightly_stress_runbook.md" + COMMAND_FILE="${OUT_DIR}/run_nightly_stress_plan.sh" + + cat >"$RUNBOOK_FILE" <<'RUNBOOK' +# Manual transition nightly/stress stress-runbook + +## Runtime baseline + +- endpoint: __ENDPOINT__ +- bucket: __JOB_BUCKET__ +- prefix template: __JOB_PREFIX__ +- tier: __TIER__ +- matrix: __MATRIX_CSV__ +- matrix command: __MATRIX_CMD__ +- notes: __NOTES__ + +## Threshold template (editable) + +- unknown failure ratio threshold: __UNKNOWN_FAILURE_RATIO_THRESHOLD__ +- queue-mismatch tolerance ratio: __QUEUE_MISMATCH_RATIO_THRESHOLD__ +- unknown failure count threshold: __UNKNOWN_FAILURE_COUNT_THRESHOLD__ + +## Failure snapshot policy + +- on startup failure (missing job_id, API error), write a timestamped snapshot under '__OUT_DIR__/failure-snapshots' +- if immediate status shows failure_reason, capture snapshot under '__OUT_DIR__/failure-snapshots' +- for each failed run, run manual_transition_journal_audit.sh and keep its outputs as evidence + +## Usage + +Use the commands below directly: + +- chmod +x __COMMAND_FILE__ +- bash __COMMAND_FILE__ + +## Files produced + +- __MATRIX_CSV__ +- __MATRIX_CMD__ +- __NOTES__ +- __RUNBOOK_FILE__ +- __COMMAND_FILE__ +- __OUT_DIR__/failure-snapshots/* +RUNBOOK + + perl -0pi -e "s#__ENDPOINT__#${ENDPOINT//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__JOB_BUCKET__#${JOB_BUCKET//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__JOB_PREFIX__#${JOB_PREFIX//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__TIER__#${TIER//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__MATRIX_CSV__#${matrix_csv//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__MATRIX_CMD__#${matrix_cmd//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__NOTES__#${notes//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__RUNBOOK_FILE__#${RUNBOOK_FILE//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__COMMAND_FILE__#${COMMAND_FILE//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__OUT_DIR__#${OUT_DIR//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__UNKNOWN_FAILURE_RATIO_THRESHOLD__#${UNKNOWN_FAILURE_RATIO_THRESHOLD//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__QUEUE_MISMATCH_RATIO_THRESHOLD__#${QUEUE_MISMATCH_RATIO_THRESHOLD//#/#}#g" "$RUNBOOK_FILE" + perl -0pi -e "s#__UNKNOWN_FAILURE_COUNT_THRESHOLD__#${UNKNOWN_FAILURE_COUNT_THRESHOLD//#/#}#g" "$RUNBOOK_FILE" + + cat >"$COMMAND_FILE" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +SOAK_MATRIX_CSV='__SOAK_MATRIX_CSV__' +ENDPOINT='__ENDPOINT__' +ADMIN_TOKEN='__ADMIN_TOKEN__' +JOB_BUCKET='__JOB_BUCKET__' +JOB_PREFIX='__JOB_PREFIX__' +TIER='__TIER__' +OUT_DIR='__OUT_DIR__' + +: "${UNKNOWN_FAILURE_RATIO_THRESHOLD:=0.0}" +: "${QUEUE_MISMATCH_RATIO_THRESHOLD:=0.0}" +: "${UNKNOWN_FAILURE_COUNT_THRESHOLD:=0}" + +SNAPSHOT_DIR="${OUT_DIR}/failure-snapshots" +mkdir -p "$SNAPSHOT_DIR" + +require_cmd() { + local cmd="$1" + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "ERROR: required command not found: $cmd" >&2 + exit 1 + fi +} + +url_encode() { + local value="$1" + jq -rn --arg v "$value" '$v|@uri' +} + +snapshot_failure() { + local run_tag="$1" + local reason="$2" + local job_id="$3" + local ts + local snapshot_dir + + ts="$(date +%Y%m%dT%H%M%S)" + snapshot_dir="${SNAPSHOT_DIR}/${run_tag}/${ts}" + mkdir -p "$snapshot_dir" + + { + echo "run_tag=${run_tag}" + echo "reason=${reason}" + echo "job_id=${job_id}" + echo "ts=${ts}" + } >"${snapshot_dir}/snapshot.meta" + + if [[ -n "$job_id" && "$job_id" != "NA" ]]; then + curl -sS ${ADMIN_TOKEN:+-H "Authorization: Bearer ${ADMIN_TOKEN}"} -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" \ + >"${snapshot_dir}/job_status.json" + ./scripts/manual_transition_journal_audit.sh --endpoint "$ENDPOINT" --job-id "$job_id" ${ADMIN_TOKEN:+--admin-token "$ADMIN_TOKEN"} --out-dir "$snapshot_dir" || true + fi + + cp "$SOAK_MATRIX_CSV" "${snapshot_dir}/source_matrix.csv" +} + +run_entry() { + local tag="$1" + local duration_min="$2" + local concurrency="$3" + local ops_per_min="$4" + local mix_name="$5" + local read_pct="$6" + local write_pct="$7" + local size="$8" + local expected_ops="$9" + local budget_status="${10}" + + local prefix + local query + local url + local response + local job_id + local status + local headers=() + + if [[ "$budget_status" == "over-budget" ]]; then + echo "[skip] ${tag} ${size} over budget: expected_ops=${expected_ops}" + return 0 + fi + + if [[ "$UNKNOWN_FAILURE_COUNT_THRESHOLD" -gt 0 && "$expected_ops" -gt "$UNKNOWN_FAILURE_COUNT_THRESHOLD" ]]; then + : + fi + + prefix="${JOB_PREFIX}/${tag}/${size}/${read_pct}r${write_pct}w" + query="bucket=$(url_encode "$JOB_BUCKET")" + query="${query}&prefix=$(url_encode "$prefix")" + query="${query}&maxObjects=100000" + query="${query}&maxDurationSeconds=$((duration_min * 60))" + query="${query}&mode=async&dryRun=false" + if [[ -n "$TIER" ]]; then + query="${query}&tier=$(url_encode "$TIER")" + fi + + url="${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/run?${query}" + + if [[ -n "$ADMIN_TOKEN" ]]; then + headers+=("-H" "Authorization: Bearer ${ADMIN_TOKEN}") + fi + + echo "==> run=${tag} size=${size} mix=${mix_name} read_pct=${read_pct} write_pct=${write_pct} duration_min=${duration_min}" + if ! response="$(curl -sS "${headers[@]}" -X POST "$url")"; then + snapshot_failure "$tag" "curl_post_failed" "NA" + return 1 + fi + job_id="$(printf '%s' "$response" | jq -r '.job_id // empty')" + if [[ -z "$job_id" || "$job_id" == "null" ]]; then + snapshot_failure "$tag" "missing_job_id" "NA" + return 1 + fi + + status="$(curl -sS "${headers[@]}" -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" | jq -r '.failure_reason // empty')" + if [[ -n "$status" && "$status" != "null" ]]; then + snapshot_failure "$tag" "failure_reason=${status}" "$job_id" + fi +} + +main() { + require_cmd bash + require_cmd curl + require_cmd jq + require_cmd awk + + if [[ ! -f "$SOAK_MATRIX_CSV" ]]; then + echo "ERROR: expected matrix file missing: $SOAK_MATRIX_CSV" >&2 + echo "Run manual_transition_soak_matrix.sh first." >&2 + exit 1 + fi + + while IFS=',' read -r window duration_min concurrency ops_per_min mix_name read_pct write_pct size expected_ops run_id budget_status; do + if [[ -z "$window" || "$window" == "window" ]]; then + continue + fi + run_entry "${window}" "${duration_min}" "${concurrency}" "${ops_per_min}" "${mix_name}" "${read_pct}" "${write_pct}" "${size}" "${expected_ops}" "${budget_status}" || true + done < <(tail -n +2 "$SOAK_MATRIX_CSV") +} + +main "$@" +EOF + + perl -0pi -e "s#__SOAK_MATRIX_CSV__#${matrix_csv//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__ENDPOINT__#${ENDPOINT//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__ADMIN_TOKEN__#${ADMIN_TOKEN//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__JOB_BUCKET__#${JOB_BUCKET//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__JOB_PREFIX__#${JOB_PREFIX//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__TIER__#${TIER//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__OUT_DIR__#${OUT_DIR//#/#}#g" "$COMMAND_FILE" + chmod +x "$COMMAND_FILE" +} + +main() { + parse_args "$@" + if [[ -z "$ENDPOINT" ]]; then + echo "ERROR: --endpoint is required" >&2 + usage + exit 1 + fi + + if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="${PROJECT_ROOT}/target/manual-transition-nightly-stress-runbook/$(date +%Y%m%dT%H%M%S)" + fi + mkdir -p "$OUT_DIR" + require_cmd awk + require_cmd jq + + main_matrix + command_template + + echo "Generated runbook: ${RUNBOOK_FILE}" +} + +main "$@" diff --git a/scripts/manual_transition_soak_matrix.sh b/scripts/manual_transition_soak_matrix.sh index 4c2602a50..902858dbb 100755 --- a/scripts/manual_transition_soak_matrix.sh +++ b/scripts/manual_transition_soak_matrix.sh @@ -133,6 +133,10 @@ parse_ratio() { echo "ERROR: mix percentages must be integers: $spec" >&2 exit 1 fi + if (( read_pct + write_pct != 100 )); then + echo "ERROR: read/write percentages must sum to 100: $spec" >&2 + exit 1 + fi echo "$name|$read_pct|$write_pct" }