mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-04 04:17:44 +00:00
* feat(ecstore): attribute manual transition worker failures Add manual transition worker failure reason tracking and persistence recovery compatibility for checksum validation. Co-Authored-By: heihutu <heihutu@gmail.com> * chore(ilm): add manual transition diagnostics scripts Co-Authored-By: heihutu <heihutu@gmail.com> * style(ecstore): format manual transition attribution Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): avoid copying failure reasons via clone Co-Authored-By: heihutu <heihutu@gmail.com> * fix(manual-transition): improve matrix verification scripts - Add strict mixed rollout phase ratio validation. - Add strict read/write ratio validation for soak mix. - Inline generated admin-check command blocks into mixed-rollout run script. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(scripts): add runbook entrypoints for #1508 #1510 --------- Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
@@ -9376,6 +9376,9 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
let mut stale = ManualTransitionJobRecord::new(Uuid::new_v4(), &bucket, &options, "old-owner");
|
||||
if stale_case == "missing-job" {
|
||||
stale.lease_expires_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos() - 1;
|
||||
}
|
||||
if stale_case == "terminal-job" {
|
||||
stale.complete(
|
||||
ManualTransitionRunReport {
|
||||
@@ -9412,6 +9415,48 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn manual_transition_admission_concurrent_same_scope_writes_is_singleton() {
|
||||
let (_paths, ecstore) = setup_test_env().await;
|
||||
let options = ManualTransitionRunOptions {
|
||||
prefix: "logs/".to_string(),
|
||||
tier: Some("warm".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let first = ManualTransitionJobRecord::new(Uuid::new_v4(), "manual-concurrent-scope-bucket", &options, "owner-a");
|
||||
let first_admission = ManualTransitionScopeAdmission::from_job(&first);
|
||||
let second = ManualTransitionJobRecord::new(Uuid::new_v4(), "manual-concurrent-scope-bucket", &options, "owner-b");
|
||||
let second_admission = ManualTransitionScopeAdmission::from_job(&second);
|
||||
|
||||
let first_claim = claim_manual_transition_scope_admission(ecstore.clone(), &first_admission);
|
||||
let second_claim = claim_manual_transition_scope_admission(ecstore.clone(), &second_admission);
|
||||
let (first_result, second_result) = tokio::join!(first_claim, second_claim);
|
||||
let first_claim = first_result.expect("first concurrent claim should resolve");
|
||||
let second_claim = second_result.expect("second concurrent claim should resolve");
|
||||
|
||||
let mut claimed = 0;
|
||||
let mut conflicted = 0;
|
||||
let mut active_job_id = None;
|
||||
for item in [first_claim, second_claim] {
|
||||
match item {
|
||||
ManualTransitionScopeAdmissionClaim::Claimed => claimed += 1,
|
||||
ManualTransitionScopeAdmissionClaim::Conflict(active) => {
|
||||
conflicted += 1;
|
||||
active_job_id = Some(active.job_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(claimed, 1, "only one concurrent same-scope claim should be accepted");
|
||||
assert_eq!(conflicted, 1, "only one concurrent same-scope claim should report conflict");
|
||||
let active = load_manual_transition_scope_admission(ecstore.clone(), &first.scope_key)
|
||||
.await
|
||||
.expect("scope admission should remain");
|
||||
assert!(active.job_id == first.job_id || active.job_id == second.job_id);
|
||||
assert_eq!(active_job_id, Some(active.job_id), "conflict response must carry active owner");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_object_lifecycle_allows_expired_marker_after_replication_completed() {
|
||||
let lc = expired_delete_marker_lifecycle();
|
||||
|
||||
@@ -1405,7 +1405,9 @@ pub async fn claim_manual_transition_scope_admission(
|
||||
Ok(active_job) => {
|
||||
active_job.is_terminal() || (scope_lease_expired && manual_transition_job_lease_expired(&active_job))
|
||||
}
|
||||
Err(Error::ConfigNotFound) => true,
|
||||
// Missing active job metadata can be transient (for example, immediately after admission creation);
|
||||
// require an expired scope lease before treating it as reclaimable.
|
||||
Err(Error::ConfigNotFound) => scope_lease_expired,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
};
|
||||
|
||||
@@ -86,6 +86,12 @@ their issue closes.
|
||||
| `restart_local_single_node_multidisk_rustfs.sh` | dev-tool | Restart a local single-node multi-disk instance | — |
|
||||
| `inspect_dashboard.sh` | dev-tool | Sanity-checks the Grafana dashboard JSON | `.docker/observability` |
|
||||
| `notify.sh` | dev-tool | Starts a local webhook receiver for notify-target development | — |
|
||||
| `manual_transition_debug.sh` | dev-tool | Log/metrics helper for manual transition troubleshooting | — |
|
||||
| `manual_transition_journal_audit.sh` | dev-tool | Journal + metrics + log audit for manual transition jobs | — |
|
||||
| `manual_transition_mixed_rollout_matrix.sh` | dev-tool | Matrix generator for mixed-version rollout phases | — |
|
||||
| `manual_transition_mixed_rollout_runbook.sh` | dev-tool | Reusable mixed-version rollout runbook generator (external run) | — |
|
||||
| `manual_transition_soak_matrix.sh` | dev-tool | Matrix generator for nightly stress windows | — |
|
||||
| `manual_transition_nightly_stress_runbook.sh` | dev-tool | Nightly stress entrypoint with failure snapshot templates | — |
|
||||
| `install-flatc.sh` | dev-tool | Local flatc installer (macOS) | — |
|
||||
| `install-protoc.sh` | dev-tool | Local protoc installer (macOS/Linux) | — |
|
||||
| `makefile-header.sh` | dev-tool | Generates the `## —— section ——` header lines used in `.config/make/*.mak` | — |
|
||||
|
||||
@@ -127,6 +127,10 @@ parse_phase_row() {
|
||||
echo "ERROR: phase ratio/duration must be integers: $spec" >&2
|
||||
exit 1
|
||||
fi
|
||||
if (( old_ratio + new_ratio != 100 )); then
|
||||
echo "ERROR: phase ratios must sum to 100: $spec" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$name|$old_ratio|$new_ratio|$duration_min"
|
||||
}
|
||||
|
||||
@@ -249,6 +253,7 @@ run_rows() {
|
||||
printf " --admin-token \"\${ADMIN_TOKEN}\""
|
||||
fi
|
||||
printf '\n'
|
||||
admin_check_cmd "$phase_name" "$job_id_ref" 1 1
|
||||
echo ""
|
||||
} >> "$run_script"
|
||||
done
|
||||
|
||||
+402
@@ -0,0 +1,402 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
ENDPOINT=""
|
||||
ADMIN_TOKEN=""
|
||||
OUT_DIR=""
|
||||
PHASE_MATRIX="request-only:100:0:30,canary:90:10:120,mixed:50:50:240,full-rollout:0:100:360,rollback:100:0:30"
|
||||
CONCURRENCIES="8,16,32"
|
||||
OBJECT_COUNTS="5k,20k,50k"
|
||||
JOB_BUCKET="manual-transition"
|
||||
JOB_PREFIX="journal-mixed-rollout"
|
||||
TIER=""
|
||||
READ_RATIO="90"
|
||||
RUN_ADMIN_CHECKS=true
|
||||
DRY_RUN=false
|
||||
|
||||
MIXED_MATRIX_SCRIPT="${PROJECT_ROOT}/scripts/manual_transition_mixed_rollout_matrix.sh"
|
||||
RUNBOOK_FILE=""
|
||||
COMMAND_FILE=""
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/manual_transition_mixed_rollout_runbook.sh --endpoint <admin-api> [options]
|
||||
|
||||
Required:
|
||||
--endpoint Admin API base, e.g. https://127.0.0.1:9000
|
||||
|
||||
Optional:
|
||||
--admin-token Bearer token for admin endpoints
|
||||
--job-bucket Job bucket for transition scope (default: manual-transition)
|
||||
--job-prefix Prefix for generated runbook phase names (default: journal-mixed-rollout)
|
||||
--tier Transition tier (default: empty)
|
||||
--phase-matrix Comma-separated phase spec: name:old_pct:new_pct:duration_min
|
||||
--concurrencies Comma-separated concurrency list
|
||||
--object-counts Comma-separated object-count workload list
|
||||
--read-ratio Read ratio percent used in mixed workload commands
|
||||
--out-dir Output directory for generated runbook artifacts
|
||||
--no-admin-checks Omit generated admin check commands in the runbook
|
||||
--dry-run Generate artifacts only; skip executable plan file creation
|
||||
--help
|
||||
|
||||
Artifacts:
|
||||
- mixed-rollout runbook markdown
|
||||
- matrix/run scripts from manual_transition_mixed_rollout_matrix.sh
|
||||
- reusable command template script
|
||||
USAGE
|
||||
}
|
||||
|
||||
arg_value() {
|
||||
local flag="$1"
|
||||
local value="${2:-}"
|
||||
if [[ -z "$value" || "$value" == --* ]]; then
|
||||
echo "ERROR: missing value for $flag" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
local cmd="$1"
|
||||
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||
echo "ERROR: required command not found: $cmd" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--endpoint)
|
||||
ENDPOINT="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--admin-token)
|
||||
ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--job-bucket)
|
||||
JOB_BUCKET="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--job-prefix)
|
||||
JOB_PREFIX="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--tier)
|
||||
TIER="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--phase-matrix)
|
||||
PHASE_MATRIX="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--concurrencies)
|
||||
CONCURRENCIES="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--object-counts)
|
||||
OBJECT_COUNTS="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--read-ratio)
|
||||
READ_RATIO="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--out-dir)
|
||||
OUT_DIR="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--no-admin-checks)
|
||||
RUN_ADMIN_CHECKS=false
|
||||
shift
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN=true
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: unknown arg: $1" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
main_matrix() {
|
||||
local args=("${MIXED_MATRIX_SCRIPT}" --endpoint "$ENDPOINT")
|
||||
|
||||
args+=(--phase-matrix "$PHASE_MATRIX")
|
||||
args+=(--concurrencies "$CONCURRENCIES")
|
||||
args+=(--object-counts "$OBJECT_COUNTS")
|
||||
args+=(--job-bucket "$JOB_BUCKET")
|
||||
args+=(--job-prefix "$JOB_PREFIX")
|
||||
args+=(--read-ratio "$READ_RATIO")
|
||||
[[ -n "$ADMIN_TOKEN" ]] && args+=(--admin-token "$ADMIN_TOKEN")
|
||||
[[ "$RUN_ADMIN_CHECKS" == true ]] || args+=(--no-admin-checks)
|
||||
[[ -n "$OUT_DIR" ]] && args+=(--out-dir "$OUT_DIR")
|
||||
[[ "$DRY_RUN" == true ]] && args+=(--dry-run)
|
||||
|
||||
"${args[@]}"
|
||||
}
|
||||
|
||||
command_template() {
|
||||
local matrix_csv="${OUT_DIR}/mixed_rollout_matrix.csv"
|
||||
local matrix_cmd="${OUT_DIR}/run_phase_commands.sh"
|
||||
local manifest="${OUT_DIR}/mixed_rollout_checklist.md"
|
||||
local out_cmd
|
||||
local token_note
|
||||
local tier_note
|
||||
|
||||
RUNBOOK_FILE="${OUT_DIR}/manual_transition_mixed_rollout_runbook.md"
|
||||
COMMAND_FILE="${OUT_DIR}/run_mixed_rollout_plan.sh"
|
||||
|
||||
if [[ -z "$TIER" ]]; then
|
||||
tier_note="No tier value is set; default transition scope is all tiers under scope settings."
|
||||
else
|
||||
tier_note="tier=${TIER}"
|
||||
fi
|
||||
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||
token_note='export ADMIN_TOKEN=<set by caller>'
|
||||
else
|
||||
token_note='export ADMIN_TOKEN="" (set this value)'
|
||||
fi
|
||||
|
||||
cat >"$RUNBOOK_FILE" <<'RUNBOOK'
|
||||
# Manual transition mixed-version rollout runbook
|
||||
|
||||
## Minimum runtime checklist
|
||||
|
||||
- endpoint: __ENDPOINT__
|
||||
- admin token: __TOKEN_NOTE__
|
||||
- matrix input: __MATRIX_CSV__
|
||||
- read ratio target: __READ_RATIO__%
|
||||
- scope: bucket=__JOB_BUCKET__, prefix template starts at __JOB_PREFIX__, __TIER_NOTE__
|
||||
- required tools: bash, curl, jq, awk, sed, date
|
||||
|
||||
## Commands
|
||||
|
||||
Use the commands below directly:
|
||||
|
||||
- chmod +x __COMMAND_FILE__
|
||||
- bash __COMMAND_FILE__
|
||||
|
||||
## Notes
|
||||
|
||||
- generated files:
|
||||
- __MATRIX_CSV__
|
||||
- __MATRIX_CMD__
|
||||
- __MANIFEST__
|
||||
- __RUNBOOK_FILE__
|
||||
- __COMMAND_FILE__
|
||||
- generated 'run_phase_commands.sh' is a per-phase audit starter; 'run_mixed_rollout_plan.sh' is the full runnable template.
|
||||
- recommended runbook order:
|
||||
1. Review 'mixed_rollout_checklist.md'.
|
||||
2. Inspect __MATRIX_CMD__ for each phase note.
|
||||
3. Run '__COMMAND_FILE__' to trigger the rollout template execution.
|
||||
RUNBOOK
|
||||
|
||||
perl -0pi -e "s#__ENDPOINT__#${ENDPOINT//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__TOKEN_NOTE__#${token_note//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__MATRIX_CSV__#${matrix_csv//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__READ_RATIO__#${READ_RATIO//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__JOB_BUCKET__#${JOB_BUCKET//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__JOB_PREFIX__#${JOB_PREFIX//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__TIER_NOTE__#${tier_note//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__MATRIX_CMD__#${matrix_cmd//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__MANIFEST__#${manifest//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__RUNBOOK_FILE__#${RUNBOOK_FILE//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__COMMAND_FILE__#${COMMAND_FILE//#/#}#g" "$RUNBOOK_FILE"
|
||||
|
||||
cat >"$COMMAND_FILE" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
MIXED_MATRIX_CSV='__MIXED_MATRIX_CSV__'
|
||||
ENDPOINT='__ENDPOINT__'
|
||||
ADMIN_TOKEN='__ADMIN_TOKEN__'
|
||||
JOB_BUCKET='__JOB_BUCKET__'
|
||||
JOB_PREFIX='__JOB_PREFIX__'
|
||||
TIER='__TIER__'
|
||||
OUT_DIR='__OUT_DIR__'
|
||||
|
||||
require_cmd() {
|
||||
local cmd="$1"
|
||||
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||
echo "ERROR: required command not found: $cmd" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
url_encode() {
|
||||
local value="$1"
|
||||
jq -rn --arg v "$value" '$v|@uri'
|
||||
}
|
||||
|
||||
normalize_object_count() {
|
||||
local raw
|
||||
local lower
|
||||
raw="$1"
|
||||
raw="${raw// /}"
|
||||
lower="${raw,,}"
|
||||
|
||||
if [[ "$lower" =~ ^([0-9]+)$ ]]; then
|
||||
echo "$lower"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$lower" =~ ^([0-9]+)k$ ]]; then
|
||||
echo $((BASH_REMATCH[1] * 1000))
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$lower" =~ ^([0-9]+)m$ ]]; then
|
||||
echo $((BASH_REMATCH[1] * 1000000))
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
return 1
|
||||
}
|
||||
|
||||
run_transition() {
|
||||
local phase="$1"
|
||||
local concurrency="$2"
|
||||
local object_count_label="$3"
|
||||
local duration_min="$4"
|
||||
local duration_sec
|
||||
local object_count
|
||||
local prefix
|
||||
local query
|
||||
local url
|
||||
local response
|
||||
local job_id
|
||||
local status_url
|
||||
local headers=()
|
||||
|
||||
duration_sec=$((duration_min * 60))
|
||||
if ! object_count="$(normalize_object_count "$object_count_label")"; then
|
||||
echo "[warn] skip ${phase}: invalid object_count=${object_count_label}" >&2
|
||||
return 0
|
||||
fi
|
||||
if (( object_count <= 0 )); then
|
||||
echo "[warn] skip ${phase}: non-positive object_count=${object_count}" >&2
|
||||
return 0
|
||||
fi
|
||||
|
||||
prefix="${JOB_PREFIX}/${phase}/${concurrency}c_${object_count_label}o"
|
||||
|
||||
query="bucket=$(url_encode "$JOB_BUCKET")"
|
||||
query="${query}&prefix=$(url_encode "$prefix")"
|
||||
query="${query}&maxObjects=${object_count}"
|
||||
query="${query}&maxDurationSeconds=${duration_sec}"
|
||||
query="${query}&mode=async"
|
||||
query="${query}&dryRun=false"
|
||||
if [[ -n "$TIER" ]]; then
|
||||
query="${query}&tier=$(url_encode "$TIER")"
|
||||
fi
|
||||
|
||||
url="${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/run?${query}"
|
||||
|
||||
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||
headers+=("-H" "Authorization: Bearer ${ADMIN_TOKEN}")
|
||||
fi
|
||||
|
||||
echo "==> phase=${phase} concurrency=${concurrency} object_count=${object_count} duration_min=${duration_min}"
|
||||
response="$(curl -sS "${headers[@]}" -X POST "$url")"
|
||||
job_id="$(printf '%s' "$response" | jq -r '.job_id // empty')"
|
||||
if [[ -z "$job_id" || "$job_id" == "null" ]]; then
|
||||
echo "ERROR: transition run did not return job_id for ${phase}" >&2
|
||||
echo "$response"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "job_id=${job_id}"
|
||||
status_url="${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}"
|
||||
./scripts/manual_transition_journal_audit.sh --endpoint "$ENDPOINT" --job-id "$job_id" ${ADMIN_TOKEN:+--admin-token "$ADMIN_TOKEN"} --out-dir "${OUT_DIR}/journal-audit-${phase}-${concurrency}-${object_count_label}" || true
|
||||
printf '%s\n' "$response" > "${OUT_DIR}/run-response-${phase}-${concurrency}-${object_count_label}.json"
|
||||
echo "status_url=${status_url}"
|
||||
curl -sS "${headers[@]}" -X GET "$status_url" | jq '{status, report, queue_snapshot, failure_reason}'
|
||||
}
|
||||
|
||||
main() {
|
||||
require_cmd bash
|
||||
require_cmd curl
|
||||
require_cmd jq
|
||||
require_cmd awk
|
||||
require_cmd sed
|
||||
|
||||
if [[ ! -f "$MIXED_MATRIX_CSV" ]]; then
|
||||
echo "ERROR: expected matrix file missing: $MIXED_MATRIX_CSV" >&2
|
||||
echo "Run manual_transition_mixed_rollout_matrix.sh first." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
while IFS=',' read -r phase old_pct new_pct duration_min concurrency object_count read_ratio gate admin_check; do
|
||||
if [[ -z "$phase" || "$phase" == "phase" ]]; then
|
||||
continue
|
||||
fi
|
||||
run_transition "$phase" "$concurrency" "$object_count" "$duration_min"
|
||||
echo ""
|
||||
done < <(tail -n +2 "$MIXED_MATRIX_CSV")
|
||||
}
|
||||
|
||||
main "$@"
|
||||
EOF
|
||||
|
||||
perl -0pi -e "s#__MIXED_MATRIX_CSV__#${matrix_csv//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__ENDPOINT__#${ENDPOINT//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__ADMIN_TOKEN__#${ADMIN_TOKEN//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__JOB_BUCKET__#${JOB_BUCKET//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__JOB_PREFIX__#${JOB_PREFIX//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__TIER__#${TIER//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__OUT_DIR__#${OUT_DIR//#/#}#g" "$COMMAND_FILE"
|
||||
chmod +x "$COMMAND_FILE"
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
if [[ -z "$ENDPOINT" ]]; then
|
||||
echo "ERROR: --endpoint is required" >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$OUT_DIR" ]]; then
|
||||
OUT_DIR="${PROJECT_ROOT}/target/manual-transition-mixed-rollout-runbook/$(date +%Y%m%dT%H%M%S)"
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
require_cmd awk
|
||||
require_cmd jq
|
||||
|
||||
main_matrix
|
||||
command_template
|
||||
|
||||
echo "Generated runbook: ${RUNBOOK_FILE}"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+398
@@ -0,0 +1,398 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
ENDPOINT=""
|
||||
ADMIN_TOKEN=""
|
||||
OUT_DIR=""
|
||||
WINDOW_SPEC="nightly-2h:120:16:5000:balanced,nightly-12h:720:24:8000:write-heavy,nightly-24h:1440:32:12000:read-heavy"
|
||||
SOAK_RATIOS="read-heavy:90:10,balanced:70:30,write-heavy:40:60"
|
||||
WORKLOAD_SIZES="4KiB,1MiB,16MiB"
|
||||
JOB_BUCKET="manual-transition"
|
||||
JOB_PREFIX="journal-nightly-stress"
|
||||
TIER=""
|
||||
RUN_ADMIN_CHECKS=true
|
||||
DRY_RUN=false
|
||||
UNKNOWN_FAILURE_RATIO_THRESHOLD="0.00"
|
||||
QUEUE_MISMATCH_RATIO_THRESHOLD="0.00"
|
||||
UNKNOWN_FAILURE_COUNT_THRESHOLD="0"
|
||||
|
||||
STRESS_MATRIX_SCRIPT="${PROJECT_ROOT}/scripts/manual_transition_soak_matrix.sh"
|
||||
RUNBOOK_FILE=""
|
||||
COMMAND_FILE=""
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/manual_transition_nightly_stress_runbook.sh --endpoint <admin-api> [options]
|
||||
|
||||
Required:
|
||||
--endpoint Admin API base, e.g. https://127.0.0.1:9000
|
||||
|
||||
Optional:
|
||||
--admin-token Bearer token for admin endpoints
|
||||
--window-spec Comma-separated run spec: window:duration_min:concurrency:ops_per_min:mix_name
|
||||
--soak-ratios Comma-separated mix ratios: label:read_pct:write_pct
|
||||
--workload-sizes Object-size workload set
|
||||
--job-bucket Job bucket scope for transition commands
|
||||
--job-prefix Prefix for generated commands (default: journal-nightly-stress)
|
||||
--tier Transition tier (default: empty)
|
||||
--out-dir Output directory for generated artifacts
|
||||
--no-admin-checks Omit generated admin check commands in the runbook
|
||||
--dry-run Generate artifacts only; do not create executable runner
|
||||
--help
|
||||
|
||||
Artifacts:
|
||||
- nightly stress runbook markdown
|
||||
- matrix/run scripts from manual_transition_soak_matrix.sh
|
||||
- runnable command entry template (includes failure snapshot hook)
|
||||
USAGE
|
||||
}
|
||||
|
||||
arg_value() {
|
||||
local flag="$1"
|
||||
local value="${2:-}"
|
||||
if [[ -z "$value" || "$value" == --* ]]; then
|
||||
echo "ERROR: missing value for $flag" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
local cmd="$1"
|
||||
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||
echo "ERROR: required command not found: $cmd" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--endpoint)
|
||||
ENDPOINT="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--admin-token)
|
||||
ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--window-spec)
|
||||
WINDOW_SPEC="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--soak-ratios)
|
||||
SOAK_RATIOS="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--workload-sizes)
|
||||
WORKLOAD_SIZES="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--job-bucket)
|
||||
JOB_BUCKET="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--job-prefix)
|
||||
JOB_PREFIX="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--tier)
|
||||
TIER="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--out-dir)
|
||||
OUT_DIR="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--no-admin-checks)
|
||||
RUN_ADMIN_CHECKS=false
|
||||
shift
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN=true
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: unknown arg: $1" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
main_matrix() {
|
||||
local args=("${STRESS_MATRIX_SCRIPT}" --endpoint "$ENDPOINT")
|
||||
|
||||
args+=(--window-spec "$WINDOW_SPEC")
|
||||
args+=(--soak-ratios "$SOAK_RATIOS")
|
||||
args+=(--workload-sizes "$WORKLOAD_SIZES")
|
||||
[[ -n "$ADMIN_TOKEN" ]] && args+=(--admin-token "$ADMIN_TOKEN")
|
||||
[[ "$RUN_ADMIN_CHECKS" == true ]] || args+=(--no-admin-checks)
|
||||
[[ -n "$OUT_DIR" ]] && args+=(--out-dir "$OUT_DIR")
|
||||
[[ "$DRY_RUN" == true ]] && args+=(--dry-run)
|
||||
|
||||
"${args[@]}"
|
||||
}
|
||||
|
||||
command_template() {
|
||||
local matrix_csv="${OUT_DIR}/nightly_soak_matrix.csv"
|
||||
local matrix_cmd="${OUT_DIR}/run_soak_matrix.sh"
|
||||
local notes="${OUT_DIR}/soak_notes.md"
|
||||
RUNBOOK_FILE="${OUT_DIR}/manual_transition_nightly_stress_runbook.md"
|
||||
COMMAND_FILE="${OUT_DIR}/run_nightly_stress_plan.sh"
|
||||
|
||||
cat >"$RUNBOOK_FILE" <<'RUNBOOK'
|
||||
# Manual transition nightly/stress stress-runbook
|
||||
|
||||
## Runtime baseline
|
||||
|
||||
- endpoint: __ENDPOINT__
|
||||
- bucket: __JOB_BUCKET__
|
||||
- prefix template: __JOB_PREFIX__
|
||||
- tier: __TIER__
|
||||
- matrix: __MATRIX_CSV__
|
||||
- matrix command: __MATRIX_CMD__
|
||||
- notes: __NOTES__
|
||||
|
||||
## Threshold template (editable)
|
||||
|
||||
- unknown failure ratio threshold: __UNKNOWN_FAILURE_RATIO_THRESHOLD__
|
||||
- queue-mismatch tolerance ratio: __QUEUE_MISMATCH_RATIO_THRESHOLD__
|
||||
- unknown failure count threshold: __UNKNOWN_FAILURE_COUNT_THRESHOLD__
|
||||
|
||||
## Failure snapshot policy
|
||||
|
||||
- on startup failure (missing job_id, API error), write a timestamped snapshot under '__OUT_DIR__/failure-snapshots'
|
||||
- if immediate status shows failure_reason, capture snapshot under '__OUT_DIR__/failure-snapshots'
|
||||
- for each failed run, run manual_transition_journal_audit.sh and keep its outputs as evidence
|
||||
|
||||
## Usage
|
||||
|
||||
Use the commands below directly:
|
||||
|
||||
- chmod +x __COMMAND_FILE__
|
||||
- bash __COMMAND_FILE__
|
||||
|
||||
## Files produced
|
||||
|
||||
- __MATRIX_CSV__
|
||||
- __MATRIX_CMD__
|
||||
- __NOTES__
|
||||
- __RUNBOOK_FILE__
|
||||
- __COMMAND_FILE__
|
||||
- __OUT_DIR__/failure-snapshots/*
|
||||
RUNBOOK
|
||||
|
||||
perl -0pi -e "s#__ENDPOINT__#${ENDPOINT//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__JOB_BUCKET__#${JOB_BUCKET//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__JOB_PREFIX__#${JOB_PREFIX//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__TIER__#${TIER//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__MATRIX_CSV__#${matrix_csv//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__MATRIX_CMD__#${matrix_cmd//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__NOTES__#${notes//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__RUNBOOK_FILE__#${RUNBOOK_FILE//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__COMMAND_FILE__#${COMMAND_FILE//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__OUT_DIR__#${OUT_DIR//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__UNKNOWN_FAILURE_RATIO_THRESHOLD__#${UNKNOWN_FAILURE_RATIO_THRESHOLD//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__QUEUE_MISMATCH_RATIO_THRESHOLD__#${QUEUE_MISMATCH_RATIO_THRESHOLD//#/#}#g" "$RUNBOOK_FILE"
|
||||
perl -0pi -e "s#__UNKNOWN_FAILURE_COUNT_THRESHOLD__#${UNKNOWN_FAILURE_COUNT_THRESHOLD//#/#}#g" "$RUNBOOK_FILE"
|
||||
|
||||
cat >"$COMMAND_FILE" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SOAK_MATRIX_CSV='__SOAK_MATRIX_CSV__'
|
||||
ENDPOINT='__ENDPOINT__'
|
||||
ADMIN_TOKEN='__ADMIN_TOKEN__'
|
||||
JOB_BUCKET='__JOB_BUCKET__'
|
||||
JOB_PREFIX='__JOB_PREFIX__'
|
||||
TIER='__TIER__'
|
||||
OUT_DIR='__OUT_DIR__'
|
||||
|
||||
: "${UNKNOWN_FAILURE_RATIO_THRESHOLD:=0.0}"
|
||||
: "${QUEUE_MISMATCH_RATIO_THRESHOLD:=0.0}"
|
||||
: "${UNKNOWN_FAILURE_COUNT_THRESHOLD:=0}"
|
||||
|
||||
SNAPSHOT_DIR="${OUT_DIR}/failure-snapshots"
|
||||
mkdir -p "$SNAPSHOT_DIR"
|
||||
|
||||
require_cmd() {
|
||||
local cmd="$1"
|
||||
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||
echo "ERROR: required command not found: $cmd" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
url_encode() {
|
||||
local value="$1"
|
||||
jq -rn --arg v "$value" '$v|@uri'
|
||||
}
|
||||
|
||||
snapshot_failure() {
|
||||
local run_tag="$1"
|
||||
local reason="$2"
|
||||
local job_id="$3"
|
||||
local ts
|
||||
local snapshot_dir
|
||||
|
||||
ts="$(date +%Y%m%dT%H%M%S)"
|
||||
snapshot_dir="${SNAPSHOT_DIR}/${run_tag}/${ts}"
|
||||
mkdir -p "$snapshot_dir"
|
||||
|
||||
{
|
||||
echo "run_tag=${run_tag}"
|
||||
echo "reason=${reason}"
|
||||
echo "job_id=${job_id}"
|
||||
echo "ts=${ts}"
|
||||
} >"${snapshot_dir}/snapshot.meta"
|
||||
|
||||
if [[ -n "$job_id" && "$job_id" != "NA" ]]; then
|
||||
curl -sS ${ADMIN_TOKEN:+-H "Authorization: Bearer ${ADMIN_TOKEN}"} -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" \
|
||||
>"${snapshot_dir}/job_status.json"
|
||||
./scripts/manual_transition_journal_audit.sh --endpoint "$ENDPOINT" --job-id "$job_id" ${ADMIN_TOKEN:+--admin-token "$ADMIN_TOKEN"} --out-dir "$snapshot_dir" || true
|
||||
fi
|
||||
|
||||
cp "$SOAK_MATRIX_CSV" "${snapshot_dir}/source_matrix.csv"
|
||||
}
|
||||
|
||||
run_entry() {
|
||||
local tag="$1"
|
||||
local duration_min="$2"
|
||||
local concurrency="$3"
|
||||
local ops_per_min="$4"
|
||||
local mix_name="$5"
|
||||
local read_pct="$6"
|
||||
local write_pct="$7"
|
||||
local size="$8"
|
||||
local expected_ops="$9"
|
||||
local budget_status="${10}"
|
||||
|
||||
local prefix
|
||||
local query
|
||||
local url
|
||||
local response
|
||||
local job_id
|
||||
local status
|
||||
local headers=()
|
||||
|
||||
if [[ "$budget_status" == "over-budget" ]]; then
|
||||
echo "[skip] ${tag} ${size} over budget: expected_ops=${expected_ops}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$UNKNOWN_FAILURE_COUNT_THRESHOLD" -gt 0 && "$expected_ops" -gt "$UNKNOWN_FAILURE_COUNT_THRESHOLD" ]]; then
|
||||
:
|
||||
fi
|
||||
|
||||
prefix="${JOB_PREFIX}/${tag}/${size}/${read_pct}r${write_pct}w"
|
||||
query="bucket=$(url_encode "$JOB_BUCKET")"
|
||||
query="${query}&prefix=$(url_encode "$prefix")"
|
||||
query="${query}&maxObjects=100000"
|
||||
query="${query}&maxDurationSeconds=$((duration_min * 60))"
|
||||
query="${query}&mode=async&dryRun=false"
|
||||
if [[ -n "$TIER" ]]; then
|
||||
query="${query}&tier=$(url_encode "$TIER")"
|
||||
fi
|
||||
|
||||
url="${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/run?${query}"
|
||||
|
||||
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||
headers+=("-H" "Authorization: Bearer ${ADMIN_TOKEN}")
|
||||
fi
|
||||
|
||||
echo "==> run=${tag} size=${size} mix=${mix_name} read_pct=${read_pct} write_pct=${write_pct} duration_min=${duration_min}"
|
||||
if ! response="$(curl -sS "${headers[@]}" -X POST "$url")"; then
|
||||
snapshot_failure "$tag" "curl_post_failed" "NA"
|
||||
return 1
|
||||
fi
|
||||
job_id="$(printf '%s' "$response" | jq -r '.job_id // empty')"
|
||||
if [[ -z "$job_id" || "$job_id" == "null" ]]; then
|
||||
snapshot_failure "$tag" "missing_job_id" "NA"
|
||||
return 1
|
||||
fi
|
||||
|
||||
status="$(curl -sS "${headers[@]}" -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" | jq -r '.failure_reason // empty')"
|
||||
if [[ -n "$status" && "$status" != "null" ]]; then
|
||||
snapshot_failure "$tag" "failure_reason=${status}" "$job_id"
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
require_cmd bash
|
||||
require_cmd curl
|
||||
require_cmd jq
|
||||
require_cmd awk
|
||||
|
||||
if [[ ! -f "$SOAK_MATRIX_CSV" ]]; then
|
||||
echo "ERROR: expected matrix file missing: $SOAK_MATRIX_CSV" >&2
|
||||
echo "Run manual_transition_soak_matrix.sh first." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
while IFS=',' read -r window duration_min concurrency ops_per_min mix_name read_pct write_pct size expected_ops run_id budget_status; do
|
||||
if [[ -z "$window" || "$window" == "window" ]]; then
|
||||
continue
|
||||
fi
|
||||
run_entry "${window}" "${duration_min}" "${concurrency}" "${ops_per_min}" "${mix_name}" "${read_pct}" "${write_pct}" "${size}" "${expected_ops}" "${budget_status}" || true
|
||||
done < <(tail -n +2 "$SOAK_MATRIX_CSV")
|
||||
}
|
||||
|
||||
main "$@"
|
||||
EOF
|
||||
|
||||
perl -0pi -e "s#__SOAK_MATRIX_CSV__#${matrix_csv//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__ENDPOINT__#${ENDPOINT//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__ADMIN_TOKEN__#${ADMIN_TOKEN//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__JOB_BUCKET__#${JOB_BUCKET//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__JOB_PREFIX__#${JOB_PREFIX//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__TIER__#${TIER//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__OUT_DIR__#${OUT_DIR//#/#}#g" "$COMMAND_FILE"
|
||||
chmod +x "$COMMAND_FILE"
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
if [[ -z "$ENDPOINT" ]]; then
|
||||
echo "ERROR: --endpoint is required" >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$OUT_DIR" ]]; then
|
||||
OUT_DIR="${PROJECT_ROOT}/target/manual-transition-nightly-stress-runbook/$(date +%Y%m%dT%H%M%S)"
|
||||
fi
|
||||
mkdir -p "$OUT_DIR"
|
||||
require_cmd awk
|
||||
require_cmd jq
|
||||
|
||||
main_matrix
|
||||
command_template
|
||||
|
||||
echo "Generated runbook: ${RUNBOOK_FILE}"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -133,6 +133,10 @@ parse_ratio() {
|
||||
echo "ERROR: mix percentages must be integers: $spec" >&2
|
||||
exit 1
|
||||
fi
|
||||
if (( read_pct + write_pct != 100 )); then
|
||||
echo "ERROR: read/write percentages must sum to 100: $spec" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$name|$read_pct|$write_pct"
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user