gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-05 12:57:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ecstore): attribute manual transition worker failures (#5324)

Browse Source 
* feat(ecstore): attribute manual transition worker failures

Add manual transition worker failure reason tracking and persistence recovery compatibility for checksum validation.

Co-Authored-By: heihutu <heihutu@gmail.com>

* chore(ilm): add manual transition diagnostics scripts

Co-Authored-By: heihutu <heihutu@gmail.com>

* style(ecstore): format manual transition attribution

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(ecstore): avoid copying failure reasons via clone

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-07-27 14:06:12 +08:00
committed by GitHub
parent f566b382a0
commit 9d84056d7b
8 changed files with 1887 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/manual_transition_debug.sh
Executable
+253
View File
@@ -0,0 +1,253 @@
#!/usr/bin/env bash
#
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -euo pipefail
ENDPOINT=""
JOB_ID=""
ADMIN_TOKEN=""
LOG_GLOB="/var/log/rustfs/*.log"
METRIC_TYPES="1"
METRIC_SAMPLES="1"
METRIC_INTERVAL=""
BUCKET_FILTER=""
SHOW_HELP=false
usage() {
cat <<'USAGE'
Usage:
scripts/manual_transition_debug.sh --endpoint <admin-api-base> --job-id <job-id> [options]
Required:
--endpoint Admin endpoint base, e.g. http://127.0.0.1:9000
--job-id Manual transition job_id (UUID)
Optional:
--admin-token Bearer token for /admin/v3 endpoints
--log-glob Log glob to scan, defaults to /var/log/rustfs/*.log
--metric-types Metrics types bitmask, defaults to 1 (SCANNER)
--metric-samples Metrics samples, defaults to 1
--metric-interval Metrics interval parameter, passed as-is to query string
--bucket Optional bucket filter for focused log output
-h, --help Show this help
Output sections:
- lifecycle_worker_state structured events (job-scoped)
- manual-transition admin job status
- lifecycle_tier_operation_failed and admin_ilm_transition_state traces
- metrics snapshot (aggregated.scanner.lifecycle_transition)
USAGE
}
require_cmd() {
local cmd="$1"
if ! command -v "$cmd" >/dev/null 2>&1; then
echo "ERROR: required command not found: $cmd" >&2
exit 1
fi
}
api_get() {
local path="$1"
local url="${ENDPOINT%/}${path}"
if [[ -n "$ADMIN_TOKEN" ]]; then
curl -sS -H "Authorization: Bearer ${ADMIN_TOKEN}" "$url"
return
fi
curl -sS "$url"
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--endpoint)
ENDPOINT="${2:-}"
shift 2
;;
--job-id)
JOB_ID="${2:-}"
shift 2
;;
--admin-token)
ADMIN_TOKEN="${2:-}"
shift 2
;;
--log-glob)
LOG_GLOB="${2:-}"
shift 2
;;
--metric-types)
METRIC_TYPES="${2:-}"
shift 2
;;
--metric-samples)
METRIC_SAMPLES="${2:-}"
shift 2
;;
--metric-interval)
METRIC_INTERVAL="${2:-}"
shift 2
;;
--bucket)
BUCKET_FILTER="${2:-}"
shift 2
;;
-h|--help)
SHOW_HELP=true
shift
;;
*)
echo "ERROR: unknown arg: $1" >&2
usage
exit 1
;;
esac
done
}
validate() {
if [[ "$SHOW_HELP" == true ]]; then
usage
exit 0
fi
if [[ -z "$ENDPOINT" || -z "$JOB_ID" ]]; then
echo "ERROR: --endpoint and --job-id are required" >&2
usage
exit 1
fi
if [[ -z "$LOG_GLOB" ]]; then
echo "ERROR: --log-glob must not be empty" >&2
exit 1
fi
}
build_admin_metrics_path() {
local path="/rustfs/admin/v3/metrics?types=${METRIC_TYPES}&n=${METRIC_SAMPLES}"
if [[ -n "$METRIC_INTERVAL" ]]; then
path="${path}&interval=${METRIC_INTERVAL}"
fi
printf '%s' "$path"
}
run_json_stream() {
local query="$1"
local filter="$2"
local log_lines; log_lines="$(mktemp)"
local log_errors; log_errors="$(mktemp)"
local log_paths=()
local match
while IFS= read -r match; do
[[ -n "$match" ]] && log_paths+=("$match")
done < <(compgen -G "$LOG_GLOB" || true)
if [[ "${#log_paths[@]}" -eq 0 ]]; then
log_paths=("$LOG_GLOB")
fi
if ! rg -n --color=never -- "$query" -- "${log_paths[@]}" >"$log_lines" 2>"$log_errors"; then
:
fi
if [[ -s "$log_errors" ]]; then
sed 's/^/DEBUG: /' "$log_errors"
fi
jq -R "$filter" <"$log_lines"
rm -f "$log_lines" "$log_errors"
}
run_worker_logs() {
local base_filter='fromjson? | select(.event == "lifecycle_worker_state")'
local manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")))'
local dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) ) | .state'
if [[ -n "$BUCKET_FILTER" ]]; then
manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")) and .bucket == env.BUCKET_FILTER)'
dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) and .bucket == env.BUCKET_FILTER) | .state'
fi
echo "## lifecycle_worker_state (all)"
run_json_stream "lifecycle_worker_state" "$base_filter"
echo "## lifecycle_worker_state (manual_transition + job filter: ${JOB_ID})"
JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \
run_json_stream "lifecycle_worker_state" "$manual_filter" || true
echo "## lifecycle_worker_state state distribution (manual_transition only)"
JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \
run_json_stream "lifecycle_worker_state" "$dist_filter" | jq -r . | sort | uniq -c | sort -nr || true
}
run_admin_job() {
echo "## manual transition job status"
api_get "/rustfs/admin/v3/ilm/transition/jobs/${JOB_ID}" \
| jq '{status, mode, job_id, failure_reason, report: .report, queue_snapshot: .queue_snapshot}'
}
run_tier_operation_failures() {
echo "## lifecycle_tier_operation_failed events"
local filter='fromjson? | select(.event == "lifecycle_tier_operation_failed") | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}'
if [[ -n "$BUCKET_FILTER" ]]; then
filter='fromjson? | select(.event == "lifecycle_tier_operation_failed" and .bucket == env.BUCKET_FILTER) | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}'
fi
JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \
run_json_stream "lifecycle_tier_operation_failed" "$filter" || true
}
run_admin_events() {
echo "## admin_ilm_transition_state (job filter)"
local filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}'
if [[ -n "$BUCKET_FILTER" ]]; then
filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID and .bucket == env.BUCKET_FILTER) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}'
fi
JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \
run_json_stream "admin_ilm_transition_state" "$filter" || true
}
run_metrics() {
echo "## metrics: scanner.lifecycle_transition"
local path
path="$(build_admin_metrics_path)"
api_get "$path" | jq '.aggregated.scanner.lifecycle_transition'
echo "## metrics: scanner lifecycle cycle"
api_get "$path" | jq '.aggregated.scanner | {lifecycle_transition: .lifecycle_transition, current_cycle_ilm_actions: .current_cycle_ilm_actions, current_cycle_lifecycle_transition_actions: .current_cycle_lifecycle_transition_actions}'
}
main() {
parse_args "$@"
validate
require_cmd curl
require_cmd jq
require_cmd rg
run_worker_logs
run_admin_events
run_tier_operation_failures
run_admin_job
run_metrics
}
main "$@"
scripts/manual_transition_journal_audit.sh
Executable
+590
View File
@@ -0,0 +1,590 @@
#!/usr/bin/env bash
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
ENDPOINT=""
JOB_ID=""
ADMIN_TOKEN=""
LOG_GLOB="/var/log/rustfs/*.log"
METRIC_TYPES="1"
METRIC_SAMPLES="1"
METRIC_INTERVAL=""
TASK_BUCKET_FILTER=""
OUT_DIR=""
SYS_BUCKET="${RUSTFS_META_BUCKET:-.rustfs.sys}"
TASK_PREFIX="ilm/manual-transition/tasks"
RESULT_PREFIX="ilm/manual-transition/results"
MC_BIN="${MC_BIN:-mc}"
MC_ALIAS=""
MC_ENDPOINT=""
MC_ACCESS_KEY=""
MC_SECRET_KEY=""
MC_ALIAS_CREATED=false
SHOW_HELP=false
DRY_RUN=false
usage() {
cat <<'USAGE'
Usage:
scripts/manual_transition_journal_audit.sh --endpoint <admin-api> --job-id <job-uuid> [options]
Required:
--endpoint Admin endpoint base, e.g. https://127.0.0.1:9000
--job-id Manual transition job_id
Optional:
--admin-token Bearer token for /admin/v3 endpoints
--log-glob Log glob, default /var/log/rustfs/*.log
--metric-types /admin/v3/metrics type flags, default 1
--metric-samples /admin/v3/metrics sample count, default 1
--metric-interval /admin/v3/metrics interval query param
--bucket-filter Optional bucket filter for logs
--sys-bucket Meta bucket for journal read (default .rustfs.sys)
--mc-alias Reuse existing mc alias (skip alias set)
--mc-endpoint mc endpoint for .rustfs.sys access
--mc-access-key mc access key
--mc-secret-key mc secret key
--out-dir Artifact output dir
--dry-run
--help Show usage
Output:
- Journal CSV/JSON samples for manual transition tasks and worker results
- Reconcile report (task_count/result_count/missing/mismatch)
- /admin/v3 lifecycle_worker_state / admin_ilm_transition_state events
- lifecycle_tier_operation_failed logs
- aggregated lifecycle transition metrics (admin endpoint sample)
USAGE
}
require_bash4() {
if (( BASH_VERSINFO[0] < 4 )); then
echo "ERROR: manual transition journal audit requires bash >= 4.0" >&2
exit 1
fi
}
arg_value() {
local flag="$1"
local value="${2:-}"
if [[ -z "$value" || "$value" == --* ]]; then
echo "ERROR: missing value for $flag" >&2
exit 1
fi
printf '%s' "$value"
}
require_cmd() {
local cmd="$1"
if ! command -v "$cmd" >/dev/null 2>&1; then
echo "ERROR: command not found: $cmd" >&2
exit 1
fi
}
require_nonempty() {
local value="$1"
local label="$2"
if [[ -z "$value" ]]; then
echo "ERROR: $label is required" >&2
exit 1
fi
}
normalize_uuid() {
local raw="${1//-/}"
printf '%s' "${raw,,}"
}
validate_uuid() {
local normalized
normalized="$(normalize_uuid "$1")"
if ! [[ "$normalized" =~ ^[0-9a-f]{32}$ ]]; then
echo "ERROR: invalid uuid (expect UUIDv4): $1" >&2
exit 1
fi
printf '%s' "$normalized"
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--job-id) JOB_ID="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--admin-token) ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--log-glob) LOG_GLOB="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--metric-types) METRIC_TYPES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--metric-samples) METRIC_SAMPLES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--metric-interval) METRIC_INTERVAL="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--bucket-filter) TASK_BUCKET_FILTER="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--sys-bucket) SYS_BUCKET="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--mc-alias) MC_ALIAS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--mc-endpoint) MC_ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--mc-access-key) MC_ACCESS_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--mc-secret-key) MC_SECRET_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--dry-run) DRY_RUN=true; shift ;;
-h|--help) SHOW_HELP=true; shift ;;
*)
echo "ERROR: unknown arg: $1" >&2
usage
exit 1
;;
esac
done
}
validate_args() {
if [[ "$SHOW_HELP" == true ]]; then
usage
exit 0
fi
require_nonempty "$ENDPOINT" "--endpoint"
require_nonempty "$JOB_ID" "--job-id"
JOB_ID="$(validate_uuid "$JOB_ID")"
if [[ "$LOG_GLOB" == "" ]]; then
echo "ERROR: --log-glob must not be empty" >&2
exit 1
fi
if [[ -z "$MC_ALIAS" ]]; then
if [[ -z "$MC_ENDPOINT" || -z "$MC_ACCESS_KEY" || -z "$MC_SECRET_KEY" ]]; then
echo "ERROR: either --mc-alias or (--mc-endpoint + --mc-access-key + --mc-secret-key) is required" >&2
exit 1
fi
fi
}
cleanup_mc_alias() {
if [[ "$MC_ALIAS_CREATED" == true ]]; then
"${MC_BIN}" alias remove "$MC_ALIAS" >/dev/null 2>&1 || true
fi
}
setup_output() {
if [[ -z "$OUT_DIR" ]]; then
OUT_DIR="${PROJECT_ROOT}/target/manual-transition-journal-audit/$(date +%Y%m%dT%H%M%S)"
fi
mkdir -p "$OUT_DIR"
SUMMARY_TXT="${OUT_DIR}/audit_summary.txt"
TASKS_CSV="${OUT_DIR}/tasks.csv"
RESULTS_CSV="${OUT_DIR}/results.csv"
RECONCILE_CSV="${OUT_DIR}/journal_reconcile.csv"
COMMANDS_TXT="${OUT_DIR}/commands.txt"
{
echo "task_key,bucket,object,version_id,storage_class,queued_at_unix_nanos,task_object"
} > "$TASKS_CSV"
{
echo "task_key,result,completion_reason,completed_at_unix_nanos,result_object"
} > "$RESULTS_CSV"
{
echo "task_key,bucket,object,version_id,storage_class,task_has_result,result,result_reason,task_job_id_match,result_job_id_match,task_path,result_path"
} > "$RECONCILE_CSV"
}
setup_mc_alias() {
if [[ "$MC_ALIAS" == "" ]]; then
MC_ALIAS="mt-journal-audit-${JOB_ID}"
MC_ALIAS_CREATED=true
if [[ "$DRY_RUN" == true ]]; then
echo "[DRY-RUN] mc alias set ${MC_ALIAS} ${MC_ENDPOINT} ${MC_ACCESS_KEY} ***"
else
"${MC_BIN}" alias set "${MC_ALIAS}" "${MC_ENDPOINT}" "${MC_ACCESS_KEY}" "${MC_SECRET_KEY}" >/dev/null
fi
fi
if ! [[ "$DRY_RUN" == true ]]; then
"${MC_BIN}" ls "${MC_ALIAS}/${SYS_BUCKET}" >/dev/null
fi
trap cleanup_mc_alias EXIT
}
api_get() {
local path="$1"
local url="${ENDPOINT%/}${path}"
if [[ -n "$ADMIN_TOKEN" ]]; then
curl -sS -H "Authorization: Bearer ${ADMIN_TOKEN}" "$url"
else
curl -sS "$url"
fi
}
build_admin_metrics_path() {
local path="/rustfs/admin/v3/metrics?types=${METRIC_TYPES}&n=${METRIC_SAMPLES}"
if [[ -n "$METRIC_INTERVAL" ]]; then
path="${path}&interval=${METRIC_INTERVAL}"
fi
printf '%s' "$path"
}
run_json_stream() {
local query="$1"
local filter="$2"
local log_lines; log_lines="$(mktemp)"
local log_errors; log_errors="$(mktemp)"
local log_paths=()
local match
while IFS= read -r match; do
[[ -n "$match" ]] && log_paths+=("$match")
done < <(compgen -G "$LOG_GLOB" || true)
if [[ "${#log_paths[@]}" -eq 0 ]]; then
log_paths=("$LOG_GLOB")
fi
if ! rg -n --color=never -- "$query" -- "${log_paths[@]}" >"$log_lines" 2>"$log_errors"; then
:
fi
if [[ -s "$log_errors" ]]; then
sed 's/^/DEBUG: /' "$log_errors"
fi
if [[ -s "$log_lines" ]]; then
jq -R "$filter" <"$log_lines"
else
echo "[]"
fi
rm -f "$log_lines" "$log_errors"
}
run_worker_logs() {
local base_filter='fromjson? | select(.event == "lifecycle_worker_state")'
local manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")))'
local dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) ) | .state'
if [[ -n "$TASK_BUCKET_FILTER" ]]; then
manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")) and .bucket == env.TASK_BUCKET_FILTER)'
dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) and .bucket == env.TASK_BUCKET_FILTER) | .state'
fi
echo "## lifecycle_worker_state all"
run_json_stream "lifecycle_worker_state" "$base_filter" || true
echo
echo "## lifecycle_worker_state manual_transition + job filter"
JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" run_json_stream "lifecycle_worker_state" "$manual_filter" || true
echo
echo "## lifecycle_worker_state state distribution"
JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" run_json_stream "lifecycle_worker_state" "$dist_filter" | jq -r . | sort | uniq -c | sort -nr || true
}
run_admin_job() {
echo "## manual transition job status"
api_get "/rustfs/admin/v3/ilm/transition/jobs/${JOB_ID}" \
| jq '{status, mode, job_id, report: .report, queue_snapshot: .queue_snapshot, cancel_requested}'
}
run_admin_events() {
echo "## admin_ilm_transition_state events"
local filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}'
if [[ -n "$TASK_BUCKET_FILTER" ]]; then
filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID and .bucket == env.TASK_BUCKET_FILTER) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}'
fi
JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" \
run_json_stream "admin_ilm_transition_state" "$filter" || true
}
run_tier_operation_failures() {
echo "## lifecycle_tier_operation_failed events"
local filter='fromjson? | select(.event == "lifecycle_tier_operation_failed") | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}'
if [[ -n "$TASK_BUCKET_FILTER" ]]; then
filter='fromjson? | select(.event == "lifecycle_tier_operation_failed" and .bucket == env.TASK_BUCKET_FILTER) | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}'
fi
JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" \
run_json_stream "lifecycle_tier_operation_failed" "$filter" || true
}
run_metrics() {
echo "## metrics: scanner.lifecycle_transition"
local path
path="$(build_admin_metrics_path)"
api_get "$path" | jq '.aggregated.scanner.lifecycle_transition'
}
journal_key_prefix() {
local prefix="$1"
local shard_a="${JOB_ID:0:2}"
local shard_b="${JOB_ID:2:2}"
printf '%s/%s/%s/%s' "$prefix" "$shard_a" "$shard_b" "$JOB_ID"
}
csv_escape() {
local value="${1:-}"
value="${value//\"/\"\"}"
printf '\"%s\"' "$value"
}
read_json_file() {
local object_key="$1"
if [[ "$DRY_RUN" == true ]]; then
echo ""
return 0
fi
"${MC_BIN}" cat "${MC_ALIAS}/${SYS_BUCKET}/${object_key}"
}
collect_journals() {
if [[ "$DRY_RUN" == true ]]; then
echo "[DRY-RUN] skip .rustfs.sys object reads."
echo "dry_run=true" > "$SUMMARY_TXT"
return 0
fi
local task_prefix result_prefix
task_prefix="$(journal_key_prefix "$TASK_PREFIX")"
result_prefix="$(journal_key_prefix "$RESULT_PREFIX")"
local task_objs
local result_objs
local object_name
local task_key
local raw
local record_task_job record_result_job
local expected_job
expected_job="$JOB_ID"
local -A task_bucket task_object task_version task_storage task_queued task_job task_path
local -A result_status result_reason result_completed result_job result_path
local -A task_seen
local -A result_seen
task_objs="$("${MC_BIN}" --json ls --recursive "${MC_ALIAS}/${SYS_BUCKET}/${task_prefix}" | jq -r 'select(.type == "file") | .key')"
result_objs="$("${MC_BIN}" --json ls --recursive "${MC_ALIAS}/${SYS_BUCKET}/${result_prefix}" | jq -r 'select(.type == "file") | .key')"
while IFS= read -r object_name; do
[[ -z "$object_name" ]] && continue
task_key="${object_name##*/}"
task_key="${task_key%.json}"
if ! [[ "$task_key" =~ ^[0-9a-f]{64}$ ]]; then
echo "WARN: ignore invalid task object name: ${object_name}" >&2
continue
fi
raw="$(read_json_file "$object_name")"
if ! record_task_job="$(printf '%s' "$raw" | jq -r '.record.job_id // empty' 2>/dev/null)"; then
echo "WARN: parse error task object: ${object_name}" >&2
continue
fi
if [[ -z "$record_task_job" ]]; then
echo "WARN: task object missing record.job_id: ${object_name}" >&2
continue
fi
task_seen["$task_key"]="1"
task_bucket["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.bucket // empty' )"
task_object["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.object // empty' )"
task_version["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.version_id // empty' )"
task_storage["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.storage_class // empty' )"
task_queued["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.queued_at_unix_nanos // empty' )"
task_job["$task_key"]="$(printf '%s' "$record_task_job")"
task_path["$task_key"]="$object_name"
printf '%s,%s,%s,%s,%s,%s,%s\n' \
"$(csv_escape "$task_key")" \
"$(csv_escape "${task_bucket[$task_key]}")" \
"$(csv_escape "${task_object[$task_key]}")" \
"$(csv_escape "${task_version[$task_key]}")" \
"$(csv_escape "${task_storage[$task_key]}")" \
"$(csv_escape "${task_queued[$task_key]}")" \
"$(csv_escape "$object_name")" >> "$TASKS_CSV"
done <<< "$task_objs"
while IFS= read -r object_name; do
[[ -z "$object_name" ]] && continue
task_key="${object_name##*/}"
task_key="${task_key%.json}"
if ! [[ "$task_key" =~ ^[0-9a-f]{64}$ ]]; then
echo "WARN: ignore invalid result object name: ${object_name}" >&2
continue
fi
raw="$(read_json_file "$object_name")"
if ! record_result_job="$(printf '%s' "$raw" | jq -r '.record.job_id // empty' 2>/dev/null)"; then
echo "WARN: parse error result object: ${object_name}" >&2
continue
fi
if [[ -z "$record_result_job" ]]; then
echo "WARN: result object missing record.job_id: ${object_name}" >&2
continue
fi
result_seen["$task_key"]="1"
result_status["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.result // .result // empty' )"
result_reason["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.failure_reason // empty' )"
result_completed["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.completed_at_unix_nanos // empty' )"
result_job["$task_key"]="$(printf '%s' "$record_result_job")"
result_path["$task_key"]="$object_name"
printf '%s,%s,%s,%s,%s\n' \
"$(csv_escape "$task_key")" \
"$(csv_escape "${result_status[$task_key]}")" \
"$(csv_escape "${result_reason[$task_key]}")" \
"$(csv_escape "${result_completed[$task_key]}")" \
"$(csv_escape "$object_name")" >> "$RESULTS_CSV"
done <<< "$result_objs"
local task_count=0
local result_count=0
local missing_count=0
local mismatch_count=0
local completed_count=0
local failed_count=0
local task_job_id_match
local result_job_id_match
local task_record_job
local result_record_job
local -A failure_by_reason=()
task_count="${#task_seen[@]}"
result_count="${#result_seen[@]}"
for task_key in "${!task_seen[@]}"; do
local result=""
local reason=""
local task_has_result="false"
if [[ -n "${result_seen[$task_key]+x}" ]]; then
task_has_result="true"
result="${result_status[$task_key]}"
reason="${result_reason[$task_key]}"
if [[ "${result}" == "Completed" ]]; then
completed_count=$((completed_count + 1))
elif [[ "${result}" == "TierFailure" ]]; then
failed_count=$((failed_count + 1))
reason="${reason:-Unknown}"
failure_by_reason["$reason"]=$(( ${failure_by_reason["$reason"]:-0} + 1 ))
fi
else
missing_count=$((missing_count + 1))
fi
task_job_id_match="false"
result_job_id_match="false"
task_record_job="${task_job[$task_key]-}"
if [[ -n "$task_record_job" && "$(normalize_uuid "$task_record_job")" == "$expected_job" ]]; then
task_job_id_match="true"
fi
result_record_job="${result_job[$task_key]-}"
if [[ -n "$result_record_job" && "$(normalize_uuid "$result_record_job")" == "$expected_job" ]]; then
result_job_id_match="true"
fi
if [[ "$task_job_id_match" != "true" ]] || [[ -n "${result_seen[$task_key]+x}" && "$result_job_id_match" != "true" ]]; then
mismatch_count=$((mismatch_count + 1))
fi
printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \
"$(csv_escape "$task_key")" \
"$(csv_escape "${task_bucket[$task_key]}")" \
"$(csv_escape "${task_object[$task_key]}")" \
"$(csv_escape "${task_version[$task_key]}")" \
"$(csv_escape "${task_storage[$task_key]}")" \
"$(csv_escape "$task_has_result")" \
"$(csv_escape "${result_status[$task_key]}")" \
"$(csv_escape "${result_reason[$task_key]}")" \
"$(csv_escape "$task_job_id_match")" \
"$(csv_escape "$result_job_id_match")" \
"$(csv_escape "${task_path[$task_key]}")" \
"$(csv_escape "${result_path[$task_key]-}")" >> "$RECONCILE_CSV"
done
local orphan_count=0
for task_key in "${!result_seen[@]}"; do
if [[ -z "${task_seen[$task_key]+x}" ]]; then
orphan_count=$((orphan_count + 1))
printf 'orphan_result_task_key=%s\n' "$task_key" >> "$SUMMARY_TXT"
fi
done
{
echo "job_id=$JOB_ID"
echo "sys_bucket=$SYS_BUCKET"
echo "expected_task_prefix=$task_prefix"
echo "expected_result_prefix=$result_prefix"
echo "task_count=${task_count}"
echo "result_count=${result_count}"
echo "missing_result_count=${missing_count}"
echo "job_id_mismatch_count=${mismatch_count}"
echo "orphan_result_count=${orphan_count}"
echo "completed_result_count=${completed_count}"
echo "failed_result_count=${failed_count}"
for reason in "${!failure_by_reason[@]}"; do
echo "result_failure_${reason// /_}=${failure_by_reason[$reason]}"
done
} > "$SUMMARY_TXT"
{
echo
echo "## journal summary"
echo "- tasks: ${task_count}"
echo "- results: ${result_count}"
echo "- missing result for task: ${missing_count}"
echo "- orphaned result rows: ${orphan_count}"
echo "- completed result: ${completed_count}"
echo "- failed result: ${failed_count}"
} >> "$SUMMARY_TXT"
}
build_command_notes() {
{
echo "# Run snippets (to use manually)"
echo
echo "curl -sS ${ENDPOINT}/rustfs/admin/v3/metrics?types=${METRIC_TYPES}&n=${METRIC_SAMPLES} | jq '.aggregated.scanner.lifecycle_transition'"
echo "scripts/manual_transition_journal_audit.sh --endpoint ${ENDPOINT} --job-id ${JOB_ID} --admin-token <token> --sys-bucket ${SYS_BUCKET}"
echo "scripts/manual_transition_journal_audit.sh --endpoint ${ENDPOINT} --job-id ${JOB_ID} --admin-token <token> --sys-bucket ${SYS_BUCKET} --out-dir ${OUT_DIR}"
if [[ "$DRY_RUN" == true ]]; then
echo "# Add --mc-endpoint, --mc-access-key, --mc-secret-key when writing to .rustfs.sys."
fi
} > "$COMMANDS_TXT"
}
main() {
parse_args "$@"
if [[ "$SHOW_HELP" == true ]]; then
usage
exit 0
fi
require_bash4
validate_args
require_cmd "$MC_BIN"
require_cmd curl
require_cmd jq
require_cmd rg
setup_output
setup_mc_alias
build_command_notes
run_worker_logs
echo
run_admin_events
echo
run_tier_operation_failures
echo
run_admin_job
echo
run_metrics
echo
collect_journals
echo "Artifacts: $OUT_DIR"
echo " - $TASKS_CSV"
echo " - $RESULTS_CSV"
echo " - $RECONCILE_CSV"
echo " - $SUMMARY_TXT"
echo " - $COMMANDS_TXT"
}
main "$@"
scripts/manual_transition_mixed_rollout_matrix.sh
Executable
+291
View File
@@ -0,0 +1,291 @@
#!/usr/bin/env bash
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
ENDPOINT=""
ADMIN_TOKEN=""
OUT_DIR=""
PHASE_MATRIX="request-only:100:0:30,canary:90:10:120,mixed:50:50:240,full-rollout:0:100:360,rollback:100:0:30"
CONCURRENCIES="8,16,32"
OBJECT_COUNTS="5k,20k,50k"
JOB_BUCKET="manual-transition"
JOB_PREFIX="journal-mixed-rollout"
READ_RATIO="90"
RUN_ADMIN_CHECKS=true
DRY_RUN=false
usage() {
cat <<'USAGE'
Usage:
scripts/manual_transition_mixed_rollout_matrix.sh --endpoint <admin-api> [options]
Required:
--endpoint Admin API base, e.g. https://127.0.0.1:9000
Optional:
--admin-token Bearer token for admin calls
--phase-matrix Comma-separated phase spec: name:old_pct:new_pct:duration_min
--concurrencies Comma-separated concurrency list
--object-counts Comma-separated queue-size list (for plan rows)
--job-bucket Manual transition source bucket for sample commands
--job-prefix Prefix for generated job notes
--read-ratio Target read ratio percentage for mixed workload
--no-admin-checks Skip generated admin check commands
--out-dir Output dir
--dry-run
--help
Examples:
scripts/manual_transition_mixed_rollout_matrix.sh \
--endpoint https://127.0.0.1:9000 \
--admin-token "$TOKEN" \
--phase-matrix "request-only:100:0:30,canary:90:10:120,full:0:100:240"
USAGE
}
arg_value() {
local flag="$1"
local value="${2:-}"
if [[ -z "$value" || "$value" == --* ]]; then
echo "ERROR: missing value for $flag" >&2
exit 1
fi
printf '%s' "$value"
}
require_cmd() {
local cmd="$1"
if ! command -v "$cmd" >/dev/null 2>&1; then
echo "ERROR: command not found: $cmd" >&2
exit 1
fi
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--admin-token) ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--phase-matrix) PHASE_MATRIX="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--concurrencies) CONCURRENCIES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--object-counts) OBJECT_COUNTS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--job-bucket) JOB_BUCKET="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--job-prefix) JOB_PREFIX="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--read-ratio) READ_RATIO="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--no-admin-checks) RUN_ADMIN_CHECKS=false; shift ;;
--dry-run) DRY_RUN=true; shift ;;
-h|--help) usage; exit 0 ;;
*)
echo "ERROR: unknown arg: $1" >&2
usage
exit 1
;;
esac
done
}
trim() {
echo "$1" | awk '{$1=$1;print}'
}
parse_phase_row() {
local spec="$1"
local name old_ratio new_ratio duration_min
name="$(echo "$spec" | cut -d':' -f1)"
old_ratio="$(echo "$spec" | cut -d':' -f2)"
new_ratio="$(echo "$spec" | cut -d':' -f3)"
duration_min="$(echo "$spec" | cut -d':' -f4)"
name="$(trim "$name")"
old_ratio="$(trim "$old_ratio")"
new_ratio="$(trim "$new_ratio")"
duration_min="$(trim "$duration_min")"
if [[ -z "$name" || -z "$old_ratio" || -z "$new_ratio" || -z "$duration_min" ]]; then
echo "ERROR: invalid phase spec: $spec" >&2
exit 1
fi
if ! [[ "$old_ratio" =~ ^[0-9]+$ && "$new_ratio" =~ ^[0-9]+$ && "$duration_min" =~ ^[0-9]+$ ]]; then
echo "ERROR: phase ratio/duration must be integers: $spec" >&2
exit 1
fi
echo "$name|$old_ratio|$new_ratio|$duration_min"
}
gate_for_phase() {
case "$1" in
request-only)
echo "No mixed-version errors; unknown failures should stay 0 in job report; log ratio must trend down."
;;
canary)
echo "old/new mismatch in worker results = 0; no Unknown job failures."
;;
mixed)
echo "request-only + canary acceptance gates both pass for 2+ windows, then proceed."
;;
full-rollout|full)
echo "full canary-to-full transition completed, no tier_failure in report, worker_result mismatch = 0."
;;
rollback)
echo "all rollout checks reverse cleanly; queued -> completed in job terminal state."
;;
*)
echo "run admin job status + journal reconcile + metric delta sanity check."
;;
esac
}
admin_check_cmd() {
local phase="$1"
local job_id_ref="$2"
local metric_types="$3"
local metric_samples="$4"
local metric_path
local job_url
local metric_url
local job_filter='.report, .queue_snapshot, .failure_reason'
local metric_filter='.aggregated.scanner.lifecycle_transition'
if [[ "$RUN_ADMIN_CHECKS" != true ]]; then
return
fi
metric_path="/rustfs/admin/v3/metrics?types=${metric_types}&n=${metric_samples}"
job_url="${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id_ref}"
metric_url="${ENDPOINT%/}${metric_path}"
{
printf '# check-%s\n' "$phase"
printf 'curl -sS'
if [[ -n "$ADMIN_TOKEN" ]]; then
printf " -H \"Authorization: Bearer \${ADMIN_TOKEN}\""
fi
printf ' -X GET "%s" | jq '\''%s'\''\n' "$job_url" "$job_filter"
printf 'curl -sS'
if [[ -n "$ADMIN_TOKEN" ]]; then
printf " -H \"Authorization: Bearer \${ADMIN_TOKEN}\""
fi
printf ' -X GET "%s" | jq '\''%s'\''\n' "$metric_url" "$metric_filter"
printf './scripts/manual_transition_journal_audit.sh --endpoint %s --job-id "%s"' "$ENDPOINT" "$job_id_ref"
if [[ -n "$ADMIN_TOKEN" ]]; then
printf " --admin-token \"\${ADMIN_TOKEN}\""
fi
printf ' --sys-bucket .rustfs.sys\n'
}
}
run_rows() {
local matrix_csv="${OUT_DIR}/mixed_rollout_matrix.csv"
local run_script="${OUT_DIR}/run_phase_commands.sh"
local checklist="${OUT_DIR}/mixed_rollout_checklist.md"
local command_manifest="${OUT_DIR}/run_phase_notes.txt"
local admin_check
local admin_check_csv
local job_id_ref
{
echo "phase,old_pct,new_pct,duration_min,concurrency,object_count,read_ratio,gate,admin_check"
} > "$matrix_csv"
{
echo "#!/usr/bin/env bash"
echo "set -euo pipefail"
echo ""
echo "# generated matrix commands: fill in JOB_ID after each manual transition run"
echo ""
} > "$run_script"
{
echo "# mixed-version rollout command template"
echo "Endpoint: ${ENDPOINT}"
echo "Bucket: ${JOB_BUCKET}"
echo "Prefix: ${JOB_PREFIX}"
echo "Read ratio: ${READ_RATIO}%"
echo ""
} > "$checklist"
while IFS=',' read -r phase_spec; do
[[ -z "$phase_spec" ]] && continue
parsed="$(parse_phase_row "$phase_spec")"
IFS='|' read -r phase_name old_ratio new_ratio duration_min <<< "$parsed"
gate="$(gate_for_phase "$phase_name")"
IFS=',' read -r -a counts <<< "$OBJECT_COUNTS"
IFS=',' read -r -a concs <<< "$CONCURRENCIES"
for conc in "${concs[@]}"; do
conc="$(trim "$conc")"
[[ -z "$conc" ]] && continue
for count in "${counts[@]}"; do
count="$(trim "$count")"
[[ -z "$count" || -z "$conc" ]] && continue
job_id_ref="<JOB_ID_${phase_name}_c${conc}_q${count}>"
admin_check="$(admin_check_cmd "$phase_name" "$job_id_ref" 1 1 | tr '\n' ';')"
admin_check_csv="${admin_check//\"/\"\"}"
echo "${phase_name},${old_ratio},${new_ratio},${duration_min},${conc},${count},${READ_RATIO},\"${gate}\",\"${admin_check_csv}\"" >> "$matrix_csv"
{
echo "# phase: ${phase_name} concurrency: ${conc} object_count: ${count}"
echo "# old:new = ${old_ratio}:${new_ratio}, duration=${duration_min}min"
printf './scripts/manual_transition_journal_audit.sh --endpoint %s --job-id "%s"' "$ENDPOINT" "$job_id_ref"
if [[ -n "$ADMIN_TOKEN" ]]; then
printf " --admin-token \"\${ADMIN_TOKEN}\""
fi
printf '\n'
echo ""
} >> "$run_script"
done
done
done < <(tr ',' '\n' <<< "$PHASE_MATRIX")
{
echo "## Mixed-version rollout matrix artifacts"
echo ""
echo "- matrix: ${matrix_csv}"
echo "- commands: ${run_script}"
echo "- checklist: ${checklist}"
} > "$command_manifest"
}
main() {
parse_args "$@"
if [[ -z "$ENDPOINT" ]]; then
echo "ERROR: --endpoint is required" >&2
usage
exit 1
fi
if [[ -z "$OUT_DIR" ]]; then
OUT_DIR="${PROJECT_ROOT}/target/manual-transition-mixed-rollout/$(date +%Y%m%dT%H%M%S)"
fi
mkdir -p "$OUT_DIR"
require_cmd awk
run_rows
if [[ "$DRY_RUN" == true ]]; then
echo "[DRY-RUN] generated files:"
fi
echo "Generated:"
echo " - ${OUT_DIR}/mixed_rollout_matrix.csv"
echo " - ${OUT_DIR}/run_phase_commands.sh"
echo " - ${OUT_DIR}/mixed_rollout_checklist.md"
echo " - ${OUT_DIR}/run_phase_notes.txt"
}
main "$@"
scripts/manual_transition_soak_matrix.sh
Executable
+262
View File
@@ -0,0 +1,262 @@
#!/usr/bin/env bash
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# software distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
ENDPOINT=""
ADMIN_TOKEN=""
OUT_DIR=""
WINDOW_SPEC="nightly-2h:120:16:5000:balanced,nightly-12h:720:24:8000:write-heavy,nightly-24h:1440:32:12000:read-heavy"
SOAK_RATIOS="read-heavy:90:10,balanced:70:30,write-heavy:40:60"
WORKLOAD_SIZES="4KiB,1MiB,16MiB"
RUN_ADMIN_CHECKS=true
COST_BUDGET="$((24*1024))"
DRY_RUN=false
usage() {
cat <<'USAGE'
Usage:
scripts/manual_transition_soak_matrix.sh --endpoint <admin-api> [options]
Required:
--endpoint Admin API base
Optional:
--admin-token Bearer token for admin calls
--window-spec CSV of <name:duration_min:concurrency:ops_per_minute:mix_name>
--workload-sizes Comma-separated object size set
--soak-ratios Comma-separated mix spec: label:read_pct:write_pct
--cost-budget Max budget units for each run, default 24576
--out-dir Output dir
--no-admin-checks Skip admin check command blocks
--dry-run
--help
USAGE
}
arg_value() {
local flag="$1"
local value="${2:-}"
if [[ -z "$value" || "$value" == --* ]]; then
echo "ERROR: missing value for $flag" >&2
exit 1
fi
printf '%s' "$value"
}
require_cmd() {
local cmd="$1"
if ! command -v "$cmd" >/dev/null 2>&1; then
echo "ERROR: command not found: $cmd" >&2
exit 1
fi
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--admin-token) ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--window-spec) WINDOW_SPEC="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--workload-sizes) WORKLOAD_SIZES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--soak-ratios) SOAK_RATIOS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--cost-budget) COST_BUDGET="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--no-admin-checks) RUN_ADMIN_CHECKS=false; shift ;;
--dry-run) DRY_RUN=true; shift ;;
-h|--help) usage; exit 0 ;;
*)
echo "ERROR: unknown arg: $1" >&2
usage
exit 1
;;
esac
done
}
trim() {
echo "$1" | awk '{$1=$1;print}'
}
parse_window() {
local spec="$1"
local name duration_min concurrency ops_per_min mix_name
name="$(echo "$spec" | cut -d':' -f1)"
duration_min="$(echo "$spec" | cut -d':' -f2)"
concurrency="$(echo "$spec" | cut -d':' -f3)"
ops_per_min="$(echo "$spec" | cut -d':' -f4)"
mix_name="$(echo "$spec" | cut -d':' -f5)"
name="$(trim "$name")"
duration_min="$(trim "$duration_min")"
concurrency="$(trim "$concurrency")"
ops_per_min="$(trim "$ops_per_min")"
mix_name="$(trim "$mix_name")"
if [[ -z "$name" || -z "$duration_min" || -z "$concurrency" || -z "$ops_per_min" || -z "$mix_name" ]]; then
echo "ERROR: invalid window spec: $spec" >&2
exit 1
fi
if ! [[ "$duration_min" =~ ^[0-9]+$ && "$concurrency" =~ ^[0-9]+$ && "$ops_per_min" =~ ^[0-9]+$ ]]; then
echo "ERROR: window duration/concurrency/ops must be integers: $spec" >&2
exit 1
fi
echo "$name|$duration_min|$concurrency|$ops_per_min|$mix_name"
}
parse_ratio() {
local spec="$1"
local name read_pct write_pct
name="$(echo "$spec" | cut -d':' -f1)"
read_pct="$(echo "$spec" | cut -d':' -f2)"
write_pct="$(echo "$spec" | cut -d':' -f3)"
name="$(trim "$name")"
read_pct="$(trim "$read_pct")"
write_pct="$(trim "$write_pct")"
if [[ -z "$name" || -z "$read_pct" || -z "$write_pct" ]]; then
echo "ERROR: invalid mix ratio: $spec" >&2
exit 1
fi
if ! [[ "$read_pct" =~ ^[0-9]+$ && "$write_pct" =~ ^[0-9]+$ ]]; then
echo "ERROR: mix percentages must be integers: $spec" >&2
exit 1
fi
echo "$name|$read_pct|$write_pct"
}
resolve_mix() {
local name="$1"
local line
line="$(awk -v target="$name" -F: '{
if ($1 == target) { print $0; exit }
}' < <(tr ',' '\n' <<< "$SOAK_RATIOS"))"
if [[ -z "$line" ]]; then
return 1
fi
parse_ratio "$line"
}
run_checks() {
local tag="$1"
local job_id="$2"
local token_hdr=""
if [[ -z "$job_id" ]]; then
job_id="<JOB_ID_${tag}>"
fi
if [[ -n "$ADMIN_TOKEN" ]]; then
token_hdr="-H \"Authorization: Bearer \${ADMIN_TOKEN}\" "
fi
if [[ "$RUN_ADMIN_CHECKS" != true ]]; then
return
fi
echo "# admin checks for ${tag}"
echo "curl -sS ${token_hdr}-X GET \"${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}\" | jq '.status, .report, .queue_snapshot, .failure_reason'"
echo "curl -sS ${token_hdr}-X GET \"${ENDPOINT%/}/rustfs/admin/v3/metrics?types=1&n=1\" | jq '.aggregated.scanner.lifecycle_transition'"
printf './scripts/manual_transition_journal_audit.sh --endpoint %s --job-id "%s"' "$ENDPOINT" "$job_id"
if [[ -n "$ADMIN_TOKEN" ]]; then
printf " --admin-token \"\${ADMIN_TOKEN}\""
fi
printf '\n'
}
write_matrix_files() {
local matrix_csv="${OUT_DIR}/nightly_soak_matrix.csv"
local run_matrix="${OUT_DIR}/run_soak_matrix.sh"
local notes="${OUT_DIR}/soak_notes.md"
local run_id=1
{
echo "window,window_duration_min,concurrency,ops_per_min,mix_name,read_pct,write_pct,size,expected_ops,expected_run_id,run_check"
} > "$matrix_csv"
echo "#!/usr/bin/env bash" > "$run_matrix"
echo "set -euo pipefail" >> "$run_matrix"
echo "" >> "$run_matrix"
{
echo "# Nightly/night soak matrix for manual transition stress"
echo "- endpoint: ${ENDPOINT}"
echo "- cost-budget: ${COST_BUDGET}"
echo ""
echo "Recommended gate for each row: no unknown failure_reason spikes, job result drift is 0, and queued/task/result counts reconcile."
echo ""
} > "$notes"
IFS=',' read -r -a sizes <<< "$WORKLOAD_SIZES"
while IFS=',' read -r window_spec; do
[[ -z "$window_spec" ]] && continue
parsed="$(parse_window "$window_spec")"
IFS='|' read -r window_name duration_min concurrency ops_per_min mix_name <<< "$parsed"
mix="$(resolve_mix "$mix_name" || true)"
if [[ -z "$mix" ]]; then
echo "ERROR: unknown mix name in window spec: ${window_name} -> ${mix_name}" >&2
exit 1
fi
IFS='|' read -r resolved_mix read_pct write_pct <<< "$mix"
for size in "${sizes[@]}"; do
size="$(trim "$size")"
[[ -z "$size" ]] && continue
expected_ops=$(( duration_min * ops_per_min ))
if (( expected_ops > COST_BUDGET )); then
budget_status="over-budget"
else
budget_status="within-budget"
fi
echo "${window_name},${duration_min},${concurrency},${ops_per_min},${resolved_mix},${read_pct},${write_pct},${size},${expected_ops},${run_id},${budget_status}" >> "$matrix_csv"
{
echo "# window=${window_name} size=${size} mix=${resolved_mix} concurrency=${concurrency} duration=${duration_min}m ops_per_min=${ops_per_min}"
echo "job_id=\"<JOB_ID_${window_name}_${size}_${concurrency}_${run_id}>\""
echo "read_pct=${read_pct}"
echo "write_pct=${write_pct}"
echo "expected_ops=${expected_ops}"
echo "budget_status=${budget_status}"
run_checks "${window_name}_${size}_${concurrency}_${run_id}" "<JOB_ID_${window_name}_${size}_${concurrency}_${run_id}>"
echo
} >> "$run_matrix"
run_id=$((run_id + 1))
done
done < <(tr ',' '\n' <<< "$WINDOW_SPEC")
{
echo "# run target:"
echo "bash ${run_matrix}"
} >> "$notes"
}
main() {
parse_args "$@"
if [[ -z "$ENDPOINT" ]]; then
echo "ERROR: --endpoint is required" >&2
usage
exit 1
fi
if [[ -z "$OUT_DIR" ]]; then
OUT_DIR="${PROJECT_ROOT}/target/manual-transition-nightly-soak/$(date +%Y%m%dT%H%M%S)"
fi
mkdir -p "$OUT_DIR"
require_cmd awk
write_matrix_files
if [[ "$DRY_RUN" == true ]]; then
echo "[DRY-RUN] generated matrix files:"
fi
echo "Generated:"
echo " - ${OUT_DIR}/nightly_soak_matrix.csv"
echo " - ${OUT_DIR}/run_soak_matrix.sh"
echo " - ${OUT_DIR}/soak_notes.md"
}
main "$@"