mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-02 11:29:17 +00:00
fix: classify manual transition tier failures (#5335)
Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
@@ -1875,12 +1875,30 @@ fn manual_transition_worker_failure_reason(err: &Error) -> ManualTransitionWorke
|
||||
if is_slow_down(err) {
|
||||
return ManualTransitionWorkerFailureReason::SlowDown;
|
||||
}
|
||||
if is_network_or_host_down(&err.to_string(), false) {
|
||||
let message = err.to_string();
|
||||
if is_remote_tier_permission_denied_error(&message) {
|
||||
return ManualTransitionWorkerFailureReason::PermissionDenied;
|
||||
}
|
||||
if is_remote_tier_network_error(&message) {
|
||||
return ManualTransitionWorkerFailureReason::Network;
|
||||
}
|
||||
ManualTransitionWorkerFailureReason::Unknown
|
||||
}
|
||||
|
||||
fn is_remote_tier_permission_denied_error(message: &str) -> bool {
|
||||
message.contains("remote tier request failed with status 401")
|
||||
|| message.contains("remote tier request failed with status 403")
|
||||
|| message.contains("InvalidAccessKeyId")
|
||||
|| message.contains("AccessDenied")
|
||||
|| message.contains("SignatureDoesNotMatch")
|
||||
}
|
||||
|
||||
fn is_remote_tier_network_error(message: &str) -> bool {
|
||||
message.contains("client error (SendRequest)")
|
||||
|| message.contains("dispatch failure")
|
||||
|| is_network_or_host_down(message, false)
|
||||
}
|
||||
|
||||
fn is_err_permission_denied(err: &Error) -> bool {
|
||||
match err {
|
||||
Error::VolumeAccessDenied | Error::FileAccessDenied | Error::PrefixAccessDenied(_, _) | Error::DiskAccessDenied => true,
|
||||
@@ -7740,6 +7758,16 @@ mod tests {
|
||||
manual_transition_worker_failure_reason(&Error::SlowDown),
|
||||
ManualTransitionWorkerFailureReason::SlowDown
|
||||
);
|
||||
assert_eq!(
|
||||
manual_transition_worker_failure_reason(&Error::Io(std::io::Error::other(
|
||||
"remote tier request failed with status 403 Forbidden: InvalidAccessKeyId",
|
||||
))),
|
||||
ManualTransitionWorkerFailureReason::PermissionDenied
|
||||
);
|
||||
assert_eq!(
|
||||
manual_transition_worker_failure_reason(&Error::Io(std::io::Error::other("client error (SendRequest)",))),
|
||||
ManualTransitionWorkerFailureReason::Network
|
||||
);
|
||||
assert_eq!(
|
||||
manual_transition_worker_failure_reason(&Error::MethodNotAllowed),
|
||||
ManualTransitionWorkerFailureReason::Unknown
|
||||
|
||||
@@ -20,6 +20,9 @@ PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
ENDPOINT=""
|
||||
ADMIN_TOKEN=""
|
||||
ACCESS_KEY=""
|
||||
SECRET_KEY=""
|
||||
AWS_SIGV4_SCOPE="aws:amz:us-east-1:s3"
|
||||
OUT_DIR=""
|
||||
LOG_GLOB="/var/log/rustfs/*.log"
|
||||
METRIC_TYPES="1"
|
||||
@@ -45,6 +48,9 @@ Required for real runs:
|
||||
|
||||
Optional:
|
||||
--admin-token Bearer token for /admin/v3 endpoints
|
||||
--access-key Access key for SigV4-signed admin/metrics calls
|
||||
--secret-key Secret key for SigV4-signed admin/metrics calls
|
||||
--aws-sigv4-scope curl --aws-sigv4 scope, default aws:amz:us-east-1:s3
|
||||
--log-glob Log glob to scan, default /var/log/rustfs/*.log
|
||||
--metric-types /admin/v3/metrics type flags, default 1
|
||||
--metric-samples /admin/v3/metrics sample count, default 1
|
||||
@@ -64,7 +70,8 @@ Output:
|
||||
Typical #1509 evidence target:
|
||||
scripts/manual_transition_failure_samples.sh \
|
||||
--endpoint https://127.0.0.1:9000 \
|
||||
--admin-token "$TOKEN" \
|
||||
--access-key "$ACCESS_KEY" \
|
||||
--secret-key "$SECRET_KEY" \
|
||||
--sample auth:<JOB_ID_AUTH>:RemoteAuth \
|
||||
--sample network:<JOB_ID_NETWORK>:RemoteNetwork \
|
||||
--min-distinct-reasons 2 \
|
||||
@@ -92,7 +99,7 @@ require_cmd() {
|
||||
|
||||
normalize_uuid() {
|
||||
local raw="${1//-/}"
|
||||
printf '%s' "${raw,,}"
|
||||
printf '%s' "$raw" | tr '[:upper:]' '[:lower:]'
|
||||
}
|
||||
|
||||
validate_uuid() {
|
||||
@@ -117,6 +124,9 @@ parse_args() {
|
||||
case "$1" in
|
||||
--endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--admin-token) ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--access-key) ACCESS_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--secret-key) SECRET_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--aws-sigv4-scope) AWS_SIGV4_SCOPE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--sample) SAMPLES+=("$(arg_value "$1" "${2:-}")"); shift 2 ;;
|
||||
--log-glob) LOG_GLOB="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--metric-types) METRIC_TYPES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
@@ -156,6 +166,14 @@ validate_args() {
|
||||
if [[ -z "$OUT_DIR" ]]; then
|
||||
OUT_DIR="${PROJECT_ROOT}/target/manual-transition-failure-samples/$(date +%Y%m%dT%H%M%S)"
|
||||
fi
|
||||
if [[ -n "$ADMIN_TOKEN" && ( -n "$ACCESS_KEY" || -n "$SECRET_KEY" ) ]]; then
|
||||
echo "ERROR: --admin-token cannot be combined with --access-key/--secret-key" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "$ACCESS_KEY" && -z "$SECRET_KEY" || -z "$ACCESS_KEY" && -n "$SECRET_KEY" ]]; then
|
||||
echo "ERROR: --access-key and --secret-key must be provided together" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
sample_label() {
|
||||
@@ -201,6 +219,8 @@ curl_json() {
|
||||
local args=(-fsS)
|
||||
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||
args+=(-H "Authorization: Bearer ${ADMIN_TOKEN}")
|
||||
elif [[ -n "$ACCESS_KEY" ]]; then
|
||||
args+=(--aws-sigv4 "$AWS_SIGV4_SCOPE" --user "${ACCESS_KEY}:${SECRET_KEY}")
|
||||
fi
|
||||
curl "${args[@]}" "$url" > "$out_file"
|
||||
}
|
||||
@@ -233,7 +253,8 @@ Replace the placeholders and run:
|
||||
```bash
|
||||
scripts/manual_transition_failure_samples.sh \
|
||||
--endpoint <admin-api> \
|
||||
--admin-token "$TOKEN" \
|
||||
--access-key "$ACCESS_KEY" \
|
||||
--secret-key "$SECRET_KEY" \
|
||||
--sample auth:<JOB_ID_AUTH>:<expected_reason_key> \
|
||||
--sample network:<JOB_ID_NETWORK>:<expected_reason_key> \
|
||||
--sample timeout:<JOB_ID_TIMEOUT>:<expected_reason_key> \
|
||||
@@ -249,7 +270,7 @@ scripts/manual_transition_failure_samples.sh \
|
||||
- terminal partial/failed counts agree with `tier_failure_by_reason`
|
||||
PLAN
|
||||
cat >"${OUT_DIR}/commands.txt" <<COMMANDS
|
||||
scripts/manual_transition_failure_samples.sh --endpoint ${ENDPOINT} --sample auth:<JOB_ID_AUTH>:<expected_reason_key> --sample network:<JOB_ID_NETWORK>:<expected_reason_key> --min-distinct-reasons ${MIN_DISTINCT_REASONS} --out-dir ${OUT_DIR}
|
||||
scripts/manual_transition_failure_samples.sh --endpoint ${ENDPOINT} --access-key <ACCESS_KEY> --secret-key <SECRET_KEY> --sample auth:<JOB_ID_AUTH>:<expected_reason_key> --sample network:<JOB_ID_NETWORK>:<expected_reason_key> --min-distinct-reasons ${MIN_DISTINCT_REASONS} --out-dir ${OUT_DIR}
|
||||
COMMANDS
|
||||
echo "[DRY-RUN] generated ${OUT_DIR}/sample_plan.md"
|
||||
echo "[DRY-RUN] generated ${OUT_DIR}/commands.txt"
|
||||
@@ -272,7 +293,7 @@ collect_logs() {
|
||||
log_paths=("$LOG_GLOB")
|
||||
fi
|
||||
|
||||
if ! rg --no-filename --color=never -- "${job_id}|lifecycle_tier_operation_failed|lifecycle_worker_state" -- "${log_paths[@]}" >"$raw_file" 2>"$err_file"; then
|
||||
if ! rg --no-filename --color=never -- "${job_id}|lifecycle_tier_operation_failed|lifecycle_worker_state" "${log_paths[@]}" >"$raw_file" 2>"$err_file"; then
|
||||
:
|
||||
fi
|
||||
|
||||
@@ -312,6 +333,7 @@ write_command_notes() {
|
||||
cat >"${OUT_DIR}/commands.txt" <<COMMANDS
|
||||
# Re-run this evidence capture
|
||||
scripts/manual_transition_failure_samples.sh --endpoint ${ENDPOINT} --min-distinct-reasons ${MIN_DISTINCT_REASONS} --out-dir ${OUT_DIR} \\
|
||||
$(if [[ -n "$ADMIN_TOKEN" ]]; then printf ' --admin-token <ADMIN_TOKEN> \\\n'; elif [[ -n "$ACCESS_KEY" ]]; then printf ' --access-key <ACCESS_KEY> --secret-key <SECRET_KEY> \\\n'; fi)
|
||||
$(printf ' --sample %q \\\n' "${SAMPLES[@]}")
|
||||
|
||||
# Per-job deep dive
|
||||
@@ -398,6 +420,7 @@ main() {
|
||||
require_cmd jq
|
||||
require_cmd rg
|
||||
require_cmd awk
|
||||
require_cmd tr
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
local summary_csv="${OUT_DIR}/failure_attribution_summary.csv"
|
||||
|
||||
@@ -20,6 +20,9 @@ PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
ENDPOINT=""
|
||||
ADMIN_TOKEN=""
|
||||
ACCESS_KEY=""
|
||||
SECRET_KEY=""
|
||||
AWS_SIGV4_SCOPE="aws:amz:us-east-1:s3"
|
||||
OUT_DIR=""
|
||||
WINDOW_SPEC="nightly-2h:120:16:5000:balanced,nightly-12h:720:24:8000:write-heavy,nightly-24h:1440:32:12000:read-heavy"
|
||||
SOAK_RATIOS="read-heavy:90:10,balanced:70:30,write-heavy:40:60"
|
||||
@@ -47,6 +50,9 @@ Required:
|
||||
|
||||
Optional:
|
||||
--admin-token Bearer token for admin endpoints
|
||||
--access-key Access key for SigV4-signed admin calls
|
||||
--secret-key Secret key for SigV4-signed admin calls
|
||||
--aws-sigv4-scope curl --aws-sigv4 scope, default aws:amz:us-east-1:s3
|
||||
--window-spec Comma-separated run spec: window:duration_min:concurrency:ops_per_min:mix_name
|
||||
--soak-ratios Comma-separated mix ratios: label:read_pct:write_pct
|
||||
--workload-sizes Object-size workload set
|
||||
@@ -94,6 +100,18 @@ parse_args() {
|
||||
ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--access-key)
|
||||
ACCESS_KEY="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--secret-key)
|
||||
SECRET_KEY="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--aws-sigv4-scope)
|
||||
AWS_SIGV4_SCOPE="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--window-spec)
|
||||
WINDOW_SPEC="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
@@ -228,6 +246,9 @@ set -euo pipefail
|
||||
SOAK_MATRIX_CSV='__SOAK_MATRIX_CSV__'
|
||||
ENDPOINT='__ENDPOINT__'
|
||||
ADMIN_TOKEN='__ADMIN_TOKEN__'
|
||||
ACCESS_KEY='__ACCESS_KEY__'
|
||||
SECRET_KEY='__SECRET_KEY__'
|
||||
AWS_SIGV4_SCOPE='__AWS_SIGV4_SCOPE__'
|
||||
JOB_BUCKET='__JOB_BUCKET__'
|
||||
JOB_PREFIX='__JOB_PREFIX__'
|
||||
TIER='__TIER__'
|
||||
@@ -238,7 +259,9 @@ OUT_DIR='__OUT_DIR__'
|
||||
: "${UNKNOWN_FAILURE_COUNT_THRESHOLD:=0}"
|
||||
|
||||
SNAPSHOT_DIR="${OUT_DIR}/failure-snapshots"
|
||||
RESULT_DIR="${OUT_DIR}/run-results"
|
||||
mkdir -p "$SNAPSHOT_DIR"
|
||||
mkdir -p "$RESULT_DIR"
|
||||
|
||||
require_cmd() {
|
||||
local cmd="$1"
|
||||
@@ -253,6 +276,19 @@ url_encode() {
|
||||
jq -rn --arg v "$value" '$v|@uri'
|
||||
}
|
||||
|
||||
curl_admin() {
|
||||
local method="$1"
|
||||
local url="$2"
|
||||
shift 2
|
||||
local args=(-sS -X "$method")
|
||||
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||
args+=(-H "Authorization: Bearer ${ADMIN_TOKEN}")
|
||||
elif [[ -n "$ACCESS_KEY" ]]; then
|
||||
args+=(--aws-sigv4 "$AWS_SIGV4_SCOPE" --user "${ACCESS_KEY}:${SECRET_KEY}")
|
||||
fi
|
||||
curl "${args[@]}" "$url" "$@"
|
||||
}
|
||||
|
||||
snapshot_failure() {
|
||||
local run_tag="$1"
|
||||
local reason="$2"
|
||||
@@ -272,7 +308,7 @@ snapshot_failure() {
|
||||
} >"${snapshot_dir}/snapshot.meta"
|
||||
|
||||
if [[ -n "$job_id" && "$job_id" != "NA" ]]; then
|
||||
curl -sS ${ADMIN_TOKEN:+-H "Authorization: Bearer ${ADMIN_TOKEN}"} -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" \
|
||||
curl_admin GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" \
|
||||
>"${snapshot_dir}/job_status.json"
|
||||
./scripts/manual_transition_journal_audit.sh --endpoint "$ENDPOINT" --job-id "$job_id" ${ADMIN_TOKEN:+--admin-token "$ADMIN_TOKEN"} --out-dir "$snapshot_dir" || true
|
||||
fi
|
||||
@@ -300,7 +336,7 @@ run_entry() {
|
||||
local status
|
||||
local status_json
|
||||
local status_info
|
||||
local headers=()
|
||||
local result_tag
|
||||
|
||||
if [[ "$budget_status" == "over-budget" ]]; then
|
||||
echo "[skip] ${tag} ${size} over budget: expected_ops=${expected_ops}"
|
||||
@@ -308,6 +344,7 @@ run_entry() {
|
||||
fi
|
||||
|
||||
prefix="${JOB_PREFIX}/${tag}/${size}/${read_pct}r${write_pct}w"
|
||||
result_tag="${tag}-${size}-${read_pct}r${write_pct}w"
|
||||
query="bucket=$(url_encode "$JOB_BUCKET")"
|
||||
query="${query}&prefix=$(url_encode "$prefix")"
|
||||
query="${query}&maxObjects=100000"
|
||||
@@ -319,32 +356,30 @@ run_entry() {
|
||||
|
||||
url="${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/run?${query}"
|
||||
|
||||
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||
headers+=("-H" "Authorization: Bearer ${ADMIN_TOKEN}")
|
||||
fi
|
||||
|
||||
echo "==> run=${tag} size=${size} mix=${mix_name} read_pct=${read_pct} write_pct=${write_pct} duration_min=${duration_min}"
|
||||
if ! response="$(curl -sS "${headers[@]}" -X POST "$url")"; then
|
||||
if ! response="$(curl_admin POST "$url")"; then
|
||||
snapshot_failure "$tag" "curl_post_failed" "NA"
|
||||
return 1
|
||||
fi
|
||||
printf '%s' "$response" >"${RESULT_DIR}/${result_tag}-run.json"
|
||||
job_id="$(printf '%s' "$response" | jq -r '.job_id // empty')"
|
||||
if [[ -z "$job_id" || "$job_id" == "null" ]]; then
|
||||
snapshot_failure "$tag" "missing_job_id" "NA"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! status_json="$(curl -sS "${headers[@]}" -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}")"; then
|
||||
if ! status_json="$(curl_admin GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}")"; then
|
||||
snapshot_failure "$tag" "job_status_fetch_failed" "$job_id"
|
||||
return 1
|
||||
fi
|
||||
if ! status_info="$(printf '%s' "$status_json" | jq -r '[.status // "", .failure_reason // "", (.report.enqueued // 0), (.report.transition_completed // 0), (.report.transition_failed // 0), (.report.tier_failure_by_reason["unknown"] // 0), (.queue_snapshot.queued // 0), (.queue_snapshot.active // 0), (.queue_snapshot.compensation_pending // 0), (.queue_snapshot.compensation_running // 0), (.queue_snapshot.queue_full // 0), (.queue_snapshot.queue_send_timeout // 0)] | map(tostring) | join(\"\\u0000\")')"; then
|
||||
printf '%s' "$status_json" >"${RESULT_DIR}/${result_tag}-status.json"
|
||||
if ! status_info="$(printf '%s' "$status_json" | jq -r '[.status // "", .failure_reason // "__none__", (.report.enqueued // 0), (.report.transition_completed // 0), (.report.transition_failed // 0), (.report.tier_failure_by_reason["unknown"] // 0), (.queue_snapshot.queued // 0), (.queue_snapshot.active // 0), (.queue_snapshot.compensation_pending // 0), (.queue_snapshot.compensation_running // 0), (.queue_snapshot.queue_full // 0), (.queue_snapshot.queue_send_timeout // 0)] | map(tostring) | @tsv')"; then
|
||||
snapshot_failure "$tag" "invalid_job_status_json" "$job_id"
|
||||
return 1
|
||||
fi
|
||||
IFS=$'\0' read -r status failure_reason report_enqueued report_completed report_failed report_unknown_failure queue_snapshot_queued queue_snapshot_active compensation_pending compensation_running queue_full queue_send_timeout <<< "$status_info"
|
||||
IFS=$'\t' read -r status failure_reason report_enqueued report_completed report_failed report_unknown_failure queue_snapshot_queued queue_snapshot_active compensation_pending compensation_running queue_full queue_send_timeout <<< "$status_info"
|
||||
|
||||
if [[ -n "$failure_reason" && "$failure_reason" != "null" ]]; then
|
||||
if [[ "$failure_reason" != "__none__" ]]; then
|
||||
snapshot_failure "$tag" "failure_reason=${failure_reason}" "$job_id"
|
||||
fi
|
||||
|
||||
@@ -374,24 +409,25 @@ run_entry() {
|
||||
unknown_ratio="$(awk -v unknown="$report_unknown_failure" -v denom="$ratio_denominator" 'BEGIN{printf "%.6f", (unknown / denom)}')"
|
||||
mismatch_ratio="$(awk -v mismatch="$mismatch_count" -v denom="$ratio_denominator" 'BEGIN{printf "%.6f", (mismatch / denom)}')"
|
||||
|
||||
if (( UNKNOWN_FAILURE_RATIO_THRESHOLD > 0 )) && \
|
||||
awk "BEGIN{exit !($unknown_ratio > ${UNKNOWN_FAILURE_RATIO_THRESHOLD})}"; then
|
||||
if awk "BEGIN{exit !(${UNKNOWN_FAILURE_RATIO_THRESHOLD} > 0 && $unknown_ratio > ${UNKNOWN_FAILURE_RATIO_THRESHOLD})}"; then
|
||||
snapshot_failure "$tag" "unknown_failure_ratio=${unknown_ratio}/expected=${ratio_denominator}/threshold=${UNKNOWN_FAILURE_RATIO_THRESHOLD}" "$job_id"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if (( UNKNOWN_FAILURE_COUNT_THRESHOLD > 0 )) && (( report_unknown_failure > UNKNOWN_FAILURE_COUNT_THRESHOLD )); then
|
||||
snapshot_failure "$tag" "unknown_failure_count=${report_unknown_failure}/threshold=${UNKNOWN_FAILURE_COUNT_THRESHOLD}" "$job_id"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if (( QUEUE_MISMATCH_RATIO_THRESHOLD > 0 )) && \
|
||||
awk "BEGIN{exit !($mismatch_ratio > ${QUEUE_MISMATCH_RATIO_THRESHOLD})}"; then
|
||||
if awk "BEGIN{exit !(${QUEUE_MISMATCH_RATIO_THRESHOLD} > 0 && $mismatch_ratio > ${QUEUE_MISMATCH_RATIO_THRESHOLD})}"; then
|
||||
snapshot_failure "$tag" "queue_mismatch_ratio=${mismatch_ratio}/expected=${ratio_denominator}/threshold=${QUEUE_MISMATCH_RATIO_THRESHOLD}" "$job_id"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Keep compatibility with older callers that monitor queue drift via full terminal status.
|
||||
curl -sS "${headers[@]}" -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" | jq '{status, report, queue_snapshot, failure_reason}'
|
||||
printf '%s' "$status_json" | jq '{job_id, status, bucket, prefix, tier, report, queue_snapshot, failure_reason}'
|
||||
}
|
||||
|
||||
is_terminal_status() {
|
||||
@@ -413,18 +449,34 @@ main() {
|
||||
require_cmd jq
|
||||
require_cmd awk
|
||||
|
||||
if [[ -n "$ADMIN_TOKEN" && ( -n "$ACCESS_KEY" || -n "$SECRET_KEY" ) ]]; then
|
||||
echo "ERROR: bearer token cannot be combined with SigV4 credentials" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "$ACCESS_KEY" && -z "$SECRET_KEY" || -z "$ACCESS_KEY" && -n "$SECRET_KEY" ]]; then
|
||||
echo "ERROR: ACCESS_KEY and SECRET_KEY must be provided together" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$SOAK_MATRIX_CSV" ]]; then
|
||||
echo "ERROR: expected matrix file missing: $SOAK_MATRIX_CSV" >&2
|
||||
echo "Run manual_transition_soak_matrix.sh first." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local failures=0
|
||||
while IFS=',' read -r window duration_min concurrency ops_per_min mix_name read_pct write_pct size expected_ops run_id budget_status; do
|
||||
if [[ -z "$window" || "$window" == "window" ]]; then
|
||||
continue
|
||||
fi
|
||||
run_entry "${window}" "${duration_min}" "${concurrency}" "${ops_per_min}" "${mix_name}" "${read_pct}" "${write_pct}" "${size}" "${expected_ops}" "${budget_status}" || true
|
||||
if ! run_entry "${window}" "${duration_min}" "${concurrency}" "${ops_per_min}" "${mix_name}" "${read_pct}" "${write_pct}" "${size}" "${expected_ops}" "${budget_status}"; then
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
done < <(tail -n +2 "$SOAK_MATRIX_CSV")
|
||||
if (( failures > 0 )); then
|
||||
echo "ERROR: ${failures} soak row(s) failed; see ${SNAPSHOT_DIR}" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -433,6 +485,9 @@ EOF
|
||||
perl -0pi -e "s#__SOAK_MATRIX_CSV__#${matrix_csv//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__ENDPOINT__#${ENDPOINT//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__ADMIN_TOKEN__#${ADMIN_TOKEN//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__ACCESS_KEY__#${ACCESS_KEY//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__SECRET_KEY__#${SECRET_KEY//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__AWS_SIGV4_SCOPE__#${AWS_SIGV4_SCOPE//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__JOB_BUCKET__#${JOB_BUCKET//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__JOB_PREFIX__#${JOB_PREFIX//#/#}#g" "$COMMAND_FILE"
|
||||
perl -0pi -e "s#__TIER__#${TIER//#/#}#g" "$COMMAND_FILE"
|
||||
@@ -454,6 +509,14 @@ main() {
|
||||
mkdir -p "$OUT_DIR"
|
||||
require_cmd awk
|
||||
require_cmd jq
|
||||
if [[ -n "$ADMIN_TOKEN" && ( -n "$ACCESS_KEY" || -n "$SECRET_KEY" ) ]]; then
|
||||
echo "ERROR: --admin-token cannot be combined with --access-key/--secret-key" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "$ACCESS_KEY" && -z "$SECRET_KEY" || -z "$ACCESS_KEY" && -n "$SECRET_KEY" ]]; then
|
||||
echo "ERROR: --access-key and --secret-key must be provided together" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
main_matrix
|
||||
command_template
|
||||
|
||||
@@ -78,6 +78,8 @@ rg -q "MIXED_MATRIX_CSV='$TMP_DIR/mixed-runbook/mixed_rollout_matrix.csv'" "$TMP
|
||||
|
||||
"$STRESS_RUNBOOK" \
|
||||
--endpoint http://127.0.0.1:9000 \
|
||||
--access-key hotadmin \
|
||||
--secret-key hotsecret \
|
||||
--window-spec quick:1:1:1:balanced \
|
||||
--soak-ratios balanced:70:30 \
|
||||
--workload-sizes 4KiB \
|
||||
@@ -86,20 +88,65 @@ rg -q "MIXED_MATRIX_CSV='$TMP_DIR/mixed-runbook/mixed_rollout_matrix.csv'" "$TMP
|
||||
|
||||
test -x "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "failure-snapshots" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "run-results" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "UNKNOWN_FAILURE_RATIO_THRESHOLD" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "QUEUE_MISMATCH_RATIO_THRESHOLD" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "UNKNOWN_FAILURE_COUNT_THRESHOLD" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "ACCESS_KEY='hotadmin'" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "SECRET_KEY='hotsecret'" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "AWS_SIGV4_SCOPE='aws:amz:us-east-1:s3'" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "curl_admin" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "failures=\\$\\(\\(failures \\+ 1\\)\\)" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
if rg -q "headers\\[@\\]" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"; then
|
||||
echo "nightly stress runner must not expand an unset headers array" >&2
|
||||
exit 1
|
||||
fi
|
||||
if rg -q "run_entry .*\\|\\| true" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"; then
|
||||
echo "nightly stress runner must not hide failed rows" >&2
|
||||
exit 1
|
||||
fi
|
||||
rg -q "snapshot_failure" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "manual_transition_failure_samples.sh" "$TMP_DIR/stress-runbook/manual_transition_nightly_stress_runbook.md"
|
||||
rg -q "is_terminal_status" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "status_json" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "@tsv" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q 'failure_reason // "__none__"' "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q 'failure_reason" != "__none__"' "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
if rg -F -q 'failure_reason // ""' "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"; then
|
||||
echo "nightly stress runner must not emit an empty TSV failure_reason field" >&2
|
||||
exit 1
|
||||
fi
|
||||
rg -q "job_id, status, bucket, prefix, tier" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
if rg -F -q 'join(\"' "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"; then
|
||||
echo "nightly stress runner must not emit escaped jq quotes" >&2
|
||||
exit 1
|
||||
fi
|
||||
rg -q "unknown_failure_ratio" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "queue_mismatch_ratio" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "report_unknown_failure" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
rg -q "Manual transition nightly/stress stress-runbook" "$TMP_DIR/stress-runbook/manual_transition_nightly_stress_runbook.md"
|
||||
rg -q "SOAK_MATRIX_CSV='$TMP_DIR/stress-runbook/nightly_soak_matrix.csv'" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"
|
||||
|
||||
bash scripts/monitor_manual_transition_ci.sh --help | rg -q "Usage:"
|
||||
if bash "$STRESS_RUNBOOK" --endpoint http://127.0.0.1:9000 --access-key hotadmin --dry-run >/tmp/manual_transition_stress_runbook.err 2>&1; then
|
||||
echo "nightly stress runbook should fail when SigV4 credentials are incomplete" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! rg -q "ERROR: --access-key and --secret-key must be provided together" /tmp/manual_transition_stress_runbook.err; then
|
||||
echo "nightly stress runbook missing incomplete SigV4 credential guard output" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if bash "$STRESS_RUNBOOK" --endpoint http://127.0.0.1:9000 --admin-token token --access-key hotadmin --secret-key hotsecret --dry-run >/tmp/manual_transition_stress_runbook.err 2>&1; then
|
||||
echo "nightly stress runbook should reject mixed bearer and SigV4 credentials" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! rg -q "ERROR: --admin-token cannot be combined with --access-key/--secret-key" /tmp/manual_transition_stress_runbook.err; then
|
||||
echo "nightly stress runbook missing mixed credential guard output" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bash scripts/monitor_manual_transition_ci.sh --help >/tmp/monitor_manual_transition_ci.help
|
||||
rg -q "Usage:" /tmp/monitor_manual_transition_ci.help
|
||||
if bash scripts/monitor_manual_transition_ci.sh --issues >/tmp/monitor_manual_transition_ci.err 2>&1; then
|
||||
echo "monitor script should fail when --issues has no value" >&2
|
||||
exit 1
|
||||
@@ -118,7 +165,8 @@ if ! rg -q "ERROR: --runs must be a positive integer" /tmp/monitor_manual_transi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bash "$FAILURE_SAMPLES" --help | rg -q "Usage:"
|
||||
bash "$FAILURE_SAMPLES" --help >/tmp/manual_transition_failure_samples.help
|
||||
rg -q "Usage:" /tmp/manual_transition_failure_samples.help
|
||||
if bash "$FAILURE_SAMPLES" --endpoint http://127.0.0.1:9000 --sample >/tmp/manual_transition_failure_samples.err 2>&1; then
|
||||
echo "failure samples script should fail when --sample has no value" >&2
|
||||
exit 1
|
||||
@@ -137,8 +185,28 @@ if ! rg -q "ERROR: --min-distinct-reasons must be a positive integer" /tmp/manua
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if bash "$FAILURE_SAMPLES" --endpoint http://127.0.0.1:9000 --access-key hotadmin --dry-run >/tmp/manual_transition_failure_samples.err 2>&1; then
|
||||
echo "failure samples script should fail when SigV4 credentials are incomplete" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! rg -q "ERROR: --access-key and --secret-key must be provided together" /tmp/manual_transition_failure_samples.err; then
|
||||
echo "failure samples script missing incomplete SigV4 credential guard output" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if bash "$FAILURE_SAMPLES" --endpoint http://127.0.0.1:9000 --admin-token token --access-key hotadmin --secret-key hotsecret --dry-run >/tmp/manual_transition_failure_samples.err 2>&1; then
|
||||
echo "failure samples script should reject mixed bearer and SigV4 credentials" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! rg -q "ERROR: --admin-token cannot be combined with --access-key/--secret-key" /tmp/manual_transition_failure_samples.err; then
|
||||
echo "failure samples script missing mixed credential guard output" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"$FAILURE_SAMPLES" \
|
||||
--endpoint http://127.0.0.1:9000 \
|
||||
--access-key hotadmin \
|
||||
--secret-key hotsecret \
|
||||
--sample auth:11111111-1111-4111-8111-111111111111:RemoteAuth \
|
||||
--sample network:22222222-2222-4222-8222-222222222222:RemoteNetwork \
|
||||
--out-dir "$TMP_DIR/failure-samples" \
|
||||
@@ -146,4 +214,5 @@ fi
|
||||
|
||||
rg -q "Manual transition failure attribution sample plan" "$TMP_DIR/failure-samples/sample_plan.md"
|
||||
rg -q "auth:<JOB_ID_AUTH>:<expected_reason_key>" "$TMP_DIR/failure-samples/sample_plan.md"
|
||||
rg -q -- "--access-key <ACCESS_KEY> --secret-key <SECRET_KEY>" "$TMP_DIR/failure-samples/commands.txt"
|
||||
rg -q -- "--min-distinct-reasons 2" "$TMP_DIR/failure-samples/commands.txt"
|
||||
|
||||
Reference in New Issue
Block a user