diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 5229a61ec..2f12513fe 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -1875,12 +1875,30 @@ fn manual_transition_worker_failure_reason(err: &Error) -> ManualTransitionWorke if is_slow_down(err) { return ManualTransitionWorkerFailureReason::SlowDown; } - if is_network_or_host_down(&err.to_string(), false) { + let message = err.to_string(); + if is_remote_tier_permission_denied_error(&message) { + return ManualTransitionWorkerFailureReason::PermissionDenied; + } + if is_remote_tier_network_error(&message) { return ManualTransitionWorkerFailureReason::Network; } ManualTransitionWorkerFailureReason::Unknown } +fn is_remote_tier_permission_denied_error(message: &str) -> bool { + message.contains("remote tier request failed with status 401") + || message.contains("remote tier request failed with status 403") + || message.contains("InvalidAccessKeyId") + || message.contains("AccessDenied") + || message.contains("SignatureDoesNotMatch") +} + +fn is_remote_tier_network_error(message: &str) -> bool { + message.contains("client error (SendRequest)") + || message.contains("dispatch failure") + || is_network_or_host_down(message, false) +} + fn is_err_permission_denied(err: &Error) -> bool { match err { Error::VolumeAccessDenied | Error::FileAccessDenied | Error::PrefixAccessDenied(_, _) | Error::DiskAccessDenied => true, @@ -7740,6 +7758,16 @@ mod tests { manual_transition_worker_failure_reason(&Error::SlowDown), ManualTransitionWorkerFailureReason::SlowDown ); + assert_eq!( + manual_transition_worker_failure_reason(&Error::Io(std::io::Error::other( + "remote tier request failed with status 403 Forbidden: InvalidAccessKeyId", + ))), + ManualTransitionWorkerFailureReason::PermissionDenied + ); + assert_eq!( + manual_transition_worker_failure_reason(&Error::Io(std::io::Error::other("client error (SendRequest)",))), + ManualTransitionWorkerFailureReason::Network + ); assert_eq!( manual_transition_worker_failure_reason(&Error::MethodNotAllowed), ManualTransitionWorkerFailureReason::Unknown diff --git a/scripts/manual_transition_failure_samples.sh b/scripts/manual_transition_failure_samples.sh index 7238daadd..786b70c36 100755 --- a/scripts/manual_transition_failure_samples.sh +++ b/scripts/manual_transition_failure_samples.sh @@ -20,6 +20,9 @@ PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" ENDPOINT="" ADMIN_TOKEN="" +ACCESS_KEY="" +SECRET_KEY="" +AWS_SIGV4_SCOPE="aws:amz:us-east-1:s3" OUT_DIR="" LOG_GLOB="/var/log/rustfs/*.log" METRIC_TYPES="1" @@ -45,6 +48,9 @@ Required for real runs: Optional: --admin-token Bearer token for /admin/v3 endpoints + --access-key Access key for SigV4-signed admin/metrics calls + --secret-key Secret key for SigV4-signed admin/metrics calls + --aws-sigv4-scope curl --aws-sigv4 scope, default aws:amz:us-east-1:s3 --log-glob Log glob to scan, default /var/log/rustfs/*.log --metric-types /admin/v3/metrics type flags, default 1 --metric-samples /admin/v3/metrics sample count, default 1 @@ -64,7 +70,8 @@ Output: Typical #1509 evidence target: scripts/manual_transition_failure_samples.sh \ --endpoint https://127.0.0.1:9000 \ - --admin-token "$TOKEN" \ + --access-key "$ACCESS_KEY" \ + --secret-key "$SECRET_KEY" \ --sample auth::RemoteAuth \ --sample network::RemoteNetwork \ --min-distinct-reasons 2 \ @@ -92,7 +99,7 @@ require_cmd() { normalize_uuid() { local raw="${1//-/}" - printf '%s' "${raw,,}" + printf '%s' "$raw" | tr '[:upper:]' '[:lower:]' } validate_uuid() { @@ -117,6 +124,9 @@ parse_args() { case "$1" in --endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;; --admin-token) ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --access-key) ACCESS_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --secret-key) SECRET_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --aws-sigv4-scope) AWS_SIGV4_SCOPE="$(arg_value "$1" "${2:-}")"; shift 2 ;; --sample) SAMPLES+=("$(arg_value "$1" "${2:-}")"); shift 2 ;; --log-glob) LOG_GLOB="$(arg_value "$1" "${2:-}")"; shift 2 ;; --metric-types) METRIC_TYPES="$(arg_value "$1" "${2:-}")"; shift 2 ;; @@ -156,6 +166,14 @@ validate_args() { if [[ -z "$OUT_DIR" ]]; then OUT_DIR="${PROJECT_ROOT}/target/manual-transition-failure-samples/$(date +%Y%m%dT%H%M%S)" fi + if [[ -n "$ADMIN_TOKEN" && ( -n "$ACCESS_KEY" || -n "$SECRET_KEY" ) ]]; then + echo "ERROR: --admin-token cannot be combined with --access-key/--secret-key" >&2 + exit 1 + fi + if [[ -n "$ACCESS_KEY" && -z "$SECRET_KEY" || -z "$ACCESS_KEY" && -n "$SECRET_KEY" ]]; then + echo "ERROR: --access-key and --secret-key must be provided together" >&2 + exit 1 + fi } sample_label() { @@ -201,6 +219,8 @@ curl_json() { local args=(-fsS) if [[ -n "$ADMIN_TOKEN" ]]; then args+=(-H "Authorization: Bearer ${ADMIN_TOKEN}") + elif [[ -n "$ACCESS_KEY" ]]; then + args+=(--aws-sigv4 "$AWS_SIGV4_SCOPE" --user "${ACCESS_KEY}:${SECRET_KEY}") fi curl "${args[@]}" "$url" > "$out_file" } @@ -233,7 +253,8 @@ Replace the placeholders and run: ```bash scripts/manual_transition_failure_samples.sh \ --endpoint \ - --admin-token "$TOKEN" \ + --access-key "$ACCESS_KEY" \ + --secret-key "$SECRET_KEY" \ --sample auth:: \ --sample network:: \ --sample timeout:: \ @@ -249,7 +270,7 @@ scripts/manual_transition_failure_samples.sh \ - terminal partial/failed counts agree with `tier_failure_by_reason` PLAN cat >"${OUT_DIR}/commands.txt" <: --sample network:: --min-distinct-reasons ${MIN_DISTINCT_REASONS} --out-dir ${OUT_DIR} +scripts/manual_transition_failure_samples.sh --endpoint ${ENDPOINT} --access-key --secret-key --sample auth:: --sample network:: --min-distinct-reasons ${MIN_DISTINCT_REASONS} --out-dir ${OUT_DIR} COMMANDS echo "[DRY-RUN] generated ${OUT_DIR}/sample_plan.md" echo "[DRY-RUN] generated ${OUT_DIR}/commands.txt" @@ -272,7 +293,7 @@ collect_logs() { log_paths=("$LOG_GLOB") fi - if ! rg --no-filename --color=never -- "${job_id}|lifecycle_tier_operation_failed|lifecycle_worker_state" -- "${log_paths[@]}" >"$raw_file" 2>"$err_file"; then + if ! rg --no-filename --color=never -- "${job_id}|lifecycle_tier_operation_failed|lifecycle_worker_state" "${log_paths[@]}" >"$raw_file" 2>"$err_file"; then : fi @@ -312,6 +333,7 @@ write_command_notes() { cat >"${OUT_DIR}/commands.txt" < \\\n'; elif [[ -n "$ACCESS_KEY" ]]; then printf ' --access-key --secret-key \\\n'; fi) $(printf ' --sample %q \\\n' "${SAMPLES[@]}") # Per-job deep dive @@ -398,6 +420,7 @@ main() { require_cmd jq require_cmd rg require_cmd awk + require_cmd tr mkdir -p "$OUT_DIR" local summary_csv="${OUT_DIR}/failure_attribution_summary.csv" diff --git a/scripts/manual_transition_nightly_stress_runbook.sh b/scripts/manual_transition_nightly_stress_runbook.sh index ab93b5473..e5700c293 100755 --- a/scripts/manual_transition_nightly_stress_runbook.sh +++ b/scripts/manual_transition_nightly_stress_runbook.sh @@ -20,6 +20,9 @@ PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" ENDPOINT="" ADMIN_TOKEN="" +ACCESS_KEY="" +SECRET_KEY="" +AWS_SIGV4_SCOPE="aws:amz:us-east-1:s3" OUT_DIR="" WINDOW_SPEC="nightly-2h:120:16:5000:balanced,nightly-12h:720:24:8000:write-heavy,nightly-24h:1440:32:12000:read-heavy" SOAK_RATIOS="read-heavy:90:10,balanced:70:30,write-heavy:40:60" @@ -47,6 +50,9 @@ Required: Optional: --admin-token Bearer token for admin endpoints + --access-key Access key for SigV4-signed admin calls + --secret-key Secret key for SigV4-signed admin calls + --aws-sigv4-scope curl --aws-sigv4 scope, default aws:amz:us-east-1:s3 --window-spec Comma-separated run spec: window:duration_min:concurrency:ops_per_min:mix_name --soak-ratios Comma-separated mix ratios: label:read_pct:write_pct --workload-sizes Object-size workload set @@ -94,6 +100,18 @@ parse_args() { ADMIN_TOKEN="$(arg_value "$1" "${2:-}")" shift 2 ;; + --access-key) + ACCESS_KEY="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --secret-key) + SECRET_KEY="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --aws-sigv4-scope) + AWS_SIGV4_SCOPE="$(arg_value "$1" "${2:-}")" + shift 2 + ;; --window-spec) WINDOW_SPEC="$(arg_value "$1" "${2:-}")" shift 2 @@ -228,6 +246,9 @@ set -euo pipefail SOAK_MATRIX_CSV='__SOAK_MATRIX_CSV__' ENDPOINT='__ENDPOINT__' ADMIN_TOKEN='__ADMIN_TOKEN__' +ACCESS_KEY='__ACCESS_KEY__' +SECRET_KEY='__SECRET_KEY__' +AWS_SIGV4_SCOPE='__AWS_SIGV4_SCOPE__' JOB_BUCKET='__JOB_BUCKET__' JOB_PREFIX='__JOB_PREFIX__' TIER='__TIER__' @@ -238,7 +259,9 @@ OUT_DIR='__OUT_DIR__' : "${UNKNOWN_FAILURE_COUNT_THRESHOLD:=0}" SNAPSHOT_DIR="${OUT_DIR}/failure-snapshots" +RESULT_DIR="${OUT_DIR}/run-results" mkdir -p "$SNAPSHOT_DIR" +mkdir -p "$RESULT_DIR" require_cmd() { local cmd="$1" @@ -253,6 +276,19 @@ url_encode() { jq -rn --arg v "$value" '$v|@uri' } +curl_admin() { + local method="$1" + local url="$2" + shift 2 + local args=(-sS -X "$method") + if [[ -n "$ADMIN_TOKEN" ]]; then + args+=(-H "Authorization: Bearer ${ADMIN_TOKEN}") + elif [[ -n "$ACCESS_KEY" ]]; then + args+=(--aws-sigv4 "$AWS_SIGV4_SCOPE" --user "${ACCESS_KEY}:${SECRET_KEY}") + fi + curl "${args[@]}" "$url" "$@" +} + snapshot_failure() { local run_tag="$1" local reason="$2" @@ -272,7 +308,7 @@ snapshot_failure() { } >"${snapshot_dir}/snapshot.meta" if [[ -n "$job_id" && "$job_id" != "NA" ]]; then - curl -sS ${ADMIN_TOKEN:+-H "Authorization: Bearer ${ADMIN_TOKEN}"} -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" \ + curl_admin GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" \ >"${snapshot_dir}/job_status.json" ./scripts/manual_transition_journal_audit.sh --endpoint "$ENDPOINT" --job-id "$job_id" ${ADMIN_TOKEN:+--admin-token "$ADMIN_TOKEN"} --out-dir "$snapshot_dir" || true fi @@ -300,7 +336,7 @@ run_entry() { local status local status_json local status_info - local headers=() + local result_tag if [[ "$budget_status" == "over-budget" ]]; then echo "[skip] ${tag} ${size} over budget: expected_ops=${expected_ops}" @@ -308,6 +344,7 @@ run_entry() { fi prefix="${JOB_PREFIX}/${tag}/${size}/${read_pct}r${write_pct}w" + result_tag="${tag}-${size}-${read_pct}r${write_pct}w" query="bucket=$(url_encode "$JOB_BUCKET")" query="${query}&prefix=$(url_encode "$prefix")" query="${query}&maxObjects=100000" @@ -319,32 +356,30 @@ run_entry() { url="${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/run?${query}" - if [[ -n "$ADMIN_TOKEN" ]]; then - headers+=("-H" "Authorization: Bearer ${ADMIN_TOKEN}") - fi - echo "==> run=${tag} size=${size} mix=${mix_name} read_pct=${read_pct} write_pct=${write_pct} duration_min=${duration_min}" - if ! response="$(curl -sS "${headers[@]}" -X POST "$url")"; then + if ! response="$(curl_admin POST "$url")"; then snapshot_failure "$tag" "curl_post_failed" "NA" return 1 fi + printf '%s' "$response" >"${RESULT_DIR}/${result_tag}-run.json" job_id="$(printf '%s' "$response" | jq -r '.job_id // empty')" if [[ -z "$job_id" || "$job_id" == "null" ]]; then snapshot_failure "$tag" "missing_job_id" "NA" return 1 fi - if ! status_json="$(curl -sS "${headers[@]}" -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}")"; then + if ! status_json="$(curl_admin GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}")"; then snapshot_failure "$tag" "job_status_fetch_failed" "$job_id" return 1 fi - if ! status_info="$(printf '%s' "$status_json" | jq -r '[.status // "", .failure_reason // "", (.report.enqueued // 0), (.report.transition_completed // 0), (.report.transition_failed // 0), (.report.tier_failure_by_reason["unknown"] // 0), (.queue_snapshot.queued // 0), (.queue_snapshot.active // 0), (.queue_snapshot.compensation_pending // 0), (.queue_snapshot.compensation_running // 0), (.queue_snapshot.queue_full // 0), (.queue_snapshot.queue_send_timeout // 0)] | map(tostring) | join(\"\\u0000\")')"; then + printf '%s' "$status_json" >"${RESULT_DIR}/${result_tag}-status.json" + if ! status_info="$(printf '%s' "$status_json" | jq -r '[.status // "", .failure_reason // "__none__", (.report.enqueued // 0), (.report.transition_completed // 0), (.report.transition_failed // 0), (.report.tier_failure_by_reason["unknown"] // 0), (.queue_snapshot.queued // 0), (.queue_snapshot.active // 0), (.queue_snapshot.compensation_pending // 0), (.queue_snapshot.compensation_running // 0), (.queue_snapshot.queue_full // 0), (.queue_snapshot.queue_send_timeout // 0)] | map(tostring) | @tsv')"; then snapshot_failure "$tag" "invalid_job_status_json" "$job_id" return 1 fi - IFS=$'\0' read -r status failure_reason report_enqueued report_completed report_failed report_unknown_failure queue_snapshot_queued queue_snapshot_active compensation_pending compensation_running queue_full queue_send_timeout <<< "$status_info" + IFS=$'\t' read -r status failure_reason report_enqueued report_completed report_failed report_unknown_failure queue_snapshot_queued queue_snapshot_active compensation_pending compensation_running queue_full queue_send_timeout <<< "$status_info" - if [[ -n "$failure_reason" && "$failure_reason" != "null" ]]; then + if [[ "$failure_reason" != "__none__" ]]; then snapshot_failure "$tag" "failure_reason=${failure_reason}" "$job_id" fi @@ -374,24 +409,25 @@ run_entry() { unknown_ratio="$(awk -v unknown="$report_unknown_failure" -v denom="$ratio_denominator" 'BEGIN{printf "%.6f", (unknown / denom)}')" mismatch_ratio="$(awk -v mismatch="$mismatch_count" -v denom="$ratio_denominator" 'BEGIN{printf "%.6f", (mismatch / denom)}')" - if (( UNKNOWN_FAILURE_RATIO_THRESHOLD > 0 )) && \ - awk "BEGIN{exit !($unknown_ratio > ${UNKNOWN_FAILURE_RATIO_THRESHOLD})}"; then + if awk "BEGIN{exit !(${UNKNOWN_FAILURE_RATIO_THRESHOLD} > 0 && $unknown_ratio > ${UNKNOWN_FAILURE_RATIO_THRESHOLD})}"; then snapshot_failure "$tag" "unknown_failure_ratio=${unknown_ratio}/expected=${ratio_denominator}/threshold=${UNKNOWN_FAILURE_RATIO_THRESHOLD}" "$job_id" + return 1 fi if (( UNKNOWN_FAILURE_COUNT_THRESHOLD > 0 )) && (( report_unknown_failure > UNKNOWN_FAILURE_COUNT_THRESHOLD )); then snapshot_failure "$tag" "unknown_failure_count=${report_unknown_failure}/threshold=${UNKNOWN_FAILURE_COUNT_THRESHOLD}" "$job_id" + return 1 fi - if (( QUEUE_MISMATCH_RATIO_THRESHOLD > 0 )) && \ - awk "BEGIN{exit !($mismatch_ratio > ${QUEUE_MISMATCH_RATIO_THRESHOLD})}"; then + if awk "BEGIN{exit !(${QUEUE_MISMATCH_RATIO_THRESHOLD} > 0 && $mismatch_ratio > ${QUEUE_MISMATCH_RATIO_THRESHOLD})}"; then snapshot_failure "$tag" "queue_mismatch_ratio=${mismatch_ratio}/expected=${ratio_denominator}/threshold=${QUEUE_MISMATCH_RATIO_THRESHOLD}" "$job_id" + return 1 fi fi fi # Keep compatibility with older callers that monitor queue drift via full terminal status. - curl -sS "${headers[@]}" -X GET "${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" | jq '{status, report, queue_snapshot, failure_reason}' + printf '%s' "$status_json" | jq '{job_id, status, bucket, prefix, tier, report, queue_snapshot, failure_reason}' } is_terminal_status() { @@ -413,18 +449,34 @@ main() { require_cmd jq require_cmd awk + if [[ -n "$ADMIN_TOKEN" && ( -n "$ACCESS_KEY" || -n "$SECRET_KEY" ) ]]; then + echo "ERROR: bearer token cannot be combined with SigV4 credentials" >&2 + exit 1 + fi + if [[ -n "$ACCESS_KEY" && -z "$SECRET_KEY" || -z "$ACCESS_KEY" && -n "$SECRET_KEY" ]]; then + echo "ERROR: ACCESS_KEY and SECRET_KEY must be provided together" >&2 + exit 1 + fi + if [[ ! -f "$SOAK_MATRIX_CSV" ]]; then echo "ERROR: expected matrix file missing: $SOAK_MATRIX_CSV" >&2 echo "Run manual_transition_soak_matrix.sh first." >&2 exit 1 fi + local failures=0 while IFS=',' read -r window duration_min concurrency ops_per_min mix_name read_pct write_pct size expected_ops run_id budget_status; do if [[ -z "$window" || "$window" == "window" ]]; then continue fi - run_entry "${window}" "${duration_min}" "${concurrency}" "${ops_per_min}" "${mix_name}" "${read_pct}" "${write_pct}" "${size}" "${expected_ops}" "${budget_status}" || true + if ! run_entry "${window}" "${duration_min}" "${concurrency}" "${ops_per_min}" "${mix_name}" "${read_pct}" "${write_pct}" "${size}" "${expected_ops}" "${budget_status}"; then + failures=$((failures + 1)) + fi done < <(tail -n +2 "$SOAK_MATRIX_CSV") + if (( failures > 0 )); then + echo "ERROR: ${failures} soak row(s) failed; see ${SNAPSHOT_DIR}" >&2 + exit 1 + fi } main "$@" @@ -433,6 +485,9 @@ EOF perl -0pi -e "s#__SOAK_MATRIX_CSV__#${matrix_csv//#/#}#g" "$COMMAND_FILE" perl -0pi -e "s#__ENDPOINT__#${ENDPOINT//#/#}#g" "$COMMAND_FILE" perl -0pi -e "s#__ADMIN_TOKEN__#${ADMIN_TOKEN//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__ACCESS_KEY__#${ACCESS_KEY//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__SECRET_KEY__#${SECRET_KEY//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__AWS_SIGV4_SCOPE__#${AWS_SIGV4_SCOPE//#/#}#g" "$COMMAND_FILE" perl -0pi -e "s#__JOB_BUCKET__#${JOB_BUCKET//#/#}#g" "$COMMAND_FILE" perl -0pi -e "s#__JOB_PREFIX__#${JOB_PREFIX//#/#}#g" "$COMMAND_FILE" perl -0pi -e "s#__TIER__#${TIER//#/#}#g" "$COMMAND_FILE" @@ -454,6 +509,14 @@ main() { mkdir -p "$OUT_DIR" require_cmd awk require_cmd jq + if [[ -n "$ADMIN_TOKEN" && ( -n "$ACCESS_KEY" || -n "$SECRET_KEY" ) ]]; then + echo "ERROR: --admin-token cannot be combined with --access-key/--secret-key" >&2 + exit 1 + fi + if [[ -n "$ACCESS_KEY" && -z "$SECRET_KEY" || -z "$ACCESS_KEY" && -n "$SECRET_KEY" ]]; then + echo "ERROR: --access-key and --secret-key must be provided together" >&2 + exit 1 + fi main_matrix command_template diff --git a/scripts/test_manual_transition_runbooks.sh b/scripts/test_manual_transition_runbooks.sh index 87744d088..c8fa74702 100755 --- a/scripts/test_manual_transition_runbooks.sh +++ b/scripts/test_manual_transition_runbooks.sh @@ -78,6 +78,8 @@ rg -q "MIXED_MATRIX_CSV='$TMP_DIR/mixed-runbook/mixed_rollout_matrix.csv'" "$TMP "$STRESS_RUNBOOK" \ --endpoint http://127.0.0.1:9000 \ + --access-key hotadmin \ + --secret-key hotsecret \ --window-spec quick:1:1:1:balanced \ --soak-ratios balanced:70:30 \ --workload-sizes 4KiB \ @@ -86,20 +88,65 @@ rg -q "MIXED_MATRIX_CSV='$TMP_DIR/mixed-runbook/mixed_rollout_matrix.csv'" "$TMP test -x "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "failure-snapshots" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q "run-results" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "UNKNOWN_FAILURE_RATIO_THRESHOLD" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "QUEUE_MISMATCH_RATIO_THRESHOLD" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "UNKNOWN_FAILURE_COUNT_THRESHOLD" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q "ACCESS_KEY='hotadmin'" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q "SECRET_KEY='hotsecret'" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q "AWS_SIGV4_SCOPE='aws:amz:us-east-1:s3'" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q "curl_admin" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q "failures=\\$\\(\\(failures \\+ 1\\)\\)" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +if rg -q "headers\\[@\\]" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"; then + echo "nightly stress runner must not expand an unset headers array" >&2 + exit 1 +fi +if rg -q "run_entry .*\\|\\| true" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"; then + echo "nightly stress runner must not hide failed rows" >&2 + exit 1 +fi rg -q "snapshot_failure" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "manual_transition_failure_samples.sh" "$TMP_DIR/stress-runbook/manual_transition_nightly_stress_runbook.md" rg -q "is_terminal_status" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "status_json" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q "@tsv" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q 'failure_reason // "__none__"' "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q 'failure_reason" != "__none__"' "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +if rg -F -q 'failure_reason // ""' "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"; then + echo "nightly stress runner must not emit an empty TSV failure_reason field" >&2 + exit 1 +fi +rg -q "job_id, status, bucket, prefix, tier" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +if rg -F -q 'join(\"' "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"; then + echo "nightly stress runner must not emit escaped jq quotes" >&2 + exit 1 +fi rg -q "unknown_failure_ratio" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "queue_mismatch_ratio" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "report_unknown_failure" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "Manual transition nightly/stress stress-runbook" "$TMP_DIR/stress-runbook/manual_transition_nightly_stress_runbook.md" rg -q "SOAK_MATRIX_CSV='$TMP_DIR/stress-runbook/nightly_soak_matrix.csv'" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" -bash scripts/monitor_manual_transition_ci.sh --help | rg -q "Usage:" +if bash "$STRESS_RUNBOOK" --endpoint http://127.0.0.1:9000 --access-key hotadmin --dry-run >/tmp/manual_transition_stress_runbook.err 2>&1; then + echo "nightly stress runbook should fail when SigV4 credentials are incomplete" >&2 + exit 1 +fi +if ! rg -q "ERROR: --access-key and --secret-key must be provided together" /tmp/manual_transition_stress_runbook.err; then + echo "nightly stress runbook missing incomplete SigV4 credential guard output" >&2 + exit 1 +fi + +if bash "$STRESS_RUNBOOK" --endpoint http://127.0.0.1:9000 --admin-token token --access-key hotadmin --secret-key hotsecret --dry-run >/tmp/manual_transition_stress_runbook.err 2>&1; then + echo "nightly stress runbook should reject mixed bearer and SigV4 credentials" >&2 + exit 1 +fi +if ! rg -q "ERROR: --admin-token cannot be combined with --access-key/--secret-key" /tmp/manual_transition_stress_runbook.err; then + echo "nightly stress runbook missing mixed credential guard output" >&2 + exit 1 +fi + +bash scripts/monitor_manual_transition_ci.sh --help >/tmp/monitor_manual_transition_ci.help +rg -q "Usage:" /tmp/monitor_manual_transition_ci.help if bash scripts/monitor_manual_transition_ci.sh --issues >/tmp/monitor_manual_transition_ci.err 2>&1; then echo "monitor script should fail when --issues has no value" >&2 exit 1 @@ -118,7 +165,8 @@ if ! rg -q "ERROR: --runs must be a positive integer" /tmp/monitor_manual_transi exit 1 fi -bash "$FAILURE_SAMPLES" --help | rg -q "Usage:" +bash "$FAILURE_SAMPLES" --help >/tmp/manual_transition_failure_samples.help +rg -q "Usage:" /tmp/manual_transition_failure_samples.help if bash "$FAILURE_SAMPLES" --endpoint http://127.0.0.1:9000 --sample >/tmp/manual_transition_failure_samples.err 2>&1; then echo "failure samples script should fail when --sample has no value" >&2 exit 1 @@ -137,8 +185,28 @@ if ! rg -q "ERROR: --min-distinct-reasons must be a positive integer" /tmp/manua exit 1 fi +if bash "$FAILURE_SAMPLES" --endpoint http://127.0.0.1:9000 --access-key hotadmin --dry-run >/tmp/manual_transition_failure_samples.err 2>&1; then + echo "failure samples script should fail when SigV4 credentials are incomplete" >&2 + exit 1 +fi +if ! rg -q "ERROR: --access-key and --secret-key must be provided together" /tmp/manual_transition_failure_samples.err; then + echo "failure samples script missing incomplete SigV4 credential guard output" >&2 + exit 1 +fi + +if bash "$FAILURE_SAMPLES" --endpoint http://127.0.0.1:9000 --admin-token token --access-key hotadmin --secret-key hotsecret --dry-run >/tmp/manual_transition_failure_samples.err 2>&1; then + echo "failure samples script should reject mixed bearer and SigV4 credentials" >&2 + exit 1 +fi +if ! rg -q "ERROR: --admin-token cannot be combined with --access-key/--secret-key" /tmp/manual_transition_failure_samples.err; then + echo "failure samples script missing mixed credential guard output" >&2 + exit 1 +fi + "$FAILURE_SAMPLES" \ --endpoint http://127.0.0.1:9000 \ + --access-key hotadmin \ + --secret-key hotsecret \ --sample auth:11111111-1111-4111-8111-111111111111:RemoteAuth \ --sample network:22222222-2222-4222-8222-222222222222:RemoteNetwork \ --out-dir "$TMP_DIR/failure-samples" \ @@ -146,4 +214,5 @@ fi rg -q "Manual transition failure attribution sample plan" "$TMP_DIR/failure-samples/sample_plan.md" rg -q "auth::" "$TMP_DIR/failure-samples/sample_plan.md" +rg -q -- "--access-key --secret-key " "$TMP_DIR/failure-samples/commands.txt" rg -q -- "--min-distinct-reasons 2" "$TMP_DIR/failure-samples/commands.txt"