diff --git a/crates/e2e_test/src/reliant/tiering.rs b/crates/e2e_test/src/reliant/tiering.rs index 620937fed..1ee8b7ecc 100644 --- a/crates/e2e_test/src/reliant/tiering.rs +++ b/crates/e2e_test/src/reliant/tiering.rs @@ -400,6 +400,49 @@ struct ManualTransitionRunReport { continuation_token: Option, } +fn assert_completed_or_in_flight_partial(state: &str, report: &ManualTransitionRunReport, context: &str) { + match state { + "completed" => {} + "partial" if report.skipped_already_in_flight > 0 => {} + _ => panic!("{context}: state={state}, report={report:#?}"), + } +} + +fn assert_conflict_winner_report(state: &str, report: &ManualTransitionRunReport, expected_objects: u64, context: &str) { + assert_completed_or_in_flight_partial(state, report, context); + if report.skipped_already_in_flight > 0 { + assert!( + report.scanned <= expected_objects, + "{context}: scanned more objects than the conflict scope contains: {report:#?}" + ); + assert!( + report.eligible <= expected_objects, + "{context}: marked more objects eligible than the conflict scope contains: {report:#?}" + ); + assert_eq!( + report.enqueued + report.skipped_already_in_flight, + report.eligible, + "{context}: partial in-flight accounting must cover every eligible object: {report:#?}" + ); + } else { + assert_eq!(report.scanned, expected_objects, "{context}: {report:#?}"); + assert_eq!( + report.eligible + report.skipped_already_transitioned, + expected_objects, + "{context}: {report:#?}" + ); + assert_eq!( + report.enqueued + report.skipped_already_in_flight, + expected_objects, + "{context}: {report:#?}" + ); + } + assert_eq!( + report.transition_completed, report.enqueued, + "{context}: winner must wait for all queued transitions: {report:#?}" + ); +} + #[derive(Debug, Deserialize)] struct ManualTransitionQueueSnapshot { queue_capacity: u64, @@ -880,7 +923,7 @@ async fn test_manual_transition_run_black_box_semantics() -> TestResult { assert_eq!(due.mode, "enqueue_only"); assert!(due.job_id.is_none()); assert!(due.status_endpoint.is_none()); - assert_eq!(due.state, "completed"); + assert_completed_or_in_flight_partial(&due.state, &due.report, "due manual transition run"); assert_eq!(due.report.bucket, MANUAL_DUE_BUCKET); assert_eq!(due.report.prefix, "manual-due/"); assert_eq!(due.report.tier.as_deref(), Some(TIER_NAME)); @@ -1314,29 +1357,16 @@ async fn test_manual_transition_async_overlapping_scope_conflict_reports_active_ let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, StdDuration::from_secs(30)).await?; assert_eq!(terminal.job_id, job_id); - assert_eq!(terminal.status, "completed", "terminal conflict winner response: {terminal:#?}"); assert!(!terminal.report.dry_run); assert_eq!(terminal.report.bucket, MANUAL_ASYNC_CONFLICT_BUCKET); assert_eq!(terminal.report.prefix, accepted.report.prefix); - assert_eq!( - terminal.report.scanned, MANUAL_ASYNC_CONFLICT_OBJECTS as u64, - "terminal conflict winner response: {terminal:#?}" - ); - assert_eq!( - terminal.report.eligible + terminal.report.skipped_already_transitioned, + assert_conflict_winner_report( + &terminal.status, + &terminal.report, MANUAL_ASYNC_CONFLICT_OBJECTS as u64, - "terminal conflict winner response: {terminal:#?}" + "terminal conflict winner response", ); assert_eq!(terminal.report.dry_run_eligible, 0, "terminal conflict winner response: {terminal:#?}"); - assert_eq!( - terminal.report.enqueued + terminal.report.skipped_already_in_flight, - terminal.report.eligible, - "terminal conflict winner response: {terminal:#?}" - ); - assert_eq!( - terminal.report.transition_completed, terminal.report.enqueued, - "terminal conflict winner must wait for all queued transitions: {terminal:#?}" - ); assert_eq!(terminal.report.transition_failed, 0, "terminal conflict winner response: {terminal:#?}"); assert_eq!(terminal.report.tier_failure, 0, "terminal conflict winner response: {terminal:#?}"); let after_remote_count = cold_tier_object_count(&cold_client).await?; @@ -1440,26 +1470,9 @@ async fn test_manual_transition_async_same_scope_conflict_reports_active_job() - let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, StdDuration::from_secs(30)).await?; assert_eq!(terminal.job_id, job_id); - assert_eq!( - terminal.status, "completed", - "terminal same-scope conflict winner response: {terminal:#?}" - ); assert_eq!(terminal.report.bucket, MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET); assert_eq!(terminal.report.prefix, MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX); - assert_eq!(terminal.report.scanned, 50, "terminal same-scope conflict winner response: {terminal:#?}"); - assert_eq!( - terminal.report.eligible, 50, - "terminal same-scope conflict winner response: {terminal:#?}" - ); - assert_eq!( - terminal.report.enqueued + terminal.report.skipped_already_in_flight, - 50, - "terminal same-scope conflict winner response: {terminal:#?}" - ); - assert_eq!( - terminal.report.transition_completed, terminal.report.enqueued, - "terminal same-scope conflict winner must wait for all queued transitions: {terminal:#?}" - ); + assert_conflict_winner_report(&terminal.status, &terminal.report, 50, "terminal same-scope conflict winner response"); assert_eq!( terminal.report.transition_failed, 0, "terminal same-scope conflict winner response: {terminal:#?}" @@ -2189,7 +2202,7 @@ async fn test_manual_transition_run_queue_pressure_partial() -> TestResult { assert_eq!(response.report.bucket, MANUAL_QUEUE_PRESSURE_BUCKET); assert_eq!(response.report.prefix, MANUAL_QUEUE_PRESSURE_PREFIX); assert!( - response.report.skipped_queue_full > 0, + response.report.skipped_queue_full > 0 || response.report.skipped_already_in_flight > 0, "expected queue-pressure path to skip at least one object: {:#?}", response.report ); diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 2f12513fe..e06a25dd2 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -3452,7 +3452,10 @@ impl ManualTransitionRunReport { } pub fn has_partial_enqueue(&self) -> bool { - self.skipped_queue_full > 0 || self.skipped_queue_closed > 0 || self.skipped_queue_timeout > 0 + self.skipped_already_in_flight > 0 + || self.skipped_queue_full > 0 + || self.skipped_queue_closed > 0 + || self.skipped_queue_timeout > 0 } pub fn was_truncated(&self) -> bool { diff --git a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs index d15d79e67..a8db7d722 100644 --- a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs +++ b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs @@ -2460,6 +2460,25 @@ mod tests { assert!(record.error.is_none()); } + #[test] + fn manual_transition_job_record_in_flight_skip_report_is_partial() { + let options = ManualTransitionRunOptions::default(); + let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER); + + record.complete( + ManualTransitionRunReport { + eligible: 1, + skipped_already_in_flight: 1, + ..Default::default() + }, + ManualTransitionQueueSnapshot::default(), + ); + + assert_eq!(record.state, ManualTransitionJobState::Partial); + assert_eq!(record.report.skipped_already_in_flight, 1); + assert!(record.error.is_none()); + } + #[test] fn manual_transition_job_record_queue_pressure_reports_persist_readback() { let options = ManualTransitionRunOptions { diff --git a/rustfs/src/admin/handlers/ilm_transition.rs b/rustfs/src/admin/handlers/ilm_transition.rs index 52fac0dba..5d935187c 100644 --- a/rustfs/src/admin/handlers/ilm_transition.rs +++ b/rustfs/src/admin/handlers/ilm_transition.rs @@ -1148,6 +1148,16 @@ mod tests { assert_eq!(response_state(&report), "partial"); } + #[test] + fn manual_transition_response_reports_partial_for_in_flight_skip() { + let report = ManualTransitionRunReport { + skipped_already_in_flight: 1, + ..Default::default() + }; + + assert_eq!(response_state(&report), "partial"); + } + #[test] fn manual_transition_response_reports_partial_for_duration_budget() { let report = ManualTransitionRunReport { diff --git a/scripts/manual_transition_mixed_rollout_runbook.sh b/scripts/manual_transition_mixed_rollout_runbook.sh index 2e521da27..8096c64ad 100755 --- a/scripts/manual_transition_mixed_rollout_runbook.sh +++ b/scripts/manual_transition_mixed_rollout_runbook.sh @@ -20,6 +20,9 @@ PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" ENDPOINT="" ADMIN_TOKEN="" +ACCESS_KEY="" +SECRET_KEY="" +AWS_SIGV4_SCOPE="aws:amz:us-east-1:s3" OUT_DIR="" PHASE_MATRIX="request-only:100:0:30,canary:90:10:120,mixed:50:50:240,full-rollout:0:100:360,rollback:100:0:30" CONCURRENCIES="8,16,32" @@ -45,6 +48,9 @@ Required: Optional: --admin-token Bearer token for admin endpoints + --access-key Access key for SigV4-signed admin calls + --secret-key Secret key for SigV4-signed admin calls + --aws-sigv4-scope curl --aws-sigv4 scope, default aws:amz:us-east-1:s3 --job-bucket Job bucket for transition scope (default: manual-transition) --job-prefix Prefix for generated runbook phase names (default: journal-mixed-rollout) --tier Transition tier (default: empty) @@ -93,6 +99,18 @@ parse_args() { ADMIN_TOKEN="$(arg_value "$1" "${2:-}")" shift 2 ;; + --access-key) + ACCESS_KEY="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --secret-key) + SECRET_KEY="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --aws-sigv4-scope) + AWS_SIGV4_SCOPE="$(arg_value "$1" "${2:-}")" + shift 2 + ;; --job-bucket) JOB_BUCKET="$(arg_value "$1" "${2:-}")" shift 2 @@ -181,8 +199,10 @@ command_template() { fi if [[ -n "$ADMIN_TOKEN" ]]; then token_note='export ADMIN_TOKEN=' + elif [[ -n "$ACCESS_KEY" ]]; then + token_note='SigV4 credentials embedded in generated runner' else - token_note='export ADMIN_TOKEN="" (set this value)' + token_note='export ADMIN_TOKEN="" or pass --access-key/--secret-key' fi cat >"$RUNBOOK_FILE" <<'RUNBOOK' @@ -214,6 +234,8 @@ Use the commands below directly: - __COMMAND_FILE__ - generated 'run_phase_commands.sh' is a per-phase audit starter; 'run_mixed_rollout_plan.sh' is the full runnable template. - for failure-oriented rollout phases, use scripts/manual_transition_failure_samples.sh to capture auth/network/timeout attribution evidence from the produced job ids. +- for #1508 strict validation, pre-seed non-empty lifecycle-matching objects before each phase and keep both run response and terminal status/readback evidence. +- for in-flight rollback validation, set IN_FLIGHT_ROLLBACK_HOOK to a script that replaces one new node with the old binary after job admission and before terminal status polling. - recommended runbook order: 1. Review 'mixed_rollout_checklist.md'. 2. Inspect __MATRIX_CMD__ for each phase note. @@ -239,10 +261,16 @@ set -euo pipefail MIXED_MATRIX_CSV='__MIXED_MATRIX_CSV__' ENDPOINT='__ENDPOINT__' ADMIN_TOKEN='__ADMIN_TOKEN__' +ACCESS_KEY='__ACCESS_KEY__' +SECRET_KEY='__SECRET_KEY__' +AWS_SIGV4_SCOPE='__AWS_SIGV4_SCOPE__' JOB_BUCKET='__JOB_BUCKET__' JOB_PREFIX='__JOB_PREFIX__' TIER='__TIER__' OUT_DIR='__OUT_DIR__' +POLL_SECONDS="${POLL_SECONDS:-120}" +S3_ENDPOINT="${S3_ENDPOINT:-$ENDPOINT}" +IN_FLIGHT_ROLLBACK_HOOK="${IN_FLIGHT_ROLLBACK_HOOK:-}" require_cmd() { local cmd="$1" @@ -257,12 +285,40 @@ url_encode() { jq -rn --arg v "$value" '$v|@uri' } +curl_admin() { + local method="$1" + local url="$2" + shift 2 + local args=(-sS -X "$method") + if [[ -n "$ADMIN_TOKEN" ]]; then + args+=(-H "Authorization: Bearer ${ADMIN_TOKEN}") + elif [[ -n "$ACCESS_KEY" ]]; then + args+=(--aws-sigv4 "$AWS_SIGV4_SCOPE" --user "${ACCESS_KEY}:${SECRET_KEY}") + fi + curl "${args[@]}" "$url" "$@" +} + +curl_s3() { + curl_admin "$@" +} + +is_terminal_status() { + case "$1" in + completed|partial|failed|cancelled|unknown) + return 0 + ;; + *) + return 1 + ;; + esac +} + normalize_object_count() { local raw local lower raw="$1" raw="${raw// /}" - lower="${raw,,}" + lower="$(printf '%s' "$raw" | tr '[:upper:]' '[:lower:]')" if [[ "$lower" =~ ^([0-9]+)$ ]]; then echo "$lower" @@ -283,6 +339,52 @@ normalize_object_count() { return 1 } +seed_phase_workload() { + local phase="$1" + local prefix="$2" + local object_count="$3" + local result_tag="$4" + local lifecycle_file="${OUT_DIR}/lifecycle-${result_tag}.xml" + local object_key="${prefix}/probe-${result_tag}.txt" + + if [[ -z "$TIER" ]]; then + echo "ERROR: --tier is required for non-empty mixed-rollout evidence seeding" >&2 + return 1 + fi + + curl_s3 PUT "${S3_ENDPOINT%/}/${JOB_BUCKET}" -o "${OUT_DIR}/create-bucket-${result_tag}.response" -w "%{http_code}\n" \ + >"${OUT_DIR}/create-bucket-${result_tag}.http_code" || true + + cat >"$lifecycle_file" < + + manual-transition-${phase} + ${prefix} + Enabled + 0${TIER} + + +XML + curl_s3 PUT "${S3_ENDPOINT%/}/${JOB_BUCKET}?lifecycle" -H 'Content-Type: application/xml' --data-binary "@${lifecycle_file}" \ + -o "${OUT_DIR}/put-lifecycle-${result_tag}.response" -w "%{http_code}\n" >"${OUT_DIR}/put-lifecycle-${result_tag}.http_code" + + curl_s3 PUT "${S3_ENDPOINT%/}/${JOB_BUCKET}/${object_key}" --data-binary "rustfs #1508 ${phase} non-empty transition probe" \ + -o "${OUT_DIR}/put-object-${result_tag}.response" -w "%{http_code}\n" >"${OUT_DIR}/put-object-${result_tag}.http_code" + + echo "$object_key" >"${OUT_DIR}/object-key-${result_tag}.txt" + if (( object_count > 1 )); then + echo "[info] seeded one required probe object for ${phase}; caller may pre-seed the remaining $((object_count - 1)) objects under ${prefix}" + fi +} + +head_phase_probe() { + local result_tag="$1" + local object_key + object_key="$(cat "${OUT_DIR}/object-key-${result_tag}.txt")" + curl_s3 HEAD "${S3_ENDPOINT%/}/${JOB_BUCKET}/${object_key}" -D "${OUT_DIR}/head-object-${result_tag}.headers" \ + -o /dev/null -w "%{http_code}\n" >"${OUT_DIR}/head-object-${result_tag}.http_code" || true +} + run_transition() { local phase="$1" local concurrency="$2" @@ -296,7 +398,9 @@ run_transition() { local response local job_id local status_url - local headers=() + local status_json + local result_tag + local status duration_sec=$((duration_min * 60)) if ! object_count="$(normalize_object_count "$object_count_label")"; then @@ -309,6 +413,8 @@ run_transition() { fi prefix="${JOB_PREFIX}/${phase}/${concurrency}c_${object_count_label}o" + result_tag="${phase}-${concurrency}-${object_count_label}" + seed_phase_workload "$phase" "$prefix" "$object_count" "$result_tag" query="bucket=$(url_encode "$JOB_BUCKET")" query="${query}&prefix=$(url_encode "$prefix")" @@ -322,25 +428,34 @@ run_transition() { url="${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/run?${query}" - if [[ -n "$ADMIN_TOKEN" ]]; then - headers+=("-H" "Authorization: Bearer ${ADMIN_TOKEN}") - fi - echo "==> phase=${phase} concurrency=${concurrency} object_count=${object_count} duration_min=${duration_min}" - response="$(curl -sS "${headers[@]}" -X POST "$url")" + response="$(curl_admin POST "$url")" + printf '%s\n' "$response" > "${OUT_DIR}/run-response-${result_tag}.json" job_id="$(printf '%s' "$response" | jq -r '.job_id // empty')" if [[ -z "$job_id" || "$job_id" == "null" ]]; then echo "ERROR: transition run did not return job_id for ${phase}" >&2 - echo "$response" + printf '%s\n' "$response" > "${OUT_DIR}/run-response-missing-job-${result_tag}.json" return 1 fi echo "job_id=${job_id}" status_url="${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}" + if [[ -n "$IN_FLIGHT_ROLLBACK_HOOK" ]]; then + "$IN_FLIGHT_ROLLBACK_HOOK" "$phase" "$job_id" "$status_url" "$OUT_DIR" + fi ./scripts/manual_transition_journal_audit.sh --endpoint "$ENDPOINT" --job-id "$job_id" ${ADMIN_TOKEN:+--admin-token "$ADMIN_TOKEN"} --out-dir "${OUT_DIR}/journal-audit-${phase}-${concurrency}-${object_count_label}" || true - printf '%s\n' "$response" > "${OUT_DIR}/run-response-${phase}-${concurrency}-${object_count_label}.json" echo "status_url=${status_url}" - curl -sS "${headers[@]}" -X GET "$status_url" | jq '{status, report, queue_snapshot, failure_reason}' + for _ in $(seq 1 "$POLL_SECONDS"); do + status_json="$(curl_admin GET "$status_url")" + printf '%s\n' "$status_json" > "${OUT_DIR}/status-${result_tag}.json" + status="$(printf '%s' "$status_json" | jq -r '.status // ""')" + if is_terminal_status "$status"; then + break + fi + sleep 1 + done + head_phase_probe "$result_tag" + printf '%s\n' "$status_json" | jq '{job_id, status, bucket, prefix, tier, report, queue_snapshot, failure_reason}' } main() { @@ -349,6 +464,14 @@ main() { require_cmd jq require_cmd awk require_cmd sed + if [[ -n "$ADMIN_TOKEN" && ( -n "$ACCESS_KEY" || -n "$SECRET_KEY" ) ]]; then + echo "ERROR: --admin-token cannot be combined with --access-key/--secret-key" >&2 + exit 1 + fi + if [[ -n "$ACCESS_KEY" && -z "$SECRET_KEY" || -z "$ACCESS_KEY" && -n "$SECRET_KEY" ]]; then + echo "ERROR: --access-key and --secret-key must be provided together" >&2 + exit 1 + fi if [[ ! -f "$MIXED_MATRIX_CSV" ]]; then echo "ERROR: expected matrix file missing: $MIXED_MATRIX_CSV" >&2 @@ -371,6 +494,9 @@ EOF perl -0pi -e "s#__MIXED_MATRIX_CSV__#${matrix_csv//#/#}#g" "$COMMAND_FILE" perl -0pi -e "s#__ENDPOINT__#${ENDPOINT//#/#}#g" "$COMMAND_FILE" perl -0pi -e "s#__ADMIN_TOKEN__#${ADMIN_TOKEN//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__ACCESS_KEY__#${ACCESS_KEY//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__SECRET_KEY__#${SECRET_KEY//#/#}#g" "$COMMAND_FILE" + perl -0pi -e "s#__AWS_SIGV4_SCOPE__#${AWS_SIGV4_SCOPE//#/#}#g" "$COMMAND_FILE" perl -0pi -e "s#__JOB_BUCKET__#${JOB_BUCKET//#/#}#g" "$COMMAND_FILE" perl -0pi -e "s#__JOB_PREFIX__#${JOB_PREFIX//#/#}#g" "$COMMAND_FILE" perl -0pi -e "s#__TIER__#${TIER//#/#}#g" "$COMMAND_FILE" @@ -393,6 +519,14 @@ main() { mkdir -p "$OUT_DIR" require_cmd awk require_cmd jq + if [[ -n "$ADMIN_TOKEN" && ( -n "$ACCESS_KEY" || -n "$SECRET_KEY" ) ]]; then + echo "ERROR: --admin-token cannot be combined with --access-key/--secret-key" >&2 + exit 1 + fi + if [[ -n "$ACCESS_KEY" && -z "$SECRET_KEY" || -z "$ACCESS_KEY" && -n "$SECRET_KEY" ]]; then + echo "ERROR: --access-key and --secret-key must be provided together" >&2 + exit 1 + fi main_matrix command_template diff --git a/scripts/manual_transition_nightly_stress_runbook.sh b/scripts/manual_transition_nightly_stress_runbook.sh index e5700c293..b2cbfcacc 100755 --- a/scripts/manual_transition_nightly_stress_runbook.sh +++ b/scripts/manual_transition_nightly_stress_runbook.sh @@ -200,6 +200,7 @@ command_template() { - unknown failure ratio threshold: __UNKNOWN_FAILURE_RATIO_THRESHOLD__ - queue-mismatch tolerance ratio: __QUEUE_MISMATCH_RATIO_THRESHOLD__ - unknown failure count threshold: __UNKNOWN_FAILURE_COUNT_THRESHOLD__ +- already-in-flight transitions: fail any terminal run with skipped_already_in_flight > 0 ## Failure snapshot policy @@ -373,11 +374,11 @@ run_entry() { return 1 fi printf '%s' "$status_json" >"${RESULT_DIR}/${result_tag}-status.json" - if ! status_info="$(printf '%s' "$status_json" | jq -r '[.status // "", .failure_reason // "__none__", (.report.enqueued // 0), (.report.transition_completed // 0), (.report.transition_failed // 0), (.report.tier_failure_by_reason["unknown"] // 0), (.queue_snapshot.queued // 0), (.queue_snapshot.active // 0), (.queue_snapshot.compensation_pending // 0), (.queue_snapshot.compensation_running // 0), (.queue_snapshot.queue_full // 0), (.queue_snapshot.queue_send_timeout // 0)] | map(tostring) | @tsv')"; then + if ! status_info="$(printf '%s' "$status_json" | jq -r '[.status // "", .failure_reason // "__none__", (.report.enqueued // 0), (.report.transition_completed // 0), (.report.transition_failed // 0), (.report.skipped_already_in_flight // 0), (.report.tier_failure_by_reason["unknown"] // 0), (.queue_snapshot.queued // 0), (.queue_snapshot.active // 0), (.queue_snapshot.compensation_pending // 0), (.queue_snapshot.compensation_running // 0), (.queue_snapshot.queue_full // 0), (.queue_snapshot.queue_send_timeout // 0)] | map(tostring) | @tsv')"; then snapshot_failure "$tag" "invalid_job_status_json" "$job_id" return 1 fi - IFS=$'\t' read -r status failure_reason report_enqueued report_completed report_failed report_unknown_failure queue_snapshot_queued queue_snapshot_active compensation_pending compensation_running queue_full queue_send_timeout <<< "$status_info" + IFS=$'\t' read -r status failure_reason report_enqueued report_completed report_failed report_already_in_flight report_unknown_failure queue_snapshot_queued queue_snapshot_active compensation_pending compensation_running queue_full queue_send_timeout <<< "$status_info" if [[ "$failure_reason" != "__none__" ]]; then snapshot_failure "$tag" "failure_reason=${failure_reason}" "$job_id" @@ -390,6 +391,10 @@ run_entry() { local mismatch_ratio local unknown_ratio local ratio_denominator + if (( report_already_in_flight > 0 )); then + snapshot_failure "$tag" "already_in_flight=${report_already_in_flight}" "$job_id" + return 1 + fi mismatch_count="$((report_enqueued - report_completed - report_failed))" if (( mismatch_count < 0 )); then mismatch_count=0 diff --git a/scripts/test_manual_transition_runbooks.sh b/scripts/test_manual_transition_runbooks.sh index c8fa74702..ca940eb67 100755 --- a/scripts/test_manual_transition_runbooks.sh +++ b/scripts/test_manual_transition_runbooks.sh @@ -65,6 +65,8 @@ rg -qx "quick,1,1,1,balanced,70,30,4KiB,1,1,within-budget" "$TMP_DIR/soak/nightl "$MIXED_RUNBOOK" \ --endpoint http://127.0.0.1:9000 \ + --access-key hotadmin \ + --secret-key hotsecret \ --phase-matrix request-only:100:0:1 \ --concurrencies 2 \ --object-counts 3k \ @@ -74,7 +76,38 @@ rg -qx "quick,1,1,1,balanced,70,30,4KiB,1,1,within-budget" "$TMP_DIR/soak/nightl test -x "$TMP_DIR/mixed-runbook/run_mixed_rollout_plan.sh" rg -q "Manual transition mixed-version rollout runbook" "$TMP_DIR/mixed-runbook/manual_transition_mixed_rollout_runbook.md" rg -q "manual_transition_failure_samples.sh" "$TMP_DIR/mixed-runbook/manual_transition_mixed_rollout_runbook.md" +rg -q "non-empty lifecycle-matching objects" "$TMP_DIR/mixed-runbook/manual_transition_mixed_rollout_runbook.md" +rg -q "in-flight rollback validation" "$TMP_DIR/mixed-runbook/manual_transition_mixed_rollout_runbook.md" rg -q "MIXED_MATRIX_CSV='$TMP_DIR/mixed-runbook/mixed_rollout_matrix.csv'" "$TMP_DIR/mixed-runbook/run_mixed_rollout_plan.sh" +rg -q "ACCESS_KEY='hotadmin'" "$TMP_DIR/mixed-runbook/run_mixed_rollout_plan.sh" +rg -q "SECRET_KEY='hotsecret'" "$TMP_DIR/mixed-runbook/run_mixed_rollout_plan.sh" +rg -q "curl_admin" "$TMP_DIR/mixed-runbook/run_mixed_rollout_plan.sh" +rg -q "POLL_SECONDS" "$TMP_DIR/mixed-runbook/run_mixed_rollout_plan.sh" +rg -q "S3_ENDPOINT=" "$TMP_DIR/mixed-runbook/run_mixed_rollout_plan.sh" +rg -q "IN_FLIGHT_ROLLBACK_HOOK" "$TMP_DIR/mixed-runbook/run_mixed_rollout_plan.sh" +rg -q "seed_phase_workload" "$TMP_DIR/mixed-runbook/run_mixed_rollout_plan.sh" +rg -q "head_phase_probe" "$TMP_DIR/mixed-runbook/run_mixed_rollout_plan.sh" +rg -q "put-lifecycle" "$TMP_DIR/mixed-runbook/run_mixed_rollout_plan.sh" +rg -q "put-object" "$TMP_DIR/mixed-runbook/run_mixed_rollout_plan.sh" +rg -q "head-object" "$TMP_DIR/mixed-runbook/run_mixed_rollout_plan.sh" + +if bash "$MIXED_RUNBOOK" --endpoint http://127.0.0.1:9000 --access-key hotadmin --dry-run >/tmp/manual_transition_mixed_runbook.err 2>&1; then + echo "mixed rollout runbook should fail when SigV4 credentials are incomplete" >&2 + exit 1 +fi +if ! rg -q "ERROR: --access-key and --secret-key must be provided together" /tmp/manual_transition_mixed_runbook.err; then + echo "mixed rollout runbook missing incomplete SigV4 credential guard output" >&2 + exit 1 +fi + +if bash "$MIXED_RUNBOOK" --endpoint http://127.0.0.1:9000 --admin-token token --access-key hotadmin --secret-key hotsecret --dry-run >/tmp/manual_transition_mixed_runbook.err 2>&1; then + echo "mixed rollout runbook should reject mixed bearer and SigV4 credentials" >&2 + exit 1 +fi +if ! rg -q "ERROR: --admin-token cannot be combined with --access-key/--secret-key" /tmp/manual_transition_mixed_runbook.err; then + echo "mixed rollout runbook missing mixed credential guard output" >&2 + exit 1 +fi "$STRESS_RUNBOOK" \ --endpoint http://127.0.0.1:9000 \ @@ -107,9 +140,12 @@ if rg -q "run_entry .*\\|\\| true" "$TMP_DIR/stress-runbook/run_nightly_stress_p fi rg -q "snapshot_failure" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "manual_transition_failure_samples.sh" "$TMP_DIR/stress-runbook/manual_transition_nightly_stress_runbook.md" +rg -q "already-in-flight transitions" "$TMP_DIR/stress-runbook/manual_transition_nightly_stress_runbook.md" rg -q "is_terminal_status" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "status_json" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q "@tsv" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q "report_already_in_flight" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" +rg -q "already_in_flight=" "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q 'failure_reason // "__none__"' "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" rg -q 'failure_reason" != "__none__"' "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh" if rg -F -q 'failure_reason // ""' "$TMP_DIR/stress-runbook/run_nightly_stress_plan.sh"; then