diff --git a/scripts/README.md b/scripts/README.md index c88f4b679..1885201f8 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -90,7 +90,7 @@ their issue closes. | `manual_transition_journal_audit.sh` | dev-tool | Journal + metrics + log audit for manual transition jobs | — | | `manual_transition_mixed_rollout_matrix.sh` | dev-tool | Matrix generator for mixed-version rollout phases | — | | `manual_transition_mixed_rollout_runbook.sh` | dev-tool | Reusable mixed-version rollout runbook generator (external run) | — | -| `manual_transition_mixed_version_docker_harness.sh` | dev-tool | Dedicated #1508 Docker harness for old/new manual-transition rollout evidence | `test_manual_transition_runbooks.sh` | +| `manual_transition_mixed_version_docker_harness.sh` | dev-tool | Dedicated #1508 Docker harness for old/new manual-transition rollout evidence with strict/baseline/blocked result classification | `test_manual_transition_runbooks.sh` | | `monitor_manual_transition_ci.sh` | dev-tool | CI workflow/status watcher for manual transition follow-up monitoring | — | | `manual_transition_soak_matrix.sh` | dev-tool | Matrix generator for nightly stress windows | — | | `manual_transition_nightly_stress_runbook.sh` | dev-tool | Nightly stress entrypoint with failure snapshot templates | — | diff --git a/scripts/manual_transition_mixed_version_docker_harness.sh b/scripts/manual_transition_mixed_version_docker_harness.sh index 0937703fc..68a611611 100755 --- a/scripts/manual_transition_mixed_version_docker_harness.sh +++ b/scripts/manual_transition_mixed_version_docker_harness.sh @@ -25,9 +25,13 @@ COLD_IMAGE="${COLD_IMAGE:-${NEW_IMAGE}}" BASE_PORT="${BASE_PORT:-19400}" OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/manual-transition-1508-docker/$(date +%Y%m%dT%H%M%S)}" KEEP_UP=false -ROLLBACK_NEW2_TO_OLD=true +ROLLBACK_PHASE="${ROLLBACK_PHASE:-after-terminal}" +OLD_NODE_PHASE="${OLD_NODE_PHASE:-initial}" WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-180}" POLL_SECONDS="${POLL_SECONDS:-180}" +TRANSITION_WORKERS="${TRANSITION_WORKERS:-2}" +TRANSITION_QUEUE_CAPACITY="${TRANSITION_QUEUE_CAPACITY:-64}" +FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT="${FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT:-false}" HOT_ACCESS_KEY="${HOT_ACCESS_KEY:-mvadmin}" HOT_SECRET_KEY="${HOT_SECRET_KEY:-mvsecret}" @@ -58,18 +62,31 @@ Options: --object-count Non-empty probe object count --tier Remote tier name --keep-up Leave Docker services running - --no-rollback Do not replace node2 with old image after job admission + --rollback-phase Rollback node2 timing: after-terminal, in-flight, none + --old-node-phase Old node1 timing: initial, before-job + --no-rollback Alias for --rollback-phase none -h, --help Show help Environment: PROJECT_NAME OLD_IMAGE NEW_IMAGE COLD_IMAGE BASE_PORT OUT_DIR KEEP_UP HOT_ACCESS_KEY HOT_SECRET_KEY COLD_ACCESS_KEY COLD_SECRET_KEY TIER_NAME TIER_BUCKET TIER_PREFIX JOB_BUCKET JOB_PREFIX OBJECT_COUNT - WAIT_TIMEOUT_SECS POLL_SECONDS AWS_SIGV4_SCOPE + ROLLBACK_PHASE WAIT_TIMEOUT_SECS POLL_SECONDS TRANSITION_WORKERS + TRANSITION_QUEUE_CAPACITY OLD_NODE_PHASE FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT AWS_SIGV4_SCOPE Artifacts: compose.yml, image inspect files, health/readiness logs, API responses, terminal status, old-node readback, container logs, summary.env. + +Result classifications: + strict_mixed_rollout_pass Real old/new images, non-empty completed transition, zero failures + baseline_tiered_storage_pass Same old/new image completed transition; useful baseline, not #1508 closure + blocked_manual_api_not_implemented Manual transition API returned 501 before job admission + blocked_manual_api_unavailable Manual transition API did not return a usable job_id + blocked_cluster_readiness_failed Docker cluster did not reach health/readiness before admission + blocked_empty_scan_or_lifecycle Job completed without lifecycle-matching transition work + blocked_manual_job_preempted_by_lifecycle_queue Lifecycle/immediate transition queued work before the job + strict_mixed_rollout_fail Mixed rollout ran but did not satisfy the strict #1508 gate USAGE } @@ -111,6 +128,28 @@ parse_positive_int() { fi } +validate_rollback_phase() { + case "$ROLLBACK_PHASE" in + after-terminal|in-flight|none) + ;; + *) + log_error "--rollback-phase must be one of: after-terminal, in-flight, none" + exit 1 + ;; + esac +} + +validate_old_node_phase() { + case "$OLD_NODE_PHASE" in + initial|before-job) + ;; + *) + log_error "--old-node-phase must be one of: initial, before-job" + exit 1 + ;; + esac +} + parse_args() { while [[ $# -gt 0 ]]; do case "$1" in @@ -150,8 +189,16 @@ parse_args() { KEEP_UP=true shift ;; + --rollback-phase) + ROLLBACK_PHASE="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --old-node-phase) + OLD_NODE_PHASE="$(arg_value "$1" "${2:-}")" + shift 2 + ;; --no-rollback) - ROLLBACK_NEW2_TO_OLD=false + ROLLBACK_PHASE=none shift ;; -h|--help) @@ -198,12 +245,18 @@ cleanup() { } write_compose_file() { + local node1_image COMPOSE_FILE="${OUT_DIR}/compose.yml" + node1_image="$OLD_IMAGE" + if [[ "$OLD_NODE_PHASE" == "before-job" ]]; then + node1_image="$NEW_IMAGE" + fi cat >"$COMPOSE_FILE" <"${OUT_DIR}/run-response.json" + curl_hot POST "$(hot_endpoint 2)/rustfs/admin/v3/ilm/transition/run?${query}" \ + -o "${OUT_DIR}/run-response.json" \ + -w "%{http_code}\n" >"${OUT_DIR}/run-response.http_code" || true + response="$(cat "${OUT_DIR}/run-response.json" 2>/dev/null || true)" + run_http_code="$(cat "${OUT_DIR}/run-response.http_code" 2>/dev/null || true)" job_id="$(printf '%s' "$response" | jq -r '.job_id // empty')" if [[ -z "$job_id" ]]; then - log_error "manual transition response omitted job_id" + log_warn "manual transition response omitted job_id, http_code=${run_http_code:-unknown}" return 1 fi printf '%s\n' "$job_id" >"${OUT_DIR}/job-id.txt" @@ -451,19 +516,25 @@ start_transition_job() { replace_node2_with_old_image() { local network network="$(network_name)" - log_info "Replacing node2 with old image ${OLD_IMAGE} for in-flight rollback readback" + log_info "Replacing node2 with old image ${OLD_IMAGE} for ${ROLLBACK_PHASE} rollback readback" compose stop node2 >/dev/null compose rm -f node2 >/dev/null docker run -d \ --name "${PROJECT_NAME}-node2-rollback-old" \ --network "$network" \ --network-alias node2 \ + --user 0:0 \ -p "$((BASE_PORT + 2)):9000" \ -e RUSTFS_ADDRESS=:9000 \ -e RUSTFS_ACCESS_KEY="$HOT_ACCESS_KEY" \ -e RUSTFS_SECRET_KEY="$HOT_SECRET_KEY" \ -e RUSTFS_VOLUMES="$(hot_volumes)" \ -e RUSTFS_SCANNER_ENABLED=false \ + -e RUSTFS_SCANNER_CYCLE=3600 \ + -e RUSTFS_SCANNER_START_DELAY_SECS=3600 \ + -e RUSTFS_MAX_TRANSITION_WORKERS="$TRANSITION_WORKERS" \ + -e RUSTFS_TRANSITION_QUEUE_CAPACITY="$TRANSITION_QUEUE_CAPACITY" \ + -e RUSTFS_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT="$FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT" \ -e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \ -e RUSTFS_OBS_LOGGER_LEVEL=warn \ -v "${PROJECT_NAME}_node2_data_0:/data/rustfs0" \ @@ -473,6 +544,37 @@ replace_node2_with_old_image() { "$OLD_IMAGE" >/dev/null } +replace_node1_with_old_image() { + local network + network="$(network_name)" + log_info "Replacing node1 with old image ${OLD_IMAGE} before manual transition job" + compose stop node1 >/dev/null + compose rm -f node1 >/dev/null + docker run -d \ + --name "${PROJECT_NAME}-node1-before-job-old" \ + --network "$network" \ + --network-alias node1 \ + --user 0:0 \ + -p "$((BASE_PORT + 1)):9000" \ + -e RUSTFS_ADDRESS=:9000 \ + -e RUSTFS_ACCESS_KEY="$HOT_ACCESS_KEY" \ + -e RUSTFS_SECRET_KEY="$HOT_SECRET_KEY" \ + -e RUSTFS_VOLUMES="$(hot_volumes)" \ + -e RUSTFS_SCANNER_ENABLED=false \ + -e RUSTFS_SCANNER_CYCLE=3600 \ + -e RUSTFS_SCANNER_START_DELAY_SECS=3600 \ + -e RUSTFS_MAX_TRANSITION_WORKERS="$TRANSITION_WORKERS" \ + -e RUSTFS_TRANSITION_QUEUE_CAPACITY="$TRANSITION_QUEUE_CAPACITY" \ + -e RUSTFS_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT="$FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT" \ + -e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \ + -e RUSTFS_OBS_LOGGER_LEVEL=warn \ + -v "${PROJECT_NAME}_node1_data_0:/data/rustfs0" \ + -v "${PROJECT_NAME}_node1_data_1:/data/rustfs1" \ + -v "${PROJECT_NAME}_node1_data_2:/data/rustfs2" \ + -v "${PROJECT_NAME}_node1_data_3:/data/rustfs3" \ + "$OLD_IMAGE" >/dev/null +} + poll_terminal_status() { local job_id="$1" local status_url status_json terminal_state @@ -512,6 +614,85 @@ head_probe() { -w "%{http_code}\n" >"${OUT_DIR}/head-object.http_code" || true } +image_id() { + local file="$1" + jq -r '.[0].Id // ""' "$file" 2>/dev/null || true +} + +is_compat_readback_code() { + case "$1" in + 200|501) + return 0 + ;; + *) + return 1 + ;; + esac +} + +classify_result() { + local terminal_state="$1" + local transition_completed="$2" + local transition_failed="$3" + local tier_failure="$4" + local old_code="$5" + local rollback_code="$6" + local run_http_code="$7" + local lifecycle_config_found="$8" + local scanned="$9" + local eligible="${10}" + local skipped_already_transitioned="${11}" + local skipped_already_in_flight="${12}" + local old_image_id new_image_id images_are_mixed readback_ok + + if [[ -f "${OUT_DIR}/readiness-failed" ]]; then + printf 'blocked_cluster_readiness_failed\n' + return + fi + + old_image_id="$(image_id "${OUT_DIR}/old-image.inspect.json")" + new_image_id="$(image_id "${OUT_DIR}/new-image.inspect.json")" + images_are_mixed=false + if [[ "$OLD_IMAGE" != "$NEW_IMAGE" && -n "$old_image_id" && -n "$new_image_id" && "$old_image_id" != "$new_image_id" ]]; then + images_are_mixed=true + fi + + readback_ok=false + if is_compat_readback_code "$old_code"; then + if [[ "$ROLLBACK_PHASE" == "none" ]] || is_compat_readback_code "$rollback_code"; then + readback_ok=true + fi + fi + + if [[ "$run_http_code" == "501" ]]; then + printf 'blocked_manual_api_not_implemented\n' + return + fi + if [[ -z "$(cat "${OUT_DIR}/job-id.txt" 2>/dev/null || true)" ]]; then + printf 'blocked_manual_api_unavailable\n' + return + fi + if [[ "$terminal_state" == "completed" && ( "$lifecycle_config_found" != "true" || "$scanned" == "0" || "$eligible" == "0" || "$transition_completed" == "0" ) ]]; then + printf 'blocked_empty_scan_or_lifecycle\n' + return + fi + if [[ "$transition_completed" == "0" && "$eligible" != "0" && "$transition_failed" == "0" && "$tier_failure" == "0" ]]; then + if [[ "$skipped_already_transitioned" != "0" || "$skipped_already_in_flight" != "0" ]]; then + printf 'blocked_manual_job_preempted_by_lifecycle_queue\n' + return + fi + fi + if [[ "$terminal_state" == "completed" && "$transition_completed" != "0" && "$transition_failed" == "0" && "$tier_failure" == "0" ]]; then + if [[ "$images_are_mixed" == "true" && "$readback_ok" == "true" ]]; then + printf 'strict_mixed_rollout_pass\n' + return + fi + printf 'baseline_tiered_storage_pass\n' + return + fi + printf 'strict_mixed_rollout_fail\n' +} + collect_logs() { local service if [[ -z "$COMPOSE_FILE" || ! -f "$COMPOSE_FILE" ]]; then @@ -520,37 +701,63 @@ collect_logs() { for service in cold node1 node2 node3 node4; do compose logs --no-color "$service" >"${OUT_DIR}/${service}.log" 2>/dev/null || true done + docker logs "${PROJECT_NAME}-node1-before-job-old" >"${OUT_DIR}/node1-before-job-old.log" 2>/dev/null || true docker logs "${PROJECT_NAME}-node2-rollback-old" >"${OUT_DIR}/node2-rollback-old.log" 2>/dev/null || true } summarize() { - local terminal_state transition_completed tier_failure transition_failed old_code rollback_code + local terminal_state transition_completed tier_failure transition_failed old_code rollback_code run_http_code lifecycle_config_found scanned eligible skipped_already_transitioned skipped_already_in_flight queue_queued queue_active result_classification terminal_state="$(cat "${OUT_DIR}/terminal-state.txt" 2>/dev/null || true)" transition_completed="$(jq -r '.report.transition_completed // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')" tier_failure="$(jq -r '.report.tier_failure // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')" transition_failed="$(jq -r '.report.transition_failed // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')" old_code="$(cat "${OUT_DIR}/old-node-status.http_code" 2>/dev/null || true)" rollback_code="$(cat "${OUT_DIR}/rollback-node2-status.http_code" 2>/dev/null || true)" + run_http_code="$(cat "${OUT_DIR}/run-response.http_code" 2>/dev/null || true)" + lifecycle_config_found="$(jq -r '.report.lifecycle_config_found // false' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf 'false')" + scanned="$(jq -r '.report.scanned // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')" + eligible="$(jq -r '.report.eligible // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')" + skipped_already_transitioned="$(jq -r '.report.skipped_already_transitioned // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')" + skipped_already_in_flight="$(jq -r '.report.skipped_already_in_flight // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')" + queue_queued="$(jq -r '.queue_snapshot.queued // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')" + queue_active="$(jq -r '.queue_snapshot.active // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')" + result_classification="$(classify_result "$terminal_state" "$transition_completed" "$transition_failed" "$tier_failure" "$old_code" "$rollback_code" "$run_http_code" "$lifecycle_config_found" "$scanned" "$eligible" "$skipped_already_transitioned" "$skipped_already_in_flight")" cat >"${OUT_DIR}/summary.env" <"${OUT_DIR}/cold-image.inspect.json" compose up -d - wait_cluster_ready + if ! wait_cluster_ready; then + printf 'cluster readiness failed before manual transition admission\n' >"${OUT_DIR}/readiness-failed" + collect_logs + summarize + return 1 + fi curl_cold PUT "$(cold_endpoint)/${TIER_BUCKET}" -o "${OUT_DIR}/create-cold-bucket.response" -w "%{http_code}\n" >"${OUT_DIR}/create-cold-bucket.http_code" create_bucket "$(hot_endpoint 2)" "$JOB_BUCKET" "${OUT_DIR}/create-hot-bucket" add_tier - put_lifecycle seed_objects - start_transition_job - if [[ "$ROLLBACK_NEW2_TO_OLD" == "true" ]]; then - replace_node2_with_old_image + put_lifecycle + if [[ "$OLD_NODE_PHASE" == "before-job" ]]; then + replace_node1_with_old_image + wait_http_ok "$(hot_endpoint 1)/health" "node1-old-live" || true + wait_http_ok "$(hot_endpoint 1)/health/ready" "node1-old-ready" || true + fi + if start_transition_job; then + if [[ "$ROLLBACK_PHASE" == "in-flight" ]]; then + replace_node2_with_old_image + fi + poll_terminal_status "$(cat "${OUT_DIR}/job-id.txt")" || true + if [[ "$ROLLBACK_PHASE" == "after-terminal" ]]; then + replace_node2_with_old_image + wait_http_ok "$(hot_endpoint 2)/health" "rollback-node2-live" || true + fi + capture_old_node_readback "$(cat "${OUT_DIR}/job-id.txt")" + head_probe + else + log_warn "Skipping terminal polling because no manual transition job was admitted" fi - poll_terminal_status "$(cat "${OUT_DIR}/job-id.txt")" - capture_old_node_readback "$(cat "${OUT_DIR}/job-id.txt")" - head_probe collect_logs summarize } diff --git a/scripts/test_manual_transition_runbooks.sh b/scripts/test_manual_transition_runbooks.sh index 248a72ef3..9ad09909a 100755 --- a/scripts/test_manual_transition_runbooks.sh +++ b/scripts/test_manual_transition_runbooks.sh @@ -208,7 +208,16 @@ bash "$MIXED_DOCKER_HARNESS" --help >/tmp/manual_transition_mixed_version_docker rg -q "mixed_version_docker_harness" /tmp/manual_transition_mixed_version_docker_harness.help rg -q -- "--old-image" /tmp/manual_transition_mixed_version_docker_harness.help rg -q -- "--new-image" /tmp/manual_transition_mixed_version_docker_harness.help +rg -q -- "--rollback-phase" /tmp/manual_transition_mixed_version_docker_harness.help +rg -q -- "--old-node-phase" /tmp/manual_transition_mixed_version_docker_harness.help rg -q -- "--no-rollback" /tmp/manual_transition_mixed_version_docker_harness.help +rg -q "strict_mixed_rollout_pass" /tmp/manual_transition_mixed_version_docker_harness.help +rg -q "baseline_tiered_storage_pass" /tmp/manual_transition_mixed_version_docker_harness.help +rg -q "blocked_manual_api_not_implemented" /tmp/manual_transition_mixed_version_docker_harness.help +rg -q "blocked_cluster_readiness_failed" /tmp/manual_transition_mixed_version_docker_harness.help +rg -q "blocked_manual_job_preempted_by_lifecycle_queue" /tmp/manual_transition_mixed_version_docker_harness.help +rg -q "OLD_NODE_PHASE" /tmp/manual_transition_mixed_version_docker_harness.help +rg -q "FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT" /tmp/manual_transition_mixed_version_docker_harness.help if bash "$FAILURE_SAMPLES" --endpoint http://127.0.0.1:9000 --sample >/tmp/manual_transition_failure_samples.err 2>&1; then echo "failure samples script should fail when --sample has no value" >&2 exit 1