mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-29 09:38:59 +00:00
test(lifecycle): classify mixed rollout harness results (#5384)
* test(lifecycle): classify mixed rollout harness results Classify Docker #1508 evidence as strict, baseline, blocked, or failed so tiered-storage baseline runs cannot be mistaken for strict mixed-version rollout closure evidence. Co-Authored-By: heihutu <heihutu@gmail.com> * test: classify Docker manual transition preemption Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
+1
-1
@@ -90,7 +90,7 @@ their issue closes.
|
||||
| `manual_transition_journal_audit.sh` | dev-tool | Journal + metrics + log audit for manual transition jobs | — |
|
||||
| `manual_transition_mixed_rollout_matrix.sh` | dev-tool | Matrix generator for mixed-version rollout phases | — |
|
||||
| `manual_transition_mixed_rollout_runbook.sh` | dev-tool | Reusable mixed-version rollout runbook generator (external run) | — |
|
||||
| `manual_transition_mixed_version_docker_harness.sh` | dev-tool | Dedicated #1508 Docker harness for old/new manual-transition rollout evidence | `test_manual_transition_runbooks.sh` |
|
||||
| `manual_transition_mixed_version_docker_harness.sh` | dev-tool | Dedicated #1508 Docker harness for old/new manual-transition rollout evidence with strict/baseline/blocked result classification | `test_manual_transition_runbooks.sh` |
|
||||
| `monitor_manual_transition_ci.sh` | dev-tool | CI workflow/status watcher for manual transition follow-up monitoring | — |
|
||||
| `manual_transition_soak_matrix.sh` | dev-tool | Matrix generator for nightly stress windows | — |
|
||||
| `manual_transition_nightly_stress_runbook.sh` | dev-tool | Nightly stress entrypoint with failure snapshot templates | — |
|
||||
|
||||
@@ -25,9 +25,13 @@ COLD_IMAGE="${COLD_IMAGE:-${NEW_IMAGE}}"
|
||||
BASE_PORT="${BASE_PORT:-19400}"
|
||||
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/manual-transition-1508-docker/$(date +%Y%m%dT%H%M%S)}"
|
||||
KEEP_UP=false
|
||||
ROLLBACK_NEW2_TO_OLD=true
|
||||
ROLLBACK_PHASE="${ROLLBACK_PHASE:-after-terminal}"
|
||||
OLD_NODE_PHASE="${OLD_NODE_PHASE:-initial}"
|
||||
WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-180}"
|
||||
POLL_SECONDS="${POLL_SECONDS:-180}"
|
||||
TRANSITION_WORKERS="${TRANSITION_WORKERS:-2}"
|
||||
TRANSITION_QUEUE_CAPACITY="${TRANSITION_QUEUE_CAPACITY:-64}"
|
||||
FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT="${FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT:-false}"
|
||||
|
||||
HOT_ACCESS_KEY="${HOT_ACCESS_KEY:-mvadmin}"
|
||||
HOT_SECRET_KEY="${HOT_SECRET_KEY:-mvsecret}"
|
||||
@@ -58,18 +62,31 @@ Options:
|
||||
--object-count <n> Non-empty probe object count
|
||||
--tier <name> Remote tier name
|
||||
--keep-up Leave Docker services running
|
||||
--no-rollback Do not replace node2 with old image after job admission
|
||||
--rollback-phase <phase> Rollback node2 timing: after-terminal, in-flight, none
|
||||
--old-node-phase <phase> Old node1 timing: initial, before-job
|
||||
--no-rollback Alias for --rollback-phase none
|
||||
-h, --help Show help
|
||||
|
||||
Environment:
|
||||
PROJECT_NAME OLD_IMAGE NEW_IMAGE COLD_IMAGE BASE_PORT OUT_DIR KEEP_UP
|
||||
HOT_ACCESS_KEY HOT_SECRET_KEY COLD_ACCESS_KEY COLD_SECRET_KEY
|
||||
TIER_NAME TIER_BUCKET TIER_PREFIX JOB_BUCKET JOB_PREFIX OBJECT_COUNT
|
||||
WAIT_TIMEOUT_SECS POLL_SECONDS AWS_SIGV4_SCOPE
|
||||
ROLLBACK_PHASE WAIT_TIMEOUT_SECS POLL_SECONDS TRANSITION_WORKERS
|
||||
TRANSITION_QUEUE_CAPACITY OLD_NODE_PHASE FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT AWS_SIGV4_SCOPE
|
||||
|
||||
Artifacts:
|
||||
compose.yml, image inspect files, health/readiness logs, API responses,
|
||||
terminal status, old-node readback, container logs, summary.env.
|
||||
|
||||
Result classifications:
|
||||
strict_mixed_rollout_pass Real old/new images, non-empty completed transition, zero failures
|
||||
baseline_tiered_storage_pass Same old/new image completed transition; useful baseline, not #1508 closure
|
||||
blocked_manual_api_not_implemented Manual transition API returned 501 before job admission
|
||||
blocked_manual_api_unavailable Manual transition API did not return a usable job_id
|
||||
blocked_cluster_readiness_failed Docker cluster did not reach health/readiness before admission
|
||||
blocked_empty_scan_or_lifecycle Job completed without lifecycle-matching transition work
|
||||
blocked_manual_job_preempted_by_lifecycle_queue Lifecycle/immediate transition queued work before the job
|
||||
strict_mixed_rollout_fail Mixed rollout ran but did not satisfy the strict #1508 gate
|
||||
USAGE
|
||||
}
|
||||
|
||||
@@ -111,6 +128,28 @@ parse_positive_int() {
|
||||
fi
|
||||
}
|
||||
|
||||
validate_rollback_phase() {
|
||||
case "$ROLLBACK_PHASE" in
|
||||
after-terminal|in-flight|none)
|
||||
;;
|
||||
*)
|
||||
log_error "--rollback-phase must be one of: after-terminal, in-flight, none"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
validate_old_node_phase() {
|
||||
case "$OLD_NODE_PHASE" in
|
||||
initial|before-job)
|
||||
;;
|
||||
*)
|
||||
log_error "--old-node-phase must be one of: initial, before-job"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
@@ -150,8 +189,16 @@ parse_args() {
|
||||
KEEP_UP=true
|
||||
shift
|
||||
;;
|
||||
--rollback-phase)
|
||||
ROLLBACK_PHASE="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--old-node-phase)
|
||||
OLD_NODE_PHASE="$(arg_value "$1" "${2:-}")"
|
||||
shift 2
|
||||
;;
|
||||
--no-rollback)
|
||||
ROLLBACK_NEW2_TO_OLD=false
|
||||
ROLLBACK_PHASE=none
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
@@ -198,12 +245,18 @@ cleanup() {
|
||||
}
|
||||
|
||||
write_compose_file() {
|
||||
local node1_image
|
||||
COMPOSE_FILE="${OUT_DIR}/compose.yml"
|
||||
node1_image="$OLD_IMAGE"
|
||||
if [[ "$OLD_NODE_PHASE" == "before-job" ]]; then
|
||||
node1_image="$NEW_IMAGE"
|
||||
fi
|
||||
cat >"$COMPOSE_FILE" <<EOF
|
||||
services:
|
||||
cold:
|
||||
image: ${COLD_IMAGE}
|
||||
hostname: cold
|
||||
user: "0:0"
|
||||
environment:
|
||||
- RUSTFS_ADDRESS=:9000
|
||||
- RUSTFS_ACCESS_KEY=${COLD_ACCESS_KEY}
|
||||
@@ -222,14 +275,20 @@ services:
|
||||
- rustfs-1508-net
|
||||
|
||||
node1:
|
||||
image: ${OLD_IMAGE}
|
||||
image: ${node1_image}
|
||||
hostname: node1
|
||||
user: "0:0"
|
||||
environment: &hot-env
|
||||
- RUSTFS_ADDRESS=:9000
|
||||
- RUSTFS_ACCESS_KEY=${HOT_ACCESS_KEY}
|
||||
- RUSTFS_SECRET_KEY=${HOT_SECRET_KEY}
|
||||
- RUSTFS_VOLUMES=$(hot_volumes)
|
||||
- RUSTFS_SCANNER_ENABLED=false
|
||||
- RUSTFS_SCANNER_CYCLE=3600
|
||||
- RUSTFS_SCANNER_START_DELAY_SECS=3600
|
||||
- RUSTFS_MAX_TRANSITION_WORKERS=${TRANSITION_WORKERS}
|
||||
- RUSTFS_TRANSITION_QUEUE_CAPACITY=${TRANSITION_QUEUE_CAPACITY}
|
||||
- RUSTFS_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT=${FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=warn
|
||||
volumes:
|
||||
@@ -245,6 +304,7 @@ services:
|
||||
node2:
|
||||
image: ${NEW_IMAGE}
|
||||
hostname: node2
|
||||
user: "0:0"
|
||||
environment: *hot-env
|
||||
volumes:
|
||||
- node2_data_0:/data/rustfs0
|
||||
@@ -259,6 +319,7 @@ services:
|
||||
node3:
|
||||
image: ${NEW_IMAGE}
|
||||
hostname: node3
|
||||
user: "0:0"
|
||||
environment: *hot-env
|
||||
volumes:
|
||||
- node3_data_0:/data/rustfs0
|
||||
@@ -273,6 +334,7 @@ services:
|
||||
node4:
|
||||
image: ${NEW_IMAGE}
|
||||
hostname: node4
|
||||
user: "0:0"
|
||||
environment: *hot-env
|
||||
volumes:
|
||||
- node4_data_0:/data/rustfs0
|
||||
@@ -433,16 +495,19 @@ seed_objects() {
|
||||
}
|
||||
|
||||
start_transition_job() {
|
||||
local query response job_id
|
||||
local query response job_id run_http_code
|
||||
query="bucket=$(url_encode "$JOB_BUCKET")"
|
||||
query="${query}&prefix=$(url_encode "${JOB_PREFIX}/")"
|
||||
query="${query}&tier=$(url_encode "$TIER_NAME")"
|
||||
query="${query}&dryRun=false&maxObjects=${OBJECT_COUNT}&mode=async"
|
||||
response="$(curl_hot POST "$(hot_endpoint 2)/rustfs/admin/v3/ilm/transition/run?${query}")"
|
||||
printf '%s\n' "$response" >"${OUT_DIR}/run-response.json"
|
||||
curl_hot POST "$(hot_endpoint 2)/rustfs/admin/v3/ilm/transition/run?${query}" \
|
||||
-o "${OUT_DIR}/run-response.json" \
|
||||
-w "%{http_code}\n" >"${OUT_DIR}/run-response.http_code" || true
|
||||
response="$(cat "${OUT_DIR}/run-response.json" 2>/dev/null || true)"
|
||||
run_http_code="$(cat "${OUT_DIR}/run-response.http_code" 2>/dev/null || true)"
|
||||
job_id="$(printf '%s' "$response" | jq -r '.job_id // empty')"
|
||||
if [[ -z "$job_id" ]]; then
|
||||
log_error "manual transition response omitted job_id"
|
||||
log_warn "manual transition response omitted job_id, http_code=${run_http_code:-unknown}"
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$job_id" >"${OUT_DIR}/job-id.txt"
|
||||
@@ -451,19 +516,25 @@ start_transition_job() {
|
||||
replace_node2_with_old_image() {
|
||||
local network
|
||||
network="$(network_name)"
|
||||
log_info "Replacing node2 with old image ${OLD_IMAGE} for in-flight rollback readback"
|
||||
log_info "Replacing node2 with old image ${OLD_IMAGE} for ${ROLLBACK_PHASE} rollback readback"
|
||||
compose stop node2 >/dev/null
|
||||
compose rm -f node2 >/dev/null
|
||||
docker run -d \
|
||||
--name "${PROJECT_NAME}-node2-rollback-old" \
|
||||
--network "$network" \
|
||||
--network-alias node2 \
|
||||
--user 0:0 \
|
||||
-p "$((BASE_PORT + 2)):9000" \
|
||||
-e RUSTFS_ADDRESS=:9000 \
|
||||
-e RUSTFS_ACCESS_KEY="$HOT_ACCESS_KEY" \
|
||||
-e RUSTFS_SECRET_KEY="$HOT_SECRET_KEY" \
|
||||
-e RUSTFS_VOLUMES="$(hot_volumes)" \
|
||||
-e RUSTFS_SCANNER_ENABLED=false \
|
||||
-e RUSTFS_SCANNER_CYCLE=3600 \
|
||||
-e RUSTFS_SCANNER_START_DELAY_SECS=3600 \
|
||||
-e RUSTFS_MAX_TRANSITION_WORKERS="$TRANSITION_WORKERS" \
|
||||
-e RUSTFS_TRANSITION_QUEUE_CAPACITY="$TRANSITION_QUEUE_CAPACITY" \
|
||||
-e RUSTFS_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT="$FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT" \
|
||||
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
||||
-e RUSTFS_OBS_LOGGER_LEVEL=warn \
|
||||
-v "${PROJECT_NAME}_node2_data_0:/data/rustfs0" \
|
||||
@@ -473,6 +544,37 @@ replace_node2_with_old_image() {
|
||||
"$OLD_IMAGE" >/dev/null
|
||||
}
|
||||
|
||||
replace_node1_with_old_image() {
|
||||
local network
|
||||
network="$(network_name)"
|
||||
log_info "Replacing node1 with old image ${OLD_IMAGE} before manual transition job"
|
||||
compose stop node1 >/dev/null
|
||||
compose rm -f node1 >/dev/null
|
||||
docker run -d \
|
||||
--name "${PROJECT_NAME}-node1-before-job-old" \
|
||||
--network "$network" \
|
||||
--network-alias node1 \
|
||||
--user 0:0 \
|
||||
-p "$((BASE_PORT + 1)):9000" \
|
||||
-e RUSTFS_ADDRESS=:9000 \
|
||||
-e RUSTFS_ACCESS_KEY="$HOT_ACCESS_KEY" \
|
||||
-e RUSTFS_SECRET_KEY="$HOT_SECRET_KEY" \
|
||||
-e RUSTFS_VOLUMES="$(hot_volumes)" \
|
||||
-e RUSTFS_SCANNER_ENABLED=false \
|
||||
-e RUSTFS_SCANNER_CYCLE=3600 \
|
||||
-e RUSTFS_SCANNER_START_DELAY_SECS=3600 \
|
||||
-e RUSTFS_MAX_TRANSITION_WORKERS="$TRANSITION_WORKERS" \
|
||||
-e RUSTFS_TRANSITION_QUEUE_CAPACITY="$TRANSITION_QUEUE_CAPACITY" \
|
||||
-e RUSTFS_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT="$FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT" \
|
||||
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
||||
-e RUSTFS_OBS_LOGGER_LEVEL=warn \
|
||||
-v "${PROJECT_NAME}_node1_data_0:/data/rustfs0" \
|
||||
-v "${PROJECT_NAME}_node1_data_1:/data/rustfs1" \
|
||||
-v "${PROJECT_NAME}_node1_data_2:/data/rustfs2" \
|
||||
-v "${PROJECT_NAME}_node1_data_3:/data/rustfs3" \
|
||||
"$OLD_IMAGE" >/dev/null
|
||||
}
|
||||
|
||||
poll_terminal_status() {
|
||||
local job_id="$1"
|
||||
local status_url status_json terminal_state
|
||||
@@ -512,6 +614,85 @@ head_probe() {
|
||||
-w "%{http_code}\n" >"${OUT_DIR}/head-object.http_code" || true
|
||||
}
|
||||
|
||||
image_id() {
|
||||
local file="$1"
|
||||
jq -r '.[0].Id // ""' "$file" 2>/dev/null || true
|
||||
}
|
||||
|
||||
is_compat_readback_code() {
|
||||
case "$1" in
|
||||
200|501)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
classify_result() {
|
||||
local terminal_state="$1"
|
||||
local transition_completed="$2"
|
||||
local transition_failed="$3"
|
||||
local tier_failure="$4"
|
||||
local old_code="$5"
|
||||
local rollback_code="$6"
|
||||
local run_http_code="$7"
|
||||
local lifecycle_config_found="$8"
|
||||
local scanned="$9"
|
||||
local eligible="${10}"
|
||||
local skipped_already_transitioned="${11}"
|
||||
local skipped_already_in_flight="${12}"
|
||||
local old_image_id new_image_id images_are_mixed readback_ok
|
||||
|
||||
if [[ -f "${OUT_DIR}/readiness-failed" ]]; then
|
||||
printf 'blocked_cluster_readiness_failed\n'
|
||||
return
|
||||
fi
|
||||
|
||||
old_image_id="$(image_id "${OUT_DIR}/old-image.inspect.json")"
|
||||
new_image_id="$(image_id "${OUT_DIR}/new-image.inspect.json")"
|
||||
images_are_mixed=false
|
||||
if [[ "$OLD_IMAGE" != "$NEW_IMAGE" && -n "$old_image_id" && -n "$new_image_id" && "$old_image_id" != "$new_image_id" ]]; then
|
||||
images_are_mixed=true
|
||||
fi
|
||||
|
||||
readback_ok=false
|
||||
if is_compat_readback_code "$old_code"; then
|
||||
if [[ "$ROLLBACK_PHASE" == "none" ]] || is_compat_readback_code "$rollback_code"; then
|
||||
readback_ok=true
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$run_http_code" == "501" ]]; then
|
||||
printf 'blocked_manual_api_not_implemented\n'
|
||||
return
|
||||
fi
|
||||
if [[ -z "$(cat "${OUT_DIR}/job-id.txt" 2>/dev/null || true)" ]]; then
|
||||
printf 'blocked_manual_api_unavailable\n'
|
||||
return
|
||||
fi
|
||||
if [[ "$terminal_state" == "completed" && ( "$lifecycle_config_found" != "true" || "$scanned" == "0" || "$eligible" == "0" || "$transition_completed" == "0" ) ]]; then
|
||||
printf 'blocked_empty_scan_or_lifecycle\n'
|
||||
return
|
||||
fi
|
||||
if [[ "$transition_completed" == "0" && "$eligible" != "0" && "$transition_failed" == "0" && "$tier_failure" == "0" ]]; then
|
||||
if [[ "$skipped_already_transitioned" != "0" || "$skipped_already_in_flight" != "0" ]]; then
|
||||
printf 'blocked_manual_job_preempted_by_lifecycle_queue\n'
|
||||
return
|
||||
fi
|
||||
fi
|
||||
if [[ "$terminal_state" == "completed" && "$transition_completed" != "0" && "$transition_failed" == "0" && "$tier_failure" == "0" ]]; then
|
||||
if [[ "$images_are_mixed" == "true" && "$readback_ok" == "true" ]]; then
|
||||
printf 'strict_mixed_rollout_pass\n'
|
||||
return
|
||||
fi
|
||||
printf 'baseline_tiered_storage_pass\n'
|
||||
return
|
||||
fi
|
||||
printf 'strict_mixed_rollout_fail\n'
|
||||
}
|
||||
|
||||
collect_logs() {
|
||||
local service
|
||||
if [[ -z "$COMPOSE_FILE" || ! -f "$COMPOSE_FILE" ]]; then
|
||||
@@ -520,37 +701,63 @@ collect_logs() {
|
||||
for service in cold node1 node2 node3 node4; do
|
||||
compose logs --no-color "$service" >"${OUT_DIR}/${service}.log" 2>/dev/null || true
|
||||
done
|
||||
docker logs "${PROJECT_NAME}-node1-before-job-old" >"${OUT_DIR}/node1-before-job-old.log" 2>/dev/null || true
|
||||
docker logs "${PROJECT_NAME}-node2-rollback-old" >"${OUT_DIR}/node2-rollback-old.log" 2>/dev/null || true
|
||||
}
|
||||
|
||||
summarize() {
|
||||
local terminal_state transition_completed tier_failure transition_failed old_code rollback_code
|
||||
local terminal_state transition_completed tier_failure transition_failed old_code rollback_code run_http_code lifecycle_config_found scanned eligible skipped_already_transitioned skipped_already_in_flight queue_queued queue_active result_classification
|
||||
terminal_state="$(cat "${OUT_DIR}/terminal-state.txt" 2>/dev/null || true)"
|
||||
transition_completed="$(jq -r '.report.transition_completed // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
tier_failure="$(jq -r '.report.tier_failure // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
transition_failed="$(jq -r '.report.transition_failed // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
old_code="$(cat "${OUT_DIR}/old-node-status.http_code" 2>/dev/null || true)"
|
||||
rollback_code="$(cat "${OUT_DIR}/rollback-node2-status.http_code" 2>/dev/null || true)"
|
||||
run_http_code="$(cat "${OUT_DIR}/run-response.http_code" 2>/dev/null || true)"
|
||||
lifecycle_config_found="$(jq -r '.report.lifecycle_config_found // false' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf 'false')"
|
||||
scanned="$(jq -r '.report.scanned // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
eligible="$(jq -r '.report.eligible // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
skipped_already_transitioned="$(jq -r '.report.skipped_already_transitioned // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
skipped_already_in_flight="$(jq -r '.report.skipped_already_in_flight // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
queue_queued="$(jq -r '.queue_snapshot.queued // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
queue_active="$(jq -r '.queue_snapshot.active // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
|
||||
result_classification="$(classify_result "$terminal_state" "$transition_completed" "$transition_failed" "$tier_failure" "$old_code" "$rollback_code" "$run_http_code" "$lifecycle_config_found" "$scanned" "$eligible" "$skipped_already_transitioned" "$skipped_already_in_flight")"
|
||||
cat >"${OUT_DIR}/summary.env" <<EOF
|
||||
project_name=${PROJECT_NAME}
|
||||
old_image=${OLD_IMAGE}
|
||||
new_image=${NEW_IMAGE}
|
||||
cold_image=${COLD_IMAGE}
|
||||
old_image_id=$(image_id "${OUT_DIR}/old-image.inspect.json")
|
||||
new_image_id=$(image_id "${OUT_DIR}/new-image.inspect.json")
|
||||
base_port=${BASE_PORT}
|
||||
tier=${TIER_NAME}
|
||||
job_bucket=${JOB_BUCKET}
|
||||
job_prefix=${JOB_PREFIX}
|
||||
object_count=${OBJECT_COUNT}
|
||||
transition_workers=${TRANSITION_WORKERS}
|
||||
transition_queue_capacity=${TRANSITION_QUEUE_CAPACITY}
|
||||
force_immediate_transition_enqueue_timeout=${FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT}
|
||||
run_http_code=${run_http_code}
|
||||
terminal_state=${terminal_state}
|
||||
lifecycle_config_found=${lifecycle_config_found}
|
||||
scanned=${scanned}
|
||||
eligible=${eligible}
|
||||
skipped_already_transitioned=${skipped_already_transitioned}
|
||||
skipped_already_in_flight=${skipped_already_in_flight}
|
||||
transition_completed=${transition_completed}
|
||||
transition_failed=${transition_failed}
|
||||
tier_failure=${tier_failure}
|
||||
queue_queued=${queue_queued}
|
||||
queue_active=${queue_active}
|
||||
old_node_status_http_code=${old_code}
|
||||
rollback_node2_status_http_code=${rollback_code}
|
||||
rollback_new2_to_old=${ROLLBACK_NEW2_TO_OLD}
|
||||
rollback_phase=${ROLLBACK_PHASE}
|
||||
old_node_phase=${OLD_NODE_PHASE}
|
||||
rollback_new2_to_old=$([[ "$ROLLBACK_PHASE" == "none" ]] && printf 'false' || printf 'true')
|
||||
result_classification=${result_classification}
|
||||
EOF
|
||||
cat "${OUT_DIR}/summary.env"
|
||||
if [[ "$terminal_state" != "completed" || "$transition_completed" == "0" || "$transition_failed" != "0" || "$tier_failure" != "0" ]]; then
|
||||
if [[ "$result_classification" != "strict_mixed_rollout_pass" ]]; then
|
||||
log_error "strict #1508 mixed-version transition gate failed; see ${OUT_DIR}"
|
||||
return 1
|
||||
fi
|
||||
@@ -560,6 +767,8 @@ main() {
|
||||
parse_args "$@"
|
||||
parse_positive_int "--base-port" "$BASE_PORT"
|
||||
parse_positive_int "--object-count" "$OBJECT_COUNT"
|
||||
validate_rollback_phase
|
||||
validate_old_node_phase
|
||||
require_cmd docker
|
||||
require_cmd curl
|
||||
require_cmd jq
|
||||
@@ -574,20 +783,37 @@ main() {
|
||||
docker image inspect "$COLD_IMAGE" >"${OUT_DIR}/cold-image.inspect.json"
|
||||
|
||||
compose up -d
|
||||
wait_cluster_ready
|
||||
if ! wait_cluster_ready; then
|
||||
printf 'cluster readiness failed before manual transition admission\n' >"${OUT_DIR}/readiness-failed"
|
||||
collect_logs
|
||||
summarize
|
||||
return 1
|
||||
fi
|
||||
|
||||
curl_cold PUT "$(cold_endpoint)/${TIER_BUCKET}" -o "${OUT_DIR}/create-cold-bucket.response" -w "%{http_code}\n" >"${OUT_DIR}/create-cold-bucket.http_code"
|
||||
create_bucket "$(hot_endpoint 2)" "$JOB_BUCKET" "${OUT_DIR}/create-hot-bucket"
|
||||
add_tier
|
||||
put_lifecycle
|
||||
seed_objects
|
||||
start_transition_job
|
||||
if [[ "$ROLLBACK_NEW2_TO_OLD" == "true" ]]; then
|
||||
replace_node2_with_old_image
|
||||
put_lifecycle
|
||||
if [[ "$OLD_NODE_PHASE" == "before-job" ]]; then
|
||||
replace_node1_with_old_image
|
||||
wait_http_ok "$(hot_endpoint 1)/health" "node1-old-live" || true
|
||||
wait_http_ok "$(hot_endpoint 1)/health/ready" "node1-old-ready" || true
|
||||
fi
|
||||
if start_transition_job; then
|
||||
if [[ "$ROLLBACK_PHASE" == "in-flight" ]]; then
|
||||
replace_node2_with_old_image
|
||||
fi
|
||||
poll_terminal_status "$(cat "${OUT_DIR}/job-id.txt")" || true
|
||||
if [[ "$ROLLBACK_PHASE" == "after-terminal" ]]; then
|
||||
replace_node2_with_old_image
|
||||
wait_http_ok "$(hot_endpoint 2)/health" "rollback-node2-live" || true
|
||||
fi
|
||||
capture_old_node_readback "$(cat "${OUT_DIR}/job-id.txt")"
|
||||
head_probe
|
||||
else
|
||||
log_warn "Skipping terminal polling because no manual transition job was admitted"
|
||||
fi
|
||||
poll_terminal_status "$(cat "${OUT_DIR}/job-id.txt")"
|
||||
capture_old_node_readback "$(cat "${OUT_DIR}/job-id.txt")"
|
||||
head_probe
|
||||
collect_logs
|
||||
summarize
|
||||
}
|
||||
|
||||
@@ -208,7 +208,16 @@ bash "$MIXED_DOCKER_HARNESS" --help >/tmp/manual_transition_mixed_version_docker
|
||||
rg -q "mixed_version_docker_harness" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q -- "--old-image" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q -- "--new-image" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q -- "--rollback-phase" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q -- "--old-node-phase" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q -- "--no-rollback" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q "strict_mixed_rollout_pass" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q "baseline_tiered_storage_pass" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q "blocked_manual_api_not_implemented" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q "blocked_cluster_readiness_failed" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q "blocked_manual_job_preempted_by_lifecycle_queue" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q "OLD_NODE_PHASE" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
rg -q "FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT" /tmp/manual_transition_mixed_version_docker_harness.help
|
||||
if bash "$FAILURE_SAMPLES" --endpoint http://127.0.0.1:9000 --sample >/tmp/manual_transition_failure_samples.err 2>&1; then
|
||||
echo "failure samples script should fail when --sample has no value" >&2
|
||||
exit 1
|
||||
|
||||
Reference in New Issue
Block a user