test(lifecycle): add manual transition rollout harnesses (#5346)

* test(lifecycle): add manual transition rollout harnesses Add a dedicated #1508 ignored E2E harness for non-empty manual transition rollout readback and a reusable Docker mixed-version harness entrypoint. Co-Authored-By: heihutu <heihutu@gmail.com> * test(lifecycle): satisfy rollout harness clippy Use unwrap_or_default for the optional transition_failed report field while preserving the zero-default status readback behavior. Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
2026-07-28 17:18:58 +00:00 · 2026-07-28 08:04:47 +08:00
parent d5df66ac4f
commit d385cea7c6
4 changed files with 708 additions and 1 deletions
@@ -2167,6 +2167,111 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
    Ok(())
 }

+#[tokio::test]
+#[serial]
+#[ignore = "manual #1508 evidence harness: starts a 4-node cluster, a remote tier, and an in-flight transition job"]
+async fn four_node_manual_transition_rollout_non_empty_restart_readback() -> TestResult {
+    init_logging();
+
+    let mut cold = RustFSTestEnvironment::new().await?;
+    cold.access_key = "inline1508coldadmin".to_string();
+    cold.secret_key = "inline1508coldsecret".to_string();
+    cold.start_rustfs_server_without_cleanup(vec![]).await?;
+    let cold_client = cold.create_s3_client();
+    cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
+
+    let mut hot = RustFSTestClusterEnvironment::new(4).await?;
+    hot.set_env("RUSTFS_SCANNER_ENABLED", "false");
+    hot.set_env("RUSTFS_SCANNER_CYCLE", "3600");
+    hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "2");
+    hot.set_env("RUSTFS_TRANSITION_QUEUE_CAPACITY", "64");
+    hot.start().await?;
+    let hot_client = hot.create_s3_client(0)?;
+
+    let tier_name = unique_tier_name();
+    add_rustfs_tier(&hot, &cold, &tier_name).await?;
+    let bucket = format!("manual-transition-1508-{}", Uuid::new_v4().simple());
+    let prefix = "transition/manual-rollout/";
+    hot_client.create_bucket().bucket(&bucket).send().await?;
+    put_lifecycle_with_transition_retry(&hot_client, &bucket, &tier_name).await?;
+    for index in 0u8..24 {
+        let key = format!("{prefix}object-{index:02}.bin");
+        hot_client
+            .put_object()
+            .bucket(&bucket)
+            .key(key)
+            .body(ByteStream::from(payload(64 * KIB, index)))
+            .send()
+            .await?;
+    }
+
+    let (accepted_status, accepted_body) =
+        start_manual_transition_job_on_node(&hot, 1, &bucket, prefix, &tier_name, false, 24).await?;
+    assert_eq!(
+        accepted_status,
+        StatusCode::ACCEPTED,
+        "non-empty #1508 transition job should be accepted by a rollout node: {}",
+        compact_body(&accepted_body)
+    );
+    let accepted: serde_json::Value = serde_json::from_str(&accepted_body)?;
+    let job_id = accepted["job_id"]
+        .as_str()
+        .ok_or_else(|| format!("accepted #1508 response omitted job_id: {accepted_body}"))?;
+    let status_endpoint = accepted["status_endpoint"]
+        .as_str()
+        .ok_or_else(|| format!("accepted #1508 response omitted status_endpoint: {accepted_body}"))?;
+
+    let terminal = wait_for_manual_transition_job_terminal(&hot, 0, job_id, false).await?;
+    assert_eq!(terminal["job_id"].as_str(), Some(job_id));
+    assert_eq!(terminal["bucket"].as_str(), Some(bucket.as_str()));
+    assert_eq!(terminal["prefix"].as_str(), Some(prefix));
+    assert_eq!(terminal["dry_run"].as_bool(), Some(false));
+    assert_eq!(
+        terminal["status"].as_str(),
+        Some("completed"),
+        "non-empty #1508 rollout transition should complete before restart readback: {terminal}"
+    );
+    let transition_failed: u64 = terminal["report"]
+        .get("transition_failed")
+        .and_then(serde_json::Value::as_u64)
+        .unwrap_or_default();
+    assert_eq!(
+        transition_failed, 0,
+        "terminal #1508 report should not hide transition failures: {terminal}"
+    );
+    assert_eq!(
+        terminal["report"]["tier_failure"].as_u64(),
+        Some(0),
+        "terminal #1508 report should not hide tier failures: {terminal}"
+    );
+    let transition_completed = terminal["report"]["transition_completed"]
+        .as_u64()
+        .ok_or_else(|| format!("terminal #1508 report omitted transition_completed: {terminal}"))?;
+    assert!(
+        transition_completed > 0,
+        "terminal #1508 report should include real completed transitions: {terminal}"
+    );
+    assert!(
+        cold_tier_object_count(&cold_client).await? > 0,
+        "cold tier should contain transitioned #1508 objects"
+    );
+
+    hot.stop_node(2)?;
+    hot.start_node(2).await?;
+    let after_restart = read_manual_transition_job_status_endpoint(&hot, 2, status_endpoint).await?;
+    assert_eq!(after_restart["job_id"].as_str(), Some(job_id));
+    assert_eq!(
+        after_restart["status"], terminal["status"],
+        "terminal #1508 status changed after rollout node restart"
+    );
+    assert_eq!(
+        after_restart["report"]["transition_completed"], terminal["report"]["transition_completed"],
+        "terminal #1508 transition count changed after rollout node restart"
+    );
+
+    Ok(())
+}
+
 #[tokio::test]
 #[serial]
 async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_transition() -> TestResult {
@@ -90,6 +90,7 @@ their issue closes.
 | `manual_transition_journal_audit.sh` | dev-tool | Journal + metrics + log audit for manual transition jobs | — |
 | `manual_transition_mixed_rollout_matrix.sh` | dev-tool | Matrix generator for mixed-version rollout phases | — |
 | `manual_transition_mixed_rollout_runbook.sh` | dev-tool | Reusable mixed-version rollout runbook generator (external run) | — |
+| `manual_transition_mixed_version_docker_harness.sh` | dev-tool | Dedicated #1508 Docker harness for old/new manual-transition rollout evidence | `test_manual_transition_runbooks.sh` |
 | `monitor_manual_transition_ci.sh` | dev-tool | CI workflow/status watcher for manual transition follow-up monitoring | — |
 | `manual_transition_soak_matrix.sh` | dev-tool | Matrix generator for nightly stress windows | — |
 | `manual_transition_nightly_stress_runbook.sh` | dev-tool | Nightly stress entrypoint with failure snapshot templates | — |
@@ -0,0 +1,595 @@
+#!/usr/bin/env bash
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+PROJECT_NAME="${PROJECT_NAME:-rustfs-1508-mixed}"
+OLD_IMAGE="${OLD_IMAGE:-rustfs/rustfs:latest}"
+NEW_IMAGE="${NEW_IMAGE:-rustfs/rustfs:latest}"
+COLD_IMAGE="${COLD_IMAGE:-${NEW_IMAGE}}"
+BASE_PORT="${BASE_PORT:-19400}"
+OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/manual-transition-1508-docker/$(date +%Y%m%dT%H%M%S)}"
+KEEP_UP=false
+ROLLBACK_NEW2_TO_OLD=true
+WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-180}"
+POLL_SECONDS="${POLL_SECONDS:-180}"
+
+HOT_ACCESS_KEY="${HOT_ACCESS_KEY:-mvadmin}"
+HOT_SECRET_KEY="${HOT_SECRET_KEY:-mvsecret}"
+COLD_ACCESS_KEY="${COLD_ACCESS_KEY:-coldadmin}"
+COLD_SECRET_KEY="${COLD_SECRET_KEY:-coldsecret}"
+TIER_NAME="${TIER_NAME:-COLDMV}"
+TIER_BUCKET="${TIER_BUCKET:-manual-transition-cold-tier}"
+TIER_PREFIX="${TIER_PREFIX:-tiered}"
+JOB_BUCKET="${JOB_BUCKET:-manual-transition-1508}"
+JOB_PREFIX="${JOB_PREFIX:-transition/manual-rollout}"
+OBJECT_COUNT="${OBJECT_COUNT:-4}"
+AWS_SIGV4_SCOPE="${AWS_SIGV4_SCOPE:-aws:amz:us-east-1:s3}"
+
+COMPOSE_FILE=""
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/manual_transition_mixed_version_docker_harness.sh [options]
+
+Options:
+  --project-name <name>        Docker compose project name
+  --old-image <image>          Old RustFS image for node1 and rollback node2
+  --new-image <image>          New RustFS image for node2-node4
+  --cold-image <image>         Cold-tier RustFS image (default: new image)
+  --base-port <port>           Host port base; cold uses base, hot nodes use base+1..4
+  --out-dir <path>             Artifact output directory
+  --object-count <n>           Non-empty probe object count
+  --tier <name>                Remote tier name
+  --keep-up                    Leave Docker services running
+  --no-rollback                Do not replace node2 with old image after job admission
+  -h, --help                   Show help
+
+Environment:
+  PROJECT_NAME OLD_IMAGE NEW_IMAGE COLD_IMAGE BASE_PORT OUT_DIR KEEP_UP
+  HOT_ACCESS_KEY HOT_SECRET_KEY COLD_ACCESS_KEY COLD_SECRET_KEY
+  TIER_NAME TIER_BUCKET TIER_PREFIX JOB_BUCKET JOB_PREFIX OBJECT_COUNT
+  WAIT_TIMEOUT_SECS POLL_SECONDS AWS_SIGV4_SCOPE
+
+Artifacts:
+  compose.yml, image inspect files, health/readiness logs, API responses,
+  terminal status, old-node readback, container logs, summary.env.
+USAGE
+}
+
+log_info() {
+  printf '[INFO] %s\n' "$*"
+}
+
+log_warn() {
+  printf '[WARN] %s\n' "$*"
+}
+
+log_error() {
+  printf '[ERROR] %s\n' "$*" >&2
+}
+
+arg_value() {
+  local flag="$1"
+  local value="${2:-}"
+  if [[ -z "$value" || "$value" == --* ]]; then
+    log_error "missing value for ${flag}"
+    exit 1
+  fi
+  printf '%s' "$value"
+}
+
+require_cmd() {
+  if ! command -v "$1" >/dev/null 2>&1; then
+    log_error "required command not found: $1"
+    exit 1
+  fi
+}
+
+parse_positive_int() {
+  local name="$1"
+  local value="$2"
+  if ! [[ "$value" =~ ^[0-9]+$ ]] || (( value <= 0 )); then
+    log_error "${name} must be a positive integer"
+    exit 1
+  fi
+}
+
+parse_args() {
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --project-name)
+        PROJECT_NAME="$(arg_value "$1" "${2:-}")"
+        shift 2
+        ;;
+      --old-image)
+        OLD_IMAGE="$(arg_value "$1" "${2:-}")"
+        shift 2
+        ;;
+      --new-image)
+        NEW_IMAGE="$(arg_value "$1" "${2:-}")"
+        shift 2
+        ;;
+      --cold-image)
+        COLD_IMAGE="$(arg_value "$1" "${2:-}")"
+        shift 2
+        ;;
+      --base-port)
+        BASE_PORT="$(arg_value "$1" "${2:-}")"
+        shift 2
+        ;;
+      --out-dir)
+        OUT_DIR="$(arg_value "$1" "${2:-}")"
+        shift 2
+        ;;
+      --object-count)
+        OBJECT_COUNT="$(arg_value "$1" "${2:-}")"
+        shift 2
+        ;;
+      --tier)
+        TIER_NAME="$(arg_value "$1" "${2:-}")"
+        shift 2
+        ;;
+      --keep-up)
+        KEEP_UP=true
+        shift
+        ;;
+      --no-rollback)
+        ROLLBACK_NEW2_TO_OLD=false
+        shift
+        ;;
+      -h|--help)
+        usage
+        exit 0
+        ;;
+      *)
+        log_error "unknown argument: $1"
+        usage
+        exit 1
+        ;;
+    esac
+  done
+}
+
+compose() {
+  docker compose --project-name "$PROJECT_NAME" -f "$COMPOSE_FILE" "$@"
+}
+
+network_name() {
+  printf '%s_rustfs-1508-net' "$PROJECT_NAME"
+}
+
+hot_volumes() {
+  printf 'http://node{1...4}:9000/data/rustfs{0...3}'
+}
+
+hot_endpoint() {
+  local node_index="$1"
+  printf 'http://127.0.0.1:%s' "$((BASE_PORT + node_index))"
+}
+
+cold_endpoint() {
+  printf 'http://127.0.0.1:%s' "$BASE_PORT"
+}
+
+cleanup() {
+  collect_logs || true
+  if [[ "$KEEP_UP" == "true" ]]; then
+    log_info "KEEP_UP=true, leaving Docker services running"
+    return
+  fi
+  compose down -v >/dev/null 2>&1 || true
+}
+
+write_compose_file() {
+  COMPOSE_FILE="${OUT_DIR}/compose.yml"
+  cat >"$COMPOSE_FILE" <<EOF
+services:
+  cold:
+    image: ${COLD_IMAGE}
+    hostname: cold
+    environment:
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_ACCESS_KEY=${COLD_ACCESS_KEY}
+      - RUSTFS_SECRET_KEY=${COLD_SECRET_KEY}
+      - RUSTFS_VOLUMES=/data/rustfs0 /data/rustfs1 /data/rustfs2 /data/rustfs3
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true
+      - RUSTFS_OBS_LOGGER_LEVEL=warn
+    volumes:
+      - cold_data_0:/data/rustfs0
+      - cold_data_1:/data/rustfs1
+      - cold_data_2:/data/rustfs2
+      - cold_data_3:/data/rustfs3
+    ports:
+      - "${BASE_PORT}:9000"
+    networks:
+      - rustfs-1508-net
+
+  node1:
+    image: ${OLD_IMAGE}
+    hostname: node1
+    environment: &hot-env
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_ACCESS_KEY=${HOT_ACCESS_KEY}
+      - RUSTFS_SECRET_KEY=${HOT_SECRET_KEY}
+      - RUSTFS_VOLUMES=$(hot_volumes)
+      - RUSTFS_SCANNER_ENABLED=false
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true
+      - RUSTFS_OBS_LOGGER_LEVEL=warn
+    volumes:
+      - node1_data_0:/data/rustfs0
+      - node1_data_1:/data/rustfs1
+      - node1_data_2:/data/rustfs2
+      - node1_data_3:/data/rustfs3
+    ports:
+      - "$((BASE_PORT + 1)):9000"
+    networks:
+      - rustfs-1508-net
+
+  node2:
+    image: ${NEW_IMAGE}
+    hostname: node2
+    environment: *hot-env
+    volumes:
+      - node2_data_0:/data/rustfs0
+      - node2_data_1:/data/rustfs1
+      - node2_data_2:/data/rustfs2
+      - node2_data_3:/data/rustfs3
+    ports:
+      - "$((BASE_PORT + 2)):9000"
+    networks:
+      - rustfs-1508-net
+
+  node3:
+    image: ${NEW_IMAGE}
+    hostname: node3
+    environment: *hot-env
+    volumes:
+      - node3_data_0:/data/rustfs0
+      - node3_data_1:/data/rustfs1
+      - node3_data_2:/data/rustfs2
+      - node3_data_3:/data/rustfs3
+    ports:
+      - "$((BASE_PORT + 3)):9000"
+    networks:
+      - rustfs-1508-net
+
+  node4:
+    image: ${NEW_IMAGE}
+    hostname: node4
+    environment: *hot-env
+    volumes:
+      - node4_data_0:/data/rustfs0
+      - node4_data_1:/data/rustfs1
+      - node4_data_2:/data/rustfs2
+      - node4_data_3:/data/rustfs3
+    ports:
+      - "$((BASE_PORT + 4)):9000"
+    networks:
+      - rustfs-1508-net
+
+networks:
+  rustfs-1508-net:
+
+volumes:
+  cold_data_0:
+  cold_data_1:
+  cold_data_2:
+  cold_data_3:
+  node1_data_0:
+  node1_data_1:
+  node1_data_2:
+  node1_data_3:
+  node2_data_0:
+  node2_data_1:
+  node2_data_2:
+  node2_data_3:
+  node3_data_0:
+  node3_data_1:
+  node3_data_2:
+  node3_data_3:
+  node4_data_0:
+  node4_data_1:
+  node4_data_2:
+  node4_data_3:
+EOF
+}
+
+curl_signed() {
+  local access_key="$1"
+  local secret_key="$2"
+  local method="$3"
+  local url="$4"
+  shift 4
+  curl -sS --aws-sigv4 "$AWS_SIGV4_SCOPE" --user "${access_key}:${secret_key}" -X "$method" "$url" "$@"
+}
+
+curl_hot() {
+  local method="$1"
+  local url="$2"
+  shift 2
+  curl_signed "$HOT_ACCESS_KEY" "$HOT_SECRET_KEY" "$method" "$url" "$@"
+}
+
+curl_cold() {
+  local method="$1"
+  local url="$2"
+  shift 2
+  curl_signed "$COLD_ACCESS_KEY" "$COLD_SECRET_KEY" "$method" "$url" "$@"
+}
+
+url_encode() {
+  jq -rn --arg v "$1" '$v|@uri'
+}
+
+wait_http_ok() {
+  local url="$1"
+  local tag="$2"
+  local start now code
+  start="$(date +%s)"
+  while true; do
+    code="$(curl -sS -o /dev/null -w '%{http_code}' "$url" || true)"
+    printf '%s %s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$url" "$code" >>"${OUT_DIR}/health-${tag}.log"
+    if [[ "$code" == "200" ]]; then
+      return 0
+    fi
+    now="$(date +%s)"
+    if (( now - start >= WAIT_TIMEOUT_SECS )); then
+      log_error "timed out waiting for ${url}"
+      return 1
+    fi
+    sleep 2
+  done
+}
+
+wait_cluster_ready() {
+  local index endpoint
+  wait_http_ok "$(cold_endpoint)/health" "cold-live"
+  wait_http_ok "$(cold_endpoint)/health/ready" "cold-ready"
+  for index in 1 2 3 4; do
+    endpoint="$(hot_endpoint "$index")"
+    wait_http_ok "${endpoint}/health" "node${index}-live"
+    wait_http_ok "${endpoint}/health/ready" "node${index}-ready"
+  done
+}
+
+create_bucket() {
+  local endpoint="$1"
+  local bucket="$2"
+  local result_file="$3"
+  curl_hot PUT "${endpoint%/}/${bucket}" -o "${result_file}.response" -w "%{http_code}\n" >"${result_file}.http_code" || true
+}
+
+add_tier() {
+  local body="${OUT_DIR}/add-tier.json"
+  cat >"$body" <<EOF
+{
+  "type": "rustfs",
+  "rustfs": {
+    "name": "${TIER_NAME}",
+    "endpoint": "http://cold:9000",
+    "accessKey": "${COLD_ACCESS_KEY}",
+    "secretKey": "${COLD_SECRET_KEY}",
+    "bucket": "${TIER_BUCKET}",
+    "prefix": "${TIER_PREFIX}",
+    "region": "us-east-1",
+    "storageClass": ""
+  }
+}
+EOF
+  curl_hot PUT "$(hot_endpoint 2)/rustfs/admin/v3/tier" \
+    -H 'Content-Type: application/json' \
+    --data-binary "@${body}" \
+    -o "${OUT_DIR}/add-tier.response" \
+    -w "%{http_code}\n" >"${OUT_DIR}/add-tier.http_code"
+}
+
+put_lifecycle() {
+  local lifecycle="${OUT_DIR}/lifecycle.xml"
+  cat >"$lifecycle" <<EOF
+<LifecycleConfiguration>
+  <Rule>
+    <ID>manual-transition-1508</ID>
+    <Filter><Prefix>${JOB_PREFIX}/</Prefix></Filter>
+    <Status>Enabled</Status>
+    <Transition><Days>0</Days><StorageClass>${TIER_NAME}</StorageClass></Transition>
+  </Rule>
+</LifecycleConfiguration>
+EOF
+  curl_hot PUT "$(hot_endpoint 2)/${JOB_BUCKET}?lifecycle" \
+    -H 'Content-Type: application/xml' \
+    --data-binary "@${lifecycle}" \
+    -o "${OUT_DIR}/put-lifecycle.response" \
+    -w "%{http_code}\n" >"${OUT_DIR}/put-lifecycle.http_code"
+}
+
+seed_objects() {
+  local index key
+  for index in $(seq 1 "$OBJECT_COUNT"); do
+    key="${JOB_PREFIX}/object-$(printf '%02d' "$index").txt"
+    printf 'rustfs #1508 mixed-version transition probe %s\n' "$index" >"${OUT_DIR}/object-${index}.txt"
+    curl_hot PUT "$(hot_endpoint 2)/${JOB_BUCKET}/${key}" \
+      --data-binary "@${OUT_DIR}/object-${index}.txt" \
+      -o "${OUT_DIR}/put-object-${index}.response" \
+      -w "%{http_code}\n" >"${OUT_DIR}/put-object-${index}.http_code"
+    printf '%s\n' "$key" >>"${OUT_DIR}/object-keys.txt"
+  done
+}
+
+start_transition_job() {
+  local query response job_id
+  query="bucket=$(url_encode "$JOB_BUCKET")"
+  query="${query}&prefix=$(url_encode "${JOB_PREFIX}/")"
+  query="${query}&tier=$(url_encode "$TIER_NAME")"
+  query="${query}&dryRun=false&maxObjects=${OBJECT_COUNT}&mode=async"
+  response="$(curl_hot POST "$(hot_endpoint 2)/rustfs/admin/v3/ilm/transition/run?${query}")"
+  printf '%s\n' "$response" >"${OUT_DIR}/run-response.json"
+  job_id="$(printf '%s' "$response" | jq -r '.job_id // empty')"
+  if [[ -z "$job_id" ]]; then
+    log_error "manual transition response omitted job_id"
+    return 1
+  fi
+  printf '%s\n' "$job_id" >"${OUT_DIR}/job-id.txt"
+}
+
+replace_node2_with_old_image() {
+  local network
+  network="$(network_name)"
+  log_info "Replacing node2 with old image ${OLD_IMAGE} for in-flight rollback readback"
+  compose stop node2 >/dev/null
+  compose rm -f node2 >/dev/null
+  docker run -d \
+    --name "${PROJECT_NAME}-node2-rollback-old" \
+    --network "$network" \
+    --network-alias node2 \
+    -p "$((BASE_PORT + 2)):9000" \
+    -e RUSTFS_ADDRESS=:9000 \
+    -e RUSTFS_ACCESS_KEY="$HOT_ACCESS_KEY" \
+    -e RUSTFS_SECRET_KEY="$HOT_SECRET_KEY" \
+    -e RUSTFS_VOLUMES="$(hot_volumes)" \
+    -e RUSTFS_SCANNER_ENABLED=false \
+    -e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
+    -e RUSTFS_OBS_LOGGER_LEVEL=warn \
+    -v "${PROJECT_NAME}_node2_data_0:/data/rustfs0" \
+    -v "${PROJECT_NAME}_node2_data_1:/data/rustfs1" \
+    -v "${PROJECT_NAME}_node2_data_2:/data/rustfs2" \
+    -v "${PROJECT_NAME}_node2_data_3:/data/rustfs3" \
+    "$OLD_IMAGE" >/dev/null
+}
+
+poll_terminal_status() {
+  local job_id="$1"
+  local status_url status_json terminal_state
+  status_url="$(hot_endpoint 3)/rustfs/admin/v3/ilm/transition/jobs/${job_id}"
+  for _ in $(seq 1 "$POLL_SECONDS"); do
+    status_json="$(curl_hot GET "$status_url")"
+    printf '%s\n' "$status_json" >"${OUT_DIR}/status-terminal.json"
+    terminal_state="$(printf '%s' "$status_json" | jq -r '.status // ""')"
+    case "$terminal_state" in
+      completed|partial|failed|cancelled|unknown)
+        printf '%s\n' "$terminal_state" >"${OUT_DIR}/terminal-state.txt"
+        return 0
+        ;;
+    esac
+    sleep 1
+  done
+  log_error "manual transition job did not reach a terminal status within ${POLL_SECONDS}s"
+  return 1
+}
+
+capture_old_node_readback() {
+  local job_id="$1"
+  curl_hot GET "$(hot_endpoint 1)/rustfs/admin/v3/ilm/transition/jobs/${job_id}" \
+    -o "${OUT_DIR}/old-node-status.response" \
+    -w "%{http_code}\n" >"${OUT_DIR}/old-node-status.http_code" || true
+  curl_hot GET "$(hot_endpoint 2)/rustfs/admin/v3/ilm/transition/jobs/${job_id}" \
+    -o "${OUT_DIR}/rollback-node2-status.response" \
+    -w "%{http_code}\n" >"${OUT_DIR}/rollback-node2-status.http_code" || true
+}
+
+head_probe() {
+  local key
+  key="$(head -n 1 "${OUT_DIR}/object-keys.txt")"
+  curl_hot HEAD "$(hot_endpoint 3)/${JOB_BUCKET}/${key}" \
+    -D "${OUT_DIR}/head-object.headers" \
+    -o /dev/null \
+    -w "%{http_code}\n" >"${OUT_DIR}/head-object.http_code" || true
+}
+
+collect_logs() {
+  local service
+  if [[ -z "$COMPOSE_FILE" || ! -f "$COMPOSE_FILE" ]]; then
+    return 0
+  fi
+  for service in cold node1 node2 node3 node4; do
+    compose logs --no-color "$service" >"${OUT_DIR}/${service}.log" 2>/dev/null || true
+  done
+  docker logs "${PROJECT_NAME}-node2-rollback-old" >"${OUT_DIR}/node2-rollback-old.log" 2>/dev/null || true
+}
+
+summarize() {
+  local terminal_state transition_completed tier_failure transition_failed old_code rollback_code
+  terminal_state="$(cat "${OUT_DIR}/terminal-state.txt" 2>/dev/null || true)"
+  transition_completed="$(jq -r '.report.transition_completed // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
+  tier_failure="$(jq -r '.report.tier_failure // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
+  transition_failed="$(jq -r '.report.transition_failed // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')"
+  old_code="$(cat "${OUT_DIR}/old-node-status.http_code" 2>/dev/null || true)"
+  rollback_code="$(cat "${OUT_DIR}/rollback-node2-status.http_code" 2>/dev/null || true)"
+  cat >"${OUT_DIR}/summary.env" <<EOF
+project_name=${PROJECT_NAME}
+old_image=${OLD_IMAGE}
+new_image=${NEW_IMAGE}
+cold_image=${COLD_IMAGE}
+base_port=${BASE_PORT}
+tier=${TIER_NAME}
+job_bucket=${JOB_BUCKET}
+job_prefix=${JOB_PREFIX}
+object_count=${OBJECT_COUNT}
+terminal_state=${terminal_state}
+transition_completed=${transition_completed}
+transition_failed=${transition_failed}
+tier_failure=${tier_failure}
+old_node_status_http_code=${old_code}
+rollback_node2_status_http_code=${rollback_code}
+rollback_new2_to_old=${ROLLBACK_NEW2_TO_OLD}
+EOF
+  cat "${OUT_DIR}/summary.env"
+  if [[ "$terminal_state" != "completed" || "$transition_completed" == "0" || "$transition_failed" != "0" || "$tier_failure" != "0" ]]; then
+    log_error "strict #1508 mixed-version transition gate failed; see ${OUT_DIR}"
+    return 1
+  fi
+}
+
+main() {
+  parse_args "$@"
+  parse_positive_int "--base-port" "$BASE_PORT"
+  parse_positive_int "--object-count" "$OBJECT_COUNT"
+  require_cmd docker
+  require_cmd curl
+  require_cmd jq
+
+  mkdir -p "$OUT_DIR"
+  write_compose_file
+  trap cleanup EXIT INT TERM
+
+  log_info "Writing artifacts under ${OUT_DIR}"
+  docker image inspect "$OLD_IMAGE" >"${OUT_DIR}/old-image.inspect.json"
+  docker image inspect "$NEW_IMAGE" >"${OUT_DIR}/new-image.inspect.json"
+  docker image inspect "$COLD_IMAGE" >"${OUT_DIR}/cold-image.inspect.json"
+
+  compose up -d
+  wait_cluster_ready
+
+  curl_cold PUT "$(cold_endpoint)/${TIER_BUCKET}" -o "${OUT_DIR}/create-cold-bucket.response" -w "%{http_code}\n" >"${OUT_DIR}/create-cold-bucket.http_code"
+  create_bucket "$(hot_endpoint 2)" "$JOB_BUCKET" "${OUT_DIR}/create-hot-bucket"
+  add_tier
+  put_lifecycle
+  seed_objects
+  start_transition_job
+  if [[ "$ROLLBACK_NEW2_TO_OLD" == "true" ]]; then
+    replace_node2_with_old_image
+  fi
+  poll_terminal_status "$(cat "${OUT_DIR}/job-id.txt")"
+  capture_old_node_readback "$(cat "${OUT_DIR}/job-id.txt")"
+  head_probe
+  collect_logs
+  summarize
+}
+
+main "$@"
@@ -8,6 +8,7 @@ SOAK_MATRIX="${PROJECT_ROOT}/scripts/manual_transition_soak_matrix.sh"
 MIXED_RUNBOOK="${PROJECT_ROOT}/scripts/manual_transition_mixed_rollout_runbook.sh"
 STRESS_RUNBOOK="${PROJECT_ROOT}/scripts/manual_transition_nightly_stress_runbook.sh"
 FAILURE_SAMPLES="${PROJECT_ROOT}/scripts/manual_transition_failure_samples.sh"
+MIXED_DOCKER_HARNESS="${PROJECT_ROOT}/scripts/manual_transition_mixed_version_docker_harness.sh"
 TMP_DIR="$(mktemp -d)"

 cleanup() {
@@ -15,7 +16,7 @@ cleanup() {
 }
 trap cleanup EXIT

-bash -n "$MIXED_MATRIX" "$SOAK_MATRIX" "$MIXED_RUNBOOK" "$STRESS_RUNBOOK" "$FAILURE_SAMPLES"
+bash -n "$MIXED_MATRIX" "$SOAK_MATRIX" "$MIXED_RUNBOOK" "$STRESS_RUNBOOK" "$FAILURE_SAMPLES" "$MIXED_DOCKER_HARNESS"

 if "$MIXED_MATRIX" \
  --endpoint http://127.0.0.1:9000 \
@@ -203,6 +204,11 @@ fi

 bash "$FAILURE_SAMPLES" --help >/tmp/manual_transition_failure_samples.help
 rg -q "Usage:" /tmp/manual_transition_failure_samples.help
+bash "$MIXED_DOCKER_HARNESS" --help >/tmp/manual_transition_mixed_version_docker_harness.help
+rg -q "mixed_version_docker_harness" /tmp/manual_transition_mixed_version_docker_harness.help
+rg -q -- "--old-image" /tmp/manual_transition_mixed_version_docker_harness.help
+rg -q -- "--new-image" /tmp/manual_transition_mixed_version_docker_harness.help
+rg -q -- "--no-rollback" /tmp/manual_transition_mixed_version_docker_harness.help
 if bash "$FAILURE_SAMPLES" --endpoint http://127.0.0.1:9000 --sample >/tmp/manual_transition_failure_samples.err 2>&1; then
  echo "failure samples script should fail when --sample has no value" >&2
  exit 1