diff --git a/crates/e2e_test/src/inline_fast_path_cluster_test.rs b/crates/e2e_test/src/inline_fast_path_cluster_test.rs index 17161ea7f..2857fa1d3 100644 --- a/crates/e2e_test/src/inline_fast_path_cluster_test.rs +++ b/crates/e2e_test/src/inline_fast_path_cluster_test.rs @@ -2167,6 +2167,111 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat Ok(()) } +#[tokio::test] +#[serial] +#[ignore = "manual #1508 evidence harness: starts a 4-node cluster, a remote tier, and an in-flight transition job"] +async fn four_node_manual_transition_rollout_non_empty_restart_readback() -> TestResult { + init_logging(); + + let mut cold = RustFSTestEnvironment::new().await?; + cold.access_key = "inline1508coldadmin".to_string(); + cold.secret_key = "inline1508coldsecret".to_string(); + cold.start_rustfs_server_without_cleanup(vec![]).await?; + let cold_client = cold.create_s3_client(); + cold_client.create_bucket().bucket(TIER_BUCKET).send().await?; + + let mut hot = RustFSTestClusterEnvironment::new(4).await?; + hot.set_env("RUSTFS_SCANNER_ENABLED", "false"); + hot.set_env("RUSTFS_SCANNER_CYCLE", "3600"); + hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "2"); + hot.set_env("RUSTFS_TRANSITION_QUEUE_CAPACITY", "64"); + hot.start().await?; + let hot_client = hot.create_s3_client(0)?; + + let tier_name = unique_tier_name(); + add_rustfs_tier(&hot, &cold, &tier_name).await?; + let bucket = format!("manual-transition-1508-{}", Uuid::new_v4().simple()); + let prefix = "transition/manual-rollout/"; + hot_client.create_bucket().bucket(&bucket).send().await?; + put_lifecycle_with_transition_retry(&hot_client, &bucket, &tier_name).await?; + for index in 0u8..24 { + let key = format!("{prefix}object-{index:02}.bin"); + hot_client + .put_object() + .bucket(&bucket) + .key(key) + .body(ByteStream::from(payload(64 * KIB, index))) + .send() + .await?; + } + + let (accepted_status, accepted_body) = + start_manual_transition_job_on_node(&hot, 1, &bucket, prefix, &tier_name, false, 24).await?; + assert_eq!( + accepted_status, + StatusCode::ACCEPTED, + "non-empty #1508 transition job should be accepted by a rollout node: {}", + compact_body(&accepted_body) + ); + let accepted: serde_json::Value = serde_json::from_str(&accepted_body)?; + let job_id = accepted["job_id"] + .as_str() + .ok_or_else(|| format!("accepted #1508 response omitted job_id: {accepted_body}"))?; + let status_endpoint = accepted["status_endpoint"] + .as_str() + .ok_or_else(|| format!("accepted #1508 response omitted status_endpoint: {accepted_body}"))?; + + let terminal = wait_for_manual_transition_job_terminal(&hot, 0, job_id, false).await?; + assert_eq!(terminal["job_id"].as_str(), Some(job_id)); + assert_eq!(terminal["bucket"].as_str(), Some(bucket.as_str())); + assert_eq!(terminal["prefix"].as_str(), Some(prefix)); + assert_eq!(terminal["dry_run"].as_bool(), Some(false)); + assert_eq!( + terminal["status"].as_str(), + Some("completed"), + "non-empty #1508 rollout transition should complete before restart readback: {terminal}" + ); + let transition_failed: u64 = terminal["report"] + .get("transition_failed") + .and_then(serde_json::Value::as_u64) + .unwrap_or_default(); + assert_eq!( + transition_failed, 0, + "terminal #1508 report should not hide transition failures: {terminal}" + ); + assert_eq!( + terminal["report"]["tier_failure"].as_u64(), + Some(0), + "terminal #1508 report should not hide tier failures: {terminal}" + ); + let transition_completed = terminal["report"]["transition_completed"] + .as_u64() + .ok_or_else(|| format!("terminal #1508 report omitted transition_completed: {terminal}"))?; + assert!( + transition_completed > 0, + "terminal #1508 report should include real completed transitions: {terminal}" + ); + assert!( + cold_tier_object_count(&cold_client).await? > 0, + "cold tier should contain transitioned #1508 objects" + ); + + hot.stop_node(2)?; + hot.start_node(2).await?; + let after_restart = read_manual_transition_job_status_endpoint(&hot, 2, status_endpoint).await?; + assert_eq!(after_restart["job_id"].as_str(), Some(job_id)); + assert_eq!( + after_restart["status"], terminal["status"], + "terminal #1508 status changed after rollout node restart" + ); + assert_eq!( + after_restart["report"]["transition_completed"], terminal["report"]["transition_completed"], + "terminal #1508 transition count changed after rollout node restart" + ); + + Ok(()) +} + #[tokio::test] #[serial] async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_transition() -> TestResult { diff --git a/scripts/README.md b/scripts/README.md index 61a6475c9..c88f4b679 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -90,6 +90,7 @@ their issue closes. | `manual_transition_journal_audit.sh` | dev-tool | Journal + metrics + log audit for manual transition jobs | — | | `manual_transition_mixed_rollout_matrix.sh` | dev-tool | Matrix generator for mixed-version rollout phases | — | | `manual_transition_mixed_rollout_runbook.sh` | dev-tool | Reusable mixed-version rollout runbook generator (external run) | — | +| `manual_transition_mixed_version_docker_harness.sh` | dev-tool | Dedicated #1508 Docker harness for old/new manual-transition rollout evidence | `test_manual_transition_runbooks.sh` | | `monitor_manual_transition_ci.sh` | dev-tool | CI workflow/status watcher for manual transition follow-up monitoring | — | | `manual_transition_soak_matrix.sh` | dev-tool | Matrix generator for nightly stress windows | — | | `manual_transition_nightly_stress_runbook.sh` | dev-tool | Nightly stress entrypoint with failure snapshot templates | — | diff --git a/scripts/manual_transition_mixed_version_docker_harness.sh b/scripts/manual_transition_mixed_version_docker_harness.sh new file mode 100755 index 000000000..0937703fc --- /dev/null +++ b/scripts/manual_transition_mixed_version_docker_harness.sh @@ -0,0 +1,595 @@ +#!/usr/bin/env bash +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +PROJECT_NAME="${PROJECT_NAME:-rustfs-1508-mixed}" +OLD_IMAGE="${OLD_IMAGE:-rustfs/rustfs:latest}" +NEW_IMAGE="${NEW_IMAGE:-rustfs/rustfs:latest}" +COLD_IMAGE="${COLD_IMAGE:-${NEW_IMAGE}}" +BASE_PORT="${BASE_PORT:-19400}" +OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/manual-transition-1508-docker/$(date +%Y%m%dT%H%M%S)}" +KEEP_UP=false +ROLLBACK_NEW2_TO_OLD=true +WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-180}" +POLL_SECONDS="${POLL_SECONDS:-180}" + +HOT_ACCESS_KEY="${HOT_ACCESS_KEY:-mvadmin}" +HOT_SECRET_KEY="${HOT_SECRET_KEY:-mvsecret}" +COLD_ACCESS_KEY="${COLD_ACCESS_KEY:-coldadmin}" +COLD_SECRET_KEY="${COLD_SECRET_KEY:-coldsecret}" +TIER_NAME="${TIER_NAME:-COLDMV}" +TIER_BUCKET="${TIER_BUCKET:-manual-transition-cold-tier}" +TIER_PREFIX="${TIER_PREFIX:-tiered}" +JOB_BUCKET="${JOB_BUCKET:-manual-transition-1508}" +JOB_PREFIX="${JOB_PREFIX:-transition/manual-rollout}" +OBJECT_COUNT="${OBJECT_COUNT:-4}" +AWS_SIGV4_SCOPE="${AWS_SIGV4_SCOPE:-aws:amz:us-east-1:s3}" + +COMPOSE_FILE="" + +usage() { + cat <<'USAGE' +Usage: + scripts/manual_transition_mixed_version_docker_harness.sh [options] + +Options: + --project-name Docker compose project name + --old-image Old RustFS image for node1 and rollback node2 + --new-image New RustFS image for node2-node4 + --cold-image Cold-tier RustFS image (default: new image) + --base-port Host port base; cold uses base, hot nodes use base+1..4 + --out-dir Artifact output directory + --object-count Non-empty probe object count + --tier Remote tier name + --keep-up Leave Docker services running + --no-rollback Do not replace node2 with old image after job admission + -h, --help Show help + +Environment: + PROJECT_NAME OLD_IMAGE NEW_IMAGE COLD_IMAGE BASE_PORT OUT_DIR KEEP_UP + HOT_ACCESS_KEY HOT_SECRET_KEY COLD_ACCESS_KEY COLD_SECRET_KEY + TIER_NAME TIER_BUCKET TIER_PREFIX JOB_BUCKET JOB_PREFIX OBJECT_COUNT + WAIT_TIMEOUT_SECS POLL_SECONDS AWS_SIGV4_SCOPE + +Artifacts: + compose.yml, image inspect files, health/readiness logs, API responses, + terminal status, old-node readback, container logs, summary.env. +USAGE +} + +log_info() { + printf '[INFO] %s\n' "$*" +} + +log_warn() { + printf '[WARN] %s\n' "$*" +} + +log_error() { + printf '[ERROR] %s\n' "$*" >&2 +} + +arg_value() { + local flag="$1" + local value="${2:-}" + if [[ -z "$value" || "$value" == --* ]]; then + log_error "missing value for ${flag}" + exit 1 + fi + printf '%s' "$value" +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + log_error "required command not found: $1" + exit 1 + fi +} + +parse_positive_int() { + local name="$1" + local value="$2" + if ! [[ "$value" =~ ^[0-9]+$ ]] || (( value <= 0 )); then + log_error "${name} must be a positive integer" + exit 1 + fi +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --project-name) + PROJECT_NAME="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --old-image) + OLD_IMAGE="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --new-image) + NEW_IMAGE="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --cold-image) + COLD_IMAGE="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --base-port) + BASE_PORT="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --out-dir) + OUT_DIR="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --object-count) + OBJECT_COUNT="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --tier) + TIER_NAME="$(arg_value "$1" "${2:-}")" + shift 2 + ;; + --keep-up) + KEEP_UP=true + shift + ;; + --no-rollback) + ROLLBACK_NEW2_TO_OLD=false + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + log_error "unknown argument: $1" + usage + exit 1 + ;; + esac + done +} + +compose() { + docker compose --project-name "$PROJECT_NAME" -f "$COMPOSE_FILE" "$@" +} + +network_name() { + printf '%s_rustfs-1508-net' "$PROJECT_NAME" +} + +hot_volumes() { + printf 'http://node{1...4}:9000/data/rustfs{0...3}' +} + +hot_endpoint() { + local node_index="$1" + printf 'http://127.0.0.1:%s' "$((BASE_PORT + node_index))" +} + +cold_endpoint() { + printf 'http://127.0.0.1:%s' "$BASE_PORT" +} + +cleanup() { + collect_logs || true + if [[ "$KEEP_UP" == "true" ]]; then + log_info "KEEP_UP=true, leaving Docker services running" + return + fi + compose down -v >/dev/null 2>&1 || true +} + +write_compose_file() { + COMPOSE_FILE="${OUT_DIR}/compose.yml" + cat >"$COMPOSE_FILE" <>"${OUT_DIR}/health-${tag}.log" + if [[ "$code" == "200" ]]; then + return 0 + fi + now="$(date +%s)" + if (( now - start >= WAIT_TIMEOUT_SECS )); then + log_error "timed out waiting for ${url}" + return 1 + fi + sleep 2 + done +} + +wait_cluster_ready() { + local index endpoint + wait_http_ok "$(cold_endpoint)/health" "cold-live" + wait_http_ok "$(cold_endpoint)/health/ready" "cold-ready" + for index in 1 2 3 4; do + endpoint="$(hot_endpoint "$index")" + wait_http_ok "${endpoint}/health" "node${index}-live" + wait_http_ok "${endpoint}/health/ready" "node${index}-ready" + done +} + +create_bucket() { + local endpoint="$1" + local bucket="$2" + local result_file="$3" + curl_hot PUT "${endpoint%/}/${bucket}" -o "${result_file}.response" -w "%{http_code}\n" >"${result_file}.http_code" || true +} + +add_tier() { + local body="${OUT_DIR}/add-tier.json" + cat >"$body" <"${OUT_DIR}/add-tier.http_code" +} + +put_lifecycle() { + local lifecycle="${OUT_DIR}/lifecycle.xml" + cat >"$lifecycle" < + + manual-transition-1508 + ${JOB_PREFIX}/ + Enabled + 0${TIER_NAME} + + +EOF + curl_hot PUT "$(hot_endpoint 2)/${JOB_BUCKET}?lifecycle" \ + -H 'Content-Type: application/xml' \ + --data-binary "@${lifecycle}" \ + -o "${OUT_DIR}/put-lifecycle.response" \ + -w "%{http_code}\n" >"${OUT_DIR}/put-lifecycle.http_code" +} + +seed_objects() { + local index key + for index in $(seq 1 "$OBJECT_COUNT"); do + key="${JOB_PREFIX}/object-$(printf '%02d' "$index").txt" + printf 'rustfs #1508 mixed-version transition probe %s\n' "$index" >"${OUT_DIR}/object-${index}.txt" + curl_hot PUT "$(hot_endpoint 2)/${JOB_BUCKET}/${key}" \ + --data-binary "@${OUT_DIR}/object-${index}.txt" \ + -o "${OUT_DIR}/put-object-${index}.response" \ + -w "%{http_code}\n" >"${OUT_DIR}/put-object-${index}.http_code" + printf '%s\n' "$key" >>"${OUT_DIR}/object-keys.txt" + done +} + +start_transition_job() { + local query response job_id + query="bucket=$(url_encode "$JOB_BUCKET")" + query="${query}&prefix=$(url_encode "${JOB_PREFIX}/")" + query="${query}&tier=$(url_encode "$TIER_NAME")" + query="${query}&dryRun=false&maxObjects=${OBJECT_COUNT}&mode=async" + response="$(curl_hot POST "$(hot_endpoint 2)/rustfs/admin/v3/ilm/transition/run?${query}")" + printf '%s\n' "$response" >"${OUT_DIR}/run-response.json" + job_id="$(printf '%s' "$response" | jq -r '.job_id // empty')" + if [[ -z "$job_id" ]]; then + log_error "manual transition response omitted job_id" + return 1 + fi + printf '%s\n' "$job_id" >"${OUT_DIR}/job-id.txt" +} + +replace_node2_with_old_image() { + local network + network="$(network_name)" + log_info "Replacing node2 with old image ${OLD_IMAGE} for in-flight rollback readback" + compose stop node2 >/dev/null + compose rm -f node2 >/dev/null + docker run -d \ + --name "${PROJECT_NAME}-node2-rollback-old" \ + --network "$network" \ + --network-alias node2 \ + -p "$((BASE_PORT + 2)):9000" \ + -e RUSTFS_ADDRESS=:9000 \ + -e RUSTFS_ACCESS_KEY="$HOT_ACCESS_KEY" \ + -e RUSTFS_SECRET_KEY="$HOT_SECRET_KEY" \ + -e RUSTFS_VOLUMES="$(hot_volumes)" \ + -e RUSTFS_SCANNER_ENABLED=false \ + -e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \ + -e RUSTFS_OBS_LOGGER_LEVEL=warn \ + -v "${PROJECT_NAME}_node2_data_0:/data/rustfs0" \ + -v "${PROJECT_NAME}_node2_data_1:/data/rustfs1" \ + -v "${PROJECT_NAME}_node2_data_2:/data/rustfs2" \ + -v "${PROJECT_NAME}_node2_data_3:/data/rustfs3" \ + "$OLD_IMAGE" >/dev/null +} + +poll_terminal_status() { + local job_id="$1" + local status_url status_json terminal_state + status_url="$(hot_endpoint 3)/rustfs/admin/v3/ilm/transition/jobs/${job_id}" + for _ in $(seq 1 "$POLL_SECONDS"); do + status_json="$(curl_hot GET "$status_url")" + printf '%s\n' "$status_json" >"${OUT_DIR}/status-terminal.json" + terminal_state="$(printf '%s' "$status_json" | jq -r '.status // ""')" + case "$terminal_state" in + completed|partial|failed|cancelled|unknown) + printf '%s\n' "$terminal_state" >"${OUT_DIR}/terminal-state.txt" + return 0 + ;; + esac + sleep 1 + done + log_error "manual transition job did not reach a terminal status within ${POLL_SECONDS}s" + return 1 +} + +capture_old_node_readback() { + local job_id="$1" + curl_hot GET "$(hot_endpoint 1)/rustfs/admin/v3/ilm/transition/jobs/${job_id}" \ + -o "${OUT_DIR}/old-node-status.response" \ + -w "%{http_code}\n" >"${OUT_DIR}/old-node-status.http_code" || true + curl_hot GET "$(hot_endpoint 2)/rustfs/admin/v3/ilm/transition/jobs/${job_id}" \ + -o "${OUT_DIR}/rollback-node2-status.response" \ + -w "%{http_code}\n" >"${OUT_DIR}/rollback-node2-status.http_code" || true +} + +head_probe() { + local key + key="$(head -n 1 "${OUT_DIR}/object-keys.txt")" + curl_hot HEAD "$(hot_endpoint 3)/${JOB_BUCKET}/${key}" \ + -D "${OUT_DIR}/head-object.headers" \ + -o /dev/null \ + -w "%{http_code}\n" >"${OUT_DIR}/head-object.http_code" || true +} + +collect_logs() { + local service + if [[ -z "$COMPOSE_FILE" || ! -f "$COMPOSE_FILE" ]]; then + return 0 + fi + for service in cold node1 node2 node3 node4; do + compose logs --no-color "$service" >"${OUT_DIR}/${service}.log" 2>/dev/null || true + done + docker logs "${PROJECT_NAME}-node2-rollback-old" >"${OUT_DIR}/node2-rollback-old.log" 2>/dev/null || true +} + +summarize() { + local terminal_state transition_completed tier_failure transition_failed old_code rollback_code + terminal_state="$(cat "${OUT_DIR}/terminal-state.txt" 2>/dev/null || true)" + transition_completed="$(jq -r '.report.transition_completed // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')" + tier_failure="$(jq -r '.report.tier_failure // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')" + transition_failed="$(jq -r '.report.transition_failed // 0' "${OUT_DIR}/status-terminal.json" 2>/dev/null || printf '0')" + old_code="$(cat "${OUT_DIR}/old-node-status.http_code" 2>/dev/null || true)" + rollback_code="$(cat "${OUT_DIR}/rollback-node2-status.http_code" 2>/dev/null || true)" + cat >"${OUT_DIR}/summary.env" <"${OUT_DIR}/old-image.inspect.json" + docker image inspect "$NEW_IMAGE" >"${OUT_DIR}/new-image.inspect.json" + docker image inspect "$COLD_IMAGE" >"${OUT_DIR}/cold-image.inspect.json" + + compose up -d + wait_cluster_ready + + curl_cold PUT "$(cold_endpoint)/${TIER_BUCKET}" -o "${OUT_DIR}/create-cold-bucket.response" -w "%{http_code}\n" >"${OUT_DIR}/create-cold-bucket.http_code" + create_bucket "$(hot_endpoint 2)" "$JOB_BUCKET" "${OUT_DIR}/create-hot-bucket" + add_tier + put_lifecycle + seed_objects + start_transition_job + if [[ "$ROLLBACK_NEW2_TO_OLD" == "true" ]]; then + replace_node2_with_old_image + fi + poll_terminal_status "$(cat "${OUT_DIR}/job-id.txt")" + capture_old_node_readback "$(cat "${OUT_DIR}/job-id.txt")" + head_probe + collect_logs + summarize +} + +main "$@" diff --git a/scripts/test_manual_transition_runbooks.sh b/scripts/test_manual_transition_runbooks.sh index ca940eb67..248a72ef3 100755 --- a/scripts/test_manual_transition_runbooks.sh +++ b/scripts/test_manual_transition_runbooks.sh @@ -8,6 +8,7 @@ SOAK_MATRIX="${PROJECT_ROOT}/scripts/manual_transition_soak_matrix.sh" MIXED_RUNBOOK="${PROJECT_ROOT}/scripts/manual_transition_mixed_rollout_runbook.sh" STRESS_RUNBOOK="${PROJECT_ROOT}/scripts/manual_transition_nightly_stress_runbook.sh" FAILURE_SAMPLES="${PROJECT_ROOT}/scripts/manual_transition_failure_samples.sh" +MIXED_DOCKER_HARNESS="${PROJECT_ROOT}/scripts/manual_transition_mixed_version_docker_harness.sh" TMP_DIR="$(mktemp -d)" cleanup() { @@ -15,7 +16,7 @@ cleanup() { } trap cleanup EXIT -bash -n "$MIXED_MATRIX" "$SOAK_MATRIX" "$MIXED_RUNBOOK" "$STRESS_RUNBOOK" "$FAILURE_SAMPLES" +bash -n "$MIXED_MATRIX" "$SOAK_MATRIX" "$MIXED_RUNBOOK" "$STRESS_RUNBOOK" "$FAILURE_SAMPLES" "$MIXED_DOCKER_HARNESS" if "$MIXED_MATRIX" \ --endpoint http://127.0.0.1:9000 \ @@ -203,6 +204,11 @@ fi bash "$FAILURE_SAMPLES" --help >/tmp/manual_transition_failure_samples.help rg -q "Usage:" /tmp/manual_transition_failure_samples.help +bash "$MIXED_DOCKER_HARNESS" --help >/tmp/manual_transition_mixed_version_docker_harness.help +rg -q "mixed_version_docker_harness" /tmp/manual_transition_mixed_version_docker_harness.help +rg -q -- "--old-image" /tmp/manual_transition_mixed_version_docker_harness.help +rg -q -- "--new-image" /tmp/manual_transition_mixed_version_docker_harness.help +rg -q -- "--no-rollback" /tmp/manual_transition_mixed_version_docker_harness.help if bash "$FAILURE_SAMPLES" --endpoint http://127.0.0.1:9000 --sample >/tmp/manual_transition_failure_samples.err 2>&1; then echo "failure samples script should fail when --sample has no value" >&2 exit 1