diff --git a/.gitignore b/.gitignore index 71fbbb008..24598ac37 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ .vscode .cursor .direnv/ +.claude/ +.tmp/ /test /logs /data diff --git a/docker-compose.decommission.yml b/docker-compose.decommission.yml index d8fc2b281..8c3f1233c 100644 --- a/docker-compose.decommission.yml +++ b/docker-compose.decommission.yml @@ -58,5 +58,24 @@ services: networks: - rustfs-decommission-network + rustfs-decommission-tier: + image: ${RUSTFS_TIER_DOCKER_IMAGE:-minio/minio:latest} + command: + - server + - /data + - --console-address + - ":9001" + ports: + - "9200:9000" + - "9201:9001" + environment: + MINIO_ROOT_USER: "minioadmin" + MINIO_ROOT_PASSWORD: "minioadmin" + volumes: + - ./deploy/data/decommission-tier:/data + restart: unless-stopped + networks: + - rustfs-decommission-network + networks: rustfs-decommission-network: diff --git a/scripts/test/decommission_docker.md b/scripts/test/decommission_docker.md index 6e7f2d972..de895dc76 100644 --- a/scripts/test/decommission_docker.md +++ b/scripts/test/decommission_docker.md @@ -5,10 +5,12 @@ This setup simulates a two-pool RustFS cluster locally with Docker so decommissi ## Topology - One RustFS container +- One disposable MinIO container for tier/lifecycle validation - Two pools - Four disks per pool - S3 API on `http://127.0.0.1:9100` - Console on `http://127.0.0.1:9101` +- Tier backend S3 API on `http://127.0.0.1:9200` Pool cmdlines used by admin decommission: diff --git a/scripts/test/decommission_docker.sh b/scripts/test/decommission_docker.sh index b8b205421..2486b584e 100755 --- a/scripts/test/decommission_docker.sh +++ b/scripts/test/decommission_docker.sh @@ -87,7 +87,7 @@ configure_alias() { up() { require_bin docker require_bin mc - mkdir -p deploy/data/decommission deploy/logs/decommission + mkdir -p deploy/data/decommission deploy/data/decommission-tier deploy/logs/decommission compose up -d wait_healthy configure_alias @@ -107,7 +107,7 @@ down() { reset() { require_bin docker compose down -v --remove-orphans || true - rm -rf deploy/data/decommission deploy/logs/decommission .tmp/decommission-validation + rm -rf deploy/data/decommission deploy/data/decommission-tier deploy/logs/decommission .tmp/decommission-validation echo "local decommission docker environment reset" } diff --git a/scripts/test/decommission_tier_lifecycle_recovery.md b/scripts/test/decommission_tier_lifecycle_recovery.md new file mode 100644 index 000000000..0a9176c62 --- /dev/null +++ b/scripts/test/decommission_tier_lifecycle_recovery.md @@ -0,0 +1,112 @@ +# Decommission Tier Lifecycle Recovery + +This manual E2E flow covers the Tier + ILM + Decommission interaction path +against a disposable local deployment. + +## Goal + +Validate that transitioned objects can be expired and recovered safely while +decommission also handles tiered objects without removing remote tier data +incorrectly. + +## What It Covers + +- RustFS tier configuration backed by a disposable MinIO service. +- Immediate transition of two object prefixes to the configured tier. +- Lifecycle expiry of transitioned objects with RustFS restart recovery. +- Concurrent GET traffic while expiry and cleanup are running. +- Decommission of a pool containing transitioned objects. +- Remote tier count checks before and after decommission. + +## What It Does Not Cover Automatically + +- True distributed node failure injection. +- Long-running high-cardinality stress with 10,000+ objects. +- Replication target side-effects. +- All lifecycle worker pause/failure modes. + +Those scenarios should be added as nightly or cluster-specific E2E coverage +after this manual flow is stable. + +## Prerequisites + +- Docker with Compose support. +- `mc` configured or available on `PATH`. +- `awscurl` available on `PATH`. +- The local RustFS image used by `docker-compose.decommission.yml`. + +If the RustFS image is not present, build it first: + +```bash +./scripts/test/decommission_docker.sh build +``` + +## Quick Start + +Run the full flow: + +```bash +./scripts/test/decommission_tier_lifecycle_recovery.sh run +``` + +Clean up all generated containers, data, logs, and state: + +```bash +./scripts/test/decommission_tier_lifecycle_recovery.sh reset +``` + +## Manual Flow + +```bash +./scripts/test/decommission_tier_lifecycle_recovery.sh up +./scripts/test/decommission_tier_lifecycle_recovery.sh prepare +./scripts/test/decommission_tier_lifecycle_recovery.sh wait-transition +./scripts/test/decommission_tier_lifecycle_recovery.sh expire-recovery +./scripts/test/decommission_tier_lifecycle_recovery.sh decommission +./scripts/test/decommission_tier_lifecycle_recovery.sh verify +``` + +## Useful Overrides + +- `OBJECT_COUNT=100` increases objects per prefix. +- `TRANSITION_DAYS=0` keeps transition immediate. +- `EXPIRY_DAYS=0` keeps expiry immediate for local validation. +- `WAIT_TIMEOUT_SECONDS=1800` gives slow machines more time. +- `GET_WORKERS=32` increases concurrent GET pressure. +- `GET_LOAD_SECONDS=300` keeps GET traffic running longer. + +Example: + +```bash +OBJECT_COUNT=100 GET_WORKERS=32 WAIT_TIMEOUT_SECONDS=1800 \ + ./scripts/test/decommission_tier_lifecycle_recovery.sh run +``` + +## Expected Output + +The flow should end with: + +- transitioned `expire/` objects no longer readable as current objects +- remote tier object count drained by at least the number of expired objects +- concurrent GET log without `NoSuchVersion` or invalid version-state errors +- transitioned `decom/` objects still readable after decommission +- remote tier object count not lower after decommission + +The script stores evidence under `.tmp/decommission-tier-lifecycle/work`, +including scanner status samples, decommission status output, and concurrent +GET stderr. + +Missed lifecycle or journal-style counters may increase during restart or queue +pressure, but recovery should leave no residual cleanup work for the expired +objects. + +## Nightly Follow-Up + +Before wiring this into nightly CI, run it repeatedly with larger `OBJECT_COUNT` +values and collect: + +- scanner status samples +- tier backend object counts before and after expiry +- GET error logs +- decommission status JSON +- total runtime and flake rate diff --git a/scripts/test/decommission_tier_lifecycle_recovery.sh b/scripts/test/decommission_tier_lifecycle_recovery.sh new file mode 100755 index 000000000..77bb12920 --- /dev/null +++ b/scripts/test/decommission_tier_lifecycle_recovery.sh @@ -0,0 +1,604 @@ +#!/usr/bin/env bash +set -euo pipefail + +COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.decommission.yml}" +RUSTFS_SERVICE_NAME="${RUSTFS_SERVICE_NAME:-rustfs-decommission-latest}" +TIER_SERVICE_NAME="${TIER_SERVICE_NAME:-rustfs-decommission-tier}" +MC_ALIAS_NAME="${MC_ALIAS_NAME:-rustfs-decom}" +TIER_ALIAS_NAME="${TIER_ALIAS_NAME:-rustfs-decom-tier}" +S3_ENDPOINT="${S3_ENDPOINT:-http://127.0.0.1:9100}" +TIER_ENDPOINT="${TIER_ENDPOINT:-http://127.0.0.1:9200}" +TIER_ENDPOINT_FROM_RUSTFS="${TIER_ENDPOINT_FROM_RUSTFS:-http://rustfs-decommission-tier:9000}" +ACCESS_KEY="${ACCESS_KEY:-rustfsadmin}" +SECRET_KEY="${SECRET_KEY:-rustfsadmin}" +TIER_ACCESS_KEY="${TIER_ACCESS_KEY:-minioadmin}" +TIER_SECRET_KEY="${TIER_SECRET_KEY:-minioadmin}" +TIER_NAME="${TIER_NAME:-COLDTIER}" +TIER_BUCKET="${TIER_BUCKET:-rustfs-tier}" +TIER_PREFIX="${TIER_PREFIX:-decommission-tier}" +POOL_CMDLINE="${POOL_CMDLINE:-/data/pool1/disk{1...4}}" +STATE_DIR="${DECOM_TIER_STATE_DIR:-.tmp/decommission-tier-lifecycle}" +OBJECT_COUNT="${OBJECT_COUNT:-12}" +TRANSITION_DAYS="${TRANSITION_DAYS:-0}" +EXPIRY_DAYS="${EXPIRY_DAYS:-0}" +WAIT_TIMEOUT_SECONDS="${WAIT_TIMEOUT_SECONDS:-900}" +POLL_INTERVAL_SECONDS="${POLL_INTERVAL_SECONDS:-5}" +GET_WORKERS="${GET_WORKERS:-8}" +GET_LOAD_SECONDS="${GET_LOAD_SECONDS:-60}" +SCANNER_SAMPLES="${SCANNER_SAMPLES:-3}" +SCANNER_INTERVAL_SECONDS="${SCANNER_INTERVAL_SECONDS:-10}" + +usage() { + cat <<'EOF' +Usage: + decommission_tier_lifecycle_recovery.sh up + decommission_tier_lifecycle_recovery.sh prepare + decommission_tier_lifecycle_recovery.sh wait-transition + decommission_tier_lifecycle_recovery.sh expire-recovery + decommission_tier_lifecycle_recovery.sh decommission + decommission_tier_lifecycle_recovery.sh verify + decommission_tier_lifecycle_recovery.sh run + decommission_tier_lifecycle_recovery.sh status + decommission_tier_lifecycle_recovery.sh down + decommission_tier_lifecycle_recovery.sh reset + +Environment: + COMPOSE_FILE Compose file. Default: docker-compose.decommission.yml + MC_ALIAS_NAME RustFS mc alias. Default: rustfs-decom + TIER_ALIAS_NAME tier backend mc alias. Default: rustfs-decom-tier + S3_ENDPOINT RustFS endpoint from host. Default: http://127.0.0.1:9100 + TIER_ENDPOINT MinIO tier endpoint from host. Default: http://127.0.0.1:9200 + TIER_ENDPOINT_FROM_RUSTFS MinIO endpoint reachable by RustFS container. + TIER_NAME RustFS tier name. Must be uppercase. Default: COLDTIER + OBJECT_COUNT Objects per lifecycle/decommission prefix. Default: 12 + TRANSITION_DAYS Lifecycle transition days. Default: 0 + EXPIRY_DAYS Lifecycle expiry days for expire/ objects. Default: 0 + WAIT_TIMEOUT_SECONDS Poll timeout for transition/expiry/decommission. Default: 900 + +The run target starts a local RustFS two-pool deployment plus a disposable +MinIO tier backend, transitions two object prefixes, restarts RustFS to exercise +recovery, expires one prefix while concurrent GET runs, then decommissions the +second prefix's pool and verifies remote tier data was not removed. +EOF +} + +require_bin() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "missing required binary: $1" >&2 + exit 1 + fi +} + +is_positive_integer() { + [[ "$1" =~ ^[1-9][0-9]*$ ]] +} + +is_nonnegative_integer() { + [[ "$1" =~ ^[0-9]+$ ]] +} + +validate_numeric_env() { + if ! is_positive_integer "$OBJECT_COUNT"; then + echo "OBJECT_COUNT must be a positive integer" >&2 + exit 1 + fi + if ! is_nonnegative_integer "$TRANSITION_DAYS" || ! is_nonnegative_integer "$EXPIRY_DAYS"; then + echo "TRANSITION_DAYS and EXPIRY_DAYS must be nonnegative integers" >&2 + exit 1 + fi + if ! is_positive_integer "$WAIT_TIMEOUT_SECONDS" || ! is_positive_integer "$POLL_INTERVAL_SECONDS"; then + echo "WAIT_TIMEOUT_SECONDS and POLL_INTERVAL_SECONDS must be positive integers" >&2 + exit 1 + fi +} + +compose() { + docker compose -f "$COMPOSE_FILE" "$@" +} + +state_file() { + printf '%s/state.env\n' "$STATE_DIR" +} + +work_dir() { + printf '%s/work\n' "$STATE_DIR" +} + +load_state() { + local file + file="$(state_file)" + if [[ ! -f "$file" ]]; then + echo "state file not found: $file; run prepare first" >&2 + exit 1 + fi + # shellcheck disable=SC1090 + source "$file" +} + +write_state() { + mkdir -p "$STATE_DIR" + cat >"$(state_file)" <&2 + exit 1 + ;; + esac + write_state +} + +urlencode() { + local raw="$1" + if command -v jq >/dev/null 2>&1; then + jq -rn --arg v "$raw" '$v|@uri' + return + fi + python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$raw" +} + +admin_request() { + local method="$1" + local path="$2" + local body_file="${3:-}" + local url="${S3_ENDPOINT%/}/rustfs/admin/v3${path}" + + if [[ -n "$body_file" ]]; then + local body + body="$(<"$body_file")" + awscurl \ + --fail-with-body \ + --service s3 \ + --region us-east-1 \ + --access_key "$ACCESS_KEY" \ + --secret_key "$SECRET_KEY" \ + -H "Content-Type: application/json" \ + -X "$method" \ + -d "$body" \ + "$url" + else + awscurl \ + --fail-with-body \ + --service s3 \ + --region us-east-1 \ + --access_key "$ACCESS_KEY" \ + --secret_key "$SECRET_KEY" \ + -X "$method" \ + "$url" + fi +} + +wait_container_ready() { + local service="$1" + local elapsed=0 + local container_id + container_id="$(compose ps -q "$service")" + if [[ -z "$container_id" ]]; then + echo "container not found for service $service" >&2 + exit 1 + fi + + while (( elapsed < WAIT_TIMEOUT_SECONDS )); do + local status + status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id")" + if [[ "$status" == "healthy" || "$status" == "running" ]]; then + echo "$service is $status" + return + fi + sleep "$POLL_INTERVAL_SECONDS" + elapsed=$((elapsed + POLL_INTERVAL_SECONDS)) + done + + echo "$service did not become ready within ${WAIT_TIMEOUT_SECONDS}s" >&2 + compose logs --tail=200 "$service" || true + exit 1 +} + +configure_aliases() { + require_bin mc + mc alias set "$MC_ALIAS_NAME" "$S3_ENDPOINT" "$ACCESS_KEY" "$SECRET_KEY" >/dev/null + mc alias set "$TIER_ALIAS_NAME" "$TIER_ENDPOINT" "$TIER_ACCESS_KEY" "$TIER_SECRET_KEY" >/dev/null + echo "configured aliases: $MC_ALIAS_NAME, $TIER_ALIAS_NAME" +} + +wait_tier_backend_ready() { + local elapsed=0 + while (( elapsed < WAIT_TIMEOUT_SECONDS )); do + if mc alias set "$TIER_ALIAS_NAME" "$TIER_ENDPOINT" "$TIER_ACCESS_KEY" "$TIER_SECRET_KEY" >/dev/null 2>&1 \ + && mc ls "$TIER_ALIAS_NAME" >/dev/null 2>&1; then + echo "tier backend is ready" + return + fi + sleep "$POLL_INTERVAL_SECONDS" + elapsed=$((elapsed + POLL_INTERVAL_SECONDS)) + done + + echo "tier backend did not become ready within ${WAIT_TIMEOUT_SECONDS}s" >&2 + compose logs --tail=200 "$TIER_SERVICE_NAME" || true + exit 1 +} + +up() { + require_bin docker + require_bin mc + mkdir -p deploy/data/decommission deploy/logs/decommission deploy/data/decommission-tier + compose up -d "$TIER_SERVICE_NAME" "$RUSTFS_SERVICE_NAME" + wait_container_ready "$TIER_SERVICE_NAME" + wait_tier_backend_ready + wait_container_ready "$RUSTFS_SERVICE_NAME" + configure_aliases + mc mb --ignore-existing "${TIER_ALIAS_NAME}/${TIER_BUCKET}" +} + +down() { + require_bin docker + compose down +} + +reset() { + require_bin docker + compose down -v --remove-orphans || true + rm -rf deploy/data/decommission deploy/logs/decommission deploy/data/decommission-tier "$STATE_DIR" + echo "tier lifecycle decommission environment reset" +} + +write_tier_config() { + local file="$1" + cat >"$file" <"$file" </dev/null 2>&1 || true + admin_request PUT "/tier?force=true" "$tier_config" + admin_request GET "/tier/$(urlencode "$TIER_NAME")" >/dev/null + + mc mb --ignore-existing "${MC_ALIAS_NAME}/${TEST_BUCKET}" + write_lifecycle_config "$lifecycle_config" false + mc ilm import "${MC_ALIAS_NAME}/${TEST_BUCKET}" <"$lifecycle_config" + + for i in $(seq 1 "$OBJECT_COUNT"); do + printf 'expiry object %s run %s\n' "$i" "$RUN_ID" >"$payload" + mc cp "$payload" "${MC_ALIAS_NAME}/${TEST_BUCKET}/${EXPIRY_PREFIX}obj-${i}.txt" >/dev/null + printf 'decommission object %s run %s\n' "$i" "$RUN_ID" >"$payload" + mc cp "$payload" "${MC_ALIAS_NAME}/${TEST_BUCKET}/${DECOM_PREFIX}obj-${i}.txt" >/dev/null + done + + echo "prepared bucket: ${TEST_BUCKET}" +} + +remote_count() { + load_state + mc find "${TIER_ALIAS_NAME}/${TIER_BUCKET}/${TIER_PREFIX}/${RUN_ID}" --type f 2>/dev/null | wc -l | tr -d ' ' +} + +wait_transition() { + require_bin mc + load_state + + local expected elapsed count + expected=$((OBJECT_COUNT * 2)) + elapsed=0 + while (( elapsed < WAIT_TIMEOUT_SECONDS )); do + count="$(remote_count)" + echo "remote tier object count: ${count}/${expected}" + if (( count >= expected )); then + update_state_value REMOTE_AFTER_TRANSITION "$count" + echo "transition completed" + return + fi + sleep "$POLL_INTERVAL_SECONDS" + elapsed=$((elapsed + POLL_INTERVAL_SECONDS)) + done + + echo "timed out waiting for transitioned objects in tier backend" >&2 + exit 1 +} + +start_get_load() { + load_state + local dir errors pids_file + dir="$(work_dir)" + mkdir -p "$dir" + errors="${dir}/get-errors.log" + pids_file="${dir}/get-load.pids" + : >"$errors" + : >"$pids_file" + + for worker in $(seq 1 "$GET_WORKERS"); do + ( + end_at=$((SECONDS + GET_LOAD_SECONDS)) + while (( SECONDS < end_at )); do + idx=$(( (RANDOM % OBJECT_COUNT) + 1 )) + mc cat "${MC_ALIAS_NAME}/${TEST_BUCKET}/${EXPIRY_PREFIX}obj-${idx}.txt" >/dev/null 2>>"$errors" || true + done + ) & + echo "$!" >>"$pids_file" + done +} + +wait_get_load() { + local pids_file + pids_file="$(work_dir)/get-load.pids" + if [[ ! -f "$pids_file" ]]; then + return + fi + while read -r pid; do + [[ -z "$pid" ]] && continue + wait "$pid" || true + done <"$pids_file" +} + +assert_no_bad_get_errors() { + local errors + errors="$(work_dir)/get-errors.log" + if [[ ! -f "$errors" ]]; then + return + fi + if grep -E 'NoSuchVersion|remote.*NoSuch|invalid.*version' "$errors" >/dev/null 2>&1; then + echo "concurrent GET observed invalid version-state errors:" >&2 + grep -E 'NoSuchVersion|remote.*NoSuch|invalid.*version' "$errors" >&2 + exit 1 + fi +} + +restart_rustfs() { + require_bin docker + compose restart "$RUSTFS_SERVICE_NAME" + wait_container_ready "$RUSTFS_SERVICE_NAME" + configure_aliases +} + +expired_count() { + load_state + local expired=0 + for i in $(seq 1 "$OBJECT_COUNT"); do + if ! mc stat "${MC_ALIAS_NAME}/${TEST_BUCKET}/${EXPIRY_PREFIX}obj-${i}.txt" >/dev/null 2>&1; then + expired=$((expired + 1)) + fi + done + printf '%s\n' "$expired" +} + +wait_expiry_and_cleanup() { + load_state + local dir lifecycle_config before expected_max elapsed expired count + dir="$(work_dir)" + lifecycle_config="${dir}/lifecycle-expire.json" + write_lifecycle_config "$lifecycle_config" true + + before="$(remote_count)" + mc ilm import "${MC_ALIAS_NAME}/${TEST_BUCKET}" <"$lifecycle_config" + restart_rustfs + start_get_load + + expected_max=$((before - OBJECT_COUNT)) + if (( expected_max < 0 )); then + expected_max=0 + fi + + elapsed=0 + while (( elapsed < WAIT_TIMEOUT_SECONDS )); do + expired="$(expired_count)" + count="$(remote_count)" + echo "expiry progress: expired=${expired}/${OBJECT_COUNT}, remote=${count}, target_remote<=${expected_max}" + if (( expired >= OBJECT_COUNT && count <= expected_max )); then + wait_get_load + assert_no_bad_get_errors + echo "lifecycle expiry and remote cleanup completed" + return + fi + sleep "$POLL_INTERVAL_SECONDS" + elapsed=$((elapsed + POLL_INTERVAL_SECONDS)) + done + + wait_get_load + assert_no_bad_get_errors + echo "timed out waiting for lifecycle expiry cleanup" >&2 + exit 1 +} + +run_decommission() { + require_bin mc + load_state + local before + before="$(remote_count)" + update_state_value REMOTE_BEFORE_DECOMMISSION "$before" + ./scripts/test/decommission_validation.sh start "$MC_ALIAS_NAME" "$POOL_CMDLINE" + ./scripts/test/decommission_validation.sh wait "$MC_ALIAS_NAME" "$POOL_CMDLINE" "$WAIT_TIMEOUT_SECONDS" +} + +verify_decommission() { + load_state + local after + for i in $(seq 1 "$OBJECT_COUNT"); do + mc stat "${MC_ALIAS_NAME}/${TEST_BUCKET}/${DECOM_PREFIX}obj-${i}.txt" >/dev/null + done + + after="$(remote_count)" + if (( after < REMOTE_BEFORE_DECOMMISSION )); then + echo "remote tier object count decreased during decommission: before=${REMOTE_BEFORE_DECOMMISSION}, after=${after}" >&2 + exit 1 + fi + + echo "decommission verification passed; remote tier count before=${REMOTE_BEFORE_DECOMMISSION}, after=${after}" +} + +capture_scanner_status() { + require_bin awscurl + load_state + local out_dir + out_dir="$(work_dir)/scanner-status" + mkdir -p "$out_dir" + + for i in $(seq 1 "$SCANNER_SAMPLES"); do + admin_request GET "/scanner/status" >"${out_dir}/scanner-status-${i}.json" || true + sleep "$SCANNER_INTERVAL_SECONDS" + done + echo "scanner status samples: ${out_dir}" +} + +capture_decommission_status() { + load_state + local out_dir + out_dir="$(work_dir)/decommission-status" + mkdir -p "$out_dir" + ./scripts/test/decommission_validation.sh status "$MC_ALIAS_NAME" "$POOL_CMDLINE" >"${out_dir}/status.txt" 2>&1 || true + echo "decommission status sample: ${out_dir}/status.txt" +} + +status() { + load_state + echo "bucket: ${TEST_BUCKET}" + echo "tier backend objects: $(remote_count)" + admin_request GET "/tier-stats?tier=$(urlencode "$TIER_NAME")" || true + echo + ./scripts/test/decommission_validation.sh status "$MC_ALIAS_NAME" "$POOL_CMDLINE" || true +} + +verify() { + load_state + assert_no_bad_get_errors + verify_decommission + capture_decommission_status + capture_scanner_status + cat <