mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
test(decommission): cover tier lifecycle recovery flows (#3494)
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
cxymds
@@ -4,6 +4,8 @@
|
||||
.vscode
|
||||
.cursor
|
||||
.direnv/
|
||||
.claude/
|
||||
.tmp/
|
||||
/test
|
||||
/logs
|
||||
/data
|
||||
|
||||
@@ -58,5 +58,24 @@ services:
|
||||
networks:
|
||||
- rustfs-decommission-network
|
||||
|
||||
rustfs-decommission-tier:
|
||||
image: ${RUSTFS_TIER_DOCKER_IMAGE:-minio/minio:latest}
|
||||
command:
|
||||
- server
|
||||
- /data
|
||||
- --console-address
|
||||
- ":9001"
|
||||
ports:
|
||||
- "9200:9000"
|
||||
- "9201:9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: "minioadmin"
|
||||
MINIO_ROOT_PASSWORD: "minioadmin"
|
||||
volumes:
|
||||
- ./deploy/data/decommission-tier:/data
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- rustfs-decommission-network
|
||||
|
||||
networks:
|
||||
rustfs-decommission-network:
|
||||
|
||||
@@ -5,10 +5,12 @@ This setup simulates a two-pool RustFS cluster locally with Docker so decommissi
|
||||
## Topology
|
||||
|
||||
- One RustFS container
|
||||
- One disposable MinIO container for tier/lifecycle validation
|
||||
- Two pools
|
||||
- Four disks per pool
|
||||
- S3 API on `http://127.0.0.1:9100`
|
||||
- Console on `http://127.0.0.1:9101`
|
||||
- Tier backend S3 API on `http://127.0.0.1:9200`
|
||||
|
||||
Pool cmdlines used by admin decommission:
|
||||
|
||||
|
||||
@@ -87,7 +87,7 @@ configure_alias() {
|
||||
up() {
|
||||
require_bin docker
|
||||
require_bin mc
|
||||
mkdir -p deploy/data/decommission deploy/logs/decommission
|
||||
mkdir -p deploy/data/decommission deploy/data/decommission-tier deploy/logs/decommission
|
||||
compose up -d
|
||||
wait_healthy
|
||||
configure_alias
|
||||
@@ -107,7 +107,7 @@ down() {
|
||||
reset() {
|
||||
require_bin docker
|
||||
compose down -v --remove-orphans || true
|
||||
rm -rf deploy/data/decommission deploy/logs/decommission .tmp/decommission-validation
|
||||
rm -rf deploy/data/decommission deploy/data/decommission-tier deploy/logs/decommission .tmp/decommission-validation
|
||||
echo "local decommission docker environment reset"
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
# Decommission Tier Lifecycle Recovery
|
||||
|
||||
This manual E2E flow covers the Tier + ILM + Decommission interaction path
|
||||
against a disposable local deployment.
|
||||
|
||||
## Goal
|
||||
|
||||
Validate that transitioned objects can be expired and recovered safely while
|
||||
decommission also handles tiered objects without removing remote tier data
|
||||
incorrectly.
|
||||
|
||||
## What It Covers
|
||||
|
||||
- RustFS tier configuration backed by a disposable MinIO service.
|
||||
- Immediate transition of two object prefixes to the configured tier.
|
||||
- Lifecycle expiry of transitioned objects with RustFS restart recovery.
|
||||
- Concurrent GET traffic while expiry and cleanup are running.
|
||||
- Decommission of a pool containing transitioned objects.
|
||||
- Remote tier count checks before and after decommission.
|
||||
|
||||
## What It Does Not Cover Automatically
|
||||
|
||||
- True distributed node failure injection.
|
||||
- Long-running high-cardinality stress with 10,000+ objects.
|
||||
- Replication target side-effects.
|
||||
- All lifecycle worker pause/failure modes.
|
||||
|
||||
Those scenarios should be added as nightly or cluster-specific E2E coverage
|
||||
after this manual flow is stable.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Docker with Compose support.
|
||||
- `mc` configured or available on `PATH`.
|
||||
- `awscurl` available on `PATH`.
|
||||
- The local RustFS image used by `docker-compose.decommission.yml`.
|
||||
|
||||
If the RustFS image is not present, build it first:
|
||||
|
||||
```bash
|
||||
./scripts/test/decommission_docker.sh build
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
Run the full flow:
|
||||
|
||||
```bash
|
||||
./scripts/test/decommission_tier_lifecycle_recovery.sh run
|
||||
```
|
||||
|
||||
Clean up all generated containers, data, logs, and state:
|
||||
|
||||
```bash
|
||||
./scripts/test/decommission_tier_lifecycle_recovery.sh reset
|
||||
```
|
||||
|
||||
## Manual Flow
|
||||
|
||||
```bash
|
||||
./scripts/test/decommission_tier_lifecycle_recovery.sh up
|
||||
./scripts/test/decommission_tier_lifecycle_recovery.sh prepare
|
||||
./scripts/test/decommission_tier_lifecycle_recovery.sh wait-transition
|
||||
./scripts/test/decommission_tier_lifecycle_recovery.sh expire-recovery
|
||||
./scripts/test/decommission_tier_lifecycle_recovery.sh decommission
|
||||
./scripts/test/decommission_tier_lifecycle_recovery.sh verify
|
||||
```
|
||||
|
||||
## Useful Overrides
|
||||
|
||||
- `OBJECT_COUNT=100` increases objects per prefix.
|
||||
- `TRANSITION_DAYS=0` keeps transition immediate.
|
||||
- `EXPIRY_DAYS=0` keeps expiry immediate for local validation.
|
||||
- `WAIT_TIMEOUT_SECONDS=1800` gives slow machines more time.
|
||||
- `GET_WORKERS=32` increases concurrent GET pressure.
|
||||
- `GET_LOAD_SECONDS=300` keeps GET traffic running longer.
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
OBJECT_COUNT=100 GET_WORKERS=32 WAIT_TIMEOUT_SECONDS=1800 \
|
||||
./scripts/test/decommission_tier_lifecycle_recovery.sh run
|
||||
```
|
||||
|
||||
## Expected Output
|
||||
|
||||
The flow should end with:
|
||||
|
||||
- transitioned `expire/` objects no longer readable as current objects
|
||||
- remote tier object count drained by at least the number of expired objects
|
||||
- concurrent GET log without `NoSuchVersion` or invalid version-state errors
|
||||
- transitioned `decom/` objects still readable after decommission
|
||||
- remote tier object count not lower after decommission
|
||||
|
||||
The script stores evidence under `.tmp/decommission-tier-lifecycle/work`,
|
||||
including scanner status samples, decommission status output, and concurrent
|
||||
GET stderr.
|
||||
|
||||
Missed lifecycle or journal-style counters may increase during restart or queue
|
||||
pressure, but recovery should leave no residual cleanup work for the expired
|
||||
objects.
|
||||
|
||||
## Nightly Follow-Up
|
||||
|
||||
Before wiring this into nightly CI, run it repeatedly with larger `OBJECT_COUNT`
|
||||
values and collect:
|
||||
|
||||
- scanner status samples
|
||||
- tier backend object counts before and after expiry
|
||||
- GET error logs
|
||||
- decommission status JSON
|
||||
- total runtime and flake rate
|
||||
+604
@@ -0,0 +1,604 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.decommission.yml}"
|
||||
RUSTFS_SERVICE_NAME="${RUSTFS_SERVICE_NAME:-rustfs-decommission-latest}"
|
||||
TIER_SERVICE_NAME="${TIER_SERVICE_NAME:-rustfs-decommission-tier}"
|
||||
MC_ALIAS_NAME="${MC_ALIAS_NAME:-rustfs-decom}"
|
||||
TIER_ALIAS_NAME="${TIER_ALIAS_NAME:-rustfs-decom-tier}"
|
||||
S3_ENDPOINT="${S3_ENDPOINT:-http://127.0.0.1:9100}"
|
||||
TIER_ENDPOINT="${TIER_ENDPOINT:-http://127.0.0.1:9200}"
|
||||
TIER_ENDPOINT_FROM_RUSTFS="${TIER_ENDPOINT_FROM_RUSTFS:-http://rustfs-decommission-tier:9000}"
|
||||
ACCESS_KEY="${ACCESS_KEY:-rustfsadmin}"
|
||||
SECRET_KEY="${SECRET_KEY:-rustfsadmin}"
|
||||
TIER_ACCESS_KEY="${TIER_ACCESS_KEY:-minioadmin}"
|
||||
TIER_SECRET_KEY="${TIER_SECRET_KEY:-minioadmin}"
|
||||
TIER_NAME="${TIER_NAME:-COLDTIER}"
|
||||
TIER_BUCKET="${TIER_BUCKET:-rustfs-tier}"
|
||||
TIER_PREFIX="${TIER_PREFIX:-decommission-tier}"
|
||||
POOL_CMDLINE="${POOL_CMDLINE:-/data/pool1/disk{1...4}}"
|
||||
STATE_DIR="${DECOM_TIER_STATE_DIR:-.tmp/decommission-tier-lifecycle}"
|
||||
OBJECT_COUNT="${OBJECT_COUNT:-12}"
|
||||
TRANSITION_DAYS="${TRANSITION_DAYS:-0}"
|
||||
EXPIRY_DAYS="${EXPIRY_DAYS:-0}"
|
||||
WAIT_TIMEOUT_SECONDS="${WAIT_TIMEOUT_SECONDS:-900}"
|
||||
POLL_INTERVAL_SECONDS="${POLL_INTERVAL_SECONDS:-5}"
|
||||
GET_WORKERS="${GET_WORKERS:-8}"
|
||||
GET_LOAD_SECONDS="${GET_LOAD_SECONDS:-60}"
|
||||
SCANNER_SAMPLES="${SCANNER_SAMPLES:-3}"
|
||||
SCANNER_INTERVAL_SECONDS="${SCANNER_INTERVAL_SECONDS:-10}"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
decommission_tier_lifecycle_recovery.sh up
|
||||
decommission_tier_lifecycle_recovery.sh prepare
|
||||
decommission_tier_lifecycle_recovery.sh wait-transition
|
||||
decommission_tier_lifecycle_recovery.sh expire-recovery
|
||||
decommission_tier_lifecycle_recovery.sh decommission
|
||||
decommission_tier_lifecycle_recovery.sh verify
|
||||
decommission_tier_lifecycle_recovery.sh run
|
||||
decommission_tier_lifecycle_recovery.sh status
|
||||
decommission_tier_lifecycle_recovery.sh down
|
||||
decommission_tier_lifecycle_recovery.sh reset
|
||||
|
||||
Environment:
|
||||
COMPOSE_FILE Compose file. Default: docker-compose.decommission.yml
|
||||
MC_ALIAS_NAME RustFS mc alias. Default: rustfs-decom
|
||||
TIER_ALIAS_NAME tier backend mc alias. Default: rustfs-decom-tier
|
||||
S3_ENDPOINT RustFS endpoint from host. Default: http://127.0.0.1:9100
|
||||
TIER_ENDPOINT MinIO tier endpoint from host. Default: http://127.0.0.1:9200
|
||||
TIER_ENDPOINT_FROM_RUSTFS MinIO endpoint reachable by RustFS container.
|
||||
TIER_NAME RustFS tier name. Must be uppercase. Default: COLDTIER
|
||||
OBJECT_COUNT Objects per lifecycle/decommission prefix. Default: 12
|
||||
TRANSITION_DAYS Lifecycle transition days. Default: 0
|
||||
EXPIRY_DAYS Lifecycle expiry days for expire/ objects. Default: 0
|
||||
WAIT_TIMEOUT_SECONDS Poll timeout for transition/expiry/decommission. Default: 900
|
||||
|
||||
The run target starts a local RustFS two-pool deployment plus a disposable
|
||||
MinIO tier backend, transitions two object prefixes, restarts RustFS to exercise
|
||||
recovery, expires one prefix while concurrent GET runs, then decommissions the
|
||||
second prefix's pool and verifies remote tier data was not removed.
|
||||
EOF
|
||||
}
|
||||
|
||||
require_bin() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
echo "missing required binary: $1" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
is_positive_integer() {
|
||||
[[ "$1" =~ ^[1-9][0-9]*$ ]]
|
||||
}
|
||||
|
||||
is_nonnegative_integer() {
|
||||
[[ "$1" =~ ^[0-9]+$ ]]
|
||||
}
|
||||
|
||||
validate_numeric_env() {
|
||||
if ! is_positive_integer "$OBJECT_COUNT"; then
|
||||
echo "OBJECT_COUNT must be a positive integer" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! is_nonnegative_integer "$TRANSITION_DAYS" || ! is_nonnegative_integer "$EXPIRY_DAYS"; then
|
||||
echo "TRANSITION_DAYS and EXPIRY_DAYS must be nonnegative integers" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! is_positive_integer "$WAIT_TIMEOUT_SECONDS" || ! is_positive_integer "$POLL_INTERVAL_SECONDS"; then
|
||||
echo "WAIT_TIMEOUT_SECONDS and POLL_INTERVAL_SECONDS must be positive integers" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
compose() {
|
||||
docker compose -f "$COMPOSE_FILE" "$@"
|
||||
}
|
||||
|
||||
state_file() {
|
||||
printf '%s/state.env\n' "$STATE_DIR"
|
||||
}
|
||||
|
||||
work_dir() {
|
||||
printf '%s/work\n' "$STATE_DIR"
|
||||
}
|
||||
|
||||
load_state() {
|
||||
local file
|
||||
file="$(state_file)"
|
||||
if [[ ! -f "$file" ]]; then
|
||||
echo "state file not found: $file; run prepare first" >&2
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck disable=SC1090
|
||||
source "$file"
|
||||
}
|
||||
|
||||
write_state() {
|
||||
mkdir -p "$STATE_DIR"
|
||||
cat >"$(state_file)" <<EOF
|
||||
RUN_ID='$RUN_ID'
|
||||
TEST_BUCKET='$TEST_BUCKET'
|
||||
EXPIRY_PREFIX='$EXPIRY_PREFIX'
|
||||
DECOM_PREFIX='$DECOM_PREFIX'
|
||||
TIER_NAME='$TIER_NAME'
|
||||
TIER_BUCKET='$TIER_BUCKET'
|
||||
TIER_PREFIX='$TIER_PREFIX'
|
||||
REMOTE_AFTER_TRANSITION='${REMOTE_AFTER_TRANSITION:-0}'
|
||||
REMOTE_BEFORE_DECOMMISSION='${REMOTE_BEFORE_DECOMMISSION:-0}'
|
||||
EOF
|
||||
}
|
||||
|
||||
update_state_value() {
|
||||
local key="$1"
|
||||
local value="$2"
|
||||
load_state
|
||||
case "$key" in
|
||||
REMOTE_AFTER_TRANSITION) REMOTE_AFTER_TRANSITION="$value" ;;
|
||||
REMOTE_BEFORE_DECOMMISSION) REMOTE_BEFORE_DECOMMISSION="$value" ;;
|
||||
*)
|
||||
echo "unsupported state key: $key" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
write_state
|
||||
}
|
||||
|
||||
urlencode() {
|
||||
local raw="$1"
|
||||
if command -v jq >/dev/null 2>&1; then
|
||||
jq -rn --arg v "$raw" '$v|@uri'
|
||||
return
|
||||
fi
|
||||
python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$raw"
|
||||
}
|
||||
|
||||
admin_request() {
|
||||
local method="$1"
|
||||
local path="$2"
|
||||
local body_file="${3:-}"
|
||||
local url="${S3_ENDPOINT%/}/rustfs/admin/v3${path}"
|
||||
|
||||
if [[ -n "$body_file" ]]; then
|
||||
local body
|
||||
body="$(<"$body_file")"
|
||||
awscurl \
|
||||
--fail-with-body \
|
||||
--service s3 \
|
||||
--region us-east-1 \
|
||||
--access_key "$ACCESS_KEY" \
|
||||
--secret_key "$SECRET_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X "$method" \
|
||||
-d "$body" \
|
||||
"$url"
|
||||
else
|
||||
awscurl \
|
||||
--fail-with-body \
|
||||
--service s3 \
|
||||
--region us-east-1 \
|
||||
--access_key "$ACCESS_KEY" \
|
||||
--secret_key "$SECRET_KEY" \
|
||||
-X "$method" \
|
||||
"$url"
|
||||
fi
|
||||
}
|
||||
|
||||
wait_container_ready() {
|
||||
local service="$1"
|
||||
local elapsed=0
|
||||
local container_id
|
||||
container_id="$(compose ps -q "$service")"
|
||||
if [[ -z "$container_id" ]]; then
|
||||
echo "container not found for service $service" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
while (( elapsed < WAIT_TIMEOUT_SECONDS )); do
|
||||
local status
|
||||
status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id")"
|
||||
if [[ "$status" == "healthy" || "$status" == "running" ]]; then
|
||||
echo "$service is $status"
|
||||
return
|
||||
fi
|
||||
sleep "$POLL_INTERVAL_SECONDS"
|
||||
elapsed=$((elapsed + POLL_INTERVAL_SECONDS))
|
||||
done
|
||||
|
||||
echo "$service did not become ready within ${WAIT_TIMEOUT_SECONDS}s" >&2
|
||||
compose logs --tail=200 "$service" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
configure_aliases() {
|
||||
require_bin mc
|
||||
mc alias set "$MC_ALIAS_NAME" "$S3_ENDPOINT" "$ACCESS_KEY" "$SECRET_KEY" >/dev/null
|
||||
mc alias set "$TIER_ALIAS_NAME" "$TIER_ENDPOINT" "$TIER_ACCESS_KEY" "$TIER_SECRET_KEY" >/dev/null
|
||||
echo "configured aliases: $MC_ALIAS_NAME, $TIER_ALIAS_NAME"
|
||||
}
|
||||
|
||||
wait_tier_backend_ready() {
|
||||
local elapsed=0
|
||||
while (( elapsed < WAIT_TIMEOUT_SECONDS )); do
|
||||
if mc alias set "$TIER_ALIAS_NAME" "$TIER_ENDPOINT" "$TIER_ACCESS_KEY" "$TIER_SECRET_KEY" >/dev/null 2>&1 \
|
||||
&& mc ls "$TIER_ALIAS_NAME" >/dev/null 2>&1; then
|
||||
echo "tier backend is ready"
|
||||
return
|
||||
fi
|
||||
sleep "$POLL_INTERVAL_SECONDS"
|
||||
elapsed=$((elapsed + POLL_INTERVAL_SECONDS))
|
||||
done
|
||||
|
||||
echo "tier backend did not become ready within ${WAIT_TIMEOUT_SECONDS}s" >&2
|
||||
compose logs --tail=200 "$TIER_SERVICE_NAME" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
up() {
|
||||
require_bin docker
|
||||
require_bin mc
|
||||
mkdir -p deploy/data/decommission deploy/logs/decommission deploy/data/decommission-tier
|
||||
compose up -d "$TIER_SERVICE_NAME" "$RUSTFS_SERVICE_NAME"
|
||||
wait_container_ready "$TIER_SERVICE_NAME"
|
||||
wait_tier_backend_ready
|
||||
wait_container_ready "$RUSTFS_SERVICE_NAME"
|
||||
configure_aliases
|
||||
mc mb --ignore-existing "${TIER_ALIAS_NAME}/${TIER_BUCKET}"
|
||||
}
|
||||
|
||||
down() {
|
||||
require_bin docker
|
||||
compose down
|
||||
}
|
||||
|
||||
reset() {
|
||||
require_bin docker
|
||||
compose down -v --remove-orphans || true
|
||||
rm -rf deploy/data/decommission deploy/logs/decommission deploy/data/decommission-tier "$STATE_DIR"
|
||||
echo "tier lifecycle decommission environment reset"
|
||||
}
|
||||
|
||||
write_tier_config() {
|
||||
local file="$1"
|
||||
cat >"$file" <<EOF
|
||||
{
|
||||
"type": "minio",
|
||||
"minio": {
|
||||
"name": "${TIER_NAME}",
|
||||
"endpoint": "${TIER_ENDPOINT_FROM_RUSTFS}",
|
||||
"accessKey": "${TIER_ACCESS_KEY}",
|
||||
"secretKey": "${TIER_SECRET_KEY}",
|
||||
"bucket": "${TIER_BUCKET}",
|
||||
"prefix": "${TIER_PREFIX}/${RUN_ID}",
|
||||
"region": "us-east-1"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
write_lifecycle_config() {
|
||||
local file="$1"
|
||||
local include_expiry="$2"
|
||||
local expiry_block=""
|
||||
if [[ "$include_expiry" == "true" ]]; then
|
||||
expiry_block='
|
||||
"Expiration": {
|
||||
"Days": '"${EXPIRY_DAYS}"'
|
||||
},'
|
||||
fi
|
||||
|
||||
cat >"$file" <<EOF
|
||||
{
|
||||
"Rules": [
|
||||
{
|
||||
"ID": "transition-expire-prefix",
|
||||
"Status": "Enabled",
|
||||
"Filter": {
|
||||
"Prefix": "${EXPIRY_PREFIX}"
|
||||
},${expiry_block}
|
||||
"Transition": {
|
||||
"Days": ${TRANSITION_DAYS},
|
||||
"StorageClass": "${TIER_NAME}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": "transition-decommission-prefix",
|
||||
"Status": "Enabled",
|
||||
"Filter": {
|
||||
"Prefix": "${DECOM_PREFIX}"
|
||||
},
|
||||
"Transition": {
|
||||
"Days": ${TRANSITION_DAYS},
|
||||
"StorageClass": "${TIER_NAME}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
prepare() {
|
||||
require_bin mc
|
||||
require_bin awscurl
|
||||
validate_numeric_env
|
||||
|
||||
RUN_ID="$(date -u +%Y%m%d%H%M%S)-$$"
|
||||
TEST_BUCKET="decom-tier-${RUN_ID}"
|
||||
EXPIRY_PREFIX="expire/"
|
||||
DECOM_PREFIX="decom/"
|
||||
REMOTE_AFTER_TRANSITION=0
|
||||
REMOTE_BEFORE_DECOMMISSION=0
|
||||
|
||||
local dir
|
||||
dir="$(work_dir)"
|
||||
mkdir -p "$dir"
|
||||
write_state
|
||||
|
||||
local tier_config lifecycle_config payload
|
||||
tier_config="${dir}/tier-config.json"
|
||||
lifecycle_config="${dir}/lifecycle-transition.json"
|
||||
payload="${dir}/payload.txt"
|
||||
|
||||
write_tier_config "$tier_config"
|
||||
admin_request DELETE "/tier/$(urlencode "$TIER_NAME")?force=true" >/dev/null 2>&1 || true
|
||||
admin_request PUT "/tier?force=true" "$tier_config"
|
||||
admin_request GET "/tier/$(urlencode "$TIER_NAME")" >/dev/null
|
||||
|
||||
mc mb --ignore-existing "${MC_ALIAS_NAME}/${TEST_BUCKET}"
|
||||
write_lifecycle_config "$lifecycle_config" false
|
||||
mc ilm import "${MC_ALIAS_NAME}/${TEST_BUCKET}" <"$lifecycle_config"
|
||||
|
||||
for i in $(seq 1 "$OBJECT_COUNT"); do
|
||||
printf 'expiry object %s run %s\n' "$i" "$RUN_ID" >"$payload"
|
||||
mc cp "$payload" "${MC_ALIAS_NAME}/${TEST_BUCKET}/${EXPIRY_PREFIX}obj-${i}.txt" >/dev/null
|
||||
printf 'decommission object %s run %s\n' "$i" "$RUN_ID" >"$payload"
|
||||
mc cp "$payload" "${MC_ALIAS_NAME}/${TEST_BUCKET}/${DECOM_PREFIX}obj-${i}.txt" >/dev/null
|
||||
done
|
||||
|
||||
echo "prepared bucket: ${TEST_BUCKET}"
|
||||
}
|
||||
|
||||
remote_count() {
|
||||
load_state
|
||||
mc find "${TIER_ALIAS_NAME}/${TIER_BUCKET}/${TIER_PREFIX}/${RUN_ID}" --type f 2>/dev/null | wc -l | tr -d ' '
|
||||
}
|
||||
|
||||
wait_transition() {
|
||||
require_bin mc
|
||||
load_state
|
||||
|
||||
local expected elapsed count
|
||||
expected=$((OBJECT_COUNT * 2))
|
||||
elapsed=0
|
||||
while (( elapsed < WAIT_TIMEOUT_SECONDS )); do
|
||||
count="$(remote_count)"
|
||||
echo "remote tier object count: ${count}/${expected}"
|
||||
if (( count >= expected )); then
|
||||
update_state_value REMOTE_AFTER_TRANSITION "$count"
|
||||
echo "transition completed"
|
||||
return
|
||||
fi
|
||||
sleep "$POLL_INTERVAL_SECONDS"
|
||||
elapsed=$((elapsed + POLL_INTERVAL_SECONDS))
|
||||
done
|
||||
|
||||
echo "timed out waiting for transitioned objects in tier backend" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
start_get_load() {
|
||||
load_state
|
||||
local dir errors pids_file
|
||||
dir="$(work_dir)"
|
||||
mkdir -p "$dir"
|
||||
errors="${dir}/get-errors.log"
|
||||
pids_file="${dir}/get-load.pids"
|
||||
: >"$errors"
|
||||
: >"$pids_file"
|
||||
|
||||
for worker in $(seq 1 "$GET_WORKERS"); do
|
||||
(
|
||||
end_at=$((SECONDS + GET_LOAD_SECONDS))
|
||||
while (( SECONDS < end_at )); do
|
||||
idx=$(( (RANDOM % OBJECT_COUNT) + 1 ))
|
||||
mc cat "${MC_ALIAS_NAME}/${TEST_BUCKET}/${EXPIRY_PREFIX}obj-${idx}.txt" >/dev/null 2>>"$errors" || true
|
||||
done
|
||||
) &
|
||||
echo "$!" >>"$pids_file"
|
||||
done
|
||||
}
|
||||
|
||||
wait_get_load() {
|
||||
local pids_file
|
||||
pids_file="$(work_dir)/get-load.pids"
|
||||
if [[ ! -f "$pids_file" ]]; then
|
||||
return
|
||||
fi
|
||||
while read -r pid; do
|
||||
[[ -z "$pid" ]] && continue
|
||||
wait "$pid" || true
|
||||
done <"$pids_file"
|
||||
}
|
||||
|
||||
assert_no_bad_get_errors() {
|
||||
local errors
|
||||
errors="$(work_dir)/get-errors.log"
|
||||
if [[ ! -f "$errors" ]]; then
|
||||
return
|
||||
fi
|
||||
if grep -E 'NoSuchVersion|remote.*NoSuch|invalid.*version' "$errors" >/dev/null 2>&1; then
|
||||
echo "concurrent GET observed invalid version-state errors:" >&2
|
||||
grep -E 'NoSuchVersion|remote.*NoSuch|invalid.*version' "$errors" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
restart_rustfs() {
|
||||
require_bin docker
|
||||
compose restart "$RUSTFS_SERVICE_NAME"
|
||||
wait_container_ready "$RUSTFS_SERVICE_NAME"
|
||||
configure_aliases
|
||||
}
|
||||
|
||||
expired_count() {
|
||||
load_state
|
||||
local expired=0
|
||||
for i in $(seq 1 "$OBJECT_COUNT"); do
|
||||
if ! mc stat "${MC_ALIAS_NAME}/${TEST_BUCKET}/${EXPIRY_PREFIX}obj-${i}.txt" >/dev/null 2>&1; then
|
||||
expired=$((expired + 1))
|
||||
fi
|
||||
done
|
||||
printf '%s\n' "$expired"
|
||||
}
|
||||
|
||||
wait_expiry_and_cleanup() {
|
||||
load_state
|
||||
local dir lifecycle_config before expected_max elapsed expired count
|
||||
dir="$(work_dir)"
|
||||
lifecycle_config="${dir}/lifecycle-expire.json"
|
||||
write_lifecycle_config "$lifecycle_config" true
|
||||
|
||||
before="$(remote_count)"
|
||||
mc ilm import "${MC_ALIAS_NAME}/${TEST_BUCKET}" <"$lifecycle_config"
|
||||
restart_rustfs
|
||||
start_get_load
|
||||
|
||||
expected_max=$((before - OBJECT_COUNT))
|
||||
if (( expected_max < 0 )); then
|
||||
expected_max=0
|
||||
fi
|
||||
|
||||
elapsed=0
|
||||
while (( elapsed < WAIT_TIMEOUT_SECONDS )); do
|
||||
expired="$(expired_count)"
|
||||
count="$(remote_count)"
|
||||
echo "expiry progress: expired=${expired}/${OBJECT_COUNT}, remote=${count}, target_remote<=${expected_max}"
|
||||
if (( expired >= OBJECT_COUNT && count <= expected_max )); then
|
||||
wait_get_load
|
||||
assert_no_bad_get_errors
|
||||
echo "lifecycle expiry and remote cleanup completed"
|
||||
return
|
||||
fi
|
||||
sleep "$POLL_INTERVAL_SECONDS"
|
||||
elapsed=$((elapsed + POLL_INTERVAL_SECONDS))
|
||||
done
|
||||
|
||||
wait_get_load
|
||||
assert_no_bad_get_errors
|
||||
echo "timed out waiting for lifecycle expiry cleanup" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
run_decommission() {
|
||||
require_bin mc
|
||||
load_state
|
||||
local before
|
||||
before="$(remote_count)"
|
||||
update_state_value REMOTE_BEFORE_DECOMMISSION "$before"
|
||||
./scripts/test/decommission_validation.sh start "$MC_ALIAS_NAME" "$POOL_CMDLINE"
|
||||
./scripts/test/decommission_validation.sh wait "$MC_ALIAS_NAME" "$POOL_CMDLINE" "$WAIT_TIMEOUT_SECONDS"
|
||||
}
|
||||
|
||||
verify_decommission() {
|
||||
load_state
|
||||
local after
|
||||
for i in $(seq 1 "$OBJECT_COUNT"); do
|
||||
mc stat "${MC_ALIAS_NAME}/${TEST_BUCKET}/${DECOM_PREFIX}obj-${i}.txt" >/dev/null
|
||||
done
|
||||
|
||||
after="$(remote_count)"
|
||||
if (( after < REMOTE_BEFORE_DECOMMISSION )); then
|
||||
echo "remote tier object count decreased during decommission: before=${REMOTE_BEFORE_DECOMMISSION}, after=${after}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "decommission verification passed; remote tier count before=${REMOTE_BEFORE_DECOMMISSION}, after=${after}"
|
||||
}
|
||||
|
||||
capture_scanner_status() {
|
||||
require_bin awscurl
|
||||
load_state
|
||||
local out_dir
|
||||
out_dir="$(work_dir)/scanner-status"
|
||||
mkdir -p "$out_dir"
|
||||
|
||||
for i in $(seq 1 "$SCANNER_SAMPLES"); do
|
||||
admin_request GET "/scanner/status" >"${out_dir}/scanner-status-${i}.json" || true
|
||||
sleep "$SCANNER_INTERVAL_SECONDS"
|
||||
done
|
||||
echo "scanner status samples: ${out_dir}"
|
||||
}
|
||||
|
||||
capture_decommission_status() {
|
||||
load_state
|
||||
local out_dir
|
||||
out_dir="$(work_dir)/decommission-status"
|
||||
mkdir -p "$out_dir"
|
||||
./scripts/test/decommission_validation.sh status "$MC_ALIAS_NAME" "$POOL_CMDLINE" >"${out_dir}/status.txt" 2>&1 || true
|
||||
echo "decommission status sample: ${out_dir}/status.txt"
|
||||
}
|
||||
|
||||
status() {
|
||||
load_state
|
||||
echo "bucket: ${TEST_BUCKET}"
|
||||
echo "tier backend objects: $(remote_count)"
|
||||
admin_request GET "/tier-stats?tier=$(urlencode "$TIER_NAME")" || true
|
||||
echo
|
||||
./scripts/test/decommission_validation.sh status "$MC_ALIAS_NAME" "$POOL_CMDLINE" || true
|
||||
}
|
||||
|
||||
verify() {
|
||||
load_state
|
||||
assert_no_bad_get_errors
|
||||
verify_decommission
|
||||
capture_decommission_status
|
||||
capture_scanner_status
|
||||
cat <<EOF
|
||||
verification passed:
|
||||
- expire/ objects are expired and remote tier cleanup reached the expected count
|
||||
- concurrent GET did not emit NoSuchVersion or invalid version-state errors
|
||||
- decom/ tiered objects remain readable after decommission
|
||||
- remote tier object count did not decrease during decommission
|
||||
|
||||
manual evidence:
|
||||
- inspect $(work_dir)/scanner-status for missed/recovery counters
|
||||
- inspect $(work_dir)/get-errors.log for any unexpected non-version errors
|
||||
EOF
|
||||
}
|
||||
|
||||
run_all() {
|
||||
up
|
||||
prepare
|
||||
wait_transition
|
||||
wait_expiry_and_cleanup
|
||||
run_decommission
|
||||
verify
|
||||
}
|
||||
|
||||
main() {
|
||||
if [[ $# -ne 1 ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$1" in
|
||||
up) up ;;
|
||||
prepare) prepare ;;
|
||||
wait-transition) wait_transition ;;
|
||||
expire-recovery) wait_expiry_and_cleanup ;;
|
||||
decommission) run_decommission ;;
|
||||
verify) verify ;;
|
||||
run) run_all ;;
|
||||
status) status ;;
|
||||
down) down ;;
|
||||
reset) reset ;;
|
||||
-h|--help) usage ;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user