mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
feat: improve legacy metadata and admin compatibility (#2202)
This commit is contained in:
weisd
@@ -5,7 +5,7 @@ RUSTFS_VOLUMES="http://node{1...4}:7000/data/rustfs{0...3} http://node{5...8
|
||||
RUSTFS_ADDRESS=":7000"
|
||||
RUSTFS_CONSOLE_ENABLE=true
|
||||
RUST_LOG=warn
|
||||
RUSTFS_OBS_LOG_DIRECTORY="/var/logs/rustfs/"
|
||||
RUSTFS_OBS_LOG_DIRECTORY="./deploy/logs"
|
||||
RUSTFS_NS_SCANNER_INTERVAL=60
|
||||
#RUSTFS_SKIP_BACKGROUND_TASK=true
|
||||
RUSTFS_COMPRESSION_ENABLED=true
|
||||
RUSTFS_COMPRESSION_ENABLED=true
|
||||
|
||||
@@ -18,7 +18,11 @@ set -euo pipefail
|
||||
# Disable proxy for localhost requests to avoid interference
|
||||
export NO_PROXY="127.0.0.1,localhost,::1"
|
||||
export no_proxy="${NO_PROXY}"
|
||||
unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY
|
||||
# Keep proxy variables for external downloads; NO_PROXY bypasses localhost.
|
||||
|
||||
# Ensure user-level Python scripts are discoverable (awscurl/tox installed via --user)
|
||||
PYTHON_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" 2>/dev/null || echo "3.14")
|
||||
export PATH="$HOME/Library/Python/${PYTHON_VERSION}/bin:$HOME/.local/bin:$PATH"
|
||||
|
||||
# Configuration
|
||||
S3_ACCESS_KEY="${S3_ACCESS_KEY:-rustfsadmin}"
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
# Local Docker Decommission Test
|
||||
|
||||
This setup simulates a two-pool RustFS cluster locally with Docker so decommission can be tested without relying on a remote environment.
|
||||
|
||||
## Topology
|
||||
|
||||
- One RustFS container
|
||||
- Two pools
|
||||
- Four disks per pool
|
||||
- S3 API on `http://127.0.0.1:9100`
|
||||
- Console on `http://127.0.0.1:9101`
|
||||
|
||||
Pool cmdlines used by admin decommission:
|
||||
|
||||
- `/data/pool0/disk{1...4}`
|
||||
- `/data/pool1/disk{1...4}`
|
||||
|
||||
## Quick Start
|
||||
|
||||
1. Start the local cluster:
|
||||
`./scripts/test/decommission_docker.sh up`
|
||||
2. Run the end-to-end smoke flow:
|
||||
`./scripts/test/decommission_docker.sh smoke`
|
||||
|
||||
## Manual Flow
|
||||
|
||||
1. Start the cluster:
|
||||
`./scripts/test/decommission_docker.sh up`
|
||||
2. Prepare test data:
|
||||
`./scripts/test/decommission_validation.sh prepare rustfs-decom`
|
||||
3. Start decommission on pool 1:
|
||||
`./scripts/test/decommission_validation.sh start rustfs-decom '/data/pool1/disk{1...4}'`
|
||||
4. Wait for completion:
|
||||
`./scripts/test/decommission_validation.sh wait rustfs-decom '/data/pool1/disk{1...4}' 900`
|
||||
5. Verify objects and versions:
|
||||
`./scripts/test/decommission_validation.sh verify rustfs-decom`
|
||||
|
||||
## Cleanup
|
||||
|
||||
- Stop containers:
|
||||
`./scripts/test/decommission_docker.sh down`
|
||||
- Remove containers, volumes, generated data, and saved validation state:
|
||||
`./scripts/test/decommission_docker.sh reset`
|
||||
|
||||
## Notes
|
||||
|
||||
- This simulates pools, not separate physical nodes.
|
||||
- It is good for validating decommission control flow and object migration behavior.
|
||||
- It is not a substitute for a real distributed failure-injection test.
|
||||
Executable
+173
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.decommission.yml}"
|
||||
SERVICE_NAME="${SERVICE_NAME:-rustfs-decommission-latest}"
|
||||
MC_ALIAS_NAME="${MC_ALIAS_NAME:-rustfs-decom}"
|
||||
S3_ENDPOINT="${S3_ENDPOINT:-http://127.0.0.1:9100}"
|
||||
ACCESS_KEY="${ACCESS_KEY:-rustfsadmin}"
|
||||
SECRET_KEY="${SECRET_KEY:-rustfsadmin}"
|
||||
DOCKERFILE_PATH="${DOCKERFILE_PATH:-Dockerfile.decommission-local}"
|
||||
DOCKER_IMAGE_NAME="${DOCKER_IMAGE_NAME:-rustfs-local:decommission-latest}"
|
||||
POOL0_CMDLINE="${POOL0_CMDLINE:-/data/pool0/disk{1...4}}"
|
||||
POOL1_CMDLINE="${POOL1_CMDLINE:-/data/pool1/disk{1...4}}"
|
||||
HEALTH_TIMEOUT_SECONDS="${HEALTH_TIMEOUT_SECONDS:-1200}"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage:
|
||||
decommission_docker.sh build
|
||||
decommission_docker.sh up
|
||||
decommission_docker.sh down
|
||||
decommission_docker.sh reset
|
||||
decommission_docker.sh info
|
||||
decommission_docker.sh smoke
|
||||
|
||||
Environment:
|
||||
COMPOSE_FILE Compose file path. Default: ${COMPOSE_FILE}
|
||||
DOCKERFILE_PATH Dockerfile path. Default: ${DOCKERFILE_PATH}
|
||||
DOCKER_IMAGE_NAME Image tag to build/use. Default: ${DOCKER_IMAGE_NAME}
|
||||
MC_ALIAS_NAME mc alias to create. Default: ${MC_ALIAS_NAME}
|
||||
S3_ENDPOINT RustFS S3 endpoint. Default: ${S3_ENDPOINT}
|
||||
POOL0_CMDLINE First pool cmdline. Default: ${POOL0_CMDLINE}
|
||||
POOL1_CMDLINE Second pool cmdline. Default: ${POOL1_CMDLINE}
|
||||
HEALTH_TIMEOUT_SECONDS Wait timeout. Default: ${HEALTH_TIMEOUT_SECONDS}
|
||||
|
||||
Smoke flow:
|
||||
1. Start a local two-pool RustFS container
|
||||
2. Configure mc alias ${MC_ALIAS_NAME}
|
||||
3. Prepare decommission test data
|
||||
4. Decommission pool 1
|
||||
5. Wait for completion and verify readability
|
||||
EOF
|
||||
}
|
||||
|
||||
require_bin() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
echo "missing required binary: $1" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
compose() {
|
||||
docker compose -f "${COMPOSE_FILE}" "$@"
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
local elapsed=0
|
||||
local container_id
|
||||
container_id="$(compose ps -q "${SERVICE_NAME}")"
|
||||
if [[ -z "${container_id}" ]]; then
|
||||
echo "container not found for service ${SERVICE_NAME}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
while (( elapsed < HEALTH_TIMEOUT_SECONDS )); do
|
||||
local status
|
||||
status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "${container_id}")"
|
||||
if [[ "${status}" == "healthy" || "${status}" == "running" ]]; then
|
||||
echo "container is ${status}"
|
||||
return
|
||||
fi
|
||||
sleep 5
|
||||
elapsed=$((elapsed + 5))
|
||||
done
|
||||
|
||||
echo "container did not become healthy within ${HEALTH_TIMEOUT_SECONDS}s" >&2
|
||||
compose logs --tail=200 "${SERVICE_NAME}" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
configure_alias() {
|
||||
require_bin mc
|
||||
mc alias set "${MC_ALIAS_NAME}" "${S3_ENDPOINT}" "${ACCESS_KEY}" "${SECRET_KEY}" >/dev/null
|
||||
echo "mc alias configured: ${MC_ALIAS_NAME} -> ${S3_ENDPOINT}"
|
||||
}
|
||||
|
||||
up() {
|
||||
require_bin docker
|
||||
require_bin mc
|
||||
mkdir -p deploy/data/decommission deploy/logs/decommission
|
||||
compose up -d
|
||||
wait_healthy
|
||||
configure_alias
|
||||
info
|
||||
}
|
||||
|
||||
build_image() {
|
||||
require_bin docker
|
||||
docker build -f "${DOCKERFILE_PATH}" -t "${DOCKER_IMAGE_NAME}" .
|
||||
}
|
||||
|
||||
down() {
|
||||
require_bin docker
|
||||
compose down
|
||||
}
|
||||
|
||||
reset() {
|
||||
require_bin docker
|
||||
compose down -v --remove-orphans || true
|
||||
rm -rf deploy/data/decommission deploy/logs/decommission .tmp/decommission-validation
|
||||
echo "local decommission docker environment reset"
|
||||
}
|
||||
|
||||
info() {
|
||||
cat <<EOF
|
||||
Local Docker decommission environment:
|
||||
compose file : ${COMPOSE_FILE}
|
||||
dockerfile : ${DOCKERFILE_PATH}
|
||||
image : ${DOCKER_IMAGE_NAME}
|
||||
mc alias : ${MC_ALIAS_NAME}
|
||||
endpoint : ${S3_ENDPOINT}
|
||||
pool 0 : ${POOL0_CMDLINE}
|
||||
pool 1 : ${POOL1_CMDLINE}
|
||||
|
||||
Recommended manual flow:
|
||||
./scripts/test/decommission_validation.sh prepare ${MC_ALIAS_NAME}
|
||||
./scripts/test/decommission_validation.sh start ${MC_ALIAS_NAME} '${POOL1_CMDLINE}'
|
||||
./scripts/test/decommission_validation.sh wait ${MC_ALIAS_NAME} '${POOL1_CMDLINE}' 900
|
||||
./scripts/test/decommission_validation.sh verify ${MC_ALIAS_NAME}
|
||||
EOF
|
||||
}
|
||||
|
||||
smoke() {
|
||||
up
|
||||
./scripts/test/decommission_validation.sh prepare "${MC_ALIAS_NAME}"
|
||||
./scripts/test/decommission_validation.sh start "${MC_ALIAS_NAME}" "${POOL1_CMDLINE}"
|
||||
./scripts/test/decommission_validation.sh wait "${MC_ALIAS_NAME}" "${POOL1_CMDLINE}" 900
|
||||
./scripts/test/decommission_validation.sh verify "${MC_ALIAS_NAME}"
|
||||
}
|
||||
|
||||
main() {
|
||||
if [[ $# -ne 1 ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$1" in
|
||||
build)
|
||||
build_image
|
||||
;;
|
||||
up)
|
||||
up
|
||||
;;
|
||||
down)
|
||||
down
|
||||
;;
|
||||
reset)
|
||||
reset
|
||||
;;
|
||||
info)
|
||||
info
|
||||
;;
|
||||
smoke)
|
||||
smoke
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -0,0 +1,57 @@
|
||||
# Decommission Validation
|
||||
|
||||
This helper covers a practical decommission smoke flow against a disposable multi-pool cluster that is already running and reachable via `mc`.
|
||||
|
||||
## What It Covers
|
||||
|
||||
- Versioned object rewrite
|
||||
- Multipart object rewrite
|
||||
- Delete-marker preservation
|
||||
- Object-lock bucket readability after decommission
|
||||
- Admin decommission start/status/cancel command flow
|
||||
|
||||
## What It Does Not Cover Automatically
|
||||
|
||||
- Remote tier object migration
|
||||
- Lifecycle-expired object skipping
|
||||
- Replication target side-effects
|
||||
- Cross-node failure injection during decommission
|
||||
|
||||
Those scenarios still need cluster-specific setup and should be verified manually after the smoke flow passes.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- `mc` installed and configured with an alias that points to the RustFS cluster
|
||||
- A disposable cluster with at least two pools
|
||||
- A pool cmdline string that matches the admin API input, for example:
|
||||
`http://server{5...8}/disk{1...4}`
|
||||
|
||||
## Recommended Flow
|
||||
|
||||
1. Prepare data:
|
||||
`./scripts/test/decommission_validation.sh prepare <alias>`
|
||||
2. Start decommission:
|
||||
`./scripts/test/decommission_validation.sh start <alias> '<pool-cmdline>'`
|
||||
3. Wait for completion:
|
||||
`./scripts/test/decommission_validation.sh wait <alias> '<pool-cmdline>' 900`
|
||||
4. Verify readable data and version listings:
|
||||
`./scripts/test/decommission_validation.sh verify <alias>`
|
||||
|
||||
## Optional Cancel Check
|
||||
|
||||
1. Start decommission on a disposable pool.
|
||||
2. Run:
|
||||
`./scripts/test/decommission_validation.sh cancel <alias> '<pool-cmdline>'`
|
||||
3. Confirm status shows `canceled=true` and not `complete=true`.
|
||||
|
||||
## Manual Extension For Tiered Objects
|
||||
|
||||
If the cluster already has a configured remote tier and ILM transition rule:
|
||||
|
||||
1. Upload an object that matches the transition rule.
|
||||
2. Wait until the object is transitioned remotely.
|
||||
3. Run the decommission flow above.
|
||||
4. Confirm:
|
||||
- the object is still readable after decommission
|
||||
- decommission completes without `failed=true`
|
||||
- no residual object/version remains on the source pool
|
||||
Executable
+434
@@ -0,0 +1,434 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
decommission_validation.sh prepare <alias>
|
||||
decommission_validation.sh start <alias> <pool-cmdline>
|
||||
decommission_validation.sh status <alias> <pool-cmdline>
|
||||
decommission_validation.sh wait <alias> <pool-cmdline> [timeout-seconds]
|
||||
decommission_validation.sh verify <alias>
|
||||
decommission_validation.sh cancel <alias> [pool-cmdline]
|
||||
|
||||
Environment:
|
||||
DECOM_STATE_DIR Directory for generated state files. Default: .tmp/decommission-validation
|
||||
DECOM_ADMIN_API Admin API mode: auto, mc, rustfs. Default: auto
|
||||
|
||||
Examples:
|
||||
./scripts/test/decommission_validation.sh prepare rustfs
|
||||
./scripts/test/decommission_validation.sh start rustfs 'http://server{5...8}/disk{1...4}'
|
||||
./scripts/test/decommission_validation.sh wait rustfs 'http://server{5...8}/disk{1...4}' 900
|
||||
./scripts/test/decommission_validation.sh verify rustfs
|
||||
EOF
|
||||
}
|
||||
|
||||
require_bin() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
echo "missing required binary: $1" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
state_dir() {
|
||||
printf '%s\n' "${DECOM_STATE_DIR:-.tmp/decommission-validation}"
|
||||
}
|
||||
|
||||
state_file() {
|
||||
local alias_name="$1"
|
||||
printf '%s/%s.env\n' "$(state_dir)" "$alias_name"
|
||||
}
|
||||
|
||||
load_state() {
|
||||
local alias_name="$1"
|
||||
local file
|
||||
file="$(state_file "$alias_name")"
|
||||
if [[ ! -f "$file" ]]; then
|
||||
echo "state file not found: $file" >&2
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck disable=SC1090
|
||||
source "$file"
|
||||
}
|
||||
|
||||
admin_alias() {
|
||||
printf '%s\n' "$1"
|
||||
}
|
||||
|
||||
admin_api_mode() {
|
||||
printf '%s\n' "${DECOM_ADMIN_API:-auto}"
|
||||
}
|
||||
|
||||
mc_alias_field() {
|
||||
local alias_name="$1"
|
||||
local field="$2"
|
||||
local alias_json
|
||||
|
||||
alias_json="$(mc alias list --json 2>/dev/null | grep -F "\"alias\":\"${alias_name}\"" | tail -n 1 || true)"
|
||||
if [[ -z "$alias_json" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if command -v jq >/dev/null 2>&1; then
|
||||
printf '%s\n' "$alias_json" | jq -r ".${field} // empty"
|
||||
return
|
||||
fi
|
||||
|
||||
printf '%s\n' "$alias_json" | sed -n "s/.*\"${field}\":\"\\([^\"]*\\)\".*/\\1/p"
|
||||
}
|
||||
|
||||
urlencode() {
|
||||
local raw="$1"
|
||||
if command -v jq >/dev/null 2>&1; then
|
||||
jq -rn --arg v "$raw" '$v|@uri'
|
||||
return
|
||||
fi
|
||||
|
||||
python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$raw"
|
||||
}
|
||||
|
||||
rustfs_admin_base() {
|
||||
local alias_name="$1"
|
||||
local url
|
||||
url="$(mc_alias_field "$alias_name" "URL")"
|
||||
if [[ -z "$url" ]]; then
|
||||
echo "unable to resolve mc alias URL for ${alias_name}" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s/rustfs/admin/v3' "${url%/}"
|
||||
}
|
||||
|
||||
rustfs_admin_request() {
|
||||
local alias_name="$1"
|
||||
local method="$2"
|
||||
local path="$3"
|
||||
local access_key secret_key
|
||||
|
||||
access_key="$(mc_alias_field "$alias_name" "accessKey")"
|
||||
secret_key="$(mc_alias_field "$alias_name" "secretKey")"
|
||||
|
||||
if [[ -z "$access_key" || -z "$secret_key" ]]; then
|
||||
echo "unable to resolve mc alias credentials for ${alias_name}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
awscurl \
|
||||
--fail-with-body \
|
||||
--service s3 \
|
||||
--region us-east-1 \
|
||||
--access_key "$access_key" \
|
||||
--secret_key "$secret_key" \
|
||||
-X "$method" \
|
||||
"$(rustfs_admin_base "$alias_name")${path}"
|
||||
}
|
||||
|
||||
detect_admin_api() {
|
||||
local alias_name="$1"
|
||||
local requested
|
||||
requested="$(admin_api_mode)"
|
||||
|
||||
case "$requested" in
|
||||
mc|rustfs)
|
||||
printf '%s\n' "$requested"
|
||||
return
|
||||
;;
|
||||
auto)
|
||||
;;
|
||||
*)
|
||||
echo "unsupported DECOM_ADMIN_API mode: ${requested}" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if ! command -v awscurl >/dev/null 2>&1; then
|
||||
printf 'mc\n'
|
||||
return
|
||||
fi
|
||||
|
||||
if rustfs_admin_request "$alias_name" GET "/pools/list" >/dev/null 2>&1; then
|
||||
printf 'rustfs\n'
|
||||
else
|
||||
printf 'mc\n'
|
||||
fi
|
||||
}
|
||||
|
||||
write_state() {
|
||||
local alias_name="$1"
|
||||
local run_id="$2"
|
||||
local basic_bucket="$3"
|
||||
local lock_bucket="$4"
|
||||
local state_path
|
||||
state_path="$(state_file "$alias_name")"
|
||||
mkdir -p "$(dirname "$state_path")"
|
||||
cat >"$state_path" <<EOF
|
||||
ALIAS_NAME='$alias_name'
|
||||
RUN_ID='$run_id'
|
||||
BASIC_BUCKET='$basic_bucket'
|
||||
LOCK_BUCKET='$lock_bucket'
|
||||
EOF
|
||||
printf 'state file: %s\n' "$state_path"
|
||||
}
|
||||
|
||||
prepare() {
|
||||
local alias_name="$1"
|
||||
|
||||
require_bin mc
|
||||
require_bin mktemp
|
||||
require_bin dd
|
||||
|
||||
local run_id
|
||||
run_id="$(date +%Y%m%d%H%M%S)-$$"
|
||||
local basic_bucket="decom-basic-${run_id}"
|
||||
local lock_bucket="decom-lock-${run_id}"
|
||||
local workdir
|
||||
workdir="$(mktemp -d)"
|
||||
|
||||
trap "rm -rf -- '$workdir'" EXIT
|
||||
|
||||
printf 'alpha-v1\n' >"${workdir}/single.txt"
|
||||
printf 'tombstone\n' >"${workdir}/delete-marker.txt"
|
||||
dd if=/dev/zero of="${workdir}/multipart.bin" bs=1M count=20 status=none
|
||||
|
||||
mc mb "${alias_name}/${basic_bucket}"
|
||||
mc version enable "${alias_name}/${basic_bucket}"
|
||||
mc cp "${workdir}/single.txt" "${alias_name}/${basic_bucket}/single.txt"
|
||||
|
||||
printf 'alpha-v2\n' >"${workdir}/single.txt"
|
||||
mc cp "${workdir}/single.txt" "${alias_name}/${basic_bucket}/single.txt"
|
||||
mc cp "${workdir}/multipart.bin" "${alias_name}/${basic_bucket}/multipart.bin"
|
||||
mc cp "${workdir}/delete-marker.txt" "${alias_name}/${basic_bucket}/delete-marker.txt"
|
||||
mc rm "${alias_name}/${basic_bucket}/delete-marker.txt"
|
||||
|
||||
mc mb --with-lock "${alias_name}/${lock_bucket}"
|
||||
mc retention set --default GOVERNANCE "1d" "${alias_name}/${lock_bucket}"
|
||||
mc cp "${workdir}/single.txt" "${alias_name}/${lock_bucket}/locked.txt"
|
||||
|
||||
write_state "$alias_name" "$run_id" "$basic_bucket" "$lock_bucket"
|
||||
|
||||
cat <<EOF
|
||||
prepared buckets:
|
||||
basic: ${basic_bucket}
|
||||
lock : ${lock_bucket}
|
||||
|
||||
next steps:
|
||||
./scripts/test/decommission_validation.sh start ${alias_name} '<pool-cmdline>'
|
||||
./scripts/test/decommission_validation.sh wait ${alias_name} '<pool-cmdline>' 900
|
||||
./scripts/test/decommission_validation.sh verify ${alias_name}
|
||||
EOF
|
||||
}
|
||||
|
||||
start_decommission() {
|
||||
local alias_name="$1"
|
||||
local pool_cmdline="$2"
|
||||
local mode
|
||||
mode="$(detect_admin_api "$alias_name")"
|
||||
|
||||
case "$mode" in
|
||||
rustfs)
|
||||
require_bin awscurl
|
||||
rustfs_admin_request "$alias_name" POST "/pools/decommission?pool=${pool_cmdline}"
|
||||
;;
|
||||
mc)
|
||||
require_bin mc
|
||||
mc admin decommission start "$(admin_alias "$alias_name")" "$pool_cmdline"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
status_decommission() {
|
||||
local alias_name="$1"
|
||||
local pool_cmdline="$2"
|
||||
local mode
|
||||
mode="$(detect_admin_api "$alias_name")"
|
||||
|
||||
case "$mode" in
|
||||
rustfs)
|
||||
require_bin awscurl
|
||||
rustfs_admin_request "$alias_name" GET "/pools/status?pool=${pool_cmdline}"
|
||||
;;
|
||||
mc)
|
||||
require_bin mc
|
||||
mc admin decommission status "$(admin_alias "$alias_name")" "$pool_cmdline"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
parse_status_field() {
|
||||
local json_input="$1"
|
||||
local field="$2"
|
||||
if command -v jq >/dev/null 2>&1; then
|
||||
printf '%s\n' "$json_input" | jq -r ".. | .${field}? // empty" 2>/dev/null | tail -n 1
|
||||
else
|
||||
printf '%s\n' "$json_input" | grep -Eo "\"${field}\"[[:space:]]*:[[:space:]]*(true|false)" | tail -n 1 | awk -F: '{gsub(/[[:space:]]/, "", $2); print $2}'
|
||||
fi
|
||||
}
|
||||
|
||||
wait_decommission() {
|
||||
local alias_name="$1"
|
||||
local pool_cmdline="$2"
|
||||
local timeout_seconds="${3:-900}"
|
||||
local elapsed=0
|
||||
local interval=5
|
||||
local mode
|
||||
|
||||
mode="$(detect_admin_api "$alias_name")"
|
||||
|
||||
while (( elapsed < timeout_seconds )); do
|
||||
local json_output
|
||||
case "$mode" in
|
||||
rustfs)
|
||||
json_output="$(rustfs_admin_request "$alias_name" GET "/pools/status?pool=${pool_cmdline}" 2>/dev/null || true)"
|
||||
;;
|
||||
mc)
|
||||
require_bin mc
|
||||
json_output="$(mc admin decommission status --json "$(admin_alias "$alias_name")" "$pool_cmdline" 2>/dev/null || true)"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ -n "$json_output" ]]; then
|
||||
printf '%s\n' "$json_output"
|
||||
else
|
||||
status_decommission "$alias_name" "$pool_cmdline" || true
|
||||
fi
|
||||
|
||||
local complete failed canceled
|
||||
complete="$(parse_status_field "$json_output" "complete")"
|
||||
failed="$(parse_status_field "$json_output" "failed")"
|
||||
canceled="$(parse_status_field "$json_output" "canceled")"
|
||||
|
||||
if [[ "$failed" == "true" ]]; then
|
||||
echo "decommission finished with failed=true" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$canceled" == "true" ]]; then
|
||||
echo "decommission finished with canceled=true" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$complete" == "true" ]]; then
|
||||
echo "decommission completed successfully"
|
||||
return
|
||||
fi
|
||||
|
||||
sleep "$interval"
|
||||
elapsed=$((elapsed + interval))
|
||||
done
|
||||
|
||||
echo "timed out waiting for decommission completion after ${timeout_seconds}s" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
verify() {
|
||||
local alias_name="$1"
|
||||
require_bin mc
|
||||
|
||||
load_state "$alias_name"
|
||||
|
||||
mc stat "${alias_name}/${BASIC_BUCKET}/single.txt"
|
||||
mc stat "${alias_name}/${BASIC_BUCKET}/multipart.bin"
|
||||
mc stat "${alias_name}/${LOCK_BUCKET}/locked.txt"
|
||||
|
||||
echo "version listing for ${BASIC_BUCKET}:"
|
||||
mc ls --versions "${alias_name}/${BASIC_BUCKET}"
|
||||
|
||||
echo
|
||||
echo "object lock bucket listing for ${LOCK_BUCKET}:"
|
||||
mc ls --versions "${alias_name}/${LOCK_BUCKET}"
|
||||
|
||||
cat <<EOF
|
||||
basic verification passed:
|
||||
- latest single object is readable
|
||||
- multipart object is readable
|
||||
- object-lock bucket object is readable
|
||||
|
||||
manual follow-up:
|
||||
- confirm delete-marker history still exists in 'mc ls --versions ${alias_name}/${BASIC_BUCKET}'
|
||||
- confirm 'decommission_validation.sh status ${alias_name} ${POOL_CMDLINE:-<pool-cmdline>}' shows complete=true and failed=false
|
||||
EOF
|
||||
}
|
||||
|
||||
cancel_decommission() {
|
||||
local alias_name="$1"
|
||||
local pool_cmdline="${2:-}"
|
||||
local mode
|
||||
mode="$(detect_admin_api "$alias_name")"
|
||||
|
||||
case "$mode" in
|
||||
rustfs)
|
||||
require_bin awscurl
|
||||
if [[ -n "$pool_cmdline" ]]; then
|
||||
rustfs_admin_request "$alias_name" POST "/pools/cancel?pool=${pool_cmdline}"
|
||||
else
|
||||
rustfs_admin_request "$alias_name" POST "/pools/cancel"
|
||||
fi
|
||||
;;
|
||||
mc)
|
||||
require_bin mc
|
||||
if [[ -n "$pool_cmdline" ]]; then
|
||||
mc admin decommission cancel "$(admin_alias "$alias_name")" "$pool_cmdline"
|
||||
else
|
||||
mc admin decommission cancel "$(admin_alias "$alias_name")"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main() {
|
||||
if [[ $# -lt 2 ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local cmd="$1"
|
||||
shift
|
||||
|
||||
case "$cmd" in
|
||||
prepare)
|
||||
if [[ $# -ne 1 ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
prepare "$1"
|
||||
;;
|
||||
start)
|
||||
if [[ $# -ne 2 ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
start_decommission "$1" "$2"
|
||||
;;
|
||||
status)
|
||||
if [[ $# -ne 2 ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
status_decommission "$1" "$2"
|
||||
;;
|
||||
wait)
|
||||
if [[ $# -lt 2 || $# -gt 3 ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
wait_decommission "$1" "$2" "${3:-900}"
|
||||
;;
|
||||
verify)
|
||||
if [[ $# -ne 1 ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
verify "$1"
|
||||
;;
|
||||
cancel)
|
||||
if [[ $# -lt 1 || $# -gt 2 ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
cancel_decommission "$1" "${2:-}"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user